From 1ea5c11b4c79385cdb880c23f87b77716db8b66a Mon Sep 17 00:00:00 2001 From: Razvan Radulescu <43811028+h3xxit@users.noreply.github.com> Date: Wed, 6 Aug 2025 18:22:52 +0200 Subject: [PATCH 1/3] Add docstrings --- src/utcp/client/client_transport_interface.py | 67 ++++ src/utcp/client/openapi_converter.py | 57 ++- src/utcp/client/tool_repository.py | 25 ++ .../tool_search_strategies/tag_search.py | 73 +++- src/utcp/client/tool_search_strategy.py | 40 +- .../transport_interfaces/cli_transport.py | 49 ++- .../transport_interfaces/http_transport.py | 41 ++ src/utcp/client/utcp_client.py | 61 ++- src/utcp/client/utcp_client_config.py | 102 +++++ src/utcp/client/variable_substitutor.py | 139 ++++++- src/utcp/shared/auth.py | 46 ++- src/utcp/shared/provider.py | 349 +++++++++++++++--- src/utcp/shared/tool.py | 274 +++++++++++++- src/utcp/shared/utcp_manual.py | 63 +++- 14 files changed, 1289 insertions(+), 97 deletions(-) diff --git a/src/utcp/client/client_transport_interface.py b/src/utcp/client/client_transport_interface.py index c8e7475..1015a8a 100644 --- a/src/utcp/client/client_transport_interface.py +++ b/src/utcp/client/client_transport_interface.py @@ -1,17 +1,84 @@ +"""Abstract interface for UTCP client transport implementations. + +This module defines the contract that all transport implementations must follow +to integrate with the UTCP client. Transport implementations handle the actual +communication with different types of tool providers (HTTP, CLI, WebSocket, etc.). +""" + from abc import ABC, abstractmethod from typing import Dict, Any, List from utcp.shared.provider import Provider from utcp.shared.tool import Tool class ClientTransportInterface(ABC): + """Abstract interface for UTCP client transport implementations. + + Defines the contract that all transport implementations must follow to + integrate with the UTCP client. Each transport handles communication + with a specific type of provider (HTTP, CLI, WebSocket, etc.). + + Transport implementations are responsible for: + - Discovering available tools from providers + - Managing provider lifecycle (registration/deregistration) + - Executing tool calls through the appropriate protocol + """ + @abstractmethod async def register_tool_provider(self, manual_provider: Provider) -> List[Tool]: + """Register a tool provider and discover its available tools. + + Connects to the provider and retrieves the list of tools it offers. + This may involve making discovery requests, parsing configuration files, + or initializing connections depending on the provider type. + + Args: + manual_provider: The provider configuration to register. + + Returns: + List of Tool objects discovered from the provider. + + Raises: + ConnectionError: If unable to connect to the provider. + ValueError: If the provider configuration is invalid. + """ pass @abstractmethod async def deregister_tool_provider(self, manual_provider: Provider) -> None: + """Deregister a tool provider and clean up resources. + + Cleanly disconnects from the provider and releases any associated + resources such as connections, processes, or file handles. + + Args: + manual_provider: The provider configuration to deregister. + + Note: + Should handle cases where the provider is already disconnected + or was never properly registered. + """ pass @abstractmethod async def call_tool(self, tool_name: str, arguments: Dict[str, Any], tool_provider: Provider) -> Any: + """Execute a tool call through this transport. + + Sends a tool invocation request to the provider using the appropriate + protocol and returns the result. Handles serialization of arguments + and deserialization of responses according to the transport type. + + Args: + tool_name: Name of the tool to call (may include provider prefix). + arguments: Dictionary of arguments to pass to the tool. + tool_provider: Provider configuration for the tool. + + Returns: + The tool's response, with type depending on the tool's output schema. + + Raises: + ToolNotFoundError: If the specified tool doesn't exist. + ValidationError: If the arguments don't match the tool's input schema. + ConnectionError: If unable to communicate with the provider. + TimeoutError: If the tool call exceeds the configured timeout. + """ pass diff --git a/src/utcp/client/openapi_converter.py b/src/utcp/client/openapi_converter.py index e0e9df3..026231a 100644 --- a/src/utcp/client/openapi_converter.py +++ b/src/utcp/client/openapi_converter.py @@ -1,3 +1,22 @@ +"""OpenAPI specification converter for UTCP tool generation. + +This module provides functionality to convert OpenAPI specifications (both 2.0 +and 3.0) into UTCP tool definitions. It handles schema resolution, authentication +mapping, and proper tool creation from REST API specifications. + +Key Features: + - OpenAPI 2.0 and 3.0 specification support + - Automatic JSON reference ($ref) resolution + - Authentication scheme mapping (API key, Basic, OAuth2) + - Input/output schema extraction from OpenAPI schemas + - URL path parameter handling + - Request body and header field mapping + - Provider name generation from specification metadata + +The converter creates UTCP tools that can be used to interact with REST APIs +defined by OpenAPI specifications, providing a bridge between OpenAPI and UTCP. +""" + import json from typing import Any, Dict, List, Optional, Tuple import sys @@ -11,9 +30,45 @@ class OpenApiConverter: - """Converts an OpenAPI JSON specification into a UtcpManual.""" + """Converts OpenAPI specifications into UTCP tool definitions. + + Processes OpenAPI 2.0 and 3.0 specifications to generate equivalent UTCP + tools, handling schema resolution, authentication mapping, and proper + HTTP provider configuration. Each operation in the OpenAPI spec becomes + a UTCP tool with appropriate input/output schemas. + + Features: + - Complete OpenAPI specification parsing + - Recursive JSON reference ($ref) resolution + - Authentication scheme conversion (API key, Basic, OAuth2) + - Input parameter and request body handling + - Response schema extraction + - URL template and path parameter support + - Provider name normalization + - Placeholder variable generation for configuration + + Architecture: + The converter works by iterating through all paths and operations + in the OpenAPI spec, extracting relevant information for each + operation, and creating corresponding UTCP tools with HTTP providers. + + Attributes: + spec: The parsed OpenAPI specification dictionary. + spec_url: Optional URL where the specification was retrieved from. + placeholder_counter: Counter for generating unique placeholder variables. + provider_name: Normalized name for the provider derived from the spec. + """ def __init__(self, openapi_spec: Dict[str, Any], spec_url: Optional[str] = None, provider_name: Optional[str] = None): + """Initialize the OpenAPI converter. + + Args: + openapi_spec: Parsed OpenAPI specification as a dictionary. + spec_url: Optional URL where the specification was retrieved from. + Used for base URL determination if servers are not specified. + provider_name: Optional custom name for the provider. If not + provided, derives name from the specification title. + """ self.spec = openapi_spec self.spec_url = spec_url # Single counter for all placeholder variables diff --git a/src/utcp/client/tool_repository.py b/src/utcp/client/tool_repository.py index 5b670d3..3a3278b 100644 --- a/src/utcp/client/tool_repository.py +++ b/src/utcp/client/tool_repository.py @@ -1,9 +1,34 @@ +"""Abstract interface for tool and provider storage. + +This module defines the contract for implementing tool repositories that store +and manage UTCP tools and their associated providers. Different implementations +can provide various storage backends such as in-memory, database, or file-based +storage. +""" + from abc import ABC, abstractmethod from typing import List, Dict, Any, Optional from utcp.shared.provider import Provider from utcp.shared.tool import Tool class ToolRepository(ABC): + """Abstract interface for tool and provider storage implementations. + + Defines the contract for repositories that manage the lifecycle and storage + of UTCP tools and providers. Repositories are responsible for: + - Persisting provider configurations and their associated tools + - Providing efficient lookup and retrieval operations + - Managing relationships between providers and tools + - Ensuring data consistency during operations + + The repository interface supports both individual and bulk operations, + allowing for flexible implementation strategies ranging from simple + in-memory storage to sophisticated database backends. + + Note: + All methods are async to support both synchronous and asynchronous + storage implementations. + """ @abstractmethod async def save_provider_with_tools(self, provider: Provider, tools: List[Tool]) -> None: """ diff --git a/src/utcp/client/tool_search_strategies/tag_search.py b/src/utcp/client/tool_search_strategies/tag_search.py index c3f763d..57fcd37 100644 --- a/src/utcp/client/tool_search_strategies/tag_search.py +++ b/src/utcp/client/tool_search_strategies/tag_search.py @@ -1,3 +1,10 @@ +"""Tag-based tool search strategy implementation. + +This module provides a search strategy that ranks tools based on tag matches +and description keyword matches. It implements a weighted scoring system where +explicit tag matches receive higher scores than description word matches. +""" + from utcp.client.tool_search_strategy import ToolSearchStrategy from typing import List, Dict, Tuple from utcp.shared.tool import Tool @@ -6,25 +13,73 @@ import asyncio class TagSearchStrategy(ToolSearchStrategy): + """Tag-based search strategy for UTCP tools. + + Implements a weighted scoring algorithm that matches search queries against + tool tags and descriptions. Explicit tag matches receive full weight while + description word matches receive reduced weight. + + Scoring Algorithm: + - Exact tag matches: Weight 1.0 + - Tag word matches: Weight equal to description_weight + - Description word matches: Weight equal to description_weight + - Only considers description words longer than 2 characters + + Examples: + >>> strategy = TagSearchStrategy(repository, description_weight=0.3) + >>> tools = await strategy.search_tools("weather api", limit=5) + >>> # Returns tools with "weather" or "api" tags/descriptions + + Attributes: + tool_repository: Repository to search for tools. + description_weight: Weight multiplier for description matches (0.0-1.0). + """ def __init__(self, tool_repository: ToolRepository, description_weight: float = 0.3): + """Initialize the tag search strategy. + + Args: + tool_repository: Repository containing tools to search. + description_weight: Weight for description word matches relative to + tag matches. Should be between 0.0 and 1.0, where 1.0 gives + equal weight to tags and descriptions. + + Raises: + ValueError: If description_weight is not between 0.0 and 1.0. + """ + if not 0.0 <= description_weight <= 1.0: + raise ValueError("description_weight must be between 0.0 and 1.0") + self.tool_repository = tool_repository # Weight for description words vs explicit tags (explicit tags have weight of 1.0) self.description_weight = description_weight async def search_tools(self, query: str, limit: int = 10) -> List[Tool]: - """ - Return tools ordered by tag occurrences in the query. - - Uses both explicit tags and words from tool descriptions (with less weight). - + """Search tools using tag and description matching. + + Implements a weighted scoring system that ranks tools based on how well + their tags and descriptions match the search query. Normalizes the query + and uses word-based matching with configurable weights. + + Scoring Details: + - Exact tag matches in query: +1.0 points + - Individual tag words matching query words: +description_weight points + - Description words matching query words: +description_weight points + - Only description words > 2 characters are considered + Args: - query: The search query string - limit: Maximum number of tools to return - + query: Search query string. Case-insensitive, word-based matching. + limit: Maximum number of tools to return. Must be >= 0. + Returns: - List of tools ordered by relevance to the query + List of Tool objects ranked by relevance score (highest first). + Empty list if no tools match or repository is empty. + + Raises: + ValueError: If limit is negative. """ + if limit < 0: + raise ValueError("limit must be non-negative") # Normalize query to lowercase and split into words query_lower = query.lower() # Extract words from the query, filtering out non-word characters diff --git a/src/utcp/client/tool_search_strategy.py b/src/utcp/client/tool_search_strategy.py index 51b92b3..b620e34 100644 --- a/src/utcp/client/tool_search_strategy.py +++ b/src/utcp/client/tool_search_strategy.py @@ -1,18 +1,48 @@ +"""Abstract interface for tool search strategies. + +This module defines the contract for implementing tool search and ranking +algorithms. Different strategies can implement various approaches such as +tag-based search, semantic search, or hybrid approaches. +""" + from abc import ABC, abstractmethod from typing import List from utcp.shared.tool import Tool class ToolSearchStrategy(ABC): + """Abstract interface for tool search implementations. + + Defines the contract for tool search strategies that can be plugged into + the UTCP client. Different implementations can provide various search + algorithms such as tag-based matching, semantic similarity, or keyword + search. + + Search strategies are responsible for: + - Interpreting search queries + - Ranking tools by relevance + - Limiting results appropriately + - Providing consistent search behavior + """ + @abstractmethod async def search_tools(self, query: str, limit: int = 10) -> List[Tool]: - """ - Search for tools relevant to the query. + """Search for tools relevant to the query. + + Executes a search against the available tools and returns the most + relevant matches ranked by the strategy's scoring algorithm. Args: - query: The search query. - limit: The maximum number of tools to return. 0 for no limit. + query: The search query string. Format depends on the strategy + (e.g., keywords, tags, natural language). + limit: Maximum number of tools to return. Use 0 for no limit. + Strategies should respect this limit for performance. Returns: - A list of tools that match the search query. + List of Tool objects ranked by relevance, limited to the + specified count. Empty list if no matches found. + + Raises: + ValueError: If the query format is invalid for this strategy. + RuntimeError: If the search operation fails unexpectedly. """ pass diff --git a/src/utcp/client/transport_interfaces/cli_transport.py b/src/utcp/client/transport_interfaces/cli_transport.py index b0c0b63..5117465 100644 --- a/src/utcp/client/transport_interfaces/cli_transport.py +++ b/src/utcp/client/transport_interfaces/cli_transport.py @@ -1,7 +1,23 @@ -""" -Command Line Interface (CLI) transport for UTCP client. +"""Command Line Interface (CLI) transport for UTCP client. + +This module provides the CLI transport implementation that enables UTCP clients +to interact with command-line tools and processes. It handles tool discovery +through startup commands, tool execution with proper argument formatting, +and output processing with JSON parsing capabilities. -This transport executes command-line tools and processes. +Key Features: + - Asynchronous command execution with timeout handling + - Tool discovery via startup commands that output UTCP manuals + - Flexible argument formatting for command-line flags + - Environment variable support for authentication and configuration + - JSON output parsing with fallback to raw text + - Cross-platform command parsing (Windows/Unix) + - Working directory control for command execution + +Security: + - Command execution is isolated through subprocess + - Environment variables can be controlled per provider + - Working directory can be restricted """ import asyncio import json @@ -20,11 +36,28 @@ class CliTransport(ClientTransportInterface): """Transport implementation for CLI-based tool providers. - - This transport executes command-line tools and processes. It supports: - - Tool discovery via startup commands that output tool definitions - - Tool execution by running commands with arguments - - Basic authentication via environment variables or command-line flags + + Handles communication with command-line tools by executing processes + and managing their input/output. Supports both tool discovery and + execution phases with comprehensive error handling and timeout management. + + Features: + - Asynchronous subprocess execution with proper cleanup + - Tool discovery through startup commands returning UTCP manuals + - Flexible argument formatting for various CLI conventions + - Environment variable injection for authentication + - JSON output parsing with graceful fallback to text + - Cross-platform command parsing and execution + - Configurable working directories and timeouts + - Process lifecycle management with proper termination + + Architecture: + CLI tools are discovered by executing the provider's command_name + and parsing the output for UTCP manual JSON. Tool calls execute + the same command with formatted arguments and return processed output. + + Attributes: + _log: Logger function for debugging and error reporting. """ def __init__(self, logger: Optional[Callable[[str], None]] = None): diff --git a/src/utcp/client/transport_interfaces/http_transport.py b/src/utcp/client/transport_interfaces/http_transport.py index 12f466a..3aa295b 100644 --- a/src/utcp/client/transport_interfaces/http_transport.py +++ b/src/utcp/client/transport_interfaces/http_transport.py @@ -1,3 +1,17 @@ +"""HTTP transport implementation for UTCP client. + +This module provides the HTTP transport implementation that handles communication +with HTTP-based tool providers. It supports RESTful APIs, authentication methods, +URL path parameters, and automatic tool discovery through various formats. + +Key Features: + - Multiple authentication methods (API key, Basic, OAuth2) + - URL path parameter substitution + - Automatic tool discovery from UTCP manuals, OpenAPI specs, and YAML + - Security enforcement (HTTPS or localhost only) + - Request/response handling with proper error management +""" + from typing import Dict, Any, List import aiohttp import json @@ -15,7 +29,34 @@ from aiohttp import ClientSession, BasicAuth as AiohttpBasicAuth class HttpClientTransport(ClientTransportInterface): + """HTTP transport implementation for UTCP client. + + Handles communication with HTTP-based tool providers, supporting various + authentication methods, URL path parameters, and automatic tool discovery. + Enforces security by requiring HTTPS or localhost connections. + + Features: + - RESTful API communication with configurable HTTP methods + - Multiple authentication: API key (header/query/cookie), Basic, OAuth2 + - URL path parameter substitution from tool arguments + - Tool discovery from UTCP manuals, OpenAPI specs, and YAML + - Request body and header field mapping from tool arguments + - OAuth2 token caching and automatic refresh + - Security validation of connection URLs + + Attributes: + _session: Optional aiohttp ClientSession for connection reuse. + _oauth_tokens: Cache of OAuth2 tokens by client_id. + _log: Logger function for debugging and error reporting. + """ + def __init__(self, logger: Optional[Callable[[str], None]] = None): + """Initialize the HTTP transport. + + Args: + logger: Optional logging function that accepts log messages. + Defaults to a no-op function if not provided. + """ self._session: Optional[aiohttp.ClientSession] = None self._oauth_tokens: Dict[str, Dict[str, Any]] = {} diff --git a/src/utcp/client/utcp_client.py b/src/utcp/client/utcp_client.py index c3e238e..9abf863 100644 --- a/src/utcp/client/utcp_client.py +++ b/src/utcp/client/utcp_client.py @@ -1,3 +1,17 @@ +"""Main UTCP client implementation. + +This module provides the primary client interface for the Universal Tool Calling +Protocol. The UtcpClient class manages multiple transport implementations, +tool repositories, search strategies, and provider configurations. + +Key Features: + - Multi-transport support (HTTP, CLI, WebSocket, etc.) + - Dynamic provider registration and deregistration + - Tool discovery and search capabilities + - Variable substitution for configuration + - Pluggable tool repositories and search strategies +""" + from pathlib import Path import re import os @@ -27,8 +41,17 @@ from utcp.client.variable_substitutor import DefaultVariableSubstitutor, VariableSubstitutor class UtcpClientInterface(ABC): - """ - Interface for a UTCP client. + """Abstract interface for UTCP client implementations. + + Defines the core contract for UTCP clients, including provider management, + tool execution, search capabilities, and variable handling. This interface + allows for different client implementations while maintaining consistency. + + The interface supports: + - Provider lifecycle management (register/deregister) + - Tool discovery and execution + - Tool search and filtering + - Configuration variable validation """ @abstractmethod def register_tool_provider(self, manual_provider: Provider) -> List[Tool]: @@ -108,6 +131,40 @@ def get_required_variables_for_tool(self, tool_name: str) -> List[str]: pass class UtcpClient(UtcpClientInterface): + """Main implementation of the UTCP client. + + The UtcpClient is the primary entry point for interacting with UTCP tool + providers. It manages multiple transport implementations, handles provider + registration, executes tool calls, and provides search capabilities. + + Key Features: + - Multi-transport architecture supporting HTTP, CLI, WebSocket, etc. + - Dynamic provider registration from configuration files + - Variable substitution for secure credential management + - Pluggable tool repositories and search strategies + - Comprehensive error handling and validation + + Architecture: + - Transport Layer: Handles protocol-specific communication + - Repository Layer: Manages tool and provider storage + - Search Layer: Provides tool discovery and filtering + - Configuration Layer: Manages settings and variable substitution + + Usage: + >>> client = await UtcpClient.create({ + ... "providers_file_path": "./providers.json" + ... }) + >>> tools = await client.search_tools("weather") + >>> result = await client.call_tool("api.get_weather", {"city": "NYC"}) + + Attributes: + transports: Dictionary mapping provider types to transport implementations. + tool_repository: Storage backend for tools and providers. + search_strategy: Algorithm for tool search and ranking. + config: Client configuration including file paths and settings. + variable_substitutor: Handler for environment variable substitution. + """ + transports: Dict[str, ClientTransportInterface] = { "http": HttpClientTransport(), "cli": CliTransport(), diff --git a/src/utcp/client/utcp_client_config.py b/src/utcp/client/utcp_client_config.py index a74cafe..7649517 100644 --- a/src/utcp/client/utcp_client_config.py +++ b/src/utcp/client/utcp_client_config.py @@ -1,29 +1,131 @@ +"""Configuration models for UTCP client setup. + +This module defines the configuration classes and variable loading mechanisms +for UTCP clients. It provides flexible variable substitution support through +multiple sources including environment files, direct configuration, and +custom variable loaders. + +The configuration system enables: + - Variable substitution in provider configurations + - Multiple variable sources with hierarchical resolution + - Environment file loading (.env files) + - Direct variable specification + - Custom variable loader implementations +""" + from abc import ABC, abstractmethod from pydantic import BaseModel, Field from typing import Optional, List, Dict, Literal, TypedDict from dotenv import dotenv_values class UtcpVariableNotFound(Exception): + """Exception raised when a required variable cannot be found. + + This exception is thrown during variable substitution when a referenced + variable cannot be resolved through any of the configured variable sources. + It provides information about which variable was missing to help with + debugging configuration issues. + + Attributes: + variable_name: The name of the variable that could not be found. + """ variable_name: str def __init__(self, variable_name: str): + """Initialize the exception with the missing variable name. + + Args: + variable_name: Name of the variable that could not be found. + """ self.variable_name = variable_name super().__init__(f"Variable {variable_name} referenced in provider configuration not found. Please add it to the environment variables or to your UTCP configuration.") class UtcpVariablesConfig(BaseModel, ABC): + """Abstract base class for variable loading configurations. + + Defines the interface for variable loaders that can retrieve variable + values from different sources such as files, databases, or external + services. Implementations provide specific loading mechanisms while + maintaining a consistent interface. + + Attributes: + type: Type identifier for the variable loader. + """ type: Literal["dotenv"] = "dotenv" @abstractmethod def get(self, key: str) -> Optional[str]: + """Retrieve a variable value by key. + + Args: + key: Variable name to retrieve. + + Returns: + Variable value if found, None otherwise. + """ pass class UtcpDotEnv(UtcpVariablesConfig): + """Environment file variable loader implementation. + + Loads variables from .env files using the dotenv format. This loader + supports the standard key=value format with optional quoting and + comment support provided by the python-dotenv library. + + Attributes: + env_file_path: Path to the .env file to load variables from. + + Example: + ```python + loader = UtcpDotEnv(env_file_path=".env") + api_key = loader.get("API_KEY") + ``` + """ env_file_path: str def get(self, key: str) -> Optional[str]: + """Load a variable from the configured .env file. + + Args: + key: Variable name to retrieve from the environment file. + + Returns: + Variable value if found in the file, None otherwise. + """ return dotenv_values(self.env_file_path).get(key) class UtcpClientConfig(BaseModel): + """Configuration model for UTCP client setup. + + Provides comprehensive configuration options for UTCP clients including + variable definitions, provider file locations, and variable loading + mechanisms. Supports hierarchical variable resolution with multiple + sources. + + Variable Resolution Order: + 1. Direct variables dictionary + 2. Custom variable loaders (in order) + 3. Environment variables + + Attributes: + variables: Direct variable definitions as key-value pairs. + These take precedence over other variable sources. + providers_file_path: Optional path to a file containing provider + configurations. Supports JSON and YAML formats. + load_variables_from: List of variable loaders to use for + variable resolution. Loaders are consulted in order. + + Example: + ```python + config = UtcpClientConfig( + variables={"API_BASE": "https://api.example.com"}, + providers_file_path="providers.yaml", + load_variables_from=[ + UtcpDotEnv(env_file_path=".env") + ] + ) + ``` + """ variables: Optional[Dict[str, str]] = Field(default_factory=dict) providers_file_path: Optional[str] = None load_variables_from: Optional[List[UtcpVariablesConfig]] = None diff --git a/src/utcp/client/variable_substitutor.py b/src/utcp/client/variable_substitutor.py index 2fc04b6..1e69716 100644 --- a/src/utcp/client/variable_substitutor.py +++ b/src/utcp/client/variable_substitutor.py @@ -1,3 +1,15 @@ +"""Variable substitution system for UTCP configuration values. + +This module provides a flexible variable substitution system that enables +dynamic replacement of placeholders in provider configurations and tool +arguments. It supports multiple variable sources including configuration +files, environment variables, and custom variable loaders. + +Variable Syntax: + Variables can be referenced using either ${VAR_NAME} or $VAR_NAME syntax. + Provider-specific variables are automatically namespaced to avoid conflicts. +""" + from abc import ABC, abstractmethod from utcp.client.utcp_client_config import UtcpClientConfig from typing import Any @@ -7,16 +19,87 @@ from typing import List, Optional class VariableSubstitutor(ABC): + """Abstract interface for variable substitution implementations. + + Defines the contract for variable substitution systems that can replace + placeholders in configuration data with actual values from various sources. + Implementations handle different variable resolution strategies and + source hierarchies. + """ + @abstractmethod - def substitute(self, obj: Any, config: UtcpClientConfig, provider_name: Optional[str] = None) -> Any: + def substitute(self, obj: dict | list | str, config: UtcpClientConfig, provider_name: Optional[str] = None) -> Any: + """Substitute variables in the given object. + + Args: + obj: Object containing potential variable references to substitute. + Can be dict, list, str, or any other type. + config: UTCP client configuration containing variable definitions + and loaders. + provider_name: Optional provider name for variable namespacing. + + Returns: + Object with all variable references replaced by their values. + + Raises: + UtcpVariableNotFound: If a referenced variable cannot be resolved. + """ pass @abstractmethod def find_required_variables(self, obj: dict | list | str, provider_name: str) -> List[str]: + """Find all variable references in the given object. + + Args: + obj: Object to scan for variable references. + provider_name: Provider name for variable namespacing. + + Returns: + List of fully-qualified variable names found in the object. + """ pass class DefaultVariableSubstitutor(VariableSubstitutor): + """Default implementation of variable substitution. + + Provides a hierarchical variable resolution system that searches for + variables in the following order: + 1. Configuration variables (exact match) + 2. Custom variable loaders (in order) + 3. Environment variables + + Features: + - Provider-specific variable namespacing + - Multiple variable syntax support: ${VAR} and $VAR + - Hierarchical variable resolution + - Recursive substitution in nested data structures + - Variable discovery for validation + + Variable Namespacing: + Provider-specific variables are prefixed with the provider name + to avoid conflicts. For example, a variable 'api_key' for provider + 'web_scraper' becomes 'web__scraper_api_key' internally. + """ def _get_variable(self, key: str, config: UtcpClientConfig, provider_name: Optional[str] = None) -> str: + """Resolve a variable value through the hierarchical resolution system. + + Searches for the variable value in the following order: + 1. Configuration variables dictionary + 2. Custom variable loaders (in registration order) + 3. Environment variables + + Args: + key: Variable name to resolve. + config: UTCP client configuration containing variable sources. + provider_name: Optional provider name for variable namespacing. + When provided, the key is prefixed with the provider name. + + Returns: + Resolved variable value as a string. + + Raises: + UtcpVariableNotFound: If the variable cannot be found in any source. + """ if provider_name: key = provider_name.replace("_", "!").replace("!", "__") + "_" + key if config.variables and key in config.variables: @@ -36,6 +119,35 @@ def _get_variable(self, key: str, config: UtcpClientConfig, provider_name: Optio raise UtcpVariableNotFound(key) def substitute(self, obj: dict | list | str, config: UtcpClientConfig, provider_name: Optional[str] = None) -> Any: + """Recursively substitute variables in nested data structures. + + Performs deep substitution on dictionaries, lists, and strings. + Non-string types are returned unchanged. String values are scanned + for variable references using ${VAR} and $VAR syntax. + + Args: + obj: Object to perform substitution on. Can be any type. + config: UTCP client configuration containing variable sources. + provider_name: Optional provider name for variable namespacing. + + Returns: + Object with all variable references replaced. Structure and + non-string values are preserved. + + Raises: + UtcpVariableNotFound: If any referenced variable cannot be resolved. + + Example: + ```python + substitutor = DefaultVariableSubstitutor() + result = substitutor.substitute( + {"url": "https://${HOST}/api", "port": 8080}, + config, + "my_provider" + ) + # Returns: {"url": "https://api.example.com/api", "port": 8080} + ``` + """ if isinstance(obj, dict): return {k: self.substitute(v, config, provider_name) for k, v in obj.items()} elif isinstance(obj, list): @@ -52,6 +164,31 @@ def replacer(match): return obj def find_required_variables(self, obj: dict | list | str, provider_name: str) -> List[str]: + """Recursively discover all variable references in a data structure. + + Scans the object for variable references using ${VAR} and $VAR syntax, + returning fully-qualified variable names with provider namespacing. + Useful for validation and dependency analysis. + + Args: + obj: Object to scan for variable references. + provider_name: Provider name used for variable namespacing. + Variable names are prefixed with this provider name. + + Returns: + List of fully-qualified variable names found in the object. + Variables are prefixed with the provider name to avoid conflicts. + + Example: + ```python + substitutor = DefaultVariableSubstitutor() + vars = substitutor.find_required_variables( + {"url": "https://${HOST}/api", "key": "$API_KEY"}, + "web_api" + ) + # Returns: ["web__api_HOST", "web__api_API_KEY"] + ``` + """ if isinstance(obj, dict): result = [] for v in obj.values(): diff --git a/src/utcp/shared/auth.py b/src/utcp/shared/auth.py index 82490d3..b06f2b9 100644 --- a/src/utcp/shared/auth.py +++ b/src/utcp/shared/auth.py @@ -1,3 +1,9 @@ +"""Authentication schemes for UTCP providers. + +This module defines the authentication models supported by UTCP providers, +including API key authentication, basic authentication, and OAuth2. +""" + from typing import Literal, Optional, TypeAlias, Union from pydantic import BaseModel, Field @@ -6,10 +12,19 @@ class ApiKeyAuth(BaseModel): """Authentication using an API key. The key can be provided directly or sourced from an environment variable. + Supports placement in headers, query parameters, or cookies. + + Attributes: + auth_type: The authentication type identifier, always "api_key". + api_key: The API key for authentication. Values starting with '$' or formatted as '${}' are + treated as an injected variable from environment or configuration. + var_name: The name of the header, query parameter, or cookie that + contains the API key. + location: Where to include the API key (header, query parameter, or cookie). """ auth_type: Literal["api_key"] = "api_key" - api_key: str = Field(..., description="The API key for authentication. If it starts with '$', it is treated as an injected variable. This is the recommended way to provide API keys.") + api_key: str = Field(..., description="The API key for authentication. Values starting with '$' or formatted as '${}' are treated as an injected variable from environment or configuration. This is the recommended way to provide API keys.") var_name: str = Field( "X-Api-Key", description="The name of the header, query parameter, cookie or other container for the API key." ) @@ -19,7 +34,16 @@ class ApiKeyAuth(BaseModel): class BasicAuth(BaseModel): - """Authentication using a username and password.""" + """Authentication using HTTP Basic Authentication. + + Uses the standard HTTP Basic Authentication scheme with username and password + encoded in the Authorization header. + + Attributes: + auth_type: The authentication type identifier, always "basic". + username: The username for basic authentication. Recommended to use injected variables. + password: The password for basic authentication. Recommended to use injected variables. + """ auth_type: Literal["basic"] = "basic" username: str = Field(..., description="The username for basic authentication.") @@ -27,7 +51,18 @@ class BasicAuth(BaseModel): class OAuth2Auth(BaseModel): - """Authentication using OAuth2.""" + """Authentication using OAuth2 client credentials flow. + + Implements the OAuth2 client credentials grant type for machine-to-machine + authentication. The client automatically handles token acquisition and refresh. + + Attributes: + auth_type: The authentication type identifier, always "oauth2". + token_url: The URL endpoint to fetch the OAuth2 access token from. Recommended to use injected variables. + client_id: The OAuth2 client identifier. Recommended to use injected variables. + client_secret: The OAuth2 client secret. Recommended to use injected variables. + scope: Optional scope parameter to limit the access token's permissions. + """ auth_type: Literal["oauth2"] = "oauth2" token_url: str = Field(..., description="The URL to fetch the OAuth2 token from.") @@ -37,3 +72,8 @@ class OAuth2Auth(BaseModel): Auth: TypeAlias = Union[ApiKeyAuth, BasicAuth, OAuth2Auth] +"""Type alias for all supported authentication schemes. + +This union type encompasses all authentication methods supported by UTCP providers. +Use this type for type hints when accepting any authentication scheme. +""" diff --git a/src/utcp/shared/provider.py b/src/utcp/shared/provider.py index 2a9b339..99e50dc 100644 --- a/src/utcp/shared/provider.py +++ b/src/utcp/shared/provider.py @@ -1,3 +1,25 @@ +"""Provider configurations for UTCP tool providers. + +This module defines the provider models and configurations for all supported +transport protocols in UTCP. Each provider type encapsulates the necessary +configuration to connect to and interact with tools through different +communication channels. + +Supported provider types: + - HTTP: RESTful HTTP/HTTPS APIs + - SSE: Server-Sent Events for streaming + - HTTP Stream: HTTP Chunked Transfer Encoding + - CLI: Command Line Interface tools + - WebSocket: Bidirectional WebSocket connections (WIP) + - gRPC: Google Remote Procedure Call (WIP) + - GraphQL: GraphQL query language + - TCP: Raw TCP socket connections + - UDP: User Datagram Protocol + - WebRTC: Web Real-Time Communication (WIP) + - MCP: Model Context Protocol + - Text: Text file-based providers +""" + from typing import Dict, Any, Optional, List, Literal, TypeAlias, Union from pydantic import BaseModel, Field from typing import Annotated @@ -14,22 +36,56 @@ 'sse', # Server-Sent Events 'http_stream', # HTTP Chunked Transfer Encoding 'cli', # Command Line Interface - 'websocket', # WebSocket bidirectional connection - 'grpc', # gRPC (Google Remote Procedure Call) + 'websocket', # WebSocket bidirectional connection (WIP) + 'grpc', # gRPC (Google Remote Procedure Call) (WIP) 'graphql', # GraphQL query language 'tcp', # Raw TCP socket 'udp', # User Datagram Protocol - 'webrtc', # Web Real-Time Communication + 'webrtc', # Web Real-Time Communication (WIP) 'mcp', # Model Context Protocol 'text', # Text file provider ] +"""Type alias for all supported provider transport types. + +This literal type defines all the communication protocols and transport +mechanisms that UTCP supports for connecting to tool providers. +""" class Provider(BaseModel): + """Base class for all UTCP tool providers. + + This is the abstract base class that all specific provider implementations + inherit from. It provides the common fields that every provider must have. + + Attributes: + name: Unique identifier for the provider. Defaults to a random UUID hex string. + Should be unique across all providers and recommended to be set to a human-readable name. + Can only contain letters, numbers and underscores. All special characters must be replaced with underscores. + provider_type: The transport protocol type used by this provider. + """ + name: str = uuid.uuid4().hex provider_type: ProviderType class HttpProvider(Provider): - """Options specific to HTTP tools""" + """Provider configuration for HTTP-based tools. + + Supports RESTful HTTP/HTTPS APIs with various HTTP methods, authentication, + custom headers, and flexible request/response handling. Supports URL path + parameters using {parameter_name} syntax. All tool arguments not mapped to + URL body, headers or query pattern parameters are passed as query parameters using '?arg_name={arg_value}'. + + Attributes: + provider_type: Always "http" for HTTP providers. + http_method: The HTTP method to use for requests. + url: The base URL for the HTTP endpoint. Supports path parameters like + "https://api.example.com/users/{user_id}/posts/{post_id}". + content_type: The Content-Type header for requests. + auth: Optional authentication configuration. + headers: Optional static headers to include in all requests. + body_field: Name of the tool argument to map to the HTTP request body. + header_fields: List of tool argument names to map to HTTP request headers. + """ provider_type: Literal["http"] = "http" http_method: Literal["GET", "POST", "PUT", "DELETE", "PATCH"] = "GET" @@ -41,7 +97,24 @@ class HttpProvider(Provider): header_fields: Optional[List[str]] = Field(default=None, description="List of input fields to be sent as request headers.") class SSEProvider(Provider): - """Options specific to Server-Sent Events tools""" + """Provider configuration for Server-Sent Events (SSE) tools. + + Enables real-time streaming of events from server to client using the + Server-Sent Events protocol. Supports automatic reconnection and + event type filtering. All tool arguments not mapped to URL body, headers + or query pattern parameters are passed as query parameters using '?arg_name={arg_value}'. + + Attributes: + provider_type: Always "sse" for SSE providers. + url: The SSE endpoint URL to connect to. + event_type: Optional filter for specific event types. If None, all events are received. + reconnect: Whether to automatically reconnect on connection loss. + retry_timeout: Timeout in milliseconds before attempting reconnection. + auth: Optional authentication configuration. + headers: Optional static headers for the initial connection. + body_field: Optional tool argument name to map to request body during connection. + header_fields: List of tool argument names to map to HTTP headers during connection. + """ provider_type: Literal["sse"] = "sse" url: str @@ -54,7 +127,25 @@ class SSEProvider(Provider): header_fields: Optional[List[str]] = Field(default=None, description="List of input fields to be sent as request headers for the initial connection.") class StreamableHttpProvider(Provider): - """Options specific to HTTP Chunked Transfer Encoding (HTTP streaming) tools""" + """Provider configuration for HTTP streaming tools. + + Uses HTTP Chunked Transfer Encoding to enable streaming of large responses + or real-time data. Useful for tools that return large datasets or provide + progressive results. All tool arguments not mapped to URL body, headers + or query pattern parameters are passed as query parameters using '?arg_name={arg_value}'. + + Attributes: + provider_type: Always "http_stream" for HTTP streaming providers. + url: The streaming HTTP endpoint URL. Supports path parameters. + http_method: The HTTP method to use (GET or POST). + content_type: The Content-Type header for requests. + chunk_size: Size of each chunk in bytes for reading the stream. + timeout: Request timeout in milliseconds. + headers: Optional static headers to include in requests. + auth: Optional authentication configuration. + body_field: Optional tool argument name to map to HTTP request body. + header_fields: List of tool argument names to map to HTTP request headers. + """ provider_type: Literal["http_stream"] = "http_stream" url: str @@ -68,7 +159,18 @@ class StreamableHttpProvider(Provider): header_fields: Optional[List[str]] = Field(default=None, description="List of input fields to be sent as request headers.") class CliProvider(Provider): - """Options specific to CLI tools""" + """Provider configuration for Command Line Interface tools. + + Enables execution of command-line tools and programs as UTCP providers. + Supports environment variable injection and custom working directories. + + Attributes: + provider_type: Always "cli" for CLI providers. + command_name: The name or path of the command to execute. + env_vars: Optional environment variables to set during command execution. + working_dir: Optional custom working directory for command execution. + auth: Always None - CLI providers don't support authentication. + """ provider_type: Literal["cli"] = "cli" command_name: str @@ -77,7 +179,20 @@ class CliProvider(Provider): auth: None = None class WebSocketProvider(Provider): - """Options specific to WebSocket tools""" + """Provider configuration for WebSocket-based tools. (WIP) + + Enables bidirectional real-time communication with WebSocket servers. + Supports custom protocols, keep-alive functionality, and authentication. + + Attributes: + provider_type: Always "websocket" for WebSocket providers. + url: The WebSocket endpoint URL (ws:// or wss://). + protocol: Optional WebSocket sub-protocol to request. + keep_alive: Whether to maintain the connection with keep-alive messages. + auth: Optional authentication configuration. + headers: Optional static headers for the WebSocket handshake. + header_fields: List of tool argument names to map to headers during handshake. + """ provider_type: Literal["websocket"] = "websocket" url: str @@ -88,7 +203,20 @@ class WebSocketProvider(Provider): header_fields: Optional[List[str]] = Field(default=None, description="List of input fields to be sent as request headers for the initial connection.") class GRPCProvider(Provider): - """Options specific to gRPC tools""" + """Provider configuration for gRPC (Google Remote Procedure Call) tools. (WIP) + + Enables communication with gRPC services using the Protocol Buffers + serialization format. Supports both secure (TLS) and insecure connections. + + Attributes: + provider_type: Always "grpc" for gRPC providers. + host: The hostname or IP address of the gRPC server. + port: The port number of the gRPC server. + service_name: The name of the gRPC service to call. + method_name: The name of the gRPC method to invoke. + use_ssl: Whether to use SSL/TLS for secure connections. + auth: Optional authentication configuration. + """ provider_type: Literal["grpc"] = "grpc" host: str @@ -99,7 +227,21 @@ class GRPCProvider(Provider): auth: Optional[Auth] = None class GraphQLProvider(Provider): - """Options specific to GraphQL tools""" + """Provider configuration for GraphQL-based tools. + + Enables communication with GraphQL endpoints supporting queries, mutations, + and subscriptions. Provides flexible query execution with custom headers + and authentication. + + Attributes: + provider_type: Always "graphql" for GraphQL providers. + url: The GraphQL endpoint URL. + operation_type: The type of GraphQL operation (query, mutation, subscription). + operation_name: Optional name for the GraphQL operation. + auth: Optional authentication configuration. + headers: Optional static headers to include in requests. + header_fields: List of tool argument names to map to HTTP request headers. + """ provider_type: Literal["graphql"] = "graphql" url: str @@ -110,21 +252,41 @@ class GraphQLProvider(Provider): header_fields: Optional[List[str]] = Field(default=None, description="List of input fields to be sent as request headers for the initial connection.") class TCPProvider(Provider): - """Options specific to raw TCP socket tools - - For request data handling: - - If request_data_format is 'json', arguments will be formatted as a JSON object and sent - - If request_data_format is 'text', the request_data_template can contain placeholders - in the format UTCP_ARG_argname_UTCP_ARG which will be replaced with the value of - the argument named 'argname' - For response data handling: - - If response_byte_format is None, raw bytes will be returned - - If response_byte_format is an encoding (e.g., 'utf-8'), bytes will be decoded to text - For TCP stream framing (choose one): - 1. Length-prefix framing: Set framing_strategy='length_prefix' and length_prefix_bytes - 2. Delimiter-based framing: Set framing_strategy='delimiter' and message_delimiter - 3. Fixed-length framing: Set framing_strategy='fixed_length' and fixed_message_length - 4. Stream-based: Set framing_strategy='stream' to read until connection closes + """Provider configuration for raw TCP socket tools. + + Enables direct communication with TCP servers using custom protocols. + Supports flexible request formatting, response decoding, and multiple + framing strategies for message boundaries. + + Request Data Handling: + - 'json' format: Arguments formatted as JSON object + - 'text' format: Template-based with UTCP_ARG_argname_UTCP_ARG placeholders + + Response Data Handling: + - If response_byte_format is None: Returns raw bytes + - If response_byte_format is encoding string: Decodes bytes to text + + TCP Stream Framing Options: + 1. Length-prefix: Set framing_strategy='length_prefix' + length_prefix_bytes + 2. Delimiter-based: Set framing_strategy='delimiter' + message_delimiter + 3. Fixed-length: Set framing_strategy='fixed_length' + fixed_message_length + 4. Stream-based: Set framing_strategy='stream' (reads until connection closes) + + Attributes: + provider_type: Always "tcp" for TCP providers. + host: The hostname or IP address of the TCP server. + port: The port number of the TCP server. + request_data_format: Format for request data ('json' or 'text'). + request_data_template: Template string for 'text' format with placeholders. + response_byte_format: Encoding for response decoding (None for raw bytes). + framing_strategy: Method for detecting message boundaries. + length_prefix_bytes: Number of bytes for length prefix (1, 2, 4, or 8). + length_prefix_endian: Byte order for length prefix ('big' or 'little'). + message_delimiter: Delimiter string for message boundaries. + fixed_message_length: Fixed length in bytes for each message. + max_response_size: Maximum bytes to read for stream-based framing. + timeout: Connection timeout in milliseconds. + auth: Always None - TCP providers don't support authentication. """ provider_type: Literal["tcp"] = "tcp" @@ -166,17 +328,30 @@ class TCPProvider(Provider): auth: None = None class UDPProvider(Provider): - """Options specific to UDP socket tools - - For request data handling: - - If request_data_format is 'json', arguments will be formatted as a JSON object and sent - - If request_data_format is 'text', the request_data_template can contain placeholders - in the format UTCP_ARG_argname_UTCP_ARG which will be replaced with the value of - the argument named 'argname' - - For response data handling: - - If response_byte_format is None, raw bytes will be returned - - If response_byte_format is an encoding (e.g., 'utf-8'), bytes will be decoded to text + """Provider configuration for UDP (User Datagram Protocol) socket tools. + + Enables communication with UDP servers using the connectionless UDP protocol. + Supports flexible request formatting, response decoding, and multi-datagram + response handling. + + Request Data Handling: + - 'json' format: Arguments formatted as JSON object + - 'text' format: Template-based with UTCP_ARG_argname_UTCP_ARG placeholders + + Response Data Handling: + - If response_byte_format is None: Returns raw bytes + - If response_byte_format is encoding string: Decodes bytes to text + + Attributes: + provider_type: Always "udp" for UDP providers. + host: The hostname or IP address of the UDP server. + port: The port number of the UDP server. + number_of_response_datagrams: Expected number of response datagrams (0 for no response). + request_data_format: Format for request data ('json' or 'text'). + request_data_template: Template string for 'text' format with placeholders. + response_byte_format: Encoding for response decoding (None for raw bytes). + timeout: Request timeout in milliseconds. + auth: Always None - UDP providers don't support authentication. """ provider_type: Literal["udp"] = "udp" @@ -190,7 +365,18 @@ class UDPProvider(Provider): auth: None = None class WebRTCProvider(Provider): - """Options specific to WebRTC tools""" + """Provider configuration for WebRTC (Web Real-Time Communication) tools. + + Enables peer-to-peer communication using WebRTC data channels. + Requires a signaling server to establish the initial connection. + + Attributes: + provider_type: Always "webrtc" for WebRTC providers. + signaling_server: URL of the signaling server for peer discovery. + peer_id: Unique identifier for this peer in the WebRTC network. + data_channel_name: Name of the data channel for tool communication. + auth: Always None - WebRTC providers don't support authentication. + """ provider_type: Literal["webrtc"] = "webrtc" signaling_server: str @@ -199,24 +385,67 @@ class WebRTCProvider(Provider): auth: None = None class McpStdioServer(BaseModel): - """Configuration for an MCP server connected via stdio.""" + """Configuration for an MCP server connected via stdio transport. + + Enables communication with Model Context Protocol servers through + standard input/output streams, typically used for local processes. + + Attributes: + transport: Always "stdio" for stdio-based MCP servers. + command: The command to execute to start the MCP server. + args: Optional command-line arguments for the MCP server. + env: Optional environment variables for the MCP server process. + """ transport: Literal["stdio"] = "stdio" command: str args: Optional[List[str]] = [] env: Optional[Dict[str, str]] = {} class McpHttpServer(BaseModel): - """Configuration for an MCP server connected via streamable HTTP.""" + """Configuration for an MCP server connected via HTTP transport. + + Enables communication with Model Context Protocol servers through + HTTP connections, typically used for remote MCP services. + + Attributes: + transport: Always "http" for HTTP-based MCP servers. + url: The HTTP endpoint URL for the MCP server. + """ transport: Literal["http"] = "http" url: str McpServer: TypeAlias = Union[McpStdioServer, McpHttpServer] +"""Type alias for MCP server configurations. + +Union type for all supported MCP server transport configurations, +including both stdio and HTTP-based servers. +""" class McpConfig(BaseModel): + """Configuration container for multiple MCP servers. + + Holds a collection of named MCP server configurations, allowing + a single MCP provider to manage multiple server connections. + + Attributes: + mcpServers: Dictionary mapping server names to their configurations. + """ + mcpServers: Dict[str, McpServer] class MCPProvider(Provider): - """Options specific to MCP tools, supporting both stdio and HTTP transports.""" + """Provider configuration for Model Context Protocol (MCP) tools. + + Enables communication with MCP servers that provide structured tool + interfaces. Supports both stdio (local process) and HTTP (remote) + transport methods. + + Attributes: + provider_type: Always "mcp" for MCP providers. + config: Configuration object containing MCP server definitions. + This follows the same format as the official MCP server configuration. + auth: Optional OAuth2 authentication for HTTP-based MCP servers. + """ provider_type: Literal["mcp"] = "mcp" config: McpConfig @@ -224,13 +453,21 @@ class MCPProvider(Provider): class TextProvider(Provider): - """Options specific to text file-based tools. + """Provider configuration for text file-based tools. + + Reads tool definitions from local text files, useful for static tool + configurations or when tools generate output files at known locations. + + Use Cases: + - Static tool definitions from configuration files + - Tools that write results to predictable file locations + - Download manuals from a remote server to allow inspection of tools + before calling them and guarantee security for high-risk environments - This provider reads tool definitions from a local text file. This is useful - when the tool call is included in the startup command, but the result of the - tool call produces a file at a static location that can be read from. It can - also be used as a UTCP tool provider to specify tools that should be used - from different other providers. + Attributes: + provider_type: Always "text" for text file providers. + file_path: Path to the file containing tool definitions. + auth: Always None - text providers don't support authentication. """ provider_type: Literal["text"] = "text" @@ -254,3 +491,23 @@ class TextProvider(Provider): ], Field(discriminator="provider_type") ] +"""Discriminated union type for all UTCP provider configurations. + +This annotated union type includes all supported provider implementations, +using 'provider_type' as the discriminator field for automatic type +resolution during deserialization. + +Supported Provider Types: + - HttpProvider: RESTful HTTP/HTTPS APIs + - SSEProvider: Server-Sent Events streaming + - StreamableHttpProvider: HTTP Chunked Transfer Encoding + - CliProvider: Command Line Interface tools + - WebSocketProvider: Bidirectional WebSocket connections + - GRPCProvider: Google Remote Procedure Call + - GraphQLProvider: GraphQL query language + - TCPProvider: Raw TCP socket connections + - UDPProvider: User Datagram Protocol + - WebRTCProvider: Web Real-Time Communication + - MCPProvider: Model Context Protocol + - TextProvider: Text file-based providers +""" diff --git a/src/utcp/shared/tool.py b/src/utcp/shared/tool.py index 3994b7c..f7f80ce 100644 --- a/src/utcp/shared/tool.py +++ b/src/utcp/shared/tool.py @@ -1,3 +1,17 @@ +"""Tool definitions and schema generation for UTCP. + +This module provides the core tool definition models and utilities for +automatic schema generation from Python functions. It supports both +manual tool definitions and decorator-based automatic tool creation. + +Key Components: + - Tool: The main tool definition model + - ToolInputOutputSchema: JSON Schema for tool inputs and outputs + - ToolContext: Global tool registry + - @utcp_tool: Decorator for automatic tool creation from functions + - Schema generation utilities for Python type hints +""" + import inspect from typing import Dict, Any, Optional, List, Set, Tuple, get_type_hints, get_origin, get_args, Union from pydantic import BaseModel, Field @@ -5,6 +19,25 @@ class ToolInputOutputSchema(BaseModel): + """JSON Schema definition for tool inputs and outputs. + + Represents a JSON Schema object that defines the structure and validation + rules for tool parameters (inputs) or return values (outputs). Compatible + with JSON Schema Draft 7. + + Attributes: + type: The JSON Schema type (object, array, string, number, boolean, null). + properties: Dictionary of property definitions for object types. + required: List of required property names for object types. + description: Human-readable description of the schema. + title: Title for the schema. + items: Schema definition for array item types. + enum: List of allowed values for enumeration types. + minimum: Minimum value for numeric types. + maximum: Maximum value for numeric types. + format: String format specification (e.g., "date", "email"). None for strings. + """ + type: str = Field(default="object") properties: Dict[str, Any] = Field(default_factory=dict) required: Optional[List[str]] = None @@ -17,6 +50,22 @@ class ToolInputOutputSchema(BaseModel): format: Optional[str] = None # For string formats class Tool(BaseModel): + """Definition of a UTCP tool. + + Represents a callable tool with its metadata, input/output schemas, + and provider configuration. Tools are the fundamental units of + functionality in the UTCP ecosystem. + + Attributes: + name: Unique identifier for the tool, typically in format "provider.tool_name". + description: Human-readable description of what the tool does. + inputs: JSON Schema defining the tool's input parameters. + outputs: JSON Schema defining the tool's return value structure. + tags: List of tags for categorization and search. + average_response_size: Optional hint about typical response size in bytes. + tool_provider: Provider configuration for accessing this tool. + """ + name: str description: str = "" inputs: ToolInputOutputSchema = Field(default_factory=ToolInputOutputSchema) @@ -26,22 +75,61 @@ class Tool(BaseModel): tool_provider: ProviderUnion class ToolContext: + """Global registry for UTCP tools. + + Maintains a centralized collection of all registered tools in the current + process. Used by the @utcp_tool decorator to automatically register tools + and by servers to discover available tools. + + Note: + This is a class-level registry using static methods. All tools + registered here are globally available within the process. + """ + tools: List[Tool] = [] @staticmethod - def add_tool(tool: Tool): - """Add a tool to the UTCP server.""" + def add_tool(tool: Tool) -> None: + """Add a tool to the global tool registry. + Args: + tool: The tool definition to register. + + Note: + Prints registration information for debugging purposes. + """ print(f"Adding tool: {tool.name} with provider: {tool.tool_provider.name if tool.tool_provider else 'None'}") ToolContext.tools.append(tool) @staticmethod def get_tools() -> List[Tool]: - """Get the list of tools available in the UTCP server.""" + """Get all tools from the global registry. + + Returns: + List of all registered Tool objects. + """ return ToolContext.tools -########## UTCP Tool Decorator ########## -def python_type_to_json_type(py_type): +def python_type_to_json_type(py_type) -> str: + """Convert Python type annotations to JSON Schema type strings. + + Maps Python type hints to their corresponding JSON Schema type names. + Handles generic types, unions, and optional types. + + Args: + py_type: Python type annotation to convert. + + Returns: + JSON Schema type string (e.g., "string", "number", "array", "object"). + + Examples: + >>> python_type_to_json_type(str) + "string" + >>> python_type_to_json_type(List[int]) + "array" + >>> python_type_to_json_type(Optional[str]) + "string" + """ origin = get_origin(py_type) args = get_args(py_type) @@ -76,9 +164,21 @@ def python_type_to_json_type(py_type): return mapping.get(py_type, "object") def get_docstring_description_input(func) -> Dict[str, Optional[str]]: - """ - Extracts descriptions for parameters from the function docstring. - Returns a dict mapping param names to their descriptions. + """Extract parameter descriptions from function docstring. + + Parses the function's docstring to extract descriptions for each parameter. + Looks for lines that start with parameter names followed by descriptions. + + Args: + func: The function to extract parameter descriptions from. + + Returns: + Dictionary mapping parameter names to their descriptions. + Parameters without descriptions are omitted. + + Example: + For a function with docstring containing "param1: Description of param1", + returns {"param1": "Description of param1"}. """ doc = func.__doc__ if not doc: @@ -93,9 +193,21 @@ def get_docstring_description_input(func) -> Dict[str, Optional[str]]: return descriptions def get_docstring_description_output(func) -> Dict[str, Optional[str]]: - """ - Extracts the return value description from the function docstring. - Returns a dict with key 'return' and its description. + """Extract return value description from function docstring. + + Parses the function's docstring to find the return value description. + Looks for lines starting with "Returns:" or "Return:". + + Args: + func: The function to extract return description from. + + Returns: + Dictionary with "return" key mapped to the description, + or empty dict if no return description is found. + + Example: + For a docstring with "Returns: The computed result", + returns {"return": "The computed result"}. """ doc = func.__doc__ if not doc: @@ -110,7 +222,21 @@ def get_docstring_description_output(func) -> Dict[str, Optional[str]]: return {"return": doc.splitlines()[i + 1].strip()} return {} -def get_param_description(cls, param_name=None): +def get_param_description(cls, param_name: Optional[str] = None) -> str: + """Extract parameter description from class docstring or field metadata. + + Attempts to find description for a parameter from various sources: + 1. Class docstring lines starting with parameter name + 2. Pydantic field descriptions + 3. Class-level description or docstring as fallback + + Args: + cls: The class to extract description from. + param_name: Optional specific parameter name to get description for. + + Returns: + Description string for the parameter or class. + """ # Try to get description for a specific param if available if param_name: # Check if there's a class variable or annotation with description @@ -124,12 +250,46 @@ def get_param_description(cls, param_name=None): # Fallback to class-level description return getattr(cls, "description", "") or (getattr(cls, "__doc__", "") or "") -def is_optional(t): +def is_optional(t) -> bool: + """Check if a type annotation represents an optional type. + + Determines if a type is Optional[T] (equivalent to Union[T, None]). + + Args: + t: Type annotation to check. + + Returns: + True if the type is optional (Union with None), False otherwise. + + Examples: + >>> is_optional(Optional[str]) + True + >>> is_optional(str) + False + >>> is_optional(Union[str, None]) + True + """ origin = get_origin(t) args = get_args(t) return origin is Union and type(None) in args -def recurse_type(param_type): +def recurse_type(param_type) -> Dict[str, Any]: + """Recursively convert Python type to JSON Schema object. + + Creates a complete JSON Schema representation of a Python type, + including nested objects, arrays, and their properties. + + Args: + param_type: Python type annotation to convert. + + Returns: + Dictionary representing the JSON Schema for the type. + Includes type, properties, items, required fields, and descriptions. + + Examples: + >>> recurse_type(List[str]) + {"type": "array", "items": {"type": "string"}, "description": "An array of items"} + """ json_type = python_type_to_json_type(param_type) # Handle array/list types @@ -176,7 +336,20 @@ def recurse_type(param_type): "description": "" } -def type_to_json_schema(param_type, param_name=None, param_description=None): +def type_to_json_schema(param_type, param_name: Optional[str] = None, param_description: Optional[Dict[str, str]] = None) -> Dict[str, Any]: + """Convert Python type to JSON Schema with description handling. + + Creates a JSON Schema representation of a Python type with appropriate + descriptions from parameter documentation or auto-generated fallbacks. + + Args: + param_type: Python type annotation to convert. + param_name: Optional parameter name for description lookup. + param_description: Optional dictionary of parameter descriptions. + + Returns: + JSON Schema dictionary with type, description, and structure information. + """ json_type = python_type_to_json_type(param_type) # Recurse for object and dict types @@ -199,7 +372,22 @@ def type_to_json_schema(param_type, param_name=None, param_description=None): return val -def generate_input_schema(func, title, description): +def generate_input_schema(func, title: Optional[str], description: Optional[str]) -> ToolInputOutputSchema: + """Generate input schema for a function's parameters. + + Analyzes a function's signature and type hints to create a JSON Schema + that describes the function's input parameters. Extracts parameter + descriptions from the function's docstring. + + Args: + func: Function to generate input schema for. + title: Optional title for the schema. + description: Optional description for the schema. + + Returns: + ToolInputOutputSchema object describing the function's input parameters. + Includes parameter types, required fields, and descriptions. + """ sig = inspect.signature(func) type_hints = get_type_hints(func) @@ -231,7 +419,22 @@ def generate_input_schema(func, title, description): return schema -def generate_output_schema(func, title, description): +def generate_output_schema(func, title: Optional[str], description: Optional[str]) -> ToolInputOutputSchema: + """Generate output schema for a function's return value. + + Analyzes a function's return type annotation to create a JSON Schema + that describes the function's output. Extracts return value description + from the function's docstring. + + Args: + func: Function to generate output schema for. + title: Optional title for the schema. + description: Optional description for the schema. + + Returns: + ToolInputOutputSchema object describing the function's return value. + Contains "result" property with the return type and description. + """ type_hints = get_type_hints(func) func_name = func.__name__ func_description = description or func.__doc__ or "" @@ -270,6 +473,43 @@ def utcp_tool( inputs: Optional[ToolInputOutputSchema] = None, outputs: Optional[ToolInputOutputSchema] = None, ): + """Decorator to convert Python functions into UTCP tools. + + Automatically generates tool definitions with input/output schemas from + function signatures and type hints. Registers the tool in the global + ToolContext for discovery. + + Args: + tool_provider: Provider configuration for accessing this tool. + name: Optional custom name for the tool. Defaults to function name. + description: Optional description. Defaults to function docstring. + tags: Optional list of tags for categorization. Defaults to ["utcp"]. + inputs: Optional manual input schema. Auto-generated if not provided. + outputs: Optional manual output schema. Auto-generated if not provided. + + Returns: + Decorator function that transforms the target function into a UTCP tool. + + Examples: + >>> @utcp_tool(HttpProvider(url="https://api.example.com")) + ... def get_weather(location: str) -> dict: + ... pass + + >>> @utcp_tool( + ... tool_provider=CliProvider(command_name="curl"), + ... name="fetch_url", + ... description="Fetch content from a URL", + ... tags=["http", "utility"] + ... ) + ... def fetch(url: str) -> str: + ... pass + + Note: + The decorated function gains additional attributes: + - input(): Returns the input schema + - output(): Returns the output schema + - tool_definition(): Returns the complete Tool object + """ def decorator(func): if tool_provider.name is None: tool_provider.name = f"{func.__name__}_provider" diff --git a/src/utcp/shared/utcp_manual.py b/src/utcp/shared/utcp_manual.py index 296a676..9a9b7e9 100644 --- a/src/utcp/shared/utcp_manual.py +++ b/src/utcp/shared/utcp_manual.py @@ -1,12 +1,44 @@ +"""UTCP manual data structure for tool discovery. + +This module defines the UtcpManual model that standardizes the format for +tool provider responses during tool discovery. It serves as the contract +between tool providers and clients for sharing available tools and their +configurations. +""" + from typing import List from pydantic import BaseModel, ConfigDict from utcp.shared.tool import Tool, ToolContext from utcp.version import __version__ - -""" -The response returned by a tool provider when queried for available tools (e.g. through the /utcp endpoint) -""" class UtcpManual(BaseModel): + """Standard format for tool provider responses during discovery. + + Represents the complete set of tools available from a provider, along + with version information for compatibility checking. This format is + returned by tool providers when clients query for available tools + (e.g., through the `/utcp` endpoint or similar discovery mechanisms). + + The manual serves as the authoritative source of truth for what tools + a provider offers and how they should be invoked. + + Attributes: + version: UTCP protocol version supported by the provider. + Defaults to the current library version. + tools: List of available tools with their complete configurations + including input/output schemas, descriptions, and metadata. + + Example: + ```python + # Create a manual from registered tools + manual = UtcpManual.create() + + # Manual with specific tools + manual = UtcpManual( + version="1.0.0", + tools=[tool1, tool2, tool3] + ) + ``` + """ version: str = __version__ tools: List[Tool] @@ -14,7 +46,28 @@ class UtcpManual(BaseModel): @staticmethod def create(version: str = __version__) -> "UtcpManual": - """Get the UTCP manual with version and tools.""" + """Create a UTCP manual from the global tool registry. + + Convenience method that creates a manual containing all tools + currently registered in the global ToolContext. This is typically + used by tool providers to generate their discovery response. + + Args: + version: UTCP protocol version to include in the manual. + Defaults to the current library version. + + Returns: + UtcpManual containing all registered tools and the specified version. + + Example: + ```python + # Create manual with default version + manual = UtcpManual.create() + + # Create manual with specific version + manual = UtcpManual.create(version="1.2.0") + ``` + """ return UtcpManual( version=version, tools=ToolContext.get_tools() From bea38dfba90b820b3425c039c5395750ab8f945e Mon Sep 17 00:00:00 2001 From: Razvan Radulescu <43811028+h3xxit@users.noreply.github.com> Date: Thu, 7 Aug 2025 11:04:41 +0200 Subject: [PATCH 2/3] Fix parsing bugs for config --- src/utcp/client/utcp_client.py | 12 ++++++------ src/utcp/client/utcp_client_config.py | 14 +++++++++++--- 2 files changed, 17 insertions(+), 9 deletions(-) diff --git a/src/utcp/client/utcp_client.py b/src/utcp/client/utcp_client.py index 9abf863..f3505c0 100644 --- a/src/utcp/client/utcp_client.py +++ b/src/utcp/client/utcp_client.py @@ -210,17 +210,17 @@ async def create(cls, config: Optional[Union[Dict[str, Any], UtcpClientConfig]] client = cls(config, tool_repository, search_strategy, DefaultVariableSubstitutor()) - # If a providers file is used, configure TextTransport to resolve relative paths from its directory - if config.providers_file_path: - providers_dir = os.path.dirname(os.path.abspath(config.providers_file_path)) - client.transports["text"] = TextTransport(base_path=providers_dir) - if client.config.variables: config_without_vars = client.config.model_copy() config_without_vars.variables = None client.config.variables = client.variable_substitutor.substitute(client.config.variables, config_without_vars) - await client.load_providers(config.providers_file_path) + # If a providers file is used, configure TextTransport to resolve relative paths from its directory + if config.providers_file_path: + providers_dir = os.path.dirname(os.path.abspath(config.providers_file_path)) + client.transports["text"] = TextTransport(base_path=providers_dir) + + await client.load_providers(config.providers_file_path) return client diff --git a/src/utcp/client/utcp_client_config.py b/src/utcp/client/utcp_client_config.py index 7649517..0ffb2fa 100644 --- a/src/utcp/client/utcp_client_config.py +++ b/src/utcp/client/utcp_client_config.py @@ -15,7 +15,7 @@ from abc import ABC, abstractmethod from pydantic import BaseModel, Field -from typing import Optional, List, Dict, Literal, TypedDict +from typing import Optional, List, Dict, Annotated, Union, Literal from dotenv import dotenv_values class UtcpVariableNotFound(Exception): @@ -51,7 +51,7 @@ class UtcpVariablesConfig(BaseModel, ABC): Attributes: type: Type identifier for the variable loader. """ - type: Literal["dotenv"] = "dotenv" + type: str @abstractmethod def get(self, key: str) -> Optional[str]: @@ -81,6 +81,7 @@ class UtcpDotEnv(UtcpVariablesConfig): api_key = loader.get("API_KEY") ``` """ + type: Literal["dotenv"] = "dotenv" env_file_path: str def get(self, key: str) -> Optional[str]: @@ -94,6 +95,13 @@ def get(self, key: str) -> Optional[str]: """ return dotenv_values(self.env_file_path).get(key) +UtcpVariablesConfigUnion = Annotated[ + Union[ + UtcpDotEnv + ], + Field(discriminator="type") +] + class UtcpClientConfig(BaseModel): """Configuration model for UTCP client setup. @@ -128,4 +136,4 @@ class UtcpClientConfig(BaseModel): """ variables: Optional[Dict[str, str]] = Field(default_factory=dict) providers_file_path: Optional[str] = None - load_variables_from: Optional[List[UtcpVariablesConfig]] = None + load_variables_from: Optional[List[UtcpVariablesConfigUnion]] = None From baf04a7e7aa4525f7a5382c98387216fb5672324 Mon Sep 17 00:00:00 2001 From: Razvan Radulescu <43811028+h3xxit@users.noreply.github.com> Date: Thu, 7 Aug 2025 11:09:15 +0200 Subject: [PATCH 3/3] 0.2.1 --- pyproject.toml | 2 +- src/utcp/version.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 37532c4..c16f332 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "utcp" -version = "0.2.0" +version = "0.2.1" authors = [ { name = "Razvan-Ion Radulescu" }, { name = "Andrei-Stefan Ghiurtu" }, diff --git a/src/utcp/version.py b/src/utcp/version.py index 675a6b3..7deadba 100644 --- a/src/utcp/version.py +++ b/src/utcp/version.py @@ -2,7 +2,7 @@ import tomli from pathlib import Path -__version__ = "0.2.0" +__version__ = "0.2.1" try: __version__ = version("utcp") except PackageNotFoundError: