feat: adapt proxy

Author: diogoncalves

examples/01_intro_to_llmstudio copy.ipynb

@@ -16,28 +16,32 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 33,
+   "execution_count": 1,
    "metadata": {},
-   "outputs": [
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "Running LLMstudio Engine on http://localhost:55189 Running LLMstudio Tracking on http://localhost:55190 \n",
-      "\n"
-     ]
-    }
-   ],
+   "outputs": [],
    "source": [
     "from llmstudio.llm.langchain import ChatLLMstudio\n",
     "from llmstudio import LLM"
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": 38,
+   "execution_count": 2,
    "metadata": {},
-   "outputs": [],
+   "outputs": [
+    {
+     "ename": "TypeError",
+     "evalue": "Can't instantiate abstract class LLMProxyProvider without an implementation for abstract method '_provider_config_name'",
+     "output_type": "error",
+     "traceback": [
+      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
+      "\u001b[0;31mTypeError\u001b[0m                                 Traceback (most recent call last)",
+      "Cell \u001b[0;32mIn[2], line 2\u001b[0m\n\u001b[1;32m      1\u001b[0m \u001b[38;5;66;03m# llm = ChatLLMstudio(model_id='openai/gpt-3.5-turbo', temperature=0)\u001b[39;00m\n\u001b[0;32m----> 2\u001b[0m llm \u001b[38;5;241m=\u001b[39m \u001b[43mChatLLMstudio\u001b[49m\u001b[43m(\u001b[49m\u001b[43mmodel_id\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;124;43m'\u001b[39;49m\u001b[38;5;124;43mvertexai/gemini-1.5-flash\u001b[39;49m\u001b[38;5;124;43m'\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mtemperature\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;241;43m0\u001b[39;49m\u001b[43m)\u001b[49m\n",
+      "File \u001b[0;32m~/fun/LLMstudio/llmstudio/llm/langchain.py:33\u001b[0m, in \u001b[0;36mChatLLMstudio.__init__\u001b[0;34m(self, model_id, **kwargs)\u001b[0m\n\u001b[1;32m     31\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21m__init__\u001b[39m(\u001b[38;5;28mself\u001b[39m, model_id: \u001b[38;5;28mstr\u001b[39m, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs):\n\u001b[1;32m     32\u001b[0m     \u001b[38;5;28msuper\u001b[39m()\u001b[38;5;241m.\u001b[39m\u001b[38;5;21m__init__\u001b[39m(model_id\u001b[38;5;241m=\u001b[39mmodel_id, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs)\n\u001b[0;32m---> 33\u001b[0m     \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mllm \u001b[38;5;241m=\u001b[39m \u001b[43mLLM\u001b[49m\u001b[43m(\u001b[49m\u001b[43mmodel_id\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mmodel_id\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n",
+      "\u001b[0;31mTypeError\u001b[0m: Can't instantiate abstract class LLMProxyProvider without an implementation for abstract method '_provider_config_name'"
+     ]
+    }
+   ],
    "source": [
     "# llm = ChatLLMstudio(model_id='openai/gpt-3.5-turbo', temperature=0)\n",
     "llm = ChatLLMstudio(model_id='vertexai/gemini-1.5-flash', temperature=0)"
@@ -52,7 +56,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 37,
+   "execution_count": 3,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -62,16 +66,16 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 35,
+   "execution_count": 4,
    "metadata": {},
    "outputs": [
     {
      "data": {
       "text/plain": [
-       "AIMessage(content='Hello! \\n\\nHow can I help you today? \\n', response_metadata={'token_usage': None, 'model_name': 'gemini-1.5-flash', 'system_fingerprint': None, 'finish_reason': 'stop', 'logprobs': None}, id='run-5945f8bd-7151-4d85-bf90-3bcc1eaabc6c-0')"
+       "AIMessage(content='Hello! 👋  How can I help you today? 😊 \\n', response_metadata={'token_usage': None, 'model_name': 'gemini-1.5-flash', 'system_fingerprint': None, 'finish_reason': 'stop', 'logprobs': None}, id='run-2e60f8e1-bba0-4cab-9e21-8ecdbe8d49a2-0')"
       ]
      },
-     "execution_count": 35,
+     "execution_count": 4,
      "metadata": {},
      "output_type": "execute_result"
     }
@@ -388,7 +392,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.11.9"
+   "version": "3.12.2"
   }
  },
  "nbformat": 4,

examples/01_intro_to_llmstudio_with_proxy.ipynb

@@ -0,0 +1,28 @@
+from llmstudio.server import start_server
+start_server()
+
+from llmstudio.engine.provider import LLMProxyProvider
+
+
+llm = LLMProxyProvider(provider="openai", host="0.0.0.0", port="8001")
+
+result = llm.chat("What's your name", model="gpt-4o")
+print(result)
+
+import asyncio
+
+# stream
+print("\nasync stream")
+async def async_stream():
+    
+    response_async = await llm.achat("What's your name", model="gpt-4o", is_stream=True)
+    async for p in response_async:
+        if "}" in p.chat_output:
+            p.chat_output
+        print("that: ",p.chat_output)
+        # pprint(p.choices[0].delta.content==p.chat_output)
+        # print("metrics: ", p.metrics)
+        # print(p)
+        if p.metrics:
+            print(p)
+asyncio.run(async_stream())

examples/03_langchain_integration.ipynb

@@ -73,7 +73,8 @@ async def achat(
         model: str,
         is_stream: Optional[bool] = False,
         retries: Optional[int] = 0,
-        parameters: Optional[dict] = {}
+        parameters: Optional[dict] = {},
+        **kwargs
     ) -> Coroutine[Any, Any, Union[ChatCompletionChunk, ChatCompletion]]:
         raise NotImplementedError("Providers needs to have achat method implemented.")
 
@@ -84,7 +85,8 @@ def chat(
         model: str,
         is_stream: Optional[bool] = False,
         retries: Optional[int] = 0,
-        parameters: Optional[dict] = {}
+        parameters: Optional[dict] = {},
+        **kwargs
     ) -> Union[ChatCompletionChunk, ChatCompletion]:
         raise NotImplementedError("Providers needs to have chat method implemented.")
 
@@ -103,7 +105,8 @@ async def achat(
         model: str,
         is_stream: Optional[bool] = False,
         retries: Optional[int] = 0,
-        parameters: Optional[dict] = {}
+        parameters: Optional[dict] = {},
+        **kwargs
     ):
 
         """Makes a chat connection with the provider's API"""
@@ -145,7 +148,8 @@ def chat(
         model: str,
         is_stream: Optional[bool] = False,
         retries: Optional[int] = 0,
-        parameters: Optional[dict] = {}
+        parameters: Optional[dict] = {},
+        **kwargs
     ):
 
         """Makes a chat connection with the provider's API"""

examples/llm_proxy.py

@@ -1,5 +1,6 @@
 import os
 import signal
+import threading
 
 import click
 
@@ -25,11 +26,14 @@ def server(ui):
 
     print("Servers are running. Press CTRL+C to stop.")
 
+    stop_event = threading.Event()
     try:
-        signal.pause()
+        stop_event.wait()  # Wait indefinitely until the event is set
     except KeyboardInterrupt:
         print("Shutting down servers...")
 
 
 if __name__ == "__main__":
-    main()
+    # main()
+    server()
+    print(4)

libs/core/llmstudio_core/providers/provider.py

@@ -9,10 +9,11 @@
 from fastapi import FastAPI, Request
 from fastapi.middleware.cors import CORSMiddleware
 from fastapi.responses import StreamingResponse
-from pydantic import BaseModel, ValidationError
+from pydantic import BaseModel
 
 from llmstudio.config import ENGINE_HOST, ENGINE_PORT
-from llmstudio.engine.providers import *
+from llmstudio_core.providers import _load_engine_config
+from llmstudio_core.providers.provider import provider_registry
 
 ENGINE_BASE_ENDPOINT = "/api/engine"
 ENGINE_HEALTH_ENDPOINT = "/health"
@@ -47,38 +48,6 @@ class EngineConfig(BaseModel):
     providers: Dict[str, ProviderConfig]
 
 
-def _load_engine_config() -> EngineConfig:
-    default_config_path = Path(os.path.join(os.path.dirname(__file__), "config.yaml"))
-    local_config_path = Path(os.getcwd(), "config.yaml")
-
-    def _merge_configs(config1, config2):
-        for key in config2:
-            if key in config1:
-                if isinstance(config1[key], dict) and isinstance(config2[key], dict):
-                    _merge_configs(config1[key], config2[key])
-                elif isinstance(config1[key], list) and isinstance(config2[key], list):
-                    config1[key].extend(config2[key])
-                else:
-                    config1[key] = config2[key]
-            else:
-                config1[key] = config2[key]
-        return config1
-
-    try:
-        default_config_data = yaml.safe_load(default_config_path.read_text())
-        local_config_data = (
-            yaml.safe_load(local_config_path.read_text())
-            if local_config_path.exists()
-            else {}
-        )
-        config_data = _merge_configs(default_config_data, local_config_data)
-        return EngineConfig(**config_data)
-    except yaml.YAMLError as e:
-        raise RuntimeError(f"Error parsing YAML configuration: {e}")
-    except ValidationError as e:
-        raise RuntimeError(f"Error in configuration data: {e}")
-
-
 def create_engine_app(
     started_event: Event, config: EngineConfig = _load_engine_config()
 ) -> FastAPI:
@@ -124,9 +93,11 @@ def get_models(provider: Optional[str] = None):
     def create_chat_handler(provider_config):
         async def chat_handler(request: Request):
             """Endpoint for chat functionality."""
-            provider_class = provider_registry.get(f"{provider_config.name}Provider")
+            provider_class = provider_registry.get(f"{provider_config.name}".lower())
             provider_instance = provider_class(provider_config)
-            return await provider_instance.chat(await request.json())
+            request_dict = await request.json()
+            result = await provider_instance.achat(**request_dict)
+            return result
 
         return chat_handler