Handle DEFAULT_META_PROVIDER_CONFIG

Author: ajhai

docker/.env.dev

@@ -25,6 +25,7 @@ DEFAULT_AZURE_OPENAI_API_KEY=
 DEFAULT_COHERE_API_KEY=
 DEFAULT_FOREFRONTAI_API_KEY=
 DEFAULT_ELEVENLABS_API_KEY=
+DEFAULT_META_PROVIDER_CONFIG=
 
 # Runner
 RUNNER_WSS_HOSTNAME=localhost

llmstack/base/models.py

@@ -102,18 +102,9 @@ def get_vendor_env_platform_defaults():
         promptly_provider_config.embeddings_generator = EmbeddingsGeneratorConfig(
             **settings.DEFAULT_EMBEDDINGS_GENERATOR_CONFIG
         )
-    if settings.DEFAULT_META_PROVIDER_CONFIG:
-        meta_provider_configs = {}
-        try:
-            meta_provider_configs = json.loads(settings.DEFAULT_META_PROVIDER_CONFIG)
-        except json.JSONDecodeError as e:
-            logger.error(
-                f"Error parsing DEFAULT_META_PROVIDER_CONFIG: {settings.DEFAULT_META_PROVIDER_CONFIG}",
-            )
-            logger.error(e)
-
+    if settings.DEFAULT_META_PROVIDER_CONFIG and isinstance(settings.DEFAULT_META_PROVIDER_CONFIG, dict):
         # Iterate over the meta provider config and add the provider configs
-        for k, v in meta_provider_configs.items():
+        for k, v in settings.DEFAULT_META_PROVIDER_CONFIG.items():
             provider_configs[k] = MetaProviderConfig(
                 deployment_config=v,
                 provider_config_source=ProviderConfigSource.PLATFORM_DEFAULT.value,

llmstack/processors/providers/meta/chat_completions.py

@@ -3,7 +3,6 @@
 from typing import List, Optional
 
 from asgiref.sync import async_to_sync
-from django.conf import settings
 from pydantic import Field
 
 from llmstack.apps.schemas import OutputTemplate
@@ -161,28 +160,9 @@ def session_data_to_persist(self) -> dict:
     def process(self) -> MessagesOutput:
         from llmstack.common.utils.sslr import LLM
 
-        deployment_configs = settings.CUSTOM_MODELS_DEPLOYMENT_CONFIG.get(
-            f"{self.provider_slug()}/{self._config.model.model_name()}"
-        )
-        if not deployment_configs:
-            raise Exception(
-                f"Model deployment config not found for {self.provider_slug()}/{self._config.model.model_name()}"
-            )
-
-        model_deployment_configs = []
-
-        if not self._config.deployment_names:
-            if deployment_configs.get("default"):
-                model_deployment_configs = [deployment_configs["default"]]
-        else:
-            for entry in self._config.deployment_names:
-                if deployment_configs.get(entry):
-                    model_deployment_configs.append(deployment_configs.get(entry))
-
-        if not model_deployment_configs:
-            raise Exception(
-                f"Model deployment config not found for {self.provider_slug()}/{self._config.model.model_name()}"
-            )
+        deployment_config = self.get_provider_config(model_slug=self._config.model)
+        if not deployment_config:
+            raise Exception(f"Model deployment config not found for {self.provider_slug()}/{self._config.model}")
 
         messages = []
 
@@ -196,7 +176,7 @@ def process(self) -> MessagesOutput:
         for message in self._input.messages:
             messages.append({"role": str(message.role), "content": str(message.message)})
 
-        client = LLM(provider="custom", deployment_config=model_deployment_configs[0])
+        client = LLM(provider="custom", deployment_config=deployment_config.model_dump().get("deployment_config"))
 
         result = client.chat.completions.create(
             messages=messages,

llmstack/server/settings.py

@@ -365,7 +365,15 @@
     "DEFAULT_GOOGLE_CUSTOM_SEARCH_CX",
     "",
 )
-DEFAULT_META_PROVIDER_CONFIG = os.getenv("DEFAULT_META_PROVIDER_CONFIG", "{}")
+DEFAULT_META_PROVIDER_CONFIG = {}
+try:
+    DEFAULT_META_PROVIDER_CONFIG = (
+        json.loads(base64.b64decode(os.getenv("DEFAULT_META_PROVIDER_CONFIG")))
+        if os.getenv("DEFAULT_META_PROVIDER_CONFIG")
+        else {}
+    )
+except Exception:
+    print("Error parsing DEFAULT_META_PROVIDER_CONFIG")
 
 WEAVIATE_URL = os.getenv("WEAVIATE_URL", "http://weaviate:8080")
 WEAVIATE_TEXT2VEC_MODULE_CONFIG = {
@@ -682,16 +690,6 @@
 
 ENABLE_JOBS = os.getenv("ENABLE_JOBS", "True") == "True"
 
-CUSTOM_MODELS_DEPLOYMENT_CONFIG = {}
-try:
-    CUSTOM_MODELS_DEPLOYMENT_CONFIG = (
-        json.loads(base64.b64decode(os.getenv("CUSTOM_MODELS_DEPLOYMENT_CONFIG")))
-        if os.getenv("CUSTOM_MODELS_DEPLOYMENT_CONFIG")
-        else {}
-    )
-except Exception:
-    print("Error parsing CUSTOM_MODELS_DEPLOYMENT_CONFIG")
-
 CONNECTION_TYPE_INTERFACE_EXCLUDED_PACKAGES = os.getenv("CONNECTION_TYPE_INTERFACE_EXCLUDED_PACKAGES", "").split(",")
 
 DEFAULT_DATA_DESTINATION_CONFIG = {"provider_slug": "weaviate", "processor_slug": "vector-store"}