update

Author: minmingzhu

examples/inference/api_server_openai/query_http_requests_tool.py

@@ -73,15 +73,14 @@
 
 messages = [
     [
-        {"role": "user", "content": "You are a helpful assistant"},
         {"role": "user", "content": "What's the weather like in Boston today?"},
     ],
 ]
 
 proxies = {"http": None, "https": None}
 
 for message in messages:
-    print(f"User: {message[1]['content']}")
+    print(f"User: {message[0]['content']}")
     print("Assistant:", end=" ", flush=True)
 
     body = {

llm_on_ray/inference/models/gpt-j-6b.yaml

@@ -14,4 +14,4 @@ ipex:
 model_description:
   model_id_or_path: EleutherAI/gpt-j-6b
   tokenizer_name_or_path: EleutherAI/gpt-j-6b
-  gpt_base_model: true
+  gpt_base_model: true

llm_on_ray/inference/predictor_deployment.py

@@ -310,9 +310,6 @@ def preprocess_prompts(self, input: Union[str, List], tools=None, tool_choice=No
         Raises:
             HTTPException: If the input prompt format is invalid or not supported.
         """
-        logger.info("preprocess_prompts")
-        logger.info(input)
-        logger.info(type(input))
 
         if isinstance(input, str):
             return input
@@ -357,7 +354,6 @@ def preprocess_prompts(self, input: Union[str, List], tools=None, tool_choice=No
             raise HTTPException(400, "Invalid prompt format.")
 
     async def __call__(self, http_request: Request) -> Union[StreamingResponse, JSONResponse, str]:
-        logger.info("PredictorDeployment call")
         self.use_openai = False
 
         try:
@@ -376,7 +372,6 @@ async def __call__(self, http_request: Request) -> Union[StreamingResponse, JSON
                 content="Empty prompt is not supported.",
             )
         config = json_request["config"] if "config" in json_request else {}
-        logger.info(input)
         # return prompt or list of prompts preprocessed
         prompts = self.preprocess_prompts(input)
 

llm_on_ray/ui/start_ui.py

@@ -31,11 +31,7 @@
 from ray.util import queue
 from llm_on_ray.inference.inference_config import all_models, ModelDescription, Prompt
 from llm_on_ray.inference.inference_config import InferenceConfig as FinetunedConfig
-from llm_on_ray.inference.chat_template_process import (
-    ChatModelGptJ,
-    ChatModelLLama,
-    ChatModelwithImage,
-)
+
 from llm_on_ray.inference.predictor_deployment import PredictorDeployment
 from llm_on_ray.ui.html_format import cpu_memory_html, ray_status_html, custom_css
 from langchain.vectorstores import FAISS