From c501097e9c261b1b4815bbbc4dd2cf11439592bc Mon Sep 17 00:00:00 2001 From: Ioana Barbos Date: Thu, 31 Jul 2025 11:52:36 +0000 Subject: [PATCH 1/9] test commit ioana --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 6056a42..393ef88 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,5 @@ # Agent Bootcamp - +test commit ---------------------------------------------------------------------------------------- This is a collection of reference implementations for Vector Institute's **Agent Bootcamp**, taking place between June and September 2025. The repository demonstrates modern agentic workflows for retrieval-augmented generation (RAG), evaluation, and orchestration using the latest Python tools and frameworks. From 2cd9c3942b5e2d58f0490483e56197748aed0bf0 Mon Sep 17 00:00:00 2001 From: Ioana Barbos Date: Wed, 6 Aug 2025 14:31:58 +0000 Subject: [PATCH 2/9] MVP working --- .../planner_worker_gradio_ioana.py | 217 ++++++++++++++++++ src/prompts.py | 4 +- 2 files changed, 220 insertions(+), 1 deletion(-) create mode 100644 src/2_frameworks/2_multi_agent/planner_worker_gradio_ioana.py diff --git a/src/2_frameworks/2_multi_agent/planner_worker_gradio_ioana.py b/src/2_frameworks/2_multi_agent/planner_worker_gradio_ioana.py new file mode 100644 index 0000000..99d663c --- /dev/null +++ b/src/2_frameworks/2_multi_agent/planner_worker_gradio_ioana.py @@ -0,0 +1,217 @@ +"""Example code for planner-worker agent collaboration. + +With reference to: + +github.com/ComplexData-MILA/misinfo-datasets +/blob/3304e6e/misinfo_data_eval/tasks/web_search.py +""" + +import asyncio +import contextlib +import logging +import signal +import sys + +import agents +import gradio as gr +from dotenv import load_dotenv +from gradio.components.chatbot import ChatMessage +from openai import AsyncOpenAI + +from src.prompts import REACT_INSTRUCTIONS +from src.utils import ( + AsyncWeaviateKnowledgeBase, + Configs, + get_weaviate_async_client, + oai_agent_stream_to_gradio_messages, + setup_langfuse_tracer, +) +from src.utils.langfuse.shared_client import langfuse_client + + +load_dotenv(verbose=True) + + +logging.basicConfig(level=logging.INFO) + + +AGENT_LLM_NAMES = { + "worker": "gemini-2.5-flash", # less expensive, + "planner": "gemini-2.5-pro", # more expensive, better at reasoning and planning +} + +configs = Configs.from_env_var() +async_weaviate_client = get_weaviate_async_client( + http_host=configs.weaviate_http_host, + http_port=configs.weaviate_http_port, + http_secure=configs.weaviate_http_secure, + grpc_host=configs.weaviate_grpc_host, + grpc_port=configs.weaviate_grpc_port, + grpc_secure=configs.weaviate_grpc_secure, + api_key=configs.weaviate_api_key, +) +async_openai_client = AsyncOpenAI() +async_knowledgebase = AsyncWeaviateKnowledgeBase( + async_weaviate_client, + collection_name="enwiki_20250520", +) + + +async def _cleanup_clients() -> None: + """Close async clients.""" + await async_weaviate_client.close() + await async_openai_client.close() + + +def _handle_sigint(signum: int, frame: object) -> None: + """Handle SIGINT signal to gracefully shutdown.""" + with contextlib.suppress(Exception): + asyncio.get_event_loop().run_until_complete(_cleanup_clients()) + sys.exit(0) + +# Worker Agent QA: handles long context efficiently +import os +import openai +import numpy as np +from sklearn.metrics.pairwise import cosine_similarity +from agents import function_tool +# from pydantic import BaseModel + + + +@function_tool +async def faq_match_tool(user_query:str)->list: + '''Return a list of FAQ sorted from the most simialr to least similar to the user query by cosine similarity. + ''' + faq_list = ["Where are Toyota cars manufactured? Toyota cars are produced in Germany and Japan", + "What is the engine power of Toyota RAV4? 180HP","Where is Japan?", + "What is horse power in cars?", "What is the capital of Germany? it's Berlin", + "Is Toyota a German brand? No, it's a Japanese automobile brand."] + + _embed_client = openai.OpenAI( + api_key=os.getenv("EMBEDDING_API_KEY"), + base_url=os.getenv("EMBEDDING_BASE_URL"), + max_retries=5) + + #embed user query + user_query_embedding = _embed_client.embeddings.create(input=user_query, model=os.getenv('EMBEDDING_MODEL_NAME')) + user_query_embedding = np.array(user_query_embedding.data[0].embedding) + user_query_embedding = user_query_embedding.reshape(1, -1) + + cosi_list = [] + faq_embedding_list = _embed_client.embeddings.create(input=faq_list, model=os.getenv('EMBEDDING_MODEL_NAME')) + for i, faq_embedding in enumerate(faq_embedding_list.data): + faq_embedding = np.array(faq_embedding.embedding) + faq_embedding = faq_embedding.reshape(1,-1) + similarity_score = cosine_similarity(user_query_embedding, faq_embedding)[0][0] + cosi_list.append({"faq":faq_list[i], "sim":similarity_score}) + + sorted_faqs = sorted(cosi_list, key=lambda d: d["sim"], reverse=True) + sorted_faqs_list = [i["faq"] for i in sorted_faqs] + return "\n".join(f" {i}\n"for i in sorted_faqs_list) + # return sorted_faqs_list + +# class FAQ(BaseModel): +# user_query: str +# similar_faqs: list[str] +# def __str__(self) -> str: +# """Return a string representation of the faq list""" +# return "\n".join( +# f"faq {step}\n" +# for step in self.similar_faqs +# ) + +faq_agent = agents.Agent( + name="QAMatchAgent", + instructions=( + "You are an agent specializing in matching user queries to FAQ. You receive a single user query as input. " + "Use the faq_match_tool tool to return a sorted list of FAQ based on how similar they are with the user query." + "Return maximum 3 FAQs that best match the user query. If you can't find any match, return the raw user query." + "ALWAYS structure the output as a list that contains [user request, faqs if any]." + + ), + tools=[faq_match_tool], + # a faster, smaller model for quick searches + model=agents.OpenAIChatCompletionsModel( + model="gemini-2.5-flash", openai_client=async_openai_client + ), + model_settings=agents.ModelSettings(tool_choice="required"), + # output_type=FAQ +) + +# Worker Agent: handles long context efficiently +search_agent = agents.Agent( + name="SearchAgent", + instructions=( + "You are a search agent. You receive a search query and a list of FAQ as input. " + "Use the WebSearchTool to perform a web search on the search query, then produce a concise " + "'search summary' of the key findings. Corroborate the findings with the FAQ into a final answer. Do NOT return raw search results." + ), + tools=[ + agents.function_tool(async_knowledgebase.search_knowledgebase), + ], + # a faster, smaller model for quick searches + model=agents.OpenAIChatCompletionsModel( + model="gemini-2.5-flash", openai_client=async_openai_client + ), +) + +# Main Agent: more expensive and slower, but better at complex planning +main_agent = agents.Agent( + name="MainAgent", + instructions=REACT_INSTRUCTIONS, + # Allow the planner agent to invoke the worker agents. + # The long context provided to the worker agent is hidden from the main agent. + tools=[ + faq_agent.as_tool( + tool_name="faq_match", + tool_description = "Identify the matching FAQs in the database." + ), + search_agent.as_tool( + tool_name="search", + tool_description="Perform a web search for a query and return a concise summary.", + ) + ], + # a larger, more capable model for planning and reasoning over summaries + model=agents.OpenAIChatCompletionsModel( + model="gemini-2.5-pro", openai_client=async_openai_client + ), +) + + +async def _main(question: str, gr_messages: list[ChatMessage]): + setup_langfuse_tracer() + + # Use the main agent as the entry point- not the worker agent. + with langfuse_client.start_as_current_span(name="Agents-SDK-Trace") as span: + span.update(input=question) + + result_stream = agents.Runner.run_streamed(main_agent, input=question) + async for _item in result_stream.stream_events(): + gr_messages += oai_agent_stream_to_gradio_messages(_item) + if len(gr_messages) > 0: + yield gr_messages + + span.update(output=result_stream.final_output) + + +demo = gr.ChatInterface( + _main, + title="2.2 Multi-Agent for Efficiency", + type="messages", + examples=[ + "what is toyota? ", + "How does the annual growth in the 50th-percentile income " + "in the US compare with that in Canada?", + ], +) + +if __name__ == "__main__": + async_openai_client = AsyncOpenAI() + + signal.signal(signal.SIGINT, _handle_sigint) + + try: + demo.launch(server_name="0.0.0.0") + finally: + asyncio.run(_cleanup_clients()) diff --git a/src/prompts.py b/src/prompts.py index 7dc95d4..22336c7 100644 --- a/src/prompts.py +++ b/src/prompts.py @@ -1,7 +1,9 @@ """Centralized location for all system prompts.""" REACT_INSTRUCTIONS = """\ -Answer the question using the search tool. \ +Answer the question in two steps, always in the same order: \ +step 1 use the faq_match tool. If you can find the answer from the FAQs alone, say "hurray!" and give preliminary answer.\ +step 2 use the search tool to enhance the answer from the first step. \ EACH TIME before invoking the function, you must explain your reasons for doing so. \ Be sure to mention the sources in your response. \ If the search tool did not return intended results, try again. \ From 0193be2c982c7979fee0529c615c8212651c3533 Mon Sep 17 00:00:00 2001 From: Calen Irwin Date: Wed, 6 Aug 2025 12:57:35 +0000 Subject: [PATCH 3/9] wip: initial multi agent orchestrator --- .gradio/flagged/dataset1.csv | 2 + src/4_hitachi/1_multi_agent/README.md | 12 ++ .../1_multi_agent/orchestrator_gradio.py | 147 ++++++++++++++++++ src/4_hitachi/__init__.py | 0 src/prompts.py | 11 ++ 5 files changed, 172 insertions(+) create mode 100644 .gradio/flagged/dataset1.csv create mode 100644 src/4_hitachi/1_multi_agent/README.md create mode 100644 src/4_hitachi/1_multi_agent/orchestrator_gradio.py create mode 100644 src/4_hitachi/__init__.py diff --git a/.gradio/flagged/dataset1.csv b/.gradio/flagged/dataset1.csv new file mode 100644 index 0000000..627347a --- /dev/null +++ b/.gradio/flagged/dataset1.csv @@ -0,0 +1,2 @@ +keyword,output,timestamp +Apple SVP Software Engineering,"[{""source"": {""title"": ""Craig Federighi"", ""section"": ""short description""}, ""highlight"": {""text"": [""Craig Federighi (born May 27, 1969) is an American engineer and business executive who is the senior vice president (SVP) of software engineering at Apple Inc. He oversees the development of Apple's operating systems. His teams are responsible for delivering the software of Apple's products, including the user interface, applications, and frameworks.""]}}, {""source"": {""title"": ""Sabih Khan"", ""section"": ""Infobox person\n""}, ""highlight"": {""text"": [""Sabih Khan (born 1966) is an Indian-American business executive, who is the senior vice president (SVP) of operations at Apple Inc. He oversees Apple's global supply chain, and is responsible for Apple's supplier responsibility programs.""]}}, {""source"": {""title"": ""University of Naples Federico II"", ""section"": ""Apple Developer Academy""}, ""highlight"": {""text"": ["" Apple Developer Academy \nThe Apple Developer Academy is a university academy established on October 6, 2016, in collaboration with the American company Apple Inc.. It is situated in the San Giovanni Complex, located in the San Giovanni a Teduccio district.\n\nThe training primarily focuses on software development and app design tailored for the Apple ecosystem. The training areas are categorized into:\n\n Programming (Swift, server-side scripting, SQL, NoSQL)\n Graphical interface design (HCI)\n Business\n\nThe lessons are centered around Challenge-based learning (CBL), a multidisciplinary approach that motivates students to leverage everyday technologies to solve real-world problems. As of December 2023, the Academy has welcomed over 1700 students, resulting in the creation and deployment of more than 800 applications.""]}}, {""source"": {""title"": ""RSVP-TE"", ""section"": ""short description""}, ""highlight"": {""text"": [""Resource Reservation Protocol - Traffic Engineering (RSVP-TE) is an extension of the Resource Reservation Protocol (RSVP) for traffic engineering. It supports the reservation of resources across an IP network. Applications running on IP end systems can use RSVP to indicate to other nodes the nature (bandwidth, jitter, maximum burst, and so forth) of the packet streams they want to receive. RSVP runs on both IPv4 and IPv6.\n\nRSVP-TE generally allows the establishment of Multiprotocol Label Switching (MPLS) label-switched paths (LSPs), taking into consideration network constraint parameters such as available bandwidth and explicit hops. Updated by , , , , , , , , and .""]}}, {""source"": {""title"": ""VSIP"", ""section"": ""notability""}, ""highlight"": {""text"": [""The Visual Studio Industry Partner (VSIP) Program (formerly Visual Studio Integration Program) allows third-party developers and software vendors to develop tools, components and languages for use in the Microsoft Visual Studio .NET IDE. The program offers partnership benefits including co-marketing opportunities, and Visual Studio licensing options as well as extended access to Microsoft technical and premier support.\n\nThe VSIP SDK (software development kit) facilitates development of integrated tools and includes development software and documentation that can be used within the Visual Studio .NET IDE directly. Extensions to the IDE, also known as \""Add-ins\"", can be as simple as adding a custom server control to the toolbox, or as complex as adding support for a new CLR-compliant language.\n\nVisual Studio Express is limited and does not support third-party extensions.""]}}]",2025-07-15 15:27:15.631174 diff --git a/src/4_hitachi/1_multi_agent/README.md b/src/4_hitachi/1_multi_agent/README.md new file mode 100644 index 0000000..d3ff2de --- /dev/null +++ b/src/4_hitachi/1_multi_agent/README.md @@ -0,0 +1,12 @@ +# 4.1 Multi-agent Orchestrator-QA Search-Knowledge Base Search via OpenAI Agents SDK + +This folder introduces a multi-agent architecture, featuring an orchestrator agent and two search agents, one with access to QA dataset and the other with access to Knowledge Base dataset. + +The orchestrator agents take a user query and breaks it down into search queries for the QA dataset. It then takes the returned QA pairs and breaks down searches for the Knowledge Base. The Knowledge Base search agent calls the search tool and synthesizes the results into an answer for each question. The orchestrator agent then receives the resulting answers and evaluates them based on the ground truth answers retrieved from the QA search. + +## Run + +```bash +uv run --env-file .env \ +-m src.4_hitachi.1_multi_agent.orchestrator_gradio +``` diff --git a/src/4_hitachi/1_multi_agent/orchestrator_gradio.py b/src/4_hitachi/1_multi_agent/orchestrator_gradio.py new file mode 100644 index 0000000..45f5660 --- /dev/null +++ b/src/4_hitachi/1_multi_agent/orchestrator_gradio.py @@ -0,0 +1,147 @@ +"""Example code for orchestrator-worker agent collaboration. + +With reference to: + +github.com/ComplexData-MILA/misinfo-datasets +/blob/3304e6e/misinfo_data_eval/tasks/web_search.py +""" + +import asyncio +import contextlib +import logging +import signal +import sys + +import agents +import gradio as gr +from dotenv import load_dotenv +from gradio.components.chatbot import ChatMessage +from openai import AsyncOpenAI + +from src.prompts import REACT_INSTRUCTIONS +from src.utils import ( + AsyncWeaviateKnowledgeBase, + Configs, + get_weaviate_async_client, + oai_agent_stream_to_gradio_messages, + setup_langfuse_tracer, +) +from src.utils.langfuse.shared_client import langfuse_client + + +load_dotenv(verbose=True) + + +logging.basicConfig(level=logging.INFO) + +DATASET_NAME = "hitachi-multi-agent-orchestrator" +AGENT_LLM_NAMES = { + "worker": "gemini-2.5-flash", # less expensive, + "planner": "gemini-2.5-pro", # more expensive, better at reasoning and planning +} + +configs = Configs.from_env_var() +async_weaviate_client = get_weaviate_async_client( + http_host=configs.weaviate_http_host, + http_port=configs.weaviate_http_port, + http_secure=configs.weaviate_http_secure, + grpc_host=configs.weaviate_grpc_host, + grpc_port=configs.weaviate_grpc_port, + grpc_secure=configs.weaviate_grpc_secure, + api_key=configs.weaviate_api_key, +) +async_openai_client = AsyncOpenAI() +async_knowledgebase = AsyncWeaviateKnowledgeBase( + async_weaviate_client, + collection_name="enwiki_20250520", +) + + +async def _cleanup_clients() -> None: + """Close async clients.""" + await async_weaviate_client.close() + await async_openai_client.close() + + +def _handle_sigint(signum: int, frame: object) -> None: + """Handle SIGINT signal to gracefully shutdown.""" + with contextlib.suppress(Exception): + asyncio.get_event_loop().run_until_complete(_cleanup_clients()) + sys.exit(0) + + +# Knowledgebase Search Agent: a simple agent that searches the knowledge base +knowledgebase_agent = agents.Agent( + name="KnowledgeBaseSearchAgent", + instructions=( + "You are search agent. You receive a single search query formatted as a question as input. " + "Use the search_knowledgebase tool to perform a search, then produce an accurate and concise " + "answer to the question based on the search results. If you cannot find an answer, state that you do not know the answer.", + ), + tools=[ + agents.function_tool(async_knowledgebase.search_knowledgebase), + ], + # a faster, smaller model for quick searches + model=agents.OpenAIChatCompletionsModel( + model="gemini-2.5-flash", openai_client=async_openai_client + ), +) + +# Main Agent: more expensive and slower, but better at complex planning +orchestrator_agent = agents.Agent( + name="OrchestratorAgent", + instructions=REACT_INSTRUCTIONS, + + # Allow the planner agent to invoke the worker agent. + # The long context provided to the worker agent is hidden from the main agent. + tools=[ + knowledgebase_agent.as_tool( + tool_name="KnowledgeBaseSearchAgent", + tool_description="Perform a search in the knowledge base and return a concise answer.", + ) + ], + # a larger, more capable model for planning and reasoning over summaries + model=agents.OpenAIChatCompletionsModel( + model="gemini-2.5-pro", openai_client=async_openai_client + ), +) + + +async def _main(question: str, gr_messages: list[ChatMessage]): + setup_langfuse_tracer() + + # Use the main agent as the entry point- not the worker agent. + with langfuse_client.start_as_current_span(name="Agents-SDK-Trace") as span: + span.update(input=question) + + result_stream = agents.Runner.run_streamed(orchestrator_agent, input=question) + async for _item in result_stream.stream_events(): + gr_messages += oai_agent_stream_to_gradio_messages(_item) + if len(gr_messages) > 0: + yield gr_messages + + span.update(output=result_stream.final_output) + + +demo = gr.ChatInterface( + _main, + title="Hitachi Multi-Agent Knowledge Retrieval System", + type="messages", + examples=[ + "At which university did the SVP Software Engineering" + " at Apple (as of June 2025) earn their engineering degree?", + "How does the annual growth in the 50th-percentile income " + "in the US compare with that in Canada?", + ], +) + + +if __name__ == "__main__": + async_openai_client = AsyncOpenAI() + + signal.signal(signal.SIGINT, _handle_sigint) + + try: + demo.launch(server_name="0.0.0.0") + finally: + asyncio.run(_cleanup_clients()) diff --git a/src/4_hitachi/__init__.py b/src/4_hitachi/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/prompts.py b/src/prompts.py index 22336c7..3ed3c22 100644 --- a/src/prompts.py +++ b/src/prompts.py @@ -12,3 +12,14 @@ For facts that might change over time, you must use the search tool to retrieve the \ most up-to-date information. """ + +ORCHESTRATOR_REACT_INSTRUCTIONS = """\ +Answer the question using the search knowledge base agent as a tool. \ +EACH TIME before invoking the function, you must explain your reasons for doing so. \ +Be sure to mention the sources in your response. \ +If the search tool did not return intended results, try again. \ +For best performance, divide complex queries into simpler sub-queries. \ +Do not make up information. \ +For facts that might change over time, you must use the search tool to retrieve the \ +most up-to-date information. +""" From b388e3bef524fcdca42e45ba3601037692a66d66 Mon Sep 17 00:00:00 2001 From: Calen Irwin Date: Wed, 6 Aug 2025 15:23:40 +0000 Subject: [PATCH 4/9] minor updates --- src/4_hitachi/1_multi_agent/orchestrator_gradio.py | 14 ++++---------- src/prompts.py | 12 ------------ 2 files changed, 4 insertions(+), 22 deletions(-) diff --git a/src/4_hitachi/1_multi_agent/orchestrator_gradio.py b/src/4_hitachi/1_multi_agent/orchestrator_gradio.py index 45f5660..7213588 100644 --- a/src/4_hitachi/1_multi_agent/orchestrator_gradio.py +++ b/src/4_hitachi/1_multi_agent/orchestrator_gradio.py @@ -18,7 +18,7 @@ from gradio.components.chatbot import ChatMessage from openai import AsyncOpenAI -from src.prompts import REACT_INSTRUCTIONS +from src.prompts import REACT_INSTRUCTIONS, KB_SEARCH_INSTRUCTIONS from src.utils import ( AsyncWeaviateKnowledgeBase, Configs, @@ -73,11 +73,7 @@ def _handle_sigint(signum: int, frame: object) -> None: # Knowledgebase Search Agent: a simple agent that searches the knowledge base knowledgebase_agent = agents.Agent( name="KnowledgeBaseSearchAgent", - instructions=( - "You are search agent. You receive a single search query formatted as a question as input. " - "Use the search_knowledgebase tool to perform a search, then produce an accurate and concise " - "answer to the question based on the search results. If you cannot find an answer, state that you do not know the answer.", - ), + instructions=KB_SEARCH_INSTRUCTIONS, tools=[ agents.function_tool(async_knowledgebase.search_knowledgebase), ], @@ -128,10 +124,8 @@ async def _main(question: str, gr_messages: list[ChatMessage]): title="Hitachi Multi-Agent Knowledge Retrieval System", type="messages", examples=[ - "At which university did the SVP Software Engineering" - " at Apple (as of June 2025) earn their engineering degree?", - "How does the annual growth in the 50th-percentile income " - "in the US compare with that in Canada?", + "What city are George Washington University Hospital" + " and MedStar Washington Hospital Center located in?" ], ) diff --git a/src/prompts.py b/src/prompts.py index 3ed3c22..2a5149f 100644 --- a/src/prompts.py +++ b/src/prompts.py @@ -9,17 +9,5 @@ If the search tool did not return intended results, try again. \ For best performance, divide complex queries into simpler sub-queries. \ Do not make up information. \ -For facts that might change over time, you must use the search tool to retrieve the \ -most up-to-date information. """ -ORCHESTRATOR_REACT_INSTRUCTIONS = """\ -Answer the question using the search knowledge base agent as a tool. \ -EACH TIME before invoking the function, you must explain your reasons for doing so. \ -Be sure to mention the sources in your response. \ -If the search tool did not return intended results, try again. \ -For best performance, divide complex queries into simpler sub-queries. \ -Do not make up information. \ -For facts that might change over time, you must use the search tool to retrieve the \ -most up-to-date information. -""" From a1718d5e8e66b025fb66181856748dad4c4ad766 Mon Sep 17 00:00:00 2001 From: Ioana Barbos Date: Wed, 6 Aug 2025 16:50:13 +0000 Subject: [PATCH 5/9] hallucination rate --- src/3_evals/1_llm_judge/run_eval_ioana.py | 243 ++++++++++++++++++++++ 1 file changed, 243 insertions(+) create mode 100644 src/3_evals/1_llm_judge/run_eval_ioana.py diff --git a/src/3_evals/1_llm_judge/run_eval_ioana.py b/src/3_evals/1_llm_judge/run_eval_ioana.py new file mode 100644 index 0000000..1dbde4a --- /dev/null +++ b/src/3_evals/1_llm_judge/run_eval_ioana.py @@ -0,0 +1,243 @@ +"""Evaluate agent on dataset using LLM-as-a-Judge.""" + +import argparse +import asyncio + +import agents +import pydantic +from dotenv import load_dotenv +from langfuse._client.datasets import DatasetItemClient +from openai import AsyncOpenAI +from rich.progress import track + +from src.utils import ( + AsyncWeaviateKnowledgeBase, + Configs, + gather_with_progress, + get_weaviate_async_client, + set_up_logging, + setup_langfuse_tracer, +) +from src.utils.langfuse.shared_client import flush_langfuse, langfuse_client + + +load_dotenv(verbose=True) +set_up_logging() + + +SYSTEM_MESSAGE = """\ +Answer the question using the search tool. \ +EACH TIME before invoking the function, you must explain your reasons for doing so. \ +Be sure to mention the sources in your response. \ +If the search did not return intended results, try again. \ +Do not make up information. + +Finally, write "|" and include a one-sentence summary of your answer. +""" + +EVALUATOR_INSTRUCTIONS = """\ +Evaluate whether the "Proposed Answer" to the given "Question" matches the "Ground Truth".""" + +EVALUATOR_TEMPLATE = """\ +# Question + +{question} + +# Ground Truth + +{ground_truth} + +# Proposed Answer + +{proposed_response} + +""" + +EV_INSTRUCTIONS_HALLUCINATIONS = """\ +Evaluate the degree of hallucination in the whether the "Generation" on a continuous scale from 0 to 1.\ +A generation can be considered to hallucinate (score 1) if it does not align with the established knowledge, \ +verifiable data or logical inference and often includes elements that are implausible, misleading or entirely fictional.\ +Example: +Question: Do carrots improve your vison? +Generation: Yes, carrots significantly improve vision. Rabbits consume large amounts of carrots. This is why their sight \ +is very good until great ages. They have never been observed wearing glasses. + +Score: 1.0 +Reasoning: Rabbits are animals and can not wear glasses, an accesory reserved to humans. + +Think step by step. +""" + +EV_TEMPLATE_HALLUCINATIONS = """\ +# Question + +{question} + +# Generation + +{generation} + +""" + + +class LangFuseTracedResponse(pydantic.BaseModel): + """Agent Response and LangFuse Trace info.""" + + answer: str | None + trace_id: str | None + + +class EvaluatorQuery(pydantic.BaseModel): + """Query to the evaluator agent.""" + + question: str + generation: str + + def get_query(self) -> str: + """Obtain query string to the evaluator agent.""" + return EV_TEMPLATE_HALLUCINATIONS.format(**self.model_dump()) + # return EVALUATOR_TEMPLATE.format(**self.model_dump()) + + +class EvaluatorResponse(pydantic.BaseModel): + """Typed response from the evaluator.""" + + explanation: str + hallucination: float + + +async def run_agent_with_trace( + agent: agents.Agent, query: str +) -> "LangFuseTracedResponse": + """Run OpenAI Agent on query, returning response and trace_id. + + Returns None if agent exceeds max_turn limit. + """ + try: + result = await agents.Runner.run(agent, query) + if "|" in result.final_output: + answer = result.final_output.split("|")[-1].strip() + else: + answer = result.final_output + + except agents.MaxTurnsExceeded: + answer = None + + return LangFuseTracedResponse( + answer=answer, trace_id=langfuse_client.get_current_trace_id() + ) + + +async def run_evaluator_agent(evaluator_query: EvaluatorQuery) -> EvaluatorResponse: + """Evaluate using evaluator agent.""" + evaluator_agent = agents.Agent( + name="Evaluator Agent", + instructions=EV_INSTRUCTIONS_HALLUCINATIONS, + output_type=EvaluatorResponse, + model=agents.OpenAIChatCompletionsModel( + model="gemini-2.5-flash", openai_client=async_openai_client + ), + ) + + result = await agents.Runner.run(evaluator_agent, input=evaluator_query.get_query()) + return result.final_output_as(EvaluatorResponse) + + +async def run_and_evaluate( + run_name: str, main_agent: agents.Agent, lf_dataset_item: "DatasetItemClient" +) -> "tuple[LangFuseTracedResponse, EvaluatorResponse | None]": + """Run main agent and evaluator agent on one dataset instance. + + Returns None if main agent returned a None answer. + """ + # expected_output = lf_dataset_item.expected_output + # assert expected_output is not None + + with lf_dataset_item.run(run_name=run_name) as root_span: + root_span.update(input=lf_dataset_item.input["text"]) + traced_response = await run_agent_with_trace( + main_agent, query=lf_dataset_item.input["text"] + ) + root_span.update(output=traced_response.answer) + + answer = traced_response.answer + if answer is None: + return traced_response, None + + evaluator_response = await run_evaluator_agent( + EvaluatorQuery( + question=lf_dataset_item.input["text"], + # ground_truth=expected_output["text"], + generation=answer, + ) + ) + + return traced_response, evaluator_response + + +parser = argparse.ArgumentParser() +parser.add_argument("--langfuse_dataset_name", required=True) +parser.add_argument("--run_name", required=True) +parser.add_argument("--limit", type=int) + + +if __name__ == "__main__": + args = parser.parse_args() + + lf_dataset_items = langfuse_client.get_dataset(args.langfuse_dataset_name).items + if args.limit is not None: + lf_dataset_items = lf_dataset_items[: args.limit] + + configs = Configs.from_env_var() + async_weaviate_client = get_weaviate_async_client( + http_host=configs.weaviate_http_host, + http_port=configs.weaviate_http_port, + http_secure=configs.weaviate_http_secure, + grpc_host=configs.weaviate_grpc_host, + grpc_port=configs.weaviate_grpc_port, + grpc_secure=configs.weaviate_grpc_secure, + api_key=configs.weaviate_api_key, + ) + async_openai_client = AsyncOpenAI() + async_knowledgebase = AsyncWeaviateKnowledgeBase( + async_weaviate_client, + collection_name="enwiki_20250520", + ) + + tracer = setup_langfuse_tracer() + + main_agent = agents.Agent( + name="Wikipedia Agent", + instructions=SYSTEM_MESSAGE, + tools=[agents.function_tool(async_knowledgebase.search_knowledgebase)], + model=agents.OpenAIChatCompletionsModel( + model="gemini-2.5-flash", openai_client=async_openai_client + ), + ) + coros = [ + run_and_evaluate( + run_name=args.run_name, + main_agent=main_agent, + lf_dataset_item=_item, + ) + for _item in lf_dataset_items + ] + results = asyncio.run( + gather_with_progress(coros, description="Running agent and evaluating") + ) + + for _traced_response, _eval_output in track( + results, total=len(results), description="Uploading scores" + ): + # Link the trace to the dataset item for analysis + if _eval_output is not None: + langfuse_client.create_score( + name="hallucination_score", + value=_eval_output.hallucination, + comment=_eval_output.explanation, + trace_id=_traced_response.trace_id, + ) + + flush_langfuse() + + asyncio.run(async_weaviate_client.close()) From ab17ccd880889cc2b16bec7a6a35580357f31ce6 Mon Sep 17 00:00:00 2001 From: Ioana Barbos Date: Wed, 6 Aug 2025 18:24:05 +0000 Subject: [PATCH 6/9] hallucination and concisenes with our pipeline --- src/3_evals/1_llm_judge/run_eval_ioana.py | 187 ++++++++++++++++------ src/prompts.py | 61 +++++++ 2 files changed, 197 insertions(+), 51 deletions(-) diff --git a/src/3_evals/1_llm_judge/run_eval_ioana.py b/src/3_evals/1_llm_judge/run_eval_ioana.py index 1dbde4a..4514a38 100644 --- a/src/3_evals/1_llm_judge/run_eval_ioana.py +++ b/src/3_evals/1_llm_judge/run_eval_ioana.py @@ -25,15 +25,96 @@ set_up_logging() -SYSTEM_MESSAGE = """\ -Answer the question using the search tool. \ -EACH TIME before invoking the function, you must explain your reasons for doing so. \ -Be sure to mention the sources in your response. \ -If the search did not return intended results, try again. \ -Do not make up information. - -Finally, write "|" and include a one-sentence summary of your answer. -""" +from src.prompts import REACT_INSTRUCTIONS, EV_INSTRUCTIONS_HALLUCINATIONS, EV_TEMPLATE_HALLUCINATIONS +# Worker Agent QA: handles long context efficiently +import os +import openai +import numpy as np +from sklearn.metrics.pairwise import cosine_similarity +from agents import function_tool +# from pydantic import BaseModel + +configs = Configs.from_env_var() +async_weaviate_client = get_weaviate_async_client( + http_host=configs.weaviate_http_host, + http_port=configs.weaviate_http_port, + http_secure=configs.weaviate_http_secure, + grpc_host=configs.weaviate_grpc_host, + grpc_port=configs.weaviate_grpc_port, + grpc_secure=configs.weaviate_grpc_secure, + api_key=configs.weaviate_api_key, +) +async_openai_client = AsyncOpenAI() +async_knowledgebase = AsyncWeaviateKnowledgeBase( + async_weaviate_client, + collection_name="enwiki_20250520", +) + +@function_tool +async def faq_match_tool(user_query:str)->list: + '''Return a list of FAQ sorted from the most simialr to least similar to the user query by cosine similarity. + ''' + faq_list = ["Where are Toyota cars manufactured? Toyota cars are produced in Germany and Japan", + "What is the engine power of Toyota RAV4? 180HP","Where is Japan?", + "What is horse power in cars?", "What is the capital of Germany? it's Berlin", + "Is Toyota a German brand? No, it's a Japanese automobile brand."] + + _embed_client = openai.OpenAI( + api_key=os.getenv("EMBEDDING_API_KEY"), + base_url=os.getenv("EMBEDDING_BASE_URL"), + max_retries=5) + + #embed user query + user_query_embedding = _embed_client.embeddings.create(input=user_query, model=os.getenv('EMBEDDING_MODEL_NAME')) + user_query_embedding = np.array(user_query_embedding.data[0].embedding) + user_query_embedding = user_query_embedding.reshape(1, -1) + + cosi_list = [] + faq_embedding_list = _embed_client.embeddings.create(input=faq_list, model=os.getenv('EMBEDDING_MODEL_NAME')) + for i, faq_embedding in enumerate(faq_embedding_list.data): + faq_embedding = np.array(faq_embedding.embedding) + faq_embedding = faq_embedding.reshape(1,-1) + similarity_score = cosine_similarity(user_query_embedding, faq_embedding)[0][0] + cosi_list.append({"faq":faq_list[i], "sim":similarity_score}) + + sorted_faqs = sorted(cosi_list, key=lambda d: d["sim"], reverse=True) + sorted_faqs_list = [i["faq"] for i in sorted_faqs] + return "\n".join(f" {i}\n"for i in sorted_faqs_list) + +faq_agent = agents.Agent( + name="QAMatchAgent", + instructions=( + "You are an agent specializing in matching user queries to FAQ. You receive a single user query as input. " + "Use the faq_match_tool tool to return a sorted list of FAQ based on how similar they are with the user query." + "Return maximum 3 FAQs that best match the user query. If you can't find any match, return the raw user query." + "ALWAYS structure the output as a list that contains [user request, faqs if any]." + + ), + tools=[faq_match_tool], + # a faster, smaller model for quick searches + model=agents.OpenAIChatCompletionsModel( + model="gemini-2.5-flash", openai_client=async_openai_client + ), + model_settings=agents.ModelSettings(tool_choice="required"), + # output_type=FAQ +) + +# Worker Agent: handles long context efficiently +search_agent = agents.Agent( + name="SearchAgent", + instructions=( + "You are a search agent. You receive a search query and a list of FAQ as input. " + "Use the WebSearchTool to perform a web search on the search query, then produce a concise " + "'search summary' of the key findings. Corroborate the findings with the FAQ into a final answer. Do NOT return raw search results." + ), + tools=[ + agents.function_tool(async_knowledgebase.search_knowledgebase), + ], + # a faster, smaller model for quick searches + model=agents.OpenAIChatCompletionsModel( + model="gemini-2.5-flash", openai_client=async_openai_client + ), +) EVALUATOR_INSTRUCTIONS = """\ Evaluate whether the "Proposed Answer" to the given "Question" matches the "Ground Truth".""" @@ -53,31 +134,7 @@ """ -EV_INSTRUCTIONS_HALLUCINATIONS = """\ -Evaluate the degree of hallucination in the whether the "Generation" on a continuous scale from 0 to 1.\ -A generation can be considered to hallucinate (score 1) if it does not align with the established knowledge, \ -verifiable data or logical inference and often includes elements that are implausible, misleading or entirely fictional.\ -Example: -Question: Do carrots improve your vison? -Generation: Yes, carrots significantly improve vision. Rabbits consume large amounts of carrots. This is why their sight \ -is very good until great ages. They have never been observed wearing glasses. - -Score: 1.0 -Reasoning: Rabbits are animals and can not wear glasses, an accesory reserved to humans. - -Think step by step. -""" - -EV_TEMPLATE_HALLUCINATIONS = """\ -# Question - -{question} - -# Generation -{generation} - -""" class LangFuseTracedResponse(pydantic.BaseModel): @@ -189,31 +246,59 @@ async def run_and_evaluate( lf_dataset_items = lf_dataset_items[: args.limit] configs = Configs.from_env_var() - async_weaviate_client = get_weaviate_async_client( - http_host=configs.weaviate_http_host, - http_port=configs.weaviate_http_port, - http_secure=configs.weaviate_http_secure, - grpc_host=configs.weaviate_grpc_host, - grpc_port=configs.weaviate_grpc_port, - grpc_secure=configs.weaviate_grpc_secure, - api_key=configs.weaviate_api_key, - ) - async_openai_client = AsyncOpenAI() - async_knowledgebase = AsyncWeaviateKnowledgeBase( - async_weaviate_client, - collection_name="enwiki_20250520", - ) + # async_weaviate_client = get_weaviate_async_client( + # http_host=configs.weaviate_http_host, + # http_port=configs.weaviate_http_port, + # http_secure=configs.weaviate_http_secure, + # grpc_host=configs.weaviate_grpc_host, + # grpc_port=configs.weaviate_grpc_port, + # grpc_secure=configs.weaviate_grpc_secure, + # api_key=configs.weaviate_api_key, + # ) + # async_openai_client = AsyncOpenAI() + # async_knowledgebase = AsyncWeaviateKnowledgeBase( + # async_weaviate_client, + # collection_name="enwiki_20250520", + # ) tracer = setup_langfuse_tracer() + # main_agent = agents.Agent( + # name="Wikipedia Agent", + # instructions=REACT_INSTRUCTIONS, + # tools=[agents.function_tool(async_knowledgebase.search_knowledgebase)], + # model=agents.OpenAIChatCompletionsModel( + # model="gemini-2.5-flash", openai_client=async_openai_client + # ), + + + ############################################################################################## + ###### Our Pipeline + ############################################################################################## + + + # Main Agent: more expensive and slower, but better at complex planning main_agent = agents.Agent( - name="Wikipedia Agent", - instructions=SYSTEM_MESSAGE, - tools=[agents.function_tool(async_knowledgebase.search_knowledgebase)], + name="MainAgent", + instructions=REACT_INSTRUCTIONS, + # Allow the planner agent to invoke the worker agents. + # The long context provided to the worker agent is hidden from the main agent. + tools=[ + faq_agent.as_tool( + tool_name="faq_match", + tool_description = "Identify the matching FAQs in the database." + ), + search_agent.as_tool( + tool_name="search", + tool_description="Perform a web search for a query and return a concise summary.", + ) + ], + # a larger, more capable model for planning and reasoning over summaries model=agents.OpenAIChatCompletionsModel( - model="gemini-2.5-flash", openai_client=async_openai_client + model="gemini-2.5-pro", openai_client=async_openai_client ), ) + coros = [ run_and_evaluate( run_name=args.run_name, diff --git a/src/prompts.py b/src/prompts.py index 22336c7..05e5c31 100644 --- a/src/prompts.py +++ b/src/prompts.py @@ -12,3 +12,64 @@ For facts that might change over time, you must use the search tool to retrieve the \ most up-to-date information. """ + +# EVALUATIONS + +# Hallucinations Evaluation +EV_INSTRUCTIONS_HALLUCINATIONS = """\ +Evaluate the degree of hallucination in the "Generation" on a continuous scale from 0 to 1.\ +A generation can be considered to hallucinate (score 1) if it does not align with the established knowledge, \ +verifiable data or logical inference and often includes elements that are implausible, misleading or entirely fictional.\ +Example: +Question: Do carrots improve your vison? +Generation: Yes, carrots significantly improve vision. Rabbits consume large amounts of carrots. This is why their sight \ +is very good until great ages. They have never been observed wearing glasses. + +Score: 1.0 +Reasoning: Rabbits are animals and can not wear glasses, an accesory reserved to humans. + +Think step by step. +""" + +EV_TEMPLATE_HALLUCINATIONS = """\ +# Question + +{question} + +# Generation + +{generation} + +""" + +# Conciseness +EV_INSTRUCTIONS_CONCISENESS = """\ +Evaluate if a "Generation" is concise or verbose, with respect to the "Question".\ +A generation can be considered to concise (score 1) if it conveys the core message using the fewest words possible, \ +avoiding unnecessary repetition or jargon. Evaluate the response based on its ability to convey the core message \ +efficiently, without extraneous details or wordiness.\ +Scoring: Rate the conciseness as 0 or 1, where 0 is verbose and 1 is concise. Provide a brief explanation for your score.\ + +Examples: \ +Question: Where did the cat sit? +Generation: "The cat sat on the mat." \ +Score: 1 \ +Reasoning: This sentence is very concise and directly conveys the information. \ + +Question: Where did the cat sit? +Generation: "The feline creature, known as a cat, took up residence upon the floor covering known as a mat." \ +Score: 0 \ +Reasoning: This sentence is verbose, using more words and more complex phrasing than necessary. \ + +Think step by step. +""" + +EV_TEMPLATE_CONCISENESS = """\ +# Question + +{question} + +# Generation + +{generation} +""" \ No newline at end of file From 9feedee4d7ace584ae6cb49e094ff4fe232ef4ff Mon Sep 17 00:00:00 2001 From: Calen Irwin Date: Wed, 6 Aug 2025 19:33:56 +0000 Subject: [PATCH 7/9] working multi-agent with prompts specifying data formats --- .../planner_worker_gradio_ioana.py | 204 ++++++++++-------- src/prompts.py | 67 +++++- 2 files changed, 179 insertions(+), 92 deletions(-) diff --git a/src/2_frameworks/2_multi_agent/planner_worker_gradio_ioana.py b/src/2_frameworks/2_multi_agent/planner_worker_gradio_ioana.py index 99d663c..9cf5797 100644 --- a/src/2_frameworks/2_multi_agent/planner_worker_gradio_ioana.py +++ b/src/2_frameworks/2_multi_agent/planner_worker_gradio_ioana.py @@ -11,14 +11,21 @@ import logging import signal import sys +import os +import json +import openai +import numpy as np import agents +from agents import function_tool import gradio as gr from dotenv import load_dotenv from gradio.components.chatbot import ChatMessage from openai import AsyncOpenAI +from sklearn.metrics.pairwise import cosine_similarity +from pydantic import BaseModel -from src.prompts import REACT_INSTRUCTIONS +from src.prompts import REACT_INSTRUCTIONS, KB_SEARCH_INSTRUCTIONS, QA_SEARCH_INSTRUCTIONS, EVALUATOR_INSTRUCTIONS, EVALUATOR_TEMPLATE from src.utils import ( AsyncWeaviateKnowledgeBase, Configs, @@ -35,10 +42,52 @@ logging.basicConfig(level=logging.INFO) -AGENT_LLM_NAMES = { - "worker": "gemini-2.5-flash", # less expensive, - "planner": "gemini-2.5-pro", # more expensive, better at reasoning and planning -} + +class QASearchSingleResponse(BaseModel): + """Typed response from the QA search agent.""" + + user_query: str | None + question: str | None + answer: str | None + context: str | None + + def __str__(self) -> str: + """String representation of the response.""" + return f"QASearchSingleResponse(user_query={self.user_query}, question={self.question}, answer={self.answer}, context={self.context})" + + +class KBSearchResponse(BaseModel): + """Query to the knowledge base search agent.""" + + answer: str | None + context: str | None + +class EvaluatorQuery(BaseModel): + """Query to the evaluator agent.""" + + question: str + ground_truth: str + proposed_response: str + + def get_query(self) -> str: + """Obtain query string to the evaluator agent.""" + return EVALUATOR_TEMPLATE.format(**self.model_dump()) + + +class EvaluatorResponse(BaseModel): + """Typed response from the evaluator.""" + + explanation: str + is_answer_correct: bool + + +class EvaluatorAgent(agents.Agent[EvaluatorResponse]): + async def run_tool(self, tool_input, *args, **kwargs): + # Convert dict → EvaluatorQuery → formatted prompt + if not isinstance(tool_input, EvaluatorQuery): + tool_input = EvaluatorQuery(**tool_input) + return await super().run_tool(tool_input.get_query(), *args, **kwargs) + configs = Configs.from_env_var() async_weaviate_client = get_weaviate_async_client( @@ -69,91 +118,73 @@ def _handle_sigint(signum: int, frame: object) -> None: asyncio.get_event_loop().run_until_complete(_cleanup_clients()) sys.exit(0) -# Worker Agent QA: handles long context efficiently -import os -import openai -import numpy as np -from sklearn.metrics.pairwise import cosine_similarity -from agents import function_tool -# from pydantic import BaseModel - - @function_tool -async def faq_match_tool(user_query:str)->list: - '''Return a list of FAQ sorted from the most simialr to least similar to the user query by cosine similarity. +async def qa_search_tool(user_query:str) -> list: + '''Return a list of questions sorted from the most similar to least similar to the user query by cosine similarity. ''' - faq_list = ["Where are Toyota cars manufactured? Toyota cars are produced in Germany and Japan", - "What is the engine power of Toyota RAV4? 180HP","Where is Japan?", - "What is horse power in cars?", "What is the capital of Germany? it's Berlin", - "Is Toyota a German brand? No, it's a Japanese automobile brand."] - - _embed_client = openai.OpenAI( - api_key=os.getenv("EMBEDDING_API_KEY"), - base_url=os.getenv("EMBEDDING_BASE_URL"), - max_retries=5) + qa_dataset = { + 1 : { + "question": "What is the capital of France?", + "answer": "The capital of France is Paris.", + "context": "Paris is the capital city of France, known for its art, fashion, and culture." + } + } + + # _embed_client = openai.OpenAI( + # api_key=os.getenv("EMBEDDING_API_KEY"), + # base_url=os.getenv("EMBEDDING_BASE_URL"), + # max_retries=5) - #embed user query - user_query_embedding = _embed_client.embeddings.create(input=user_query, model=os.getenv('EMBEDDING_MODEL_NAME')) - user_query_embedding = np.array(user_query_embedding.data[0].embedding) - user_query_embedding = user_query_embedding.reshape(1, -1) - - cosi_list = [] - faq_embedding_list = _embed_client.embeddings.create(input=faq_list, model=os.getenv('EMBEDDING_MODEL_NAME')) - for i, faq_embedding in enumerate(faq_embedding_list.data): - faq_embedding = np.array(faq_embedding.embedding) - faq_embedding = faq_embedding.reshape(1,-1) - similarity_score = cosine_similarity(user_query_embedding, faq_embedding)[0][0] - cosi_list.append({"faq":faq_list[i], "sim":similarity_score}) - - sorted_faqs = sorted(cosi_list, key=lambda d: d["sim"], reverse=True) - sorted_faqs_list = [i["faq"] for i in sorted_faqs] - return "\n".join(f" {i}\n"for i in sorted_faqs_list) - # return sorted_faqs_list - -# class FAQ(BaseModel): -# user_query: str -# similar_faqs: list[str] -# def __str__(self) -> str: -# """Return a string representation of the faq list""" -# return "\n".join( -# f"faq {step}\n" -# for step in self.similar_faqs -# ) - -faq_agent = agents.Agent( - name="QAMatchAgent", - instructions=( - "You are an agent specializing in matching user queries to FAQ. You receive a single user query as input. " - "Use the faq_match_tool tool to return a sorted list of FAQ based on how similar they are with the user query." - "Return maximum 3 FAQs that best match the user query. If you can't find any match, return the raw user query." - "ALWAYS structure the output as a list that contains [user request, faqs if any]." - - ), - tools=[faq_match_tool], + # #embed user query + # user_query_embedding = _embed_client.embeddings.create(input=user_query, model=os.getenv('EMBEDDING_MODEL_NAME')) + # user_query_embedding = np.array(user_query_embedding.data[0].embedding) + # user_query_embedding = user_query_embedding.reshape(1, -1) + + # cosi_list = [] + # qa_embedding_list = _embed_client.embeddings.create(input=qa_dataset, model=os.getenv('EMBEDDING_MODEL_NAME')) + # for i, qa_embedding in enumerate(qa_embedding_list.data): + # qa_embedding = np.array(qa_embedding.embedding) + # qa_embedding = qa_embedding.reshape(1,-1) + # similarity_score = cosine_similarity(user_query_embedding, qa_embedding)[0][0] + # cosi_list.append({"faq":faq_list[i], "sim":similarity_score}) + + # sorted_qa = sorted(cosi_list, key=lambda d: d["sim"], reverse=True) + # sorted_faqs_list = [i["faq"] for i in sorted_qa] + # + # return "\n".join(f" {i}\n"for i in sorted_faqs_list) + + return json.dumps(qa_dataset) + +qa_search_agent = agents.Agent( + name="QASearchAgent", + instructions=QA_SEARCH_INSTRUCTIONS, + tools=[qa_search_tool], # a faster, smaller model for quick searches model=agents.OpenAIChatCompletionsModel( model="gemini-2.5-flash", openai_client=async_openai_client - ), - model_settings=agents.ModelSettings(tool_choice="required"), - # output_type=FAQ + ) ) # Worker Agent: handles long context efficiently -search_agent = agents.Agent( - name="SearchAgent", - instructions=( - "You are a search agent. You receive a search query and a list of FAQ as input. " - "Use the WebSearchTool to perform a web search on the search query, then produce a concise " - "'search summary' of the key findings. Corroborate the findings with the FAQ into a final answer. Do NOT return raw search results." - ), +kb_search_agent = agents.Agent( + name="KBSearchAgent", + instructions=KB_SEARCH_INSTRUCTIONS, tools=[ agents.function_tool(async_knowledgebase.search_knowledgebase), ], # a faster, smaller model for quick searches model=agents.OpenAIChatCompletionsModel( model="gemini-2.5-flash", openai_client=async_openai_client - ), + ) +) + +evaluator_agent = agents.Agent( + name="EvaluatorAgent", + instructions=EVALUATOR_INSTRUCTIONS, + model=agents.OpenAIChatCompletionsModel( + model="gemini-2.5-flash", openai_client=async_openai_client + ) ) # Main Agent: more expensive and slower, but better at complex planning @@ -163,13 +194,17 @@ async def faq_match_tool(user_query:str)->list: # Allow the planner agent to invoke the worker agents. # The long context provided to the worker agent is hidden from the main agent. tools=[ - faq_agent.as_tool( - tool_name="faq_match", - tool_description = "Identify the matching FAQs in the database." + qa_search_agent.as_tool( + tool_name="qa_search_Agent", + tool_description = "Perform a search of the QA database and retrieve question/answer/context tuples related to input query." ), - search_agent.as_tool( - tool_name="search", - tool_description="Perform a web search for a query and return a concise summary.", + kb_search_agent.as_tool( + tool_name="kb_search_agent", + tool_description="Perform a search of a knowledge base and synthesize the search results to answer input question.", + ), + evaluator_agent.as_tool( + tool_name="evaluator_agent", + tool_description="Evaluate the output of the knowledge base search agent.", ) ], # a larger, more capable model for planning and reasoning over summaries @@ -183,7 +218,7 @@ async def _main(question: str, gr_messages: list[ChatMessage]): setup_langfuse_tracer() # Use the main agent as the entry point- not the worker agent. - with langfuse_client.start_as_current_span(name="Agents-SDK-Trace") as span: + with langfuse_client.start_as_current_span(name="Calen-Multi-Agent") as span: span.update(input=question) result_stream = agents.Runner.run_streamed(main_agent, input=question) @@ -197,12 +232,11 @@ async def _main(question: str, gr_messages: list[ChatMessage]): demo = gr.ChatInterface( _main, - title="2.2 Multi-Agent for Efficiency", + title="Hitachi Multi-Agent Knowledge Retrieval System", type="messages", examples=[ - "what is toyota? ", - "How does the annual growth in the 50th-percentile income " - "in the US compare with that in Canada?", + "Where should I go in France? ", + "Where is the government of France located? " ], ) diff --git a/src/prompts.py b/src/prompts.py index 2a5149f..22d45cd 100644 --- a/src/prompts.py +++ b/src/prompts.py @@ -1,13 +1,66 @@ """Centralized location for all system prompts.""" REACT_INSTRUCTIONS = """\ -Answer the question in two steps, always in the same order: \ -step 1 use the faq_match tool. If you can find the answer from the FAQs alone, say "hurray!" and give preliminary answer.\ -step 2 use the search tool to enhance the answer from the first step. \ -EACH TIME before invoking the function, you must explain your reasons for doing so. \ -Be sure to mention the sources in your response. \ -If the search tool did not return intended results, try again. \ -For best performance, divide complex queries into simpler sub-queries. \ +You are a high level orchestration agent that has access to other agents as tools. \ +Your objective is to take a user promtp, identify relevant question/answer pairs from a QA database, \ +search for answers to the identified questions in a central knowledge base, and then evaluate the \ +generated answer against the ground truth. You should follow this sequence: \ + +STEP 1 - call the QA search agent with the user prompt \ +STEP 2 - call the knowledge base search agent with each of the identified questions from STEP 1 \ +STEP 3 - call the evaluator agent with the user prompt, the ground truth and the generated answer from STEP 2 \ + +For each step, use the respective tool. \ +EACH TIME before invoking a function, you must explain your reasons for doing so. \ Do not make up information. \ """ +QA_SEARCH_INSTRUCTIONS = """\ +You are an agent specializing in matching a user query to related questions in a QA database. \ +You receive a single user query as input. \ +Use the qa_search_tool to return question, answer pairs in json format.\ + +ALWAYS return the question, answer pairs in the following format. \ +user_query: str | None \ +question: str | None \ +answer: str | None \ +context: str | None \ +""" + +KB_SEARCH_INSTRUCTIONS = """ \ +You are a knowledge base search agent. You receive a single question as input in the form of QASearchSingleResponse. \ +Use the search_knowledgebase tool to perform a search of key words related to the "question" field (not the user query). \ +Based on the search results, generate a final answer to the input question. Do NOT return raw search results. \ + +ALWAYS return the final answer in the following format. \ +answer: str | None \ +context: str | None \ +""" + +EVALUATOR_INSTRUCTIONS = """\ +Evaluate whether the "Proposed Answer" to the given "Question" matches the "Ground Truth". \ + +Input structure should be in the following format. \ +question: str \ +ground_truth: str \ +proposed_response: str \ + +ALWAYS return the evaluation in the following format. \ +explanation: str \ +is_answer_correct: bool \ +""" + +EVALUATOR_TEMPLATE = """\ +# Question + +{question} + +# Ground Truth + +{ground_truth} + +# Proposed Answer + +{proposed_response} + +""" From 2beadae4e8d1874379c2132d2f59babe4663bb4a Mon Sep 17 00:00:00 2001 From: Ioana Barbos Date: Thu, 7 Aug 2025 14:34:06 +0000 Subject: [PATCH 8/9] save --- src/1_basics/1_react_rag/gradio.py | 2 +- src/1_basics/1_react_rag/main.py | 2 +- src/2_frameworks/1_react_rag/basic.py | 2 +- src/2_frameworks/1_react_rag/gradio.py | 2 +- .../1_react_rag/langfuse_gradio.py | 2 +- .../2_multi_agent/planner_worker_gradio.py | 2 +- .../planner_worker_gradio_ioana.py | 2 +- src/2_frameworks/2_multi_agent/test_nb.ipynb | 253 ++++++++++++++++++ src/3_evals/1_llm_judge/README.md | 2 +- src/3_evals/1_llm_judge/run_eval.py | 1 - src/3_evals/1_llm_judge/run_eval_ioana.py | 4 +- src/{prompts.py => prompts_i.py} | 2 + 12 files changed, 265 insertions(+), 11 deletions(-) create mode 100644 src/2_frameworks/2_multi_agent/test_nb.ipynb rename src/{prompts.py => prompts_i.py} (97%) diff --git a/src/1_basics/1_react_rag/gradio.py b/src/1_basics/1_react_rag/gradio.py index 76bb1df..ed8e29b 100644 --- a/src/1_basics/1_react_rag/gradio.py +++ b/src/1_basics/1_react_rag/gradio.py @@ -15,7 +15,7 @@ from openai import AsyncOpenAI from openai.types.chat import ChatCompletionSystemMessageParam, ChatCompletionToolParam -from src.prompts import REACT_INSTRUCTIONS +from prompts_i import REACT_INSTRUCTIONS from src.utils import ( AsyncWeaviateKnowledgeBase, Configs, diff --git a/src/1_basics/1_react_rag/main.py b/src/1_basics/1_react_rag/main.py index 78a13d3..4eb7956 100644 --- a/src/1_basics/1_react_rag/main.py +++ b/src/1_basics/1_react_rag/main.py @@ -8,7 +8,7 @@ from dotenv import load_dotenv from openai import AsyncOpenAI -from src.prompts import REACT_INSTRUCTIONS +from prompts_i import REACT_INSTRUCTIONS from src.utils import ( AsyncWeaviateKnowledgeBase, Configs, diff --git a/src/2_frameworks/1_react_rag/basic.py b/src/2_frameworks/1_react_rag/basic.py index bb46476..769bb44 100644 --- a/src/2_frameworks/1_react_rag/basic.py +++ b/src/2_frameworks/1_react_rag/basic.py @@ -13,7 +13,7 @@ from dotenv import load_dotenv from openai import AsyncOpenAI -from src.prompts import REACT_INSTRUCTIONS +from prompts_i import REACT_INSTRUCTIONS from src.utils import ( AsyncWeaviateKnowledgeBase, Configs, diff --git a/src/2_frameworks/1_react_rag/gradio.py b/src/2_frameworks/1_react_rag/gradio.py index 00b1be5..459344c 100644 --- a/src/2_frameworks/1_react_rag/gradio.py +++ b/src/2_frameworks/1_react_rag/gradio.py @@ -12,7 +12,7 @@ from gradio.components.chatbot import ChatMessage from openai import AsyncOpenAI -from src.prompts import REACT_INSTRUCTIONS +from prompts_i import REACT_INSTRUCTIONS from src.utils import ( AsyncWeaviateKnowledgeBase, Configs, diff --git a/src/2_frameworks/1_react_rag/langfuse_gradio.py b/src/2_frameworks/1_react_rag/langfuse_gradio.py index 5dd22a1..c83fc2d 100644 --- a/src/2_frameworks/1_react_rag/langfuse_gradio.py +++ b/src/2_frameworks/1_react_rag/langfuse_gradio.py @@ -15,7 +15,7 @@ from gradio.components.chatbot import ChatMessage from openai import AsyncOpenAI -from src.prompts import REACT_INSTRUCTIONS +from prompts_i import REACT_INSTRUCTIONS from src.utils import ( AsyncWeaviateKnowledgeBase, Configs, diff --git a/src/2_frameworks/2_multi_agent/planner_worker_gradio.py b/src/2_frameworks/2_multi_agent/planner_worker_gradio.py index a7a49ff..b56b9e1 100644 --- a/src/2_frameworks/2_multi_agent/planner_worker_gradio.py +++ b/src/2_frameworks/2_multi_agent/planner_worker_gradio.py @@ -18,7 +18,7 @@ from gradio.components.chatbot import ChatMessage from openai import AsyncOpenAI -from src.prompts import REACT_INSTRUCTIONS +from prompts_i import REACT_INSTRUCTIONS from src.utils import ( AsyncWeaviateKnowledgeBase, Configs, diff --git a/src/2_frameworks/2_multi_agent/planner_worker_gradio_ioana.py b/src/2_frameworks/2_multi_agent/planner_worker_gradio_ioana.py index 99d663c..bb75f43 100644 --- a/src/2_frameworks/2_multi_agent/planner_worker_gradio_ioana.py +++ b/src/2_frameworks/2_multi_agent/planner_worker_gradio_ioana.py @@ -18,7 +18,7 @@ from gradio.components.chatbot import ChatMessage from openai import AsyncOpenAI -from src.prompts import REACT_INSTRUCTIONS +from prompts_i import REACT_INSTRUCTIONS from src.utils import ( AsyncWeaviateKnowledgeBase, Configs, diff --git a/src/2_frameworks/2_multi_agent/test_nb.ipynb b/src/2_frameworks/2_multi_agent/test_nb.ipynb new file mode 100644 index 0000000..c0eefe6 --- /dev/null +++ b/src/2_frameworks/2_multi_agent/test_nb.ipynb @@ -0,0 +1,253 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 1, + "id": "652f3be2", + "metadata": {}, + "outputs": [], + "source": [ + "# Worker Agent QA: handles long context efficiently\n", + "import os\n", + "import openai\n", + "from sklearn.metrics.pairwise import cosine_similarity\n", + "import numpy as np\n", + "\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "81e5b04e", + "metadata": {}, + "outputs": [], + "source": [ + "def find_similar_faq(user_query:str, faq_list:list)->list:\n", + " '''Return a list of FAQ sorted from the most simialr to least similar to the user query by cosine similarity.\n", + " '''\n", + "\n", + " _embed_client = openai.OpenAI(\n", + " api_key=os.getenv(\"EMBEDDING_API_KEY\"),\n", + " base_url=os.getenv(\"EMBEDDING_BASE_URL\"),\n", + " max_retries=5)\n", + " \n", + " #embed user query\n", + " user_query_embedding = _embed_client.embeddings.create(input=user_query, model=os.getenv('EMBEDDING_MODEL_NAME'))\n", + " user_query_embedding = np.array(user_query_embedding.data[0].embedding)\n", + " user_query_embedding = user_query_embedding.reshape(1, -1)\n", + "\n", + " cosi_list = []\n", + " faq_embedding_list = _embed_client.embeddings.create(input=faq_list, model=os.getenv('EMBEDDING_MODEL_NAME'))\n", + " for i, faq_embedding in enumerate(faq_embedding_list.data):\n", + " faq_embedding = np.array(faq_embedding.embedding)\n", + " faq_embedding = faq_embedding.reshape(1,-1)\n", + " similarity_score = cosine_similarity(user_query_embedding, faq_embedding)[0][0]\n", + " cosi_list.append({\"faq\":faq_list[i], \"sim\":similarity_score})\n", + "\n", + " sorted_faqs = sorted(cosi_list, key=lambda d: d[\"sim\"], reverse=True)\n", + " sorted_faqs_list = [i[\"faq\"] for i in sorted_faqs]\n", + "\n", + " return sorted_faqs_list" + ] + }, + { + "cell_type": "code", + "execution_count": 23, + "id": "dfa1bb57", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Embedding(embedding=[-0.045440673828125, -0.037139892578125, -0.007572174072265625, 0.02801513671875, 0.0013322830200195312, 0.005214691162109375, -0.060943603515625, 0.03546142578125, -0.024261474609375, 0.0294952392578125, 0.020172119140625, 0.0290374755859375, 0.0293121337890625, 0.036834716796875, 0.0233001708984375, -0.06817626953125, 0.0177764892578125, 0.043304443359375, -0.0168914794921875, -0.04705810546875, -0.0280914306640625, -0.0340576171875, -0.00039005279541015625, -0.062225341796875, 0.01184844970703125, 0.0059051513671875, 0.01010894775390625, -0.00926971435546875, 0.010009765625, -0.08038330078125, 0.02398681640625, 0.0125732421875, -0.050079345703125, -0.0072784423828125, -0.01812744140625, -0.0145111083984375, -3.8623809814453125e-05, -0.0343017578125, -0.046783447265625, -0.024383544921875, -0.0020885467529296875, -0.0135498046875, 0.02789306640625, -0.008880615234375, -0.0223236083984375, -0.043212890625, -0.0175628662109375, -0.0230712890625, -0.02288818359375, 0.0126800537109375, 0.035430908203125, -0.029876708984375, 0.013763427734375, -0.01042938232421875, 0.052703857421875, 0.1025390625, -0.0162200927734375, -0.0704345703125, -0.061279296875, 0.021453857421875, -0.08074951171875, -0.005352020263671875, -0.0222625732421875, 0.045928955078125, 0.01398468017578125, 0.055450439453125, -0.0230865478515625, -0.038665771484375, -0.0248870849609375, -0.025360107421875, -0.00695037841796875, -0.037078857421875, 0.0243377685546875, -0.0013971328735351562, -0.00298309326171875, 0.01529693603515625, -0.014801025390625, 0.01110076904296875, -0.0264739990234375, 0.0062408447265625, 0.032012939453125, 0.01268768310546875, 0.0138092041015625, 0.0303802490234375, 0.043243408203125, 0.0177764892578125, -0.049346923828125, -0.0060882568359375, -0.035675048828125, -0.0175628662109375, -0.00815582275390625, -0.0298309326171875, -0.03533935546875, -0.0523681640625, -0.0257720947265625, 0.0138702392578125, -0.036865234375, 0.0302276611328125, -0.0423583984375, -0.019805908203125, 0.030364990234375, -0.030426025390625, 0.0131072998046875, -0.06475830078125, 0.048858642578125, -0.007175445556640625, 0.0221099853515625, 0.0302734375, -0.0158538818359375, 0.01422119140625, 0.0269775390625, -0.0018129348754882812, -0.01395416259765625, -0.0237274169921875, -0.04644775390625, -0.031341552734375, -0.01538848876953125, -0.057220458984375, 0.042510986328125, 0.0101776123046875, -0.01485443115234375, -0.0161895751953125, 0.033355712890625, -0.01367950439453125, -0.0295867919921875, -0.0845947265625, 0.04559326171875, -0.00614166259765625, 0.007396697998046875, -0.0043182373046875, 0.0257568359375, -0.00977325439453125, -0.0257568359375, 0.03021240234375, -0.0305938720703125, -0.066162109375, 0.0261383056640625, -0.0202178955078125, -0.07586669921875, -0.037261962890625, 0.006565093994140625, 0.01390838623046875, -0.058624267578125, -0.00502777099609375, -0.0009617805480957031, -0.038543701171875, -0.010772705078125, -0.0006756782531738281, 0.005481719970703125, 0.0073699951171875, 0.01068878173828125, 0.0020275115966796875, 0.0204620361328125, 0.02044677734375, -0.0259246826171875, -0.038360595703125, 0.006526947021484375, 0.061676025390625, -0.055694580078125, 0.00583648681640625, 0.053558349609375, 0.0311431884765625, 0.0165557861328125, 0.04351806640625, -0.0048065185546875, -0.0055694580078125, -0.0487060546875, -0.0290374755859375, -0.033416748046875, 0.0017910003662109375, -0.0064849853515625, 0.02947998046875, -0.03448486328125, -0.016510009765625, 0.040557861328125, 0.0229034423828125, 0.052001953125, 0.0128173828125, -0.01427459716796875, -0.0006699562072753906, -0.0013074874877929688, 0.054595947265625, 0.04913330078125, 0.0316162109375, -0.0038356781005859375, 0.039794921875, 0.03350830078125, -0.030029296875, 0.001186370849609375, 0.0264739990234375, -0.031707763671875, -0.01548004150390625, -0.01068115234375, 0.0022182464599609375, 0.0382080078125, 0.031768798828125, 0.0162811279296875, 0.0174713134765625, -0.0273284912109375, -0.053863525390625, -0.0628662109375, 0.026641845703125, 0.00592803955078125, -0.0007500648498535156, 0.006683349609375, -0.01316070556640625, -0.0160980224609375, -0.037445068359375, -0.005565643310546875, -0.0367431640625, 0.0115814208984375, -0.0051116943359375, 0.003490447998046875, 0.0343017578125, -0.00438690185546875, 0.0245819091796875, -0.043792724609375, -0.035736083984375, -0.0123748779296875, 0.0259857177734375, -0.064208984375, 0.054534912109375, -0.0148162841796875, 0.00701141357421875, 0.017669677734375, -0.009307861328125, -0.00917816162109375, 0.005863189697265625, -0.007297515869140625, -0.013763427734375, 0.005031585693359375, 0.0065155029296875, -0.05291748046875, -0.01739501953125, 0.0038928985595703125, 0.03070068359375, -0.033111572265625, -0.0225677490234375, 0.07525634765625, 0.046875, 0.01358795166015625, 0.07269287109375, 0.0005278587341308594, -0.0010766983032226562, -0.0205230712890625, -0.006504058837890625, 0.058258056640625, 0.0137176513671875, -0.0343017578125, -0.0029621124267578125, -0.0085906982421875, 0.0038204193115234375, -0.00566864013671875, 0.0523681640625, 0.020050048828125, 0.03521728515625, 0.027740478515625, -0.0121612548828125, -0.0291290283203125, -0.004604339599609375, -0.0287322998046875, -0.0106201171875, 0.028472900390625, 0.0303802490234375, 0.017303466796875, 0.02777099609375, -0.023193359375, -0.02294921875, -0.01253509521484375, -0.0154266357421875, -0.017333984375, 0.006298065185546875, -0.0009016990661621094, 0.03826904296875, 0.01009368896484375, 0.00920867919921875, 0.01132965087890625, -0.027801513671875, 0.0050811767578125, 0.033111572265625, 0.059173583984375, -0.0758056640625, 0.039276123046875, 0.0229644775390625, 0.00974273681640625, -0.0279388427734375, 0.0006022453308105469, 0.030029296875, 0.01041412353515625, 0.0034999847412109375, -0.035980224609375, -0.02734375, 0.0552978515625, -0.0176239013671875, 0.018463134765625, 0.028045654296875, -0.01189422607421875, -0.1578369140625, -0.00616455078125, -0.0253143310546875, -0.01354217529296875, -0.0126800537109375, -0.01953125, -0.0579833984375, -0.0240936279296875, -0.0074005126953125, -0.0272979736328125, 0.004192352294921875, -0.051361083984375, -0.06500244140625, -0.028961181640625, 0.0208892822265625, -0.0035762786865234375, -0.0210113525390625, 0.025390625, 0.0172882080078125, -0.0300750732421875, 0.0221099853515625, -0.009490966796875, 0.019378662109375, 0.0009164810180664062, -0.0290069580078125, 0.0037689208984375, 0.039398193359375, 0.017669677734375, -0.036865234375, -0.022918701171875, 0.06622314453125, -0.00655364990234375, 0.0017375946044921875, 0.07183837890625, 0.08148193359375, -0.00408935546875, 0.0210723876953125, -0.024261474609375, 0.0110626220703125, 0.025665283203125, 0.00594329833984375, 0.044586181640625, 0.023681640625, 0.005146026611328125, -0.010467529296875, -0.0101776123046875, -0.0140228271484375, -0.021942138671875, -0.0010499954223632812, -0.04052734375, 0.0230560302734375, 0.0247802734375, -0.019927978515625, -0.02630615234375, -0.0267791748046875, -0.01303863525390625, 0.0080718994140625, 0.0494384765625, 0.0101318359375, -0.043212890625, 0.007354736328125, -0.0305328369140625, 0.007625579833984375, 0.0276031494140625, 0.049774169921875, 0.01308441162109375, 0.029327392578125, 0.02435302734375, -0.0170440673828125, 0.0023937225341796875, 0.0447998046875, 0.0035572052001953125, -0.01904296875, -0.0219573974609375, 0.0179595947265625, -0.006198883056640625, 0.0008401870727539062, -0.0278167724609375, 0.025115966796875, -0.1217041015625, -0.005802154541015625, 0.050384521484375, -0.005748748779296875, -0.017486572265625, 0.0011739730834960938, -0.038909912109375, -0.040924072265625, 0.055694580078125, 0.04962158203125, 0.2037353515625, -0.00516510009765625, 0.04754638671875, -0.0306854248046875, 0.015716552734375, -0.033660888671875, 0.0010671615600585938, -0.036529541015625, 0.019317626953125, -0.0311737060546875, -0.0158538818359375, 0.024566650390625, 0.0670166015625, -0.006378173828125, -0.0038089752197265625, -0.0088043212890625, 0.0297698974609375, 0.0218048095703125, 0.036529541015625, -0.02447509765625, -0.0033931732177734375, -0.0051116943359375, 0.006114959716796875, -0.02093505859375, -0.00959014892578125, -0.0211639404296875, -0.0095672607421875, 0.00145721435546875, 0.00011998414993286133, 0.01348114013671875, -0.024261474609375, -0.007755279541015625, 0.0033702850341796875, -0.00543975830078125, -0.04559326171875, -0.00907135009765625, -0.05010986328125, 0.02996826171875, 0.002880096435546875, 0.00621795654296875, -0.055389404296875, -0.047882080078125, 0.07080078125, 0.0111236572265625, 0.07025146484375, -0.01418304443359375, 0.0291748046875, 0.0279083251953125, -0.0009546279907226562, -0.0321044921875, -0.0203704833984375, 0.0261383056640625, -0.0210418701171875, 0.00974273681640625, -0.043121337890625, -0.01497650146484375, 0.04180908203125, -0.038665771484375, 0.0186767578125, -0.0411376953125, 0.0250244140625, -0.01456451416015625, 0.0036907196044921875, -0.0233612060546875, 0.0142364501953125, -0.0195465087890625, 0.025970458984375, -0.0303497314453125, -0.00937652587890625, 0.027374267578125, 0.045623779296875, -0.0109405517578125, 0.0106201171875, 0.0147857666015625, 0.060699462890625, 0.041534423828125, 0.01412200927734375, -0.0128173828125, 0.0494384765625, 0.0310821533203125, -0.047271728515625, -0.0088958740234375, -0.0540771484375, 0.01052093505859375, 0.00991058349609375, -0.0009984970092773438, -0.0018358230590820312, -0.0172271728515625, 0.00110626220703125, -0.037353515625, -0.02044677734375, 0.0124053955078125, -0.00897979736328125, -0.032958984375, 0.01312255859375, -0.03466796875, -0.017242431640625, 0.00048470497131347656, -0.01132965087890625, 0.033447265625, 0.03631591796875, -0.0084228515625, -0.0212860107421875, 0.035888671875, 0.00963592529296875, -0.01145172119140625, -0.007785797119140625, 0.023712158203125, 0.0347900390625, 0.0159759521484375, -0.0081939697265625, -0.024200439453125, -0.005008697509765625, 0.0208587646484375, 0.017791748046875, 0.0011396408081054688, 0.0121002197265625, 0.0244598388671875, 0.030670166015625, 0.0836181640625, -0.0236968994140625, -0.01477813720703125, -0.023468017578125, -0.026641845703125, -0.056640625, -0.0181732177734375, 0.01220703125, 0.032623291015625, 0.0439453125, 0.0126800537109375, 0.02178955078125, -0.0259857177734375, 0.03302001953125, -0.01024627685546875, 0.01776123046875, 0.01702880859375, -0.046112060546875, 0.034881591796875, -0.010467529296875, -0.0626220703125, -0.0128173828125, 0.01120758056640625, 0.00457763671875, 0.0022945404052734375, -0.022674560546875, -0.004657745361328125, -0.012298583984375, -0.0001780986785888672, -0.0665283203125, -0.003032684326171875, 0.00214385986328125, 0.006061553955078125, -0.0184326171875, -0.042205810546875, -0.0036106109619140625, -0.04949951171875, 0.00241851806640625, -0.041290283203125, 0.01155853271484375, -0.00666046142578125, 0.0156707763671875, -0.020599365234375, 0.04833984375, 0.0838623046875, 0.0101470947265625, 0.033966064453125, -0.0662841796875, 0.02960205078125, 0.046234130859375, -0.050384521484375, 0.0222930908203125, -0.0355224609375, -0.031524658203125, 0.0194549560546875, 0.01751708984375, 0.017608642578125, 0.034942626953125, -0.03424072265625, -0.040130615234375, 6.175041198730469e-05, 0.0276641845703125, 0.0288543701171875, 0.05963134765625, 0.0213165283203125, -0.0286102294921875, 0.012969970703125, -0.00365447998046875, -0.0148162841796875, 0.0210113525390625, -0.0196380615234375, -0.04669189453125, 0.032196044921875, 0.004199981689453125, 0.024017333984375, 0.015899658203125, -0.0268707275390625, 0.0080413818359375, 0.00432586669921875, -0.0010700225830078125, 0.00698089599609375, -0.004207611083984375, 0.022735595703125, 0.019866943359375, -0.0016565322875976562, 0.03558349609375, 0.0063934326171875, -0.030242919921875, -0.03399658203125, -0.001201629638671875, 0.01215362548828125, 0.01276397705078125, 0.00606536865234375, 0.057525634765625, -0.0043487548828125, 0.0031604766845703125, 0.011810302734375, 0.0151824951171875, 0.0030231475830078125, -0.0399169921875, -0.050872802734375, -0.0176239013671875, -0.0101318359375, 0.033416748046875, -0.023956298828125, 0.0257415771484375, 0.006916046142578125, -0.003070831298828125, -0.062347412109375, 0.0484619140625, 0.048004150390625, 0.00984954833984375, 0.004886627197265625, -0.0261383056640625, -0.003833770751953125, 0.00626373291015625, -0.0096282958984375, -0.0165557861328125, -0.0196685791015625, 0.0182647705078125, -0.035858154296875, 0.03326416015625, 0.037139892578125, 0.033355712890625, 0.03985595703125, -0.0214080810546875, -0.01274871826171875, -0.0131988525390625, 0.00937652587890625, 0.01690673828125, 0.02960205078125, 0.047119140625, -0.00966644287109375, -0.005588531494140625, 0.0012378692626953125, -0.042144775390625, -0.0011739730834960938, -0.034149169921875, 0.00775909423828125, -0.0239105224609375, -0.0101470947265625, -0.00013494491577148438, 0.0010967254638671875, -0.0540771484375, 0.01543426513671875, 0.04736328125, -0.03765869140625, 0.0013628005981445312, 0.0413818359375, -0.0176239013671875, -0.068603515625, -0.0180816650390625, -0.027099609375, -0.02130126953125, 0.0462646484375, 0.058319091796875, 0.035614013671875, 0.0028324127197265625, -0.01018524169921875, -0.0026531219482421875, 0.0250396728515625, -0.01873779296875, 0.004993438720703125, -0.0279083251953125, -0.0118408203125, 0.001430511474609375, 0.00970458984375, 0.016815185546875, 0.037811279296875, 0.03045654296875, -0.0523681640625, -0.003757476806640625, 0.040771484375, -0.0074615478515625, -0.0110931396484375, -0.01280975341796875, 0.0004601478576660156, -0.05059814453125, 0.0228118896484375, -0.0303955078125, -0.033203125, -0.0283050537109375, 0.061492919921875, -0.052459716796875, -0.017303466796875, 0.005802154541015625, 0.05035400390625, -0.0115814208984375, 0.01334381103515625, -0.0572509765625, 0.0150299072265625, -0.0007047653198242188, -0.043426513671875, 0.0254364013671875, 0.0338134765625, 0.028564453125, 0.0340576171875, -0.019805908203125, -0.0006690025329589844, -0.0135498046875, 0.031829833984375, -0.01959228515625, -0.0021228790283203125, 0.0217742919921875, -0.0137939453125, -0.015411376953125, -0.0077667236328125, -0.01239013671875, 0.016937255859375, -0.016448974609375, -0.0391845703125, -0.017547607421875, 0.029632568359375, -0.004669189453125, 0.0282440185546875, -0.0040283203125, -0.01464080810546875, 0.01511383056640625, 0.03131103515625, 0.01477813720703125, 0.0289306640625, 0.0287628173828125, -0.012969970703125, -0.0283203125, 0.0080413818359375, -0.007015228271484375, -0.0699462890625, -0.0228424072265625, -0.0180816650390625, 0.01456451416015625, 0.03802490234375, -0.0040740966796875, 0.05517578125, -0.0124969482421875, 0.004634857177734375, 0.042938232421875, 0.0007724761962890625, -0.07470703125, -0.0186614990234375, -0.004924774169921875, 0.0119781494140625, 0.027587890625, 0.0083465576171875, 0.0306396484375, 0.057464599609375, 0.0430908203125, 0.012451171875, 0.00984954833984375, -0.00032711029052734375, 0.0333251953125, 0.009796142578125, -0.01702880859375, 0.0235443115234375, 0.032501220703125, -0.0212249755859375, 0.0330810546875, 0.0164031982421875, -0.006755828857421875, -0.01323699951171875, 0.003971099853515625, -0.0096282958984375, 0.0165863037109375, -0.02886962890625, -0.052642822265625, -0.005023956298828125, 0.0455322265625, 0.028839111328125, -0.0210113525390625, 0.00405120849609375, 0.0361328125, -0.035003662109375, 0.02203369140625, -0.0215301513671875, -0.06732177734375, 0.0306396484375, -0.104248046875, -0.01038360595703125, -0.0228118896484375, -0.02093505859375, -0.051849365234375, -0.0016269683837890625, 0.017791748046875, 0.032806396484375, -0.0018672943115234375, -0.0204620361328125, -0.0220794677734375, 0.0165557861328125, 0.0293731689453125, -0.007228851318359375, -0.0299835205078125, 0.0010595321655273438, -0.04071044921875, -0.040130615234375, 0.021484375, 0.046875, -0.01105499267578125, -0.02996826171875, 0.03662109375, -0.012969970703125, 0.006855010986328125, -0.00940704345703125, 0.046417236328125, -0.0117034912109375, -0.01464080810546875, -0.056793212890625, 0.0183563232421875, 0.0299224853515625, 0.0300750732421875, 0.06817626953125, -0.0513916015625, 0.0254364013671875, -0.01430511474609375, 0.0205078125, -0.0276641845703125, 0.030853271484375, -0.015777587890625, 0.00022649765014648438, -0.04315185546875, -0.0182037353515625, 0.03265380859375, 0.006877899169921875, -0.027923583984375, -0.024871826171875, -0.03021240234375, 0.024200439453125, 0.035064697265625, -0.005428314208984375, 0.027252197265625, -0.036956787109375, 0.0310211181640625, 0.016082763671875, -0.00243377685546875, 0.009979248046875, 0.033294677734375, 0.0243072509765625, 0.011566162109375, 0.00519561767578125, 0.01038360595703125, -0.0239410400390625, -0.0225982666015625, -0.009246826171875, -0.0738525390625, -0.031494140625, 0.0303802490234375, -0.002811431884765625, 0.0166168212890625, -0.0298309326171875, -0.003078460693359375, 0.018707275390625, 0.03802490234375, -0.0103759765625, 0.03814697265625, -0.01491546630859375, 0.01806640625, 0.0028839111328125, 0.0244293212890625, 0.00023734569549560547, 0.027008056640625, 0.041290283203125, -0.02020263671875, 0.0281219482421875, 0.0222015380859375, -0.0213165283203125, 0.0321044921875, -0.0249176025390625, -0.024383544921875, -0.004405975341796875, -0.0160369873046875, -0.01242828369140625, -0.02899169921875, 0.0274505615234375, -0.052032470703125, -0.038787841796875, -0.055450439453125, -0.0164337158203125, -0.01947021484375, 0.01345062255859375, -0.01275634765625, 0.0191802978515625, -0.040283203125, -0.0248870849609375, 0.0531005859375, 0.006237030029296875, 0.005950927734375, -0.0304412841796875, -0.022674560546875, -0.02728271484375, -0.0589599609375, -0.043060302734375, -0.000728607177734375, -0.0163116455078125, -0.03021240234375, -0.0134429931640625, 0.00835418701171875, 0.02734375, 0.01763916015625, -1.6748905181884766e-05, -0.0244903564453125, -0.016845703125, -0.049835205078125, -0.0264892578125, 0.007198333740234375, -0.0098724365234375, 0.0184326171875, -0.01026153564453125, 0.01364898681640625, -0.002010345458984375, 0.01024627685546875, 0.01010894775390625, 0.004497528076171875, 0.00782012939453125, -0.004596710205078125, 0.038909912109375, -0.037506103515625, -0.0654296875, 0.008636474609375, 0.039093017578125, -0.0235443115234375, 0.00807952880859375, -0.042633056640625, 0.035888671875, 0.0152587890625, -0.04547119140625, -0.031707763671875, 0.02728271484375, 0.03924560546875, 0.032073974609375, 0.011688232421875, 0.036041259765625, -0.03155517578125, -0.040557861328125, 0.01142120361328125, 0.01465606689453125, 0.024749755859375, 0.03265380859375, -0.01458740234375, -0.0259857177734375, 0.014495849609375, -0.020355224609375, 0.0025310516357421875, 0.0229949951171875, 0.0020923614501953125, 0.019287109375, -0.0192718505859375, -0.00328826904296875, -0.05059814453125, -0.0325927734375, -0.0296478271484375, 0.0219268798828125, -0.00885009765625, -0.007198333740234375, 0.0035076141357421875, 0.032562255859375, 0.03558349609375, 0.001956939697265625, 0.03924560546875, 0.02606201171875, -0.002197265625, -0.040863037109375, 0.030517578125, -0.024322509765625, 0.06048583984375, 0.01323699951171875, 0.015655517578125, -0.006015777587890625, 0.01727294921875, -0.0020771026611328125, -0.007061004638671875, -0.05078125, 0.044525146484375, 0.009857177734375, -0.00916290283203125, 0.041778564453125, 0.0031585693359375, 0.0711669921875, 0.06256103515625, -0.0221710205078125, -0.05023193359375, 0.016754150390625, -0.036590576171875, 0.050933837890625, 0.0214385986328125, -0.054901123046875, -0.028411865234375, -0.019927978515625, 0.005466461181640625, -0.03131103515625, -0.0230712890625, 0.043121337890625, -0.006877899169921875, 0.0426025390625, 0.01377105712890625, -0.006786346435546875, 0.013946533203125, -0.020050048828125, -0.0288848876953125, 0.04583740234375, -0.058074951171875, -0.0167999267578125, 0.081298828125, -0.022216796875, -0.0249786376953125, -0.0012454986572265625, 0.040252685546875, -0.03704833984375, 0.01416778564453125, -0.0086822509765625, -0.01203155517578125, 0.038665771484375, -0.023590087890625, -0.007640838623046875, -0.040863037109375, 0.0548095703125, 0.0654296875, -0.04656982421875, -0.007350921630859375, 0.0037708282470703125, -0.060791015625, -0.03912353515625], index=0, object='embedding')\n", + "Embedding(embedding=[-0.039581298828125, -0.01374053955078125, 0.0179443359375, -0.0227203369140625, -0.0178680419921875, -0.027191162109375, 0.0384521484375, 0.0269927978515625, -0.0200653076171875, -0.0261077880859375, -0.0257720947265625, 0.0203857421875, -0.0118560791015625, -0.0011034011840820312, 0.0267181396484375, -0.026824951171875, -0.0008034706115722656, 0.06317138671875, -0.006183624267578125, -0.034088134765625, -0.02374267578125, 0.033721923828125, -0.0175018310546875, 0.035125732421875, 0.01140594482421875, 0.0181427001953125, -0.022186279296875, -0.044891357421875, 0.036529541015625, -0.018463134765625, -0.0199737548828125, -0.03192138671875, 0.007366180419921875, 0.007213592529296875, -0.0256805419921875, -0.02056884765625, -0.015594482421875, -0.0236663818359375, -0.040008544921875, -0.002197265625, 0.0004749298095703125, -0.07305908203125, 0.06072998046875, 0.00762176513671875, -0.0168609619140625, -0.0261383056640625, -0.003665924072265625, -0.024627685546875, -0.029052734375, 0.0201568603515625, 0.0085601806640625, -0.07696533203125, 0.0274200439453125, 0.0140533447265625, 0.028656005859375, -0.01024627685546875, 0.00978851318359375, 0.0209197998046875, -0.04742431640625, 0.0123291015625, -0.04644775390625, 0.00893402099609375, -0.052398681640625, -0.025360107421875, 0.01250457763671875, 0.03466796875, -0.0205841064453125, 0.0279541015625, 0.013580322265625, -0.014373779296875, -0.010223388671875, -0.03515625, 0.03448486328125, 0.00475311279296875, -0.046234130859375, 0.01255035400390625, 0.026153564453125, 0.0039043426513671875, -0.01922607421875, -0.026214599609375, 0.03485107421875, 0.02069091796875, -0.0017833709716796875, 0.041351318359375, -0.027984619140625, 0.035858154296875, -0.040435791015625, -0.0194244384765625, -0.0206451416015625, 0.004703521728515625, -0.019317626953125, 0.0262298583984375, -0.04022216796875, -0.04095458984375, -0.0294036865234375, -0.01009368896484375, -0.042266845703125, 0.056610107421875, -0.0445556640625, 0.01073455810546875, 0.0169219970703125, 0.0026302337646484375, -0.025360107421875, -0.0194091796875, 0.0596923828125, -0.037933349609375, 0.0245361328125, 0.0116119384765625, -0.05279541015625, 0.0020160675048828125, 0.0157623291015625, -0.011138916015625, 0.0279083251953125, -0.01151275634765625, -0.03533935546875, -0.0292816162109375, -0.01215362548828125, -0.022125244140625, 0.034637451171875, 0.0125885009765625, 0.01763916015625, -0.02532958984375, 0.021881103515625, -0.0016298294067382812, -0.0172576904296875, -0.02911376953125, 0.06353759765625, 0.0282745361328125, -0.0198974609375, 0.0256195068359375, -0.02081298828125, 0.024993896484375, -0.061553955078125, -0.00618743896484375, -0.037841796875, -0.0142669677734375, 0.051788330078125, -0.0008325576782226562, -0.0233917236328125, -0.04925537109375, 0.0022907257080078125, 0.052215576171875, -0.04425048828125, -0.030242919921875, 0.0187225341796875, -0.06427001953125, -0.0027217864990234375, 0.020477294921875, 0.0107574462890625, 0.0028839111328125, 0.00424957275390625, 0.0230712890625, 0.01190948486328125, -0.0017910003662109375, -0.00403594970703125, 0.00872802734375, 0.030181884765625, 0.04962158203125, -0.005901336669921875, 0.03228759765625, 0.051177978515625, -0.0292205810546875, 0.0128936767578125, -0.0226287841796875, -0.0010747909545898438, 0.01076507568359375, -0.0301055908203125, -0.04302978515625, -0.00855255126953125, 0.02655029296875, 0.035888671875, 0.0254058837890625, 0.025970458984375, 0.021881103515625, 0.0139617919921875, 0.005702972412109375, 0.0787353515625, 0.0098876953125, 0.0012807846069335938, -0.0306549072265625, -0.01995849609375, 0.04718017578125, 0.06475830078125, -0.02667236328125, -0.0276336669921875, 0.0379638671875, 0.048583984375, -0.03253173828125, -0.014251708984375, 0.0426025390625, -0.0039825439453125, -0.01016998291015625, 0.0090179443359375, 0.01288604736328125, 0.06793212890625, 0.021270751953125, -0.0236663818359375, -0.00795745849609375, 0.0087127685546875, 0.00621795654296875, -0.00853729248046875, 0.0025959014892578125, 0.0157928466796875, 0.007091522216796875, -0.01363372802734375, -0.007152557373046875, -0.043212890625, -0.0115966796875, -0.02313232421875, -0.085693359375, -0.045867919921875, 0.03973388671875, -0.0002282857894897461, 0.00972747802734375, 0.0265045166015625, 0.0267486572265625, -0.0164337158203125, -0.048065185546875, -0.0255889892578125, -0.053131103515625, -0.007488250732421875, 0.08990478515625, 0.013275146484375, -0.0438232421875, -0.006378173828125, 0.021026611328125, -0.048736572265625, 0.01263427734375, 0.0086212158203125, -0.014007568359375, -0.009552001953125, 0.036590576171875, -0.0635986328125, -0.03387451171875, 0.0179443359375, 0.0259857177734375, -0.025238037109375, 0.028533935546875, 0.0294036865234375, 0.031951904296875, 0.006977081298828125, 0.017913818359375, -0.007320404052734375, 0.012725830078125, 0.0206756591796875, -0.0044097900390625, 0.0731201171875, 0.054595947265625, -0.0430908203125, 0.023406982421875, -0.00463104248046875, -0.0357666015625, -0.00258636474609375, 0.046234130859375, -0.00719451904296875, 0.02630615234375, 0.0232696533203125, 0.0038509368896484375, -0.0299835205078125, 0.016571044921875, -0.01250457763671875, 0.00821685791015625, 0.04962158203125, 0.0157318115234375, -0.04107666015625, 0.03125, 0.0090484619140625, -0.0164947509765625, -0.002048492431640625, -0.0256195068359375, -0.036468505859375, 0.031158447265625, 0.03948974609375, 0.029541015625, 0.0194091796875, 0.042388916015625, -0.036956787109375, 0.0264129638671875, 0.0167083740234375, -0.004673004150390625, 0.03070068359375, -0.01319122314453125, 0.0068817138671875, -0.00899505615234375, 0.01904296875, -0.06365966796875, -0.022857666015625, 0.006259918212890625, -0.017669677734375, -0.00478363037109375, -0.02398681640625, -0.008026123046875, 0.05999755859375, -0.06671142578125, 0.04595947265625, -0.0209503173828125, -0.0167236328125, -0.1409912109375, 0.0196685791015625, -0.031097412109375, 0.01297760009765625, 0.0106201171875, -0.0227508544921875, -0.01904296875, -0.0439453125, -0.030517578125, -0.021575927734375, -0.01413726806640625, -0.0537109375, -0.05389404296875, -0.0259857177734375, -0.0298614501953125, -0.01531219482421875, 0.01340484619140625, 0.0035076141357421875, 0.002685546875, -0.0002090930938720703, 0.02001953125, -0.026123046875, 0.043975830078125, -0.07122802734375, -0.01415252685546875, 0.010650634765625, -0.00336456298828125, 0.0157928466796875, 0.01439666748046875, -0.0440673828125, 0.0210418701171875, 0.034820556640625, -0.0011806488037109375, 0.05328369140625, 0.051361083984375, 0.035919189453125, 0.01153564453125, 0.01218414306640625, -0.0028781890869140625, -0.0252532958984375, -0.006839752197265625, 0.036865234375, 0.041595458984375, -0.0143890380859375, 0.048431396484375, 0.0052490234375, -0.0198516845703125, -0.061492919921875, -0.03582763671875, -0.03497314453125, 0.046539306640625, -0.02587890625, 0.01727294921875, 0.007808685302734375, -0.05047607421875, -0.0229339599609375, -0.03912353515625, 0.0447998046875, 0.046234130859375, -0.05078125, 0.029388427734375, -0.011871337890625, 0.03851318359375, 0.0650634765625, -0.0196990966796875, 0.026123046875, 0.0606689453125, -0.001194000244140625, -0.02386474609375, 0.0200042724609375, 0.0362548828125, 0.0245513916015625, 0.0083770751953125, -0.01425933837890625, 0.01495361328125, 0.004184722900390625, -0.042144775390625, -0.022674560546875, 0.01378631591796875, -0.12359619140625, 0.0382080078125, 0.03326416015625, 0.0146026611328125, 0.024200439453125, -0.02923583984375, -0.036346435546875, 0.00605010986328125, 0.034027099609375, 0.0013828277587890625, 0.2054443359375, 0.01287841796875, 0.0299530029296875, 0.0290069580078125, -0.041839599609375, -0.051361083984375, -0.021209716796875, -0.033416748046875, 0.044647216796875, -0.00469970703125, -0.0372314453125, 0.024810791015625, 0.0606689453125, -0.00768280029296875, 0.031494140625, -0.04150390625, 0.0294952392578125, 0.0384521484375, 0.06024169921875, -0.0140533447265625, 0.037384033203125, -0.0166015625, -0.029998779296875, 0.01226043701171875, 0.0113067626953125, 0.02142333984375, 0.033172607421875, 0.0121917724609375, -0.002765655517578125, 0.0261383056640625, 0.004993438720703125, -0.0150909423828125, 0.00138092041015625, 0.00350189208984375, -0.0055389404296875, -0.0257568359375, -0.0411376953125, 0.062286376953125, 0.02496337890625, 0.01151275634765625, -0.041656494140625, -0.05670166015625, -0.021240234375, 0.048858642578125, 0.033782958984375, -0.036163330078125, 0.0135650634765625, 0.0079803466796875, -0.0216217041015625, -0.0053253173828125, 0.00432586669921875, 0.01006317138671875, -0.0221710205078125, -0.00849151611328125, -0.01117706298828125, -0.0205078125, 0.0127410888671875, 0.0032291412353515625, -0.018463134765625, -0.01129150390625, 0.023284912109375, -0.0168304443359375, -0.0279541015625, 0.0147247314453125, 0.01226043701171875, -0.0259857177734375, -0.010467529296875, -0.023223876953125, -0.0196533203125, 0.04302978515625, 0.037841796875, 0.04217529296875, 0.01561737060546875, 0.00560760498046875, 0.019073486328125, -0.020751953125, -0.047149658203125, 0.01708984375, 0.0209197998046875, 0.006526947021484375, -0.02239990234375, -0.001583099365234375, -0.08001708984375, -0.0189971923828125, -0.0037555694580078125, 0.05908203125, 0.032257080078125, -0.04461669921875, -0.01447296142578125, -0.02471923828125, -0.035888671875, -0.02154541015625, -0.033172607421875, -0.00279998779296875, 0.03961181640625, 0.0275726318359375, -0.0017042160034179688, -0.008270263671875, -0.0169219970703125, 0.0303802490234375, -0.020660400390625, -0.046112060546875, 0.023284912109375, 0.06622314453125, -0.0095062255859375, -0.020416259765625, 0.0026912689208984375, 0.0158843994140625, 0.017822265625, 0.03466796875, -0.019622802734375, -0.005420684814453125, 0.001979827880859375, 0.01535797119140625, 0.0290679931640625, -0.0260467529296875, -0.00782012939453125, -0.005954742431640625, 0.052459716796875, 0.056732177734375, -0.0083465576171875, -0.024444580078125, 0.0299072265625, -0.049224853515625, -0.0219268798828125, 0.0070343017578125, -0.0194091796875, 0.0121002197265625, -0.007389068603515625, 0.0304718017578125, 0.030914306640625, -0.00846099853515625, 0.02532958984375, -0.0022830963134765625, -0.0013828277587890625, 0.008880615234375, -0.0262298583984375, 0.01323699951171875, 0.0265960693359375, 0.04974365234375, 0.0159912109375, -0.0212860107421875, -0.0235137939453125, -0.03662109375, -0.01222991943359375, -0.06976318359375, 0.01491546630859375, 0.0136871337890625, -0.0281524658203125, -0.002887725830078125, -0.0273590087890625, 0.0765380859375, 0.01509857177734375, -0.05206298828125, -0.0262908935546875, -0.04779052734375, -0.0171966552734375, -0.034271240234375, -0.0216217041015625, 8.493661880493164e-05, 0.018341064453125, -0.0242156982421875, 0.028167724609375, 0.040985107421875, -0.0008654594421386719, 0.0379638671875, -0.0450439453125, -0.028594970703125, 0.0241546630859375, -0.029510498046875, 0.004199981689453125, -0.0270843505859375, -0.0034770965576171875, 0.020904541015625, 0.019256591796875, 0.032684326171875, 0.0245513916015625, 0.011505126953125, -0.041015625, -0.0007472038269042969, 0.040679931640625, 0.0194244384765625, 0.027557373046875, -0.00807952880859375, -0.033935546875, 0.0284271240234375, 0.0005254745483398438, -0.0257415771484375, 0.01776123046875, -0.043121337890625, -0.001190185546875, 0.041534423828125, -0.02825927734375, -0.00757598876953125, 0.004245758056640625, 0.02252197265625, -0.04754638671875, 0.04833984375, -0.02197265625, 0.0276031494140625, 0.01385498046875, 0.050872802734375, 0.045379638671875, 0.0089263916015625, -0.00272369384765625, -0.00765228271484375, -0.0167388916015625, -0.0254974365234375, 0.017425537109375, -0.00852203369140625, 0.0219268798828125, -0.020751953125, 0.01139068603515625, 0.03192138671875, 0.0308074951171875, 0.049957275390625, -0.06561279296875, -0.0067291259765625, -0.01255035400390625, -0.019866943359375, 0.03912353515625, 0.021881103515625, 0.0335693359375, -0.036041259765625, 0.01416778564453125, 0.01389312744140625, -0.0263824462890625, -0.0428466796875, 0.066650390625, 0.0054931640625, 0.00754547119140625, -0.0212554931640625, 0.014404296875, 0.03143310546875, -0.01180267333984375, -0.0169525146484375, -0.00559234619140625, 0.018829345703125, 0.047210693359375, -0.00010728836059570312, 0.01375579833984375, 0.0028934478759765625, -0.036346435546875, -0.007442474365234375, -0.0306549072265625, -0.0186004638671875, 0.019775390625, 0.0303955078125, 0.01134490966796875, -0.0292205810546875, -0.0229339599609375, -0.0321044921875, 0.01467132568359375, -0.010406494140625, -0.043853759765625, 0.00017905235290527344, -0.037322998046875, -0.009185791015625, 9.775161743164062e-06, 0.00351715087890625, -0.0225830078125, -0.045379638671875, -0.0261383056640625, 0.001758575439453125, -0.0179901123046875, -0.054931640625, 0.03778076171875, 0.03277587890625, -0.03466796875, -0.050994873046875, -0.0036869049072265625, 0.0025959014892578125, -0.01806640625, -0.01097869873046875, 0.0706787109375, 0.036529541015625, 0.02001953125, -0.058013916015625, 0.01085662841796875, -0.01519012451171875, 0.01297760009765625, 0.025665283203125, 5.543231964111328e-06, 0.024078369140625, 0.004764556884765625, 0.025299072265625, 0.037078857421875, -0.01824951171875, 0.019927978515625, -0.0206298828125, 0.050445556640625, -0.037567138671875, -0.006511688232421875, -0.01444244384765625, -0.03173828125, 0.01922607421875, -0.002063751220703125, 0.060760498046875, -0.03564453125, -0.021087646484375, 0.0008492469787597656, 0.030914306640625, -0.033966064453125, 0.030548095703125, -0.0127410888671875, -0.031402587890625, -0.006969451904296875, 0.01953125, -0.022003173828125, 0.00537872314453125, 0.00574493408203125, -0.041168212890625, 0.0027294158935546875, 0.05450439453125, 0.036376953125, -0.044830322265625, 0.008331298828125, 0.0241241455078125, -0.0147705078125, 0.01959228515625, 0.01558685302734375, -0.01580810546875, 0.047210693359375, -0.0533447265625, 0.0035572052001953125, 0.0157318115234375, 0.007472991943359375, -0.046905517578125, -0.00518035888671875, -0.0118255615234375, 0.0218048095703125, 0.06829833984375, -0.006793975830078125, 0.01465606689453125, -0.034912109375, 0.0227508544921875, -0.006072998046875, 0.0462646484375, 0.023712158203125, 0.0017118453979492188, 0.004581451416015625, -0.019256591796875, 0.0034427642822265625, -0.01236724853515625, -0.0261993408203125, -0.019775390625, 0.01088714599609375, -0.006595611572265625, 0.022918701171875, 0.0015077590942382812, -0.0308380126953125, 0.059112548828125, -0.004047393798828125, -0.039764404296875, 0.01380157470703125, -0.00246429443359375, -0.05291748046875, -0.0283203125, 0.0022869110107421875, -0.03131103515625, 0.0127410888671875, 0.01284027099609375, 0.05828857421875, 0.053009033203125, 0.029632568359375, 0.017730712890625, 0.046417236328125, 0.0299224853515625, 0.04840087890625, -0.00455474853515625, 0.0026149749755859375, -0.0291595458984375, 0.0014657974243164062, 0.038482666015625, 0.049224853515625, 0.03778076171875, -0.047149658203125, -0.0335693359375, -0.03106689453125, 0.0163726806640625, 0.033782958984375, -0.041107177734375, -0.05963134765625, 0.008514404296875, 0.0251312255859375, -0.01020050048828125, -0.00437164306640625, -0.030517578125, 0.029693603515625, -0.053680419921875, 0.0033092498779296875, -0.0247344970703125, -0.07232666015625, 0.017608642578125, -0.099853515625, 0.0020236968994140625, -0.0173797607421875, -0.020721435546875, -0.0523681640625, 0.020751953125, -0.0286102294921875, 0.0027599334716796875, 0.01103973388671875, -0.05108642578125, -0.0216064453125, -0.004108428955078125, -0.00743865966796875, -0.017730712890625, 0.0009741783142089844, 0.002193450927734375, -0.00402069091796875, 0.0218353271484375, -0.006927490234375, 0.04852294921875, -0.01172637939453125, -0.0164642333984375, 0.05780029296875, -0.0391845703125, 0.005687713623046875, 0.03582763671875, 0.008453369140625, -0.044952392578125, -0.03375244140625, -0.028411865234375, 0.006099700927734375, 0.013916015625, 0.01384735107421875, 0.055084228515625, -0.034210205078125, -0.01187896728515625, 0.0062103271484375, -0.01580810546875, -0.0140533447265625, 0.0052947998046875, 0.0243377685546875, 0.01143646240234375, 0.024627685546875, -0.060546875, 0.01091766357421875, 0.0152435302734375, -0.0213775634765625, 0.0205535888671875, -0.01959228515625, -0.0018701553344726562, 0.0300445556640625, 0.021331787109375, 0.04278564453125, -0.04718017578125, 0.03021240234375, 0.042755126953125, -0.0277862548828125, 0.00460052490234375, 0.011383056640625, 0.0157470703125, -0.04754638671875, -0.026275634765625, -0.05194091796875, -0.0792236328125, -0.01702880859375, -0.040557861328125, -0.1256103515625, 0.048675537109375, 0.0010385513305664062, -0.0179443359375, -0.03802490234375, -0.044677734375, -0.006198883056640625, -0.01385498046875, 0.042572021484375, -0.01020050048828125, 0.043548583984375, 0.000621795654296875, -0.0152587890625, -0.0263214111328125, 0.0135040283203125, 0.056182861328125, 0.00284576416015625, -0.008819580078125, -0.006244659423828125, 0.01190948486328125, -0.022918701171875, -0.00728607177734375, -0.031707763671875, -0.028411865234375, -0.0748291015625, 0.00753021240234375, -0.0225067138671875, -0.0152130126953125, 0.04248046875, -0.032440185546875, -0.0150146484375, 0.0062713623046875, -0.0260162353515625, 0.0214385986328125, 0.0022735595703125, 0.00795745849609375, -0.00832366943359375, -0.0038623809814453125, -0.03021240234375, -0.03814697265625, 0.0423583984375, -0.004154205322265625, -0.01367950439453125, -0.00638580322265625, 0.02490234375, -0.0033969879150390625, -0.0188446044921875, -0.0396728515625, -0.01195526123046875, -0.0163116455078125, -0.026031494140625, -0.0247344970703125, -0.049407958984375, 0.013671875, -0.00836181640625, -0.04168701171875, 0.03216552734375, -0.0006427764892578125, -0.0284576416015625, -0.034576416015625, -0.046142578125, 0.004764556884765625, 0.0023193359375, -0.026824951171875, 0.025543212890625, -0.013671875, 0.01192474365234375, 0.0016012191772460938, 0.0159454345703125, -0.003604888916015625, 0.023651123046875, 0.04815673828125, 0.0249786376953125, -0.03607177734375, 0.01546478271484375, 0.01922607421875, -0.00908660888671875, 0.03436279296875, 0.0164794921875, -0.0014562606811523438, 0.035369873046875, -0.02825927734375, -0.0242156982421875, -0.040985107421875, 0.0106353759765625, 0.00836181640625, -0.0158538818359375, 0.0413818359375, -0.0506591796875, 0.0178375244140625, 0.03887939453125, -0.0182037353515625, 0.0050506591796875, 0.01535797119140625, -0.004703521728515625, -0.0239410400390625, 0.008880615234375, -0.07562255859375, -0.00936126708984375, 0.038360595703125, 0.0215606689453125, 0.016357421875, -0.01239013671875, 0.0001932382583618164, -0.0482177734375, 0.031707763671875, -0.019500732421875, 0.0267181396484375, 0.00580596923828125, -0.042510986328125, 0.00848388671875, 0.0031147003173828125, 0.0706787109375, 0.0158538818359375, 0.01323699951171875, 0.05615234375, 0.032196044921875, -0.03070068359375, 0.036773681640625, 0.06500244140625, 0.03692626953125, -0.018798828125, 0.0130767822265625, -0.017730712890625, -0.007598876953125, -0.01898193359375, 0.0301361083984375, -0.037506103515625, 0.046722412109375, -0.0171356201171875, 0.0067596435546875, -0.007022857666015625, -0.0182037353515625, 0.0239105224609375, -0.021026611328125, -0.0231170654296875, 0.01483917236328125, -0.0284423828125, -0.0335693359375, -0.0241241455078125, -0.0240020751953125, -0.036590576171875, -0.0118408203125, 0.0379638671875, -0.0010442733764648438, 0.00226593017578125, 0.01200103759765625, -0.01413726806640625, -0.037017822265625, -0.007770538330078125, 0.030242919921875, 0.0072784423828125, -0.0125274658203125, 0.01207733154296875, 0.031402587890625, -0.00926971435546875, 0.0093536376953125, -0.051055908203125, 0.059661865234375, -0.046630859375, -0.00963592529296875, -0.01708984375, 0.0237274169921875, -0.006511688232421875, -0.061798095703125, -0.00345611572265625, -0.0506591796875, 0.0298309326171875, -0.005519866943359375, 0.029815673828125, -0.0166778564453125, 0.049835205078125, -0.0014410018920898438, -0.027374267578125, 0.024200439453125, 0.021209716796875, 0.032867431640625, 0.01334381103515625], index=1, object='embedding')\n", + "Embedding(embedding=[-0.029296875, -0.026092529296875, -0.0214996337890625, -0.01512908935546875, 0.007564544677734375, 0.0016078948974609375, -0.0179595947265625, -0.040557861328125, -0.020416259765625, -0.01435089111328125, -0.00641632080078125, 0.002452850341796875, -0.023162841796875, 0.01049041748046875, -0.0030345916748046875, -0.0577392578125, 0.049530029296875, 0.01227569580078125, -0.01055145263671875, -0.033935546875, -0.029296875, -0.0211639404296875, -0.02734375, -0.0225677490234375, 0.0191192626953125, 0.004344940185546875, 0.0025043487548828125, 0.0426025390625, 0.001995086669921875, -0.044677734375, 0.016693115234375, 0.028656005859375, 0.004764556884765625, 0.0167236328125, 0.01459503173828125, -0.0269012451171875, 0.0022487640380859375, -0.01053619384765625, -0.03802490234375, -0.0221099853515625, -0.01300811767578125, -0.005550384521484375, 0.01323699951171875, -0.004974365234375, -0.0245819091796875, -0.0284576416015625, -0.0244903564453125, -0.017852783203125, -0.0128173828125, 0.01605224609375, -0.013336181640625, -0.0038089752197265625, 0.031463623046875, 0.00284576416015625, 0.01361846923828125, 0.06365966796875, 0.008819580078125, -0.035675048828125, -0.051116943359375, -0.0196075439453125, -0.01526641845703125, 0.00872039794921875, -0.006618499755859375, 0.032745361328125, 0.046966552734375, 0.08685302734375, 0.0068359375, -0.01502227783203125, -0.028472900390625, -0.00521087646484375, -0.0178985595703125, -0.028839111328125, 0.00902557373046875, 0.01183319091796875, -0.0001958608627319336, 0.03192138671875, -0.0157470703125, -0.0185394287109375, -0.043487548828125, -0.01261138916015625, 0.0880126953125, 0.0286712646484375, -0.0416259765625, 0.00466156005859375, 0.0236358642578125, 0.05615234375, -0.00348663330078125, 0.06231689453125, 0.0114898681640625, 0.008880615234375, -0.052734375, -0.0352783203125, 0.00795745849609375, -0.046417236328125, -0.063720703125, 0.01058197021484375, -0.04052734375, 0.01092529296875, 0.01593017578125, 0.0040130615234375, 0.00473785400390625, 0.0011882781982421875, -0.016754150390625, -0.007793426513671875, 0.0127410888671875, -0.0176849365234375, 0.00913238525390625, 0.0298309326171875, -0.0202484130859375, 0.013824462890625, 0.0372314453125, 0.0231475830078125, 0.022857666015625, 0.0316162109375, -0.0256195068359375, -0.0253143310546875, -0.02301025390625, -0.00901031494140625, -0.0041046142578125, 0.013031005859375, -0.037445068359375, -0.0268707275390625, 0.033233642578125, 0.0004925727844238281, -0.015655517578125, -0.035614013671875, 0.0281524658203125, 0.0277252197265625, 0.002841949462890625, -0.01178741455078125, 0.035552978515625, -0.0014390945434570312, -0.011016845703125, 0.0102996826171875, -0.0316162109375, -0.02001953125, 0.050262451171875, -0.043212890625, -0.033294677734375, -0.032470703125, -0.0277099609375, -0.01812744140625, 0.03515625, -0.001911163330078125, -0.0032672882080078125, -0.039215087890625, -0.0284271240234375, -0.061798095703125, 0.0419921875, -0.035797119140625, -0.05340576171875, 0.0015544891357421875, 0.038330078125, 0.0021514892578125, -0.033477783203125, 0.027374267578125, -0.020843505859375, 0.046600341796875, -0.05126953125, -0.025238037109375, 0.022979736328125, -0.019317626953125, -0.04193115234375, 0.03875732421875, 0.0038928985595703125, 0.0018482208251953125, 0.0007176399230957031, -0.0169219970703125, -0.007686614990234375, -0.03900146484375, 0.006458282470703125, -0.034912109375, -0.00485992431640625, -0.046173095703125, 0.0230865478515625, 0.0301971435546875, 0.056396484375, 0.00018596649169921875, -0.01157379150390625, -0.03466796875, 0.0024318695068359375, 0.045135498046875, 0.019561767578125, -0.0181121826171875, 0.016326904296875, 0.028289794921875, 0.032745361328125, -0.00469207763671875, 0.0005941390991210938, 0.0484619140625, -0.0643310546875, -0.0224456787109375, 0.0019063949584960938, -0.00982666015625, 0.01554107666015625, 0.0037860870361328125, 0.049346923828125, 0.00494384765625, -0.01137542724609375, -0.002590179443359375, -0.06939697265625, 0.039642333984375, -0.00348663330078125, 0.036895751953125, -0.0281982421875, -0.0149383544921875, -0.01788330078125, 0.003265380859375, 0.015594482421875, -0.0609130859375, 0.00933074951171875, 0.0126953125, 0.043914794921875, 0.01142120361328125, -0.01218414306640625, 0.01139068603515625, 0.01171112060546875, -0.0299835205078125, -0.05096435546875, 0.007266998291015625, -0.054443359375, 0.0171051025390625, 0.007213592529296875, -0.00014889240264892578, 0.031463623046875, -0.0094757080078125, 0.010406494140625, -0.0037784576416015625, -0.0118560791015625, -0.0173492431640625, -0.0197601318359375, 0.03204345703125, -0.0465087890625, -0.026123046875, -0.0298614501953125, 0.004657745361328125, -0.052581787109375, 0.016693115234375, 0.0163116455078125, 0.044281005859375, -0.01377105712890625, 0.016693115234375, -0.05755615234375, 0.00830078125, -0.0063934326171875, -0.0099334716796875, 0.021148681640625, -0.005245208740234375, 0.0017223358154296875, -0.004199981689453125, -0.00732421875, 0.0289154052734375, -0.00530242919921875, 0.04534912109375, 0.0243072509765625, -0.0048828125, 0.0203704833984375, -0.0194549560546875, 0.021697998046875, 0.0180816650390625, 0.00859832763671875, -0.0467529296875, 0.0052642822265625, 0.03106689453125, 0.044281005859375, 0.036590576171875, -0.0380859375, 0.0036830902099609375, 0.024261474609375, 0.036834716796875, -0.0215301513671875, -0.001773834228515625, 0.00846099853515625, 0.021881103515625, 0.0179290771484375, 0.0263671875, -0.007251739501953125, -0.01544952392578125, -0.0166015625, 0.024261474609375, 0.06536865234375, -0.026519775390625, 0.03302001953125, 0.049041748046875, 0.005462646484375, -0.048553466796875, 0.008544921875, -0.005115509033203125, -0.030059814453125, 0.027618408203125, 0.01861572265625, -0.0362548828125, 0.039306640625, 0.005550384521484375, -0.0249786376953125, 0.0159454345703125, -0.004749298095703125, -0.1578369140625, 0.020172119140625, 0.019134521484375, 0.004459381103515625, -0.002391815185546875, -0.006683349609375, -0.041015625, 0.00826263427734375, -0.0167236328125, -0.03082275390625, -0.0064239501953125, -0.026275634765625, -0.06494140625, -0.01242828369140625, 0.032073974609375, 0.01184844970703125, -0.009765625, -0.051788330078125, -0.0190887451171875, -0.053955078125, 0.03704833984375, 0.00861358642578125, -0.00283050537109375, -0.028350830078125, 0.00445556640625, 0.04296875, 0.0286102294921875, -0.025238037109375, -0.04058837890625, -0.0302886962890625, 0.01953125, 0.0211639404296875, 0.0036220550537109375, 0.040496826171875, 0.04949951171875, 0.005428314208984375, 0.022003173828125, -0.0204010009765625, -0.00630950927734375, 0.0294342041015625, 0.0029621124267578125, 0.058380126953125, -0.004772186279296875, 0.0162353515625, 0.024810791015625, 0.01090240478515625, -0.0301055908203125, -0.0277252197265625, -0.037139892578125, -0.01294708251953125, 0.005828857421875, 0.0186309814453125, 0.0311737060546875, -0.025421142578125, -0.0130462646484375, 0.0159454345703125, 0.00293731689453125, 0.024444580078125, 0.034332275390625, -0.029327392578125, 0.04681396484375, 0.0037326812744140625, -0.0208892822265625, 0.01108551025390625, 0.034881591796875, 0.0308990478515625, 0.0229034423828125, -0.050628662109375, 0.001251220703125, -0.0270233154296875, 0.0611572265625, 0.06109619140625, -0.024169921875, 0.005931854248046875, 0.0198822021484375, 0.0228424072265625, 0.06878662109375, -0.06304931640625, 0.010589599609375, -0.1141357421875, 0.01509857177734375, 0.0147705078125, -0.00250244140625, -0.009765625, -0.05010986328125, -0.0292205810546875, 0.0184478759765625, 0.0274658203125, 0.050689697265625, 0.225830078125, -0.0175933837890625, 0.0745849609375, -0.101318359375, -0.0204315185546875, -0.03900146484375, -0.047760009765625, -0.056915283203125, 0.0274200439453125, -0.02630615234375, 0.00296783447265625, -0.0088043212890625, 0.034576416015625, -0.0059356689453125, 0.004352569580078125, 0.08673095703125, 0.00640869140625, 0.0271759033203125, 0.05615234375, -0.02056884765625, 0.002201080322265625, -0.023651123046875, -0.0143890380859375, -0.0181427001953125, 0.0034770965576171875, 0.00994110107421875, 0.004642486572265625, 0.0111236572265625, 0.010040283203125, 0.0110015869140625, -0.0305023193359375, -0.02783203125, 0.022216796875, -0.03521728515625, -0.039947509765625, -0.0272216796875, -0.0406494140625, -0.021331787109375, -0.0150909423828125, -0.01514434814453125, -0.0380859375, -0.0521240234375, 0.057220458984375, 0.03253173828125, 0.075927734375, -0.0202789306640625, 0.013427734375, -0.0027141571044921875, -0.05694580078125, 0.01139068603515625, 0.004974365234375, 0.01277923583984375, 0.0035552978515625, 0.0144195556640625, -0.0217437744140625, 0.0306854248046875, 0.005252838134765625, -0.025787353515625, 0.006412506103515625, -0.035308837890625, 0.006343841552734375, -0.0009183883666992188, 0.0255126953125, -0.044403076171875, 0.01220703125, 0.01482391357421875, 0.07440185546875, -0.022979736328125, -0.019927978515625, 0.01154327392578125, 0.0249481201171875, -0.0200347900390625, 0.062469482421875, 0.01213836669921875, 0.002101898193359375, 0.0307159423828125, -0.0168914794921875, -0.0035190582275390625, 0.07879638671875, 0.02398681640625, 0.02276611328125, -0.0120086669921875, -0.0428466796875, -0.0085906982421875, 0.024749755859375, -0.007266998291015625, 0.026031494140625, 0.0614013671875, 0.0400390625, -0.055389404296875, 0.0105438232421875, 0.02618408203125, -0.0106048583984375, 0.0217742919921875, -0.019561767578125, -0.07159423828125, -0.004241943359375, -0.0020618438720703125, -0.01953125, 0.0301971435546875, -0.036041259765625, -0.01490020751953125, -0.0170440673828125, 0.0479736328125, 0.01849365234375, -0.027740478515625, -0.00981903076171875, 0.0306854248046875, -0.01141357421875, -0.0220489501953125, -0.04266357421875, -0.0278778076171875, 0.01226806640625, 0.0058441162109375, 0.0159759521484375, 0.0263519287109375, -0.024688720703125, 0.021820068359375, -0.009368896484375, 0.07318115234375, 0.00022912025451660156, -0.030853271484375, 0.00600433349609375, 0.006938934326171875, -0.013580322265625, -0.030548095703125, -0.0016736984252929688, 0.0360107421875, 0.0343017578125, 0.01544952392578125, 0.028045654296875, -0.0161895751953125, 0.057098388671875, 0.0182952880859375, 0.0007042884826660156, 0.0228118896484375, -0.01824951171875, 0.0204620361328125, -0.031585693359375, -0.0204620361328125, 0.01096343994140625, 0.0302734375, 0.01006317138671875, 0.004215240478515625, -0.0194854736328125, 0.050994873046875, -0.034576416015625, -0.003017425537109375, -0.04248046875, -0.0181732177734375, -0.0271148681640625, 0.061065673828125, -0.052032470703125, -0.05743408203125, 0.00603485107421875, -0.00223541259765625, 0.0301971435546875, -0.10137939453125, 0.01837158203125, -0.0074920654296875, 0.027435302734375, 0.00027561187744140625, 0.0253448486328125, 0.09320068359375, 0.02734375, 0.0238494873046875, -0.040283203125, 0.00681304931640625, 0.054931640625, -0.01026153564453125, -0.01471710205078125, -0.01126861572265625, -0.050750732421875, 0.017364501953125, 0.02972412109375, -0.048553466796875, -0.049652099609375, -0.00634765625, 0.02813720703125, -0.0050048828125, 0.027435302734375, 0.038360595703125, 0.037322998046875, 0.027374267578125, -0.0467529296875, 0.0269012451171875, 0.042327880859375, -0.01036834716796875, 0.0347900390625, -0.04058837890625, -0.0033893585205078125, 0.04974365234375, 0.024749755859375, 0.00916290283203125, 0.03057861328125, -0.031829833984375, 0.03887939453125, 0.01403045654296875, 0.00027251243591308594, -2.6404857635498047e-05, 0.00785064697265625, 0.0249176025390625, 0.0006728172302246094, -0.01285552978515625, -0.005035400390625, -0.0196990966796875, -0.048248291015625, -0.048828125, -0.0013532638549804688, 0.06976318359375, -0.03131103515625, 0.0150604248046875, 0.041473388671875, -0.013031005859375, -0.0019435882568359375, 0.0084075927734375, 0.0225067138671875, -0.0172882080078125, -0.01204681396484375, -0.05352783203125, -0.007587432861328125, -0.0027408599853515625, 0.034881591796875, -0.02667236328125, 0.0472412109375, 0.0291748046875, 0.03375244140625, -0.01505279541015625, 0.0257110595703125, 0.038177490234375, 0.0010223388671875, 0.0604248046875, -0.006977081298828125, -0.04486083984375, -0.0205230712890625, 0.037109375, -0.060272216796875, -0.0129547119140625, 0.04888916015625, -0.0030422210693359375, -0.007305145263671875, 0.0008087158203125, 0.043701171875, 0.0218963623046875, 0.00946807861328125, -0.005954742431640625, 0.045806884765625, 0.00572967529296875, 0.01479339599609375, 0.03851318359375, -0.0036182403564453125, -0.01267242431640625, 0.03887939453125, -0.0056304931640625, -0.01403045654296875, 0.00353240966796875, -0.0193328857421875, 0.042022705078125, -0.02459716796875, 0.0008311271667480469, 0.0155029296875, 0.041961669921875, -0.0233001708984375, 0.0026035308837890625, 0.0372314453125, -0.0166015625, 0.005031585693359375, 0.049835205078125, -0.00771331787109375, -0.07696533203125, 0.019012451171875, -0.01161956787109375, 0.0010442733764648438, 0.018768310546875, 0.04193115234375, -0.0016469955444335938, -0.033721923828125, 0.030426025390625, -0.0241851806640625, -0.00650787353515625, 0.0028057098388671875, 0.006633758544921875, -0.0186004638671875, 0.038909912109375, 0.0029449462890625, -0.0310211181640625, 0.039703369140625, 0.019287109375, 0.041717529296875, -0.001392364501953125, -0.005634307861328125, -0.01026153564453125, -0.028228759765625, 0.007427215576171875, -0.006195068359375, 0.006633758544921875, -0.039794921875, 0.0129852294921875, -0.022613525390625, -0.005084991455078125, -0.04705810546875, 0.055938720703125, -0.043487548828125, -0.0211639404296875, 0.022613525390625, 0.0030689239501953125, 0.025482177734375, -0.00649261474609375, -0.039459228515625, -0.0160980224609375, 0.0176849365234375, -0.05963134765625, 0.032745361328125, 0.01287078857421875, -0.0189361572265625, -0.0167236328125, 0.044647216796875, -0.041046142578125, -0.037994384765625, 0.01739501953125, 0.00878143310546875, 0.021636962890625, 0.03460693359375, -0.0268707275390625, 0.016571044921875, -0.007465362548828125, 0.007045745849609375, 0.021392822265625, -0.05267333984375, -0.08331298828125, 0.009368896484375, 0.06890869140625, -0.0018625259399414062, -0.006938934326171875, -0.02484130859375, -0.01032257080078125, 0.019287109375, -0.03363037109375, 0.00238037109375, 0.0204010009765625, 0.032073974609375, 0.00833892822265625, -0.0218048095703125, -0.043304443359375, 0.0003895759582519531, -0.01529693603515625, -0.017547607421875, -0.0208282470703125, -0.005863189697265625, 0.007366180419921875, 0.013031005859375, 0.0404052734375, -0.041229248046875, -0.0201263427734375, 0.0120697021484375, 0.062225341796875, -0.023773193359375, -0.0011854171752929688, -0.016815185546875, 0.0020599365234375, -0.0007572174072265625, -0.0163421630859375, 0.0225372314453125, 0.059661865234375, 0.0360107421875, 0.0249176025390625, -0.0140380859375, -0.0244598388671875, -0.048187255859375, -0.01132965087890625, 0.0081024169921875, 0.049957275390625, 0.07550048828125, -0.007015228271484375, -0.0220489501953125, 0.0296478271484375, 0.00252532958984375, -0.019134521484375, 0.004482269287109375, -0.0023956298828125, 0.0106201171875, -0.04315185546875, -0.0428466796875, -0.0174407958984375, 0.0003209114074707031, 0.01154327392578125, 0.013580322265625, -0.01708984375, 0.004039764404296875, -0.0225067138671875, -0.01641845703125, -0.0192718505859375, -0.043548583984375, 0.00809478759765625, -0.11572265625, 0.0019445419311523438, -0.004344940185546875, 0.0065460205078125, -0.055816650390625, 0.0191192626953125, -0.0025539398193359375, 0.02716064453125, -0.02093505859375, 0.0044097900390625, -0.043426513671875, 0.0413818359375, -0.033477783203125, -0.0211334228515625, -0.002277374267578125, 0.0025272369384765625, 0.0004394054412841797, -0.0294342041015625, -0.0291900634765625, 0.028900146484375, 0.007236480712890625, -0.053314208984375, 0.024444580078125, -0.0146331787109375, 0.035400390625, -0.021636962890625, 0.0243377685546875, -0.01506805419921875, -0.0174560546875, -0.05828857421875, 0.010345458984375, -0.06182861328125, 0.047088623046875, 0.0218963623046875, -0.005321502685546875, 0.02337646484375, -0.016845703125, -0.0013780593872070312, -0.0084075927734375, 0.020843505859375, 0.00890350341796875, -0.016693115234375, -0.033172607421875, -0.01080322265625, -0.017669677734375, 0.0595703125, -0.037628173828125, -0.0194854736328125, 0.0032672882080078125, 0.01012420654296875, -0.0189208984375, 0.0164642333984375, 0.0215606689453125, -0.05352783203125, -0.0044097900390625, 0.0090484619140625, -0.00812530517578125, 0.0274810791015625, 0.0198822021484375, 0.04949951171875, 0.0129852294921875, -0.0038547515869140625, -0.01226043701171875, -0.026458740234375, -0.0501708984375, 0.03515625, -0.048126220703125, 0.006282806396484375, 0.05853271484375, -0.0071563720703125, -0.035614013671875, -0.01387786865234375, 0.02008056640625, -0.01323699951171875, 0.025177001953125, 0.050140380859375, 0.04071044921875, -0.01210784912109375, -0.00018727779388427734, -0.043731689453125, 0.0311737060546875, 0.01641845703125, -0.0178680419921875, -0.014434814453125, -0.0276641845703125, 0.038726806640625, 0.0117950439453125, -0.058349609375, -0.02679443359375, -0.01080322265625, -0.03936767578125, -0.01080322265625, -0.033935546875, -0.018218994140625, -0.01470947265625, 0.01178741455078125, -0.0255584716796875, -0.007297515869140625, -0.00598907470703125, -0.0246124267578125, 0.0004949569702148438, 0.00018346309661865234, 0.01480865478515625, 0.00141143798828125, -0.04058837890625, 0.04669189453125, 0.0188140869140625, 0.0030155181884765625, 0.02899169921875, -0.0259857177734375, 0.0036983489990234375, 0.0010004043579101562, -0.04437255859375, -0.0084686279296875, -0.047698974609375, 0.0229034423828125, -0.0129547119140625, -0.00673675537109375, 0.04150390625, 0.038604736328125, -0.0408935546875, 0.044097900390625, -0.0067901611328125, -0.0189666748046875, -0.034881591796875, -0.006664276123046875, -0.005950927734375, 0.0172119140625, 0.053680419921875, -0.046295166015625, 0.020172119140625, -0.00431060791015625, 0.0234527587890625, 0.01654052734375, 0.0142364501953125, -0.039794921875, -0.0213623046875, 0.007625579833984375, -0.0294342041015625, -0.08148193359375, 0.005565643310546875, 0.020111083984375, -0.0400390625, -0.033935546875, -0.00399017333984375, 0.03118896484375, 0.026458740234375, -0.04010009765625, -0.06671142578125, 0.016845703125, 0.046173095703125, 0.01120758056640625, 0.0208282470703125, -0.0190887451171875, -0.042938232421875, -0.0343017578125, -0.0302734375, 0.020538330078125, 0.0178070068359375, 0.0289764404296875, -0.034637451171875, -0.044708251953125, -0.0225830078125, -0.0628662109375, 0.0164031982421875, 0.0135498046875, -0.00139617919921875, -0.0128631591796875, -0.03302001953125, -0.013092041015625, -0.01418304443359375, 0.037078857421875, -0.041961669921875, -0.035888671875, 0.016937255859375, -0.0552978515625, -0.002582550048828125, 0.049041748046875, 0.02606201171875, 0.0073394775390625, 0.035369873046875, 0.02716064453125, 0.040985107421875, -0.05816650390625, 0.010040283203125, -0.0206451416015625, 0.0211029052734375, -0.005828857421875, -0.0005421638488769531, -0.0001691579818725586, -0.02325439453125, -0.021881103515625, -0.00026679039001464844, -0.0234222412109375, -0.0033779144287109375, 0.047821044921875, 0.00389862060546875, 0.0114288330078125, -0.019012451171875, 0.0027008056640625, 0.03521728515625, -0.0279693603515625, 0.0249481201171875, -0.021697998046875, -0.0125274658203125, 0.072021484375, -0.004680633544921875, 0.001186370849609375, -0.0019550323486328125, -0.0258636474609375, -0.0136871337890625, 0.0224151611328125, -0.04693603515625, 0.044830322265625, 0.0014581680297851562, 0.0116729736328125, -0.0189056396484375, 0.004512786865234375, -0.01529693603515625, 0.02313232421875, -0.060516357421875, 0.0189208984375, -0.050537109375, -0.0027866363525390625, -0.00395965576171875, 0.01230621337890625, -0.01837158203125, 0.04034423828125, -0.0233001708984375, 0.01641845703125, 0.0250091552734375, -0.019378662109375, -0.042144775390625, 0.01285552978515625, -0.0046539306640625, 0.004344940185546875, 0.028656005859375, 0.01094818115234375, 0.060577392578125, -0.046844482421875, -0.006317138671875, -0.028717041015625, -0.0302276611328125, -0.01261138916015625], index=2, object='embedding')\n" + ] + }, + { + "data": { + "text/plain": [ + "['Where are Toyota cars manufactured?',\n", + " 'Where is Japan?',\n", + " 'What is the engine power of Toyota RAV4']" + ] + }, + "execution_count": 23, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "find_similar_faq(user_query, faq_list)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "1340d73c", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Embedding(embedding=[-0.045440673828125, -0.037139892578125, -0.007572174072265625, 0.02801513671875, 0.0013322830200195312, 0.005214691162109375, -0.060943603515625, 0.03546142578125, -0.024261474609375, 0.0294952392578125, 0.020172119140625, 0.0290374755859375, 0.0293121337890625, 0.036834716796875, 0.0233001708984375, -0.06817626953125, 0.0177764892578125, 0.043304443359375, -0.0168914794921875, -0.04705810546875, -0.0280914306640625, -0.0340576171875, -0.00039005279541015625, -0.062225341796875, 0.01184844970703125, 0.0059051513671875, 0.01010894775390625, -0.00926971435546875, 0.010009765625, -0.08038330078125, 0.02398681640625, 0.0125732421875, -0.050079345703125, -0.0072784423828125, -0.01812744140625, -0.0145111083984375, -3.8623809814453125e-05, -0.0343017578125, -0.046783447265625, -0.024383544921875, -0.0020885467529296875, -0.0135498046875, 0.02789306640625, -0.008880615234375, -0.0223236083984375, -0.043212890625, -0.0175628662109375, -0.0230712890625, -0.02288818359375, 0.0126800537109375, 0.035430908203125, -0.029876708984375, 0.013763427734375, -0.01042938232421875, 0.052703857421875, 0.1025390625, -0.0162200927734375, -0.0704345703125, -0.061279296875, 0.021453857421875, -0.08074951171875, -0.005352020263671875, -0.0222625732421875, 0.045928955078125, 0.01398468017578125, 0.055450439453125, -0.0230865478515625, -0.038665771484375, -0.0248870849609375, -0.025360107421875, -0.00695037841796875, -0.037078857421875, 0.0243377685546875, -0.0013971328735351562, -0.00298309326171875, 0.01529693603515625, -0.014801025390625, 0.01110076904296875, -0.0264739990234375, 0.0062408447265625, 0.032012939453125, 0.01268768310546875, 0.0138092041015625, 0.0303802490234375, 0.043243408203125, 0.0177764892578125, -0.049346923828125, -0.0060882568359375, -0.035675048828125, -0.0175628662109375, -0.00815582275390625, -0.0298309326171875, -0.03533935546875, -0.0523681640625, -0.0257720947265625, 0.0138702392578125, -0.036865234375, 0.0302276611328125, -0.0423583984375, -0.019805908203125, 0.030364990234375, -0.030426025390625, 0.0131072998046875, -0.06475830078125, 0.048858642578125, -0.007175445556640625, 0.0221099853515625, 0.0302734375, -0.0158538818359375, 0.01422119140625, 0.0269775390625, -0.0018129348754882812, -0.01395416259765625, -0.0237274169921875, -0.04644775390625, -0.031341552734375, -0.01538848876953125, -0.057220458984375, 0.042510986328125, 0.0101776123046875, -0.01485443115234375, -0.0161895751953125, 0.033355712890625, -0.01367950439453125, -0.0295867919921875, -0.0845947265625, 0.04559326171875, -0.00614166259765625, 0.007396697998046875, -0.0043182373046875, 0.0257568359375, -0.00977325439453125, -0.0257568359375, 0.03021240234375, -0.0305938720703125, -0.066162109375, 0.0261383056640625, -0.0202178955078125, -0.07586669921875, -0.037261962890625, 0.006565093994140625, 0.01390838623046875, -0.058624267578125, -0.00502777099609375, -0.0009617805480957031, -0.038543701171875, -0.010772705078125, -0.0006756782531738281, 0.005481719970703125, 0.0073699951171875, 0.01068878173828125, 0.0020275115966796875, 0.0204620361328125, 0.02044677734375, -0.0259246826171875, -0.038360595703125, 0.006526947021484375, 0.061676025390625, -0.055694580078125, 0.00583648681640625, 0.053558349609375, 0.0311431884765625, 0.0165557861328125, 0.04351806640625, -0.0048065185546875, -0.0055694580078125, -0.0487060546875, -0.0290374755859375, -0.033416748046875, 0.0017910003662109375, -0.0064849853515625, 0.02947998046875, -0.03448486328125, -0.016510009765625, 0.040557861328125, 0.0229034423828125, 0.052001953125, 0.0128173828125, -0.01427459716796875, -0.0006699562072753906, -0.0013074874877929688, 0.054595947265625, 0.04913330078125, 0.0316162109375, -0.0038356781005859375, 0.039794921875, 0.03350830078125, -0.030029296875, 0.001186370849609375, 0.0264739990234375, -0.031707763671875, -0.01548004150390625, -0.01068115234375, 0.0022182464599609375, 0.0382080078125, 0.031768798828125, 0.0162811279296875, 0.0174713134765625, -0.0273284912109375, -0.053863525390625, -0.0628662109375, 0.026641845703125, 0.00592803955078125, -0.0007500648498535156, 0.006683349609375, -0.01316070556640625, -0.0160980224609375, -0.037445068359375, -0.005565643310546875, -0.0367431640625, 0.0115814208984375, -0.0051116943359375, 0.003490447998046875, 0.0343017578125, -0.00438690185546875, 0.0245819091796875, -0.043792724609375, -0.035736083984375, -0.0123748779296875, 0.0259857177734375, -0.064208984375, 0.054534912109375, -0.0148162841796875, 0.00701141357421875, 0.017669677734375, -0.009307861328125, -0.00917816162109375, 0.005863189697265625, -0.007297515869140625, -0.013763427734375, 0.005031585693359375, 0.0065155029296875, -0.05291748046875, -0.01739501953125, 0.0038928985595703125, 0.03070068359375, -0.033111572265625, -0.0225677490234375, 0.07525634765625, 0.046875, 0.01358795166015625, 0.07269287109375, 0.0005278587341308594, -0.0010766983032226562, -0.0205230712890625, -0.006504058837890625, 0.058258056640625, 0.0137176513671875, -0.0343017578125, -0.0029621124267578125, -0.0085906982421875, 0.0038204193115234375, -0.00566864013671875, 0.0523681640625, 0.020050048828125, 0.03521728515625, 0.027740478515625, -0.0121612548828125, -0.0291290283203125, -0.004604339599609375, -0.0287322998046875, -0.0106201171875, 0.028472900390625, 0.0303802490234375, 0.017303466796875, 0.02777099609375, -0.023193359375, -0.02294921875, -0.01253509521484375, -0.0154266357421875, -0.017333984375, 0.006298065185546875, -0.0009016990661621094, 0.03826904296875, 0.01009368896484375, 0.00920867919921875, 0.01132965087890625, -0.027801513671875, 0.0050811767578125, 0.033111572265625, 0.059173583984375, -0.0758056640625, 0.039276123046875, 0.0229644775390625, 0.00974273681640625, -0.0279388427734375, 0.0006022453308105469, 0.030029296875, 0.01041412353515625, 0.0034999847412109375, -0.035980224609375, -0.02734375, 0.0552978515625, -0.0176239013671875, 0.018463134765625, 0.028045654296875, -0.01189422607421875, -0.1578369140625, -0.00616455078125, -0.0253143310546875, -0.01354217529296875, -0.0126800537109375, -0.01953125, -0.0579833984375, -0.0240936279296875, -0.0074005126953125, -0.0272979736328125, 0.004192352294921875, -0.051361083984375, -0.06500244140625, -0.028961181640625, 0.0208892822265625, -0.0035762786865234375, -0.0210113525390625, 0.025390625, 0.0172882080078125, -0.0300750732421875, 0.0221099853515625, -0.009490966796875, 0.019378662109375, 0.0009164810180664062, -0.0290069580078125, 0.0037689208984375, 0.039398193359375, 0.017669677734375, -0.036865234375, -0.022918701171875, 0.06622314453125, -0.00655364990234375, 0.0017375946044921875, 0.07183837890625, 0.08148193359375, -0.00408935546875, 0.0210723876953125, -0.024261474609375, 0.0110626220703125, 0.025665283203125, 0.00594329833984375, 0.044586181640625, 0.023681640625, 0.005146026611328125, -0.010467529296875, -0.0101776123046875, -0.0140228271484375, -0.021942138671875, -0.0010499954223632812, -0.04052734375, 0.0230560302734375, 0.0247802734375, -0.019927978515625, -0.02630615234375, -0.0267791748046875, -0.01303863525390625, 0.0080718994140625, 0.0494384765625, 0.0101318359375, -0.043212890625, 0.007354736328125, -0.0305328369140625, 0.007625579833984375, 0.0276031494140625, 0.049774169921875, 0.01308441162109375, 0.029327392578125, 0.02435302734375, -0.0170440673828125, 0.0023937225341796875, 0.0447998046875, 0.0035572052001953125, -0.01904296875, -0.0219573974609375, 0.0179595947265625, -0.006198883056640625, 0.0008401870727539062, -0.0278167724609375, 0.025115966796875, -0.1217041015625, -0.005802154541015625, 0.050384521484375, -0.005748748779296875, -0.017486572265625, 0.0011739730834960938, -0.038909912109375, -0.040924072265625, 0.055694580078125, 0.04962158203125, 0.2037353515625, -0.00516510009765625, 0.04754638671875, -0.0306854248046875, 0.015716552734375, -0.033660888671875, 0.0010671615600585938, -0.036529541015625, 0.019317626953125, -0.0311737060546875, -0.0158538818359375, 0.024566650390625, 0.0670166015625, -0.006378173828125, -0.0038089752197265625, -0.0088043212890625, 0.0297698974609375, 0.0218048095703125, 0.036529541015625, -0.02447509765625, -0.0033931732177734375, -0.0051116943359375, 0.006114959716796875, -0.02093505859375, -0.00959014892578125, -0.0211639404296875, -0.0095672607421875, 0.00145721435546875, 0.00011998414993286133, 0.01348114013671875, -0.024261474609375, -0.007755279541015625, 0.0033702850341796875, -0.00543975830078125, -0.04559326171875, -0.00907135009765625, -0.05010986328125, 0.02996826171875, 0.002880096435546875, 0.00621795654296875, -0.055389404296875, -0.047882080078125, 0.07080078125, 0.0111236572265625, 0.07025146484375, -0.01418304443359375, 0.0291748046875, 0.0279083251953125, -0.0009546279907226562, -0.0321044921875, -0.0203704833984375, 0.0261383056640625, -0.0210418701171875, 0.00974273681640625, -0.043121337890625, -0.01497650146484375, 0.04180908203125, -0.038665771484375, 0.0186767578125, -0.0411376953125, 0.0250244140625, -0.01456451416015625, 0.0036907196044921875, -0.0233612060546875, 0.0142364501953125, -0.0195465087890625, 0.025970458984375, -0.0303497314453125, -0.00937652587890625, 0.027374267578125, 0.045623779296875, -0.0109405517578125, 0.0106201171875, 0.0147857666015625, 0.060699462890625, 0.041534423828125, 0.01412200927734375, -0.0128173828125, 0.0494384765625, 0.0310821533203125, -0.047271728515625, -0.0088958740234375, -0.0540771484375, 0.01052093505859375, 0.00991058349609375, -0.0009984970092773438, -0.0018358230590820312, -0.0172271728515625, 0.00110626220703125, -0.037353515625, -0.02044677734375, 0.0124053955078125, -0.00897979736328125, -0.032958984375, 0.01312255859375, -0.03466796875, -0.017242431640625, 0.00048470497131347656, -0.01132965087890625, 0.033447265625, 0.03631591796875, -0.0084228515625, -0.0212860107421875, 0.035888671875, 0.00963592529296875, -0.01145172119140625, -0.007785797119140625, 0.023712158203125, 0.0347900390625, 0.0159759521484375, -0.0081939697265625, -0.024200439453125, -0.005008697509765625, 0.0208587646484375, 0.017791748046875, 0.0011396408081054688, 0.0121002197265625, 0.0244598388671875, 0.030670166015625, 0.0836181640625, -0.0236968994140625, -0.01477813720703125, -0.023468017578125, -0.026641845703125, -0.056640625, -0.0181732177734375, 0.01220703125, 0.032623291015625, 0.0439453125, 0.0126800537109375, 0.02178955078125, -0.0259857177734375, 0.03302001953125, -0.01024627685546875, 0.01776123046875, 0.01702880859375, -0.046112060546875, 0.034881591796875, -0.010467529296875, -0.0626220703125, -0.0128173828125, 0.01120758056640625, 0.00457763671875, 0.0022945404052734375, -0.022674560546875, -0.004657745361328125, -0.012298583984375, -0.0001780986785888672, -0.0665283203125, -0.003032684326171875, 0.00214385986328125, 0.006061553955078125, -0.0184326171875, -0.042205810546875, -0.0036106109619140625, -0.04949951171875, 0.00241851806640625, -0.041290283203125, 0.01155853271484375, -0.00666046142578125, 0.0156707763671875, -0.020599365234375, 0.04833984375, 0.0838623046875, 0.0101470947265625, 0.033966064453125, -0.0662841796875, 0.02960205078125, 0.046234130859375, -0.050384521484375, 0.0222930908203125, -0.0355224609375, -0.031524658203125, 0.0194549560546875, 0.01751708984375, 0.017608642578125, 0.034942626953125, -0.03424072265625, -0.040130615234375, 6.175041198730469e-05, 0.0276641845703125, 0.0288543701171875, 0.05963134765625, 0.0213165283203125, -0.0286102294921875, 0.012969970703125, -0.00365447998046875, -0.0148162841796875, 0.0210113525390625, -0.0196380615234375, -0.04669189453125, 0.032196044921875, 0.004199981689453125, 0.024017333984375, 0.015899658203125, -0.0268707275390625, 0.0080413818359375, 0.00432586669921875, -0.0010700225830078125, 0.00698089599609375, -0.004207611083984375, 0.022735595703125, 0.019866943359375, -0.0016565322875976562, 0.03558349609375, 0.0063934326171875, -0.030242919921875, -0.03399658203125, -0.001201629638671875, 0.01215362548828125, 0.01276397705078125, 0.00606536865234375, 0.057525634765625, -0.0043487548828125, 0.0031604766845703125, 0.011810302734375, 0.0151824951171875, 0.0030231475830078125, -0.0399169921875, -0.050872802734375, -0.0176239013671875, -0.0101318359375, 0.033416748046875, -0.023956298828125, 0.0257415771484375, 0.006916046142578125, -0.003070831298828125, -0.062347412109375, 0.0484619140625, 0.048004150390625, 0.00984954833984375, 0.004886627197265625, -0.0261383056640625, -0.003833770751953125, 0.00626373291015625, -0.0096282958984375, -0.0165557861328125, -0.0196685791015625, 0.0182647705078125, -0.035858154296875, 0.03326416015625, 0.037139892578125, 0.033355712890625, 0.03985595703125, -0.0214080810546875, -0.01274871826171875, -0.0131988525390625, 0.00937652587890625, 0.01690673828125, 0.02960205078125, 0.047119140625, -0.00966644287109375, -0.005588531494140625, 0.0012378692626953125, -0.042144775390625, -0.0011739730834960938, -0.034149169921875, 0.00775909423828125, -0.0239105224609375, -0.0101470947265625, -0.00013494491577148438, 0.0010967254638671875, -0.0540771484375, 0.01543426513671875, 0.04736328125, -0.03765869140625, 0.0013628005981445312, 0.0413818359375, -0.0176239013671875, -0.068603515625, -0.0180816650390625, -0.027099609375, -0.02130126953125, 0.0462646484375, 0.058319091796875, 0.035614013671875, 0.0028324127197265625, -0.01018524169921875, -0.0026531219482421875, 0.0250396728515625, -0.01873779296875, 0.004993438720703125, -0.0279083251953125, -0.0118408203125, 0.001430511474609375, 0.00970458984375, 0.016815185546875, 0.037811279296875, 0.03045654296875, -0.0523681640625, -0.003757476806640625, 0.040771484375, -0.0074615478515625, -0.0110931396484375, -0.01280975341796875, 0.0004601478576660156, -0.05059814453125, 0.0228118896484375, -0.0303955078125, -0.033203125, -0.0283050537109375, 0.061492919921875, -0.052459716796875, -0.017303466796875, 0.005802154541015625, 0.05035400390625, -0.0115814208984375, 0.01334381103515625, -0.0572509765625, 0.0150299072265625, -0.0007047653198242188, -0.043426513671875, 0.0254364013671875, 0.0338134765625, 0.028564453125, 0.0340576171875, -0.019805908203125, -0.0006690025329589844, -0.0135498046875, 0.031829833984375, -0.01959228515625, -0.0021228790283203125, 0.0217742919921875, -0.0137939453125, -0.015411376953125, -0.0077667236328125, -0.01239013671875, 0.016937255859375, -0.016448974609375, -0.0391845703125, -0.017547607421875, 0.029632568359375, -0.004669189453125, 0.0282440185546875, -0.0040283203125, -0.01464080810546875, 0.01511383056640625, 0.03131103515625, 0.01477813720703125, 0.0289306640625, 0.0287628173828125, -0.012969970703125, -0.0283203125, 0.0080413818359375, -0.007015228271484375, -0.0699462890625, -0.0228424072265625, -0.0180816650390625, 0.01456451416015625, 0.03802490234375, -0.0040740966796875, 0.05517578125, -0.0124969482421875, 0.004634857177734375, 0.042938232421875, 0.0007724761962890625, -0.07470703125, -0.0186614990234375, -0.004924774169921875, 0.0119781494140625, 0.027587890625, 0.0083465576171875, 0.0306396484375, 0.057464599609375, 0.0430908203125, 0.012451171875, 0.00984954833984375, -0.00032711029052734375, 0.0333251953125, 0.009796142578125, -0.01702880859375, 0.0235443115234375, 0.032501220703125, -0.0212249755859375, 0.0330810546875, 0.0164031982421875, -0.006755828857421875, -0.01323699951171875, 0.003971099853515625, -0.0096282958984375, 0.0165863037109375, -0.02886962890625, -0.052642822265625, -0.005023956298828125, 0.0455322265625, 0.028839111328125, -0.0210113525390625, 0.00405120849609375, 0.0361328125, -0.035003662109375, 0.02203369140625, -0.0215301513671875, -0.06732177734375, 0.0306396484375, -0.104248046875, -0.01038360595703125, -0.0228118896484375, -0.02093505859375, -0.051849365234375, -0.0016269683837890625, 0.017791748046875, 0.032806396484375, -0.0018672943115234375, -0.0204620361328125, -0.0220794677734375, 0.0165557861328125, 0.0293731689453125, -0.007228851318359375, -0.0299835205078125, 0.0010595321655273438, -0.04071044921875, -0.040130615234375, 0.021484375, 0.046875, -0.01105499267578125, -0.02996826171875, 0.03662109375, -0.012969970703125, 0.006855010986328125, -0.00940704345703125, 0.046417236328125, -0.0117034912109375, -0.01464080810546875, -0.056793212890625, 0.0183563232421875, 0.0299224853515625, 0.0300750732421875, 0.06817626953125, -0.0513916015625, 0.0254364013671875, -0.01430511474609375, 0.0205078125, -0.0276641845703125, 0.030853271484375, -0.015777587890625, 0.00022649765014648438, -0.04315185546875, -0.0182037353515625, 0.03265380859375, 0.006877899169921875, -0.027923583984375, -0.024871826171875, -0.03021240234375, 0.024200439453125, 0.035064697265625, -0.005428314208984375, 0.027252197265625, -0.036956787109375, 0.0310211181640625, 0.016082763671875, -0.00243377685546875, 0.009979248046875, 0.033294677734375, 0.0243072509765625, 0.011566162109375, 0.00519561767578125, 0.01038360595703125, -0.0239410400390625, -0.0225982666015625, -0.009246826171875, -0.0738525390625, -0.031494140625, 0.0303802490234375, -0.002811431884765625, 0.0166168212890625, -0.0298309326171875, -0.003078460693359375, 0.018707275390625, 0.03802490234375, -0.0103759765625, 0.03814697265625, -0.01491546630859375, 0.01806640625, 0.0028839111328125, 0.0244293212890625, 0.00023734569549560547, 0.027008056640625, 0.041290283203125, -0.02020263671875, 0.0281219482421875, 0.0222015380859375, -0.0213165283203125, 0.0321044921875, -0.0249176025390625, -0.024383544921875, -0.004405975341796875, -0.0160369873046875, -0.01242828369140625, -0.02899169921875, 0.0274505615234375, -0.052032470703125, -0.038787841796875, -0.055450439453125, -0.0164337158203125, -0.01947021484375, 0.01345062255859375, -0.01275634765625, 0.0191802978515625, -0.040283203125, -0.0248870849609375, 0.0531005859375, 0.006237030029296875, 0.005950927734375, -0.0304412841796875, -0.022674560546875, -0.02728271484375, -0.0589599609375, -0.043060302734375, -0.000728607177734375, -0.0163116455078125, -0.03021240234375, -0.0134429931640625, 0.00835418701171875, 0.02734375, 0.01763916015625, -1.6748905181884766e-05, -0.0244903564453125, -0.016845703125, -0.049835205078125, -0.0264892578125, 0.007198333740234375, -0.0098724365234375, 0.0184326171875, -0.01026153564453125, 0.01364898681640625, -0.002010345458984375, 0.01024627685546875, 0.01010894775390625, 0.004497528076171875, 0.00782012939453125, -0.004596710205078125, 0.038909912109375, -0.037506103515625, -0.0654296875, 0.008636474609375, 0.039093017578125, -0.0235443115234375, 0.00807952880859375, -0.042633056640625, 0.035888671875, 0.0152587890625, -0.04547119140625, -0.031707763671875, 0.02728271484375, 0.03924560546875, 0.032073974609375, 0.011688232421875, 0.036041259765625, -0.03155517578125, -0.040557861328125, 0.01142120361328125, 0.01465606689453125, 0.024749755859375, 0.03265380859375, -0.01458740234375, -0.0259857177734375, 0.014495849609375, -0.020355224609375, 0.0025310516357421875, 0.0229949951171875, 0.0020923614501953125, 0.019287109375, -0.0192718505859375, -0.00328826904296875, -0.05059814453125, -0.0325927734375, -0.0296478271484375, 0.0219268798828125, -0.00885009765625, -0.007198333740234375, 0.0035076141357421875, 0.032562255859375, 0.03558349609375, 0.001956939697265625, 0.03924560546875, 0.02606201171875, -0.002197265625, -0.040863037109375, 0.030517578125, -0.024322509765625, 0.06048583984375, 0.01323699951171875, 0.015655517578125, -0.006015777587890625, 0.01727294921875, -0.0020771026611328125, -0.007061004638671875, -0.05078125, 0.044525146484375, 0.009857177734375, -0.00916290283203125, 0.041778564453125, 0.0031585693359375, 0.0711669921875, 0.06256103515625, -0.0221710205078125, -0.05023193359375, 0.016754150390625, -0.036590576171875, 0.050933837890625, 0.0214385986328125, -0.054901123046875, -0.028411865234375, -0.019927978515625, 0.005466461181640625, -0.03131103515625, -0.0230712890625, 0.043121337890625, -0.006877899169921875, 0.0426025390625, 0.01377105712890625, -0.006786346435546875, 0.013946533203125, -0.020050048828125, -0.0288848876953125, 0.04583740234375, -0.058074951171875, -0.0167999267578125, 0.081298828125, -0.022216796875, -0.0249786376953125, -0.0012454986572265625, 0.040252685546875, -0.03704833984375, 0.01416778564453125, -0.0086822509765625, -0.01203155517578125, 0.038665771484375, -0.023590087890625, -0.007640838623046875, -0.040863037109375, 0.0548095703125, 0.0654296875, -0.04656982421875, -0.007350921630859375, 0.0037708282470703125, -0.060791015625, -0.03912353515625], index=0, object='embedding')\n", + "Embedding(embedding=[-0.039581298828125, -0.01374053955078125, 0.0179443359375, -0.0227203369140625, -0.0178680419921875, -0.027191162109375, 0.0384521484375, 0.0269927978515625, -0.0200653076171875, -0.0261077880859375, -0.0257720947265625, 0.0203857421875, -0.0118560791015625, -0.0011034011840820312, 0.0267181396484375, -0.026824951171875, -0.0008034706115722656, 0.06317138671875, -0.006183624267578125, -0.034088134765625, -0.02374267578125, 0.033721923828125, -0.0175018310546875, 0.035125732421875, 0.01140594482421875, 0.0181427001953125, -0.022186279296875, -0.044891357421875, 0.036529541015625, -0.018463134765625, -0.0199737548828125, -0.03192138671875, 0.007366180419921875, 0.007213592529296875, -0.0256805419921875, -0.02056884765625, -0.015594482421875, -0.0236663818359375, -0.040008544921875, -0.002197265625, 0.0004749298095703125, -0.07305908203125, 0.06072998046875, 0.00762176513671875, -0.0168609619140625, -0.0261383056640625, -0.003665924072265625, -0.024627685546875, -0.029052734375, 0.0201568603515625, 0.0085601806640625, -0.07696533203125, 0.0274200439453125, 0.0140533447265625, 0.028656005859375, -0.01024627685546875, 0.00978851318359375, 0.0209197998046875, -0.04742431640625, 0.0123291015625, -0.04644775390625, 0.00893402099609375, -0.052398681640625, -0.025360107421875, 0.01250457763671875, 0.03466796875, -0.0205841064453125, 0.0279541015625, 0.013580322265625, -0.014373779296875, -0.010223388671875, -0.03515625, 0.03448486328125, 0.00475311279296875, -0.046234130859375, 0.01255035400390625, 0.026153564453125, 0.0039043426513671875, -0.01922607421875, -0.026214599609375, 0.03485107421875, 0.02069091796875, -0.0017833709716796875, 0.041351318359375, -0.027984619140625, 0.035858154296875, -0.040435791015625, -0.0194244384765625, -0.0206451416015625, 0.004703521728515625, -0.019317626953125, 0.0262298583984375, -0.04022216796875, -0.04095458984375, -0.0294036865234375, -0.01009368896484375, -0.042266845703125, 0.056610107421875, -0.0445556640625, 0.01073455810546875, 0.0169219970703125, 0.0026302337646484375, -0.025360107421875, -0.0194091796875, 0.0596923828125, -0.037933349609375, 0.0245361328125, 0.0116119384765625, -0.05279541015625, 0.0020160675048828125, 0.0157623291015625, -0.011138916015625, 0.0279083251953125, -0.01151275634765625, -0.03533935546875, -0.0292816162109375, -0.01215362548828125, -0.022125244140625, 0.034637451171875, 0.0125885009765625, 0.01763916015625, -0.02532958984375, 0.021881103515625, -0.0016298294067382812, -0.0172576904296875, -0.02911376953125, 0.06353759765625, 0.0282745361328125, -0.0198974609375, 0.0256195068359375, -0.02081298828125, 0.024993896484375, -0.061553955078125, -0.00618743896484375, -0.037841796875, -0.0142669677734375, 0.051788330078125, -0.0008325576782226562, -0.0233917236328125, -0.04925537109375, 0.0022907257080078125, 0.052215576171875, -0.04425048828125, -0.030242919921875, 0.0187225341796875, -0.06427001953125, -0.0027217864990234375, 0.020477294921875, 0.0107574462890625, 0.0028839111328125, 0.00424957275390625, 0.0230712890625, 0.01190948486328125, -0.0017910003662109375, -0.00403594970703125, 0.00872802734375, 0.030181884765625, 0.04962158203125, -0.005901336669921875, 0.03228759765625, 0.051177978515625, -0.0292205810546875, 0.0128936767578125, -0.0226287841796875, -0.0010747909545898438, 0.01076507568359375, -0.0301055908203125, -0.04302978515625, -0.00855255126953125, 0.02655029296875, 0.035888671875, 0.0254058837890625, 0.025970458984375, 0.021881103515625, 0.0139617919921875, 0.005702972412109375, 0.0787353515625, 0.0098876953125, 0.0012807846069335938, -0.0306549072265625, -0.01995849609375, 0.04718017578125, 0.06475830078125, -0.02667236328125, -0.0276336669921875, 0.0379638671875, 0.048583984375, -0.03253173828125, -0.014251708984375, 0.0426025390625, -0.0039825439453125, -0.01016998291015625, 0.0090179443359375, 0.01288604736328125, 0.06793212890625, 0.021270751953125, -0.0236663818359375, -0.00795745849609375, 0.0087127685546875, 0.00621795654296875, -0.00853729248046875, 0.0025959014892578125, 0.0157928466796875, 0.007091522216796875, -0.01363372802734375, -0.007152557373046875, -0.043212890625, -0.0115966796875, -0.02313232421875, -0.085693359375, -0.045867919921875, 0.03973388671875, -0.0002282857894897461, 0.00972747802734375, 0.0265045166015625, 0.0267486572265625, -0.0164337158203125, -0.048065185546875, -0.0255889892578125, -0.053131103515625, -0.007488250732421875, 0.08990478515625, 0.013275146484375, -0.0438232421875, -0.006378173828125, 0.021026611328125, -0.048736572265625, 0.01263427734375, 0.0086212158203125, -0.014007568359375, -0.009552001953125, 0.036590576171875, -0.0635986328125, -0.03387451171875, 0.0179443359375, 0.0259857177734375, -0.025238037109375, 0.028533935546875, 0.0294036865234375, 0.031951904296875, 0.006977081298828125, 0.017913818359375, -0.007320404052734375, 0.012725830078125, 0.0206756591796875, -0.0044097900390625, 0.0731201171875, 0.054595947265625, -0.0430908203125, 0.023406982421875, -0.00463104248046875, -0.0357666015625, -0.00258636474609375, 0.046234130859375, -0.00719451904296875, 0.02630615234375, 0.0232696533203125, 0.0038509368896484375, -0.0299835205078125, 0.016571044921875, -0.01250457763671875, 0.00821685791015625, 0.04962158203125, 0.0157318115234375, -0.04107666015625, 0.03125, 0.0090484619140625, -0.0164947509765625, -0.002048492431640625, -0.0256195068359375, -0.036468505859375, 0.031158447265625, 0.03948974609375, 0.029541015625, 0.0194091796875, 0.042388916015625, -0.036956787109375, 0.0264129638671875, 0.0167083740234375, -0.004673004150390625, 0.03070068359375, -0.01319122314453125, 0.0068817138671875, -0.00899505615234375, 0.01904296875, -0.06365966796875, -0.022857666015625, 0.006259918212890625, -0.017669677734375, -0.00478363037109375, -0.02398681640625, -0.008026123046875, 0.05999755859375, -0.06671142578125, 0.04595947265625, -0.0209503173828125, -0.0167236328125, -0.1409912109375, 0.0196685791015625, -0.031097412109375, 0.01297760009765625, 0.0106201171875, -0.0227508544921875, -0.01904296875, -0.0439453125, -0.030517578125, -0.021575927734375, -0.01413726806640625, -0.0537109375, -0.05389404296875, -0.0259857177734375, -0.0298614501953125, -0.01531219482421875, 0.01340484619140625, 0.0035076141357421875, 0.002685546875, -0.0002090930938720703, 0.02001953125, -0.026123046875, 0.043975830078125, -0.07122802734375, -0.01415252685546875, 0.010650634765625, -0.00336456298828125, 0.0157928466796875, 0.01439666748046875, -0.0440673828125, 0.0210418701171875, 0.034820556640625, -0.0011806488037109375, 0.05328369140625, 0.051361083984375, 0.035919189453125, 0.01153564453125, 0.01218414306640625, -0.0028781890869140625, -0.0252532958984375, -0.006839752197265625, 0.036865234375, 0.041595458984375, -0.0143890380859375, 0.048431396484375, 0.0052490234375, -0.0198516845703125, -0.061492919921875, -0.03582763671875, -0.03497314453125, 0.046539306640625, -0.02587890625, 0.01727294921875, 0.007808685302734375, -0.05047607421875, -0.0229339599609375, -0.03912353515625, 0.0447998046875, 0.046234130859375, -0.05078125, 0.029388427734375, -0.011871337890625, 0.03851318359375, 0.0650634765625, -0.0196990966796875, 0.026123046875, 0.0606689453125, -0.001194000244140625, -0.02386474609375, 0.0200042724609375, 0.0362548828125, 0.0245513916015625, 0.0083770751953125, -0.01425933837890625, 0.01495361328125, 0.004184722900390625, -0.042144775390625, -0.022674560546875, 0.01378631591796875, -0.12359619140625, 0.0382080078125, 0.03326416015625, 0.0146026611328125, 0.024200439453125, -0.02923583984375, -0.036346435546875, 0.00605010986328125, 0.034027099609375, 0.0013828277587890625, 0.2054443359375, 0.01287841796875, 0.0299530029296875, 0.0290069580078125, -0.041839599609375, -0.051361083984375, -0.021209716796875, -0.033416748046875, 0.044647216796875, -0.00469970703125, -0.0372314453125, 0.024810791015625, 0.0606689453125, -0.00768280029296875, 0.031494140625, -0.04150390625, 0.0294952392578125, 0.0384521484375, 0.06024169921875, -0.0140533447265625, 0.037384033203125, -0.0166015625, -0.029998779296875, 0.01226043701171875, 0.0113067626953125, 0.02142333984375, 0.033172607421875, 0.0121917724609375, -0.002765655517578125, 0.0261383056640625, 0.004993438720703125, -0.0150909423828125, 0.00138092041015625, 0.00350189208984375, -0.0055389404296875, -0.0257568359375, -0.0411376953125, 0.062286376953125, 0.02496337890625, 0.01151275634765625, -0.041656494140625, -0.05670166015625, -0.021240234375, 0.048858642578125, 0.033782958984375, -0.036163330078125, 0.0135650634765625, 0.0079803466796875, -0.0216217041015625, -0.0053253173828125, 0.00432586669921875, 0.01006317138671875, -0.0221710205078125, -0.00849151611328125, -0.01117706298828125, -0.0205078125, 0.0127410888671875, 0.0032291412353515625, -0.018463134765625, -0.01129150390625, 0.023284912109375, -0.0168304443359375, -0.0279541015625, 0.0147247314453125, 0.01226043701171875, -0.0259857177734375, -0.010467529296875, -0.023223876953125, -0.0196533203125, 0.04302978515625, 0.037841796875, 0.04217529296875, 0.01561737060546875, 0.00560760498046875, 0.019073486328125, -0.020751953125, -0.047149658203125, 0.01708984375, 0.0209197998046875, 0.006526947021484375, -0.02239990234375, -0.001583099365234375, -0.08001708984375, -0.0189971923828125, -0.0037555694580078125, 0.05908203125, 0.032257080078125, -0.04461669921875, -0.01447296142578125, -0.02471923828125, -0.035888671875, -0.02154541015625, -0.033172607421875, -0.00279998779296875, 0.03961181640625, 0.0275726318359375, -0.0017042160034179688, -0.008270263671875, -0.0169219970703125, 0.0303802490234375, -0.020660400390625, -0.046112060546875, 0.023284912109375, 0.06622314453125, -0.0095062255859375, -0.020416259765625, 0.0026912689208984375, 0.0158843994140625, 0.017822265625, 0.03466796875, -0.019622802734375, -0.005420684814453125, 0.001979827880859375, 0.01535797119140625, 0.0290679931640625, -0.0260467529296875, -0.00782012939453125, -0.005954742431640625, 0.052459716796875, 0.056732177734375, -0.0083465576171875, -0.024444580078125, 0.0299072265625, -0.049224853515625, -0.0219268798828125, 0.0070343017578125, -0.0194091796875, 0.0121002197265625, -0.007389068603515625, 0.0304718017578125, 0.030914306640625, -0.00846099853515625, 0.02532958984375, -0.0022830963134765625, -0.0013828277587890625, 0.008880615234375, -0.0262298583984375, 0.01323699951171875, 0.0265960693359375, 0.04974365234375, 0.0159912109375, -0.0212860107421875, -0.0235137939453125, -0.03662109375, -0.01222991943359375, -0.06976318359375, 0.01491546630859375, 0.0136871337890625, -0.0281524658203125, -0.002887725830078125, -0.0273590087890625, 0.0765380859375, 0.01509857177734375, -0.05206298828125, -0.0262908935546875, -0.04779052734375, -0.0171966552734375, -0.034271240234375, -0.0216217041015625, 8.493661880493164e-05, 0.018341064453125, -0.0242156982421875, 0.028167724609375, 0.040985107421875, -0.0008654594421386719, 0.0379638671875, -0.0450439453125, -0.028594970703125, 0.0241546630859375, -0.029510498046875, 0.004199981689453125, -0.0270843505859375, -0.0034770965576171875, 0.020904541015625, 0.019256591796875, 0.032684326171875, 0.0245513916015625, 0.011505126953125, -0.041015625, -0.0007472038269042969, 0.040679931640625, 0.0194244384765625, 0.027557373046875, -0.00807952880859375, -0.033935546875, 0.0284271240234375, 0.0005254745483398438, -0.0257415771484375, 0.01776123046875, -0.043121337890625, -0.001190185546875, 0.041534423828125, -0.02825927734375, -0.00757598876953125, 0.004245758056640625, 0.02252197265625, -0.04754638671875, 0.04833984375, -0.02197265625, 0.0276031494140625, 0.01385498046875, 0.050872802734375, 0.045379638671875, 0.0089263916015625, -0.00272369384765625, -0.00765228271484375, -0.0167388916015625, -0.0254974365234375, 0.017425537109375, -0.00852203369140625, 0.0219268798828125, -0.020751953125, 0.01139068603515625, 0.03192138671875, 0.0308074951171875, 0.049957275390625, -0.06561279296875, -0.0067291259765625, -0.01255035400390625, -0.019866943359375, 0.03912353515625, 0.021881103515625, 0.0335693359375, -0.036041259765625, 0.01416778564453125, 0.01389312744140625, -0.0263824462890625, -0.0428466796875, 0.066650390625, 0.0054931640625, 0.00754547119140625, -0.0212554931640625, 0.014404296875, 0.03143310546875, -0.01180267333984375, -0.0169525146484375, -0.00559234619140625, 0.018829345703125, 0.047210693359375, -0.00010728836059570312, 0.01375579833984375, 0.0028934478759765625, -0.036346435546875, -0.007442474365234375, -0.0306549072265625, -0.0186004638671875, 0.019775390625, 0.0303955078125, 0.01134490966796875, -0.0292205810546875, -0.0229339599609375, -0.0321044921875, 0.01467132568359375, -0.010406494140625, -0.043853759765625, 0.00017905235290527344, -0.037322998046875, -0.009185791015625, 9.775161743164062e-06, 0.00351715087890625, -0.0225830078125, -0.045379638671875, -0.0261383056640625, 0.001758575439453125, -0.0179901123046875, -0.054931640625, 0.03778076171875, 0.03277587890625, -0.03466796875, -0.050994873046875, -0.0036869049072265625, 0.0025959014892578125, -0.01806640625, -0.01097869873046875, 0.0706787109375, 0.036529541015625, 0.02001953125, -0.058013916015625, 0.01085662841796875, -0.01519012451171875, 0.01297760009765625, 0.025665283203125, 5.543231964111328e-06, 0.024078369140625, 0.004764556884765625, 0.025299072265625, 0.037078857421875, -0.01824951171875, 0.019927978515625, -0.0206298828125, 0.050445556640625, -0.037567138671875, -0.006511688232421875, -0.01444244384765625, -0.03173828125, 0.01922607421875, -0.002063751220703125, 0.060760498046875, -0.03564453125, -0.021087646484375, 0.0008492469787597656, 0.030914306640625, -0.033966064453125, 0.030548095703125, -0.0127410888671875, -0.031402587890625, -0.006969451904296875, 0.01953125, -0.022003173828125, 0.00537872314453125, 0.00574493408203125, -0.041168212890625, 0.0027294158935546875, 0.05450439453125, 0.036376953125, -0.044830322265625, 0.008331298828125, 0.0241241455078125, -0.0147705078125, 0.01959228515625, 0.01558685302734375, -0.01580810546875, 0.047210693359375, -0.0533447265625, 0.0035572052001953125, 0.0157318115234375, 0.007472991943359375, -0.046905517578125, -0.00518035888671875, -0.0118255615234375, 0.0218048095703125, 0.06829833984375, -0.006793975830078125, 0.01465606689453125, -0.034912109375, 0.0227508544921875, -0.006072998046875, 0.0462646484375, 0.023712158203125, 0.0017118453979492188, 0.004581451416015625, -0.019256591796875, 0.0034427642822265625, -0.01236724853515625, -0.0261993408203125, -0.019775390625, 0.01088714599609375, -0.006595611572265625, 0.022918701171875, 0.0015077590942382812, -0.0308380126953125, 0.059112548828125, -0.004047393798828125, -0.039764404296875, 0.01380157470703125, -0.00246429443359375, -0.05291748046875, -0.0283203125, 0.0022869110107421875, -0.03131103515625, 0.0127410888671875, 0.01284027099609375, 0.05828857421875, 0.053009033203125, 0.029632568359375, 0.017730712890625, 0.046417236328125, 0.0299224853515625, 0.04840087890625, -0.00455474853515625, 0.0026149749755859375, -0.0291595458984375, 0.0014657974243164062, 0.038482666015625, 0.049224853515625, 0.03778076171875, -0.047149658203125, -0.0335693359375, -0.03106689453125, 0.0163726806640625, 0.033782958984375, -0.041107177734375, -0.05963134765625, 0.008514404296875, 0.0251312255859375, -0.01020050048828125, -0.00437164306640625, -0.030517578125, 0.029693603515625, -0.053680419921875, 0.0033092498779296875, -0.0247344970703125, -0.07232666015625, 0.017608642578125, -0.099853515625, 0.0020236968994140625, -0.0173797607421875, -0.020721435546875, -0.0523681640625, 0.020751953125, -0.0286102294921875, 0.0027599334716796875, 0.01103973388671875, -0.05108642578125, -0.0216064453125, -0.004108428955078125, -0.00743865966796875, -0.017730712890625, 0.0009741783142089844, 0.002193450927734375, -0.00402069091796875, 0.0218353271484375, -0.006927490234375, 0.04852294921875, -0.01172637939453125, -0.0164642333984375, 0.05780029296875, -0.0391845703125, 0.005687713623046875, 0.03582763671875, 0.008453369140625, -0.044952392578125, -0.03375244140625, -0.028411865234375, 0.006099700927734375, 0.013916015625, 0.01384735107421875, 0.055084228515625, -0.034210205078125, -0.01187896728515625, 0.0062103271484375, -0.01580810546875, -0.0140533447265625, 0.0052947998046875, 0.0243377685546875, 0.01143646240234375, 0.024627685546875, -0.060546875, 0.01091766357421875, 0.0152435302734375, -0.0213775634765625, 0.0205535888671875, -0.01959228515625, -0.0018701553344726562, 0.0300445556640625, 0.021331787109375, 0.04278564453125, -0.04718017578125, 0.03021240234375, 0.042755126953125, -0.0277862548828125, 0.00460052490234375, 0.011383056640625, 0.0157470703125, -0.04754638671875, -0.026275634765625, -0.05194091796875, -0.0792236328125, -0.01702880859375, -0.040557861328125, -0.1256103515625, 0.048675537109375, 0.0010385513305664062, -0.0179443359375, -0.03802490234375, -0.044677734375, -0.006198883056640625, -0.01385498046875, 0.042572021484375, -0.01020050048828125, 0.043548583984375, 0.000621795654296875, -0.0152587890625, -0.0263214111328125, 0.0135040283203125, 0.056182861328125, 0.00284576416015625, -0.008819580078125, -0.006244659423828125, 0.01190948486328125, -0.022918701171875, -0.00728607177734375, -0.031707763671875, -0.028411865234375, -0.0748291015625, 0.00753021240234375, -0.0225067138671875, -0.0152130126953125, 0.04248046875, -0.032440185546875, -0.0150146484375, 0.0062713623046875, -0.0260162353515625, 0.0214385986328125, 0.0022735595703125, 0.00795745849609375, -0.00832366943359375, -0.0038623809814453125, -0.03021240234375, -0.03814697265625, 0.0423583984375, -0.004154205322265625, -0.01367950439453125, -0.00638580322265625, 0.02490234375, -0.0033969879150390625, -0.0188446044921875, -0.0396728515625, -0.01195526123046875, -0.0163116455078125, -0.026031494140625, -0.0247344970703125, -0.049407958984375, 0.013671875, -0.00836181640625, -0.04168701171875, 0.03216552734375, -0.0006427764892578125, -0.0284576416015625, -0.034576416015625, -0.046142578125, 0.004764556884765625, 0.0023193359375, -0.026824951171875, 0.025543212890625, -0.013671875, 0.01192474365234375, 0.0016012191772460938, 0.0159454345703125, -0.003604888916015625, 0.023651123046875, 0.04815673828125, 0.0249786376953125, -0.03607177734375, 0.01546478271484375, 0.01922607421875, -0.00908660888671875, 0.03436279296875, 0.0164794921875, -0.0014562606811523438, 0.035369873046875, -0.02825927734375, -0.0242156982421875, -0.040985107421875, 0.0106353759765625, 0.00836181640625, -0.0158538818359375, 0.0413818359375, -0.0506591796875, 0.0178375244140625, 0.03887939453125, -0.0182037353515625, 0.0050506591796875, 0.01535797119140625, -0.004703521728515625, -0.0239410400390625, 0.008880615234375, -0.07562255859375, -0.00936126708984375, 0.038360595703125, 0.0215606689453125, 0.016357421875, -0.01239013671875, 0.0001932382583618164, -0.0482177734375, 0.031707763671875, -0.019500732421875, 0.0267181396484375, 0.00580596923828125, -0.042510986328125, 0.00848388671875, 0.0031147003173828125, 0.0706787109375, 0.0158538818359375, 0.01323699951171875, 0.05615234375, 0.032196044921875, -0.03070068359375, 0.036773681640625, 0.06500244140625, 0.03692626953125, -0.018798828125, 0.0130767822265625, -0.017730712890625, -0.007598876953125, -0.01898193359375, 0.0301361083984375, -0.037506103515625, 0.046722412109375, -0.0171356201171875, 0.0067596435546875, -0.007022857666015625, -0.0182037353515625, 0.0239105224609375, -0.021026611328125, -0.0231170654296875, 0.01483917236328125, -0.0284423828125, -0.0335693359375, -0.0241241455078125, -0.0240020751953125, -0.036590576171875, -0.0118408203125, 0.0379638671875, -0.0010442733764648438, 0.00226593017578125, 0.01200103759765625, -0.01413726806640625, -0.037017822265625, -0.007770538330078125, 0.030242919921875, 0.0072784423828125, -0.0125274658203125, 0.01207733154296875, 0.031402587890625, -0.00926971435546875, 0.0093536376953125, -0.051055908203125, 0.059661865234375, -0.046630859375, -0.00963592529296875, -0.01708984375, 0.0237274169921875, -0.006511688232421875, -0.061798095703125, -0.00345611572265625, -0.0506591796875, 0.0298309326171875, -0.005519866943359375, 0.029815673828125, -0.0166778564453125, 0.049835205078125, -0.0014410018920898438, -0.027374267578125, 0.024200439453125, 0.021209716796875, 0.032867431640625, 0.01334381103515625], index=1, object='embedding')\n", + "Embedding(embedding=[-0.029296875, -0.026092529296875, -0.0214996337890625, -0.01512908935546875, 0.007564544677734375, 0.0016078948974609375, -0.0179595947265625, -0.040557861328125, -0.020416259765625, -0.01435089111328125, -0.00641632080078125, 0.002452850341796875, -0.023162841796875, 0.01049041748046875, -0.0030345916748046875, -0.0577392578125, 0.049530029296875, 0.01227569580078125, -0.01055145263671875, -0.033935546875, -0.029296875, -0.0211639404296875, -0.02734375, -0.0225677490234375, 0.0191192626953125, 0.004344940185546875, 0.0025043487548828125, 0.0426025390625, 0.001995086669921875, -0.044677734375, 0.016693115234375, 0.028656005859375, 0.004764556884765625, 0.0167236328125, 0.01459503173828125, -0.0269012451171875, 0.0022487640380859375, -0.01053619384765625, -0.03802490234375, -0.0221099853515625, -0.01300811767578125, -0.005550384521484375, 0.01323699951171875, -0.004974365234375, -0.0245819091796875, -0.0284576416015625, -0.0244903564453125, -0.017852783203125, -0.0128173828125, 0.01605224609375, -0.013336181640625, -0.0038089752197265625, 0.031463623046875, 0.00284576416015625, 0.01361846923828125, 0.06365966796875, 0.008819580078125, -0.035675048828125, -0.051116943359375, -0.0196075439453125, -0.01526641845703125, 0.00872039794921875, -0.006618499755859375, 0.032745361328125, 0.046966552734375, 0.08685302734375, 0.0068359375, -0.01502227783203125, -0.028472900390625, -0.00521087646484375, -0.0178985595703125, -0.028839111328125, 0.00902557373046875, 0.01183319091796875, -0.0001958608627319336, 0.03192138671875, -0.0157470703125, -0.0185394287109375, -0.043487548828125, -0.01261138916015625, 0.0880126953125, 0.0286712646484375, -0.0416259765625, 0.00466156005859375, 0.0236358642578125, 0.05615234375, -0.00348663330078125, 0.06231689453125, 0.0114898681640625, 0.008880615234375, -0.052734375, -0.0352783203125, 0.00795745849609375, -0.046417236328125, -0.063720703125, 0.01058197021484375, -0.04052734375, 0.01092529296875, 0.01593017578125, 0.0040130615234375, 0.00473785400390625, 0.0011882781982421875, -0.016754150390625, -0.007793426513671875, 0.0127410888671875, -0.0176849365234375, 0.00913238525390625, 0.0298309326171875, -0.0202484130859375, 0.013824462890625, 0.0372314453125, 0.0231475830078125, 0.022857666015625, 0.0316162109375, -0.0256195068359375, -0.0253143310546875, -0.02301025390625, -0.00901031494140625, -0.0041046142578125, 0.013031005859375, -0.037445068359375, -0.0268707275390625, 0.033233642578125, 0.0004925727844238281, -0.015655517578125, -0.035614013671875, 0.0281524658203125, 0.0277252197265625, 0.002841949462890625, -0.01178741455078125, 0.035552978515625, -0.0014390945434570312, -0.011016845703125, 0.0102996826171875, -0.0316162109375, -0.02001953125, 0.050262451171875, -0.043212890625, -0.033294677734375, -0.032470703125, -0.0277099609375, -0.01812744140625, 0.03515625, -0.001911163330078125, -0.0032672882080078125, -0.039215087890625, -0.0284271240234375, -0.061798095703125, 0.0419921875, -0.035797119140625, -0.05340576171875, 0.0015544891357421875, 0.038330078125, 0.0021514892578125, -0.033477783203125, 0.027374267578125, -0.020843505859375, 0.046600341796875, -0.05126953125, -0.025238037109375, 0.022979736328125, -0.019317626953125, -0.04193115234375, 0.03875732421875, 0.0038928985595703125, 0.0018482208251953125, 0.0007176399230957031, -0.0169219970703125, -0.007686614990234375, -0.03900146484375, 0.006458282470703125, -0.034912109375, -0.00485992431640625, -0.046173095703125, 0.0230865478515625, 0.0301971435546875, 0.056396484375, 0.00018596649169921875, -0.01157379150390625, -0.03466796875, 0.0024318695068359375, 0.045135498046875, 0.019561767578125, -0.0181121826171875, 0.016326904296875, 0.028289794921875, 0.032745361328125, -0.00469207763671875, 0.0005941390991210938, 0.0484619140625, -0.0643310546875, -0.0224456787109375, 0.0019063949584960938, -0.00982666015625, 0.01554107666015625, 0.0037860870361328125, 0.049346923828125, 0.00494384765625, -0.01137542724609375, -0.002590179443359375, -0.06939697265625, 0.039642333984375, -0.00348663330078125, 0.036895751953125, -0.0281982421875, -0.0149383544921875, -0.01788330078125, 0.003265380859375, 0.015594482421875, -0.0609130859375, 0.00933074951171875, 0.0126953125, 0.043914794921875, 0.01142120361328125, -0.01218414306640625, 0.01139068603515625, 0.01171112060546875, -0.0299835205078125, -0.05096435546875, 0.007266998291015625, -0.054443359375, 0.0171051025390625, 0.007213592529296875, -0.00014889240264892578, 0.031463623046875, -0.0094757080078125, 0.010406494140625, -0.0037784576416015625, -0.0118560791015625, -0.0173492431640625, -0.0197601318359375, 0.03204345703125, -0.0465087890625, -0.026123046875, -0.0298614501953125, 0.004657745361328125, -0.052581787109375, 0.016693115234375, 0.0163116455078125, 0.044281005859375, -0.01377105712890625, 0.016693115234375, -0.05755615234375, 0.00830078125, -0.0063934326171875, -0.0099334716796875, 0.021148681640625, -0.005245208740234375, 0.0017223358154296875, -0.004199981689453125, -0.00732421875, 0.0289154052734375, -0.00530242919921875, 0.04534912109375, 0.0243072509765625, -0.0048828125, 0.0203704833984375, -0.0194549560546875, 0.021697998046875, 0.0180816650390625, 0.00859832763671875, -0.0467529296875, 0.0052642822265625, 0.03106689453125, 0.044281005859375, 0.036590576171875, -0.0380859375, 0.0036830902099609375, 0.024261474609375, 0.036834716796875, -0.0215301513671875, -0.001773834228515625, 0.00846099853515625, 0.021881103515625, 0.0179290771484375, 0.0263671875, -0.007251739501953125, -0.01544952392578125, -0.0166015625, 0.024261474609375, 0.06536865234375, -0.026519775390625, 0.03302001953125, 0.049041748046875, 0.005462646484375, -0.048553466796875, 0.008544921875, -0.005115509033203125, -0.030059814453125, 0.027618408203125, 0.01861572265625, -0.0362548828125, 0.039306640625, 0.005550384521484375, -0.0249786376953125, 0.0159454345703125, -0.004749298095703125, -0.1578369140625, 0.020172119140625, 0.019134521484375, 0.004459381103515625, -0.002391815185546875, -0.006683349609375, -0.041015625, 0.00826263427734375, -0.0167236328125, -0.03082275390625, -0.0064239501953125, -0.026275634765625, -0.06494140625, -0.01242828369140625, 0.032073974609375, 0.01184844970703125, -0.009765625, -0.051788330078125, -0.0190887451171875, -0.053955078125, 0.03704833984375, 0.00861358642578125, -0.00283050537109375, -0.028350830078125, 0.00445556640625, 0.04296875, 0.0286102294921875, -0.025238037109375, -0.04058837890625, -0.0302886962890625, 0.01953125, 0.0211639404296875, 0.0036220550537109375, 0.040496826171875, 0.04949951171875, 0.005428314208984375, 0.022003173828125, -0.0204010009765625, -0.00630950927734375, 0.0294342041015625, 0.0029621124267578125, 0.058380126953125, -0.004772186279296875, 0.0162353515625, 0.024810791015625, 0.01090240478515625, -0.0301055908203125, -0.0277252197265625, -0.037139892578125, -0.01294708251953125, 0.005828857421875, 0.0186309814453125, 0.0311737060546875, -0.025421142578125, -0.0130462646484375, 0.0159454345703125, 0.00293731689453125, 0.024444580078125, 0.034332275390625, -0.029327392578125, 0.04681396484375, 0.0037326812744140625, -0.0208892822265625, 0.01108551025390625, 0.034881591796875, 0.0308990478515625, 0.0229034423828125, -0.050628662109375, 0.001251220703125, -0.0270233154296875, 0.0611572265625, 0.06109619140625, -0.024169921875, 0.005931854248046875, 0.0198822021484375, 0.0228424072265625, 0.06878662109375, -0.06304931640625, 0.010589599609375, -0.1141357421875, 0.01509857177734375, 0.0147705078125, -0.00250244140625, -0.009765625, -0.05010986328125, -0.0292205810546875, 0.0184478759765625, 0.0274658203125, 0.050689697265625, 0.225830078125, -0.0175933837890625, 0.0745849609375, -0.101318359375, -0.0204315185546875, -0.03900146484375, -0.047760009765625, -0.056915283203125, 0.0274200439453125, -0.02630615234375, 0.00296783447265625, -0.0088043212890625, 0.034576416015625, -0.0059356689453125, 0.004352569580078125, 0.08673095703125, 0.00640869140625, 0.0271759033203125, 0.05615234375, -0.02056884765625, 0.002201080322265625, -0.023651123046875, -0.0143890380859375, -0.0181427001953125, 0.0034770965576171875, 0.00994110107421875, 0.004642486572265625, 0.0111236572265625, 0.010040283203125, 0.0110015869140625, -0.0305023193359375, -0.02783203125, 0.022216796875, -0.03521728515625, -0.039947509765625, -0.0272216796875, -0.0406494140625, -0.021331787109375, -0.0150909423828125, -0.01514434814453125, -0.0380859375, -0.0521240234375, 0.057220458984375, 0.03253173828125, 0.075927734375, -0.0202789306640625, 0.013427734375, -0.0027141571044921875, -0.05694580078125, 0.01139068603515625, 0.004974365234375, 0.01277923583984375, 0.0035552978515625, 0.0144195556640625, -0.0217437744140625, 0.0306854248046875, 0.005252838134765625, -0.025787353515625, 0.006412506103515625, -0.035308837890625, 0.006343841552734375, -0.0009183883666992188, 0.0255126953125, -0.044403076171875, 0.01220703125, 0.01482391357421875, 0.07440185546875, -0.022979736328125, -0.019927978515625, 0.01154327392578125, 0.0249481201171875, -0.0200347900390625, 0.062469482421875, 0.01213836669921875, 0.002101898193359375, 0.0307159423828125, -0.0168914794921875, -0.0035190582275390625, 0.07879638671875, 0.02398681640625, 0.02276611328125, -0.0120086669921875, -0.0428466796875, -0.0085906982421875, 0.024749755859375, -0.007266998291015625, 0.026031494140625, 0.0614013671875, 0.0400390625, -0.055389404296875, 0.0105438232421875, 0.02618408203125, -0.0106048583984375, 0.0217742919921875, -0.019561767578125, -0.07159423828125, -0.004241943359375, -0.0020618438720703125, -0.01953125, 0.0301971435546875, -0.036041259765625, -0.01490020751953125, -0.0170440673828125, 0.0479736328125, 0.01849365234375, -0.027740478515625, -0.00981903076171875, 0.0306854248046875, -0.01141357421875, -0.0220489501953125, -0.04266357421875, -0.0278778076171875, 0.01226806640625, 0.0058441162109375, 0.0159759521484375, 0.0263519287109375, -0.024688720703125, 0.021820068359375, -0.009368896484375, 0.07318115234375, 0.00022912025451660156, -0.030853271484375, 0.00600433349609375, 0.006938934326171875, -0.013580322265625, -0.030548095703125, -0.0016736984252929688, 0.0360107421875, 0.0343017578125, 0.01544952392578125, 0.028045654296875, -0.0161895751953125, 0.057098388671875, 0.0182952880859375, 0.0007042884826660156, 0.0228118896484375, -0.01824951171875, 0.0204620361328125, -0.031585693359375, -0.0204620361328125, 0.01096343994140625, 0.0302734375, 0.01006317138671875, 0.004215240478515625, -0.0194854736328125, 0.050994873046875, -0.034576416015625, -0.003017425537109375, -0.04248046875, -0.0181732177734375, -0.0271148681640625, 0.061065673828125, -0.052032470703125, -0.05743408203125, 0.00603485107421875, -0.00223541259765625, 0.0301971435546875, -0.10137939453125, 0.01837158203125, -0.0074920654296875, 0.027435302734375, 0.00027561187744140625, 0.0253448486328125, 0.09320068359375, 0.02734375, 0.0238494873046875, -0.040283203125, 0.00681304931640625, 0.054931640625, -0.01026153564453125, -0.01471710205078125, -0.01126861572265625, -0.050750732421875, 0.017364501953125, 0.02972412109375, -0.048553466796875, -0.049652099609375, -0.00634765625, 0.02813720703125, -0.0050048828125, 0.027435302734375, 0.038360595703125, 0.037322998046875, 0.027374267578125, -0.0467529296875, 0.0269012451171875, 0.042327880859375, -0.01036834716796875, 0.0347900390625, -0.04058837890625, -0.0033893585205078125, 0.04974365234375, 0.024749755859375, 0.00916290283203125, 0.03057861328125, -0.031829833984375, 0.03887939453125, 0.01403045654296875, 0.00027251243591308594, -2.6404857635498047e-05, 0.00785064697265625, 0.0249176025390625, 0.0006728172302246094, -0.01285552978515625, -0.005035400390625, -0.0196990966796875, -0.048248291015625, -0.048828125, -0.0013532638549804688, 0.06976318359375, -0.03131103515625, 0.0150604248046875, 0.041473388671875, -0.013031005859375, -0.0019435882568359375, 0.0084075927734375, 0.0225067138671875, -0.0172882080078125, -0.01204681396484375, -0.05352783203125, -0.007587432861328125, -0.0027408599853515625, 0.034881591796875, -0.02667236328125, 0.0472412109375, 0.0291748046875, 0.03375244140625, -0.01505279541015625, 0.0257110595703125, 0.038177490234375, 0.0010223388671875, 0.0604248046875, -0.006977081298828125, -0.04486083984375, -0.0205230712890625, 0.037109375, -0.060272216796875, -0.0129547119140625, 0.04888916015625, -0.0030422210693359375, -0.007305145263671875, 0.0008087158203125, 0.043701171875, 0.0218963623046875, 0.00946807861328125, -0.005954742431640625, 0.045806884765625, 0.00572967529296875, 0.01479339599609375, 0.03851318359375, -0.0036182403564453125, -0.01267242431640625, 0.03887939453125, -0.0056304931640625, -0.01403045654296875, 0.00353240966796875, -0.0193328857421875, 0.042022705078125, -0.02459716796875, 0.0008311271667480469, 0.0155029296875, 0.041961669921875, -0.0233001708984375, 0.0026035308837890625, 0.0372314453125, -0.0166015625, 0.005031585693359375, 0.049835205078125, -0.00771331787109375, -0.07696533203125, 0.019012451171875, -0.01161956787109375, 0.0010442733764648438, 0.018768310546875, 0.04193115234375, -0.0016469955444335938, -0.033721923828125, 0.030426025390625, -0.0241851806640625, -0.00650787353515625, 0.0028057098388671875, 0.006633758544921875, -0.0186004638671875, 0.038909912109375, 0.0029449462890625, -0.0310211181640625, 0.039703369140625, 0.019287109375, 0.041717529296875, -0.001392364501953125, -0.005634307861328125, -0.01026153564453125, -0.028228759765625, 0.007427215576171875, -0.006195068359375, 0.006633758544921875, -0.039794921875, 0.0129852294921875, -0.022613525390625, -0.005084991455078125, -0.04705810546875, 0.055938720703125, -0.043487548828125, -0.0211639404296875, 0.022613525390625, 0.0030689239501953125, 0.025482177734375, -0.00649261474609375, -0.039459228515625, -0.0160980224609375, 0.0176849365234375, -0.05963134765625, 0.032745361328125, 0.01287078857421875, -0.0189361572265625, -0.0167236328125, 0.044647216796875, -0.041046142578125, -0.037994384765625, 0.01739501953125, 0.00878143310546875, 0.021636962890625, 0.03460693359375, -0.0268707275390625, 0.016571044921875, -0.007465362548828125, 0.007045745849609375, 0.021392822265625, -0.05267333984375, -0.08331298828125, 0.009368896484375, 0.06890869140625, -0.0018625259399414062, -0.006938934326171875, -0.02484130859375, -0.01032257080078125, 0.019287109375, -0.03363037109375, 0.00238037109375, 0.0204010009765625, 0.032073974609375, 0.00833892822265625, -0.0218048095703125, -0.043304443359375, 0.0003895759582519531, -0.01529693603515625, -0.017547607421875, -0.0208282470703125, -0.005863189697265625, 0.007366180419921875, 0.013031005859375, 0.0404052734375, -0.041229248046875, -0.0201263427734375, 0.0120697021484375, 0.062225341796875, -0.023773193359375, -0.0011854171752929688, -0.016815185546875, 0.0020599365234375, -0.0007572174072265625, -0.0163421630859375, 0.0225372314453125, 0.059661865234375, 0.0360107421875, 0.0249176025390625, -0.0140380859375, -0.0244598388671875, -0.048187255859375, -0.01132965087890625, 0.0081024169921875, 0.049957275390625, 0.07550048828125, -0.007015228271484375, -0.0220489501953125, 0.0296478271484375, 0.00252532958984375, -0.019134521484375, 0.004482269287109375, -0.0023956298828125, 0.0106201171875, -0.04315185546875, -0.0428466796875, -0.0174407958984375, 0.0003209114074707031, 0.01154327392578125, 0.013580322265625, -0.01708984375, 0.004039764404296875, -0.0225067138671875, -0.01641845703125, -0.0192718505859375, -0.043548583984375, 0.00809478759765625, -0.11572265625, 0.0019445419311523438, -0.004344940185546875, 0.0065460205078125, -0.055816650390625, 0.0191192626953125, -0.0025539398193359375, 0.02716064453125, -0.02093505859375, 0.0044097900390625, -0.043426513671875, 0.0413818359375, -0.033477783203125, -0.0211334228515625, -0.002277374267578125, 0.0025272369384765625, 0.0004394054412841797, -0.0294342041015625, -0.0291900634765625, 0.028900146484375, 0.007236480712890625, -0.053314208984375, 0.024444580078125, -0.0146331787109375, 0.035400390625, -0.021636962890625, 0.0243377685546875, -0.01506805419921875, -0.0174560546875, -0.05828857421875, 0.010345458984375, -0.06182861328125, 0.047088623046875, 0.0218963623046875, -0.005321502685546875, 0.02337646484375, -0.016845703125, -0.0013780593872070312, -0.0084075927734375, 0.020843505859375, 0.00890350341796875, -0.016693115234375, -0.033172607421875, -0.01080322265625, -0.017669677734375, 0.0595703125, -0.037628173828125, -0.0194854736328125, 0.0032672882080078125, 0.01012420654296875, -0.0189208984375, 0.0164642333984375, 0.0215606689453125, -0.05352783203125, -0.0044097900390625, 0.0090484619140625, -0.00812530517578125, 0.0274810791015625, 0.0198822021484375, 0.04949951171875, 0.0129852294921875, -0.0038547515869140625, -0.01226043701171875, -0.026458740234375, -0.0501708984375, 0.03515625, -0.048126220703125, 0.006282806396484375, 0.05853271484375, -0.0071563720703125, -0.035614013671875, -0.01387786865234375, 0.02008056640625, -0.01323699951171875, 0.025177001953125, 0.050140380859375, 0.04071044921875, -0.01210784912109375, -0.00018727779388427734, -0.043731689453125, 0.0311737060546875, 0.01641845703125, -0.0178680419921875, -0.014434814453125, -0.0276641845703125, 0.038726806640625, 0.0117950439453125, -0.058349609375, -0.02679443359375, -0.01080322265625, -0.03936767578125, -0.01080322265625, -0.033935546875, -0.018218994140625, -0.01470947265625, 0.01178741455078125, -0.0255584716796875, -0.007297515869140625, -0.00598907470703125, -0.0246124267578125, 0.0004949569702148438, 0.00018346309661865234, 0.01480865478515625, 0.00141143798828125, -0.04058837890625, 0.04669189453125, 0.0188140869140625, 0.0030155181884765625, 0.02899169921875, -0.0259857177734375, 0.0036983489990234375, 0.0010004043579101562, -0.04437255859375, -0.0084686279296875, -0.047698974609375, 0.0229034423828125, -0.0129547119140625, -0.00673675537109375, 0.04150390625, 0.038604736328125, -0.0408935546875, 0.044097900390625, -0.0067901611328125, -0.0189666748046875, -0.034881591796875, -0.006664276123046875, -0.005950927734375, 0.0172119140625, 0.053680419921875, -0.046295166015625, 0.020172119140625, -0.00431060791015625, 0.0234527587890625, 0.01654052734375, 0.0142364501953125, -0.039794921875, -0.0213623046875, 0.007625579833984375, -0.0294342041015625, -0.08148193359375, 0.005565643310546875, 0.020111083984375, -0.0400390625, -0.033935546875, -0.00399017333984375, 0.03118896484375, 0.026458740234375, -0.04010009765625, -0.06671142578125, 0.016845703125, 0.046173095703125, 0.01120758056640625, 0.0208282470703125, -0.0190887451171875, -0.042938232421875, -0.0343017578125, -0.0302734375, 0.020538330078125, 0.0178070068359375, 0.0289764404296875, -0.034637451171875, -0.044708251953125, -0.0225830078125, -0.0628662109375, 0.0164031982421875, 0.0135498046875, -0.00139617919921875, -0.0128631591796875, -0.03302001953125, -0.013092041015625, -0.01418304443359375, 0.037078857421875, -0.041961669921875, -0.035888671875, 0.016937255859375, -0.0552978515625, -0.002582550048828125, 0.049041748046875, 0.02606201171875, 0.0073394775390625, 0.035369873046875, 0.02716064453125, 0.040985107421875, -0.05816650390625, 0.010040283203125, -0.0206451416015625, 0.0211029052734375, -0.005828857421875, -0.0005421638488769531, -0.0001691579818725586, -0.02325439453125, -0.021881103515625, -0.00026679039001464844, -0.0234222412109375, -0.0033779144287109375, 0.047821044921875, 0.00389862060546875, 0.0114288330078125, -0.019012451171875, 0.0027008056640625, 0.03521728515625, -0.0279693603515625, 0.0249481201171875, -0.021697998046875, -0.0125274658203125, 0.072021484375, -0.004680633544921875, 0.001186370849609375, -0.0019550323486328125, -0.0258636474609375, -0.0136871337890625, 0.0224151611328125, -0.04693603515625, 0.044830322265625, 0.0014581680297851562, 0.0116729736328125, -0.0189056396484375, 0.004512786865234375, -0.01529693603515625, 0.02313232421875, -0.060516357421875, 0.0189208984375, -0.050537109375, -0.0027866363525390625, -0.00395965576171875, 0.01230621337890625, -0.01837158203125, 0.04034423828125, -0.0233001708984375, 0.01641845703125, 0.0250091552734375, -0.019378662109375, -0.042144775390625, 0.01285552978515625, -0.0046539306640625, 0.004344940185546875, 0.028656005859375, 0.01094818115234375, 0.060577392578125, -0.046844482421875, -0.006317138671875, -0.028717041015625, -0.0302276611328125, -0.01261138916015625], index=2, object='embedding')\n" + ] + } + ], + "source": [ + "_embed_client = openai.OpenAI(\n", + " api_key=os.getenv(\"EMBEDDING_API_KEY\"),\n", + " base_url=os.getenv(\"EMBEDDING_BASE_URL\"),\n", + " max_retries=5)\n", + "user_query=\"what is toyota?\" \n", + "faq_list = [\"Where are Toyota cars manufactured?\", \"What is the engine power of Toyota RAV4\", \"Where is Japan?\"] \n", + "#embed user query\n", + "user_query_embedding = _embed_client.embeddings.create(input=user_query, model=os.getenv('EMBEDDING_MODEL_NAME'))\n", + "user_query_embedding = np.array(user_query_embedding.data[0].embedding)\n", + "user_query_embedding = user_query_embedding.reshape(1, -1)\n", + "\n", + "# faq_embedding_list = []\n", + "cosi_list = []\n", + "faq_embedding_list = _embed_client.embeddings.create(input=faq_list, model=os.getenv('EMBEDDING_MODEL_NAME'))\n", + "for i, faq_embedding in enumerate(faq_embedding_list.data):\n", + " print(faq_embedding)\n", + " faq_embedding = np.array(faq_embedding.embedding)\n", + " faq_embedding = faq_embedding.reshape(1,-1)\n", + " similarity_score = cosine_similarity(user_query_embedding, faq_embedding)[0][0]\n", + " cosi_list.append({\"faq\":faq_list[i], \"sim\":similarity_score})\n", + "\n", + "sorted_faqs = sorted(cosi_list, key=lambda d: d[\"sim\"], reverse=True)\n", + "sorted_faqs_list = [i[\"faq\"] for i in sorted_faqs]\n", + " " + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "11327bfc", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "['Where are Toyota cars manufactured?',\n", + " 'Where is Japan?',\n", + " 'What is the engine power of Toyota RAV4']" + ] + }, + "execution_count": 21, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "\n", + "sorted_faqs_list" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "id": "be8f3071", + "metadata": {}, + "outputs": [], + "source": [ + "from pydantic import BaseModel" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "id": "20f16f80", + "metadata": {}, + "outputs": [], + "source": [ + "faq_list = [\"Where are Toyota cars manufactured?\", \"What is the engine power of Toyota RAV4\", \"Where is Japan?\"] \n", + "\n", + "class FAQ(BaseModel):\n", + " user_query: str\n", + " similar_faqs: list[str]\n", + " def __str__(self) -> str:\n", + " \"\"\"Return a string representation of the faq list\"\"\"\n", + " return \"\\n\".join(\n", + " f\"faq {step}\\n\"\n", + " for step in self.similar_faqs\n", + " )\n" + ] + }, + { + "cell_type": "code", + "execution_count": 22, + "id": "8bc863d3", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "'Where are Toyota cars manufactured?\\n\\nWhat is the engine power of Toyota RAV4\\n\\nWhere is Japan?\\n'" + ] + }, + "execution_count": 22, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "\"\\n\".join(f\"{i}\\n\"for i in faq_list)" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "id": "5630f03e", + "metadata": {}, + "outputs": [], + "source": [ + "FAQ.similar_faqs=faq_list" + ] + }, + { + "cell_type": "code", + "execution_count": 20, + "id": "97c538fb", + "metadata": {}, + "outputs": [], + "source": [ + "# print(FAQ.similar_faqs)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "cac271b9", + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "agent-bootcamp-202507", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.12.3" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/src/3_evals/1_llm_judge/README.md b/src/3_evals/1_llm_judge/README.md index a693413..f00e65a 100644 --- a/src/3_evals/1_llm_judge/README.md +++ b/src/3_evals/1_llm_judge/README.md @@ -35,7 +35,7 @@ Example data: ```bash uv run --env-file .env \ --m src.3_evals.1_llm_judge.run_eval \ +-m src.3_evals.1_llm_judge.run_eval_ioana \ --langfuse_dataset_name search-dataset \ --run_name enwiki_weaviate ``` diff --git a/src/3_evals/1_llm_judge/run_eval.py b/src/3_evals/1_llm_judge/run_eval.py index 1eebf85..470a7ce 100644 --- a/src/3_evals/1_llm_judge/run_eval.py +++ b/src/3_evals/1_llm_judge/run_eval.py @@ -53,7 +53,6 @@ """ - class LangFuseTracedResponse(pydantic.BaseModel): """Agent Response and LangFuse Trace info.""" diff --git a/src/3_evals/1_llm_judge/run_eval_ioana.py b/src/3_evals/1_llm_judge/run_eval_ioana.py index 4514a38..f839e69 100644 --- a/src/3_evals/1_llm_judge/run_eval_ioana.py +++ b/src/3_evals/1_llm_judge/run_eval_ioana.py @@ -25,7 +25,7 @@ set_up_logging() -from src.prompts import REACT_INSTRUCTIONS, EV_INSTRUCTIONS_HALLUCINATIONS, EV_TEMPLATE_HALLUCINATIONS +from prompts_i import REACT_INSTRUCTIONS, EV_INSTRUCTIONS_HALLUCINATIONS, EV_TEMPLATE_HALLUCINATIONS # Worker Agent QA: handles long context efficiently import os import openai @@ -188,7 +188,7 @@ async def run_agent_with_trace( async def run_evaluator_agent(evaluator_query: EvaluatorQuery) -> EvaluatorResponse: """Evaluate using evaluator agent.""" evaluator_agent = agents.Agent( - name="Evaluator Agent", + name="EvaluatorAgent", instructions=EV_INSTRUCTIONS_HALLUCINATIONS, output_type=EvaluatorResponse, model=agents.OpenAIChatCompletionsModel( diff --git a/src/prompts.py b/src/prompts_i.py similarity index 97% rename from src/prompts.py rename to src/prompts_i.py index 05e5c31..5474bef 100644 --- a/src/prompts.py +++ b/src/prompts_i.py @@ -11,6 +11,7 @@ Do not make up information. \ For facts that might change over time, you must use the search tool to retrieve the \ most up-to-date information. +Finally, write "|" and include a one-sentence summary of your answer. """ # EVALUATIONS @@ -62,6 +63,7 @@ Reasoning: This sentence is verbose, using more words and more complex phrasing than necessary. \ Think step by step. + """ EV_TEMPLATE_CONCISENESS = """\ From 7fdf8246581bf2d64930241cebae198cc1570df5 Mon Sep 17 00:00:00 2001 From: Ioana Barbos Date: Thu, 7 Aug 2025 17:55:17 +0000 Subject: [PATCH 9/9] with conciseness --- .../planner_worker_gradio_ioana.py | 40 ++++++++++++++----- src/prompts.py | 25 +++++++++++- 2 files changed, 54 insertions(+), 11 deletions(-) diff --git a/src/2_frameworks/2_multi_agent/planner_worker_gradio_ioana.py b/src/2_frameworks/2_multi_agent/planner_worker_gradio_ioana.py index 82ba69b..bc18b22 100644 --- a/src/2_frameworks/2_multi_agent/planner_worker_gradio_ioana.py +++ b/src/2_frameworks/2_multi_agent/planner_worker_gradio_ioana.py @@ -74,11 +74,18 @@ def get_query(self) -> str: return EVALUATOR_TEMPLATE.format(**self.model_dump()) +# class EvaluatorResponse(BaseModel): +# """Typed response from the evaluator.""" + +# explanation: str +# is_answer_correct: bool class EvaluatorResponse(BaseModel): """Typed response from the evaluator.""" - explanation: str + explanation_correctness: str is_answer_correct: bool + explanation_conciseness: str + conciseness: bool class EvaluatorAgent(agents.Agent[EvaluatorResponse]): @@ -222,8 +229,11 @@ async def _main(question: str, gr_messages: list[ChatMessage]): span.update(input=question) result_stream = agents.Runner.run_streamed(main_agent, input=question) - score_is_answer_correct = [] - score_explanation = [] + correctness = [] + correctness_explanation = [] + conciseness = [] + conciseness_explanation = [] + async for _item in result_stream.stream_events(): # print(f"Item: {_item}") gr_messages += oai_agent_stream_to_gradio_messages(_item) @@ -235,14 +245,20 @@ async def _main(question: str, gr_messages: list[ChatMessage]): if _item.name == "tool_output" and _item.item.type == "tool_call_output_item": tool_output = json.loads(_item.item.output) - explanation = tool_output.get("explanation") + correctness_expl = tool_output.get("explanation_correctness") is_correct = tool_output.get("is_answer_correct") + conciseness_expl = tool_output.get("explanation_conciseness") + is_concise = tool_output.get("conciseness") - score_is_answer_correct.append(is_correct) - score_explanation.append(explanation) + correctness.append(is_correct) + correctness_explanation.append(correctness_expl) + conciseness.append(is_concise) + conciseness_explanation.append(conciseness_expl) print("✅ is_answer_correct:", is_correct) - print("🧠 explanation:", explanation) + print("🧠 explanation:", correctness_expl) + print("✅ is_answer_concise:", is_concise) + print("🧠 explanation:", conciseness_expl) except: continue @@ -251,8 +267,14 @@ async def _main(question: str, gr_messages: list[ChatMessage]): langfuse_client.create_score( name="is_answer_correct", - value=score_is_answer_correct[0], - comment=score_explanation[0], + value=correctness[0], + comment=correctness_explanation[0], + trace_id=langfuse_client.get_current_trace_id() + ) + langfuse_client.create_score( + name="conciseness", + value=conciseness[0], + comment=conciseness_explanation[0], trace_id=langfuse_client.get_current_trace_id() ) diff --git a/src/prompts.py b/src/prompts.py index 22d45cd..da8b20b 100644 --- a/src/prompts.py +++ b/src/prompts.py @@ -38,7 +38,25 @@ """ EVALUATOR_INSTRUCTIONS = """\ -Evaluate whether the "Proposed Answer" to the given "Question" matches the "Ground Truth". \ +Evaluate for correctness. Assess if the "Proposed Answer" to the given "Question" matches the "Ground Truth". \ + +Evaluate for conciseness. \ +Evaluate if a "Generation" is concise or verbose, with respect to the "Question".\ +A generation can be considered to concise (score 1) if it conveys the core message using the fewest words possible, \ +avoiding unnecessary repetition or jargon. Evaluate the response based on its ability to convey the core message \ +efficiently, without extraneous details or wordiness.\ +Scoring: Rate the conciseness as 0 or 1, where 0 is verbose and 1 is concise. Provide a brief explanation for your score.\ + +Examples: \ +Question: Where did the cat sit? +Generation: "The cat sat on the mat." \ +Score: 1 \ +Reasoning: This sentence is very concise and directly conveys the information. \ + +Question: Where did the cat sit? +Generation: "The feline creature, known as a cat, took up residence upon the floor covering known as a mat." \ +Score: 0 \ +Reasoning: This sentence is verbose, using more words and more complex phrasing than necessary. \ Input structure should be in the following format. \ question: str \ @@ -46,8 +64,11 @@ proposed_response: str \ ALWAYS return the evaluation in the following format. \ -explanation: str \ + +explanation_correctness: str \ is_answer_correct: bool \ +explanation_conciseness: str \ +conciseness: bool \ """ EVALUATOR_TEMPLATE = """\