VERSION 1.0.0b1

examples/01_intro_to_llmstudio_with_proxy.ipynb

@@ -0,0 +1,355 @@
+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "# Intro to LLMstudio\n"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## start proxy server\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Running LLMstudio Proxy on http://0.0.0.0:8001 \n",
+      "Running LLMstudio Tracking on http://0.0.0.0:8002 \n"
+     ]
+    }
+   ],
+   "source": [
+    "from llmstudio.server import start_servers\n",
+    "start_servers()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Connected to LLMStudio Proxy @ 0.0.0.0:8001\n"
+     ]
+    }
+   ],
+   "source": [
+    "from llmstudio_proxy.provider import LLMProxyProvider as LLM\n",
+    "from llmstudio_proxy.provider import ProxyConfig\n",
+    "\n",
+    "llm = LLM(provider=\"openai\", \n",
+    "          proxy_config=ProxyConfig(host=\"0.0.0.0\", port=\"8001\"))"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 3,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "result = llm.chat(\"olá\", model=\"gpt-4o\")\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 4,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "('Olá! Como posso ajudar você hoje?',\n",
+       " {'input_tokens': 2,\n",
+       "  'output_tokens': 11,\n",
+       "  'total_tokens': 13,\n",
+       "  'cost_usd': 0.000175,\n",
+       "  'latency_s': 0.7408609390258789,\n",
+       "  'time_to_first_token_s': 0.6484076976776123,\n",
+       "  'inter_token_latency_s': 0.009322775734795464,\n",
+       "  'tokens_per_second': 13.497809741661507})"
+      ]
+     },
+     "execution_count": 4,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "result.chat_output, result.metrics"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "### Import clients\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 5,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from llmstudio import LLM\n",
+    "from pprint import pprint"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 6,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "host='0.0.0.0' port='8001' url=None username=None password=None\n",
+      "Connected to LLMStudio Proxy @ 0.0.0.0:8001\n"
+     ]
+    }
+   ],
+   "source": [
+    "# You can set OPENAI_API_KEY and ANTHROPIC_API_KEY on .env file\n",
+    "from llmstudio_proxy.provider import ProxyConfig\n",
+    "proxy = ProxyConfig(host=\"0.0.0.0\", port=\"8001\")\n",
+    "print(proxy)\n",
+    "\n",
+    "openai = LLM(\"openai\", proxy_config=proxy)\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 7,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "<llmstudio_proxy.provider.LLMProxyProvider at 0x11711c6b0>"
+      ]
+     },
+     "execution_count": 7,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "openai._provider\n"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "### Chat (non-stream)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 8,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "ChatCompletion(id='6efb4a6d-74c2-477a-9f97-a1f9857e100e', choices=[Choice(finish_reason='stop', index=0, logprobs=None, message=ChatCompletionMessage(content='I’m called Assistant. How can I help you today?', refusal=None, role='assistant', function_call=None, tool_calls=None))], created=1729085340, model='gpt-4o', object='chat.completion', service_tier=None, system_fingerprint=None, usage=None, chat_input=\"What's your name\", chat_output='I’m called Assistant. How can I help you today?', chat_output_stream='', context=[{'role': 'user', 'content': \"What's your name\"}], provider='openai', deployment='gpt-4o-2024-08-06', timestamp=1729085340.6024141, parameters={}, metrics={'input_tokens': 4, 'output_tokens': 12, 'total_tokens': 16, 'cost_usd': 0.0002, 'latency_s': 0.8470461368560791, 'time_to_first_token_s': 0.7443199157714844, 'inter_token_latency_s': 0.007879862418541541, 'tokens_per_second': 16.52802532334638})"
+      ]
+     },
+     "execution_count": 8,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "openai.chat(\"What's your name\", model=\"gpt-4o\")"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "#### Async version"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 9,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "ChatCompletion(id='0c959150-5d79-442b-b491-6663e5608b92', choices=[Choice(finish_reason='stop', index=0, logprobs=None, message=ChatCompletionMessage(content='I’m an AI language model and don’t have a personal name, but you can call me Assistant if you’d like! How can I help you today?', refusal=None, role='assistant', function_call=None, tool_calls=None))], created=1729085340, model='gpt-4o', object='chat.completion', service_tier=None, system_fingerprint=None, usage=None, chat_input=\"What's your name\", chat_output='I’m an AI language model and don’t have a personal name, but you can call me Assistant if you’d like! How can I help you today?', chat_output_stream='', context=[{'role': 'user', 'content': \"What's your name\"}], provider='openai', deployment='gpt-4o-2024-08-06', timestamp=1729085341.564915, parameters={}, metrics={'input_tokens': 4, 'output_tokens': 32, 'total_tokens': 36, 'cost_usd': 0.0005, 'latency_s': 0.9470148086547852, 'time_to_first_token_s': 0.5814306735992432, 'inter_token_latency_s': 0.011006882696440725, 'tokens_per_second': 35.902289688897575})"
+      ]
+     },
+     "execution_count": 9,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "await openai.achat(\"What's your name\", model=\"gpt-4o\")"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "### Chat (stream)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 10,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\n",
+      "\n",
+      "Space, the vast expanse that stretches beyond Earth's atmosphere, is a realm of infinite possibilities and\n",
+      "\n",
+      " mysteries. It is a place where the laws of physics seem to take on new dimensions, presenting challenges\n",
+      "\n",
+      " and wonders that have captivated humanity for centuries. Space exploration has allowed us to glimpse the beauty and complexity\n",
+      "\n",
+      " of our solar system, with its diverse planets, moons, and other celestial bodies, each offering unique\n",
+      "\n",
+      " insights into the formation and evolution of our universe. The discovery of exoplanets orbiting distant stars\n",
+      "\n",
+      " has ignited the imagination of scientists and dreamers alike, pondering the existence of life beyond Earth.\n",
+      "\n",
+      " As technology advances, our ability to explore and understand the cosmos continues to expand, promising to unlock secrets\n",
+      "\n",
+      " of the universe's origin, the nature of dark matter, and the potential for future human habitation beyond\n",
+      "\n",
+      " our home planet. Space, with its mesmerizing beauty and endless intrigue, remains one of the final front\n",
+      "\n",
+      "iers that invites exploration and discovery.\n",
+      "\n",
+      "## Metrics:\n",
+      "{'cost_usd': 0.00289,\n",
+      " 'input_tokens': 8,\n",
+      " 'inter_token_latency_s': 0.01732337538571281,\n",
+      " 'latency_s': 3.8832650184631348,\n",
+      " 'output_tokens': 190,\n",
+      " 'time_to_first_token_s': 0.6428868770599365,\n",
+      " 'tokens_per_second': 48.41286883747225,\n",
+      " 'total_tokens': 198}\n"
+     ]
+    }
+   ],
+   "source": [
+    "response = openai.chat(\"Write a paragfraph about space\", model=\"gpt-4o\", is_stream=True)\n",
+    "for i, chunk in enumerate(response):\n",
+    "    if i%20==0:\n",
+    "        print(\"\\n\")\n",
+    "    if not chunk.metrics:\n",
+    "        print(chunk.chat_output_stream, end=\"\", flush=True)\n",
+    "    else:\n",
+    "        print(\"\\n\\n## Metrics:\")\n",
+    "        pprint(chunk.metrics)\n"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "#### Async version"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 11,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\n",
+      "\n",
+      "Space, the vast and seemingly infinite expanse that lies beyond our planet's atmosphere, captivates\n",
+      "\n",
+      " the imagination and fuels scientific inquiry. It is home to an array of celestial bodies, including stars,\n",
+      "\n",
+      " planets, moons, and galaxies, each with its own mysteries waiting to be unraveled. The empt\n",
+      "\n",
+      "iness of space challenges our understanding of physics and ignites the pursuit of knowledge about the universe's origin\n",
+      "\n",
+      " and fate. Dark matter and dark energy, which together form a significant portion of the cosmos, remain\n",
+      "\n",
+      " elusive and continue to stimulate research and debate among astronomers. Beyond the scientific aspect, space is a\n",
+      "\n",
+      " source of wonder and inspiration in art, literature, and philosophy, prompting humanity to reflect on its place\n",
+      "\n",
+      " among the stars and the potential for life beyond Earth. As we venture further into this cosmic frontier,\n",
+      "\n",
+      " the quest for discovery reveals not just the universe's secrets, but also the boundless potential of human\n",
+      "\n",
+      " ingenuity and exploration.\n",
+      "\n",
+      "## Metrics:\n",
+      "{'cost_usd': 0.0001116,\n",
+      " 'input_tokens': 8,\n",
+      " 'inter_token_latency_s': 0.05799643889717434,\n",
+      " 'latency_s': 11.739921808242798,\n",
+      " 'output_tokens': 184,\n",
+      " 'time_to_first_token_s': 1.0663559436798096,\n",
+      " 'tokens_per_second': 15.758196947283615,\n",
+      " 'total_tokens': 192}\n"
+     ]
+    }
+   ],
+   "source": [
+    "i=0\n",
+    "async for chunk in await openai.achat(\"Write a paragfraph about space\", model=\"gpt-4o-mini\", is_stream=True):\n",
+    "    if i%20==0:\n",
+    "        print(\"\\n\")\n",
+    "    if not chunk.metrics:\n",
+    "        print(chunk.chat_output_stream, end=\"\", flush=True)\n",
+    "    else:\n",
+    "        print(\"\\n\\n## Metrics:\")\n",
+    "        pprint(chunk.metrics)\n",
+    "    i+=1\n"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.12.2"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}