diff --git a/README.md b/README.md index f6e277e5..315347cc 100644 --- a/README.md +++ b/README.md @@ -13,7 +13,7 @@ Open Chat Playground (OCP) is a web UI that is able to connect virtually any LLM - [ ] [Docker Model Runner](https://docs.docker.com/ai/model-runner) - [ ] [Foundry Local](https://learn.microsoft.com/azure/ai-foundry/foundry-local/what-is-foundry-local) - [x] [Hugging Face](https://huggingface.co/docs) -- [ ] [Ollama](https://github.com/ollama/ollama/tree/main/docs) +- [x] [Ollama](https://github.com/ollama/ollama/tree/main/docs) - [ ] [Anthropic](https://docs.anthropic.com) - [ ] [Naver](https://api.ncloud-docs.com/docs/ai-naver-clovastudio-summary) - [x] [LG](https://github.com/LG-AI-EXAONE) @@ -63,6 +63,7 @@ Open Chat Playground (OCP) is a web UI that is able to connect virtually any LLM - [Use Azure AI Foundry](./docs/azure-ai-foundry.md#run-on-local-machine) - [Use GitHub Models](./docs/github-models.md#run-on-local-machine) - [Use Hugging Face](./docs/hugging-face.md#run-on-local-machine) +- [Use Ollama](./docs/ollama.md#run-on-local-machine) - [Use LG](./docs/lg.md#run-on-local-machine) - [Use OpenAI](./docs/openai.md#run-on-local-machine) @@ -71,6 +72,7 @@ Open Chat Playground (OCP) is a web UI that is able to connect virtually any LLM - [Use Azure AI Foundry](./docs/azure-ai-foundry.md#run-in-local-container) - [Use GitHub Models](./docs/github-models.md#run-in-local-container) - [Use Hugging Face](./docs/hugging-face.md#run-in-local-container) +- [Use Ollama](./docs/ollama.md#run-on-local-container) - [Use LG](./docs/lg.md#run-in-local-container) - [Use OpenAI](./docs/openai.md#run-in-local-container) @@ -79,6 +81,7 @@ Open Chat Playground (OCP) is a web UI that is able to connect virtually any LLM - [Use Azure AI Foundry](./docs/azure-ai-foundry.md#run-on-azure) - [Use GitHub Models](./docs/github-models.md#run-on-azure) - [Use Hugging Face](./docs/hugging-face.md#run-on-azure) +- [Use Ollama](./docs/ollama.md#run-on-azure) - [Use LG](./docs/lg.md#run-on-azure) - [Use OpenAI](./docs/openai.md#run-on-azure) diff --git a/docs/README.md b/docs/README.md index fa29a638..9289ad5b 100644 --- a/docs/README.md +++ b/docs/README.md @@ -3,5 +3,6 @@ - [Azure AI Foundry](azure-ai-foundry.md) - [GitHub Models](github-models.md) - [Hugging Face](hugging-face.md) +- [Ollama](ollama.md) - [LG](lg.md) - [OpenAI](openai.md) diff --git a/docs/ollama.md b/docs/ollama.md new file mode 100644 index 00000000..01ae65a9 --- /dev/null +++ b/docs/ollama.md @@ -0,0 +1,218 @@ +# OpenChat Playground with Ollama + +This page describes how to run OpenChat Playground (OCP) with Ollama integration. + +## Get the repository root + +1. Get the repository root. + + ```bash + # bash/zsh + REPOSITORY_ROOT=$(git rev-parse --show-toplevel) + ``` + + ```powershell + # PowerShell + $REPOSITORY_ROOT = git rev-parse --show-toplevel + ``` + +## Run on local machine + +1. Make sure you are at the repository root. + + ```bash + cd $REPOSITORY_ROOT + ``` + +1. Make sure Ollama is installed and running on your local machine. If not, install Ollama from [ollama.com](https://ollama.com/) and start the service. + + ```bash + # Start Ollama service + ollama serve + ``` + +1. Pull the model you want to use. The default model OCP uses is "llama3.2" + + ```bash + # Example: Pull llama3.2 model + ollama pull llama3.2 + + # Or pull other models + ollama pull mistral + ollama pull phi3 + ollama pull qwen + ``` + +1. Run the app. + + ```bash + # bash/zsh + dotnet run --project $REPOSITORY_ROOT/src/OpenChat.PlaygroundApp -- \ + --connector-type Ollama + ``` + + ```powershell + # PowerShell + dotnet run --project $REPOSITORY_ROOT\src\OpenChat.PlaygroundApp -- ` + --connector-type Ollama + ``` + + +1. Open your web browser, navigate to `http://localhost:5280`, and enter prompts. + +## Run on local container + +This approach runs OpenChat Playground in a container while connecting to Ollama running on the host machine. + +1. Make sure you are at the repository root. + + ```bash + cd $REPOSITORY_ROOT + ``` + +1. Configure Ollama to accept connections from containers. + + ```bash + ollama serve + ``` + +1. Pull the model you want to use. + + ```bash + # bash/zsh + ollama pull llama3.2 + + # Verify Ollama is accessible + curl http://localhost:11434/api/version + ``` + + ```powershell + # PowerShell + ollama pull llama3.2 + + # Verify Ollama is accessible + Invoke-RestMethod -Uri http://localhost:11434/api/version + ``` + + +1. Build a container. + + ```bash + docker build -f Dockerfile -t openchat-playground:latest . + ``` + +1. Run the app. + + ```bash + # bash/zsh - from locally built container + docker run -i --rm -p 8080:8080 openchat-playground:latest \ + --connector-type Ollama \ + --base-url http://host.docker.internal:11434 \ + ``` + + ```powershell + # PowerShell - from locally built container + docker run -i --rm -p 8080:8080 openchat-playground:latest ` + --connector-type Ollama ` + --base-url http://host.docker.internal:11434 + ``` + + ```bash + # bash/zsh - from GitHub Container Registry + docker run -i --rm -p 8080:8080 ghcr.io/aliencube/open-chat-playground/openchat-playground:latest + \ + --connector-type Ollama \ + --base-url http://host.docker.internal:11434 + ``` + + ```powershell + # PowerShell - from GitHub Container Registry + docker run -i --rm -p 8080:8080 ghcr.io/aliencube/open-chat-playground/openchat-playground:latest ` + --connector-type Ollama ` + --base-url http://host.docker.internal:11434 + ``` + Alternatively, if you want to run with a different model, say [qwen], make sure you've already downloaded the model by running the `ollama pull qwen` command. + + ```bash + ollama pull qwen + ``` + + ```bash + # bash/zsh - from locally built container + docker run -i --rm -p 8080:8080 openchat-playground:latest \ + --connector-type Ollama \ + --base-url http://host.docker.internal:11434 \ + --model qwen + ``` + + ```powershell + # PowerShell - from locally built container (with a different model) + docker run -i --rm -p 8080:8080 openchat-playground:latest ` + --connector-type Ollama ` + --base-url http://host.docker.internal:11434 ` + --model qwen + ``` + + > **NOTE**: Use `host.docker.internal:11434` to connect to Ollama running on the host machine from inside the container. + +1. Open your web browser, navigate to `http://localhost:8080`, and enter prompts. + +## Run on Azure + +1. Make sure you are at the repository root. + + ```bash + cd $REPOSITORY_ROOT + ``` + +1. Login to Azure. + + ```bash + # Login to Azure Dev CLI + azd auth login + ``` + +1. Check login status. + + ```bash + # Azure Dev CLI + azd auth login --check-status + ``` + +1. Initialize `azd` template. + + ```bash + azd init + ``` + + > **NOTE**: You will be asked to provide environment name for provisioning. + +1. Set Ollama configuration to azd environment variables. + + **Azure-hosted Ollama (Automatic Deployment)** + + ```bash + # Set connector type to Ollama + azd env set CONNECTOR_TYPE "Ollama" + + # Set a specific model + azd env set OLLAMA_MODEL "llama3.2" + + # BaseUrl is automatically configured - no need to set OLLAMA_BASE_URL + ``` + + > **NOTE**: When deploying to Azure, the Ollama server will be automatically provisioned and deployed as a container with GPU support. The BaseUrl will be automatically configured to connect to the deployed Ollama instance. + +1. Run the following commands in order to provision and deploy the app. + + ```bash + azd up + ``` + + > **NOTE**: You will be asked to provide Azure subscription and location for deployment. + +1. Clean up all the resources. + + ```bash + azd down --force --purge + ``` diff --git a/infra/main.parameters.json b/infra/main.parameters.json index c9c1a5b9..f70ba0ca 100644 --- a/infra/main.parameters.json +++ b/infra/main.parameters.json @@ -29,6 +29,9 @@ "huggingFaceModel": { "value": "${HUGGING_FACE_MODEL=hf.co/Qwen/Qwen3-0.6B-GGUF}" }, + "ollamaModel": { + "value": "${OLLAMA_MODEL=llama3.2}" + }, "lgModel": { "value": "${LG_MODEL=hf.co/LGAI-EXAONE/EXAONE-4.0-1.2B-GGUF}" }, diff --git a/infra/resources.bicep b/infra/resources.bicep index 0483ad36..de1bba60 100644 --- a/infra/resources.bicep +++ b/infra/resources.bicep @@ -392,4 +392,4 @@ module ollama 'br/public:avm/res/app/container-app:0.18.1' = if (useOllama == tr } output AZURE_CONTAINER_REGISTRY_ENDPOINT string = containerRegistry.outputs.loginServer -output AZURE_RESOURCE_OPENCHAT_PLAYGROUNDAPP_ID string = openchatPlaygroundApp.outputs.resourceId +output AZURE_RESOURCE_OPENCHAT_PLAYGROUNDAPP_ID string = openchatPlaygroundApp.outputs.resourceId \ No newline at end of file diff --git a/src/OpenChat.PlaygroundApp/Abstractions/LanguageModelConnector.cs b/src/OpenChat.PlaygroundApp/Abstractions/LanguageModelConnector.cs index f2857bac..583aba8c 100644 --- a/src/OpenChat.PlaygroundApp/Abstractions/LanguageModelConnector.cs +++ b/src/OpenChat.PlaygroundApp/Abstractions/LanguageModelConnector.cs @@ -39,6 +39,7 @@ public static async Task CreateChatClientAsync(AppSettings settings ConnectorType.AzureAIFoundry => new AzureAIFoundryConnector(settings), ConnectorType.GitHubModels => new GitHubModelsConnector(settings), ConnectorType.HuggingFace => new HuggingFaceConnector(settings), + ConnectorType.Ollama => new OllamaConnector(settings), ConnectorType.LG => new LGConnector(settings), ConnectorType.OpenAI => new OpenAIConnector(settings), _ => throw new NotSupportedException($"Connector type '{settings.ConnectorType}' is not supported.") diff --git a/src/OpenChat.PlaygroundApp/Connectors/OllamaConnector.cs b/src/OpenChat.PlaygroundApp/Connectors/OllamaConnector.cs new file mode 100644 index 00000000..2201d3c2 --- /dev/null +++ b/src/OpenChat.PlaygroundApp/Connectors/OllamaConnector.cs @@ -0,0 +1,61 @@ +using Microsoft.Extensions.AI; + +using OllamaSharp; + +using OpenChat.PlaygroundApp.Abstractions; +using OpenChat.PlaygroundApp.Configurations; + +namespace OpenChat.PlaygroundApp.Connectors; + +/// +/// This represents the connector entity for Ollama. +/// +public class OllamaConnector(AppSettings settings) : LanguageModelConnector(settings.Ollama) +{ + private readonly AppSettings _appSettings = settings ?? throw new ArgumentNullException(nameof(settings)); + /// + public override bool EnsureLanguageModelSettingsValid() + { + var settings = this.Settings as OllamaSettings; + if (settings is null) + { + throw new InvalidOperationException("Missing configuration: Ollama."); + } + + if (string.IsNullOrWhiteSpace(settings.BaseUrl!.Trim()) == true) + { + throw new InvalidOperationException("Missing configuration: Ollama:BaseUrl."); + } + + if (string.IsNullOrWhiteSpace(settings.Model!.Trim()) == true) + { + throw new InvalidOperationException("Missing configuration: Ollama:Model."); + } + + return true; + } + + /// + public override async Task GetChatClientAsync() + { + var settings = this.Settings as OllamaSettings; + var baseUrl = settings!.BaseUrl!; + var model = settings!.Model!; + + var config = new OllamaApiClient.Configuration + { + Uri = new Uri(baseUrl), + Model = model, + }; + + var chatClient = new OllamaApiClient(config); + var pulls = chatClient.PullModelAsync(model); + await foreach (var pull in pulls) + { + Console.WriteLine($"Pull status: {pull!.Status}"); + } + + Console.WriteLine($"The {this._appSettings.ConnectorType} connector created with model: {settings.Model}"); + return await Task.FromResult(chatClient).ConfigureAwait(false); + } +} \ No newline at end of file diff --git a/test/OpenChat.PlaygroundApp.Tests/Connectors/OllamaConnectorTests.cs b/test/OpenChat.PlaygroundApp.Tests/Connectors/OllamaConnectorTests.cs new file mode 100644 index 00000000..e2241bb8 --- /dev/null +++ b/test/OpenChat.PlaygroundApp.Tests/Connectors/OllamaConnectorTests.cs @@ -0,0 +1,194 @@ +using Microsoft.Extensions.AI; +using OpenChat.PlaygroundApp.Abstractions; +using OpenChat.PlaygroundApp.Configurations; +using OpenChat.PlaygroundApp.Connectors; + +namespace OpenChat.PlaygroundApp.Tests.Connectors; + +public class OllamaConnectorTests +{ + private const string BaseUrl = "https://test.ollama"; + private const string Model = "test-model"; + private static AppSettings BuildAppSettings(string? baseUrl = BaseUrl, string? model = Model) + { + return new AppSettings + { + ConnectorType = ConnectorType.Ollama, + Ollama = new OllamaSettings + { + BaseUrl = baseUrl, + Model = model + } + }; + } + + [Trait("Category", "UnitTest")] + [Theory] + [InlineData(typeof(LanguageModelConnector), typeof(OllamaConnector), true)] + [InlineData(typeof(OllamaConnector), typeof(LanguageModelConnector), false)] + public void Given_BaseType_Then_It_Should_Be_AssignableFrom_DerivedType(Type baseType, Type derivedType, bool expected) + { + // Act + var result = baseType.IsAssignableFrom(derivedType); + + // Assert + result.ShouldBe(expected); + } + + [Trait("Category", "UnitTest")] + [Fact] + public void Given_Settings_Is_Null_When_EnsureLanguageModelSettingsValid_Invoked_Then_It_Should_Throw() + { + // Arrange + var appSettings = new AppSettings { ConnectorType = ConnectorType.Ollama, Ollama = null }; + var connector = new OllamaConnector(appSettings); + + // Act + Action action = () => connector.EnsureLanguageModelSettingsValid(); + + // Assert + action.ShouldThrow() + .Message.ShouldContain("Ollama"); + } + + [Trait("Category", "UnitTest")] + [Theory] + [InlineData(null, typeof(NullReferenceException), "Object reference not set to an instance of an object")] + [InlineData("", typeof(InvalidOperationException), "Ollama:BaseUrl")] + [InlineData(" ", typeof(InvalidOperationException), "Ollama:BaseUrl")] + [InlineData("\n", typeof(InvalidOperationException), "Ollama:BaseUrl")] + [InlineData("\t", typeof(InvalidOperationException), "Ollama:BaseUrl")] + public void Given_Invalid_BaseUrl_When_EnsureLanguageModelSettingsValid_Invoked_Then_It_Should_Throw(string? baseUrl, Type expectedType, string expectedMessage) + { + // Arrange + var appSettings = BuildAppSettings(baseUrl: baseUrl); + var connector = new OllamaConnector(appSettings); + + // Act + Action action = () => connector.EnsureLanguageModelSettingsValid(); + + // Assert + action.ShouldThrow(expectedType) + .Message.ShouldContain(expectedMessage); + } + + [Trait("Category", "UnitTest")] + [Theory] + [InlineData(null, typeof(NullReferenceException), "Object reference not set to an instance of an object")] + [InlineData("", typeof(InvalidOperationException), "Ollama:Model")] + [InlineData(" ", typeof(InvalidOperationException), "Ollama:Model")] + public void Given_Invalid_Model_When_EnsureLanguageModelSettingsValid_Invoked_Then_It_Should_Throw(string? model, Type expectedType, string expectedMessage) + { + // Arrange + var appSettings = BuildAppSettings(model: model); + var connector = new OllamaConnector(appSettings); + + // Act + Action action = () => connector.EnsureLanguageModelSettingsValid(); + + // Assert + action.ShouldThrow(expectedType) + .Message.ShouldContain(expectedMessage); + } + + [Trait("Category", "UnitTest")] + [Fact] + public void Given_Valid_Settings_When_EnsureLanguageModelSettingsValid_Invoked_Then_It_Should_Return_True() + { + // Arrange + var appSettings = BuildAppSettings(); + var connector = new OllamaConnector(appSettings); + + // Act + var result = connector.EnsureLanguageModelSettingsValid(); + + // Assert + result.ShouldBeTrue(); + } + + [Trait("Category", "UnitTest")] + [Fact] + public async Task Given_Valid_Settings_When_GetChatClientAsync_Invoked_Then_It_Should_Return_ChatClient() + { + // Arrange + var settings = BuildAppSettings(); + var connector = new OllamaConnector(settings); + + // Act + var client = await connector.GetChatClientAsync(); + + // Assert + client.ShouldNotBeNull(); + } + + [Trait("Category", "UnitTest")] + [Theory] + [InlineData(null, typeof(ArgumentNullException), "null")] + [InlineData("", typeof(UriFormatException), "empty")] + [InlineData("http://", typeof(UriFormatException), "Invalid URI")] + public void Given_Missing_BaseUrl_When_GetChatClientAsync_Invoked_Then_It_Should_Throw(string? baseUrl, Type expected, string message) + { + // Arrange + var settings = BuildAppSettings(baseUrl: baseUrl); + var connector = new OllamaConnector(settings); + + // Act + Func action = async () => await connector.GetChatClientAsync(); + + // Assert + action.ShouldThrow(expected) + .Message.ShouldContain(message); + } + + [Trait("Category", "UnitTest")] + [Fact] + public async Task Given_Valid_AppSettings_When_CreateChatClientAsync_Invoked_Then_It_Should_Return_ChatClient() + { + // Arrange + var appSettings = BuildAppSettings(); + + // Act + var client = await LanguageModelConnector.CreateChatClientAsync(appSettings); + + // Assert + client.ShouldNotBeNull(); + client.ShouldBeAssignableTo(); + } + + [Trait("Category", "UnitTest")] + [Theory] + [InlineData(null, null, typeof(NullReferenceException))] + [InlineData("", Model, typeof(InvalidOperationException))] + [InlineData(" ", Model, typeof(InvalidOperationException))] + [InlineData(BaseUrl, null, typeof(NullReferenceException))] + [InlineData(BaseUrl, "", typeof(InvalidOperationException))] + [InlineData(BaseUrl, " ", typeof(InvalidOperationException))] + public async Task Given_Invalid_Ollama_Settings_When_CreateChatClientAsync_Invoked_Then_It_Should_Throw(string? baseUrl, string? model, Type expectedExceptionType) + { + // Arrange + var settings = BuildAppSettings(baseUrl: baseUrl, model: model); + + // Act + Func action = async () => await LanguageModelConnector.CreateChatClientAsync(settings); + + // Assert + await action.ShouldThrowAsync(expectedExceptionType); + } + + [Trait("Category", "UnitTest")] + [Theory] + [InlineData(ConnectorType.Unknown)] + [InlineData(ConnectorType.AmazonBedrock)] + [InlineData(ConnectorType.GoogleVertexAI)] + public async Task Given_Unsupported_ConnectorType_When_CreateChatClientAsync_Invoked_Then_It_Should_Throw(ConnectorType connectorType) + { + // Arrange + var appSettings = new AppSettings { ConnectorType = connectorType }; + + // Act + Func action = async () => await LanguageModelConnector.CreateChatClientAsync(appSettings); + + // Assert + await action.ShouldThrowAsync(); + } +} \ No newline at end of file