diff --git a/.gitignore b/.gitignore new file mode 100644 index 00000000..abdb4197 --- /dev/null +++ b/.gitignore @@ -0,0 +1 @@ +catalogs/ \ No newline at end of file diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index bd554b58..36554566 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -2,10 +2,17 @@ Thank you for your interest in contributing to the official Docker MCP Registry. This document outlines how to contribute to this project. +## Prerequisites + +- Go v1.24+ +- [Docker Desktop](https://www.docker.com/products/docker-desktop/) +- [Task](https://taskfile.dev/) + + ## 🔄 Pull request process overview - Fork the repository to your own GitHub account and clone it locally. - Repository includes a `servers` folder where you should add a new folder with a `server.yaml` inside. -- Repository includes a `scripts` folder with bash scripts to automate some of the steps. +- Repository includes a `scripts` folder with bash scripts and Go code to automate some of the steps. - Correctly format your commit messages, see Commit message guidelines below. _Note: All commits must include a Signed-off-by trailer at the end of each commit message to indicate that the contributor agrees to the Developer Certificate of Origin._ - Open a PR by ensuring the title and its description reflect the content of the PR. - Ensure that CI passes, if it fails, fix the failures. @@ -22,76 +29,72 @@ Add your entry by creating a new folder following the `owner@name` template, and - The GitHub URL of your project. The project needs to have a valid Dockerfile. - A brief description of your MCP Server. - A category for the MCP server, one of: -* 'ai' -* 'data-visualization' -* 'database' -* 'devops' -* 'ecommerce' -* 'finance' -* 'games' -* 'communication' -* 'monitoring' -* 'productivity' -* 'search' - -#### 🚀 Generate folder and `server.yaml` using `new-server.sh` script -You can use our script to automate the creation of the files. Let's assume we have a new MCP Server to access my org's database. The MCP is called `My-ORGDB-MCP` and the GitHub repo is located at: `https://github.com/myorg/my-orgdb-mcp` - -You can call the tool passing the MCP server name, category, and github url. + * 'ai' + * 'data-visualization' + * 'database' + * 'devops' + * 'ecommerce' + * 'finance' + * 'games' + * 'communication' + * 'monitoring' + * 'productivity' + * 'search' + +#### 🚀 Generate folder and `server.yaml` using `task create` +You can use our command to automate the creation of the files. Let's assume we have a new MCP Server to access my org's database. My server's GitHub repo is located at: `https://github.com/myorg/my-orgdb-mcp` + +You can call the creation tool passing the category (required), and github url. If your server requires any environment variables, pass them at the end with `-e KEY=value`. ``` -./scripts/new-server.sh My-ORGDB-MCP databases https://github.com/myorg/my-orgdb-mcp +task create -- --category database https://github.com/myorg/my-orgdb-mcp -e API_TOKEN=test ``` -This will create a directory under `servers` as follows: `./servers/my-orgdb-mcp` and inside you will find a `server.yaml` file with your MCP definition. +This will build an image using the Dockerfile at the root of the repository, run it while verifying the MCP server is able to list tools, and then create the necessary files. It will create a directory under `servers` as follows: `./servers/my-orgdb-mcp` and inside you will find a `server.yaml` file with your MCP definition. ``` -server: - name: test01 - image: mcp/test01 +name: my-orgdb-mcp +image: mcp/my-orgdb-mcp type: server meta: - category: test + category: database tags: - - test - highlighted: false + - database about: - title: test01 + title: My OrgDB MCP (TODO) + description: TODO (only to provide a better description than the upstream project) icon: https://avatars.githubusercontent.com/u/182288589?s=200&v=4 source: - project: https://github.com/docker/mcp-registry - branch: main -# config: -# description: TODO -# secrets: -# - name: test01.secret_name -# env: TEST01 -# example: TODO -# env: -# - name: ENV_VAR_NAME -# example: TODO -# value: '{{test01.env_var_name}}' -# parameters: -# type: object -# properties: -# param_name: -# type: string -# required: -# - param_name + project: https://github.com/myorg/my-orgdb-mcp +config: + description: Configure the connection to TODO + secrets: + - name: my-orgdb-mcp.api_token + env: API_TOKEN + example: ``` -If you want to provide a specific Docker image built by your organisation, you can pass it to the script as follows: +If you want to provide a specific Docker image built by your organisation instead of having Docker build the image, you can specify it with the `--image` flag: ``` -IMAGE_NAME=myorg/myimage ./scripts/new-server.sh My-ORGDB-MCP databases https://github.com/myorg/my-orgdb-mcp +task create -- --category database --image myorg/my-mcp https://github.com/myorg/my-orgdb-mcp -e API_TOKEN=test ``` -As you can see, the configuration block has been commented out. If you need to pass environmental variables or secrets, please uncomment the necessary lines. - 🔒 If you don't provide a Docker image, we will build the image for you and host it in [Docker Hub's `mcp` namespace](https://hub.docker.com/u/mcp), the benefits are: image will include cryptographic signatures, provenance tracking, SBOMs, and automatic security updates. Otherwise, self-built images still benefit from container isolation but won't include the enhanced security features of Docker-built images. ### 3️⃣ Run & Test your MCP Server locally -🚧 tbd +After creating your server file with `task create`, you will be given instructions for running it locally. In the case of my-orgdb-mcp, we would run the following commands next. + +``` +task catalog -- my-orgdb-mcp +docker mcp catalog import $PWD/catalogs/my-orgdb-mcp/catalog.yaml +``` + +Now, if we go into the MCP Toolkit on Docker Desktop, we'll see our new MCP server there! We can configure and enable it there, and test it against configured clients. Once we're done testing, we can restore it back to the original Docker catalog. + +``` +docker mcp catalog reset +``` ### 4️⃣ Create `commit` and raise the Pull Request 🚧 tbd diff --git a/Taskfile.yml b/Taskfile.yml new file mode 100644 index 00000000..5ff2612e --- /dev/null +++ b/Taskfile.yml @@ -0,0 +1,10 @@ +version: "3" + +tasks: + create: + desc: Create a new mcp server definition + cmd: go run ./cmd/create {{.CLI_ARGS}} + + catalog: + desc: Generate a test catalog + cmd: go run ./cmd/catalog {{.CLI_ARGS}} \ No newline at end of file diff --git a/cmd/catalog/main.go b/cmd/catalog/main.go new file mode 100644 index 00000000..0c7ff240 --- /dev/null +++ b/cmd/catalog/main.go @@ -0,0 +1,74 @@ +package main + +import ( + "context" + "fmt" + "log" + "os" + "path/filepath" + + "github.com/docker/mcp-registry/pkg/catalog" + "github.com/docker/mcp-registry/pkg/servers" +) + +func main() { + if len(os.Args) != 2 { + fmt.Println("Usage: catalog ") + os.Exit(1) + } + + name := os.Args[1] + + if err := run(name); err != nil { + log.Fatal(err) + } +} + +func run(name string) error { + serverFile := filepath.Join("servers", name, "server.yaml") + server, err := servers.Read(serverFile) + if err != nil { + if os.IsNotExist(err) { + fmt.Fprintf(os.Stderr, "server.yaml for %s not found. Run `task create -- ` to create a new server definition first.\n", name) + } + return err + } + + tile, err := catalog.ToTile(context.Background(), server) + if err != nil { + return err + } + + catalogDir := filepath.Join("catalogs", name) + if err := os.MkdirAll(catalogDir, 0755); err != nil { + return err + } + + if err := writeCatalog(name, catalogDir, tile); err != nil { + return err + } + + return nil +} + +func writeCatalog(name, catalogDir string, tile catalog.Tile) error { + catalogFile := filepath.Join(catalogDir, "catalog.yaml") + + if err := catalog.WriteYaml(catalogFile, catalog.TopLevel{ + Version: catalog.Version, + Name: "docker-mcp", // overwrite the default catalog + DisplayName: "Local Test Catalog", + Registry: catalog.TileList{ + { + Name: name, + Tile: tile, + }, + }, + }); err != nil { + return err + } + + fmt.Printf("Catalog written to %s\n", catalogFile) + + return nil +} diff --git a/cmd/create/main.go b/cmd/create/main.go new file mode 100644 index 00000000..cc23239f --- /dev/null +++ b/cmd/create/main.go @@ -0,0 +1,303 @@ +package main + +import ( + "bytes" + "context" + "flag" + "fmt" + "log" + "os" + "os/exec" + "os/signal" + "path/filepath" + "strings" + "syscall" + + "gopkg.in/yaml.v3" + + "github.com/docker/mcp-registry/internal/licenses" + "github.com/docker/mcp-registry/internal/mcp" + "github.com/docker/mcp-registry/pkg/github" + "github.com/docker/mcp-registry/pkg/servers" +) + +func main() { + name := flag.String("name", "", "Name of the mcp server, name is guessed if not provided") + category := flag.String("category", "", "Category for the mcp server (required) - [ai, data-visualization, database, devops, ecommerce, finance, games, communication, monitoring, productivity, search]") + image := flag.String("image", "", "Image to use for the mcp server, instead of building from the repository") + build := flag.Bool("build", true, "Build the image") + listTools := flag.Bool("tools", true, "List the tools") + + flag.Parse() + args := flag.Args() + + if *category == "" { + flag.Usage() + os.Exit(1) + } + + ctx, cancel := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM) + defer cancel() + + url := "" + var additionalArgs []string + if len(args) > 0 { + url = args[0] + additionalArgs = args[1:] + } + + if err := run(ctx, url, *name, *category, *image, *build, *listTools, additionalArgs); err != nil { + log.Fatal(err) + } +} + +func run(ctx context.Context, buildURL, name, category, userProvidedImage string, build, listTools bool, args []string) error { + projectURL := buildURL + + client := github.New() + repository, err := client.GetProjectRepository(ctx, projectURL) + if err != nil { + return err + } + + tags := github.FindTags(repository.Topics) + + detectedInfo, err := github.DetectBranchAndDirectory(projectURL, repository) + if err != nil { + return err + } + + branch := detectedInfo.Branch + directory := detectedInfo.Directory + projectURL = detectedInfo.ProjectURL + + upstream := "" + if repository.GetParent() != nil { + upstream = repository.GetParent().GetHTMLURL() + } + + sha, err := client.GetCommitSHA1(ctx, projectURL, branch) + if err != nil { + return err + } + + refProjectURL := projectURL + if upstream != "" { + refProjectURL = upstream + } + + guessedName := guessName(projectURL) + if name == "" { + name = strings.ToLower(guessedName) + } + + tag := "mcp/" + name + if userProvidedImage != "" { + tag = userProvidedImage + } + + title := strings.ToUpper(guessedName[0:1]) + guessedName[1:] + if !strings.Contains(repository.GetDescription(), title+" MCP Server") { + title += " (TODO)" + } + + if !licenses.IsValid(repository.License) { + fmt.Println("[WARNING] Project", projectURL, "is licensed under", repository.License.GetName(), "which may be incompatible with some tools") + } + + if build && userProvidedImage == "" { + gitURL := projectURL + ".git#" + if branch != "" { + gitURL += branch + } + if directory != "" && directory != "." { + gitURL += ":" + directory + } + + cmd := exec.CommandContext(ctx, "docker", "buildx", "build", "--secret", "id=GIT_AUTH_TOKEN", "-t", "check", "-t", tag, "--label", "org.opencontainers.image.revision="+sha, gitURL) + cmd.Env = []string{"GIT_AUTH_TOKEN=" + os.Getenv("GITHUB_TOKEN"), "PATH=" + os.Getenv("PATH")} + cmd.Dir = os.TempDir() + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + + if err := cmd.Run(); err != nil { + return err + } + } + + // Find the working directory + cmd := exec.CommandContext(ctx, "docker", "inspect", "--format", "{{.Config.WorkingDir}}", "check") + out, err := cmd.CombinedOutput() + if err != nil { + return fmt.Errorf("getting working directory: %w\n%s", err, out) + } + + if listTools { + var ( + secrets []servers.Secret + env []servers.Env + command []string + ) + for i := 0; i < len(args); i += 1 { + if args[i] == "-e" { + kv := args[i+1] + parts := strings.SplitN(kv, "=", 2) + + if strings.HasSuffix(parts[0], "_TOKEN") || strings.HasSuffix(parts[0], "_KEY") { + secrets = append(secrets, servers.Secret{ + Name: secretName(name, parts[0]), + Env: parts[0], + Example: "<" + parts[0] + ">", + }) + } else { + env = append(env, servers.Env{ + Name: parts[0], + Example: parts[1], + }) + } + i += 1 + } else { + command = append(command, args[i]) + } + } + + icon, err := client.FindIcon(ctx, refProjectURL) + if err != nil { + return err + } + + if branch == "main" { + branch = "" + } + + env, schema := servers.CreateSchema(name, env) + + server := servers.Server{ + Name: name, + Image: tag, + Type: "server", + Meta: servers.Meta{ + Category: category, + Tags: tags, + }, + About: servers.About{ + Icon: icon, + Title: title, + Description: "TODO (only to provide a better description than the upstream project)", + }, + Source: servers.Source{ + Project: projectURL, + Upstream: upstream, + Branch: branch, + Directory: directory, + }, + Run: servers.Run{ + Command: command, + }, + Config: servers.Config{ + Description: "Configure the connection to TODO", + Secrets: secrets, + Env: env, + Parameters: schema, + }, + } + + tools, err := mcp.Tools(ctx, server, false, false, false) + if err != nil { + return err + } + + if len(tools) == 0 { + fmt.Println() + fmt.Println("No tools found.") + } else { + fmt.Println() + fmt.Println(len(tools), "tools found.") + } + + fmt.Printf("\n-----------------------------------------\n\n") + + if exists, err := checkLocalServerExists(name); err != nil { + return err + } else if exists { + fmt.Printf("[WARNING] Server for %s already exists, overwriting...\n", name) + } + + serverDir := filepath.Join("servers", server.Name) + _ = os.Mkdir(serverDir, 0755) + + serverFile := filepath.Join(serverDir, "server.yaml") + + var buf bytes.Buffer + encoder := yaml.NewEncoder(&buf) + encoder.SetIndent(2) + if err := encoder.Encode(server); err != nil { + return err + } + + if err := os.WriteFile(serverFile, buf.Bytes(), 0644); err != nil { + return fmt.Errorf("writing server config: %w", err) + } + + fmt.Printf("Server definition written to %s.\n", serverFile) + + fmt.Printf(` +----------------------------------------- + +What to do next? + + 1. Review %[2]s and make sure no TODO remains. + + 2. Test out your server in Docker Desktop by generating a catalog and importing it: + + task catalog -- %[1]s + docker mcp catalog import $PWD/catalogs/%[1]s/catalog.yaml + + 3. After doing so, you should be able to test it with the MCP Toolkit. + + 4. Reset your catalog after testing: + + docker mcp catalog reset + + 5. Open a Pull Request with the %[2]s file. +`, name, serverFile) + } + + return nil +} + +func guessName(projectURL string) string { + parts := strings.Split(strings.ToLower(projectURL), "/") + name := parts[len(parts)-1] + + name = strings.TrimPrefix(name, "mcp-server-") + name = strings.TrimPrefix(name, "mcp-") + name = strings.TrimPrefix(name, "server-") + + name = strings.TrimSuffix(name, "-mcp-server") + name = strings.TrimSuffix(name, "-mcp") + name = strings.TrimSuffix(name, "-server") + + return name +} + +func secretName(server, name string) string { + return server + "." + strings.TrimPrefix(strings.ToLower(name), strings.ToLower(server)+"_") +} + +func checkLocalServerExists(name string) (bool, error) { + entries, err := os.ReadDir("servers") + if err != nil { + return false, err + } + + for _, entry := range entries { + if entry.IsDir() { + if entry.Name() == name { + return true, nil + } + } + } + + return false, nil +} diff --git a/go.mod b/go.mod new file mode 100644 index 00000000..a4e028c0 --- /dev/null +++ b/go.mod @@ -0,0 +1,15 @@ +module github.com/docker/mcp-registry + +go 1.24.0 + +require ( + github.com/google/go-github/v70 v70.0.0 + github.com/mark3labs/mcp-go v0.25.0 + gopkg.in/yaml.v3 v3.0.1 +) + +require ( + github.com/google/go-querystring v1.1.0 // indirect + github.com/spf13/cast v1.7.1 // indirect + github.com/yosida95/uritemplate/v3 v3.0.2 // indirect +) diff --git a/go.sum b/go.sum new file mode 100644 index 00000000..ebd44eaa --- /dev/null +++ b/go.sum @@ -0,0 +1,32 @@ +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHkI4W8= +github.com/frankban/quicktest v1.14.6/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0= +github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= +github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= +github.com/google/go-github/v70 v70.0.0 h1:/tqCp5KPrcvqCc7vIvYyFYTiCGrYvaWoYMGHSQbo55o= +github.com/google/go-github/v70 v70.0.0/go.mod h1:xBUZgo8MI3lUL/hwxl3hlceJW1U8MVnXP3zUyI+rhQY= +github.com/google/go-querystring v1.1.0 h1:AnCroh3fv4ZBgVIf1Iwtovgjaw/GiKJo8M8yD/fhyJ8= +github.com/google/go-querystring v1.1.0/go.mod h1:Kcdr2DB4koayq7X8pmAG4sNG59So17icRSOU623lUBU= +github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/mark3labs/mcp-go v0.25.0 h1:UUpcMT3L5hIhuDy7aifj4Bphw4Pfx1Rf8mzMXDe8RQw= +github.com/mark3labs/mcp-go v0.25.0/go.mod h1:rXqOudj/djTORU/ThxYx8fqEVj/5pvTuuebQ2RC7uk4= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/rogpeppe/go-internal v1.9.0 h1:73kH8U+JUqXU8lRuOHeVHaa/SZPifC7BkcraZVejAe8= +github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs= +github.com/spf13/cast v1.7.1 h1:cuNEagBQEHWN1FnbGEjCXL2szYEXqfJPbP2HNUaca9Y= +github.com/spf13/cast v1.7.1/go.mod h1:ancEpBxwJDODSW/UG4rDrAqiKolqNNh2DX3mk86cAdo= +github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= +github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/yosida95/uritemplate/v3 v3.0.2 h1:Ed3Oyj9yrmi9087+NczuL5BwkIc4wvTb5zIM+UJPGz4= +github.com/yosida95/uritemplate/v3 v3.0.2/go.mod h1:ILOh0sOhIJR3+L/8afwt/kE++YT040gmv5BQTMR2HP4= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/internal/licenses/check.go b/internal/licenses/check.go new file mode 100644 index 00000000..7ec0de70 --- /dev/null +++ b/internal/licenses/check.go @@ -0,0 +1,14 @@ +package licenses + +import ( + "strings" + + "github.com/google/go-github/v70/github" +) + +func IsValid(license *github.License) bool { + if license != nil && (strings.HasPrefix(license.GetKey(), "gpl") || strings.HasPrefix(license.GetKey(), "agpl") || strings.HasPrefix(license.GetKey(), "npl")) { + return false + } + return true +} diff --git a/internal/mcp/client.go b/internal/mcp/client.go new file mode 100644 index 00000000..aff38dd1 --- /dev/null +++ b/internal/mcp/client.go @@ -0,0 +1,176 @@ +package mcp + +import ( + "context" + "fmt" + "os/exec" + "time" + + "github.com/docker/mcp-registry/pkg/servers" + "github.com/mark3labs/mcp-go/mcp" +) + +type client struct { + image string + pull bool + env []servers.Env + secrets []servers.Secret + args []string + command []string + + c *stdioMCPClient +} + +func newClient(image string, pull bool, env []servers.Env, secrets []servers.Secret, args []string, command []string) *client { + return &client{ + image: image, + pull: pull, + env: env, + secrets: secrets, + args: args, + command: command, + } +} + +func (cl *client) Start(ctx context.Context, debug bool) error { + if cl.c != nil { + return fmt.Errorf("already started %s", cl.image) + } + + if cl.pull { + output, err := exec.CommandContext(ctx, "docker", "pull", cl.image).CombinedOutput() + if err != nil { + return fmt.Errorf("pulling image %s: %w (%s)", cl.image, err, string(output)) + } + } + + args := []string{"run", "--rm", "-i", "--init", "--cap-drop=ALL"} + args = append(args, cl.args...) + for _, env := range cl.env { + args = append(args, "-e", env.Name) + } + for _, secret := range cl.secrets { + args = append(args, "-e", secret.Env) + } + args = append(args, cl.image) + for _, arg := range cl.command { + args = append(args, replacePlaceholders(arg, cl.env, cl.secrets)) + } + c := newMCPClient("docker", toEnviron(cl.env, cl.secrets), args...) + cl.c = c + + initRequest := mcp.InitializeRequest{} + initRequest.Params.ProtocolVersion = mcp.LATEST_PROTOCOL_VERSION + initRequest.Params.ClientInfo = mcp.Implementation{ + Name: "docker", + Version: "1.0.0", + } + + ctx, cancel := context.WithTimeout(ctx, 60*time.Second) + defer cancel() + + if _, err := c.Initialize(ctx, initRequest, debug); err != nil { + return fmt.Errorf("initializing %s: %w", cl.image, err) + } + return nil +} + +func (cl *client) ListTools(ctx context.Context) ([]mcp.Tool, error) { + if cl.c == nil { + return nil, fmt.Errorf("listing tools %s: not started", cl.image) + } + + response, err := cl.c.ListTools(ctx, mcp.ListToolsRequest{}) + if err != nil { + return nil, fmt.Errorf("listing tools %s: %w", cl.image, err) + } + + return response.Tools, nil +} + +func (cl *client) ListPrompts(ctx context.Context) ([]mcp.Prompt, error) { + if cl.c == nil { + return nil, fmt.Errorf("listing tools %s: not started", cl.image) + } + + response, err := cl.c.ListPrompts(ctx, mcp.ListPromptsRequest{}) + if err != nil { + return nil, fmt.Errorf("listing tools %s: %w", cl.image, err) + } + + return response.Prompts, nil +} + +func (cl *client) CallTool(ctx context.Context, name string, args map[string]any) (*mcp.CallToolResult, error) { + if cl.c == nil { + return nil, fmt.Errorf("calling tool %s: not started", name) + } + + request := mcp.CallToolRequest{} + request.Params.Name = name + request.Params.Arguments = args + if request.Params.Arguments == nil { + request.Params.Arguments = map[string]any{} + } + // MCP servers return an error if the args are empty so we make sure + // there is at least one argument + if len(request.Params.Arguments) == 0 { + request.Params.Arguments["args"] = "..." + } + + result, err := cl.c.CallTool(ctx, request) + if err != nil { + return nil, fmt.Errorf("calling tool %s on %s: %w", name, cl.image, err) + } + + return result, nil +} + +func (cl *client) Close(deleteImage bool) error { + if cl.c == nil { + return fmt.Errorf("closing %s: not started", cl.image) + } + if err := cl.c.Close(); err != nil { + return err + } + + if deleteImage { + output, err := exec.Command("docker", "rmi", "-f", cl.image).CombinedOutput() + if err != nil { + return fmt.Errorf("failed removing image %s: %w (%s)", cl.image, err, string(output)) + } + } + + return nil +} + +func replacePlaceholders(arg string, env []servers.Env, secrets []servers.Secret) string { + // TODO(dga): Temporary fix + if arg == "{{filesystem.paths|volume-target|into}}" { + return "." + } + + for _, env := range env { + if arg == "$"+env.Name { + return fmt.Sprintf("%v", env.Example) + } + } + for _, secret := range secrets { + if arg == "$"+secret.Env { + return secret.Example + } + } + + return arg +} + +func toEnviron(env []servers.Env, secrets []servers.Secret) []string { + var environ []string + for _, env := range env { + environ = append(environ, fmt.Sprintf("%s=%s", env.Name, env.Example)) + } + for _, secret := range secrets { + environ = append(environ, secret.Env+"="+secret.Example) + } + return environ +} diff --git a/internal/mcp/helper.go b/internal/mcp/helper.go new file mode 100644 index 00000000..e73ae79d --- /dev/null +++ b/internal/mcp/helper.go @@ -0,0 +1,182 @@ +package mcp + +import ( + "context" + "fmt" + "slices" + "sort" + "strings" + + "github.com/mark3labs/mcp-go/mcp" + + "github.com/docker/mcp-registry/pkg/servers" +) + +func Tools(ctx context.Context, server servers.Server, pull, cleanup, debug bool) ([]Tool, error) { + var ( + args []string + extraEnv []servers.Env + ) + if len(server.Requirement) > 0 { + cancel, sidecarID, env, err := runRequirement(ctx, server.Requirement) + if err != nil { + return nil, err + } + defer cancel() + + for _, e := range env { + parts := strings.SplitN(e, "=", 2) + + extraEnv = append(extraEnv, servers.Env{ + Name: parts[0], + Example: parts[1], + }) + } + + args = append(args, "--network", "container:"+sidecarID) + } + + env := append(server.Config.Env, extraEnv...) + for name, value := range server.Run.Env { + env = append(env, servers.Env{ + Name: name, + Value: value, + }) + } + + c := newClient(server.Image, pull, env, server.Config.Secrets, args, server.Run.Command) + if err := c.Start(ctx, debug); err != nil { + return nil, err + } + + tools, err := c.ListTools(ctx) + if err != nil { + c.Close(cleanup) + return nil, err + } + + err = c.Close(cleanup) + if err != nil { + return nil, err + } + + sort.Slice(tools, func(i, j int) bool { return tools[i].Name < tools[j].Name }) + + var list []Tool + for _, tool := range tools { + var arguments []ToolArgument + var requiredPropertyNames []string + var optionalPropertyNames []string + for name := range tool.InputSchema.Properties { + if slices.Contains(tool.InputSchema.Required, name) { + requiredPropertyNames = append(requiredPropertyNames, name) + } else { + optionalPropertyNames = append(optionalPropertyNames, name) + } + } + sort.Strings(requiredPropertyNames) + sort.Strings(optionalPropertyNames) + + propertyNames := append(requiredPropertyNames, optionalPropertyNames...) + + for _, name := range propertyNames { + v := tool.InputSchema.Properties[name] + + // Type + argumentType := "string" + rawType := v.(map[string]any)["type"] + if rawType != "" && rawType != nil { + if str, ok := rawType.(string); ok { + argumentType = str + } + } + + // Item types + var items *Items + if argumentType == "array" { + itemsType := "string" + if rawItems, found := v.(map[string]any)["items"]; found { + if kv, ok := rawItems.(map[string]any); ok { + if rawItemsType, found := kv["type"]; found { + if str, ok := rawItemsType.(string); ok { + itemsType = str + } + } + } + } + items = &Items{ + Type: itemsType, + } + } + + // Description + desc := v.(map[string]any)["description"] + + // Properties + arguments = append(arguments, ToolArgument{ + Name: name, + Type: argumentType, + Items: items, + Optional: !slices.Contains(tool.InputSchema.Required, name), + Description: argumentDescription(name, desc, tool.Description), + }) + } + + // Annotations + var annotations *ToolAnnotations + if tool.Annotations != (mcp.ToolAnnotation{}) { + annotations = &ToolAnnotations{ + Title: tool.Annotations.Title, + ReadOnlyHint: tool.Annotations.ReadOnlyHint, + DestructiveHint: tool.Annotations.DestructiveHint, + IdempotentHint: tool.Annotations.IdempotentHint, + OpenWorldHint: tool.Annotations.OpenWorldHint, + } + } + + list = append(list, Tool{ + Name: tool.Name, + Description: removeArgs(tool.Description), + Arguments: arguments, + Annotations: annotations, + }) + } + + return list, nil +} + +func removeArgs(input string) string { + var result []string + + for line := range strings.SplitSeq(input, "\n") { + if strings.HasPrefix(strings.ToLower(strings.TrimSpace(line)), "args:") { + break + } + if strings.TrimSpace(line) == "" { + result = append(result, "") + } else { + result = append(result, line) + } + } + + return strings.TrimSpace(strings.Join(result, "\n")) +} + +func argumentDescription(name string, description any, toolDescription string) string { + if description != nil && description != "" { + return fmt.Sprintf("%s", description) + } + return extractDescription(toolDescription, name) +} + +func extractDescription(input string, name string) string { + for line := range strings.SplitSeq(input, "\n") { + line = strings.TrimSpace(line) + + if strings.HasPrefix(strings.ToLower(line), name+":") { + return strings.TrimSpace(strings.TrimPrefix(line, name+":")) + } + } + + return "" +} diff --git a/internal/mcp/requirements.go b/internal/mcp/requirements.go new file mode 100644 index 00000000..2274cbfe --- /dev/null +++ b/internal/mcp/requirements.go @@ -0,0 +1,78 @@ +package mcp + +import ( + "bytes" + "context" + "fmt" + "math/rand" + "os/exec" + "strings" + "syscall" + "time" +) + +func runRequirement(ctx context.Context, requirement string) (func(), string, []string, error) { + if requirement != "neo4j" { + return nil, "", nil, fmt.Errorf("unsupported requirement: %s", requirement) + } + + // Pull first to not count the pull duration in the timeout. + cmdPull := exec.CommandContext(ctx, "docker", "pull", "neo4j") + if err := cmdPull.Run(); err != nil { + return nil, "", nil, fmt.Errorf("failed to pull Neo4j: %w", err) + } + + // Run neo4j as a sidecar. + ctxRequirement, cancel := context.WithCancel(ctx) + + var stdout bytes.Buffer + + containerName := fmt.Sprintf("neo4j-%s", randString(8)) + cmd := exec.CommandContext(ctxRequirement, "docker", "run", "--name", containerName, "--rm", "--init", "-e", "NEO4J_AUTH=none", "neo4j") + cmd.Stdout = &stdout + cmd.Cancel = func() error { + return cmd.Process.Signal(syscall.SIGTERM) + } + + if err := cmd.Start(); err != nil { + cancel() + return nil, "", nil, err + } + + start := time.Now() + started := false +waitStarted: + for { + select { + case <-ctx.Done(): + cancel() + return nil, "", nil, ctx.Err() + case <-time.After(100 * time.Millisecond): + if strings.Contains(stdout.String(), "Started.") { + started = true + break waitStarted + } + if time.Since(start) > 30*time.Second { + break waitStarted + } + } + } + + if !started { + cancel() + return nil, "", nil, fmt.Errorf("failed to start Neo4j: [%s]", stdout.String()) + } + + return cancel, containerName, []string{"NEO4J_URL=bolt://localhost:7687"}, nil +} + +func randString(n int) string { + const letterBytes = "abcdefghijklmnopqrstuvwxyz" + + b := make([]byte, n) + for i := range b { + b[i] = letterBytes[rand.Intn(len(letterBytes))] + } + + return string(b) +} diff --git a/internal/mcp/stdio.go b/internal/mcp/stdio.go new file mode 100644 index 00000000..70771f7a --- /dev/null +++ b/internal/mcp/stdio.go @@ -0,0 +1,233 @@ +package mcp + +import ( + "bufio" + "bytes" + "context" + "encoding/json" + "errors" + "fmt" + "io" + "os" + "os/exec" + "sync" + "sync/atomic" + "syscall" + + "github.com/mark3labs/mcp-go/mcp" +) + +type stdioMCPClient struct { + command string + env []string + args []string + + stdin io.WriteCloser + requestID atomic.Int64 + responses sync.Map + close func() error + initialized atomic.Bool +} + +func newMCPClient(command string, env []string, args ...string) *stdioMCPClient { + return &stdioMCPClient{ + command: command, + env: env, + args: args, + } +} + +func (c *stdioMCPClient) Initialize(ctx context.Context, request mcp.InitializeRequest, debug bool) (*mcp.InitializeResult, error) { + if c.initialized.Load() { + return nil, fmt.Errorf("client already initialized") + } + + ctxCmd, cancel := context.WithCancel(context.WithoutCancel(ctx)) + cmd := exec.CommandContext(ctxCmd, c.command, c.args...) + cmd.Env = c.env + cmd.Cancel = func() error { + return cmd.Process.Signal(syscall.SIGTERM) + } + + var stderr bytes.Buffer + if debug { + cmd.Stderr = io.MultiWriter(&stderr, os.Stderr) + } else { + cmd.Stderr = &stderr + } + + stdin, err := cmd.StdinPipe() + if err != nil { + cancel() + return nil, fmt.Errorf("failed to create stdin pipe: %w", err) + } + c.stdin = stdin + + stdout, err := cmd.StdoutPipe() + if err != nil { + cancel() + return nil, fmt.Errorf("failed to create stdout pipe: %w", err) + } + + if err := cmd.Start(); err != nil { + cancel() + return nil, fmt.Errorf("failed to start command: %w", err) + } + + c.close = func() error { + cancel() + return nil + } + go func() { + cmd.Wait() + cancel() + }() + go func() { + c.readResponses(bufio.NewReader(stdout)) + }() + + var result mcp.InitializeResult + errs := make(chan error) + go func() { + <-ctxCmd.Done() + errs <- errors.New(stderr.String()) + }() + go func() { + errs <- func() error { + params := struct { + ProtocolVersion string `json:"protocolVersion"` + ClientInfo mcp.Implementation `json:"clientInfo"` + Capabilities mcp.ClientCapabilities `json:"capabilities"` + }{ + ProtocolVersion: request.Params.ProtocolVersion, + ClientInfo: request.Params.ClientInfo, + Capabilities: request.Params.Capabilities, + } + + response, err := c.sendRequest(ctx, "initialize", params) + if err != nil { + return err + } + + if err := json.Unmarshal(*response, &result); err != nil { + return fmt.Errorf("failed to unmarshal response: %w", err) + } + + encoder := json.NewEncoder(stdin) + if err := encoder.Encode(mcp.JSONRPCNotification{ + JSONRPC: mcp.JSONRPC_VERSION, + Notification: mcp.Notification{ + Method: "notifications/initialized", + }, + }); err != nil { + return fmt.Errorf("failed to marshal initialized notification: %w", err) + } + + c.initialized.Store(true) + return nil + }() + }() + + return &result, <-errs +} + +func (c *stdioMCPClient) Close() error { + return c.close() +} + +func (c *stdioMCPClient) readResponses(stdout *bufio.Reader) error { + for { + buf, err := stdout.ReadBytes('\n') + if err != nil { + return err + } + + var baseMessage BaseMessage + if err := json.Unmarshal(buf, &baseMessage); err != nil { + continue + } + + if baseMessage.ID == nil { + continue + } + + if ch, ok := c.responses.LoadAndDelete(*baseMessage.ID); ok { + responseChan := ch.(chan RPCResponse) + + if baseMessage.Error != nil { + responseChan <- RPCResponse{ + Error: &baseMessage.Error.Message, + } + } else { + responseChan <- RPCResponse{ + Response: &baseMessage.Result, + } + } + } + } +} + +func (c *stdioMCPClient) sendRequest(ctx context.Context, method string, params any) (*json.RawMessage, error) { + id := c.requestID.Add(1) + responseChan := make(chan RPCResponse, 1) + c.responses.Store(id, responseChan) + + encoder := json.NewEncoder(c.stdin) + if err := encoder.Encode(mcp.JSONRPCRequest{ + JSONRPC: mcp.JSONRPC_VERSION, + ID: id, + Request: mcp.Request{ + Method: method, + }, + Params: params, + }); err != nil { + return nil, fmt.Errorf("failed to encode request: %w", err) + } + + select { + case <-ctx.Done(): + return nil, ctx.Err() + case response := <-responseChan: + if response.Error != nil { + return nil, errors.New(*response.Error) + } + return response.Response, nil + } +} + +func (c *stdioMCPClient) ListTools(ctx context.Context, request mcp.ListToolsRequest) (*mcp.ListToolsResult, error) { + response, err := c.sendRequest(ctx, "tools/list", request.Params) + if err != nil { + return nil, err + } + + var result mcp.ListToolsResult + if err := json.Unmarshal(*response, &result); err != nil { + return nil, fmt.Errorf("failed to unmarshal response: %w", err) + } + + return &result, nil +} + +func (c *stdioMCPClient) ListPrompts(ctx context.Context, request mcp.ListPromptsRequest) (*mcp.ListPromptsResult, error) { + response, err := c.sendRequest(ctx, "prompts/list", request.Params) + if err != nil { + return nil, err + } + + var result mcp.ListPromptsResult + if err := json.Unmarshal(*response, &result); err != nil { + return nil, fmt.Errorf("failed to unmarshal response: %w", err) + } + + return &result, nil +} + +func (c *stdioMCPClient) CallTool(ctx context.Context, request mcp.CallToolRequest) (*mcp.CallToolResult, error) { + response, err := c.sendRequest(ctx, "tools/call", request.Params) + if err != nil { + return nil, err + } + + return mcp.ParseCallToolResult(response) +} diff --git a/internal/mcp/types.go b/internal/mcp/types.go new file mode 100644 index 00000000..bc026e05 --- /dev/null +++ b/internal/mcp/types.go @@ -0,0 +1,46 @@ +package mcp + +import "encoding/json" + +type RPCResponse struct { + Error *string + Response *json.RawMessage +} + +type BaseMessage struct { + JSONRPC string `json:"jsonrpc"` + ID *int64 `json:"id,omitempty"` + Method string `json:"method,omitempty"` + Result json.RawMessage `json:"result,omitempty"` + Error *struct { + Code int `json:"code"` + Message string `json:"message"` + } `json:"error,omitempty"` +} + +type Tool struct { + Name string `json:"name" yaml:"name"` + Description string `json:"description,omitempty" yaml:"description,omitempty"` + Arguments []ToolArgument `json:"arguments,omitempty" yaml:"arguments,omitempty"` + Annotations *ToolAnnotations `json:"annotations,omitempty" yaml:"annotations,omitempty"` +} + +type ToolArgument struct { + Name string `json:"name" yaml:"name"` + Type string `json:"type" yaml:"type"` + Items *Items `json:"items,omitempty" yaml:"items,omitempty"` + Description string `json:"desc" yaml:"desc"` + Optional bool `json:"optional,omitempty" yaml:"optional,omitempty"` +} + +type ToolAnnotations struct { + Title string `json:"title,omitempty" yaml:"title,omitempty"` + ReadOnlyHint bool `json:"readOnlyHint,omitempty" yaml:"readOnlyHint,omitempty"` + DestructiveHint bool `json:"destructiveHint,omitempty" yaml:"destructiveHint,omitempty"` + IdempotentHint bool `json:"idempotentHint,omitempty" yaml:"idempotentHint,omitempty"` + OpenWorldHint bool `json:"openWorldHint,omitempty" yaml:"openWorldHint,omitempty"` +} + +type Items struct { + Type string `json:"type,omitempty" yaml:"type,omitempty"` +} diff --git a/pkg/catalog/tile.go b/pkg/catalog/tile.go new file mode 100644 index 00000000..21780f79 --- /dev/null +++ b/pkg/catalog/tile.go @@ -0,0 +1,177 @@ +package catalog + +import ( + "context" + "fmt" + "strconv" + "strings" + "time" + + "github.com/docker/mcp-registry/internal/licenses" + "github.com/docker/mcp-registry/pkg/github" + "github.com/docker/mcp-registry/pkg/servers" + + "github.com/docker/mcp-registry/pkg/hub" +) + +func ToTile(ctx context.Context, server servers.Server) (Tile, error) { + description := server.About.Description + source := "" + upstream := server.GetUpstream() + owner := "docker" + license := "Apache License 2.0" + githubStars := 0 + + if server.Type == "server" { + client := github.NewFromServer(server) + repository, err := client.GetProjectRepository(ctx, upstream) + if err != nil { + return Tile{}, err + } + if !licenses.IsValid(repository.License) { + panic(fmt.Sprintf("Project %s is licensed under %s which may be incompatible with some tools", upstream, repository.License)) + } + + if description == "" { + description = repository.GetDescription() + } + source = server.GetSourceURL() + owner = repository.Owner.GetLogin() + license = repository.License.GetName() + githubStars = repository.GetStargazersCount() + } + + if description == "" { + return Tile{}, fmt.Errorf("no description found for: %s", server.Name) + } + + var secrets []Secret + for _, s := range server.Config.Secrets { + required := false + if s.Required != nil { + required = *s.Required + } + + secrets = append(secrets, Secret{ + Name: s.Name, + Env: s.Env, + Example: s.Example, + Required: required, + }) + } + + var env []Env + for _, e := range server.Config.Env { + env = append(env, Env{ + Name: e.Name, + Value: e.Value, + }) + } + for name, value := range server.Run.Env { + env = append(env, Env{ + Name: name, + Value: value, + }) + } + + var config []Config + if len(server.Config.Parameters.Properties) > 0 { + if server.Config.Description == "" { + return Tile{}, fmt.Errorf("no config description found for: %s", server.Name) + } + + catalogConfig := Config{ + Name: server.Name, + Description: server.Config.Description, + Type: server.Config.Parameters.Type, + Properties: server.Config.Parameters.Properties, + Required: server.Config.Parameters.Required, + AnyOf: server.Config.Parameters.AnyOf, + } + for i, property := range server.Config.Parameters.Properties { + property.Schema.Default = nil + server.Config.Parameters.Properties[i] = property + if property.Schema.Type == "" { + panic("no type found for: " + property.Name + " in " + server.Name) + } + } + + config = append(config, catalogConfig) + } + + if server.About.Title == "" { + return Tile{}, fmt.Errorf("no title found for: %s", server.Name) + } + + image := server.Image + + if server.Type == "server" && image == "" { + return Tile{}, fmt.Errorf("no image for server: %s", server.Name) + } + if server.Type == "poci" && image != "" { + return Tile{}, fmt.Errorf("pocis don't have images: %s", server.Name) + } + + pullCount := 0 + starCount := 0 + if strings.HasPrefix(image, "mcp/") { + repoInfo, err := hub.GetRepositoryInfo(ctx, server.Image) + if err != nil { + return Tile{}, err + } + pullCount = repoInfo.PullCount + starCount = repoInfo.StarCount + } + + meta := Metadata{ + Category: server.Meta.Category, + Tags: server.Meta.Tags, + Owner: owner, + License: license, + GitHubStars: githubStars, + Pulls: pullCount, + Stars: starCount, + } + + var oauth OAuth + if len(server.OAuth) > 0 { + for _, provider := range server.OAuth { + oauth.Providers = append(oauth.Providers, OAuthProvider{ + Provider: provider.Provider, + Secret: provider.Secret, + Env: provider.Env, + }) + } + } + + dateAdded := time.Now().Format(time.RFC3339) + + return Tile{ + Description: addDot(strings.TrimSpace(strings.ReplaceAll(description, "\n", " "))), + Title: server.About.Title, + Type: server.Type, + Image: image, + DateAdded: &dateAdded, + ReadmeURL: "http://desktop.docker.com/mcp/catalog/v" + strconv.Itoa(Version) + "/readme/" + server.Name + ".md", + ToolsURL: "http://desktop.docker.com/mcp/catalog/v" + strconv.Itoa(Version) + "/tools/" + server.Name + ".json", + Source: source, + Upstream: upstream, + Icon: server.About.Icon, + Secrets: secrets, + Env: env, + Command: server.Run.Command, + Volumes: server.Run.Volumes, + DisableNetwork: server.Run.DisableNetwork, + AllowHosts: server.Run.AllowHosts, + Config: config, + Metadata: meta, + OAuth: oauth, + }, nil +} + +func addDot(text string) string { + if strings.HasSuffix(text, ".") { + return text + } + return text + "." +} diff --git a/pkg/catalog/types.go b/pkg/catalog/types.go new file mode 100644 index 00000000..0ece2c94 --- /dev/null +++ b/pkg/catalog/types.go @@ -0,0 +1,176 @@ +package catalog + +import ( + "bytes" + "encoding/json" + + "github.com/docker/mcp-registry/pkg/servers" + "gopkg.in/yaml.v3" +) + +const ( + Version = 2 + Name = "docker-mcp" + DisplayName = "Docker MCP Catalog" +) + +type TileWithOrder struct { + Tile `json:",inline" yaml:",inline"` + Order int `json:"order" yaml:"order"` +} + +type TopLevel struct { + Version int `json:"version" yaml:"version"` + Name string `json:"name" yaml:"name"` + DisplayName string `json:"displayName" yaml:"displayName"` + Registry TileList `json:"registry" yaml:"registry"` +} + +type TileList []TileEntry + +func (tl *TileList) UnmarshalYAML(value *yaml.Node) error { + for i := 0; i < len(value.Content); i += 2 { + keyNode := value.Content[i] + valNode := value.Content[i+1] + + var name string + if err := keyNode.Decode(&name); err != nil { + return err + } + + var tile Tile + if err := valNode.Decode(&tile); err != nil { + return err + } + + *tl = append(*tl, TileEntry{ + Name: name, + Tile: tile, + }) + } + return nil +} + +func (tl TileList) MarshalYAML() (interface{}, error) { + mapNode := &yaml.Node{ + Kind: yaml.MappingNode, + Content: []*yaml.Node{}, + } + + for _, entry := range tl { + // Key node: the tile name + keyNode := &yaml.Node{ + Kind: yaml.ScalarNode, + Value: entry.Name, + } + + // Value node: marshal the Tile + valNode := &yaml.Node{} + if err := valNode.Encode(entry.Tile); err != nil { + return nil, err + } + + mapNode.Content = append(mapNode.Content, keyNode, valNode) + } + + return mapNode, nil +} + +func (tl TileList) MarshalJSON() ([]byte, error) { + var buf bytes.Buffer + buf.WriteByte('{') + for i, entry := range tl { + if i > 0 { + buf.WriteByte(',') + } + keyBytes, err := json.Marshal(entry.Name) + if err != nil { + return nil, err + } + valBytes, err := json.Marshal(entry.Tile) + if err != nil { + return nil, err + } + buf.Write(keyBytes) + buf.WriteByte(':') + buf.Write(valBytes) + } + buf.WriteByte('}') + return buf.Bytes(), nil +} + +type TileEntry struct { + Name string `json:"name" yaml:"name"` + Tile Tile `json:",inline" yaml:",inline"` +} + +type Tile struct { + Name string `json:"name,omitempty" yaml:"name,omitempty"` + Description string `json:"description" yaml:"description"` + Title string `json:"title" yaml:"title"` + Type string `json:"type" yaml:"type"` + DateAdded *string `json:"dateAdded,omitempty" yaml:"dateAdded,omitempty"` + Image string `json:"image,omitempty" yaml:"image,omitempty"` + // TODO(dga): Remove it when the UI is ready. It's not used but it's still validated. + Ref string `json:"ref" yaml:"ref"` + ReadmeURL string `json:"readme,omitempty" yaml:"readme,omitempty"` + ToolsURL string `json:"toolsUrl,omitempty" yaml:"toolsUrl,omitempty"` + // TODO(dga): The UI ignores tiles without a source. An empty one is ok. Put back omitempty when this is fixed + // Source string `json:"source,omitempty" yaml:"source,omitempty"` + Source string `json:"source" yaml:"source"` + Upstream string `json:"upstream,omitempty" yaml:"upstream,omitempty"` + Icon string `json:"icon" yaml:"icon"` + Tools []servers.Tool `json:"tools" yaml:"tools"` + Secrets []Secret `json:"secrets,omitempty" yaml:"secrets,omitempty"` + Env []Env `json:"env,omitempty" yaml:"env,omitempty"` + Command []string `json:"command,omitempty" yaml:"command,omitempty"` + Volumes []string `json:"volumes,omitempty" yaml:"volumes,omitempty"` + DisableNetwork bool `json:"disableNetwork,omitempty" yaml:"disableNetwork,omitempty"` + AllowHosts []string `json:"allowHosts,omitempty" yaml:"allowHosts,omitempty"` + Prompts int `json:"prompts" yaml:"prompts"` + Resources map[string]any `json:"resources" yaml:"resources"` + Config []Config `json:"config,omitempty" yaml:"config,omitempty"` + Metadata Metadata `json:"metadata,omitempty" yaml:"metadata,omitempty"` + OAuth OAuth `json:"oauth,omitempty" yaml:"oauth,omitempty"` +} + +type Metadata struct { + Pulls int `json:"pulls,omitempty" yaml:"pulls,omitempty"` + Stars int `json:"stars,omitempty" yaml:"stars,omitempty"` + GitHubStars int `json:"githubStars,omitempty" yaml:"githubStars,omitempty"` + Category string `json:"category,omitempty" yaml:"category,omitempty"` + Tags []string `json:"tags,omitempty" yaml:"tags,omitempty"` + License string `json:"license,omitempty" yaml:"license,omitempty"` + Owner string `json:"owner,omitempty" yaml:"owner,omitempty"` +} + +type OAuth struct { + Providers []OAuthProvider `json:"providers,omitempty" yaml:"providers,omitempty"` +} + +type OAuthProvider struct { + Provider string `json:"provider" yaml:"provider"` + Secret string `json:"secret,omitempty" yaml:"secret,omitempty"` + Env string `json:"env,omitempty" yaml:"env,omitempty"` +} + +type Config struct { + Name string `json:"name" yaml:"name"` + Description string `json:"description" yaml:"description"` + Type string `json:"type" yaml:"type"` + Properties servers.SchemaList `json:"properties,omitempty" yaml:"properties,omitempty"` + AnyOf []servers.AnyOf `json:"anyOf,omitempty" yaml:"anyOf,omitempty"` + Required []string `json:"required,omitempty" yaml:"required,omitempty"` +} + +type Secret struct { + Name string `json:"name" yaml:"name"` + Env string `json:"env" yaml:"env"` + Example string `json:"example" yaml:"example"` + Required bool `json:"required,omitempty" yaml:"required,omitempty"` +} + +type Env struct { + Name string `json:"name" yaml:"name"` + Value string `json:"value" yaml:"value"` +} diff --git a/pkg/catalog/yaml.go b/pkg/catalog/yaml.go new file mode 100644 index 00000000..6e63c5c2 --- /dev/null +++ b/pkg/catalog/yaml.go @@ -0,0 +1,19 @@ +package catalog + +import ( + "bytes" + "os" + + "gopkg.in/yaml.v3" +) + +func WriteYaml(filename string, topLevel TopLevel) error { + var buf bytes.Buffer + encoder := yaml.NewEncoder(&buf) + encoder.SetIndent(2) + if err := encoder.Encode(topLevel); err != nil { + return err + } + + return os.WriteFile(filename, buf.Bytes(), 0644) +} diff --git a/pkg/github/github.go b/pkg/github/github.go new file mode 100644 index 00000000..58b3db7f --- /dev/null +++ b/pkg/github/github.go @@ -0,0 +1,177 @@ +package github + +import ( + "context" + "errors" + "fmt" + "net/url" + "os" + "strings" + "time" + + "github.com/docker/mcp-registry/pkg/servers" + "github.com/google/go-github/v70/github" +) + +func NewFromServer(server servers.Server) *Client { + // A couple of public GitHub repos can't be accessed with authentication if running on GHActions... + // See https://github.com/xaf/omni/issues/670 + if server.Name == "shopify" || server.Name == "heroku" { + return NewUnauthenticated() + } + return New() +} + +func New() *Client { + return &Client{ + gh: github.NewClient(nil).WithAuthToken(os.Getenv("GITHUB_TOKEN")), + } +} + +func NewUnauthenticated() *Client { + return &Client{ + gh: github.NewClient(nil), + } +} + +type Client struct { + gh *github.Client +} + +func (c *Client) GetProjectRepository(ctx context.Context, project string) (*github.Repository, error) { + owner, repo, err := extractOrgAndProject(project) + if err != nil { + return nil, err + } + + for { + repository, _, err := c.gh.Repositories.Get(ctx, owner, repo) + if sleepOnRateLimitError(ctx, err) { + continue + } + + return repository, err + } +} + +func (c *Client) GetCommitSHA1(ctx context.Context, project, branch string) (string, error) { + owner, repo, err := extractOrgAndProject(project) + if err != nil { + return "", err + } + + for { + sha, _, err := c.gh.Repositories.GetCommitSHA1(ctx, owner, repo, branch, "") + if sleepOnRateLimitError(ctx, err) { + continue + } + + return sha, err + } +} + +func (c *Client) FindIcon(ctx context.Context, projectURL string) (string, error) { + repository, err := c.GetProjectRepository(ctx, projectURL) + if err != nil { + return "", err + } + + return repository.Owner.GetAvatarURL(), nil +} + +func sleepOnRateLimitError(ctx context.Context, err error) bool { + var rateLimitErr *github.RateLimitError + if !errors.As(err, &rateLimitErr) { + return false + } + + sleepDelay := time.Until(rateLimitErr.Rate.Reset.Time) + fmt.Printf("Rate limit exceeded, waiting %d seconds for reset...\n", int64(sleepDelay.Seconds())) + + select { + case <-ctx.Done(): + case <-time.After(sleepDelay): + } + + return true +} + +func extractOrgAndProject(rawURL string) (string, string, error) { + parsedURL, err := url.Parse(rawURL) + if err != nil { + return "", "", err + } + + parts := strings.Split(strings.Trim(parsedURL.Path, "/"), "/") + if len(parts) < 2 { + return "", "", fmt.Errorf("URL path doesn't contain enough segments: %s", rawURL) + } + + org := parts[0] + project := parts[1] + + return org, project, nil +} + +func FindTags(topics []string) []string { + if len(topics) == 0 { + return []string{"TODO"} + } + + var tags []string + for _, topic := range topics { + if topic != "mcp" && topic != "mcp-server" { + tags = append(tags, topic) + } + } + + return tags +} + +type DetectedInfo struct { + ProjectURL string + Branch string + Directory string +} + +func DetectBranchAndDirectory(projectURL string, repository *github.Repository) (DetectedInfo, error) { + u, err := url.Parse(projectURL) + if err != nil { + return DetectedInfo{}, err + } + + var branch string + var directory string + parts := strings.Split(strings.Trim(u.Path, "/"), "/") + if len(parts) >= 4 && parts[2] == "tree" { + projectURL = u.Scheme + "://" + u.Host + "/" + parts[0] + "/" + parts[1] + if parts[3] == "main" { // Should match with any valid branch + branch = parts[3] + directory = strings.Join(parts[4:], "/") + } else { + branch = strings.Join(parts[3:], "/") + } + } else if len(parts) >= 4 && parts[2] == "blob" { + projectURL = u.Scheme + "://" + u.Host + "/" + parts[0] + "/" + parts[1] + if parts[3] == "main" { // Should match with any valid branch + branch = parts[3] + directory = strings.Join(parts[4:], "/") + } else { + branch = strings.Join(parts[3:], "/") + } + } else if len(parts) == 4 && parts[2] == "pull" { + projectURL = u.Scheme + "://" + u.Host + "/" + parts[0] + "/" + parts[1] + branch = "refs/pull/" + parts[3] + "/merge" + } else if len(parts) == 4 && parts[2] == "commit" { + projectURL = u.Scheme + "://" + u.Host + "/" + parts[0] + "/" + parts[1] + branch = parts[3] + } else { + branch = repository.GetDefaultBranch() + } + + return DetectedInfo{ + ProjectURL: projectURL, + Branch: branch, + Directory: directory, + }, nil +} diff --git a/pkg/hub/hub.go b/pkg/hub/hub.go new file mode 100644 index 00000000..e0cab49d --- /dev/null +++ b/pkg/hub/hub.go @@ -0,0 +1,29 @@ +package hub + +import ( + "context" + "encoding/json" + "net/http" +) + +func GetRepositoryInfo(ctx context.Context, repo string) (*repositoryResponse, error) { + req, err := http.NewRequestWithContext(ctx, http.MethodGet, "https://hub.docker.com/v2/repositories/"+repo+"/", nil) + if err != nil { + return nil, err + } + + req.Header.Set("Content-Type", "application/json") + + response, err := http.DefaultClient.Do(req) + if err != nil { + return nil, err + } + defer response.Body.Close() + + var repoResp repositoryResponse + if err := json.NewDecoder(response.Body).Decode(&repoResp); err != nil { + return nil, err + } + + return &repoResp, nil +} diff --git a/pkg/hub/types.go b/pkg/hub/types.go new file mode 100644 index 00000000..f16bcf1f --- /dev/null +++ b/pkg/hub/types.go @@ -0,0 +1,7 @@ +package hub + +type repositoryResponse struct { + PullCount int `json:"pull_count"` + StarCount int `json:"star_count"` + LastUpdated string `json:"last_updated"` +} diff --git a/pkg/servers/server.go b/pkg/servers/server.go new file mode 100644 index 00000000..132ced63 --- /dev/null +++ b/pkg/servers/server.go @@ -0,0 +1,85 @@ +package servers + +import "strings" + +func (s *Server) GetContext() string { + base := s.Source.Project + ".git" + + if s.GetBranch() != "main" { + base += "#" + s.Source.Branch + } else { + base += "#" + } + + if s.Source.Directory != "" && s.Source.Directory != "." { + base += ":" + s.Source.Directory + } + + return strings.TrimSuffix(base, "#") +} + +func (s *Server) GetSourceURL() string { + source := s.Source.Project + "/tree/" + s.GetBranch() + if s.Source.Directory != "" { + source += "/" + s.Source.Directory + } + return source +} + +func (s *Server) GetUpstream() string { + if s.Source.Upstream != "" { + return s.Source.Upstream + } + return s.Source.Project +} + +func (s *Server) GetBranch() string { + if s.Source.Branch == "" { + return "main" + } + return s.Source.Branch +} + +func (s *Server) GetDockerfileUrl() string { + base := s.Source.Project + "/blob/" + s.GetBranch() + if s.Source.Directory != "" { + base += "/" + s.Source.Directory + } + return base + "/" + s.GetDockerfile() +} + +func (s *Server) GetDockerfile() string { + if s.Source.Dockerfile == "" { + return "Dockerfile" + } + return s.Source.Dockerfile +} + +func CreateSchema(server string, env []Env) ([]Env, Schema) { + schema := Schema{} + if len(env) == 0 { + return nil, schema + } + + var updatedEnv []Env + schema.Type = "object" + for _, e := range env { + + name := strings.TrimPrefix(strings.ToLower(e.Name), strings.ToLower(server)+"_") + + schema.Properties = append(schema.Properties, SchemaEntry{ + Name: name, + Schema: Schema{ + Type: "string", + }, + }) + + updatedEnv = append(updatedEnv, Env{ + Name: e.Name, + Value: "{{" + name + "}}", + Example: e.Example, + }) + } + + return updatedEnv, schema +} diff --git a/pkg/servers/types.go b/pkg/servers/types.go new file mode 100644 index 00000000..5d87c10d --- /dev/null +++ b/pkg/servers/types.go @@ -0,0 +1,174 @@ +package servers + +import ( + "gopkg.in/yaml.v3" +) + +type Server struct { + Name string `yaml:"name" json:"name"` + Image string `yaml:"image,omitempty" json:"image,omitempty"` + Type string `yaml:"type" json:"type"` + Meta Meta `yaml:"meta,omitempty" json:"meta,omitempty"` + About About `yaml:"about,omitempty" json:"about,omitempty"` + Source Source `yaml:"source,omitempty" json:"source,omitempty"` + Run Run `yaml:"run,omitempty" json:"run,omitempty"` + Config Config `yaml:"config,omitempty" json:"config,omitempty"` + OAuth []OAuthProvider `yaml:"oauth,omitempty" json:"oauth,omitempty"` + Tools []Tool `yaml:"tools,omitempty" json:"tools,omitempty"` + Requirement string `yaml:"requirement,omitempty" json:"requirement,omitempty"` +} + +type Secret struct { + Name string `yaml:"name" json:"name"` + Env string `yaml:"env" json:"env"` + Example string `yaml:"example,omitempty" json:"example,omitempty"` + Required *bool `yaml:"required,omitempty" json:"required,omitempty"` +} + +type Env struct { + Name string `yaml:"name" json:"name"` + Example any `yaml:"example,omitempty" json:"example,omitempty"` + Value string `yaml:"value,omitempty" json:"value,omitempty"` +} + +type AnyOf struct { + Required []string `yaml:"required,omitempty" json:"required,omitempty"` +} + +type Schema struct { + Type string `yaml:"type" json:"type"` + Description string `yaml:"description,omitempty" json:"description,omitempty"` + Properties SchemaList `yaml:"properties,omitempty" json:"properties,omitempty"` + Required []string `yaml:"required,omitempty" json:"required,omitempty"` + Items Items `yaml:"items,omitempty" json:"items,omitempty"` + AnyOf []AnyOf `yaml:"anyOf,omitempty" json:"anyOf,omitempty"` + Default any `yaml:"default,omitempty" json:"default,omitempty"` +} + +type Items struct { + Type string `yaml:"type" json:"type"` +} + +type About struct { + Title string `yaml:"title,omitempty" json:"title,omitempty"` + Description string `yaml:"description,omitempty" json:"description,omitempty"` + Icon string `yaml:"icon,omitempty" json:"icon,omitempty"` +} + +type Source struct { + Project string `yaml:"project,omitempty" json:"project,omitempty"` + Upstream string `yaml:"upstream,omitempty" json:"upstream,omitempty"` + Branch string `yaml:"branch,omitempty" json:"branch,omitempty"` + Directory string `yaml:"directory,omitempty" json:"directory,omitempty"` + Dockerfile string `yaml:"dockerfile,omitempty" json:"dockerfile,omitempty"` +} + +type Run struct { + Command []string `yaml:"command,omitempty" json:"command,omitempty"` + Volumes []string `yaml:"volumes,omitempty" json:"volumes,omitempty"` + Env map[string]string `yaml:"env,omitempty" json:"env,omitempty"` + AllowHosts []string `yaml:"allowHosts,omitempty" json:"allowHosts,omitempty"` + DisableNetwork bool `yaml:"disableNetwork,omitempty" json:"disableNetwork,omitempty"` +} + +type Config struct { + Description string `yaml:"description,omitempty" json:"description,omitempty"` + Secrets []Secret `yaml:"secrets,omitempty" json:"secrets,omitempty"` + Env []Env `yaml:"env,omitempty" json:"env,omitempty"` + Parameters Schema `yaml:"parameters,omitempty" json:"parameters,omitempty"` + AnyOf []AnyOf `yaml:"anyOf,omitempty" json:"anyOf,omitempty"` +} + +type Tool struct { + Name string `yaml:"name" json:"name"` + Description string `yaml:"description,omitempty" json:"description,omitempty"` + Parameters Parameters `yaml:"parameters,omitempty" json:"parameters,omitempty"` + Container Container `yaml:"container,omitempty" json:"container,omitempty"` +} + +type Parameters struct { + Type string `yaml:"type" json:"type"` + Properties Properties `yaml:"properties" json:"properties"` + Required []string `yaml:"required" json:"required"` +} + +type Properties map[string]Property + +type Property struct { + Type string `yaml:"type" json:"type"` + Description string `yaml:"description" json:"description"` + Items *Items `yaml:"items,omitempty" json:"items,omitempty"` +} + +type Container struct { + Image string `yaml:"image,omitempty" json:"image,omitempty"` + Command []string `yaml:"command,omitempty" json:"command,omitempty"` + Volumes []string `yaml:"volumes,omitempty" json:"volumes,omitempty"` +} + +type Meta struct { + Category string `yaml:"category,omitempty" json:"category,omitempty"` + Tags []string `yaml:"tags,omitempty" json:"tags,omitempty"` + Highlighted bool `yaml:"highlighted,omitempty" json:"highlighted,omitempty"` +} + +type OAuthProvider struct { + Provider string `yaml:"provider,omitempty" json:"provider,omitempty"` + Secret string `yaml:"secret,omitempty" json:"secret,omitempty"` + Env string `yaml:"env,omitempty" json:"env,omitempty"` +} + +type SchemaEntry struct { + Schema Schema `yaml:",inline"` + Name string `yaml:"name"` +} + +type SchemaList []SchemaEntry + +func (tl *SchemaList) UnmarshalYAML(value *yaml.Node) error { + for i := 0; i < len(value.Content); i += 2 { + keyNode := value.Content[i] + valNode := value.Content[i+1] + + var name string + if err := keyNode.Decode(&name); err != nil { + return err + } + + var schema Schema + if err := valNode.Decode(&schema); err != nil { + return err + } + + *tl = append(*tl, SchemaEntry{ + Name: name, + Schema: schema, + }) + } + return nil +} + +func (tl SchemaList) MarshalYAML() (interface{}, error) { + mapNode := &yaml.Node{ + Kind: yaml.MappingNode, + Content: []*yaml.Node{}, + } + + for _, entry := range tl { + // Key node: the tile name + keyNode := &yaml.Node{ + Kind: yaml.ScalarNode, + Value: entry.Name, + } + + // Value node: marshal the Schema + valNode := &yaml.Node{} + if err := valNode.Encode(entry.Schema); err != nil { + return nil, err + } + + mapNode.Content = append(mapNode.Content, keyNode, valNode) + } + + return mapNode, nil +} diff --git a/pkg/servers/yaml.go b/pkg/servers/yaml.go new file mode 100644 index 00000000..086fb7e7 --- /dev/null +++ b/pkg/servers/yaml.go @@ -0,0 +1,24 @@ +package servers + +import ( + "fmt" + "os" + + "gopkg.in/yaml.v3" +) + +func Read(path string) (Server, error) { + file, err := os.Open(path) + if err != nil { + return Server{}, err + } + defer file.Close() + + var server Server + decoder := yaml.NewDecoder(file) + if err := decoder.Decode(&server); err != nil { + return Server{}, fmt.Errorf("failed to decode server file %s: %w", path, err) + } + + return server, nil +} diff --git a/scripts/new-server.sh b/scripts/new-server.sh deleted file mode 100755 index cf20c657..00000000 --- a/scripts/new-server.sh +++ /dev/null @@ -1,65 +0,0 @@ -#!/bin/bash - -set -e - - -NEW_SERVER_NAME=$(echo "$1" | tr '[:upper:]' '[:lower:]') -NEW_SERVER_NAME_UPPER=$(echo "$1" | tr '[:lower:]' '[:upper:]') - -PATH_TO_SERVER="./servers/$NEW_SERVER_NAME" -CATEGORY=$(echo "$2" | tr '[:upper:]' '[:lower:]') - -# IF IMAGE_NAME IS NOT PROVIDED, USE THE DEFAULT ONE -IMAGE_NAME=${IMAGE_NAME:-"mcp/$NEW_SERVER_NAME"} - - -if [[ -f "$NEW_SERVER_NAME" ]]; then - echo "❌ File already exists: $NEW_SERVER_NAME" - exit 1 -fi - -echo "Creating new server: $NEW_SERVER_NAME" - -mkdir -p "$PATH_TO_SERVER" - -echo "Server created successfully" - -echo "Creating server.yaml" - -echo "server: - name: $NEW_SERVER_NAME - image: $IMAGE_NAME -type: server -meta: - category: $CATEGORY - tags: - - $CATEGORY - highlighted: false -about: - title: $NEW_SERVER_NAME - icon: https://avatars.githubusercontent.com/u/182288589?s=200&v=4 -source: - project: $3 - branch: main -# config: -# description: "TODO" -# secrets: -# - name: $NEW_SERVER_NAME.secret_name -# env: $NEW_SERVER_NAME_UPPER -# example: "TODO" -# env: -# - name: ENV_VAR_NAME -# example: "TODO" -# value: '{{$NEW_SERVER_NAME.env_var_name}}' -# parameters: -# type: object -# properties: -# param_name: -# type: string -# required: -# - param_name - - " > "$PATH_TO_SERVER/server.yaml" - -echo "$NEW_SERVER_NAME created successfully" -