Merge pull request #1411 from janhq/j/add-read-remote-yaml

chore: add read remote yaml function

Author: namchuai

.gitignore

@@ -19,4 +19,5 @@ platform/package-lock.json
 .vscode
 platform/command
 platform/src/infrastructure/commanders/test/test_data
-**/vcpkg_installed
+**/vcpkg_installed
+engine/test.db

engine/commands/model_list_cmd.cc

@@ -12,7 +12,6 @@ namespace commands {
 void ModelListCmd::Exec() {
   namespace fs = std::filesystem;
   namespace fmu = file_manager_utils;
-  auto models_path = file_manager_utils::GetModelsContainerPath();
   cortex::db::Models modellist_handler;
   config::YamlHandler yaml_handler;
   tabulate::Table table;

engine/commands/run_cmd.h

@@ -1,6 +1,6 @@
 #pragma once
+
 #include <string>
-#include "nlohmann/json.hpp"
 #include "services/engine_service.h"
 #include "services/model_service.h"
 

engine/database/models.cc

@@ -1,12 +1,8 @@
 #include "models.h"
 #include <algorithm>
-#include <filesystem>
-#include <fstream>
 #include <iostream>
 #include <sstream>
-#include <stdexcept>
 #include "database.h"
-#include "utils/file_manager_utils.h"
 #include "utils/result.hpp"
 #include "utils/scope_exit.h"
 
@@ -52,8 +48,7 @@ bool Models::IsUnique(const std::vector<ModelEntry>& entries,
   return std::none_of(
       entries.begin(), entries.end(), [&](const ModelEntry& entry) {
         return entry.model == model_id || entry.model_alias == model_id ||
-               entry.model == model_alias ||
-               entry.model_alias == model_alias;
+               entry.model == model_alias || entry.model_alias == model_alias;
       });
 }
 
@@ -186,8 +181,7 @@ cpp::result<bool, std::string> Models::AddModelEntry(ModelEntry new_entry,
       std::cout << "Test: " << model_list.error();
       return cpp::fail(model_list.error());
     }
-    if (IsUnique(model_list.value(), new_entry.model,
-                 new_entry.model_alias)) {
+    if (IsUnique(model_list.value(), new_entry.model, new_entry.model_alias)) {
       if (use_short_alias) {
         new_entry.model_alias =
             GenerateShortenedAlias(new_entry.model, model_list.value());

engine/utils/curl_utils.h

@@ -0,0 +1,77 @@
+#include <curl/curl.h>
+#include <nlohmann/json.hpp>
+#include <string>
+#include "utils/logging_utils.h"
+#include "utils/result.hpp"
+#include "yaml-cpp/yaml.h"
+
+namespace curl_utils {
+namespace {
+size_t WriteCallback(void* contents, size_t size, size_t nmemb,
+                     std::string* output) {
+  size_t totalSize = size * nmemb;
+  output->append((char*)contents, totalSize);
+  return totalSize;
+}
+}  // namespace
+
+inline cpp::result<std::string, std::string> SimpleGet(const std::string& url) {
+  CURL* curl;
+  CURLcode res;
+  std::string readBuffer;
+
+  // Initialize libcurl
+  curl_global_init(CURL_GLOBAL_DEFAULT);
+  curl = curl_easy_init();
+
+  if (!curl) {
+    return cpp::fail("Failed to init CURL");
+  }
+  curl_easy_setopt(curl, CURLOPT_URL, url.c_str());
+
+  // Set write function callback and data buffer
+  curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, WriteCallback);
+  curl_easy_setopt(curl, CURLOPT_WRITEDATA, &readBuffer);
+
+  // Perform the request
+  res = curl_easy_perform(curl);
+
+  if (res != CURLE_OK) {
+    return cpp::fail("CURL request failed: " +
+                     static_cast<std::string>(curl_easy_strerror(res)));
+  }
+
+  curl_easy_cleanup(curl);
+  return readBuffer;
+}
+
+inline cpp::result<YAML::Node, std::string> ReadRemoteYaml(
+    const std::string& url) {
+  auto result = SimpleGet(url);
+  if (result.has_error()) {
+    return cpp::fail(result.error());
+  }
+
+  try {
+    return YAML::Load(result.value());
+  } catch (const std::exception& e) {
+    return cpp::fail("YAML from " + url +
+                     " parsing error: " + std::string(e.what()));
+  }
+}
+
+inline cpp::result<nlohmann::json, std::string> SimpleGetJson(
+    const std::string& url) {
+  auto result = SimpleGet(url);
+  if (result.has_error()) {
+    return cpp::fail(result.error());
+  }
+
+  try {
+    return nlohmann::json::parse(result.value());
+  } catch (const std::exception& e) {
+    return cpp::fail("JSON from " + url +
+                     " parsing error: " + std::string(e.what()));
+  }
+}
+}  // namespace curl_utils

engine/utils/huggingface_utils.h

@@ -1,9 +1,9 @@
 #pragma once
 
-#include <httplib.h>
 #include <optional>
 #include <string>
 #include <vector>
+#include "utils/curl_utils.h"
 #include "utils/json.hpp"
 #include "utils/result.hpp"
 #include "utils/url_parser.h"
@@ -58,17 +58,13 @@ GetModelRepositoryBranches(const std::string& author,
       .host = kHuggingfaceHost,
       .pathParams = {"api", "models", author, modelName, "refs"}};
 
-  httplib::Client cli(url_obj.GetProtocolAndHost());
-  auto res = cli.Get(url_obj.GetPathAndQuery());
-  if (res->status != httplib::StatusCode::OK_200) {
+  auto result = curl_utils::SimpleGetJson(url_obj.ToFullPath());
+  if (result.has_error()) {
     return cpp::fail("Failed to get model repository branches: " + author +
                      "/" + modelName);
   }
 
-  using json = nlohmann::json;
-  auto body = json::parse(res->body);
-  auto branches_json = body["branches"];
-
+  auto branches_json = result.value()["branches"];
   std::vector<HuggingFaceBranch> branches{};
 
   for (const auto& branch : branches_json) {
@@ -94,15 +90,13 @@ GetHuggingFaceModelRepoInfo(const std::string& author,
                       .host = kHuggingfaceHost,
                       .pathParams = {"api", "models", author, modelName}};
 
-  httplib::Client cli(url_obj.GetProtocolAndHost());
-  auto res = cli.Get(url_obj.GetPathAndQuery());
-  if (res->status != httplib::StatusCode::OK_200) {
+  auto result = curl_utils::SimpleGetJson(url_obj.ToFullPath());
+  if (result.has_error()) {
     return cpp::fail("Failed to get model repository info: " + author + "/" +
                      modelName);
   }
 
-  using json = nlohmann::json;
-  auto body = json::parse(res->body);
+  auto body = result.value();
 
   std::optional<HuggingFaceGgufInfo> gguf = std::nullopt;
   auto gguf_info = body["gguf"];