fix: add support image url for jan (#1798)

Author: namchuai

engine/common/message.h

@@ -137,6 +137,27 @@ struct Message : JsonSerializable {
         if (root["content"].isArray() && !root["content"].empty()) {
           if (root["content"][0]["type"].asString() == "text") {
             message.content = ParseContents(std::move(root["content"])).value();
+          } else if (root["content"][0]["type"].asString() == "image") {
+            // deprecated, for supporting jan and should be removed in the future
+            auto text_str = root["content"][0]["text"]["value"].asString();
+            auto img_url =
+                root["content"][0]["text"]["annotations"][0].asString();
+            auto text_content = std::make_unique<OpenAi::TextContent>();
+            {
+              auto text = OpenAi::Text();
+              auto empty_annotations =
+                  std::vector<std::unique_ptr<Annotation>>();
+              text.value = std::move(text_str);
+              text.annotations = std::move(empty_annotations);
+              text_content->text = std::move(text);
+            }
+
+            auto image_url_obj = OpenAi::ImageUrl(img_url, "auto");
+            auto image_url_content = std::make_unique<OpenAi::ImageUrlContent>(
+                "image_url", std::move(image_url_obj));
+
+            message.content.push_back(std::move(text_content));
+            message.content.push_back(std::move(image_url_content));
           } else {
             // deprecated, for supporting jan and should be removed in the future
             // check if annotations is empty

engine/common/message_content_image_url.h

@@ -4,14 +4,21 @@
 
 namespace OpenAi {
 
-struct ImageUrl {
-  // The external URL of the image, must be a supported image types: jpeg, jpg, png, gif, webp.
+struct ImageUrl : public JsonSerializable {
+  /**
+   * The external URL of the image, must be a supported image types:
+   * jpeg, jpg, png, gif, webp.
+   */
   std::string url;
 
-  // Specifies the detail level of the image. low uses fewer tokens, you can opt in to high resolution using high. Default value is auto
+  /**
+   * Specifies the detail level of the image. low uses fewer tokens, you
+   * can opt in to high resolution using high. Default value is auto
+   */
   std::string detail;
 
-  ImageUrl() = default;
+  ImageUrl(const std::string& url, const std::string& detail = "auto")
+      : url{url}, detail{detail} {}
 
   ImageUrl(ImageUrl&&) noexcept = default;
 
@@ -20,13 +27,25 @@ struct ImageUrl {
   ImageUrl(const ImageUrl&) = delete;
 
   ImageUrl& operator=(const ImageUrl&) = delete;
+
+  cpp::result<Json::Value, std::string> ToJson() override {
+    try {
+      Json::Value root;
+      root["url"] = url;
+      root["detail"] = detail;
+      return root;
+    } catch (const std::exception& e) {
+      return cpp::fail(std::string("ToJson failed: ") + e.what());
+    }
+  }
 };
 
 // References an image URL in the content of a message.
 struct ImageUrlContent : Content {
 
   // The type of the content part.
-  ImageUrlContent(const std::string& type) : Content(type) {}
+  explicit ImageUrlContent(const std::string& type, ImageUrl&& image_url)
+      : Content(type), image_url{std::move(image_url)} {}
 
   ImageUrlContent(ImageUrlContent&&) noexcept = default;
 
@@ -38,18 +57,18 @@ struct ImageUrlContent : Content {
 
   ImageUrl image_url;
 
+  ~ImageUrlContent() override = default;
+
   static cpp::result<ImageUrlContent, std::string> FromJson(
       Json::Value&& json) {
     if (json.empty()) {
       return cpp::fail("Json string is empty");
     }
 
     try {
-      ImageUrlContent content{"image_url"};
-      ImageUrl image_url;
-      image_url.url = std::move(json["image_url"]["url"].asString());
-      image_url.detail = std::move(json["image_url"]["detail"].asString());
-      content.image_url = std::move(image_url);
+      auto image_url = ImageUrl(json["image_url"]["url"].asString(),
+                                json["image_url"]["detail"].asString());
+      ImageUrlContent content{"image_url", std::move(image_url)};
       return content;
     } catch (const std::exception& e) {
       return cpp::fail(std::string("FromJson failed: ") + e.what());
@@ -60,8 +79,7 @@ struct ImageUrlContent : Content {
     try {
       Json::Value json;
       json["type"] = type;
-      json["image_url"]["url"] = image_url.url;
-      json["image_url"]["detail"] = image_url.detail;
+      json["image_url"] = image_url.ToJson().value();
       return json;
     } catch (const std::exception& e) {
       return cpp::fail(std::string("ToJson failed: ") + e.what());

engine/common/message_content_text.h

@@ -122,7 +122,6 @@ struct FilePathWrapper : Annotation {
 
 struct Text : JsonSerializable {
   // The data that makes up the text.
-
   Text() = default;
 
   Text(Text&&) noexcept = default;
@@ -214,6 +213,8 @@ struct TextContent : Content {
 
   Text text;
 
+  ~TextContent() override = default;
+
   static cpp::result<TextContent, std::string> FromJson(Json::Value&& json) {
     if (json.empty()) {
       return cpp::fail("Json string is empty");

engine/test/components/test_models_db.cc

@@ -1,6 +1,5 @@
 #include "database/models.h"
 #include "gtest/gtest.h"
-#include "utils/file_manager_utils.h"
 
 namespace cortex::db {
 namespace {
@@ -122,4 +121,4 @@ TEST_F(ModelsTestSuite, TestHasModel) {
   EXPECT_TRUE(model_list_.DeleteModelEntry(kTestModel.model).value());
 }
 
-}  // namespace cortex::db
+}  // namespace cortex::db