From abb5d7f43a202e75bf449fbe0489f28238f5c2fb Mon Sep 17 00:00:00 2001 From: Thuandz Date: Thu, 26 Sep 2024 00:46:56 +0700 Subject: [PATCH 1/8] fix/mistral-nemo-chat-template --- engine/config/chat_template_renderer.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/engine/config/chat_template_renderer.h b/engine/config/chat_template_renderer.h index f40894f7b..63a47ecf3 100644 --- a/engine/config/chat_template_renderer.h +++ b/engine/config/chat_template_renderer.h @@ -123,7 +123,7 @@ static int32_t llama_chat_apply_template_internal( ss << content << "\n"; } } else if (role == "user") { - ss << content << " [/INST]"; + ss << "[INST] " << content << " [/INST]"; } else { ss << (space_around_response ? " " : "") << content << (space_around_response ? " " : "") << ""; From 96d9fb744b8c0844b91696fef3ed57c5c5248132 Mon Sep 17 00:00:00 2001 From: nguyenhoangthuan99 Date: Tue, 1 Oct 2024 09:40:40 +0700 Subject: [PATCH 2/8] Fix: set max context length to 8192 --- engine/config/gguf_parser.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/engine/config/gguf_parser.cc b/engine/config/gguf_parser.cc index 178e4b652..aac6b358e 100644 --- a/engine/config/gguf_parser.cc +++ b/engine/config/gguf_parser.cc @@ -582,8 +582,8 @@ void GGUFHandler::ModelConfigFromMetadata() { model_config_.model = name; model_config_.id = name; model_config_.version = std::to_string(version); - model_config_.max_tokens = max_tokens; - model_config_.ctx_len = max_tokens; + model_config_.max_tokens = std::min(8192, max_tokens); + model_config_.ctx_len = std::min(8192, max_tokens); model_config_.ngl = ngl; } From c1c0af6a4c44df63a203be0c7db09c5d3dad2ff3 Mon Sep 17 00:00:00 2001 From: nguyenhoangthuan99 Date: Tue, 1 Oct 2024 10:01:25 +0700 Subject: [PATCH 3/8] Fix: CI build window --- engine/config/gguf_parser.cc | 1 + 1 file changed, 1 insertion(+) diff --git a/engine/config/gguf_parser.cc b/engine/config/gguf_parser.cc index aac6b358e..609bb6182 100644 --- a/engine/config/gguf_parser.cc +++ b/engine/config/gguf_parser.cc @@ -12,6 +12,7 @@ #ifdef _WIN32 #include #include +#include #else #include // For memory-mapped file #include // For file descriptors From f7caccefa97c378ec1ca5097d03e52e581ce6a82 Mon Sep 17 00:00:00 2001 From: nguyenhoangthuan99 Date: Tue, 1 Oct 2024 10:06:23 +0700 Subject: [PATCH 4/8] Fix: CI build window --- engine/config/gguf_parser.cc | 1 + 1 file changed, 1 insertion(+) diff --git a/engine/config/gguf_parser.cc b/engine/config/gguf_parser.cc index 609bb6182..639537868 100644 --- a/engine/config/gguf_parser.cc +++ b/engine/config/gguf_parser.cc @@ -8,6 +8,7 @@ #include #include #include +#include #ifdef _WIN32 #include From 2cb2d0df45a774522e559c9af76751ee1911885d Mon Sep 17 00:00:00 2001 From: nguyenhoangthuan99 Date: Tue, 1 Oct 2024 10:36:16 +0700 Subject: [PATCH 5/8] Fix: log download and CI build window --- engine/config/gguf_parser.cc | 7 ++++--- engine/services/download_service.cc | 10 ++++++---- engine/services/download_service.h | 2 +- 3 files changed, 11 insertions(+), 8 deletions(-) diff --git a/engine/config/gguf_parser.cc b/engine/config/gguf_parser.cc index 639537868..1ad603261 100644 --- a/engine/config/gguf_parser.cc +++ b/engine/config/gguf_parser.cc @@ -1,3 +1,4 @@ +#include #include #include #include @@ -8,7 +9,6 @@ #include #include #include -#include #ifdef _WIN32 #include @@ -27,6 +27,7 @@ #include "trantor/utils/Logger.h" namespace config { +#define MIN(a, b) ((a) < (b) ? (a) : (b)) void GGUFHandler::OpenFile(const std::string& file_path) { #ifdef _WIN32 HANDLE file_handle_ = INVALID_HANDLE_VALUE; @@ -584,8 +585,8 @@ void GGUFHandler::ModelConfigFromMetadata() { model_config_.model = name; model_config_.id = name; model_config_.version = std::to_string(version); - model_config_.max_tokens = std::min(8192, max_tokens); - model_config_.ctx_len = std::min(8192, max_tokens); + model_config_.max_tokens = MIN(8192, max_tokens); + model_config_.ctx_len = MIN(8192, max_tokens); model_config_.ngl = ngl; } diff --git a/engine/services/download_service.cc b/engine/services/download_service.cc index 5ae5974a9..731ddd844 100644 --- a/engine/services/download_service.cc +++ b/engine/services/download_service.cc @@ -67,7 +67,7 @@ cpp::result DownloadService::AddDownloadTask( std::optional dl_err_msg = std::nullopt; for (const auto& item : task.items) { CLI_LOG("Start downloading: " + item.localPath.filename().string()); - auto result = Download(task.id, item, true); + auto result = Download(task.id, item, true, task.type); if (result.has_error()) { dl_err_msg = result.error(); break; @@ -121,7 +121,7 @@ cpp::result DownloadService::AddAsyncDownloadTask( std::optional dl_err_msg = std::nullopt; for (const auto& item : task.items) { CTL_INF("Start downloading: " + item.localPath.filename().string()); - auto result = Download(task.id, item, false); + auto result = Download(task.id, item, false, task.type); if (result.has_error()) { dl_err_msg = result.error(); break; @@ -147,7 +147,7 @@ cpp::result DownloadService::AddAsyncDownloadTask( cpp::result DownloadService::Download( const std::string& download_id, const DownloadItem& download_item, - bool allow_resume) noexcept { + bool allow_resume, std::optional download_type) noexcept { CTL_INF("Absolute file output: " << download_item.localPath.string()); CURL* curl; @@ -232,7 +232,9 @@ cpp::result DownloadService::Download( fclose(file); curl_easy_cleanup(curl); - CLI_LOG("Model " << download_id << " downloaded successfully!") + if (download_type.has_value() && download_type == DownloadType::Model) { + CLI_LOG("Model " << download_id << " downloaded successfully!") + } return {}; } diff --git a/engine/services/download_service.h b/engine/services/download_service.h index 7d44185f5..06d4e0c4c 100644 --- a/engine/services/download_service.h +++ b/engine/services/download_service.h @@ -79,7 +79,7 @@ class DownloadService { cpp::result Download(const std::string& download_id, const DownloadItem& download_item, - bool allow_resume) noexcept; + bool allow_resume,std::optional download_type) noexcept; curl_off_t GetLocalFileSize(const std::filesystem::path& path) const; }; From c3493be61e194020424c35aa900abe5d7cfdc695 Mon Sep 17 00:00:00 2001 From: nguyenhoangthuan99 Date: Tue, 1 Oct 2024 11:02:16 +0700 Subject: [PATCH 6/8] Fix: CI build window --- engine/config/gguf_parser.cc | 8 +++++--- engine/services/download_service.h | 6 +++--- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/engine/config/gguf_parser.cc b/engine/config/gguf_parser.cc index 1ad603261..b64b48d43 100644 --- a/engine/config/gguf_parser.cc +++ b/engine/config/gguf_parser.cc @@ -27,7 +27,9 @@ #include "trantor/utils/Logger.h" namespace config { -#define MIN(a, b) ((a) < (b) ? (a) : (b)) +#define NOMINMAX +constexpr int kDefaultMaxContextLength = 8192; + void GGUFHandler::OpenFile(const std::string& file_path) { #ifdef _WIN32 HANDLE file_handle_ = INVALID_HANDLE_VALUE; @@ -585,8 +587,8 @@ void GGUFHandler::ModelConfigFromMetadata() { model_config_.model = name; model_config_.id = name; model_config_.version = std::to_string(version); - model_config_.max_tokens = MIN(8192, max_tokens); - model_config_.ctx_len = MIN(8192, max_tokens); + model_config_.max_tokens = std::min(kDefaultMaxContextLength, max_tokens); + model_config_.ctx_len = std::min(kDefaultMaxContextLength, max_tokens); model_config_.ngl = ngl; } diff --git a/engine/services/download_service.h b/engine/services/download_service.h index 06d4e0c4c..e0d7ab929 100644 --- a/engine/services/download_service.h +++ b/engine/services/download_service.h @@ -77,9 +77,9 @@ class DownloadService { cpp::result VerifyDownloadTask( DownloadTask& task) const noexcept; - cpp::result Download(const std::string& download_id, - const DownloadItem& download_item, - bool allow_resume,std::optional download_type) noexcept; + cpp::result Download( + const std::string& download_id, const DownloadItem& download_item, + bool allow_resume, std::optional download_type) noexcept; curl_off_t GetLocalFileSize(const std::filesystem::path& path) const; }; From 72a03a066214f2cfbb525d7a5b2163d5c5ec2e11 Mon Sep 17 00:00:00 2001 From: nguyenhoangthuan99 Date: Tue, 1 Oct 2024 11:11:03 +0700 Subject: [PATCH 7/8] Fix: CI build window --- engine/config/gguf_parser.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/engine/config/gguf_parser.cc b/engine/config/gguf_parser.cc index b64b48d43..3d7cd53df 100644 --- a/engine/config/gguf_parser.cc +++ b/engine/config/gguf_parser.cc @@ -587,8 +587,8 @@ void GGUFHandler::ModelConfigFromMetadata() { model_config_.model = name; model_config_.id = name; model_config_.version = std::to_string(version); - model_config_.max_tokens = std::min(kDefaultMaxContextLength, max_tokens); - model_config_.ctx_len = std::min(kDefaultMaxContextLength, max_tokens); + model_config_.max_tokens = std::min(kDefaultMaxContextLength, max_tokens); + model_config_.ctx_len = std::min(kDefaultMaxContextLength, max_tokens); model_config_.ngl = ngl; } From 3d7cf40cf50ed64b0d0ce7e5ae0f37d963472958 Mon Sep 17 00:00:00 2001 From: nguyenhoangthuan99 Date: Tue, 1 Oct 2024 11:50:05 +0700 Subject: [PATCH 8/8] Fix: download log for model and engines --- engine/services/download_service.cc | 15 ++++++--------- engine/services/download_service.h | 2 +- engine/services/model_service.cc | 6 +++++- 3 files changed, 12 insertions(+), 11 deletions(-) diff --git a/engine/services/download_service.cc b/engine/services/download_service.cc index 731ddd844..d72384193 100644 --- a/engine/services/download_service.cc +++ b/engine/services/download_service.cc @@ -67,14 +67,14 @@ cpp::result DownloadService::AddDownloadTask( std::optional dl_err_msg = std::nullopt; for (const auto& item : task.items) { CLI_LOG("Start downloading: " + item.localPath.filename().string()); - auto result = Download(task.id, item, true, task.type); + auto result = Download(task.id, item, true); if (result.has_error()) { dl_err_msg = result.error(); break; } } if (dl_err_msg.has_value()) { - CTL_ERR(dl_err_msg.value()); + // CTL_ERR(dl_err_msg.value()); return cpp::fail(dl_err_msg.value()); } @@ -121,7 +121,7 @@ cpp::result DownloadService::AddAsyncDownloadTask( std::optional dl_err_msg = std::nullopt; for (const auto& item : task.items) { CTL_INF("Start downloading: " + item.localPath.filename().string()); - auto result = Download(task.id, item, false, task.type); + auto result = Download(task.id, item, false); if (result.has_error()) { dl_err_msg = result.error(); break; @@ -147,7 +147,7 @@ cpp::result DownloadService::AddAsyncDownloadTask( cpp::result DownloadService::Download( const std::string& download_id, const DownloadItem& download_item, - bool allow_resume, std::optional download_type) noexcept { + bool allow_resume) noexcept { CTL_INF("Absolute file output: " << download_item.localPath.string()); CURL* curl; @@ -183,7 +183,7 @@ cpp::result DownloadService::Download( CLI_LOG("Resuming download.."); } else { CLI_LOG("Start over.."); - return {}; + return cpp::fail("Cancelled Resume download!"); } } else { CLI_LOG(download_item.localPath.filename().string() @@ -195,7 +195,7 @@ cpp::result DownloadService::Download( if (answer == "Y" || answer == "y" || answer.empty()) { CLI_LOG("Re-downloading.."); } else { - return {}; + return cpp::fail("Cancelled Re-download!"); } } } @@ -232,9 +232,6 @@ cpp::result DownloadService::Download( fclose(file); curl_easy_cleanup(curl); - if (download_type.has_value() && download_type == DownloadType::Model) { - CLI_LOG("Model " << download_id << " downloaded successfully!") - } return {}; } diff --git a/engine/services/download_service.h b/engine/services/download_service.h index e0d7ab929..f73258aad 100644 --- a/engine/services/download_service.h +++ b/engine/services/download_service.h @@ -79,7 +79,7 @@ class DownloadService { cpp::result Download( const std::string& download_id, const DownloadItem& download_item, - bool allow_resume, std::optional download_type) noexcept; + bool allow_resume) noexcept; curl_off_t GetLocalFileSize(const std::filesystem::path& path) const; }; diff --git a/engine/services/model_service.cc b/engine/services/model_service.cc index 344c9506a..80440bffb 100644 --- a/engine/services/model_service.cc +++ b/engine/services/model_service.cc @@ -239,8 +239,10 @@ cpp::result ModelService::HandleUrl( } else { auto result = download_service_.AddDownloadTask(downloadTask, on_finished); if (result.has_error()) { - CTL_ERR(result.error()); + // CTL_ERR(result.error()); return cpp::fail(result.error()); + } else { + CLI_LOG("Model " << model_id << " downloaded successfully!") } return unique_model_id; } @@ -292,6 +294,8 @@ cpp::result ModelService::DownloadModelFromCortexso( if (result.has_error()) { return cpp::fail(result.error()); + } else { + CLI_LOG("Model " << model_id << " downloaded successfully!") } return model_id;