Skip to content
This repository was archived by the owner on Jul 4, 2025. It is now read-only.

Commit 4a02ef5

Browse files
committed
feat: add conditional model parameters based on path
When the model path contains both "jan" and "nano" (case-insensitive), automatically add speculative decoding parameters to adjust generation behavior. This improves flexibility by enabling environment-specific configurations without manual parameter tuning. Also includes necessary headers for string manipulation and fixes whitespace in ctx_len handling.
1 parent d9ea600 commit 4a02ef5

File tree

1 file changed

+28
-8
lines changed

1 file changed

+28
-8
lines changed

engine/extensions/local-engine/local_engine.cc

Lines changed: 28 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,8 @@
11
#include "local_engine.h"
2+
#include <algorithm>
23
#include <random>
34
#include <thread>
5+
#include <string.h>
46
#include <unordered_set>
57
#include "utils/curl_utils.h"
68
#include "utils/json_helper.h"
@@ -37,13 +39,29 @@ int GenerateRandomInteger(int min, int max) {
3739

3840
std::vector<std::string> ConvertJsonToParamsVector(const Json::Value& root) {
3941
std::vector<std::string> res;
40-
std::string errors;
4142

4243
for (const auto& member : root.getMemberNames()) {
4344
if (member == "model_path" || member == "llama_model_path") {
4445
if (!root[member].isNull()) {
46+
const std::string path = root[member].asString();
4547
res.push_back("--model");
46-
res.push_back(root[member].asString());
48+
res.push_back(path);
49+
50+
// If path contains both "Jan" and "nano", case-insensitive, add special params
51+
std::string lowered = path;
52+
std::transform(lowered.begin(), lowered.end(), lowered.begin(), [](unsigned char c) {
53+
return std::tolower(c);
54+
});
55+
if (lowered.find("jan") != std::string::npos && lowered.find("nano") != std::string::npos) {
56+
res.push_back("--temp");
57+
res.push_back("0.7");
58+
res.push_back("--top-p");
59+
res.push_back("0.8");
60+
res.push_back("--top-k");
61+
res.push_back("20");
62+
res.push_back("--min-p");
63+
res.push_back("0");
64+
}
4765
}
4866
continue;
4967
} else if (kIgnoredParams.find(member) != kIgnoredParams.end()) {
@@ -76,13 +94,14 @@ std::vector<std::string> ConvertJsonToParamsVector(const Json::Value& root) {
7694
}
7795
continue;
7896
} else if (member == "ctx_len") {
79-
if (!root[member].isNull()) {
80-
res.push_back("--ctx-size");
81-
res.push_back(root[member].asString());
82-
}
83-
continue;
97+
if (!root[member].isNull()) {
98+
res.push_back("--ctx-size");
99+
res.push_back(root[member].asString());
100+
}
101+
continue;
84102
}
85103

104+
// Generic handling for other members
86105
res.push_back("--" + member);
87106
if (root[member].isString()) {
88107
res.push_back(root[member].asString());
@@ -101,14 +120,15 @@ std::vector<std::string> ConvertJsonToParamsVector(const Json::Value& root) {
101120
ss << "\"" << value.asString() << "\"";
102121
first = false;
103122
}
104-
ss << "] ";
123+
ss << "]";
105124
res.push_back(ss.str());
106125
}
107126
}
108127

109128
return res;
110129
}
111130

131+
112132
constexpr const auto kMinDataChunkSize = 6u;
113133

114134
struct OaiInfo {

0 commit comments

Comments
 (0)