Merge pull request #248 from janhq/247-feat-should-allow-turning-off-caching

add caching enabled to hotfix cache issues

Author: tikikun

controllers/llamaCPP.cc

@@ -178,7 +178,7 @@ void llamaCPP::chatCompletion(
 
   if (jsonBody) {
     // Default values to enable auto caching
-    data["cache_prompt"] = true;
+    data["cache_prompt"] = caching_enabled;
     data["n_keep"] = -1;
 
     // Passing load value
@@ -390,6 +390,7 @@ bool llamaCPP::loadModelImpl(const Json::Value &jsonBody) {
             .asInt();
     params.cont_batching = jsonBody.get("cont_batching", false).asBool();
 
+    this->caching_enabled = jsonBody.get("caching_enabled", false).asBool();
     this->user_prompt = jsonBody.get("user_prompt", "USER: ").asString();
     this->ai_prompt = jsonBody.get("ai_prompt", "ASSISTANT: ").asString();
     this->system_prompt =

controllers/llamaCPP.h

@@ -1908,5 +1908,6 @@ class llamaCPP : public drogon::HttpController<llamaCPP> {
   std::string system_prompt;
   std::string pre_prompt;
   int repeat_last_n;
+  bool caching_enabled;
 };
 }; // namespace inferences