Merge pull request #288 from janhq/287-bug-not-enough-threads-for-non-inference-tasks

feat: add more threads for core services

Author: tikikun

controllers/llamaCPP.cc

@@ -435,7 +435,7 @@ bool llamaCPP::loadModelImpl(const Json::Value &jsonBody) {
   gpt_params params;
 
   // By default will setting based on number of handlers
-  int drogon_thread = drogon::app().getThreadNum() - 1;
+  int drogon_thread = drogon::app().getThreadNum() - 5;
   LOG_INFO << "Drogon thread is:" << drogon_thread;
   if (jsonBody) {
     if (!jsonBody["mmproj"].isNull()) {

main.cc

@@ -39,7 +39,7 @@ int main(int argc, char *argv[]) {
   LOG_INFO << "Server started, listening at: " << host << ":" << port;
   LOG_INFO << "Please load your model";
   drogon::app().addListener(host, port);
-  drogon::app().setThreadNum(thread_num + 1);
+  drogon::app().setThreadNum(thread_num + 5);
   LOG_INFO << "Number of thread is:" << drogon::app().getThreadNum();
 
   drogon::app().run();