(EAI-1190) fix eval experiments (#845)

* makeGenerateResponse args made optional

* fix all eval experiments

* using prebuild script to check for errors in complete package

Author: yakubova92

packages/chatbot-server-mongodb-public/package.json

@@ -14,6 +14,8 @@
     ".": "./build/lib.js"
   },
   "scripts": {
+    "build:check": "tsc --noEmit --project tsconfig.json",
+    "prebuild": "npm run build:check",
     "build": "rm -rf ./build/ && tsc -b tsconfig.build.json",
     "dev": "nodemon src/index.ts",
     "format": "prettier ./src --write",

packages/chatbot-server-mongodb-public/src/eval/bin/generateEvalCasesYamlFromCSV.ts

@@ -124,7 +124,7 @@ async function main({
   );
   const urlsNotIngested = await pageStore.getMissingPagesByUrl({
     expectedUrls,
-    urlTransformer: normalizeUrl,
+    urlTransformer: (url) => normalizeUrl({ url }),
   });
   if (urlsNotIngested.length > 0) {
     console.warn(

packages/chatbot-server-mongodb-public/src/eval/experiments/allScorersTest.eval.ts

@@ -11,7 +11,6 @@ import fs from "fs";
 import path from "path";
 import { makeConversationEval } from "../ConversationEval";
 import { makeGenerateResponse } from "../../config";
-import { addMessageToConversationStream } from "../../processors/generateResponseWithSearchTool";
 
 async function conversationEval() {
   // Get all the conversation eval cases from YAML

packages/chatbot-server-mongodb-public/src/eval/experiments/architectureCenter.eval.ts

@@ -11,7 +11,6 @@ import fs from "fs";
 import path from "path";
 import { makeConversationEval } from "../ConversationEval";
 import { makeGenerateResponse } from "../../config";
-import { addMessageToConversationStream } from "../../processors/generateResponseWithSearchTool";
 
 async function conversationEval() {
   // Get ONLY architecture center conversations

packages/chatbot-server-mongodb-public/src/eval/experiments/customSystemPrompt.eval.ts

@@ -9,7 +9,6 @@ import {
 } from "../evalHelpers";
 import { makeConversationEval } from "../ConversationEval";
 import { closeDbConnections, makeGenerateResponse } from "../../config";
-import { responsesApiStream } from "../../processors/generateResponseWithSearchTool";
 
 const conversationEvalCases: ConversationEvalCase[] = [
   // Test 1: Basic custom system prompt override

packages/chatbot-server-mongodb-public/src/eval/experiments/dotcomQuestionsTest.eval.ts

@@ -11,7 +11,6 @@ import fs from "fs";
 import path from "path";
 import { makeConversationEval } from "../ConversationEval";
 import { makeGenerateResponse } from "../../config";
-import { addMessageToConversationStream } from "../../processors/generateResponseWithSearchTool";
 
 async function conversationEval() {
   // Get dotcom question set eval cases from YAML

packages/chatbot-server-mongodb-public/src/tools/search.eval.ts

@@ -2,6 +2,7 @@ import "dotenv/config";
 import {
   Eval,
   EvalCase,
+  EvalParameters,
   EvalScorer,
   EvalTask,
 } from "mongodb-rag-core/braintrust";
@@ -62,8 +63,10 @@ const { k } = retrievalConfig.findNearestNeighborsOptions;
 const retrieveRelevantContentEvalTask: EvalTask<
   RetrievalEvalCaseInput,
   RetrievalTaskOutput,
-  RetrievalEvalCaseExpected
-> = async function (data) {
+  RetrievalEvalCaseExpected,
+  void,
+  EvalParameters
+> = async function (data: RetrievalEvalCaseInput) {
   // TODO: (EAI-991) implement retrieval task for evaluation
   const extractedMetadata: MongoDbSearchToolArgs = {
     productName: null,

packages/mongodb-rag-core/src/index.ts

@@ -24,3 +24,4 @@ export * from "./VectorStore";
 export * from "./arrayFilters";
 export * from "./assertEnvVars";
 export * from "./getEnv";
+export * from "./mongoDbMetadata";