chore: cleanup (#44)

Author: David-Kunz

lib/embedding-lib.js

@@ -1,4 +1,5 @@
-import fs from 'fs'
+import fs from 'fs/promises'
+import { constants } from 'fs'
 import path from 'path'
 import { fileURLToPath } from 'url'
 import * as ort from 'onnxruntime-web'
@@ -10,18 +11,14 @@ const MODEL_DIR = path.resolve(__dirname, '..', 'models')
 
 const FILES = ['onnx/model.onnx', 'tokenizer.json', 'tokenizer_config.json']
 
-function saveFile(buffer, outputPath) {
-  return new Promise((resolve, reject) => {
-    fs.writeFile(outputPath, Buffer.from(buffer), err => {
-      if (err) reject(err)
-      else resolve()
-    })
-  })
+async function saveFile(buffer, outputPath) {
+  await fs.writeFile(outputPath, Buffer.from(buffer))
 }
 
-function fileExists(filePath) {
+async function fileExists(filePath) {
   try {
-    return fs.existsSync(filePath)
+    await fs.access(filePath, constants.F_OK)
+    return true
   } catch {
     return false
   }
@@ -44,13 +41,15 @@ async function downloadFile(url, outputPath) {
 }
 
 async function downloadModelIfNeeded() {
-  if (!fs.existsSync(MODEL_DIR)) {
-    fs.mkdirSync(MODEL_DIR, { recursive: true })
+  try {
+    await fs.access(MODEL_DIR)
+  } catch {
+    await fs.mkdir(MODEL_DIR, { recursive: true })
   }
 
   for (const file of FILES) {
     const filePath = path.join(MODEL_DIR, path.basename(file))
-    if (!fileExists(filePath)) {
+    if (!(await fileExists(filePath))) {
       const url = `https://huggingface.co/${MODEL_NAME}/resolve/main/${file}`
       await downloadFile(url, filePath)
     }
@@ -65,12 +64,8 @@ async function forceRedownloadModel() {
   // Delete all model files to force re-download
   for (const file of FILES) {
     const filePath = path.join(MODEL_DIR, path.basename(file))
-    try {
-      if (fileExists(filePath)) {
-        fs.unlinkSync(filePath)
-      }
-    } catch {
-      // Ignore deletion errors, we'll overwrite anyway
+    if (await fileExists(filePath)) {
+      await fs.unlink(filePath).catch(() => {})
     }
   }
 
@@ -84,11 +79,11 @@ async function initializeModelAndVocab() {
 
   const loadModelAndVocab = async () => {
     // Load model as buffer for onnxruntime-web
-    const modelBuffer = fs.readFileSync(modelPath)
+    const modelBuffer = await fs.readFile(modelPath)
     session = await ort.InferenceSession.create(modelBuffer)
 
     // Try to parse tokenizer JSON
-    const tokenizerJson = JSON.parse(fs.readFileSync(vocabPath, 'utf-8'))
+    const tokenizerJson = JSON.parse(await fs.readFile(vocabPath, 'utf-8'))
 
     // Validate tokenizer structure
     if (!tokenizerJson.model || !tokenizerJson.model.vocab) {
@@ -112,7 +107,6 @@ async function initializeModelAndVocab() {
   } catch (error) {
     // Model or tokenizer is corrupted, force re-download
     // eslint-disable-next-line no-console
-    console.warn('Model corruption detected, re-downloading...', error.message)
     await forceRedownloadModel()
 
     // Retry initialization after re-download
@@ -347,15 +341,13 @@ async function processChunkedEmbeddings(chunks, session) {
     const validIds = ids.filter(id => {
       const isValid = typeof id === 'number' && !isNaN(id) && isFinite(id)
       if (!isValid) {
-        // eslint-disable-next-line no-console
-        console.warn(`Invalid token ID detected: ${id} (type: ${typeof id})`)
+        throw new Error(`Invalid token ID detected: ${id} (type: ${typeof id})`)
       }
       return isValid
     })
 
     if (validIds.length !== ids.length) {
-      // eslint-disable-next-line no-console
-      console.warn(`Filtered out ${ids.length - validIds.length} invalid token IDs`)
+      throw new Error(`Found ${ids.length - validIds.length} invalid token IDs`)
     }
 
     const inputIds = new BigInt64Array(validIds.map(i => BigInt(i)))

scripts/createEmbeddings.js

@@ -28,11 +28,11 @@ async function main() {
   // can also run independently, stored/read on file system
   // await createSnippets();
   await createSnippetsEmbeddings()
-  // afterwards, copy ./docs/* to https://github.tools.sap/cap/docs-resources -> public/embeddings/
+  // afterwards, copy to capire @ public/embeddings/
 }
 
 async function createSnippetsEmbeddings() {
-  const chunks = JSON.parse(await fs.readFile('/Users/d065023/SAPDevelop/chunking/code-snippets.json'))
+  const chunks = JSON.parse(await fs.readFile('<ADAPT!>/code-snippets.json'))
   await createEmbeddings(
     'code-chunks',
     chunks.map(c => chunkToText(c))