feat(storage): implement SQLite database layer with tests

.husky/pre-commit

@@ -1,2 +1,3 @@
 pnpm lint-staged
+pnpm lint
 pnpm typecheck

WORKFLOW.md

@@ -206,6 +206,64 @@ Issue: #<number>
 - **Breaking Changes**: API changes
 - **Known Limitations**: What doesn't work yet
 
+## Commit Structure Preference
+
+### Atomic Commits (Always)
+
+**Every commit should be atomic:**
+- ✅ Complete logical unit of work
+- ✅ Can be reviewed independently
+- ✅ Can be reverted without breaking things
+- ✅ Tests pass at each commit
+- ✅ Working state maintained
+
+**Commit frequently, but keep commits atomic.**
+
+### Single Commit for Cohesive Features (Default)
+
+**For cohesive features, use a single atomic commit:**
+
+```
+feat(scope): implement feature with tests
+
+- Implementation details
+- Tests included
+- All related changes together
+```
+
+**When to use:**
+- ✅ New features/modules (e.g., new package)
+- ✅ Cohesive changes (< 1000 lines)
+- ✅ Implementation + tests belong together
+- ✅ Single logical unit of work
+
+**Rationale:**
+- Easier to review as one unit
+- Tests validate implementation immediately
+- Cleaner git history
+- Simpler to revert if needed
+
+### Multiple Commits (When Appropriate)
+
+**Split into multiple atomic commits when:**
+- 🔀 Multiple unrelated concerns (e.g., DB + API + UI)
+- 🔀 Large features (1000+ lines) that benefit from incremental review
+- 🔀 Risky changes needing staged rollout
+- 🔀 Tests require significant refactoring separate from implementation
+
+**Example split:**
+```
+feat(module): implement core functionality
+feat(module): add comprehensive test suite
+```
+
+**Avoid splitting for:**
+- ❌ Small cohesive features
+- ❌ Implementation + tests (they belong together)
+- ❌ Artificial separation (e.g., "code" vs "tests")
+
+**Key Principle:** Atomic ≠ Small. Atomic = Complete logical unit. A cohesive feature is one atomic unit.
+
 ## Testing Standards
 
 ### Coverage Goals

package.json

@@ -46,6 +46,7 @@
     "@biomejs/biome": "2.3.8",
     "@tsconfig/node-lts": "^24.0.0",
     "@types/node": "^24.10.1",
+    "@vitest/coverage-v8": "^4.0.15",
     "lint-staged": "16.2.7",
     "typescript": "^5.9.3",
     "vitest": "^4.0.15"

packages/core/src/index.test.ts → packages/core/src/__tests__/index.test.ts

@@ -1,5 +1,5 @@
 import { describe, expect, it } from 'vitest';
-import { CoreService, createCoreService } from './index';
+import { CoreService, createCoreService } from '../index';
 
 describe('CoreService', () => {
   it('should create a CoreService instance', () => {

packages/storage/drizzle.config.ts

@@ -0,0 +1,7 @@
+import { defineConfig } from 'drizzle-kit';
+
+export default defineConfig({
+  schema: './src/schema.ts',
+  out: './drizzle',
+  dialect: 'sqlite',
+});

packages/storage/drizzle/0000_blue_legion.sql

@@ -0,0 +1,8 @@
+CREATE TABLE `documents` (
+	`id` text PRIMARY KEY NOT NULL,
+	`path` text NOT NULL,
+	`hash` text,
+	`status` text DEFAULT 'pending' NOT NULL,
+	`data` text NOT NULL,
+	`created_at` integer DEFAULT '"2025-12-07T11:19:23.284Z"' NOT NULL
+);

packages/storage/drizzle/meta/0000_snapshot.json

@@ -0,0 +1,72 @@
+{
+  "version": "6",
+  "dialect": "sqlite",
+  "id": "6eba05ff-3086-4f83-bc86-506b17db1c03",
+  "prevId": "00000000-0000-0000-0000-000000000000",
+  "tables": {
+    "documents": {
+      "name": "documents",
+      "columns": {
+        "id": {
+          "name": "id",
+          "type": "text",
+          "primaryKey": true,
+          "notNull": true,
+          "autoincrement": false
+        },
+        "path": {
+          "name": "path",
+          "type": "text",
+          "primaryKey": false,
+          "notNull": true,
+          "autoincrement": false
+        },
+        "hash": {
+          "name": "hash",
+          "type": "text",
+          "primaryKey": false,
+          "notNull": false,
+          "autoincrement": false
+        },
+        "status": {
+          "name": "status",
+          "type": "text",
+          "primaryKey": false,
+          "notNull": true,
+          "autoincrement": false,
+          "default": "'pending'"
+        },
+        "data": {
+          "name": "data",
+          "type": "text",
+          "primaryKey": false,
+          "notNull": true,
+          "autoincrement": false
+        },
+        "created_at": {
+          "name": "created_at",
+          "type": "integer",
+          "primaryKey": false,
+          "notNull": true,
+          "autoincrement": false,
+          "default": "'\"2025-12-07T11:19:23.284Z\"'"
+        }
+      },
+      "indexes": {},
+      "foreignKeys": {},
+      "compositePrimaryKeys": {},
+      "uniqueConstraints": {},
+      "checkConstraints": {}
+    }
+  },
+  "views": {},
+  "enums": {},
+  "_meta": {
+    "schemas": {},
+    "tables": {},
+    "columns": {}
+  },
+  "internal": {
+    "indexes": {}
+  }
+}

packages/storage/drizzle/meta/_journal.json

@@ -0,0 +1,13 @@
+{
+  "version": "7",
+  "dialect": "sqlite",
+  "entries": [
+    {
+      "idx": 0,
+      "version": "6",
+      "when": 1765106363288,
+      "tag": "0000_blue_legion",
+      "breakpoints": true
+    }
+  ]
+}

packages/storage/package.json

@@ -0,0 +1,40 @@
+{
+  "name": "@doc-agent/storage",
+  "version": "0.1.0",
+  "type": "module",
+  "private": true,
+  "main": "./dist/index.js",
+  "types": "./dist/index.d.ts",
+  "exports": {
+    ".": {
+      "types": "./dist/index.d.ts",
+      "import": "./dist/index.js",
+      "require": "./dist/index.js"
+    }
+  },
+  "scripts": {
+    "build": "tsc",
+    "dev": "tsc --watch",
+    "clean": "rm -rf dist",
+    "typecheck": "tsc --noEmit",
+    "lint": "biome lint ./src",
+    "format": "biome format --write ./src",
+    "test": "vitest run",
+    "generate": "drizzle-kit generate",
+    "migrate": "drizzle-kit migrate"
+  },
+  "dependencies": {
+    "@doc-agent/core": "workspace:*",
+    "@lytics/kero": "^1.0.0",
+    "better-sqlite3": "^11.6.0",
+    "drizzle-orm": "^0.36.4",
+    "env-paths": "^3.0.0",
+    "zod": "^3.23.8"
+  },
+  "devDependencies": {
+    "@types/better-sqlite3": "^7.6.12",
+    "@types/node": "^22.10.1",
+    "drizzle-kit": "^0.28.1",
+    "typescript": "^5.9.3"
+  }
+}

packages/storage/src/__tests__/db.test.ts

@@ -0,0 +1,120 @@
+import * as fs from 'node:fs';
+import os from 'node:os';
+import * as path from 'node:path';
+import Database from 'better-sqlite3';
+import { drizzle } from 'drizzle-orm/better-sqlite3';
+import { beforeEach, describe, expect, it, vi } from 'vitest';
+import { createDb, ensureDirectoryExists, getDbPath, runMigrations } from '../db.js';
+
+// Mock the migrator module
+vi.mock('drizzle-orm/better-sqlite3/migrator', () => ({
+  migrate: vi.fn(),
+}));
+
+// Mock the logger using hoisted function
+const mockError = vi.hoisted(() => vi.fn());
+vi.mock('@lytics/kero', () => ({
+  default: {
+    createLogger: () => ({
+      error: mockError,
+      warn: vi.fn(),
+      info: vi.fn(),
+      debug: vi.fn(),
+    }),
+  },
+}));
+
+describe('db', () => {
+  beforeEach(() => {
+    mockError.mockClear();
+  });
+
+  describe('ensureDirectoryExists', () => {
+    it('should create directory if it does not exist', () => {
+      const tempDir = path.join(os.tmpdir(), `doc-agent-test-${Date.now()}`);
+      expect(fs.existsSync(tempDir)).toBe(false);
+
+      ensureDirectoryExists(tempDir);
+
+      expect(fs.existsSync(tempDir)).toBe(true);
+
+      // Cleanup
+      fs.rmdirSync(tempDir);
+    });
+
+    it('should not fail if directory already exists', () => {
+      const tempDir = path.join(os.tmpdir(), `doc-agent-test-${Date.now()}`);
+      fs.mkdirSync(tempDir, { recursive: true });
+
+      expect(() => ensureDirectoryExists(tempDir)).not.toThrow();
+      expect(fs.existsSync(tempDir)).toBe(true);
+
+      // Cleanup
+      fs.rmdirSync(tempDir);
+    });
+  });
+
+  describe('getDbPath', () => {
+    it('should return a valid database path', () => {
+      const dbPath = getDbPath();
+      expect(dbPath).toBeTruthy();
+      expect(dbPath).toContain('doc-agent.db');
+      expect(typeof dbPath).toBe('string');
+    });
+
+    it('should accept custom data directory', () => {
+      const customDir = path.join(os.tmpdir(), `doc-agent-custom-${Date.now()}`);
+      const dbPath = getDbPath(customDir);
+
+      expect(dbPath).toContain('doc-agent.db');
+      expect(dbPath).toContain(customDir);
+      expect(fs.existsSync(customDir)).toBe(true);
+
+      // Cleanup
+      fs.rmdirSync(customDir);
+    });
+  });
+
+  describe('runMigrations', () => {
+    it('should handle migration failures gracefully', async () => {
+      const { migrate } = await import('drizzle-orm/better-sqlite3/migrator');
+      const db = drizzle(new Database(':memory:'), { schema: {} });
+
+      // Create a temp directory that exists but has no valid migrations
+      const tempMigrationsDir = path.join(os.tmpdir(), `migrations-test-${Date.now()}`);
+      fs.mkdirSync(tempMigrationsDir, { recursive: true });
+
+      // Make migrate throw an error
+      vi.mocked(migrate).mockImplementation(() => {
+        throw new Error('Test migration failure');
+      });
+
+      // This should not throw, just log the error
+      expect(() => runMigrations(db, tempMigrationsDir)).not.toThrow();
+      expect(mockError).toHaveBeenCalledWith(expect.any(Error), 'Migration failed');
+
+      // Cleanup
+      fs.rmdirSync(tempMigrationsDir);
+    });
+
+    it('should skip migrations if folder does not exist', () => {
+      const db = drizzle(new Database(':memory:'), { schema: {} });
+      const nonExistentDir = path.join(os.tmpdir(), `non-existent-${Date.now()}`);
+
+      // Should not throw
+      expect(() => runMigrations(db, nonExistentDir)).not.toThrow();
+    });
+  });
+
+  describe('createDb', () => {
+    it('should accept custom connection string', () => {
+      const db = createDb(':memory:');
+      expect(db).toBeDefined();
+    });
+
+    it('should use default path when no connection string provided', () => {
+      const db = createDb();
+      expect(db).toBeDefined();
+    });
+  });
+});