[Refactor]: Refactor the file synchronizer to use Merkle DAG

Signed-off-by: ShawnZheng <shawn.zheng@zilliz.com>

Author: Shawnzheng011019

packages/core/src/context.ts

@@ -351,7 +351,10 @@ export class CodeContext {
     updateIgnorePatterns(ignorePatterns: string[]): void {
         // Merge with default patterns, avoiding duplicates
         const mergedPatterns = [...DEFAULT_IGNORE_PATTERNS, ...ignorePatterns];
-        this.ignorePatterns = [...new Set(mergedPatterns)]; // Remove duplicates
+        const uniquePatterns: string[] = [];
+        const patternSet = new Set(mergedPatterns);
+        patternSet.forEach(pattern => uniquePatterns.push(pattern));
+        this.ignorePatterns = uniquePatterns;
         console.log(`🚫 Updated ignore patterns: ${ignorePatterns.length} from .gitignore + ${DEFAULT_IGNORE_PATTERNS.length} default = ${this.ignorePatterns.length} total patterns`);
     }
 

packages/core/src/splitter/ast-splitter.ts

@@ -31,7 +31,7 @@ export class AstCodeSplitter implements Splitter {
         if (chunkSize) this.chunkSize = chunkSize;
         if (chunkOverlap) this.chunkOverlap = chunkOverlap;
         this.parser = new Parser();
-        
+
         // Initialize fallback splitter
         const { LangChainCodeSplitter } = require('./langchain-splitter');
         this.langchainFallback = new LangChainCodeSplitter(chunkSize, chunkOverlap);
@@ -47,7 +47,7 @@ export class AstCodeSplitter implements Splitter {
 
         try {
             console.log(`🌳 Using AST splitter for ${language} file: ${filePath || 'unknown'}`);
-            
+
             this.parser.setLanguage(langConfig.parser);
             const tree = this.parser.parse(code);
 
@@ -58,7 +58,7 @@ export class AstCodeSplitter implements Splitter {
 
             // Extract chunks based on AST nodes
             const chunks = this.extractChunks(tree.rootNode, code, langConfig.nodeTypes, language, filePath);
-            
+
             // If chunks are too large, split them further
             const refinedChunks = await this.refineChunks(chunks, code);
 
@@ -100,8 +100,8 @@ export class AstCodeSplitter implements Splitter {
     }
 
     private extractChunks(
-        node: Parser.SyntaxNode, 
-        code: string, 
+        node: Parser.SyntaxNode,
+        code: string,
         splittableTypes: string[],
         language: string,
         filePath?: string
@@ -255,7 +255,7 @@ export class AstCodeSplitter implements Splitter {
      */
     static isLanguageSupported(language: string): boolean {
         const supportedLanguages = [
-            'javascript', 'js', 'typescript', 'ts', 'python', 'py', 
+            'javascript', 'js', 'typescript', 'ts', 'python', 'py',
             'java', 'cpp', 'c++', 'c', 'go', 'rust', 'rs'
         ];
         return supportedLanguages.includes(language.toLowerCase());

packages/core/src/sync/merkle.ts

@@ -1,93 +1,99 @@
 import * as crypto from 'crypto';
 
-export class MerkleNode {
+export interface MerkleDAGNode {
+    id: string;
     hash: string;
-    left: MerkleNode | null;
-    right: MerkleNode | null;
-
-    constructor(hash: string, left: MerkleNode | null = null, right: MerkleNode | null = null) {
-        this.hash = hash;
-        this.left = left;
-        this.right = right;
-    }
-
-    static serializeNode(node: MerkleNode | null): any {
-        if (!node) return null;
-        return {
-            hash: node.hash,
-            left: MerkleNode.serializeNode(node.left),
-            right: MerkleNode.serializeNode(node.right)
-        };
-    }
-
-    static deserializeNode(data: any): MerkleNode | null {
-        if (!data) return null;
-        return new MerkleNode(
-            data.hash,
-            MerkleNode.deserializeNode(data.left),
-            MerkleNode.deserializeNode(data.right)
-        );
-    }
+    data: string;
+    parents: string[];
+    children: string[];
 }
 
-export class MerkleTree {
-    root: MerkleNode;
-    leaves: MerkleNode[];
+export class MerkleDAG {
+    nodes: Map<string, MerkleDAGNode>;
+    rootIds: string[];
 
-    constructor(data: string[]) {
-        const leaves = data.map(d => new MerkleNode(this.hash(d)));
-        this.leaves = leaves;
-        this.root = this.buildTree(leaves);
+    constructor() {
+        this.nodes = new Map();
+        this.rootIds = [];
     }
 
     private hash(data: string): string {
         return crypto.createHash('sha256').update(data).digest('hex');
     }
 
-    private buildTree(nodes: MerkleNode[]): MerkleNode {
-        if (nodes.length === 0) {
-            return new MerkleNode(this.hash(''));
-        }
-        if (nodes.length === 1) {
-            return nodes[0];
-        }
+    public addNode(data: string, parentId?: string): string {
+        const nodeId = this.hash(data);
+        const node: MerkleDAGNode = {
+            id: nodeId,
+            hash: nodeId,
+            data,
+            parents: [],
+            children: []
+        };
 
-        const parents: MerkleNode[] = [];
-        for (let i = 0; i < nodes.length; i += 2) {
-            const left = nodes[i];
-            const right = (i + 1 < nodes.length) ? nodes[i + 1] : left;
-            const parentHash = this.hash(left.hash + right.hash);
-            parents.push(new MerkleNode(parentHash, left, right));
+        // If there's a parent, create the relationship
+        if (parentId) {
+            const parentNode = this.nodes.get(parentId);
+            if (parentNode) {
+                node.parents.push(parentId);
+                parentNode.children.push(nodeId);
+                this.nodes.set(parentId, parentNode);
+            }
+        } else {
+            // If no parent, it's a root node
+            this.rootIds.push(nodeId);
         }
 
-        return this.buildTree(parents);
+        this.nodes.set(nodeId, node);
+        return nodeId;
     }
 
-    public getRootHash(): string {
-        return this.root.hash;
+    public getNode(nodeId: string): MerkleDAGNode | undefined {
+        return this.nodes.get(nodeId);
     }
 
-    public static compare(tree1: MerkleTree, tree2: MerkleTree): { added: string[], removed: string[], modified: string[] } {
-        const C1 = new Map(tree1.leaves.map(l => [l.hash, l]));
-        const C2 = new Map(tree2.leaves.map(l => [l.hash, l]));
+    public getAllNodes(): MerkleDAGNode[] {
+        return Array.from(this.nodes.values());
+    }
 
-        const added = Array.from(C2.keys()).filter(k => !C1.has(k));
-        const removed = Array.from(C1.keys()).filter(k => !C2.has(k));
-        
-        return { added, removed, modified: [] };
+    public getRootNodes(): MerkleDAGNode[] {
+        return this.rootIds.map(id => this.nodes.get(id)!).filter(Boolean);
+    }
+
+    public getLeafNodes(): MerkleDAGNode[] {
+        return Array.from(this.nodes.values()).filter(node => node.children.length === 0);
     }
 
     public serialize(): any {
         return {
-            root: MerkleNode.serializeNode(this.root),
-            leaves: this.leaves.map(l => MerkleNode.serializeNode(l))
+            nodes: Array.from(this.nodes.entries()),
+            rootIds: this.rootIds
         };
     }
 
-    static deserialize(data: any): MerkleTree {
-        const tree = Object.create(MerkleTree.prototype);
-        tree.root = MerkleNode.deserializeNode(data.root);
-        tree.leaves = (data.leaves || []).map((l: any) => MerkleNode.deserializeNode(l));
-        return tree;
+    public static deserialize(data: any): MerkleDAG {
+        const dag = new MerkleDAG();
+        dag.nodes = new Map(data.nodes);
+        dag.rootIds = data.rootIds;
+        return dag;
+    }
+
+    public static compare(dag1: MerkleDAG, dag2: MerkleDAG): { added: string[], removed: string[], modified: string[] } {
+        const nodes1 = new Map(Array.from(dag1.getAllNodes()).map(n => [n.id, n]));
+        const nodes2 = new Map(Array.from(dag2.getAllNodes()).map(n => [n.id, n]));
+
+        const added = Array.from(nodes2.keys()).filter(k => !nodes1.has(k));
+        const removed = Array.from(nodes1.keys()).filter(k => !nodes2.has(k));
+        
+        // For modified, we'll check if the data has changed for nodes that exist in both
+        const modified: string[] = [];
+        for (const [id, node1] of Array.from(nodes1.entries())) {
+            const node2 = nodes2.get(id);
+            if (node2 && node1.data !== node2.data) {
+                modified.push(id);
+            }
+        }
+
+        return { added, removed, modified };
     }
 }