Integrate tree-sitter queries into the search (#1944)

* feat(wip): include parse data in query and search

* Revert "feat(wip): include parse data in query and search"

This reverts commit 5cf95d9.

* feat(wip): reintroduce tree-sitter-specific wip

* feat(wip): some more tree-sitter framework

* feat: attach tree-sitter node ids to our nodes for conversion

* feat: initial tree-sitter query impl

* test: assignment test yay

* feat: allow specifying what capture names to match

* refactor: use proper logging for search generators

* feat: allow capture names to contain the @

* refactor: fix up stuff based on the review

* refactor: combine ParserInformation and ParserMetadata

* refactor: remove unnecessary as

* test: some additional tests

* refactor: rename fromTreeSitterQuery to syntax

* feat-fix: fixed wiki compile

Author: Ellpeck

src/cli/repl/commands/repl-execute.ts

@@ -1,9 +1,10 @@
 import type { ReplCommand, ReplCommandInformation, ReplOutput } from './repl-main';
 import { ColorEffect, Colors, FontStyles, italic } from '../../../util/text/ansi';
 import type { ReadonlyFlowrAnalysisProvider } from '../../../project/flowr-analyzer';
+import type { RShellInformation } from '../../../r-bridge/parser';
 
 export async function tryExecuteRShellCommand({ output, analyzer, allowRSessionAccess, remainingLine }: ReplCommandInformation) {
-	const parserInfo = await analyzer.parserInformation();
+	const parserInfo = analyzer.parserInformation();
 	if(!allowRSessionAccess){
 		output.stderr(`${output.formatter.format('You are not allowed to execute arbitrary R code.', { style: FontStyles.Bold, color: Colors.Red, effect: ColorEffect.Foreground })} 
 If you want to do so, please restart flowR with the ${output.formatter.format('--r-session-access', { style: FontStyles.Bold })} flag${ parserInfo.name !== 'r-shell' ? '. Additionally, please enable the r-shell engine, e.g., with ' + output.formatter.format('--default-engine r-shell', { style: FontStyles.Bold }) : ''}. Please be careful of the security implications of this action. When running flowR with npm, you have to use an extra ${output.formatter.format('--', { style: FontStyles.Bold })} to separate flowR from npm arguments.`);
@@ -16,7 +17,7 @@ If you want to do so, please restart flowR with the ${output.formatter.format('-
 
 async function executeRShellCommand(output: ReplOutput, analyzer: ReadonlyFlowrAnalysisProvider, statement: string) {
 	try {
-		const result = await analyzer.sendCommandWithOutput(statement, {
+		const result = await (analyzer.parserInformation() as RShellInformation).sendCommandWithOutput(statement, {
 			from:                    'both',
 			automaticallyTrimOutput: true
 		});

src/cli/repl/commands/repl-parse.ts

@@ -167,7 +167,7 @@ export const parseCommand: ReplCodeCommand = {
 	},
 	fn: async({ output, analyzer }) => {
 		const result = await analyzer.parse();
-		const parserInfo = await analyzer.parserInformation();
+		const parserInfo = analyzer.parserInformation();
 
 		if(parserInfo.name === 'r-shell') {
 			const object = convertPreparedParsedData(prepareParsedData(result.parsed as unknown as string));

src/core/steps/all/core/01-parse-tree-sitter.ts

@@ -5,18 +5,6 @@ import type { Tree } from 'web-tree-sitter';
 import type { ParseRequiredInput } from '../../../../r-bridge/parser';
 import { parseRequests } from '../../../../r-bridge/parser';
 
-export interface ParseStepOutputTS {
-    readonly parsed: Tree
-}
-
-export interface TreeSitterParseJson {
-	readonly '.meta': {
-		readonly tokenCount:           number,
-		readonly tokenCountNoComments: number
-	},
-	readonly str: string
-}
-
 export const PARSE_WITH_TREE_SITTER_STEP = {
 	name:              'parse',
 	humanReadableName: 'parse with tree-sitter',

src/core/steps/all/core/11-normalize-tree-sitter.ts

@@ -11,11 +11,12 @@ import type { DeepReadonly } from 'ts-essentials';
 import { normalizeTreeSitter } from '../../../../r-bridge/lang-4.x/ast/parser/json/parser';
 import type { NormalizeRequiredInput } from './10-normalize';
 import { getCurrentRequestFile } from './10-normalize';
-import type { ParseStepOutputTS } from './01-parse-tree-sitter';
 import type { FlowrConfigOptions } from '../../../../config';
+import type { ParseStepOutput } from '../../../../r-bridge/parser';
+import type { Tree } from 'web-tree-sitter';
 
-function processor(results: { 'parse'?: ParseStepOutputTS }, input: Partial<NormalizeRequiredInput>, config: FlowrConfigOptions) {
-	return normalizeTreeSitter(results['parse'] as ParseStepOutputTS, input.getId, config, input.overwriteFilePath ?? getCurrentRequestFile(input.request));
+function processor(results: { 'parse'?: ParseStepOutput<Tree> }, input: Partial<NormalizeRequiredInput>, config: FlowrConfigOptions) {
+	return normalizeTreeSitter(results['parse'] as ParseStepOutput<Tree>, input.getId, config, input.overwriteFilePath ?? getCurrentRequestFile(input.request));
 }
 
 export const NORMALIZE_TREE_SITTER = {

src/project/flowr-analyzer.ts

@@ -1,6 +1,9 @@
 import type { FlowrConfigOptions } from '../config';
 
-import type { KnownParser, ParseStepOutput, RShellInformation, TreeSitterInformation } from '../r-bridge/parser';
+import type {
+	KnownParser, KnownParserInformation,
+	ParseStepOutput
+} from '../r-bridge/parser';
 import type { Queries, QueryResults, SupportedQueryTypes } from '../queries/query';
 import { executeQueries } from '../queries/query';
 import type { ControlFlowInformation } from '../control-flow/control-flow-graph';
@@ -14,9 +17,6 @@ import type { GetSearchElements } from '../search/flowr-search-executor';
 import { runSearch } from '../search/flowr-search-executor';
 import type { FlowrAnalyzerContext, ReadOnlyFlowrAnalyzerContext } from './context/flowr-analyzer-context';
 import { CfgKind } from './cfg-kind';
-import type { OutputCollectorConfiguration } from '../r-bridge/shell';
-import { RShell } from '../r-bridge/shell';
-import { guard } from '../util/assert';
 import type { RAnalysisRequest } from './context/flowr-analyzer-files-context';
 
 /**
@@ -48,16 +48,11 @@ export interface FlowrAnalysisProvider extends ReadonlyFlowrAnalysisProvider {
  * This allows us to exchange the underlying implementation of the analyzer without affecting the APIs.
  */
 export interface ReadonlyFlowrAnalysisProvider {
-	/**
-     * Get the name of the parser used by the analyzer.
-	 */
-    parserInformation():   Promise<TreeSitterInformation | RShellInformation>;
-	/**
-	 * Sends a command to the underlying R engine and collects the output.
-	 * @param command     - The command to send to the R engine.
-	 * @param addonConfig - Additional configuration for the output collector.
-	 */
-	sendCommandWithOutput(command: string, addonConfig?: Partial<OutputCollectorConfiguration>): Promise<string[]>;
+    /**
+     * Returns a set of additional data and helper functions exposed by the underlying {@link KnownParser},
+     * including the parser's {@link BaseParserInformation.name} and corresponding version information.
+     */
+    parserInformation(): KnownParserInformation
 	/**
 	 * Returns a read-only version of the project context information.
 	 * This is the preferred method for users that want to inspect the context.
@@ -120,6 +115,7 @@ export class FlowrAnalyzer<Parser extends KnownParser = KnownParser> implements
 	/** The cache used for storing analysis results */
 	private readonly cache:      FlowrAnalyzerCache<Parser>;
 	private readonly ctx:        FlowrAnalyzerContext;
+	private parserInfo:          KnownParserInformation | undefined;
 
 	/**
      * Create a new analyzer instance.
@@ -141,15 +137,9 @@ export class FlowrAnalyzer<Parser extends KnownParser = KnownParser> implements
 		return this.ctx;
 	}
 
-	public async parserInformation(): Promise<TreeSitterInformation | RShellInformation> {
-		return this.parser.name === 'r-shell' ?
-			{ name: 'r-shell', rVersion: await (this.parser as RShell).rVersion() }
-			: { name: 'tree-sitter' };
-	}
-
-	public async sendCommandWithOutput(command: string, addonConfig?: Partial<OutputCollectorConfiguration>): Promise<string[]> {
-		guard(this.parser instanceof RShell, 'sendCommandWithOutput can only be used with RShell parsers!');
-		return this.parser.sendCommandWithOutput(command, addonConfig);
+	public parserInformation(): KnownParserInformation {
+		this.parserInfo ??= this.parser.information(this);
+		return this.parserInfo;
 	}
 
 	public inspectContext(): ReadOnlyFlowrAnalyzerContext {
@@ -208,4 +198,4 @@ export class FlowrAnalyzer<Parser extends KnownParser = KnownParser> implements
 	public close() {
 		return this.parser?.close();
 	}
-}
+}

src/r-bridge/lang-4.x/ast/parser/json/parser.ts

@@ -5,11 +5,11 @@ import { decorateAst, deterministicCountingIdGenerator } from '../../model/proce
 import type { NoInfo, RNode } from '../../model/model';
 import { normalizeRootObjToAst } from '../main/internal/structure/normalize-root';
 import type { NormalizerData } from '../main/normalizer-data';
-import type { ParseStepOutputTS } from '../../../../../core/steps/all/core/01-parse-tree-sitter';
 import { normalizeTreeSitterTreeToAst } from '../../../tree-sitter/tree-sitter-normalize';
 import type { ParseStepOutput } from '../../../../parser';
 import type { FlowrConfigOptions } from '../../../../../config';
 import { getEngineConfig } from '../../../../../config';
+import type { Tree } from 'web-tree-sitter';
 
 export const parseLog = log.getSubLogger({ name: 'ast-parser' });
 
@@ -44,7 +44,7 @@ export function normalizeButNotDecorated(
  * Tree-Sitter pendant to {@link normalize}.
  */
 export function normalizeTreeSitter(
-	{ parsed }: ParseStepOutputTS,
+	{ parsed }: ParseStepOutput<Tree>,
 	getId: IdGenerator<NoInfo> = deterministicCountingIdGenerator(0),
 	config: FlowrConfigOptions,
 	file?: string

src/r-bridge/lang-4.x/tree-sitter/tree-sitter-executor.ts

@@ -1,10 +1,12 @@
+import type { Query, QueryCapture, Tree } from 'web-tree-sitter';
 import Parser from 'web-tree-sitter';
 
 import type { RParseRequest } from '../../retriever';
-import type { SyncParser } from '../../parser';
+import type { SyncParser, TreeSitterInformation } from '../../parser';
 import type { TreeSitterEngineConfig } from '../../../config';
 import { log } from '../../../util/log';
 import fs from 'fs';
+import type { FlowrAnalysisProvider } from '../../../project/flowr-analyzer';
 
 export const DEFAULT_TREE_SITTER_R_WASM_PATH = './node_modules/@eagleoutice/tree-sitter-r/tree-sitter-r.wasm';
 export const DEFAULT_TREE_SITTER_WASM_PATH = './node_modules/web-tree-sitter/tree-sitter.wasm';
@@ -17,7 +19,7 @@ const wasmLog = log.getSubLogger({ name: 'tree-sitter-wasm' });
 export class TreeSitterExecutor implements SyncParser<Parser.Tree> {
 
 	public readonly name = 'tree-sitter';
-	public readonly parser:  Parser;
+	private readonly parser: Parser;
 	private static language: Parser.Language;
 
 	/**
@@ -54,6 +56,16 @@ export class TreeSitterExecutor implements SyncParser<Parser.Tree> {
 		return Promise.resolve('none');
 	}
 
+	public information(analyzer: FlowrAnalysisProvider): TreeSitterInformation {
+		return {
+			name:            'tree-sitter',
+			grammarVersion:  this.treeSitterVersion(),
+			treeSitterQuery: async(source: Query | string, force?: boolean) => {
+				return this.query(source, (await analyzer.parse(force)).parsed as Tree);
+			}
+		};
+	}
+
 	public treeSitterVersion(): number {
 		return this.parser.getLanguage().version;
 	}
@@ -68,6 +80,16 @@ export class TreeSitterExecutor implements SyncParser<Parser.Tree> {
 		return this.parser.parse(sourceCode);
 	}
 
+	public createQuery(source: string): Query {
+		return this.parser.getLanguage().query(source);
+	}
+
+	public query(source: Query | string, tree: Parser.Tree): QueryCapture[] {
+		const query = typeof source === 'string' ? this.createQuery(source) : source;
+		const matches = query.matches(tree.rootNode);
+		return matches.flatMap(m => m.captures);
+	}
+
 	public close(): void {
 		this.parser.delete();
 	}