From 9eea5b09a140ce757bb48f0f269566fce57f4b1b Mon Sep 17 00:00:00 2001 From: Cody Bennett Date: Tue, 21 Nov 2023 08:15:14 -0600 Subject: [PATCH 1/8] feat(AST): WGSL --- README.md | 6 +++--- src/generator.ts | 6 +++--- src/parser.ts | 2 +- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index 8d8cac6..4b3b2e4 100644 --- a/README.md +++ b/README.md @@ -228,7 +228,7 @@ minify(`#version 300 es\nin vec2 c;out vec4 data[gl_MaxDrawBuffers];void main(){ ## Parse -Parses a string of GLSL (WGSL is WIP) code into an [AST](#ast). +Parses a string of GLSL or WGSL code into an [AST](#ast). ```ts const ast: AST[] = parse(code: string) @@ -236,11 +236,11 @@ const ast: AST[] = parse(code: string) ## Generate -Generates a string of GLSL (WGSL is WIP) code from an [AST](#ast). +Generates a string of GLSL or WGSL code from an [AST](#ast). ```ts const code: string = generate(ast: AST[], { - target: 'GLSL' // | 'WGSL' + target: 'GLSL' | 'WGSL' }) ``` diff --git a/src/generator.ts b/src/generator.ts index b1abc15..a58722d 100644 --- a/src/generator.ts +++ b/src/generator.ts @@ -153,14 +153,14 @@ function format(node: AST | null): string { } export interface GenerateOptions { - target: 'GLSL' // | 'WGSL' + target: 'GLSL' | 'WGSL' } /** - * Generates a string of GLSL (WGSL WIP) code from an [AST](https://en.wikipedia.org/wiki/Abstract_syntax_tree). + * Generates a string of GLSL or WGSL code from an [AST](https://en.wikipedia.org/wiki/Abstract_syntax_tree). */ export function generate(ast: AST[], options: GenerateOptions): string { - let code = '#version 300 es\n' + let code = options.target === 'GLSL' ? '#version 300 es\n' : '' for (let i = 0; i < ast.length; i++) { code += punctuate(format(ast[i])) diff --git a/src/parser.ts b/src/parser.ts index 3bb484b..97e5146 100644 --- a/src/parser.ts +++ b/src/parser.ts @@ -551,7 +551,7 @@ function parseBlock(): BlockStatement { const DIRECTIVE_REGEX = /(^\s*#[^\\]*?)(\n|\/[\/\*])/gm /** - * Parses a string of GLSL (WGSL WIP) code into an [AST](https://en.wikipedia.org/wiki/Abstract_syntax_tree). + * Parses a string of GLSL or WGSL code into an [AST](https://en.wikipedia.org/wiki/Abstract_syntax_tree). */ export function parse(code: string): AST[] { // Remove (implicit) version header From 508572541e774b35bbd5e646225d0dbfbba92646 Mon Sep 17 00:00:00 2001 From: Cody Bennett Date: Mon, 17 Feb 2025 02:52:24 -0600 Subject: [PATCH 2/8] Merge branch 'main' into feat/wgsl --- LICENSE | 2 +- README.md | 554 ++++++++--- demo/index.html | 7 +- package.json | 24 +- src/ast.ts | 443 +++++++-- src/constants.ts | 112 ++- src/generator.ts | 262 +++-- src/index.ts | 12 +- src/minifier.ts | 51 +- src/parser.ts | 969 +++++++++++-------- src/tokenizer.ts | 37 +- src/visitor.ts | 131 +++ tests/__snapshots__/index.test.ts.snap | 217 ++++- tests/generator.test.ts | 263 +++++ tests/index.test.ts | 19 +- tests/parser.test.ts | 1220 +++++++++++++++-------- tests/tokenizer.test.ts | 72 ++ tests/traversal.test.ts | 68 ++ tsconfig.json | 3 +- vite.config.ts | 17 +- yarn.lock | 1230 +++++++++++++----------- 21 files changed, 3905 insertions(+), 1808 deletions(-) create mode 100644 src/visitor.ts create mode 100644 tests/generator.test.ts create mode 100644 tests/tokenizer.test.ts create mode 100644 tests/traversal.test.ts diff --git a/LICENSE b/LICENSE index ed8a2d2..df9521d 100644 --- a/LICENSE +++ b/LICENSE @@ -1,6 +1,6 @@ MIT License -Copyright (c) 2023 Cody Bennett +Copyright (c) 2023-2025 Cody Bennett Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal diff --git a/README.md b/README.md index 4b3b2e4..bc12c3a 100644 --- a/README.md +++ b/README.md @@ -13,33 +13,57 @@ Tools and IntelliSense for GLSL and WGSL. - [Minify](#minify) - [Parse](#parse) - [Generate](#generate) +- [Visit](#visit) - [AST](#ast) - - [Literal](#literal) + - [Node Objects](#node-objects) - [Identifier](#identifier) - - [Type](#type) - - [VariableDeclaration](#variabledeclaration) - - [VariableDeclarator](#variabledeclarator) - - [StructDeclaration](#structdeclaration) - - [FunctionDeclaration](#functiondeclaration) - - [UnaryExpression](#unaryexpression) - - [BinaryExpression](#binaryexpression) - - [TernaryExpression](#ternaryexpression) - - [CallExpression](#callexpression) - - [MemberExpression](#memberexpression) - - [ArrayExpression](#arrayexpression) - - [BlockStatement](#blockstatement) - - [IfStatement](#ifstatement) - - [ForStatement](#forstatement) - - [WhileStatement](#whilestatement) - - [DoWhileStatement](#dowhilestatement) - - [SwitchStatement](#switchstatement) - - [SwitchCase](#switchcase) - - [ReturnStatement](#returnstatement) - - [PreprocessorStatement](#preprocessorstatement) - - [PrecisionStatement](#precisionstatement) - - [ContinueStatement](#continuestatement) - - [BreakStatement](#breakstatement) - - [DiscardStatement](#discardstatement) + - [Literal](#literal) + - [ArraySpecifier](#arrayspecifier) + - [Program](#program) + - Statements + - [ExpressionStatement](#expressionstatement) + - [BlockStatement](#blockstatement) + - [DiscardStatement](#discardstatement) + - [PreprocessorStatement](#preprocessorstatement) + - [PrecisionQualifierStatement](#precisionqualifierstatement) + - [InvariantQualifierStatement](#invariantqualifierstatement) + - [LayoutQualifierStatement](#layoutqualifierstatement) + - Control Flow + - [ReturnStatement](#returnstatement) + - [BreakStatement](#breakstatement) + - [ContinueStatement](#continuestatement) + - Choice + - [IfStatement](#ifstatement) + - [SwitchStatement](#switchstatement) + - [SwitchCase](#switchcase) + - Loops + - [WhileStatement](#whilestatement) + - [DoWhileStatement](#dowhilestatement) + - [ForStatement](#forstatement) + - Declarations + - [FunctionDeclaration](#functiondeclaration) + - [FunctionParameter](#functionparameter) + - [VariableDeclaration](#variabledeclaration) + - [VariableDeclarator](#variabledeclarator) + - [StructuredBufferDeclaration](#structuredbufferdeclaration) + - [StructDeclaration](#structdeclaration) + - Expressions + - [ArrayExpression](#arrayexpression) + - Unary Operations + - [UnaryExpression](#unaryexpression) + - [UnaryOperator](#unaryoperator) + - [UpdateExpression](#updateexpression) + - [UpdateOperator](#updateoperator) + - Binary Operations + - [BinaryExpression](#binaryexpression) + - [BinaryOperator](#binaryoperator) + - [AssignmentExpression](#assignmentexpression) + - [AssignmentOperator](#assignmentoperator) + - [LogicalExpression](#logicalexpression) + - [LogicalOperator](#logicaloperator) + - [MemberExpression](#memberexpression) + - [ConditionalExpression](#conditionalexpression) + - [CallExpression](#callexpression) ## Installation @@ -228,208 +252,250 @@ minify(`#version 300 es\nin vec2 c;out vec4 data[gl_MaxDrawBuffers];void main(){ ## Parse -Parses a string of GLSL or WGSL code into an [AST](#ast). +Parses a string of GLSL (WGSL is WIP) code into an [AST](#ast). ```ts -const ast: AST[] = parse(code: string) +const ast: Program = parse(code: string) ``` ## Generate -Generates a string of GLSL or WGSL code from an [AST](#ast). +Generates a string of GLSL (WGSL is WIP) code from an [AST](#ast). ```ts -const code: string = generate(ast: AST[], { - target: 'GLSL' | 'WGSL' +const code: string = generate(program: Program, { + target: 'GLSL' // | 'WGSL' }) ``` +## Visit + +Recurses through an [AST](https://en.wikipedia.org/wiki/Abstract_syntax_tree), calling a visitor object on matching nodes. + +```ts +visit( + program: Program, + visitors: { + Program: { + enter(node, ancestors) { + // Called before any descendant nodes are processed + }, + exit(node, ancestors) { + // Called after all nodes are processed + } + }, + Identifier(node, ancestors) { + // Called before any descendant nodes are processed (alias to enter) + } + } satisfies Visitors +) +``` + ## AST An [Abstract Syntax Tree](https://en.wikipedia.org/wiki/Abstract_syntax_tree) loosely based on [ESTree](https://github.com/estree/estree) for GLSL and WGSL grammars. -### Literal +### Node Objects -A shader literal representing a `bool`, `float`, `int`, or `uint` type. +AST nodes extend `Node` objects which implement the following abstract interface: ```ts -class Literal { - value: string +interface Node { + type: string } ``` +The `type` field is a string representing the AST variant type which can determine the interface a node implements. + ### Identifier A variable identifier. ```ts -class Identifier { +interface Identifier extends Node { + type: 'Identifier' + name: string +} +``` + +### Literal + +A shader literal representing a `bool`, `float`, `int`, or `uint` type. + +```ts +interface Literal extends Node { + type: 'Literal' value: string } ``` -### Type +### ArraySpecifier -Represents a type specifier and its parameters (WGSL specific). +An array and its dimensions. ```ts -class Type { - name: string - parameters: (Type | Literal | Identifier)[] | null +interface ArraySpecifier extends Node { + type: 'ArraySpecifier' + typeSpecifier: Identifier + dimensions: (Literal | Identifier | null)[] } ``` -### VariableDeclaration +### Program -A variable declaration with an optional binding layout, type qualifiers, kind (WGSL only), and declarators (e.g. a comma-separated list). +Represents the root of an AST. ```ts -class VariableDeclaration { - layout: Record | null - qualifiers: string[] - kind: 'var' | 'let' | 'const' | null - type: Type | Identifier - declarations: VariableDeclarator[] +interface Program extends Node { + type: 'Program' + body: Statement[] } ``` -#### VariableDeclarator +### ExpressionStatement -A single named declarator as part of a `VariableDeclaration`. +An expression as a standalone statement. ```ts -class VariableDeclarator { - name: string - value: AST | null +interface ExpressionStatement extends Node { + type: 'ExpressionStatement' + expression: Expression } ``` -### StructDeclaration +### BlockStatement -A struct declaration. Can be used as a type or constructor. +A block statement. ```ts -class StructDeclaration { - name: string - members: VariableDeclaration[] +interface BlockStatement extends Node { + type: 'BlockStatement' + body: Statement[] } ``` -### FunctionDeclaration +### DiscardStatement -A function declaration with an optional type qualifier and arguments. +A discard statement in fragment shaders. ```ts -class FunctionDeclaration { - name: string - type: Type | Identifier - qualifiers: string[] - args: VariableDeclaration[] - body: BlockStatement | null +interface DiscardStatement extends Node { + type: 'DiscardStatement' } ``` -### UnaryExpression +### PreprocessorStatement -A unary expression with a left or right handed operator. +A GLSL preprocessor statement with an optional value. ```ts -class UnaryExpression { - operator: string - left: AST | null - right: AST | null +interface PreprocessorStatement extends Node { + type: 'PreprocessorStatement' + name: string + value: Expression[] | null } ``` -### BinaryExpression +### PrecisionQualifierStatement -A binary expression with a left and right operand. +A GLSL precision qualifier statement. ```ts -class BinaryExpression { - operator: string - left: AST - right: AST +interface PrecisionQualifierStatement extends Node { + type: 'PrecisionQualifierStatement' + precision: PrecisionQualifier + typeSpecifier: Identifier } ``` -### TernaryExpression +### InvariantQualifierStatement -A ternary or conditional expression. +A GLSL invariant qualifier statement. ```ts -class TernaryExpression { - test: AST - consequent: AST - alternate: AST +interface InvariantQualifierStatement extends Node { + type: 'InvariantQualifierStatement' + typeSpecifier: Identifier } ``` -### CallExpression +### LayoutQualifierStatement -A call expression. +A layout qualifier statement. ```ts -class CallExpression { - callee: AST - args: AST[] +interface LayoutQualifierStatement extends Node { + type: 'LayoutQualifierStatement' + layout: Record + qualifier: StorageQualifier } ``` -### MemberExpression +### ReturnStatement -A member expression. +A return statement with an optional argument. ```ts -class MemberExpression { - object: AST - property: AST +interface ReturnStatement extends Node { + type: 'ReturnStatement' + argument: Expression | null } ``` -### ArrayExpression +### BreakStatement -An array expression. `members` can be empty if uninitialized. +A break statement. ```ts -class ArrayExpression { - type: Type - members: AST[] +interface BreakStatement extends Node { + type: 'BreakStatement' } ``` -### BlockStatement +### ContinueStatement -A block statement. +A continue statement. ```ts -class BlockStatement { - body: AST[] +interface ContinueStatement extends Node { + type: 'ContinueStatement' } ``` ### IfStatement -An if statement. +An if-else statement. ```ts -class IfStatement { - test: AST - consequent: AST - alternate: AST | null +interface IfStatement extends Node { + type: 'IfStatement' + test: Expression + consequent: Statement + alternate: Statement | null } ``` -### ForStatement +### SwitchStatement -A for statement. +A switch statement. + +```ts +interface SwitchStatement extends Node { + type: 'SwitchStatement' + discriminant: Expression + cases: SwitchCase[] +} +``` + +#### SwitchCase + +A switch-case statement. `test` is null for a `default` case. ```ts -class ForStatement { - init: AST | null - test: AST | null - update: AST | null - body: AST +interface SwitchCase extends Node { + type: 'SwitchCase' + test: Expression | null + consequent: Statement[] } ``` @@ -438,9 +504,10 @@ class ForStatement { A while statement. ```ts -class WhileStatement { - test: AST - body: AST +interface WhileStatement extends Node { + type: 'WhileStatement' + test: Expression + body: Statement } ``` @@ -449,86 +516,265 @@ class WhileStatement { A do-while statement. ```ts -class DoWhileStatement { - test: AST - body: AST +interface DoWhileStatement extends Node { + type: 'DoWhileStatement' + body: Statement + test: Expression } ``` -### SwitchStatement +### ForStatement -A switch statement. +A for statement. ```ts -class SwitchStatement { - discriminant: AST - cases: SwitchCase[] +interface ForStatement extends Node { + type: 'ForStatement' + init: VariableDeclaration | Expression | null + test: Expression | null + update: Expression | null + body: Statement } ``` -#### SwitchCase +### FunctionDeclaration -A switch-case statement. `test` is null for a `default` case. +A function declaration. `body` is null for overloads. ```ts -class SwitchCase { - test: AST | null - consequent: AST[] +interface FunctionDeclaration extends Node { + type: 'FunctionDeclaration' + id: Identifier + qualifiers: PrecisionQualifier[] + typeSpecifier: Identifier | ArraySpecifier + params: FunctionParameter[] + body: BlockStatement | null } ``` -### ReturnStatement +#### FunctionParameter -A return statement with an optional argument. +A function parameter within a function declaration. ```ts -class ReturnStatement { - argument: Literal | Identifier | UnaryExpression | null +interface FunctionParameter extends Node { + type: 'FunctionParameter' + id: Identifier | null + qualifiers: (ConstantQualifier | ParameterQualifier | PrecisionQualifier)[] + typeSpecifier: Identifier | ArraySpecifier } ``` -### PreprocessorStatement +### VariableDeclaration -A GLSL preprocessor statement with an optional value. +A variable declaration. ```ts -class PreprocessorStatement { - name: string - value: AST[] | null +interface VariableDeclaration extends Node { + type: 'VariableDeclaration' + declarations: VariableDeclarator[] } ``` -### PrecisionStatement +#### VariableDeclarator -A GLSL precision statement. +A variable declarator within a variable declaration. ```ts -class PrecisionStatement { - precision: 'lowp' | 'mediump' | 'highp' - type: Type +interface VariableDeclarator extends Node { + type: 'VariableDeclarator' + id: Identifier + qualifiers: (ConstantQualifier | InterpolationQualifier | StorageQualifier | PrecisionQualifier)[] + typeSpecifier: Identifier | ArraySpecifier + layout: Record | null + init: Expression | null } ``` -### ContinueStatement +### StructuredBufferDeclaration -A continue statement. +A buffer interface declaration with optional layout and qualifiers. ```ts -class ContinueStatement {} +interface StructuredBufferDeclaration extends Node { + type: 'StructuredBufferDeclaration' + id: Identifier | null + qualifiers: (InterfaceStorageQualifier | MemoryQualifier | LayoutQualifier)[] + typeSpecifier: Identifier | ArraySpecifier + layout: Record | null + members: VariableDeclaration[] +} ``` -### BreakStatement +### StructDeclaration -A break statement. +A struct declaration. Can be used as a type or constructor. ```ts -class BreakStatement {} +interface StructDeclaration extends Node { + type: 'StructDeclaration' + id: Identifier + members: VariableDeclaration[] +} ``` -### DiscardStatement +### ArrayExpression + +An array initialization expression. + +```ts +interface ArrayExpression extends Node { + type: 'ArrayExpression' + typeSpecifier: ArraySpecifier + elements: Expression[] +} +``` + +### UnaryExpression + +A unary expression with a left or right handed operator. + +```ts +interface UnaryExpression extends Node { + type: 'UnaryExpression' + operator: UnaryOperator + prefix: boolean + argument: Expression +} +``` + +#### UnaryOperator + +```ts +type UnaryOperator = '-' | '+' | '!' | '~' +``` + +### UpdateExpression + +An update expression with an optionally prefixed operator. + +```ts +interface UpdateExpression extends Node { + type: 'UpdateExpression' + operator: UpdateOperator + argument: Expression + prefix: boolean +} +``` -A discard statement. +#### UpdateOperator ```ts -class DiscardStatement {} +type UpdateOperator = '++' | '--' +``` + +### BinaryExpression + +A binary expression with a left and right operand. + +```ts +interface BinaryExpression extends Node { + type: 'BinaryExpression' + operator: BinaryOperator + left: Expression + right: Expression +} +``` + +#### BinaryOperator + +```ts +type BinaryOperator = + | '==' + | '!=' + | '<' + | '<=' + | '>' + | '>=' + | '<<' + | '>>' + | '+' + | '-' + | '*' + | '/' + | '%' + | '|' + | '^' + | '&' +``` + +### AssignmentExpression + +An assignment expression. + +```ts +interface AssignmentExpression extends Node { + type: 'AssignmentExpression' + operator: AssignmentOperator + left: Expression + right: Expression +} +``` + +#### AssignmentOperator + +```ts +type AssignmentOperator = '=' | '+=' | '-=' | '*=' | '/=' | '%=' | '<<=' | '>>=' | '>>>=' | '|=' | '^=' | '&=' +``` + +### LogicalExpression + +A logical operation between two expressions. + +```ts +interface LogicalExpression extends Node { + type: 'LogicalExpression' + operator: LogicalOperator + left: Expression + right: Expression +} +``` + +#### LogicalOperator + +```ts +type LogicalOperator = '||' | '&&' | '^^' +``` + +### MemberExpression + +A member expression. + +```ts +interface MemberExpression extends Node { + type: 'MemberExpression' + object: Expression + property: Expression + computed: boolean +} +``` + +### ConditionalExpression + +A conditional expression or ternary. + +```ts +interface ConditionalExpression extends Node { + type: 'ConditionalExpression' + test: Expression + alternate: Expression + consequent: Expression +} +``` + +### CallExpression + +A function call expression or struct initialization. + +```ts +interface CallExpression extends Node { + type: 'CallExpression' + callee: Expression + arguments: Expression[] +} ``` diff --git a/demo/index.html b/demo/index.html index ca3d0d8..448012e 100644 --- a/demo/index.html +++ b/demo/index.html @@ -6,10 +6,7 @@ } -