From 961c187ff76fab2ca15055434d551dd8d70382d3 Mon Sep 17 00:00:00 2001 From: rakeshup Date: Thu, 16 Jan 2020 21:20:58 +0530 Subject: [PATCH 1/2] Started work on ast to markup mapping --- packages/idyll-ast/src/converters/index.js | 1 + packages/idyll-compiler/src/index.js | 6 ++- packages/idyll-compiler/src/lexer.js | 38 +++++++++---------- .../idyll-compiler/src/processors/post.js | 28 +++++++++++++- 4 files changed, 52 insertions(+), 21 deletions(-) diff --git a/packages/idyll-ast/src/converters/index.js b/packages/idyll-ast/src/converters/index.js index a8b69b701..eefbbcca1 100644 --- a/packages/idyll-ast/src/converters/index.js +++ b/packages/idyll-ast/src/converters/index.js @@ -113,6 +113,7 @@ function inverseConvertHelper(arrayElement, id) { } else { elementJson.type = 'component'; elementJson.name = arrayElement[0]; + elementJson.raw = arrayElement[3]; if (arrayElement[1].length !== 0) { elementJson.properties = {}; arrayElement[1].forEach(property => { diff --git a/packages/idyll-compiler/src/index.js b/packages/idyll-compiler/src/index.js index c195741de..021b8fe65 100644 --- a/packages/idyll-compiler/src/index.js +++ b/packages/idyll-compiler/src/index.js @@ -8,7 +8,8 @@ const { cleanResults, makeFullWidth, wrapText, - autoLinkify + autoLinkify, + linkMarkupToNode } = require('./processors/post'); const { convertV1ToV2 } = require('idyll-ast').converters; const matter = require('gray-matter'); @@ -44,7 +45,10 @@ module.exports = function(input, options, alias, callback) { return new Promise((resolve, reject) => reject(err)); } + const referenceMarkup = linkMarkupToNode(lexResults.positions, content); + let astTransform = Processor(output, options) + .pipe(referenceMarkup) .pipe(hoistVariables) .pipe(flattenChildren) .pipe(makeFullWidth) diff --git a/packages/idyll-compiler/src/lexer.js b/packages/idyll-compiler/src/lexer.js index b61f784a5..a56139cf0 100644 --- a/packages/idyll-compiler/src/lexer.js +++ b/packages/idyll-compiler/src/lexer.js @@ -26,6 +26,7 @@ const shouldBreak = text => { let currentInput = null; const lex = function(options, alias = {}) { + const positions = []; let { row, column, outer, skipLists, inComponent, gotName } = Object.assign( {}, { @@ -66,13 +67,19 @@ const lex = function(options, alias = {}) { ) ].join('|'); }; - var updatePosition = function(lexeme) { + var updatePosition = function(lexeme, shouldNotPush) { + if (!shouldNotPush) { + positions.push([row, column]); + } var lines = lexeme.split('\n'); row += lines.length - 1; if (lines.length > 1) { - column = 0; + column = 1; } column += lines[lines.length - 1].length; + if (!shouldNotPush) { + positions.push([row, column]); + } }; // Rules at the front are pre-processed, @@ -163,22 +170,18 @@ const lex = function(options, alias = {}) { return ['INLINE_CODE'].concat(formatToken(text.trim())); }); - lexer.addRule(/[\s\n]*(#{1,6})\s*([^\n\[]+)[\n\s]*/gm, function( - lexeme, - hashes, - text - ) { + lexer.addRule(/(#{1,6})\s*([^\n\[]+)/gm, function(lexeme, hashes, text) { if (this.reject) return; updatePosition(lexeme); - return ['BREAK', 'HEADER_' + hashes.length] + return ['HEADER_' + hashes.length] .concat(recurse(text, { skipLists: true })) .concat(['HEADER_END']); }); - lexer.addRule(/[\s\n]*>\s*([^\n\[]+)[\n\s]*/gm, function(lexeme, text) { + lexer.addRule(/>\s*([^\n\[]+)/gm, function(lexeme, text) { if (this.reject) return; updatePosition(lexeme); - return ['BREAK', 'QUOTE_START'] + return ['QUOTE_START'] .concat(recurse(text, { skipLists: true })) .concat(['QUOTE_END']); }); @@ -315,14 +318,14 @@ const lex = function(options, alias = {}) { } }); - lexer.addRule(/\/(\n?[^`\*\[\/\n\]!\\\d_])*/gm, function(lexeme) { + lexer.addRule(/\/([^`\*\[\/\n\]!\\\d_])*/gm, function(lexeme) { this.reject = inComponent || lexeme.trim() === ''; if (this.reject) return; updatePosition(lexeme); return ['WORDS'].concat(formatToken(lexeme)); }); - lexer.addRule(/(\n?[^`\*\[\/\n\]!\\\d_])+/, function(lexeme) { + lexer.addRule(/([^`\*\[\/\n\]!\\\d_])+/, function(lexeme) { this.reject = inComponent || lexeme.trim() === ''; if (this.reject) return; updatePosition(lexeme); @@ -330,7 +333,7 @@ const lex = function(options, alias = {}) { }); // Match on separately so we can greedily match the // other tags. - lexer.addRule(/[!\d\*_`] */, function(lexeme) { + lexer.addRule(/[!\d\*_`]\s*/, function(lexeme) { this.reject = inComponent || lexeme.trim() === ''; if (this.reject) return; updatePosition(lexeme); @@ -343,10 +346,10 @@ const lex = function(options, alias = {}) { return ['WORDS'].concat(formatToken(lexeme)); }); - lexer.addRule(/\s*\n{2,}\s*/, function(lexeme) { + lexer.addRule(/\s*\n+?\s*/, function(lexeme) { this.reject = inComponent; if (this.reject) return; - updatePosition(lexeme); + updatePosition(lexeme, true); return ['BREAK']; }); @@ -443,21 +446,18 @@ const lex = function(options, alias = {}) { lexer.addRule(/\s*$/, function(lexeme) { this.reject = !outer; if (this.reject) return; - updatePosition(lexeme); + updatePosition(lexeme, true); return ['EOF']; }); return function(str) { currentInput = str; - var vals = []; var output = []; - var positions = []; lexer.input = str.trim(); var token = lexer.lex(); while (token) { output.push(token); - positions.push([row, column]); token = lexer.lex(); } return { diff --git a/packages/idyll-compiler/src/processors/post.js b/packages/idyll-compiler/src/processors/post.js index c904eee01..1dd05a4e7 100644 --- a/packages/idyll-compiler/src/processors/post.js +++ b/packages/idyll-compiler/src/processors/post.js @@ -60,6 +60,9 @@ const cleanResults = (ast, options) => { if (rawNodes.indexOf(name) > -1) { return node; } + if (node[3]) { + return [node[0], node[1], cleanResults(node[2], options), node[3]]; + } return [node[0], node[1], cleanResults(node[2], options)]; }); }; @@ -256,6 +259,28 @@ function getHyperLinksFromText(textNode) { return textNode.match(regexURL); } +const linkMarkupToNode = (lexPositions, content) => { + return ast => { + const positions = [...lexPositions]; + const lines = content.split('\n'); + return ast.map(node => { + let [startRow, startColumn] = positions.shift(); + let [endRow, endColumn] = positions.shift(); + let markup = lines[startRow - 1].substring(startColumn - 1); + while (startRow < endRow) { + startRow++; + if (startRow === endRow) { + markup += lines[startRow - 1].substring(0, endColumn - 1); + } else { + markup += lines[startRow - 1]; + } + } + node[3] = markup; + return node; + }); + }; +}; + module.exports = { cleanResults, flattenChildren, @@ -266,5 +291,6 @@ module.exports = { autoLinkifyHelper, hyperLinkifiedVersion, seperateTextAndHyperLink, - getHyperLinksFromText + getHyperLinksFromText, + linkMarkupToNode }; From f2ce85f54557d228ea457dd50c415dcc188c2c7d Mon Sep 17 00:00:00 2001 From: rakeshup Date: Fri, 17 Jan 2020 00:35:43 +0530 Subject: [PATCH 2/2] Still work in progress, output extremely unstable --- packages/idyll-ast/src/converters/index.js | 1 + packages/idyll-compiler/src/lexer.js | 32 +++++++++---------- .../idyll-compiler/src/processors/post.js | 7 +++- 3 files changed, 23 insertions(+), 17 deletions(-) diff --git a/packages/idyll-ast/src/converters/index.js b/packages/idyll-ast/src/converters/index.js index eefbbcca1..2d6d7974a 100644 --- a/packages/idyll-ast/src/converters/index.js +++ b/packages/idyll-ast/src/converters/index.js @@ -103,6 +103,7 @@ function inverseConvertHelper(arrayElement, id) { elementJson.value = arrayElement; } else if (['var', 'derived', 'data', 'meta'].indexOf(arrayElement[0]) > -1) { elementJson.type = arrayElement[0]; + elementJson.raw = arrayElement[3]; elementJson.properties = {}; arrayElement[1].forEach(property => { elementJson.properties[property[0]] = { diff --git a/packages/idyll-compiler/src/lexer.js b/packages/idyll-compiler/src/lexer.js index a56139cf0..cfe12e0e2 100644 --- a/packages/idyll-compiler/src/lexer.js +++ b/packages/idyll-compiler/src/lexer.js @@ -67,8 +67,8 @@ const lex = function(options, alias = {}) { ) ].join('|'); }; - var updatePosition = function(lexeme, shouldNotPush) { - if (!shouldNotPush) { + var updatePosition = function(lexeme, shouldNotPush, isStart, isEnd) { + if (!shouldNotPush && !isEnd) { positions.push([row, column]); } var lines = lexeme.split('\n'); @@ -77,7 +77,7 @@ const lex = function(options, alias = {}) { column = 1; } column += lines[lines.length - 1].length; - if (!shouldNotPush) { + if (!shouldNotPush && !isStart) { positions.push([row, column]); } }; @@ -312,7 +312,7 @@ const lex = function(options, alias = {}) { }); lexer.addRule(/(\n\s*\/\/[^\n]*|\/\/\s+[^\n]*)/, function(lexeme) { - updatePosition(lexeme); + updatePosition(lexeme, true); if (lexeme.startsWith('\n')) { return ['BREAK']; } @@ -354,13 +354,13 @@ const lex = function(options, alias = {}) { }); lexer.addRule(/[ \t\n]+/, function(lexeme) { - updatePosition(lexeme); + updatePosition(lexeme, true); }); lexer.addRule(/\[/, function(lexeme) { inComponent = true; if (this.reject) return; - updatePosition(lexeme); + updatePosition(lexeme, false, true); return ['OPEN_BRACKET']; }); @@ -368,7 +368,7 @@ const lex = function(options, alias = {}) { inComponent = false; gotName = false; if (this.reject) return; - updatePosition(lexeme); + updatePosition(lexeme, false, false, true); var ret = ['CLOSE_BRACKET']; if (trailingSpace) { ret = ret.concat(['WORDS']).concat(formatToken(trailingSpace)); @@ -379,14 +379,14 @@ const lex = function(options, alias = {}) { lexer.addRule(/\//, function(lexeme) { this.reject = !inComponent; if (this.reject) return; - updatePosition(lexeme); + updatePosition(lexeme, true); return ['FORWARD_SLASH']; }); lexer.addRule(/true|false/, function(lexeme) { this.reject = !inComponent; if (this.reject) return; - updatePosition(lexeme); + updatePosition(lexeme, true); return ['BOOLEAN'].concat(formatToken(lexeme)); }); @@ -396,21 +396,21 @@ const lex = function(options, alias = {}) { this.reject = !inComponent || gotName; if (this.reject) return; gotName = true; - updatePosition(lexeme); + updatePosition(lexeme, true); return ['COMPONENT_NAME'].concat(formatToken(lexeme)); } ); lexer.addRule(/[^+\-0-9:\s\/\]"'`\.][^:\s\/\]"'`\.]*/, function(lexeme) { this.reject = !inComponent; if (this.reject) return; - updatePosition(lexeme); + updatePosition(lexeme, true); return ['COMPONENT_WORD'].concat(formatToken(lexeme)); }); lexer.addRule(/`[^`]*`/, function(lexeme) { this.reject = !inComponent; if (this.reject) return; - updatePosition(lexeme); + updatePosition(lexeme, true); return ['EXPRESSION'].concat(formatToken(lexeme)); }); @@ -419,27 +419,27 @@ const lex = function(options, alias = {}) { (lexeme.match(new RegExp(/\./, 'g')) || []).length >= 2; this.reject = !inComponent || multiplePeriods; if (this.reject) return; - updatePosition(lexeme); + updatePosition(lexeme, true); return ['NUMBER'].concat(formatToken(lexeme)); }); lexer.addRule(/"[^"]*"/, function(lexeme) { this.reject = !inComponent; if (this.reject) return; - updatePosition(lexeme); + updatePosition(lexeme, true); return ['STRING'].concat(formatToken(lexeme)); }); lexer.addRule(/'([^']*)'/, function(lexeme, str) { this.reject = !inComponent; if (this.reject) return; - updatePosition(lexeme); + updatePosition(lexeme, true); return ['STRING'].concat(formatToken('"' + str + '"')); }); lexer.addRule(/:/, function(lexeme) { this.reject = !inComponent; if (this.reject) return; - updatePosition(lexeme); + updatePosition(lexeme, true); return ['PARAM_SEPARATOR']; }); diff --git a/packages/idyll-compiler/src/processors/post.js b/packages/idyll-compiler/src/processors/post.js index 1dd05a4e7..5093daece 100644 --- a/packages/idyll-compiler/src/processors/post.js +++ b/packages/idyll-compiler/src/processors/post.js @@ -266,7 +266,12 @@ const linkMarkupToNode = (lexPositions, content) => { return ast.map(node => { let [startRow, startColumn] = positions.shift(); let [endRow, endColumn] = positions.shift(); - let markup = lines[startRow - 1].substring(startColumn - 1); + let markup; + if (startRow !== endRow) { + markup = lines[startRow - 1].substring(startColumn - 1); + } else { + markup = lines[startRow - 1].substring(startColumn - 1, endColumn - 1); + } while (startRow < endRow) { startRow++; if (startRow === endRow) {