Skip to content

Commit

Permalink
wip
Browse files Browse the repository at this point in the history
  • Loading branch information
@DavidAnson
DavidAnson committed Oct 23, 2024
1 parent d94b78e commit 758c92b
Show file tree
Hide file tree
Showing 4 changed files with 92 additions and 32 deletions.
62 changes: 46 additions & 16 deletions demo/markdownlint-browser.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -558,12 +558,15 @@ module.exports.expandTildePath = expandTildePath;



// Symbol for identifing the flat tokens array from micromark parse
module.exports.flatTokensSymbol = Symbol("flat-tokens");

// Symbol for identifying the htmlFlow token from micromark parse
module.exports.htmlFlowSymbol = Symbol("html-flow");

// Symbol for identifing the token lists map from micromark parse
module.exports.tokenListsSymbol = Symbol("token-lists");

// Symbol for identifying the token sequence number for micromark parse
module.exports.tokenSequenceSymbol = Symbol("token-sequence");

// Regular expression for matching common newline characters
// See NEWLINES_RE in markdown-it/lib/rules_core/normalize.js
module.exports.newLineRe = /\r\n?|\n/g;
Expand Down Expand Up @@ -684,7 +687,7 @@ module.exports = {



const { flatTokensSymbol, htmlFlowSymbol } = __webpack_require__(/*! ./shared.js */ "../helpers/shared.js");
const { htmlFlowSymbol, tokenListsSymbol, tokenSequenceSymbol } = __webpack_require__(/*! ./shared.js */ "../helpers/shared.js");

/** @typedef {import("markdownlint-micromark").TokenType} TokenType */
/** @typedef {import("../lib/markdownlint.js").MicromarkToken} Token */
Expand Down Expand Up @@ -806,11 +809,18 @@ function filterByPredicate(tokens, allowed, transformChildren) {
* @returns {Token[]} Filtered tokens.
*/
function filterByTypes(tokens, types, htmlFlow) {
const predicate = (token) =>
(htmlFlow || !inHtmlFlow(token)) && types.includes(token.type);
const flatTokens = tokens[flatTokensSymbol];
if (flatTokens) {
return flatTokens.filter(predicate);
const predicate = (token) => (htmlFlow || !inHtmlFlow(token)) && types.includes(token.type);
const tokenLists = tokens[tokenListsSymbol];
if (tokenLists) {
let filtered = [];
for (const type of types) {
const tokenList = tokenLists.get(type) || [];
// Avoid stack overflow of Array.push(...spread)
// eslint-disable-next-line unicorn/prefer-spread
filtered = filtered.concat(htmlFlow ? tokenList : tokenList.filter((token) => !inHtmlFlow(token)));
}
filtered.sort((a, b) => a[tokenSequenceSymbol] - b[tokenSequenceSymbol]);
return filtered;
}
return filterByPredicate(tokens, predicate);
}
Expand Down Expand Up @@ -995,14 +1005,15 @@ module.exports = {

const micromark = __webpack_require__(/*! markdownlint-micromark */ "markdownlint-micromark");
const { isHtmlFlowComment } = __webpack_require__(/*! ./micromark-helpers.cjs */ "../helpers/micromark-helpers.cjs");
const { flatTokensSymbol, htmlFlowSymbol, newLineRe } = __webpack_require__(/*! ./shared.js */ "../helpers/shared.js");
const { htmlFlowSymbol, newLineRe, tokenListsSymbol, tokenSequenceSymbol } = __webpack_require__(/*! ./shared.js */ "../helpers/shared.js");

/** @typedef {import("markdownlint-micromark").Construct} Construct */
/** @typedef {import("markdownlint-micromark").Event} Event */
/** @typedef {import("markdownlint-micromark").ParseOptions} MicromarkParseOptions */
/** @typedef {import("markdownlint-micromark").State} State */
/** @typedef {import("markdownlint-micromark").Token} Token */
/** @typedef {import("markdownlint-micromark").Tokenizer} Tokenizer */
/** @typedef {import("markdownlint-micromark").TokenType} TokenType */
/** @typedef {import("../lib/markdownlint.js").MicromarkToken} MicromarkToken */

/**
Expand Down Expand Up @@ -1166,13 +1177,28 @@ function getEvents(
}
}

/**
* Gets the sequence number for a token list map.
*
* @param {Map<TokenType, MicromarkToken[]>} tokenLists Token list map.
* @returns {number} Sequence number.
*/
function getSequence(tokenLists) {
let sequence = 0;
for (const tokenList of tokenLists.values()) {
sequence += tokenList.length;
}
return sequence;
}

/**
* Parses a Markdown document and returns micromark tokens (internal).
*
* @param {string} markdown Markdown document.
* @param {ParseOptions} [parseOptions] Options.
* @param {MicromarkParseOptions} [micromarkParseOptions] Options for micromark.
* @param {number} [lineDelta] Offset for start/end line.
* @param {Map<TokenType, MicromarkToken[]>} [tokenLists] Token list map.
* @param {MicromarkToken} [ancestor] Parent of top-most tokens.
* @returns {MicromarkToken[]} Micromark tokens.
*/
Expand All @@ -1181,6 +1207,7 @@ function parseInternal(
parseOptions = {},
micromarkParseOptions = {},
lineDelta = 0,
tokenLists = new Map(),
ancestor = undefined
) {
// Get options
Expand All @@ -1191,7 +1218,7 @@ function parseInternal(

// Create Token objects
const document = [];
let flatTokens = [];
let sequence = getSequence(tokenLists);
/** @type {MicromarkToken} */
const root = {
"type": "data",
Expand Down Expand Up @@ -1229,11 +1256,14 @@ function parseInternal(
"children": [],
"parent": ((previous === root) ? (ancestor || null) : previous)
};
Object.defineProperty(current, tokenSequenceSymbol, { "value": sequence++ });
if (ancestor) {
Object.defineProperty(current, htmlFlowSymbol, { "value": true });
}
previous.children.push(current);
flatTokens.push(current);
const tokenList = tokenLists.get(type) || [];
tokenList.push(current);
tokenLists.set(type, tokenList);
if ((current.type === "htmlFlow") && !isHtmlFlowComment(current)) {
skipHtmlFlowChildren = true;
if (!reparseOptions || !lines) {
Expand All @@ -1257,12 +1287,12 @@ function parseInternal(
parseOptions,
reparseOptions,
current.startLine - 1,
tokenLists,
current
);
current.children = tokens;
// Avoid stack overflow of Array.push(...spread)
// eslint-disable-next-line unicorn/prefer-spread
flatTokens = flatTokens.concat(tokens[flatTokensSymbol]);
// Reset sequence
sequence = getSequence(tokenLists);
}
} else if (kind === "exit") {
if (type === "htmlFlow") {
Expand All @@ -1280,7 +1310,7 @@ function parseInternal(
}

// Return document
Object.defineProperty(document, flatTokensSymbol, { "value": flatTokens });
Object.defineProperty(document, tokenListsSymbol, { "value": tokenLists });
if (freezeTokens) {
Object.freeze(document);
}
Expand Down
19 changes: 13 additions & 6 deletions helpers/micromark-helpers.cjs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@

"use strict";

const { flatTokensSymbol, htmlFlowSymbol } = require("./shared.js");
const { htmlFlowSymbol, tokenListsSymbol, tokenSequenceSymbol } = require("./shared.js");

/** @typedef {import("markdownlint-micromark").TokenType} TokenType */
/** @typedef {import("../lib/markdownlint.js").MicromarkToken} Token */
Expand Down Expand Up @@ -124,11 +124,18 @@ function filterByPredicate(tokens, allowed, transformChildren) {
* @returns {Token[]} Filtered tokens.
*/
function filterByTypes(tokens, types, htmlFlow) {
const predicate = (token) =>
(htmlFlow || !inHtmlFlow(token)) && types.includes(token.type);
const flatTokens = tokens[flatTokensSymbol];
if (flatTokens) {
return flatTokens.filter(predicate);
const predicate = (token) => (htmlFlow || !inHtmlFlow(token)) && types.includes(token.type);
const tokenLists = tokens[tokenListsSymbol];
if (tokenLists) {
let filtered = [];
for (const type of types) {
const tokenList = tokenLists.get(type) || [];
// Avoid stack overflow of Array.push(...spread)
// eslint-disable-next-line unicorn/prefer-spread
filtered = filtered.concat(htmlFlow ? tokenList : tokenList.filter((token) => !inHtmlFlow(token)));
}
filtered.sort((a, b) => a[tokenSequenceSymbol] - b[tokenSequenceSymbol]);
return filtered;
}
return filterByPredicate(tokens, predicate);
}
Expand Down
34 changes: 27 additions & 7 deletions helpers/micromark-parse.cjs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,14 +4,15 @@

const micromark = require("markdownlint-micromark");
const { isHtmlFlowComment } = require("./micromark-helpers.cjs");
const { flatTokensSymbol, htmlFlowSymbol, newLineRe } = require("./shared.js");
const { htmlFlowSymbol, newLineRe, tokenListsSymbol, tokenSequenceSymbol } = require("./shared.js");

/** @typedef {import("markdownlint-micromark").Construct} Construct */
/** @typedef {import("markdownlint-micromark").Event} Event */
/** @typedef {import("markdownlint-micromark").ParseOptions} MicromarkParseOptions */
/** @typedef {import("markdownlint-micromark").State} State */
/** @typedef {import("markdownlint-micromark").Token} Token */
/** @typedef {import("markdownlint-micromark").Tokenizer} Tokenizer */
/** @typedef {import("markdownlint-micromark").TokenType} TokenType */
/** @typedef {import("../lib/markdownlint.js").MicromarkToken} MicromarkToken */

/**
Expand Down Expand Up @@ -175,13 +176,28 @@ function getEvents(
}
}

/**
* Gets the sequence number for a token list map.
*
* @param {Map<TokenType, MicromarkToken[]>} tokenLists Token list map.
* @returns {number} Sequence number.
*/
function getSequence(tokenLists) {
let sequence = 0;
for (const tokenList of tokenLists.values()) {
sequence += tokenList.length;
}
return sequence;
}

/**
* Parses a Markdown document and returns micromark tokens (internal).
*
* @param {string} markdown Markdown document.
* @param {ParseOptions} [parseOptions] Options.
* @param {MicromarkParseOptions} [micromarkParseOptions] Options for micromark.
* @param {number} [lineDelta] Offset for start/end line.
* @param {Map<TokenType, MicromarkToken[]>} [tokenLists] Token list map.
* @param {MicromarkToken} [ancestor] Parent of top-most tokens.
* @returns {MicromarkToken[]} Micromark tokens.
*/
Expand All @@ -190,6 +206,7 @@ function parseInternal(
parseOptions = {},
micromarkParseOptions = {},
lineDelta = 0,
tokenLists = new Map(),
ancestor = undefined
) {
// Get options
Expand All @@ -200,7 +217,7 @@ function parseInternal(

// Create Token objects
const document = [];
let flatTokens = [];
let sequence = getSequence(tokenLists);
/** @type {MicromarkToken} */
const root = {
"type": "data",
Expand Down Expand Up @@ -238,11 +255,14 @@ function parseInternal(
"children": [],
"parent": ((previous === root) ? (ancestor || null) : previous)
};
Object.defineProperty(current, tokenSequenceSymbol, { "value": sequence++ });
if (ancestor) {
Object.defineProperty(current, htmlFlowSymbol, { "value": true });
}
previous.children.push(current);
flatTokens.push(current);
const tokenList = tokenLists.get(type) || [];
tokenList.push(current);
tokenLists.set(type, tokenList);
if ((current.type === "htmlFlow") && !isHtmlFlowComment(current)) {
skipHtmlFlowChildren = true;
if (!reparseOptions || !lines) {
Expand All @@ -266,12 +286,12 @@ function parseInternal(
parseOptions,
reparseOptions,
current.startLine - 1,
tokenLists,
current
);
current.children = tokens;
// Avoid stack overflow of Array.push(...spread)
// eslint-disable-next-line unicorn/prefer-spread
flatTokens = flatTokens.concat(tokens[flatTokensSymbol]);
// Reset sequence
sequence = getSequence(tokenLists);
}
} else if (kind === "exit") {
if (type === "htmlFlow") {
Expand All @@ -289,7 +309,7 @@ function parseInternal(
}

// Return document
Object.defineProperty(document, flatTokensSymbol, { "value": flatTokens });
Object.defineProperty(document, tokenListsSymbol, { "value": tokenLists });
if (freezeTokens) {
Object.freeze(document);
}
Expand Down
9 changes: 6 additions & 3 deletions helpers/shared.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,12 +2,15 @@

"use strict";

// Symbol for identifing the flat tokens array from micromark parse
module.exports.flatTokensSymbol = Symbol("flat-tokens");

// Symbol for identifying the htmlFlow token from micromark parse
module.exports.htmlFlowSymbol = Symbol("html-flow");

// Symbol for identifing the token lists map from micromark parse
module.exports.tokenListsSymbol = Symbol("token-lists");

// Symbol for identifying the token sequence number for micromark parse
module.exports.tokenSequenceSymbol = Symbol("token-sequence");

// Regular expression for matching common newline characters
// See NEWLINES_RE in markdown-it/lib/rules_core/normalize.js
module.exports.newLineRe = /\r\n?|\n/g;
Expand Down

0 comments on commit 758c92b

Please sign in to comment.