From 511ef43e6deee90c4c2e4c99d75cee07691d0fb4 Mon Sep 17 00:00:00 2001 From: Lukas Domagala Date: Fri, 19 Aug 2022 16:44:52 +0200 Subject: [PATCH 1/6] rich comment handling in notebooks --- src/NotebookProvider.ts | 63 ++++++++++++++++++++++++++++++++-- src/cursor-doc/token-cursor.ts | 12 +++++++ 2 files changed, 73 insertions(+), 2 deletions(-) diff --git a/src/NotebookProvider.ts b/src/NotebookProvider.ts index 9ef273c02..68507caca 100644 --- a/src/NotebookProvider.ts +++ b/src/NotebookProvider.ts @@ -4,6 +4,7 @@ import { prettyPrint } from '../out/cljs-lib/cljs-lib'; import * as tokenCursor from './cursor-doc/token-cursor'; import * as repl from './api/repl'; import _ = require('lodash'); +import { isInteger } from 'lodash'; export class NotebookProvider implements vscode.NotebookSerializer { private readonly decoder = new TextDecoder(); @@ -30,9 +31,17 @@ export class NotebookProvider implements vscode.NotebookSerializer { } } +function substring(content: string, [start, end]) { + if (isInteger(start) && isInteger(end)) { + return content.substring(start, end); + } + return ''; +} + function parseClojure(content: string): vscode.NotebookCellData[] { const cursor = tokenCursor.createStringCursor(content); const topLevelRanges = cursor.rangesForTopLevelForms().flat(); + if (topLevelRanges.length) { topLevelRanges[0] = 0; } @@ -48,18 +57,68 @@ function parseClojure(content: string): vscode.NotebookCellData[] { // start of file to end of top level sexp pairs const allRanges = _.zip(_.dropRight([_.first(topLevelRanges), ...fullRanges], 1), fullRanges); - const ranges = allRanges.map(([start, end]) => { + const ranges = allRanges.flatMap(([start, end]) => { + const endForm = cursor.doc.getTokenCursor(end - 1); + const afterForm = cursor.doc.getTokenCursor(end); + + if (endForm.getFunctionName() === 'comment') { + const commentRange = afterForm.rangeForCurrentForm(0); + const commentStartCursor = cursor.doc.getTokenCursor(commentRange[0]); + const commentCells = []; + let count = 0; + + commentStartCursor.downList(); + commentStartCursor.forwardSexp(); + + while (commentStartCursor.forwardSexp()) { + commentCells.push({ + value: substring( + content, + commentStartCursor.rangeForDefun(commentStartCursor.offsetStart) + ), + kind: vscode.NotebookCellKind.Code, + languageId: 'clojure', + metadata: { + richComment: { index: count }, + }, + }); + count++; + } + commentCells.forEach((x) => (x.metadata.richComment.count = count)); + + return commentCells; + } return { value: content.substring(start, end), kind: vscode.NotebookCellKind.Code, languageId: 'clojure', }; }); + return ranges; } function writeCellsToClojure(cells: vscode.NotebookCellData[]) { - return cells.map((x) => x.value).join(''); + return cells.reduce((acc, x) => { + if (x.kind === vscode.NotebookCellKind.Code) { + let result: string = x.value; + if (x.metadata?.richComment) { + if (x.metadata.richComment.index === 0) { + result = '\n(comment\n'.concat(result); + } + + if (x.metadata.richComment.index === x.metadata.richComment.count - 1) { + result = result.concat(')'); + } else { + result = result.concat('\n'); + } + } + + return acc.concat(result); + } else { + return acc.concat(x.value); + } + }, ''); } export class NotebookKernel { diff --git a/src/cursor-doc/token-cursor.ts b/src/cursor-doc/token-cursor.ts index 97cfab567..a592ca0b8 100644 --- a/src/cursor-doc/token-cursor.ts +++ b/src/cursor-doc/token-cursor.ts @@ -794,6 +794,18 @@ export class LispTokenCursor extends TokenCursor { return ranges; } + rangesForCommentForms(offset: number): [number, number][] { + const cursor = this.doc.getTokenCursor(offset); + const ranges: [number, number][] = []; + while (cursor.forwardSexp()) { + const end = cursor.offsetStart; + cursor.backwardSexp(); + ranges.push([cursor.offsetStart, end]); + cursor.forwardSexp(); + } + return ranges; + } + isWhiteSpace(): boolean { return tokenIsWhiteSpace(this.getToken()); } From 71e49184edf1f095bc2a1c1912c2dfdffaefe112 Mon Sep 17 00:00:00 2001 From: Lukas Domagala Date: Fri, 19 Aug 2022 18:01:03 +0200 Subject: [PATCH 2/6] better whitespace handling in rich comments --- src/NotebookProvider.ts | 45 ++++++++++++++++++++++++----------------- 1 file changed, 26 insertions(+), 19 deletions(-) diff --git a/src/NotebookProvider.ts b/src/NotebookProvider.ts index 68507caca..c44d0604a 100644 --- a/src/NotebookProvider.ts +++ b/src/NotebookProvider.ts @@ -65,33 +65,44 @@ function parseClojure(content: string): vscode.NotebookCellData[] { const commentRange = afterForm.rangeForCurrentForm(0); const commentStartCursor = cursor.doc.getTokenCursor(commentRange[0]); const commentCells = []; - let count = 0; + let previouseEnd = start; commentStartCursor.downList(); commentStartCursor.forwardSexp(); while (commentStartCursor.forwardSexp()) { + const range = commentStartCursor.rangeForDefun(commentStartCursor.offsetStart); + let leading = ''; + + leading = content.substring(previouseEnd, range[0]); + previouseEnd = range[1]; + commentCells.push({ - value: substring( - content, - commentStartCursor.rangeForDefun(commentStartCursor.offsetStart) - ), + value: substring(content, range), kind: vscode.NotebookCellKind.Code, languageId: 'clojure', metadata: { - richComment: { index: count }, + leading: leading, + range, + richComment: true, + trailing: '', }, }); - count++; } - commentCells.forEach((x) => (x.metadata.richComment.count = count)); + + _.last(commentCells).metadata.trailing = content.substring(previouseEnd, end); return commentCells; } + return { value: content.substring(start, end), kind: vscode.NotebookCellKind.Code, languageId: 'clojure', + metadata: { + leading: '', + trailing: '', + }, }; }); @@ -101,17 +112,13 @@ function parseClojure(content: string): vscode.NotebookCellData[] { function writeCellsToClojure(cells: vscode.NotebookCellData[]) { return cells.reduce((acc, x) => { if (x.kind === vscode.NotebookCellKind.Code) { - let result: string = x.value; - if (x.metadata?.richComment) { - if (x.metadata.richComment.index === 0) { - result = '\n(comment\n'.concat(result); - } - - if (x.metadata.richComment.index === x.metadata.richComment.count - 1) { - result = result.concat(')'); - } else { - result = result.concat('\n'); - } + let result = ''; + + // created inside the notebook + if (undefined === x.metadata.leading) { + result = '\n\n' + x.value; + } else { + result = x.metadata.leading + x.value + x.metadata.trailing; } return acc.concat(result); From fb35441d1f931282fedab1786e33c122f7f40a34 Mon Sep 17 00:00:00 2001 From: Lukas Domagala Date: Fri, 19 Aug 2022 18:23:23 +0200 Subject: [PATCH 3/6] fix indent of new cells --- src/NotebookProvider.ts | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/src/NotebookProvider.ts b/src/NotebookProvider.ts index c44d0604a..7438874c4 100644 --- a/src/NotebookProvider.ts +++ b/src/NotebookProvider.ts @@ -73,6 +73,7 @@ function parseClojure(content: string): vscode.NotebookCellData[] { while (commentStartCursor.forwardSexp()) { const range = commentStartCursor.rangeForDefun(commentStartCursor.offsetStart); let leading = ''; + const indent = commentStartCursor.doc.getRowCol(range[0])[1]; // will break with tabs? leading = content.substring(previouseEnd, range[0]); previouseEnd = range[1]; @@ -83,6 +84,7 @@ function parseClojure(content: string): vscode.NotebookCellData[] { languageId: 'clojure', metadata: { leading: leading, + indent, range, richComment: true, trailing: '', @@ -100,6 +102,7 @@ function parseClojure(content: string): vscode.NotebookCellData[] { kind: vscode.NotebookCellKind.Code, languageId: 'clojure', metadata: { + indent: 0, leading: '', trailing: '', }, @@ -110,13 +113,15 @@ function parseClojure(content: string): vscode.NotebookCellData[] { } function writeCellsToClojure(cells: vscode.NotebookCellData[]) { - return cells.reduce((acc, x) => { + return cells.reduce((acc, x, index) => { if (x.kind === vscode.NotebookCellKind.Code) { let result = ''; // created inside the notebook if (undefined === x.metadata.leading) { - result = '\n\n' + x.value; + const indent = index > 0 ? _.repeat(' ', cells[index - 1].metadata.indent) : ''; + + result = '\n\n' + indent + x.value; } else { result = x.metadata.leading + x.value + x.metadata.trailing; } From 7fcc022048ade2c99b4ae6f17b12b3f2232ce0f8 Mon Sep 17 00:00:00 2001 From: Lukas Domagala Date: Sat, 20 Aug 2022 16:36:28 +0200 Subject: [PATCH 4/6] notebook markdown comments --- src/NotebookProvider.ts | 110 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 110 insertions(+) diff --git a/src/NotebookProvider.ts b/src/NotebookProvider.ts index 7438874c4..66fec9789 100644 --- a/src/NotebookProvider.ts +++ b/src/NotebookProvider.ts @@ -57,6 +57,109 @@ function parseClojure(content: string): vscode.NotebookCellData[] { // start of file to end of top level sexp pairs const allRanges = _.zip(_.dropRight([_.first(topLevelRanges), ...fullRanges], 1), fullRanges); + let offset = 0; + let cells = []; + + while (cursor.forwardSexp()) { + const start = offset; + const end = cursor.offsetStart; + offset = end; + + const endForm = cursor.doc.getTokenCursor(end - 1); + const afterForm = cursor.doc.getTokenCursor(end); + + if (endForm.getFunctionName() === 'comment') { + const commentRange = afterForm.rangeForCurrentForm(0); + const commentStartCursor = cursor.doc.getTokenCursor(commentRange[0]); + const commentCells = []; + let previouseEnd = start; + + commentStartCursor.downList(); + commentStartCursor.forwardSexp(); + + while (commentStartCursor.forwardSexp()) { + const range = commentStartCursor.rangeForDefun(commentStartCursor.offsetStart); + + let leading = ''; + const indent = commentStartCursor.doc.getRowCol(range[0])[1]; // will break with tabs? + + leading = content.substring(previouseEnd, range[0]); + previouseEnd = range[1]; + + commentCells.push({ + value: substring(content, range), + kind: vscode.NotebookCellKind.Code, + languageId: 'clojure', + metadata: { + leading: leading, + indent, + range, + richComment: true, + trailing: '', + }, + }); + } + + _.last(commentCells).metadata.trailing = content.substring(previouseEnd, end); + + cells = cells.concat(commentCells); + + continue; + } + + const range = cursor.rangeForDefun(cursor.offsetStart); + + const leading = content.substring(start, range[0]); + + if (leading.indexOf(';; ') === -1) { + cells.push({ + value: leading, + kind: vscode.NotebookCellKind.Markup, + languageId: 'markdown', + metadata: { + indent: 0, + range, + leading: '', + trailing: '', + }, + }); + } else { + cells.push({ + value: leading.replace(/;; /g, ''), + kind: vscode.NotebookCellKind.Markup, + languageId: 'markdown', + metadata: { + indent: 0, + range, + markdownComment: true, + leading: '', + trailing: '', + }, + }); + } + + cells.push({ + value: substring(content, range), + kind: vscode.NotebookCellKind.Code, + languageId: 'clojure', + metadata: { + indent: 0, + range, + leading: '', + trailing: '', + }, + }); + } + + _.last(cells).metadata.trailing = content.substring( + _.last(cells).metadata.range[1], + content.length + ); + + console.log(cells); + + return cells; + const ranges = allRanges.flatMap(([start, end]) => { const endForm = cursor.doc.getTokenCursor(end - 1); const afterForm = cursor.doc.getTokenCursor(end); @@ -128,6 +231,13 @@ function writeCellsToClojure(cells: vscode.NotebookCellData[]) { return acc.concat(result); } else { + if (x.metadata.markdownComment) { + let result = x.value.replace(/\n(?!$)/g, '\n;; '); + if (index === 0) { + result = ';; ' + result; + } + return acc.concat(result); + } return acc.concat(x.value); } }, ''); From 1afc713b59f965ec4cbafd932c5cb1607f60669a Mon Sep 17 00:00:00 2001 From: Lukas Domagala Date: Sun, 21 Aug 2022 19:26:07 +0200 Subject: [PATCH 5/6] remove unused old code --- src/NotebookProvider.ts | 71 ----------------------------------------- 1 file changed, 71 deletions(-) diff --git a/src/NotebookProvider.ts b/src/NotebookProvider.ts index 66fec9789..9115fb1d0 100644 --- a/src/NotebookProvider.ts +++ b/src/NotebookProvider.ts @@ -40,23 +40,6 @@ function substring(content: string, [start, end]) { function parseClojure(content: string): vscode.NotebookCellData[] { const cursor = tokenCursor.createStringCursor(content); - const topLevelRanges = cursor.rangesForTopLevelForms().flat(); - - if (topLevelRanges.length) { - topLevelRanges[0] = 0; - } - - // grab only the ends of ranges, so we can include all of the file in the notebook - const fullRanges = _.filter(topLevelRanges, (_, index) => { - return index % 2 !== 0; - }); - - // last range should include end of file - fullRanges[fullRanges.length - 1] = content.length; - - // start of file to end of top level sexp pairs - const allRanges = _.zip(_.dropRight([_.first(topLevelRanges), ...fullRanges], 1), fullRanges); - let offset = 0; let cells = []; @@ -159,60 +142,6 @@ function parseClojure(content: string): vscode.NotebookCellData[] { console.log(cells); return cells; - - const ranges = allRanges.flatMap(([start, end]) => { - const endForm = cursor.doc.getTokenCursor(end - 1); - const afterForm = cursor.doc.getTokenCursor(end); - - if (endForm.getFunctionName() === 'comment') { - const commentRange = afterForm.rangeForCurrentForm(0); - const commentStartCursor = cursor.doc.getTokenCursor(commentRange[0]); - const commentCells = []; - let previouseEnd = start; - - commentStartCursor.downList(); - commentStartCursor.forwardSexp(); - - while (commentStartCursor.forwardSexp()) { - const range = commentStartCursor.rangeForDefun(commentStartCursor.offsetStart); - let leading = ''; - const indent = commentStartCursor.doc.getRowCol(range[0])[1]; // will break with tabs? - - leading = content.substring(previouseEnd, range[0]); - previouseEnd = range[1]; - - commentCells.push({ - value: substring(content, range), - kind: vscode.NotebookCellKind.Code, - languageId: 'clojure', - metadata: { - leading: leading, - indent, - range, - richComment: true, - trailing: '', - }, - }); - } - - _.last(commentCells).metadata.trailing = content.substring(previouseEnd, end); - - return commentCells; - } - - return { - value: content.substring(start, end), - kind: vscode.NotebookCellKind.Code, - languageId: 'clojure', - metadata: { - indent: 0, - leading: '', - trailing: '', - }, - }; - }); - - return ranges; } function writeCellsToClojure(cells: vscode.NotebookCellData[]) { From 2e2805b213888503e9cfdad69e5d6be258b903ad Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20Str=C3=B6mberg?= Date: Mon, 22 Aug 2022 11:54:08 +0200 Subject: [PATCH 6/6] Add anoter notebook test data file --- test-data/notebook2.clj | 79 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 79 insertions(+) create mode 100644 test-data/notebook2.clj diff --git a/test-data/notebook2.clj b/test-data/notebook2.clj new file mode 100644 index 000000000..dcdde78ed --- /dev/null +++ b/test-data/notebook2.clj @@ -0,0 +1,79 @@ +(ns notebook) + +"string1" "string2" +;; A line comment +;; block + +(def hover-map + {:stuff "in-here" + :ohter "yeah" + ;; Yada yada + :deeper {:a 1, "foo" :bar, [1 2 3] (vec (range 2000))}}) +;; Foo + +;; Line comment, line 1 +;; +;; Line comment, line 2 +(defn foo [] + (println "bar")) + +(foo) + +;; A line comment + +hover-map + +" testing

HTML Image

" +; foo +(comment + (+ 40 2) + + 42 + + 42/1 + + "forty-two") + +(def forty-two + (+ + 40 + 1 + 1)) + +; c3 + +;c4 + +"one" + +;c5 + +"two" + +;c6 +"three" +;c7 + +"four" +;c8 +:five + +;c9 +six + +:seven +"eight" +nine + +ten :eleven "twelve" + +; c10 + +:thirteen + +"fourteen" + +;c11 +;c12 + +:fifteen "sixteen" seventeen \ No newline at end of file