experiment: rascal formatter based on Box and layoutDiff

src/org/rascalmpl/interpreter/IEvaluatorContext.java

@@ -23,7 +23,6 @@
 import java.util.Optional;
 import java.util.Stack;
 
-import org.checkerframework.checker.nullness.qual.Nullable;
 import org.rascalmpl.ast.AbstractAST;
 import org.rascalmpl.debug.IRascalMonitor;
 import org.rascalmpl.exceptions.StackTrace;

src/org/rascalmpl/library/List.rsc

@@ -655,6 +655,29 @@ tuple[list[&T],list[&T]] split(list[&T] l) {
 	return <take(half,l), drop(half,l)>;
 }
 
+@synopsis{Groups sublists for consecutive elements which are `similar`}
+@description{
+This function does not change the order of the elements. Only elements
+which are similar end-up in a sub-list with more than one element. The
+elements which are not similar to their siblings, end up in singleton
+lists.
+}
+@examples{
+```rascal-shell
+import List;
+bool bothEvenOrBothOdd(int a, int b) = (a % 2 == 0 && b % 2 == 0) || (a % 2 == 1 && b % 2 == 1);
+group([1,7,3,6,2,9], bothEvenOrBothOdd);
+```
+}
+public list[list[&T]] group(list[&T] input, bool (&T a, &T b) similar) {
+  lres = while ([hd, *tl] := input) {
+      sim = [hd, *takeWhile(tl, bool (&T a) { return similar(a, hd); })];
+	    append sim;
+	    input = drop(size(sim), input);
+  }
+
+  return lres; 
+}
 
 @synopsis{Sum the elements of a list.}
 @examples{

src/org/rascalmpl/library/analysis/diff/edits/HiFiLayoutDiff.rsc

@@ -27,6 +27,8 @@ module analysis::diff::edits::HiFiLayoutDiff
 extend analysis::diff::edits::HiFiTreeDiff;
 import ParseTree; // this should not be necessary because imported by HiFiTreeDiff
 import String; // this should not be be necessary because imported by HiFiTreeDiff
+import lang::rascal::grammar::definition::Characters;
+import IO;
 
 @synopsis{Normalization choices for case-insensitive literals.}
 data CaseInsensitivity
@@ -106,22 +108,31 @@ list[TextEdit] layoutDiff(Tree original, Tree formatted, bool recoverComments =
     default list[TextEdit] rec(
         Tree t:appl(Production p, list[Tree] argsA),
         appl(p /* must be the same by the above assert */, list[Tree] argsB)) 
-        = [*rec(a, b) | <a, b> <- zip2(argsA, argsB)]; 
+        = [*rec(argsA[i], argsB[i]) | i <- [0..size(argsA)]]; 
 
     // first add required locations to layout nodes
-    original = reposition(original, markLit=true, markLayout=true, markSubLayout=true);
+    // TODO: check if indeed repositioning is never needed
+    // original = reposition(original, markLit=true, markLayout=true, markSubLayout=true);
 
     return rec(original, formatted);
 }
 
+private Symbol newlineClass = \char-class([range(10,10)]);
+
 @synopsis{Make sure the new layout still contains all the source code comments of the original layout}
 @description{
-This algorithm uses the @category("Comments") tag to detect source code comments inside layout substrings. If the original
+This algorithm uses the `@category(/[cC]omments/)` tag to detect source code comments inside layout substrings. If the original
 layout contains comments, we re-introduce the comments at the expected level of indentation. New comments present in the 
 replacement are kept and will overwrite any original comments. 
 
-This trick is complicated by the syntax of multiline comments and single line comments that have
-to end with a newline.
+There are corner cases with respect to the original comments:
+* the single line comment that does not end with a newline itself, yet it must always end with a newline after it.
+* multiple single line comments after each other
+
+Then there are corner cases with respect to the replacement whitespace:
+* the last line of the replacement whitespace is special. This is the indentation to use for all comments.
+* but there could be no newlines in the replacement whitespace; and still there is a single line comment to be included.
+Now we need to infer an indentation level for what follows the comment from "thin air". 
 }
 @benefits{
 * if comments are kepts and formatted by tools like Tree2Box, then this algorithm does not overwrite these.
@@ -132,7 +143,14 @@ to end with a newline.
 * if comments are not marked with `@category("Comment")` in the original grammar, then this algorithm recovers nothing.
 }
 private str learnComments(Tree original, Tree replacement) {
-    originalComments = ["<c>" | /c:appl(prod(_,_,{\tag("category"(/^[Cc]omment$/)), *_}), _) := original];
+    bool mustEndWithNewline(lit("\n"))                     = true;
+    bool mustEndWithNewline(conditional(Symbol s, _))      = mustEndWithNewline(s);
+    // if a comment can not contain newline characters, but everything else, then it must be followed by one:
+    bool mustEndWithNewline(\iter(Symbol cc:\char-class(_)))      = intersection(cc, newlineClass) != newlineClass;
+    bool mustEndWithNewline(\iter-star(Symbol cc:\char-class(_))) = intersection(cc, newlineClass) != newlineClass;
+    default  bool mustEndWithNewline(_)                    = false;
+
+    originalComments = [<s, s[-1] == "\n" || mustEndWithNewline(lastSym)> | /c:appl(prod(_,[*_,Symbol lastSym],{\tag("category"(/^[Cc]omment$/)), *_}), _) := original, str s := "<c>"];
 
     if (originalComments == []) {
         // if the original did not contain comments, stick with the replacements
@@ -146,23 +164,42 @@ private str learnComments(Tree original, Tree replacement) {
         return "<replacement>";
     }
 
-    // At this point, we know that: (a) comments are not present in the replacement and (b) they used to be there in the original.
-    // So the old comments are going to be the new output. however, we want to learn indentation from the replacement.
+    // At this point, we know that: 
+    //   (a) comments are not present in the replacement and 
+    //   (b) they used to be there in the original.
+    // So the old comments are going to be copied to the new output.
+    // But, we want to indent them using the style of the replacement.
+
+    // The last line of the replacement string typically has the indentation for the construct that follows:
+    //   |    // a comment 
+    //   |    if (true) {
+    //    ^^^^
+    //      newIndent
+    //
+    // However, if the replacement string is on a single line, then we don't have the indentation
+    // for the string on the next line readily available. In this case we indent the next line
+    // to the start column of the replacement layout, as a proxy.
+
+    str replString = "<replacement>";
+    str newIndent  = split("\n", replString)[-1] ? "";
 
-    // Drop the last newline of single-line comments, because we don't want two newlines in the output for every comment:
-    str dropEndNl(str line:/^.*\n$/) = (line[..-1]);
-    default str dropEndNl(str line)  = line;
+    if (/\n/ !:= replString) {
+        // no newline in the repl string, so no indentation available for what follows the comment...
+        newIndent = "<for (_ <- [0..replacement@\loc.begin.column]) {> <}>";
+    }
 
-    // the first line of the replacement ,is the indentation to use.
-    str replString = "<replacement>";
-    str replacementIndent = /^\n+$/ !:= replString
-        ? split("\n", replString)[0]
-        : "";
-
-    // trimming each line makes sure we forget about the original indentation, and drop accidental spaces after comment lines
-    return replString + indent(replacementIndent,
-            "<for (c <- originalComments, str line <- split("\n", dropEndNl(c))) {><indent(replacementIndent, trim(line), indentFirstLine=true)>
-           '<}>"[..-1], indentFirstLine=false) + replString;
+    // we always place sequential comments vertically, because we don't know if we are dealing
+    // we a single line comment that has to end with newline by follow restriction or by a literal "\n".
+    // TODO: a deeper analysis of the comment rule that's in use could also be used to discover this.
+    str trimmedOriginals = "<for (<c, newLine> <- originalComments) {><trim(c)><if (newLine) {>
+                           '<}><}>";
+
+    // we wrap the comment with the formatted whitespace to assure the proper indentation
+    // of its first line, and the proper indentation of what comes after this layout node
+    return replString 
+        + indent(newIndent, trimmedOriginals, indentFirstLine=false) 
+        + newIndent
+        ;
 }
 
 private Symbol delabel(label(_, Symbol t)) = t;

src/org/rascalmpl/library/lang/box/syntax/Box.rsc

@@ -10,6 +10,8 @@
 @synopsis{An abstract declarative language for two dimensional text layout}
 module lang::box::\syntax::Box
 
+import List;
+
 @synopsis{Every kind of boxes encodes one or more parameterized two-dimensional text constraints.}
 @description{
 * `H` puts their elements next to each other one the same line separated by `hs` spaces.
@@ -22,8 +24,10 @@ module lang::box::\syntax::Box
 * `SPACE` produces `space` spaces
 * `L` produces A literal word. This word may only contain printable characters and no spaces; this is a required property that the formatting algorithm depends on for correctness.
 * `U` splices its contents in the surrounding box, for automatic flattening of overly nested structures in syntax trees.
-* `G` is an additional group-by feature that reduces tot the above core features
-* `SL` is a convenience box for separated syntax lists based on `G`
+* `G` is an additional group-by feature for `list[Box]` that reduces tot the above core features. You can use it to wrap another
+box around every `gs` elements.
+* `AG` is an additional group-by feature for array `Row`s that reduces to the above core features. You can use it to wrap a `R` row
+around every `gs` elements and then construct an `A` around those rows.
 * `NULL()` is the group that will dissappear from its context, useful for skipping content. It is based on the `U` box.
 }
 @benefits{
@@ -38,27 +42,44 @@ set on every `I` Box according to the current preferences of the user.
 * `U(boxes)` is rendered as `H(boxes)` if it's the outermost Box.
 }
 data Box(int hs=1, int vs=0, int is=4)
-    = H(list[Box] boxes)
-    | V(list[Box] boxes)
-    | HOV(list[Box] boxes)
-    | HV(list[Box] boxes)
-    | I(list[Box] boxes)
-    | WD(list[Box] boxes)
-    | A(list[Row] rows, list[Alignment] columns=[l() | [R(list[Box] cs), *_] := rows, _ <- cs] /* learns the amount of columns from the first row */)
+    = _H(list[Box] boxes)
+    | _V(list[Box] boxes)
+    | _HOV(list[Box] boxes)
+    | _HV(list[Box] boxes)
+    | _I(list[Box] boxes) 
+    | _WD(list[Box] boxes)
+    | _A(list[Row] rows, Box rs=NULL(), list[Alignment] columns=[])
+    | _AG(list[Box] boxes, int gs=2, list[Alignment] columns=[], Box rs=NULL())
     | SPACE(int space)
     | L(str word)
-    | U(list[Box] boxes)
-    | G(list[Box] boxes, Box(list[Box]) op = H, int gs=2)
+    | _U(list[Box] boxes)
+    | _G(list[Box] boxes, bool backwards=false, int gs=2, Box op = H([]))
     | NULL()
     ;
 
+Box H(Box boxes..., int hs=1) = _H(boxes, hs=hs);
+Box V(Box boxes..., int vs=0) = _V(boxes, vs=vs);
+Box HOV(Box boxes..., int hs=1, int vs=0) = _HOV(boxes, hs=hs, vs=vs);
+Box HV(Box boxes..., int hs=1, int vs=0) = _HV(boxes, hs=hs, vs=vs);
+Box I(Box boxes...) = _I(boxes);
+Box WD(Box boxes...) = _WD(boxes);
+Box A(Row rows..., Box rs=NULL(), list[Alignment] columns=[])
+    = _A(rows, rs=rs, columns=columns);
+Box AG(Box boxes..., int gs=2, list[Alignment] columns=[], Box rs=NULL())
+    = _AG(boxes, gs=gs, columns=columns, rs=rs);
+Box U(Box boxes...) = _U(boxes);
+Box G(Box boxes..., bool backwards=false, int gs=2, Box op = H([]))
+    = _G(boxes, backwards=backwards, gs=gs, op=op);
+
 @synopsis{A row is a list of boxes that go into an `A` array/table.}
 @description{
 Rows do not have parameters. These are set on the `A` level instead,
 or per cell Box. 
 }
 data Row = R(list[Box] cells);
 
+// Row R(Box cells...) = _R(cells);
+
 data Alignment = l() | r() | c();  
 
 @synopsis{NULL can be used to return a Box that will completely dissappear in the surrounding context.}
@@ -81,4 +102,50 @@ algorithm starts counting boxes and widths.
 * Do not use `NULL` for empty Row cells, unless you do want your cells aligned to the left and filled up to the right with empty H boxes.
 * NULL will be formatted as `H([])` if it's the outermost Box.
 }
-Box NULL() = U([]);
+Box NULL() = U([]);
+
+@synopsis{Convenience box for adding separators to an existing box list}
+@description{
+Each element is wrapped by the `op` operator together with the next separator.
+The resulting list is wrapped by a G box, of which the elements will be spliced
+into their context. 
+}
+Box SL(list[Box] boxes, Box sep, Box op = H([], hs=0))
+  = G([b, sep | b <- boxes][..-1], op=op, gs=2);
+
+@synopsis{Flatten and fold U and G boxes to simplify the Box structure}
+@description{
+U and G and AG boxes greatly simplify the Box tree before it is formatted. This
+happens "just-in-time" for efficiency reasons. However, from a Box tree
+with many U and G boxes it can become hard to see what the actual formatting
+constraints are going to be. 
+
+This function applies the semantics of G and U and returns a Box that renders
+exactly the same output, but with a lot less nested structure. 
+}
+@benefits{
+* useful to debug complex `toBox` mappings
+* formatting semantics preserving transformation
+}
+@pitfalls{
+* only useful for debugging purposes, because it becomes a pipeline bottleneck otherwise.
+}
+Box debUG(Box b) {
+    list[Box] groupBy([], int _gs, Box _op) = [];
+    list[Box] groupBy(list[Box] boxes:[Box _, *_], int gs, Box op)
+        = [op[boxes=boxes[..gs]], *groupBy(boxes[gs..], gs, op)];
+
+    list[Box] groupByBackward([], int _gs, Box _op) = [];
+    list[Box] groupByBackward(list[Box] boxes:[Box _, *_], int gs, Box op)
+        = [op[boxes=boxes[..size(boxes) mod gs]], *groupBy(boxes[size(boxes) mod gs..], gs, op)];
+
+    list[Row] groupRows([], int _gs) = [];
+    list[Row] groupRows(list[Box] boxes:[Box _, *_], int gs)
+        = [R(boxes[..gs]), *groupRows(boxes[gs..], gs)];
+
+    return innermost visit(b) {
+        case [*Box pre, _U([*Box mid]), *Box post]           => [*pre, *mid, *post]
+        case _G(list[Box] boxes, gs=gs, op=op, backwards=bw) => _U(bw ? groupByBackward(boxes, gs, op) : groupBy(boxes, gs, op))
+        case _AG(list[Box] boxes, gs=gs, columns=cs, rs=rs)  => A(groupRows(boxes, gs), columns=cs, rs=rs)
+    }
+}