Ability to run parsers for symbols like A*, A+, {A ","}+ and A? without introducing a dummy non-terminal

.settings/org.eclipse.core.resources.prefs

@@ -5,7 +5,6 @@ encoding//src/org/rascalmpl/library/experiments/vis2/lib/nv.d3.css=UTF-8
 encoding//src/org/rascalmpl/library/experiments/vis2/lib/nv.d3.js=UTF-8
 encoding//src/org/rascalmpl/library/experiments/vis2/lib/reset.css=UTF-8
 encoding//src/org/rascalmpl/library/lang/rascal/tests/functionality/DataType.rsc=UTF-8
-encoding//target/generated-resources=UTF-8
 encoding//test/org/rascalmpl/test/data=UTF-8
 encoding/<project>=UTF-8
 encoding/META-INF=UTF-8

.vscode/launch.json

@@ -4,6 +4,41 @@
     // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387
     "version": "0.2.0",
     "configurations": [
+        {
+            "type": "java",
+            "name": "$GENERATED_PARSER$244372483",
+            "request": "launch",
+            "mainClass": "org.rascalmpl.test.parser.$GENERATED_PARSER$244372483",
+            "projectName": "rascal"
+        },
+        {
+            "type": "java",
+            "name": "$GENERATED_PARSER$279786030",
+            "request": "launch",
+            "mainClass": "org.rascalmpl.test.parser.$GENERATED_PARSER$279786030",
+            "projectName": "rascal"
+        },
+        {
+            "type": "java",
+            "name": "TopLevelStarListEmpty",
+            "request": "launch",
+            "mainClass": "org.rascalmpl.test.parser.TopLevelStarListEmpty",
+            "projectName": "rascal"
+        },
+        {
+            "type": "java",
+            "name": "TopLevelStarList",
+            "request": "launch",
+            "mainClass": "org.rascalmpl.test.parser.TopLevelStarList",
+            "projectName": "rascal"
+        },
+        {
+            "type": "java",
+            "name": "NonTerminalStarListEmpty",
+            "request": "launch",
+            "mainClass": "org.rascalmpl.test.parser.NonTerminalStarListEmpty",
+            "projectName": "rascal"
+        },
         {
             "type": "java",
             "name": "RascalCheck",

pom.xml

@@ -3,12 +3,12 @@
 
     <groupId>org.rascalmpl</groupId>
     <artifactId>rascal</artifactId>
-    <version>0.43.0-RC13-SNAPSHOT</version>
+    <version>0.43.0-RC16-PREVIEW-REGULAR-PARSERS-SNAPSHOT</version>
     <packaging>jar</packaging>
 
     <scm>
         <developerConnection>scm:git:ssh://git@github.com/usethesource/rascal.git</developerConnection>
-        <tag>v0.43.0-RC11</tag>
+        <tag>v0.43.0-RC14-PREVIEW-REGULAR-PARSERS</tag>
     </scm>
 
     <!-- dependency resolution configuration (usethesource) -->

rascal.code-workspace

@@ -7,5 +7,10 @@
 			"path": "../typepal"
 		}
 	],
-	"settings": {}
+	"settings": {
+		"files.watcherExclude": {
+			"/target/": true,
+			"/.git/": true,
+		},
+	}
 }

src/org/rascalmpl/exceptions/JavaCompilation.java

@@ -18,7 +18,7 @@ public class JavaCompilation extends RuntimeException {
 	private final long column;
 
 	public JavaCompilation(String message, long line, long column, String source, Exception cause) {
-		super("Java compilation failed due to " + message, cause);
+		super("Java compilation failed due to " + message + " at " + line + ", " + column, cause);
 		this.source = source;
 		this.line = line;
 		this.column = column;

src/org/rascalmpl/exceptions/RuntimeExceptionFactory.java

@@ -696,6 +696,10 @@ public static Throw jsonParseError(ISourceLocation file, int line, int col, Stri
 	public static Throw parseError(ISourceLocation loc, AbstractAST ast, StackTrace trace) {
 		return new Throw(VF.constructor(ParseError, loc), ast != null ? ast.getLocation() : null, trace);
 	}	
+
+	public static Throw parseError(ISourceLocation loc, ISourceLocation caller, StackTrace trace) {
+		return new Throw(VF.constructor(ParseError, loc), caller, trace);
+	}
 
 	// pathNotFound
 

src/org/rascalmpl/exceptions/Throw.java

@@ -35,10 +35,11 @@
  * of exception values, not different kind of Java classes.
  */
 public final class Throw extends ControlException {
+	public static final String RASCAL_FILL_IN_LATER = "rascalfillinlater";
 	private static final long serialVersionUID = -7290501865940548332L;
 	private final IValue exception;
-	private volatile ISourceLocation loc;
-	private volatile StackTrace trace;
+	private ISourceLocation loc;
+	private StackTrace trace;
 
 	/**
 	 * Make a new Rascal exception.
@@ -50,26 +51,23 @@ public final class Throw extends ControlException {
 	public Throw(IValue value, ISourceLocation loc, StackTrace trace) {
 		super(toString(value, 4096));
 		this.exception = value;
-		this.loc = loc;
+		this.loc = loc != null ? loc : URIUtil.rootLocation(RASCAL_FILL_IN_LATER);
 		if(trace == null) {
 			trace = StackTrace.EMPTY_STACK_TRACE;
 		}
 		this.trace = trace;
 	}
 
 	/**
-     * Make a new Rascal exception.
+     * Make a new Rascal exception, where location and trace
+	 * are to be filled in later by the runtime, automatically.
      * 
      * @param value The Rascal exception value
-     * @param loc A source location, or null if unavailable
-     * @param trace A stack trace, or null
      */
     public Throw(IValue value) {
         super(toString(value, 4096));
         this.exception = value;
-        // TODO: convert top stack frame location to rascal loc 
-        this.loc = URIUtil.rootLocation("TODO");
-        // TODO: convert JVM trace to Rascal trace..
+        this.loc = URIUtil.rootLocation(RASCAL_FILL_IN_LATER);
         this.trace = StackTrace.EMPTY_STACK_TRACE;
     }
 
@@ -106,7 +104,7 @@ public StackTrace getTrace() {
 	 * @param trace The new trace, or null for an empty trace
 	 */
 	public void setTrace(StackTrace trace) {
-		if(trace == null) {
+		if (trace == null) {
 			trace = StackTrace.EMPTY_STACK_TRACE;
 		}
 		this.trace = trace;
@@ -130,6 +128,7 @@ public ISourceLocation getLocation() {
 	 * @param loc The source location, or null
 	 */
 	public void setLocation(ISourceLocation loc) {
+		assert loc != null;
 		this.loc = loc;
 	}
 

src/org/rascalmpl/interpreter/result/JavaMethod.java

@@ -156,7 +156,7 @@ public Result<IValue> call(Type[] actualStaticTypes, IValue[] actuals, Map<Strin
 
 		try {
 			ctx.pushEnv(getName());
-
+			ctx.setCurrentAST(getAst()); // otherwise it becomes the final parameter of the call site
 			Environment env = ctx.getCurrentEnvt();
 			Map<Type, Type> renamings = Collections.emptyMap();
 
@@ -327,10 +327,11 @@ public IValue invoke(Object[] oActuals) {
 				trace.addAll(th.getTrace());
 
 				ISourceLocation loc = th.getLocation();
-				if (loc == null || loc.getScheme().equals("TODO")) {
+				if (loc == null || loc.getScheme().equals(Throw.RASCAL_FILL_IN_LATER)) {
+				  // this happens when Throw is created without a current AST location and a trace.
 				  loc = getAst().getLocation();
 				}
-				trace.add(loc, null);
+				trace.add(loc, name);
 
 				th.setLocation(loc);
 				trace.addAll(eval.getStackTrace());

src/org/rascalmpl/interpreter/result/RascalFunction.java

@@ -348,7 +348,6 @@ public Result<IValue> call(Type[] actualStaticTypes, IValue[] actuals, Map<Strin
                             // formals are now bound by side effect of the pattern matcher
                             try {
                                 bindKeywordArgs(keyArgValues);
-
                                 result = runBody();
                                 storeMemoizedResult(actuals, keyArgValues, result);
                                 return result;

src/org/rascalmpl/interpreter/utils/IUPTRAstToSymbolConstructor.java

@@ -123,6 +123,7 @@ public IConstructor visitExpressionCallOrTree(CallOrTree x) {
 
 		if (name.equals("alt")) {
 			ISet set = vf.set();
+
 			Expression.Set arg = (Set) x.getArguments().get(0);
 			for(Expression y: arg.getElements0()){
 				set = set.insert(y.accept(this));

src/org/rascalmpl/interpreter/utils/ReadEvalPrintDialogMessages.java

@@ -164,10 +164,10 @@ public static void throwMessage(PrintWriter out, Throw e, StandardTextWriter pre
                 prettyPrinter.write(e.getException(), lros);
             }
 		}
-		catch(/*IOLimitReachedException*/ RuntimeException iolrex){
+		catch (/*IOLimitReachedException*/ RuntimeException iolrex){
 			// This is fine, ignore.
 		}
-		catch(IOException ioex){
+		catch (IOException ioex){
 			// This can/should never happen.
 		}
 

src/org/rascalmpl/library/lang/rascal/grammar/ConcreteSyntax.rsc

@@ -10,6 +10,7 @@
 @synopsis{This module provides functionality for merging the Rascal grammar and arbitrary user-defined grammars}
 module lang::rascal::grammar::ConcreteSyntax
 
+import IO;
 import ValueIO;
 import ParseTree;
 import Grammar;
@@ -35,79 +36,116 @@ public Grammar addHoles(Grammar object) = compose(object, grammar({}, holes(obje
   which nonterminal names are lexical and which are not. Since nonterminal names are unique between lex and normal nonterminals
   we make sure that the literal that is generated is first normalized to remove all lex names.}
 public set[Production] holes(Grammar object) {
-  // syntax N = @holeType=<N> [-1] "N" ":" [0-9]+ [-1];
-  return  { regular(iter(\char-class([range(48,57)]))), 
-            prod(label("$MetaHole",getTargetSymbol(nont)),
-                 [ \char-class([range(0,0)]),
-                   lit("<removeConditionals(denormalize(nont))>"),lit(":"),iter(\char-class([range(48,57)])),
-                   \char-class([range(0,0)])
-                 ],{Attr::\tag("holeType"(nont))})  // TODO: added qualifier to help compiler
-          | Symbol nont <- object.rules, quotable(nont)
-          };
+    // syntax N = @holeType=<N> [\a00] "N" ":" [0-9]+ [\a00];
+    return
+        {regular(iter(\char-class([range(48, 57)]))), prod(
+                                                          label("$MetaHole", getTargetSymbol(nont)),
+                                                          [
+                                                              \char-class([range(0, 0)]),
+                                                              lit("<removeConditionals(denormalize(nont))>"),
+                                                              lit(":"),
+                                                              iter(\char-class([range(48, 57)])),
+                                                              \char-class([range(0, 0)])
+                                                          ],
+                                                          { Attr::\tag("holeType"(nont)) }
+                                                      )
+        // TODO: added qualifier to help compiler
+        | Symbol nont <- object.rules, quotable(nont)
+        };
 }
 
 @synopsis{This function is called by the Rascal interpreter to generate a string that can be parsed by the rules generated by the
   holes function in this module.}
-public str createHole(ConcretePart hole, int idx) = createHole(hole.hole, idx);
-public str createHole(ConcreteHole hole, int idx) = "\u0000<denormalize(sym2symbol(hole.symbol))>:<idx>\u0000";
+str createHole(ConcretePart hole, int idx) = createHole(hole.hole, idx);
+str createHole(ConcreteHole hole, int idx)
+    = "\u0000<denormalize(sym2symbol(hole.symbol))>:<idx>\u0000";
 
-
-@synopsis{In Rascal programs with type literals, it's hard to see easily if it is a lex or sort, so we "denormalize" here.
-  The same goes for the introduction of layout non-terminals in lists. We do not know which non-terminal is introduced,
-  so we remove this here to create a canonical 'source-level' type.}
-private Symbol denormalize(Symbol s) = visit (s) { 
-  case \lex(n)                                      => \sort(n)
-  case \iter-seps(u,[layouts(_),t,layouts(_)])      => \iter-seps(u,[t])
-  case \iter-star-seps(u,[layouts(_),t,layouts(_)]) => \iter-star-seps(u,[t])
-  case \iter-seps(u,[layouts(_)])                   => \iter(u)
-  case \iter-star-seps(u,[layouts(_)])              => \iter-star(u)
-  case \seq(ss)                                     => seq([t | t <- ss, !(t is layouts)])
-};
+@synopsis{Normalize accidental differences between symbol literals to avoid parse errors in concrete syntax fragments}
+@description{
+* In Rascal programs with type literals, it's hard to see easily if it is a lex or sort, so we "denormalize" here.
+* The same goes for the introduction of layout non-terminals in lists. We do not know which non-terminal is introduced,
+so we remove this here to create a canonical 'source-level' type.
+* Finally we brush away the difference between + and * lists here, since + plus lists mau occur where a star list is.
+}
+private
+Symbol denormalize(Symbol s)
+    = visit(s) {
+          case \lex(n) => \sort(n)
+          case \iter-seps(u, [layouts(_), t, layouts(_)]) => \iter-seps(u, [t])
+          case \iter-star-seps(u, [layouts(_), t, layouts(_)]) => \iter-star-seps(u, [t])
+          case \iter-seps(u, [layouts(_)]) => \iter(u)
+          case \iter-star-seps(u, [layouts(_)]) => \iter-star(u)
+          case \seq(ss) => seq([t | t <- ss, !(t is layouts)])
+      };
 
 private Symbol removeConditionals(Symbol sym) = visit(sym) {
-  case conditional(Symbol s, _) => s
-};
+                                                    case conditional(Symbol s, _) => s
+                                                };
 
 @synopsis{This is needed such that list variables can be repeatedly used as elements of the same list}
 private Symbol getTargetSymbol(Symbol sym) {
-  switch(sym) {
-    case Symbol::\iter(s) : return s;
-    case Symbol::\iter-star(s) : return s;  
-    case Symbol::\iter-seps(s, _) : return s; 
-    case Symbol::\iter-star-seps(s, _) : return s; 
-    case Symbol::\opt(s) : return s; 
-    default: return sym;
-  } 
+    switch(sym) {
+        case Symbol::\iter(Symbol s):
+            return s;
+        case Symbol::\iter-star(Symbol s):
+            return s;
+        case Symbol::\iter-seps(Symbol s, _):
+            return s;
+        case Symbol::\iter-star-seps(Symbol s, _):
+            return s;
+        case Symbol::\opt(Symbol s):
+            return s;
+        case Symbol::\alt({Symbol s, _}):
+            return s;
+        default:
+            return sym;
+    }
 }
 
 // TODO, rewritten quotable for the benefit of the compiler
 // Offending case:
 //data Symbol
-//        =  \lit()   
-//        | \parameter() 
+//        =  \lit()
+//        | \parameter()
 //        | \parameterized-lex(list[Symbol] parameters)
 //        ;
-//     
-//private bool quotable(Symbol x) { 
-//    return 
-//       \lit() := x 
+//
+//private bool quotable(Symbol x) {
+//    return
+//       \lit() := x
 //       &&
-//       \parameterized-lex([\parameter()]) !:= x
+//       \parameterized-lex([\parameter(_,_)]) !:= x
 //       ;
 //}
-
 @synopsis{This decides for which part of the grammar we can write anti-quotes}
-private bool quotable(Symbol x) { 
-    switch(x){ 
-        case \lit(_): return false;
-        case \empty(): return false;
-        case \cilit(_): return false;
-        case \char-class(_): return false;
-        case \layouts(_): return false;
-        case \keywords(_): return false;
-        case \start(_): return false;
-        case \parameterized-sort(_,[\parameter(_,_),*_]): return false;
-        case \parameterized-lex(_,[\parameter(_,_),*_]): return false;
-        default: return true;
-     };
+private bool quotable(Symbol x) {
+    switch(x) {
+        case \lit(_):
+            return false;
+        case \empty():
+            return false;
+        // TODO: this corner case should be allowed
+        case \cilit(_):
+            return false;
+        // TODO: why not?
+        case \char-class(_):
+            return false;
+        // TODO: this should definitely be allowed
+        case \layouts(_):
+            return false;
+        // TODO: why not?
+        case \keywords(_):
+            return false;
+        case \start(_):
+            return false;
+        case \parameterized-sort(_, [\parameter(_, _), *_]):
+            return false;
+        // TODO this is hard but should be allowed
+        case \parameterized-lex(_, [\parameter(_, _), *_]):
+            return false;
+        // TODO this is hard but should be allowed
+        default:
+            return true;
+    }
+    ;
 }