New upstream version 4.7.2

13 files changed, 115 insertions, 52 deletions

diff --git a/tool/src/org/antlr/v4/Tool.java b/tool/src/org/antlr/v4/Tool.java
index 86c49e1..d2c95f5 100644
--- a/tool/src/org/antlr/v4/Tool.java
+++ b/tool/src/org/antlr/v4/Tool.java
@@ -397,7 +397,7 @@ public class Tool {
 
 		if ( generate_ATN_dot ) generateATNs(g);
 
-		if ( g.tool.getNumErrors()==0 ) generateInterpreterData(g);
+		if (gencode && g.tool.getNumErrors()==0 ) generateInterpreterData(g);
 
 		// PERFORM GRAMMAR ANALYSIS ON ATN: BUILD DECISION DFAs
 		AnalysisPipeline anal = new AnalysisPipeline(g);
@@ -838,7 +838,7 @@ public class Tool {
 		// or just or the relative path recorded for the parent grammar. This means
 		// that when we write the tokens files, or the .java files for imported grammars
 		// taht we will write them in the correct place.
-		if (fileNameWithPath.lastIndexOf(File.separatorChar) == -1) {
+		if ((fileNameWithPath == null) || (fileNameWithPath.lastIndexOf(File.separatorChar) == -1)) {
 			// No path is included in the file name, so make the file
 			// directory the same as the parent grammar (which might sitll be just ""
 			// but when it is not, we will write the file in the correct place.
diff --git a/tool/src/org/antlr/v4/codegen/Target.java b/tool/src/org/antlr/v4/codegen/Target.java
index 025b42a..3634172 100644
--- a/tool/src/org/antlr/v4/codegen/Target.java
+++ b/tool/src/org/antlr/v4/codegen/Target.java
@@ -26,6 +26,8 @@ import org.stringtemplate.v4.STGroupFile;
 import org.stringtemplate.v4.StringRenderer;
 import org.stringtemplate.v4.misc.STMessage;
 
+import java.net.URL;
+
 /** */
 public abstract class Target {
 	/** For pure strings of Java 16-bit Unicode char, how can we display
diff --git a/tool/src/org/antlr/v4/codegen/model/ElementFrequenciesVisitor.java b/tool/src/org/antlr/v4/codegen/model/ElementFrequenciesVisitor.java
index ccd5c80..ef1aae2 100644
--- a/tool/src/org/antlr/v4/codegen/model/ElementFrequenciesVisitor.java
+++ b/tool/src/org/antlr/v4/codegen/model/ElementFrequenciesVisitor.java
@@ -149,6 +149,16 @@ public class ElementFrequenciesVisitor extends GrammarTreeVisitor {
 		minFrequencies.peek().add(ref.getText());
 	}
 
+	@Override
+	public void stringRef(TerminalAST ref) {
+		String tokenName = ref.g.getTokenName(ref.getText());
+
+		if (tokenName != null && !tokenName.startsWith("T__")) {
+			frequencies.peek().add(tokenName);
+			minFrequencies.peek().add(tokenName);
+		}
+	}
+
 	/*
 	 * Parser rules
 	 */
diff --git a/tool/src/org/antlr/v4/codegen/model/RuleFunction.java b/tool/src/org/antlr/v4/codegen/model/RuleFunction.java
index d8b69b5..d6b745f 100644
--- a/tool/src/org/antlr/v4/codegen/model/RuleFunction.java
+++ b/tool/src/org/antlr/v4/codegen/model/RuleFunction.java
@@ -7,6 +7,7 @@
 package org.antlr.v4.codegen.model;
 
 import org.antlr.runtime.RecognitionException;
+import org.antlr.runtime.tree.CommonTree;
 import org.antlr.runtime.tree.CommonTreeNodeStream;
 import org.antlr.v4.codegen.OutputModelFactory;
 import org.antlr.v4.codegen.model.decl.AltLabelStructDecl;
@@ -32,6 +33,7 @@ import org.antlr.v4.tool.Rule;
 import org.antlr.v4.tool.ast.ActionAST;
 import org.antlr.v4.tool.ast.AltAST;
 import org.antlr.v4.tool.ast.GrammarAST;
+import org.antlr.v4.tool.ast.PredAST;
 
 import java.util.ArrayList;
 import java.util.Collection;
@@ -43,6 +45,7 @@ import java.util.Map;
 import java.util.Set;
 
 import static org.antlr.v4.parse.ANTLRParser.RULE_REF;
+import static org.antlr.v4.parse.ANTLRParser.STRING_LITERAL;
 import static org.antlr.v4.parse.ANTLRParser.TOKEN_REF;
 
 /** */
@@ -166,7 +169,7 @@ public class RuleFunction extends OutputModelObject {
 	}
 
 	/** for all alts, find which ref X or r needs List
-	   Must see across alts.  If any alt needs X or r as list, then
+	   Must see across alts. If any alt needs X or r as list, then
 	   define as list.
 	 */
 	public Set<Decl> getDeclsForAllElements(List<AltAST> altASTs) {
@@ -174,21 +177,24 @@ public class RuleFunction extends OutputModelObject {
 		Set<String> nonOptional = new HashSet<String>();
 		List<GrammarAST> allRefs = new ArrayList<GrammarAST>();
 		boolean firstAlt = true;
+		IntervalSet reftypes = new IntervalSet(RULE_REF, TOKEN_REF, STRING_LITERAL);
 		for (AltAST ast : altASTs) {
-			IntervalSet reftypes = new IntervalSet(RULE_REF, TOKEN_REF);
-			List<GrammarAST> refs = ast.getNodesWithType(reftypes);
+			List<GrammarAST> refs = getRuleTokens(ast.getNodesWithType(reftypes));
 			allRefs.addAll(refs);
 			Pair<FrequencySet<String>, FrequencySet<String>> minAndAltFreq = getElementFrequenciesForAlt(ast);
 			FrequencySet<String> minFreq = minAndAltFreq.a;
 			FrequencySet<String> altFreq = minAndAltFreq.b;
 			for (GrammarAST t : refs) {
-				String refLabelName = t.getText();
-				if ( altFreq.count(refLabelName)>1 ) {
-					needsList.add(refLabelName);
-				}
+				String refLabelName = getName(t);
+
+				if (refLabelName != null) {
+					if (altFreq.count(refLabelName) > 1) {
+						needsList.add(refLabelName);
+					}
 
-				if (firstAlt && minFreq.count(refLabelName) != 0) {
-					nonOptional.add(refLabelName);
+					if (firstAlt && minFreq.count(refLabelName) != 0) {
+						nonOptional.add(refLabelName);
+					}
 				}
 			}
 
@@ -202,7 +208,12 @@ public class RuleFunction extends OutputModelObject {
 		}
 		Set<Decl> decls = new LinkedHashSet<Decl>();
 		for (GrammarAST t : allRefs) {
-			String refLabelName = t.getText();
+			String refLabelName = getName(t);
+
+			if (refLabelName == null) {
+				continue;
+			}
+
 			List<Decl> d = getDeclForAltElement(t,
 												refLabelName,
 												needsList.contains(refLabelName),
@@ -212,6 +223,35 @@ public class RuleFunction extends OutputModelObject {
 		return decls;
 	}
 
+	private List<GrammarAST> getRuleTokens(List<GrammarAST> refs) {
+		List<GrammarAST> result = new ArrayList<>(refs.size());
+		for (GrammarAST ref : refs) {
+			CommonTree r = ref;
+
+			boolean ignore = false;
+			while (r != null) {
+				// Ignore string literals in predicates
+				if (r instanceof PredAST) {
+					ignore = true;
+					break;
+				}
+				r = r.parent;
+			}
+
+			if (!ignore) {
+				result.add(ref);
+			}
+		}
+
+		return result;
+	}
+
+	private String getName(GrammarAST token) {
+		String tokenText = token.getText();
+		String tokenName = token.getType() != STRING_LITERAL ? tokenText : token.g.getTokenName(tokenText);
+		return tokenName == null || tokenName.startsWith("T__") ? null : tokenName; // Do not include tokens with auto generated names
+	}
+
 	/** Given list of X and r refs in alt, compute how many of each there are */
 	protected Pair<FrequencySet<String>, FrequencySet<String>> getElementFrequenciesForAlt(AltAST ast) {
 		try {
diff --git a/tool/src/org/antlr/v4/codegen/target/CSharpTarget.java b/tool/src/org/antlr/v4/codegen/target/CSharpTarget.java
index e7a2322..82f65c6 100644
--- a/tool/src/org/antlr/v4/codegen/target/CSharpTarget.java
+++ b/tool/src/org/antlr/v4/codegen/target/CSharpTarget.java
@@ -28,7 +28,7 @@ public class CSharpTarget extends Target {
 
     @Override
     public String getVersion() {
-        return "4.7.1";
+        return "4.7.2";
     }
 
 	@Override
diff --git a/tool/src/org/antlr/v4/codegen/target/CppTarget.java b/tool/src/org/antlr/v4/codegen/target/CppTarget.java
index 6ef32c2..839a362 100644
--- a/tool/src/org/antlr/v4/codegen/target/CppTarget.java
+++ b/tool/src/org/antlr/v4/codegen/target/CppTarget.java
@@ -50,7 +50,7 @@ public class CppTarget extends Target {
 	}
 
 	public String getVersion() {
-		return "4.7.1";
+		return "4.7.2";
 	}
 
     public boolean needsHeader() { return true; }
diff --git a/tool/src/org/antlr/v4/codegen/target/GoTarget.java b/tool/src/org/antlr/v4/codegen/target/GoTarget.java
index be4e3cc..89a40de 100644
--- a/tool/src/org/antlr/v4/codegen/target/GoTarget.java
+++ b/tool/src/org/antlr/v4/codegen/target/GoTarget.java
@@ -71,7 +71,7 @@ public class GoTarget extends Target {
 
 	@Override
 	public String getVersion() {
-		return "4.7.1";
+		return "4.7.2";
 	}
 
 	public Set<String> getBadWords() {
diff --git a/tool/src/org/antlr/v4/codegen/target/JavaScriptTarget.java b/tool/src/org/antlr/v4/codegen/target/JavaScriptTarget.java
index adbfc7c..75886fd 100644
--- a/tool/src/org/antlr/v4/codegen/target/JavaScriptTarget.java
+++ b/tool/src/org/antlr/v4/codegen/target/JavaScriptTarget.java
@@ -51,7 +51,7 @@ public class JavaScriptTarget extends Target {
 
     @Override
     public String getVersion() {
-        return "4.7.1";
+        return "4.7.2";
     }
 
     public Set<String> getBadWords() {
diff --git a/tool/src/org/antlr/v4/codegen/target/Python2Target.java b/tool/src/org/antlr/v4/codegen/target/Python2Target.java
index 3fc6a35..1beaf8c 100644
--- a/tool/src/org/antlr/v4/codegen/target/Python2Target.java
+++ b/tool/src/org/antlr/v4/codegen/target/Python2Target.java
@@ -24,27 +24,28 @@ import java.util.Set;
  */
 public class Python2Target extends Target {
 	protected static final String[] python2Keywords = {
-		"abs", "all", "any", "apply", "as",
-		"bin", "bool", "buffer", "bytearray",
-		"callable", "chr", "classmethod", "coerce", "compile", "complex",
-		"del", "delattr", "dict", "dir", "divmod",
-		"enumerate", "eval", "execfile",
-		"file", "filter", "float", "format", "frozenset",
-		"getattr", "globals",
+		"abs", "all", "and", "any", "apply", "as", "assert",
+		"bin", "bool", "break", "buffer", "bytearray",
+		"callable", "chr", "class", "classmethod", "coerce", "compile", "complex", "continue",
+		"def", "del", "delattr", "dict", "dir", "divmod",
+		"elif", "else", "enumerate", "eval", "except", "exec", "execfile",
+		"file", "filter", "finally", "float", "for", "format", "from", "frozenset",
+		"getattr", "global", "globals",
 		"hasattr", "hash", "help", "hex",
-		"id", "input", "int", "intern", "isinstance", "issubclass", "iter",
-		"len", "list", "locals",
-		"map", "max", "min", "next",
+		"id", "if", "import", "in", "input", "int", "intern", "is", "isinstance", "issubclass", "iter",
+		"lambda", "len", "list", "locals",
+		"map", "max", "min", "next", "not",
 		"memoryview",
-		"object", "oct", "open", "ord",
-		"pow", "print", "property",
-		"range", "raw_input", "reduce", "reload", "repr", "return", "reversed", "round",
+		"object", "oct", "open", "or", "ord",
+		"pass", "pow", "print", "property",
+		"raise", "range", "raw_input", "reduce", "reload", "repr", "return", "reversed", "round",
 		"set", "setattr", "slice", "sorted", "staticmethod", "str", "sum", "super",
-		"tuple", "type",
+		"try", "tuple", "type",
 		"unichr", "unicode",
 		"vars",
-		"with",
+		"while", "with",
 		"xrange",
+		"yield",
 		"zip",
 		"__import__",
 		"True", "False", "None"
@@ -94,7 +95,7 @@ public class Python2Target extends Target {
 
 	@Override
 	public String getVersion() {
-		return "4.7.1";
+		return "4.7.2";
 	}
 
 	public Set<String> getBadWords() {
diff --git a/tool/src/org/antlr/v4/codegen/target/Python3Target.java b/tool/src/org/antlr/v4/codegen/target/Python3Target.java
index 388269a..753f67c 100644
--- a/tool/src/org/antlr/v4/codegen/target/Python3Target.java
+++ b/tool/src/org/antlr/v4/codegen/target/Python3Target.java
@@ -24,26 +24,27 @@ import java.util.Set;
  */
 public class Python3Target extends Target {
 	protected static final String[] python3Keywords = {
-		"abs", "all", "any", "apply", "as",
-		"bin", "bool", "buffer", "bytearray",
-		"callable", "chr", "classmethod", "coerce", "compile", "complex",
-		"del", "delattr", "dict", "dir", "divmod",
-		"enumerate", "eval", "execfile",
-		"file", "filter", "float", "format", "frozenset",
-		"getattr", "globals",
+		"abs", "all", "and", "any", "apply", "as", "assert",
+		"bin", "bool", "break", "buffer", "bytearray",
+		"callable", "chr", "class", "classmethod", "coerce", "compile", "complex", "continue",
+		"def", "del", "delattr", "dict", "dir", "divmod",
+		"elif", "else", "enumerate", "eval", "execfile", "except",
+		"file", "filter", "finally", "float", "for", "format", "from", "frozenset",
+		"getattr", "global", "globals",
 		"hasattr", "hash", "help", "hex",
-		"id", "input", "int", "intern", "isinstance", "issubclass", "iter",
-		"len", "list", "locals",
-		"map", "max", "min", "next",
-		"memoryview",
-		"object", "oct", "open", "ord",
-		"pow", "print", "property",
-		"range", "raw_input", "reduce", "reload", "repr", "return", "reversed", "round",
+		"id", "if", "import", "in", "input", "int", "intern", "is", "isinstance", "issubclass", "iter",
+		"lambda", "len", "list", "locals",
+		"map", "max", "min", "memoryview",
+		"next", "nonlocal", "not",
+		"object", "oct", "open", "or", "ord",
+		"pass", "pow", "print", "property",
+		"raise", "range", "raw_input", "reduce", "reload", "repr", "return", "reversed", "round",
 		"set", "setattr", "slice", "sorted", "staticmethod", "str", "sum", "super",
-		"tuple", "type",
+		"try", "tuple", "type",
 		"unichr", "unicode",
 		"vars",
-		"with",
+		"with", "while",
+		"yield",
 		"zip",
 		"__import__",
 		"True", "False", "None"
@@ -96,7 +97,7 @@ public class Python3Target extends Target {
 
 	@Override
 	public String getVersion() {
-		return "4.7.1";
+		return "4.7.2";
 	}
 
 	/** Avoid grammar symbols in this set to prevent conflicts in gen'd code. */
diff --git a/tool/src/org/antlr/v4/codegen/target/SwiftTarget.java b/tool/src/org/antlr/v4/codegen/target/SwiftTarget.java
index 5e28506..b192344 100644
--- a/tool/src/org/antlr/v4/codegen/target/SwiftTarget.java
+++ b/tool/src/org/antlr/v4/codegen/target/SwiftTarget.java
@@ -87,7 +87,7 @@ public class SwiftTarget extends Target {
 
     @Override
     public String getVersion() {
-        return "4.7.1"; // Java and tool versions move in lock step
+        return "4.7.2"; // Java and tool versions move in lock step
     }
 
     public Set<String> getBadWords() {
diff --git a/tool/src/org/antlr/v4/tool/ErrorManager.java b/tool/src/org/antlr/v4/tool/ErrorManager.java
index 0230d2b..7421266 100644
--- a/tool/src/org/antlr/v4/tool/ErrorManager.java
+++ b/tool/src/org/antlr/v4/tool/ErrorManager.java
@@ -231,8 +231,7 @@ public class ErrorManager {
             setFormat("antlr"); // recurse on this rule, trying the default message format
             return;
         }
-
-        format = new STGroupFile(fileName, "UTF-8");
+        format = new STGroupFile(url, "UTF-8", '<', '>');
         format.load();
 
         if ( !initSTListener.errors.isEmpty() ) {
diff --git a/tool/src/org/antlr/v4/tool/Grammar.java b/tool/src/org/antlr/v4/tool/Grammar.java
index b86d622..fc98fcf 100644
--- a/tool/src/org/antlr/v4/tool/Grammar.java
+++ b/tool/src/org/antlr/v4/tool/Grammar.java
@@ -631,6 +631,16 @@ public class Grammar implements AttributeResolver {
 		return i;
 	}
 
+	public String getTokenName(String literal) {
+		Grammar grammar = this;
+		while (grammar != null) {
+			if (grammar.stringLiteralToTypeMap.containsKey(literal))
+				return grammar.getTokenName(grammar.stringLiteralToTypeMap.get(literal));
+			grammar = grammar.parent;
+		}
+		return null;
+	}
+
 	/** Given a token type, get a meaningful name for it such as the ID
 	 *  or string literal.  If this is a lexer and the ttype is in the
 	 *  char vocabulary, compute an ANTLR-valid (possibly escaped) char literal.