diff options
Diffstat (limited to 'tool/src')
227 files changed, 2233 insertions, 815 deletions
diff --git a/tool/src/org/antlr/v4/Tool.java b/tool/src/org/antlr/v4/Tool.java index 93a59ff..86c49e1 100644 --- a/tool/src/org/antlr/v4/Tool.java +++ b/tool/src/org/antlr/v4/Tool.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ @@ -27,6 +27,8 @@ import org.antlr.v4.parse.ToolANTLRParser; import org.antlr.v4.parse.v3TreeGrammarException; import org.antlr.v4.runtime.RuntimeMetaData; import org.antlr.v4.runtime.misc.LogManager; +import org.antlr.v4.runtime.misc.IntegerList; +import org.antlr.v4.runtime.atn.ATNSerializer; import org.antlr.v4.semantics.SemanticPipeline; import org.antlr.v4.tool.ANTLRMessage; import org.antlr.v4.tool.ANTLRToolListener; @@ -115,26 +117,28 @@ public class Tool { public Map<String, String> grammarOptions = null; public boolean warnings_are_errors = false; public boolean longMessages = false; + public boolean exact_output_dir = false; public static Option[] optionDefs = { - new Option("outputDirectory", "-o", OptionArgType.STRING, "specify output directory where all output is generated"), - new Option("libDirectory", "-lib", OptionArgType.STRING, "specify location of grammars, tokens files"), - new Option("generate_ATN_dot", "-atn", "generate rule augmented transition network diagrams"), - new Option("grammarEncoding", "-encoding", OptionArgType.STRING, "specify grammar file encoding; e.g., euc-jp"), - new Option("msgFormat", "-message-format", OptionArgType.STRING, "specify output style for messages in antlr, gnu, vs2005"), - new Option("longMessages", "-long-messages", "show exception details when available for errors and warnings"), - new Option("gen_listener", "-listener", "generate parse tree listener (default)"), - new Option("gen_listener", "-no-listener", "don't generate parse tree listener"), - new Option("gen_visitor", "-visitor", "generate parse tree visitor"), - new Option("gen_visitor", "-no-visitor", "don't generate parse tree visitor (default)"), - new Option("genPackage", "-package", OptionArgType.STRING, "specify a package/namespace for the generated code"), - new Option("gen_dependencies", "-depend", "generate file dependencies"), - new Option("", "-D<option>=value", "set/override a grammar-level option"), - new Option("warnings_are_errors", "-Werror", "treat warnings as errors"), - new Option("launch_ST_inspector", "-XdbgST", "launch StringTemplate visualizer on generated code"), + new Option("outputDirectory", "-o", OptionArgType.STRING, "specify output directory where all output is generated"), + new Option("libDirectory", "-lib", OptionArgType.STRING, "specify location of grammars, tokens files"), + new Option("generate_ATN_dot", "-atn", "generate rule augmented transition network diagrams"), + new Option("grammarEncoding", "-encoding", OptionArgType.STRING, "specify grammar file encoding; e.g., euc-jp"), + new Option("msgFormat", "-message-format", OptionArgType.STRING, "specify output style for messages in antlr, gnu, vs2005"), + new Option("longMessages", "-long-messages", "show exception details when available for errors and warnings"), + new Option("gen_listener", "-listener", "generate parse tree listener (default)"), + new Option("gen_listener", "-no-listener", "don't generate parse tree listener"), + new Option("gen_visitor", "-visitor", "generate parse tree visitor"), + new Option("gen_visitor", "-no-visitor", "don't generate parse tree visitor (default)"), + new Option("genPackage", "-package", OptionArgType.STRING, "specify a package/namespace for the generated code"), + new Option("gen_dependencies", "-depend", "generate file dependencies"), + new Option("", "-D<option>=value", "set/override a grammar-level option"), + new Option("warnings_are_errors", "-Werror", "treat warnings as errors"), + new Option("launch_ST_inspector", "-XdbgST", "launch StringTemplate visualizer on generated code"), new Option("ST_inspector_wait_for_close", "-XdbgSTWait", "wait for STViz to close before continuing"), - new Option("force_atn", "-Xforce-atn", "use the ATN simulator for all predictions"), - new Option("log", "-Xlog", "dump lots of logging info to antlr-timestamp.log"), + new Option("force_atn", "-Xforce-atn", "use the ATN simulator for all predictions"), + new Option("log", "-Xlog", "dump lots of logging info to antlr-timestamp.log"), + new Option("exact_output_dir", "-Xexact-output-dir", "all output goes into -o dir regardless of paths/package"), }; // helper vars for option management @@ -191,8 +195,11 @@ public class Tool { public Tool(String[] args) { this.args = args; errMgr = new ErrorManager(this); - errMgr.setFormat(msgFormat); + // We have to use the default message format until we have + // parsed the -message-format command line option. + errMgr.setFormat("antlr"); handleArgs(); + errMgr.setFormat(msgFormat); } protected void handleArgs() { @@ -246,7 +253,7 @@ public class Tool { haveOutputDir = true; if (outDir.exists() && !outDir.isDirectory()) { errMgr.toolError(ErrorType.OUTPUT_DIR_IS_FILE, outputDirectory); - libDirectory = "."; + outputDirectory = "."; } } else { @@ -390,6 +397,8 @@ public class Tool { if ( generate_ATN_dot ) generateATNs(g); + if ( g.tool.getNumErrors()==0 ) generateInterpreterData(g); + // PERFORM GRAMMAR ANALYSIS ON ATN: BUILD DECISION DFAs AnalysisPipeline anal = new AnalysisPipeline(g); anal.process(); @@ -693,6 +702,64 @@ public class Tool { } } + private void generateInterpreterData(Grammar g) { + StringBuilder content = new StringBuilder(); + + content.append("token literal names:\n"); + String[] names = g.getTokenLiteralNames(); + for (String name : names) { + content.append(name + "\n"); + } + content.append("\n"); + + content.append("token symbolic names:\n"); + names = g.getTokenSymbolicNames(); + for (String name : names) { + content.append(name + "\n"); + } + content.append("\n"); + + content.append("rule names:\n"); + names = g.getRuleNames(); + for (String name : names) { + content.append(name + "\n"); + } + content.append("\n"); + + if ( g.isLexer() ) { + content.append("channel names:\n"); + content.append("DEFAULT_TOKEN_CHANNEL\n"); + content.append("HIDDEN\n"); + for (String channel : g.channelValueToNameList) { + content.append(channel + "\n"); + } + content.append("\n"); + + content.append("mode names:\n"); + for (String mode : ((LexerGrammar)g).modes.keySet()) { + content.append(mode + "\n"); + } + } + content.append("\n"); + + IntegerList serializedATN = ATNSerializer.getSerialized(g.atn); + content.append("atn:\n"); + content.append(serializedATN.toString()); + + try { + Writer fw = getOutputFileWriter(g, g.name + ".interp"); + try { + fw.write(content.toString()); + } + finally { + fw.close(); + } + } + catch (IOException ioe) { + errMgr.toolError(ErrorType.CANNOT_WRITE_FILE, ioe); + } + } + /** This method is used by all code generators to create new output * files. If the outputDir set by -o is not present it will be created. * The final filename is sensitive to the output directory and @@ -758,6 +825,10 @@ public class Tool { * @param fileNameWithPath path to input source */ public File getOutputDirectory(String fileNameWithPath) { + if ( exact_output_dir ) { + return new_getOutputDirectory(fileNameWithPath); + } + File outputDir; String fileDirectory; @@ -783,12 +854,12 @@ public class Tool { // -o . /usr/lib/t.g4 => ./T.java if (fileDirectory != null && (new File(fileDirectory).isAbsolute() || - fileDirectory.startsWith("~"))) { // isAbsolute doesn't count this :( + fileDirectory.startsWith("~"))) { // isAbsolute doesn't count this :( // somebody set the dir, it takes precendence; write new file there outputDir = new File(outputDirectory); } else { - // -o /tmp subdir/t.g4 => /tmp/subdir/t.g4 + // -o /tmp subdir/t.g4 => /tmp/subdir/T.java if (fileDirectory != null) { outputDir = new File(outputDirectory, fileDirectory); } @@ -807,6 +878,37 @@ public class Tool { return outputDir; } + /** @since 4.7.1 in response to -Xexact-output-dir */ + public File new_getOutputDirectory(String fileNameWithPath) { + File outputDir; + String fileDirectory; + + if (fileNameWithPath.lastIndexOf(File.separatorChar) == -1) { + // No path is included in the file name, so make the file + // directory the same as the parent grammar (which might still be just "" + // but when it is not, we will write the file in the correct place. + fileDirectory = "."; + } + else { + fileDirectory = fileNameWithPath.substring(0, fileNameWithPath.lastIndexOf(File.separatorChar)); + } + if ( haveOutputDir ) { + // -o /tmp /var/lib/t.g4 => /tmp/T.java + // -o subdir/output /usr/lib/t.g4 => subdir/output/T.java + // -o . /usr/lib/t.g4 => ./T.java + // -o /tmp subdir/t.g4 => /tmp/T.java + outputDir = new File(outputDirectory); + } + else { + // they didn't specify a -o dir so just write to location + // where grammar is, absolute or relative, this will only happen + // with command line invocation as build tools will always + // supply an output directory. + outputDir = new File(fileDirectory); + } + return outputDir; + } + protected void writeDOTFile(Grammar g, Rule r, String dot) throws IOException { writeDOTFile(g, r.g.name + "." + r.name, dot); } diff --git a/tool/src/org/antlr/v4/analysis/AnalysisPipeline.java b/tool/src/org/antlr/v4/analysis/AnalysisPipeline.java index 6c7b97a..df27ee7 100644 --- a/tool/src/org/antlr/v4/analysis/AnalysisPipeline.java +++ b/tool/src/org/antlr/v4/analysis/AnalysisPipeline.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/tool/src/org/antlr/v4/analysis/LeftRecursionDetector.java b/tool/src/org/antlr/v4/analysis/LeftRecursionDetector.java index c180083..6a4ac00 100644 --- a/tool/src/org/antlr/v4/analysis/LeftRecursionDetector.java +++ b/tool/src/org/antlr/v4/analysis/LeftRecursionDetector.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/tool/src/org/antlr/v4/analysis/LeftRecursiveRuleAltInfo.java b/tool/src/org/antlr/v4/analysis/LeftRecursiveRuleAltInfo.java index 980fcaf..04f2dcf 100644 --- a/tool/src/org/antlr/v4/analysis/LeftRecursiveRuleAltInfo.java +++ b/tool/src/org/antlr/v4/analysis/LeftRecursiveRuleAltInfo.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/tool/src/org/antlr/v4/analysis/LeftRecursiveRuleAnalyzer.java b/tool/src/org/antlr/v4/analysis/LeftRecursiveRuleAnalyzer.java index a2040e7..ae8a441 100644 --- a/tool/src/org/antlr/v4/analysis/LeftRecursiveRuleAnalyzer.java +++ b/tool/src/org/antlr/v4/analysis/LeftRecursiveRuleAnalyzer.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/tool/src/org/antlr/v4/analysis/LeftRecursiveRuleTransformer.java b/tool/src/org/antlr/v4/analysis/LeftRecursiveRuleTransformer.java index 610c550..dd4dce6 100644 --- a/tool/src/org/antlr/v4/analysis/LeftRecursiveRuleTransformer.java +++ b/tool/src/org/antlr/v4/analysis/LeftRecursiveRuleTransformer.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/tool/src/org/antlr/v4/automata/ATNFactory.java b/tool/src/org/antlr/v4/automata/ATNFactory.java index a7de4ff..c0c0642 100644 --- a/tool/src/org/antlr/v4/automata/ATNFactory.java +++ b/tool/src/org/antlr/v4/automata/ATNFactory.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/tool/src/org/antlr/v4/automata/ATNOptimizer.java b/tool/src/org/antlr/v4/automata/ATNOptimizer.java index 36e5009..6c72016 100644 --- a/tool/src/org/antlr/v4/automata/ATNOptimizer.java +++ b/tool/src/org/antlr/v4/automata/ATNOptimizer.java @@ -1,15 +1,17 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ package org.antlr.v4.automata; +import org.antlr.v4.misc.CharSupport; import org.antlr.v4.runtime.atn.ATN; import org.antlr.v4.runtime.atn.ATNState; import org.antlr.v4.runtime.atn.AtomTransition; import org.antlr.v4.runtime.atn.BlockEndState; +import org.antlr.v4.runtime.atn.CodePointTransitions; import org.antlr.v4.runtime.atn.DecisionState; import org.antlr.v4.runtime.atn.EpsilonTransition; import org.antlr.v4.runtime.atn.NotSetTransition; @@ -96,17 +98,24 @@ public class ATNOptimizer { if (matchTransition instanceof NotSetTransition) { throw new UnsupportedOperationException("Not yet implemented."); } - IntervalSet set = matchTransition.label(); - int minElem = set.getMinElement(); - int maxElem = set.getMaxElement(); - for (int k = minElem; k <= maxElem; k++) { - if (matchSet.contains(k)) { - char setMin = (char) set.getMinElement(); - char setMax = (char) set.getMaxElement(); - // TODO: Token is missing (i.e. position in source will not be displayed). - g.tool.errMgr.grammarError(ErrorType.CHARACTERS_COLLISION_IN_SET, g.fileName, - null, (char) minElem + "-" + (char) maxElem, "[" + setMin + "-" + setMax + "]"); - break; + IntervalSet set = matchTransition.label(); + List<Interval> intervals = set.getIntervals(); + int n = intervals.size(); + for (int k = 0; k < n; k++) { + Interval setInterval = intervals.get(k); + int a = setInterval.a; + int b = setInterval.b; + if (a != -1 && b != -1) { + for (int v = a; v <= b; v++) { + if (matchSet.contains(v)) { + // TODO: Token is missing (i.e. position in source will not be displayed). + g.tool.errMgr.grammarError(ErrorType.CHARACTERS_COLLISION_IN_SET, g.fileName, + null, + CharSupport.getANTLRCharLiteralForChar(v), + CharSupport.getIntervalSetEscapedString(matchSet)); + break; + } + } } } matchSet.addAll(set); @@ -115,11 +124,11 @@ public class ATNOptimizer { Transition newTransition; if (matchSet.getIntervals().size() == 1) { if (matchSet.size() == 1) { - newTransition = new AtomTransition(blockEndState, matchSet.getMinElement()); + newTransition = CodePointTransitions.createWithCodePoint(blockEndState, matchSet.getMinElement()); } else { Interval matchInterval = matchSet.getIntervals().get(0); - newTransition = new RangeTransition(blockEndState, matchInterval.a, matchInterval.b); + newTransition = CodePointTransitions.createWithCodePointRange(blockEndState, matchInterval.a, matchInterval.b); } } else { diff --git a/tool/src/org/antlr/v4/automata/ATNPrinter.java b/tool/src/org/antlr/v4/automata/ATNPrinter.java index f3b44a1..acad6e7 100644 --- a/tool/src/org/antlr/v4/automata/ATNPrinter.java +++ b/tool/src/org/antlr/v4/automata/ATNPrinter.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/tool/src/org/antlr/v4/automata/ATNVisitor.java b/tool/src/org/antlr/v4/automata/ATNVisitor.java index 70eb6cb..953ff36 100644 --- a/tool/src/org/antlr/v4/automata/ATNVisitor.java +++ b/tool/src/org/antlr/v4/automata/ATNVisitor.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/tool/src/org/antlr/v4/automata/LexerATNFactory.java b/tool/src/org/antlr/v4/automata/LexerATNFactory.java index b596df5..fb332e5 100644 --- a/tool/src/org/antlr/v4/automata/LexerATNFactory.java +++ b/tool/src/org/antlr/v4/automata/LexerATNFactory.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ @@ -10,6 +10,7 @@ import org.antlr.runtime.CommonToken; import org.antlr.runtime.Token; import org.antlr.v4.codegen.CodeGenerator; import org.antlr.v4.misc.CharSupport; +import org.antlr.v4.misc.EscapeSequenceParsing; import org.antlr.v4.parse.ANTLRParser; import org.antlr.v4.runtime.IntStream; import org.antlr.v4.runtime.Lexer; @@ -17,6 +18,7 @@ import org.antlr.v4.runtime.atn.ATN; import org.antlr.v4.runtime.atn.ATNState; import org.antlr.v4.runtime.atn.ActionTransition; import org.antlr.v4.runtime.atn.AtomTransition; +import org.antlr.v4.runtime.atn.CodePointTransitions; import org.antlr.v4.runtime.atn.LexerAction; import org.antlr.v4.runtime.atn.LexerChannelAction; import org.antlr.v4.runtime.atn.LexerCustomAction; @@ -27,7 +29,6 @@ import org.antlr.v4.runtime.atn.LexerPushModeAction; import org.antlr.v4.runtime.atn.LexerSkipAction; import org.antlr.v4.runtime.atn.LexerTypeAction; import org.antlr.v4.runtime.atn.NotSetTransition; -import org.antlr.v4.runtime.atn.RangeTransition; import org.antlr.v4.runtime.atn.RuleStartState; import org.antlr.v4.runtime.atn.SetTransition; import org.antlr.v4.runtime.atn.TokensStartState; @@ -48,6 +49,7 @@ import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Objects; import java.util.Set; public class LexerATNFactory extends ParserATNFactory { @@ -255,7 +257,7 @@ public class LexerATNFactory extends ParserATNFactory { int t1 = CharSupport.getCharValueFromGrammarCharLiteral(a.getText()); int t2 = CharSupport.getCharValueFromGrammarCharLiteral(b.getText()); checkRange(a, b, t1, t2); - left.addTransition(new RangeTransition(right, t1, t2)); + left.addTransition(CodePointTransitions.createWithCodePointRange(right, t1, t2)); a.atnState = left; b.atnState = left; return new Handle(left, right); @@ -301,8 +303,9 @@ public class LexerATNFactory extends ParserATNFactory { Transition transition; if (set.getIntervals().size() == 1) { Interval interval = set.getIntervals().get(0); - transition = new RangeTransition(right, interval.a, interval.b); - } else { + transition = CodePointTransitions.createWithCodePointRange(right, interval.a, interval.b); + } + else { transition = new SetTransition(right, set); } @@ -343,26 +346,27 @@ public class LexerATNFactory extends ParserATNFactory { String chars = stringLiteralAST.getText(); ATNState left = newState(stringLiteralAST); ATNState right; - chars = CharSupport.getStringFromGrammarStringLiteral(chars); - if (chars == null) { - g.tool.errMgr.grammarError(ErrorType.INVALID_ESCAPE_SEQUENCE, - g.fileName, stringLiteralAST.getToken()); + String s = CharSupport.getStringFromGrammarStringLiteral(chars); + if (s == null) { + // the lexer will already have given an error return new Handle(left, left); } - int n = chars.length(); + int n = s.length(); ATNState prev = left; right = null; - for (int i = 0; i < n; i++) { + for (int i = 0; i < n; ) { right = newState(stringLiteralAST); - prev.addTransition(new AtomTransition(right, chars.charAt(i))); + int codePoint = s.codePointAt(i); + prev.addTransition(CodePointTransitions.createWithCodePoint(right, codePoint)); prev = right; + i += Character.charCount(codePoint); } stringLiteralAST.atnState = left; return new Handle(left, right); } - /** [Aa\t \u1234a-z\]\-] char sets */ + /** [Aa\t \u1234a-z\]\p{Letter}\-] char sets */ @Override public Handle charSetLiteral(GrammarAST charSetAST) { ATNState left = newState(charSetAST); @@ -373,10 +377,68 @@ public class LexerATNFactory extends ParserATNFactory { return new Handle(left, right); } + private static class CharSetParseState { + enum Mode { + NONE, + ERROR, + PREV_CODE_POINT, + PREV_PROPERTY + } + + public static final CharSetParseState NONE = new CharSetParseState(Mode.NONE, false, -1, IntervalSet.EMPTY_SET); + public static final CharSetParseState ERROR = new CharSetParseState(Mode.ERROR, false, -1, IntervalSet.EMPTY_SET); + + public final Mode mode; + public final boolean inRange; + public final int prevCodePoint; + public final IntervalSet prevProperty; + + public CharSetParseState( + Mode mode, + boolean inRange, + int prevCodePoint, + IntervalSet prevProperty) { + this.mode = mode; + this.inRange = inRange; + this.prevCodePoint = prevCodePoint; + this.prevProperty = prevProperty; + } + + @Override + public String toString() { + return String.format( + "%s mode=%s inRange=%s prevCodePoint=%d prevProperty=%s", + super.toString(), + mode, + inRange, + prevCodePoint, + prevProperty); + } + + @Override + public boolean equals(Object other) { + if (!(other instanceof CharSetParseState)) { + return false; + } + CharSetParseState that = (CharSetParseState) other; + if (this == that) { + return true; + } + return Objects.equals(this.mode, that.mode) && + Objects.equals(this.inRange, that.inRange) && + Objects.equals(this.prevCodePoint, that.prevCodePoint) && + Objects.equals(this.prevProperty, that.prevProperty); + } + + @Override + public int hashCode() { + return Objects.hash(mode, inRange, prevCodePoint, prevProperty); + } + } + public IntervalSet getSetFromCharSetLiteral(GrammarAST charSetAST) { String chars = charSetAST.getText(); chars = chars.substring(1, chars.length() - 1); - String cset = '"' + chars + '"'; IntervalSet set = new IntervalSet(); if (chars.length() == 0) { @@ -384,51 +446,127 @@ public class LexerATNFactory extends ParserATNFactory { g.fileName, charSetAST.getToken(), "[]"); return set; } - // unescape all valid escape char like \n, leaving escaped dashes as '\-' - // so we can avoid seeing them as '-' range ops. - chars = CharSupport.getStringFromGrammarStringLiteral(cset); - if (chars == null) { - g.tool.errMgr.grammarError(ErrorType.INVALID_ESCAPE_SEQUENCE, - g.fileName, charSetAST.getToken()); - return set; - } + + CharSetParseState state = CharSetParseState.NONE; + int n = chars.length(); - // now make x-y become set of char - for (int i = 0; i < n; i++) { - int c = chars.charAt(i); - if (c == '\\' && i+1 < n && chars.charAt(i+1) == '-') { // \- - checkSetCollision(charSetAST, set, '-'); - set.add('-'); - i++; + for (int i = 0; i < n; ) { + if (state.mode == CharSetParseState.Mode.ERROR) { + return new IntervalSet(); + } + int c = chars.codePointAt(i); + int offset = Character.charCount(c); + if (c == '\\') { + EscapeSequenceParsing.Result escapeParseResult = + EscapeSequenceParsing.parseEscape(chars, i); + switch (escapeParseResult.type) { + case INVALID: + String invalid = chars.substring(escapeParseResult.startOffset, + escapeParseResult.startOffset+escapeParseResult.parseLength); + g.tool.errMgr.grammarError(ErrorType.INVALID_ESCAPE_SEQUENCE, + g.fileName, charSetAST.getToken(), invalid); + state = CharSetParseState.ERROR; + break; + case CODE_POINT: + state = applyPrevStateAndMoveToCodePoint(charSetAST, set, state, escapeParseResult.codePoint); + break; + case PROPERTY: + state = applyPrevStateAndMoveToProperty(charSetAST, set, state, escapeParseResult.propertyIntervalSet); + break; + } + offset = escapeParseResult.parseLength; } - else if (i+2 < n && chars.charAt(i+1) == '-') { // range x-y - int x = c; - int y = chars.charAt(i+2); - if (x <= y) { - checkSetCollision(charSetAST, set, x, y); - set.add(x,y); + else if (c == '-' && !state.inRange && i != 0 && i != n - 1) { + if (state.mode == CharSetParseState.Mode.PREV_PROPERTY) { + g.tool.errMgr.grammarError(ErrorType.UNICODE_PROPERTY_NOT_ALLOWED_IN_RANGE, + g.fileName, charSetAST.getToken(), charSetAST.getText()); + state = CharSetParseState.ERROR; } else { - g.tool.errMgr.grammarError(ErrorType.EMPTY_STRINGS_AND_SETS_NOT_ALLOWED, - g.fileName, charSetAST.getToken(), "[" + (char) x + "-" + (char) y + "]"); + state = new CharSetParseState(state.mode, true, state.prevCodePoint, state.prevProperty); } - i += 2; } else { - checkSetCollision(charSetAST, set, c); - set.add(c); + state = applyPrevStateAndMoveToCodePoint(charSetAST, set, state, c); } + i += offset; + } + if (state.mode == CharSetParseState.Mode.ERROR) { + return new IntervalSet(); } + // Whether or not we were in a range, we'll add the last code point found to the set. + applyPrevState(charSetAST, set, state); return set; } - protected void checkSetCollision(GrammarAST ast, IntervalSet set, int el) { - if (set.contains(el)) { - g.tool.errMgr.grammarError(ErrorType.CHARACTERS_COLLISION_IN_SET, g.fileName, ast.getToken(), - (char)el, ast.getText()); + private CharSetParseState applyPrevStateAndMoveToCodePoint( + GrammarAST charSetAST, + IntervalSet set, + CharSetParseState state, + int codePoint) { + if (state.inRange) { + if (state.prevCodePoint > codePoint) { + g.tool.errMgr.grammarError( + ErrorType.EMPTY_STRINGS_AND_SETS_NOT_ALLOWED, + g.fileName, + charSetAST.getToken(), + CharSupport.getRangeEscapedString(state.prevCodePoint, codePoint)); + } + checkSetCollision(charSetAST, set, state.prevCodePoint, codePoint); + set.add(state.prevCodePoint, codePoint); + state = CharSetParseState.NONE; + } + else { + applyPrevState(charSetAST, set, state); + state = new CharSetParseState( + CharSetParseState.Mode.PREV_CODE_POINT, + false, + codePoint, + IntervalSet.EMPTY_SET); + } + return state; + } + + private CharSetParseState applyPrevStateAndMoveToProperty( + GrammarAST charSetAST, + IntervalSet set, + CharSetParseState state, + IntervalSet property) { + if (state.inRange) { + g.tool.errMgr.grammarError(ErrorType.UNICODE_PROPERTY_NOT_ALLOWED_IN_RANGE, + g.fileName, charSetAST.getToken(), charSetAST.getText()); + return CharSetParseState.ERROR; + } + else { + applyPrevState(charSetAST, set, state); + state = new CharSetParseState( + CharSetParseState.Mode.PREV_PROPERTY, + false, + -1, + property); + } + return state; + } + + private void applyPrevState(GrammarAST charSetAST, IntervalSet set, CharSetParseState state) { + switch (state.mode) { + case NONE: + case ERROR: + break; + case PREV_CODE_POINT: + checkSetCollision(charSetAST, set, state.prevCodePoint); + set.add(state.prevCodePoint); + break; + case PREV_PROPERTY: + set.addAll(state.prevProperty); + break; } } + protected void checkSetCollision(GrammarAST ast, IntervalSet set, int el) { + checkSetCollision(ast, set, el, el); + } + protected void checkSetCollision(GrammarAST ast, IntervalSet set, int a, int b) { for (int i = a; i <= b; i++) { if (set.contains(i)) { @@ -453,7 +591,7 @@ public class LexerATNFactory extends ParserATNFactory { setText = sb.toString(); } g.tool.errMgr.grammarError(ErrorType.CHARACTERS_COLLISION_IN_SET, g.fileName, ast.getToken(), - (char)a + "-" + (char)b, setText); + CharSupport.getRangeEscapedString(a, b), setText); break; } } @@ -538,23 +676,30 @@ public class LexerATNFactory extends ParserATNFactory { if (command.equals("skip")) { if (ruleCommands.contains("more")) { firstCommand = "more"; - } else if (ruleCommands.contains("type")) { + } + else if (ruleCommands.contains("type")) { firstCommand = "type"; - } else if (ruleCommands.contains("channel")) { + } + else if (ruleCommands.contains("channel")) { firstCommand = "channel"; } - } else if (command.equals("more")) { + } + else if (command.equals("more")) { if (ruleCommands.contains("skip")) { firstCommand = "skip"; - } else if (ruleCommands.contains("type")) { + } + else if (ruleCommands.contains("type")) { firstCommand = "type"; - } else if (ruleCommands.contains("channel")) { + } + else if (ruleCommands.contains("channel")) { firstCommand = "channel"; } - } else if (command.equals("type") || command.equals("channel")) { + } + else if (command.equals("type") || command.equals("channel")) { if (ruleCommands.contains("more")) { firstCommand = "more"; - } else if (ruleCommands.contains("skip")) { + } + else if (ruleCommands.contains("skip")) { firstCommand = "skip"; } } diff --git a/tool/src/org/antlr/v4/automata/ParserATNFactory.java b/tool/src/org/antlr/v4/automata/ParserATNFactory.java index 2cfaccc..830b7fd 100644 --- a/tool/src/org/antlr/v4/automata/ParserATNFactory.java +++ b/tool/src/org/antlr/v4/automata/ParserATNFactory.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ @@ -233,7 +233,12 @@ public class ParserATNFactory implements ATNFactory { @Override public Handle range(GrammarAST a, GrammarAST b) { - throw new UnsupportedOperationException("This construct is not valid in parsers."); + g.tool.errMgr.grammarError(ErrorType.TOKEN_RANGE_IN_PARSER, g.fileName, + a.getToken(), + a.getToken().getText(), + b.getToken().getText()); + // From a..b, yield ATN for just a. + return tokenRef((TerminalAST)a); } protected int getTokenType(GrammarAST atom) { diff --git a/tool/src/org/antlr/v4/automata/TailEpsilonRemover.java b/tool/src/org/antlr/v4/automata/TailEpsilonRemover.java index b534b43..c71e950 100644 --- a/tool/src/org/antlr/v4/automata/TailEpsilonRemover.java +++ b/tool/src/org/antlr/v4/automata/TailEpsilonRemover.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ @@ -45,7 +45,8 @@ public class TailEpsilonRemover extends ATNVisitor { // skip over q if (p.transition(0) instanceof RuleTransition) { ((RuleTransition) p.transition(0)).followState = r; - } else { + } + else { p.transition(0).target = r; } _atn.removeState(q); diff --git a/tool/src/org/antlr/v4/codegen/ActionTranslator.java b/tool/src/org/antlr/v4/codegen/ActionTranslator.java index d9d8208..b214c40 100644 --- a/tool/src/org/antlr/v4/codegen/ActionTranslator.java +++ b/tool/src/org/antlr/v4/codegen/ActionTranslator.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/tool/src/org/antlr/v4/codegen/BlankOutputModelFactory.java b/tool/src/org/antlr/v4/codegen/BlankOutputModelFactory.java index 148d2d5..a040c13 100644 --- a/tool/src/org/antlr/v4/codegen/BlankOutputModelFactory.java +++ b/tool/src/org/antlr/v4/codegen/BlankOutputModelFactory.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/tool/src/org/antlr/v4/codegen/CodeGenPipeline.java b/tool/src/org/antlr/v4/codegen/CodeGenPipeline.java index eb3e998..1f97ddd 100644 --- a/tool/src/org/antlr/v4/codegen/CodeGenPipeline.java +++ b/tool/src/org/antlr/v4/codegen/CodeGenPipeline.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/tool/src/org/antlr/v4/codegen/CodeGenerator.java b/tool/src/org/antlr/v4/codegen/CodeGenerator.java index 68115c1..bc1080f 100644 --- a/tool/src/org/antlr/v4/codegen/CodeGenerator.java +++ b/tool/src/org/antlr/v4/codegen/CodeGenerator.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/tool/src/org/antlr/v4/codegen/CodeGeneratorExtension.java b/tool/src/org/antlr/v4/codegen/CodeGeneratorExtension.java index 53cf2ea..1e21cc9 100644 --- a/tool/src/org/antlr/v4/codegen/CodeGeneratorExtension.java +++ b/tool/src/org/antlr/v4/codegen/CodeGeneratorExtension.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/tool/src/org/antlr/v4/codegen/DefaultOutputModelFactory.java b/tool/src/org/antlr/v4/codegen/DefaultOutputModelFactory.java index 3f84914..78ef4f6 100644 --- a/tool/src/org/antlr/v4/codegen/DefaultOutputModelFactory.java +++ b/tool/src/org/antlr/v4/codegen/DefaultOutputModelFactory.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/tool/src/org/antlr/v4/codegen/LexerFactory.java b/tool/src/org/antlr/v4/codegen/LexerFactory.java index b071861..3a87b8b 100644 --- a/tool/src/org/antlr/v4/codegen/LexerFactory.java +++ b/tool/src/org/antlr/v4/codegen/LexerFactory.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/tool/src/org/antlr/v4/codegen/OutputModelController.java b/tool/src/org/antlr/v4/codegen/OutputModelController.java index 8a38b66..5126670 100644 --- a/tool/src/org/antlr/v4/codegen/OutputModelController.java +++ b/tool/src/org/antlr/v4/codegen/OutputModelController.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ @@ -400,7 +400,7 @@ public class OutputModelController { public List<SrcOp> wildcard(GrammarAST ast, GrammarAST labelAST) { List<SrcOp> ops = delegate.wildcard(ast, labelAST); for (CodeGeneratorExtension ext : extensions) { - ops = ext.set(ops); + ops = ext.wildcard(ops); } return ops; } diff --git a/tool/src/org/antlr/v4/codegen/OutputModelFactory.java b/tool/src/org/antlr/v4/codegen/OutputModelFactory.java index cab452d..8bce642 100644 --- a/tool/src/org/antlr/v4/codegen/OutputModelFactory.java +++ b/tool/src/org/antlr/v4/codegen/OutputModelFactory.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/tool/src/org/antlr/v4/codegen/OutputModelWalker.java b/tool/src/org/antlr/v4/codegen/OutputModelWalker.java index fc3d80b..b740e16 100644 --- a/tool/src/org/antlr/v4/codegen/OutputModelWalker.java +++ b/tool/src/org/antlr/v4/codegen/OutputModelWalker.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/tool/src/org/antlr/v4/codegen/ParserFactory.java b/tool/src/org/antlr/v4/codegen/ParserFactory.java index 4086886..36742fc 100644 --- a/tool/src/org/antlr/v4/codegen/ParserFactory.java +++ b/tool/src/org/antlr/v4/codegen/ParserFactory.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ @@ -33,6 +33,7 @@ import org.antlr.v4.codegen.model.SemPred; import org.antlr.v4.codegen.model.SrcOp; import org.antlr.v4.codegen.model.StarBlock; import org.antlr.v4.codegen.model.TestSetInline; +import org.antlr.v4.codegen.model.Wildcard; import org.antlr.v4.codegen.model.decl.Decl; import org.antlr.v4.codegen.model.decl.RuleContextDecl; import org.antlr.v4.codegen.model.decl.TokenDecl; diff --git a/tool/src/org/antlr/v4/codegen/Target.java b/tool/src/org/antlr/v4/codegen/Target.java index 9a2fc5d..025b42a 100644 --- a/tool/src/org/antlr/v4/codegen/Target.java +++ b/tool/src/org/antlr/v4/codegen/Target.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ @@ -9,6 +9,7 @@ package org.antlr.v4.codegen; import org.antlr.v4.Tool; import org.antlr.v4.codegen.model.RuleFunction; import org.antlr.v4.codegen.model.SerializedATN; +import org.antlr.v4.misc.CharSupport; import org.antlr.v4.misc.Utils; import org.antlr.v4.parse.ANTLRParser; import org.antlr.v4.runtime.RuntimeMetaData; @@ -146,17 +147,22 @@ public abstract class Target { if ( quoted ) { buf.append('"'); } - for (int i=0; i<s.length(); i++) { - int c = s.charAt(i); + for (int i=0; i<s.length(); ) { + int c = s.codePointAt(i); if ( c!='\'' && // don't escape single quotes in strings for java c<targetCharValueEscape.length && targetCharValueEscape[c]!=null ) { buf.append(targetCharValueEscape[c]); } - else { - buf.append((char)c); + else if (shouldUseUnicodeEscapeForCodePointInDoubleQuotedString(c)) { + appendUnicodeEscapedCodePoint(i, buf); + } + else + { + buf.appendCodePoint(c); } + i += Character.charCount(c); } if ( quoted ) { buf.append('"'); @@ -164,6 +170,12 @@ public abstract class Target { return buf.toString(); } + /** + * Escape the Unicode code point appropriately for this language + * and append the escaped value to {@code sb}. + */ + abstract protected void appendUnicodeEscapedCodePoint(int codePoint, StringBuilder sb); + public String getTargetStringLiteralFromString(String s) { return getTargetStringLiteralFromString(s, true); } @@ -194,18 +206,21 @@ public abstract class Target { if ( addQuotes ) sb.append('"'); - for (int i = 1; i < is.length() -1; i++) { - if (is.charAt(i) == '\\') { + for (int i = 1; i < is.length() -1; ) { + int codePoint = is.codePointAt(i); + int toAdvance = Character.charCount(codePoint); + if (codePoint == '\\') { // Anything escaped is what it is! We assume that // people know how to escape characters correctly. However // we catch anything that does not need an escape in Java (which // is what the default implementation is dealing with and remove // the escape. The C target does this for instance. // - switch (is.charAt(i+1)) { + int escapedCodePoint = is.codePointAt(i+toAdvance); + toAdvance++; + switch (escapedCodePoint) { // Pass through any escapes that Java also needs // - case '"': case 'n': case 'r': case 't': @@ -214,32 +229,50 @@ public abstract class Target { case '\\': // Pass the escape through sb.append('\\'); + sb.appendCodePoint(escapedCodePoint); break; - case 'u': // Assume unnnn - // Pass the escape through as double \\ - // so that Java leaves as \u0000 string not char - sb.append('\\'); - sb.append('\\'); + case 'u': // Either unnnn or u{nnnnnn} + if (is.charAt(i+toAdvance) == '{') { + while (is.charAt(i+toAdvance) != '}') { + toAdvance++; + } + toAdvance++; + } + else { + toAdvance += 4; + } + if ( i+toAdvance <= is.length() ) { // we might have an invalid \\uAB or something + String fullEscape = is.substring(i, i+toAdvance); + appendUnicodeEscapedCodePoint( + CharSupport.getCharValueFromCharInGrammarLiteral(fullEscape), + sb); + } break; - default: - // Remove the escape by virtue of not adding it here - // Thus \' becomes ' and so on + if (shouldUseUnicodeEscapeForCodePointInDoubleQuotedString(escapedCodePoint)) { + appendUnicodeEscapedCodePoint(escapedCodePoint, sb); + } + else { + sb.appendCodePoint(escapedCodePoint); + } break; } - - // Go past the \ character - i++; - } else { - // Characters that don't need \ in ANTLR 'strings' but do in Java - if (is.charAt(i) == '"') { - // We need to escape " in Java - sb.append('\\'); + } + else { + if (codePoint == 0x22) { + // ANTLR doesn't escape " in literal strings, + // but every other language needs to do so. + sb.append("\\\""); + } + else if (shouldUseUnicodeEscapeForCodePointInDoubleQuotedString(codePoint)) { + appendUnicodeEscapedCodePoint(codePoint, sb); + } + else { + sb.appendCodePoint(codePoint); } } - // Add in the next character, which may have been escaped - sb.append(is.charAt(i)); + i += toAdvance; } if ( addQuotes ) sb.append('"'); @@ -247,6 +280,19 @@ public abstract class Target { return sb.toString(); } + private static boolean shouldUseUnicodeEscapeForCodePointInDoubleQuotedString(int codePoint) { + // We don't want anyone passing 0x0A (newline) or 0x22 + // (double-quote) here because Java treats \\u000A as + // a literal newline and \\u0022 as a literal + // double-quote, so Unicode escaping doesn't help. + assert codePoint != 0x0A && codePoint != 0x22; + + return + codePoint < 0x20 || // control characters up to but not including space + codePoint == 0x5C || // backslash + codePoint >= 0x7F; // DEL and beyond (keeps source code 7-bit US-ASCII) + } + /** Assume 16-bit char */ public String encodeIntAsCharEscape(int v) { if (v < Character.MIN_VALUE || v > Character.MAX_VALUE) { diff --git a/tool/src/org/antlr/v4/codegen/UnicodeEscapes.java b/tool/src/org/antlr/v4/codegen/UnicodeEscapes.java new file mode 100644 index 0000000..8c88b4a --- /dev/null +++ b/tool/src/org/antlr/v4/codegen/UnicodeEscapes.java @@ -0,0 +1,38 @@ +/* + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. + * Use of this file is governed by the BSD 3-clause license that + * can be found in the LICENSE.txt file in the project root. + */ + +package org.antlr.v4.codegen; + +/** + * Utility class to escape Unicode code points using various + * languages' syntaxes. + */ +public abstract class UnicodeEscapes { + static public void appendJavaStyleEscapedCodePoint(int codePoint, StringBuilder sb) { + if (Character.isSupplementaryCodePoint(codePoint)) { + // char is not an 'integral' type, so we have to explicitly convert + // to int before passing to the %X formatter or else it throws. + sb.append(String.format("\\u%04X", (int)Character.highSurrogate(codePoint))); + sb.append(String.format("\\u%04X", (int)Character.lowSurrogate(codePoint))); + } + else { + sb.append(String.format("\\u%04X", codePoint)); + } + } + + static public void appendPythonStyleEscapedCodePoint(int codePoint, StringBuilder sb) { + if (Character.isSupplementaryCodePoint(codePoint)) { + sb.append(String.format("\\U%08X", codePoint)); + } + else { + sb.append(String.format("\\u%04X", codePoint)); + } + } + + static public void appendSwiftStyleEscapedCodePoint(int codePoint, StringBuilder sb) { + sb.append(String.format("\\u{%04X}", codePoint)); + } +} diff --git a/tool/src/org/antlr/v4/codegen/model/Action.java b/tool/src/org/antlr/v4/codegen/model/Action.java index 5e1e0ef..ada06ae 100644 --- a/tool/src/org/antlr/v4/codegen/model/Action.java +++ b/tool/src/org/antlr/v4/codegen/model/Action.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/tool/src/org/antlr/v4/codegen/model/AddToLabelList.java b/tool/src/org/antlr/v4/codegen/model/AddToLabelList.java index a492ee9..27a169a 100644 --- a/tool/src/org/antlr/v4/codegen/model/AddToLabelList.java +++ b/tool/src/org/antlr/v4/codegen/model/AddToLabelList.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/tool/src/org/antlr/v4/codegen/model/AltBlock.java b/tool/src/org/antlr/v4/codegen/model/AltBlock.java index a632921..db0654f 100644 --- a/tool/src/org/antlr/v4/codegen/model/AltBlock.java +++ b/tool/src/org/antlr/v4/codegen/model/AltBlock.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/tool/src/org/antlr/v4/codegen/model/ArgAction.java b/tool/src/org/antlr/v4/codegen/model/ArgAction.java index c52c72c..29dc530 100644 --- a/tool/src/org/antlr/v4/codegen/model/ArgAction.java +++ b/tool/src/org/antlr/v4/codegen/model/ArgAction.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/tool/src/org/antlr/v4/codegen/model/BaseListenerFile.java b/tool/src/org/antlr/v4/codegen/model/BaseListenerFile.java index 206fcee..6dc6546 100644 --- a/tool/src/org/antlr/v4/codegen/model/BaseListenerFile.java +++ b/tool/src/org/antlr/v4/codegen/model/BaseListenerFile.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/tool/src/org/antlr/v4/codegen/model/BaseVisitorFile.java b/tool/src/org/antlr/v4/codegen/model/BaseVisitorFile.java index 9fff357..6c41307 100644 --- a/tool/src/org/antlr/v4/codegen/model/BaseVisitorFile.java +++ b/tool/src/org/antlr/v4/codegen/model/BaseVisitorFile.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/tool/src/org/antlr/v4/codegen/model/CaptureNextToken.java b/tool/src/org/antlr/v4/codegen/model/CaptureNextToken.java index de49cdc..cbc0f28 100644 --- a/tool/src/org/antlr/v4/codegen/model/CaptureNextToken.java +++ b/tool/src/org/antlr/v4/codegen/model/CaptureNextToken.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/tool/src/org/antlr/v4/codegen/model/CaptureNextTokenType.java b/tool/src/org/antlr/v4/codegen/model/CaptureNextTokenType.java index 094d941..41f08ec 100644 --- a/tool/src/org/antlr/v4/codegen/model/CaptureNextTokenType.java +++ b/tool/src/org/antlr/v4/codegen/model/CaptureNextTokenType.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/tool/src/org/antlr/v4/codegen/model/Choice.java b/tool/src/org/antlr/v4/codegen/model/Choice.java index 9fb8a46..6b0d279 100644 --- a/tool/src/org/antlr/v4/codegen/model/Choice.java +++ b/tool/src/org/antlr/v4/codegen/model/Choice.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/tool/src/org/antlr/v4/codegen/model/CodeBlockForAlt.java b/tool/src/org/antlr/v4/codegen/model/CodeBlockForAlt.java index 12023a7..638f035 100644 --- a/tool/src/org/antlr/v4/codegen/model/CodeBlockForAlt.java +++ b/tool/src/org/antlr/v4/codegen/model/CodeBlockForAlt.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/tool/src/org/antlr/v4/codegen/model/CodeBlockForOuterMostAlt.java b/tool/src/org/antlr/v4/codegen/model/CodeBlockForOuterMostAlt.java index ef3e9d6..236b1fe 100644 --- a/tool/src/org/antlr/v4/codegen/model/CodeBlockForOuterMostAlt.java +++ b/tool/src/org/antlr/v4/codegen/model/CodeBlockForOuterMostAlt.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/tool/src/org/antlr/v4/codegen/model/DispatchMethod.java b/tool/src/org/antlr/v4/codegen/model/DispatchMethod.java index f7b3290..c68b936 100644 --- a/tool/src/org/antlr/v4/codegen/model/DispatchMethod.java +++ b/tool/src/org/antlr/v4/codegen/model/DispatchMethod.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/tool/src/org/antlr/v4/codegen/model/ElementFrequenciesVisitor.java b/tool/src/org/antlr/v4/codegen/model/ElementFrequenciesVisitor.java index ab5b86d..ccd5c80 100644 --- a/tool/src/org/antlr/v4/codegen/model/ElementFrequenciesVisitor.java +++ b/tool/src/org/antlr/v4/codegen/model/ElementFrequenciesVisitor.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ @@ -21,12 +21,30 @@ import java.util.Deque; import java.util.Map; public class ElementFrequenciesVisitor extends GrammarTreeVisitor { + /** + * This special value means "no set", and is used by {@link #minFrequencies} + * to ensure that {@link #combineMin} doesn't merge an empty set (all zeros) + * with the results of the first alternative. + */ + private static final FrequencySet<String> SENTINEL = new FrequencySet<String>(); + final Deque<FrequencySet<String>> frequencies; + private final Deque<FrequencySet<String>> minFrequencies; public ElementFrequenciesVisitor(TreeNodeStream input) { super(input); frequencies = new ArrayDeque<FrequencySet<String>>(); frequencies.push(new FrequencySet<String>()); + minFrequencies = new ArrayDeque<FrequencySet<String>>(); + minFrequencies.push(SENTINEL); + } + + FrequencySet<String> getMinFrequencies() { + assert minFrequencies.size() == 1; + assert minFrequencies.peek() != SENTINEL; + assert SENTINEL.isEmpty(); + + return minFrequencies.peek(); } /** During code gen, we can assume tree is in good shape */ @@ -62,6 +80,31 @@ public class ElementFrequenciesVisitor extends GrammarTreeVisitor { } /** + * Generate a frequency set as the union of two input sets. If an + * element is contained in both sets, the value for the output will be + * the minimum of the two input values. + * + * @param a The first set. + * @param b The second set. If this set is {@link #SENTINEL}, it is treated + * as though no second set were provided. + * @return The union of the two sets, with the minimum value chosen + * whenever both sets contain the same key. + */ + protected static FrequencySet<String> combineMin(FrequencySet<String> a, FrequencySet<String> b) { + if (b == SENTINEL) { + return a; + } + + assert a != SENTINEL; + FrequencySet<String> result = combineAndClip(a, b, Integer.MAX_VALUE); + for (Map.Entry<String, MutableInt> entry : result.entrySet()) { + entry.getValue().v = Math.min(a.count(entry.getKey()), b.count(entry.getKey())); + } + + return result; + } + + /** * Generate a frequency set as the union of two input sets, with the * values clipped to a specified maximum value. If an element is * contained in both sets, the value for the output, prior to clipping, @@ -97,11 +140,13 @@ public class ElementFrequenciesVisitor extends GrammarTreeVisitor { @Override public void tokenRef(TerminalAST ref) { frequencies.peek().add(ref.getText()); + minFrequencies.peek().add(ref.getText()); } @Override public void ruleRef(GrammarAST ref, ActionAST arg) { frequencies.peek().add(ref.getText()); + minFrequencies.peek().add(ref.getText()); } /* @@ -111,21 +156,50 @@ public class ElementFrequenciesVisitor extends GrammarTreeVisitor { @Override protected void enterAlternative(AltAST tree) { frequencies.push(new FrequencySet<String>()); + minFrequencies.push(new FrequencySet<String>()); } @Override protected void exitAlternative(AltAST tree) { frequencies.push(combineMax(frequencies.pop(), frequencies.pop())); + minFrequencies.push(combineMin(minFrequencies.pop(), minFrequencies.pop())); } @Override protected void enterElement(GrammarAST tree) { frequencies.push(new FrequencySet<String>()); + minFrequencies.push(new FrequencySet<String>()); } @Override protected void exitElement(GrammarAST tree) { frequencies.push(combineAndClip(frequencies.pop(), frequencies.pop(), 2)); + minFrequencies.push(combineAndClip(minFrequencies.pop(), minFrequencies.pop(), 2)); + } + + @Override + protected void enterBlockSet(GrammarAST tree) { + frequencies.push(new FrequencySet<String>()); + minFrequencies.push(new FrequencySet<String>()); + } + + @Override + protected void exitBlockSet(GrammarAST tree) { + for (Map.Entry<String, MutableInt> entry : frequencies.peek().entrySet()) { + // This visitor counts a block set as a sequence of elements, not a + // sequence of alternatives of elements. Reset the count back to 1 + // for all items when leaving the set to ensure duplicate entries in + // the set are treated as a maximum of one item. + entry.getValue().v = 1; + } + + if (minFrequencies.peek().size() > 1) { + // Everything is optional + minFrequencies.peek().clear(); + } + + frequencies.push(combineAndClip(frequencies.pop(), frequencies.pop(), 2)); + minFrequencies.push(combineAndClip(minFrequencies.pop(), minFrequencies.pop(), 2)); } @Override @@ -135,6 +209,12 @@ public class ElementFrequenciesVisitor extends GrammarTreeVisitor { entry.getValue().v = 2; } } + + if (tree.getType() == CLOSURE || tree.getType() == OPTIONAL) { + // Everything inside a closure is optional, so the minimum + // number of occurrences for all elements is 0. + minFrequencies.peek().clear(); + } } /* @@ -144,21 +224,25 @@ public class ElementFrequenciesVisitor extends GrammarTreeVisitor { @Override protected void enterLexerAlternative(GrammarAST tree) { frequencies.push(new FrequencySet<String>()); + minFrequencies.push(new FrequencySet<String>()); } @Override protected void exitLexerAlternative(GrammarAST tree) { frequencies.push(combineMax(frequencies.pop(), frequencies.pop())); + minFrequencies.push(combineMin(minFrequencies.pop(), minFrequencies.pop())); } @Override protected void enterLexerElement(GrammarAST tree) { frequencies.push(new FrequencySet<String>()); + minFrequencies.push(new FrequencySet<String>()); } @Override protected void exitLexerElement(GrammarAST tree) { frequencies.push(combineAndClip(frequencies.pop(), frequencies.pop(), 2)); + minFrequencies.push(combineAndClip(minFrequencies.pop(), minFrequencies.pop(), 2)); } @Override @@ -168,5 +252,11 @@ public class ElementFrequenciesVisitor extends GrammarTreeVisitor { entry.getValue().v = 2; } } + + if (tree.getType() == CLOSURE) { + // Everything inside a closure is optional, so the minimum + // number of occurrences for all elements is 0. + minFrequencies.peek().clear(); + } } } diff --git a/tool/src/org/antlr/v4/codegen/model/ExceptionClause.java b/tool/src/org/antlr/v4/codegen/model/ExceptionClause.java index 0eb7ddc..18aac15 100644 --- a/tool/src/org/antlr/v4/codegen/model/ExceptionClause.java +++ b/tool/src/org/antlr/v4/codegen/model/ExceptionClause.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/tool/src/org/antlr/v4/codegen/model/InvokeRule.java b/tool/src/org/antlr/v4/codegen/model/InvokeRule.java index bf83499..fb580cc 100644 --- a/tool/src/org/antlr/v4/codegen/model/InvokeRule.java +++ b/tool/src/org/antlr/v4/codegen/model/InvokeRule.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/tool/src/org/antlr/v4/codegen/model/LL1AltBlock.java b/tool/src/org/antlr/v4/codegen/model/LL1AltBlock.java index 3437388..d1b6d3a 100644 --- a/tool/src/org/antlr/v4/codegen/model/LL1AltBlock.java +++ b/tool/src/org/antlr/v4/codegen/model/LL1AltBlock.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/tool/src/org/antlr/v4/codegen/model/LL1Choice.java b/tool/src/org/antlr/v4/codegen/model/LL1Choice.java index 9ef47d2..745ae4e 100644 --- a/tool/src/org/antlr/v4/codegen/model/LL1Choice.java +++ b/tool/src/org/antlr/v4/codegen/model/LL1Choice.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/tool/src/org/antlr/v4/codegen/model/LL1Loop.java b/tool/src/org/antlr/v4/codegen/model/LL1Loop.java index 839a14d..78cfabd 100644 --- a/tool/src/org/antlr/v4/codegen/model/LL1Loop.java +++ b/tool/src/org/antlr/v4/codegen/model/LL1Loop.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/tool/src/org/antlr/v4/codegen/model/LL1OptionalBlock.java b/tool/src/org/antlr/v4/codegen/model/LL1OptionalBlock.java index 80f8d0b..652967e 100644 --- a/tool/src/org/antlr/v4/codegen/model/LL1OptionalBlock.java +++ b/tool/src/org/antlr/v4/codegen/model/LL1OptionalBlock.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/tool/src/org/antlr/v4/codegen/model/LL1OptionalBlockSingleAlt.java b/tool/src/org/antlr/v4/codegen/model/LL1OptionalBlockSingleAlt.java index 50579ca..6d4fbd2 100644 --- a/tool/src/org/antlr/v4/codegen/model/LL1OptionalBlockSingleAlt.java +++ b/tool/src/org/antlr/v4/codegen/model/LL1OptionalBlockSingleAlt.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/tool/src/org/antlr/v4/codegen/model/LL1PlusBlockSingleAlt.java b/tool/src/org/antlr/v4/codegen/model/LL1PlusBlockSingleAlt.java index 227753f..dcfaf11 100644 --- a/tool/src/org/antlr/v4/codegen/model/LL1PlusBlockSingleAlt.java +++ b/tool/src/org/antlr/v4/codegen/model/LL1PlusBlockSingleAlt.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/tool/src/org/antlr/v4/codegen/model/LL1StarBlockSingleAlt.java b/tool/src/org/antlr/v4/codegen/model/LL1StarBlockSingleAlt.java index fb787c1..27d46ee 100644 --- a/tool/src/org/antlr/v4/codegen/model/LL1StarBlockSingleAlt.java +++ b/tool/src/org/antlr/v4/codegen/model/LL1StarBlockSingleAlt.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/tool/src/org/antlr/v4/codegen/model/LabeledOp.java b/tool/src/org/antlr/v4/codegen/model/LabeledOp.java index b8d5441..73d24d7 100644 --- a/tool/src/org/antlr/v4/codegen/model/LabeledOp.java +++ b/tool/src/org/antlr/v4/codegen/model/LabeledOp.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/tool/src/org/antlr/v4/codegen/model/LeftRecursiveRuleFunction.java b/tool/src/org/antlr/v4/codegen/model/LeftRecursiveRuleFunction.java index abe3f7f..cf6420e 100644 --- a/tool/src/org/antlr/v4/codegen/model/LeftRecursiveRuleFunction.java +++ b/tool/src/org/antlr/v4/codegen/model/LeftRecursiveRuleFunction.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/tool/src/org/antlr/v4/codegen/model/Lexer.java b/tool/src/org/antlr/v4/codegen/model/Lexer.java index 2bc103e..35170e2 100644 --- a/tool/src/org/antlr/v4/codegen/model/Lexer.java +++ b/tool/src/org/antlr/v4/codegen/model/Lexer.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/tool/src/org/antlr/v4/codegen/model/LexerFile.java b/tool/src/org/antlr/v4/codegen/model/LexerFile.java index 3fae78b..9e02c60 100644 --- a/tool/src/org/antlr/v4/codegen/model/LexerFile.java +++ b/tool/src/org/antlr/v4/codegen/model/LexerFile.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/tool/src/org/antlr/v4/codegen/model/ListenerDispatchMethod.java b/tool/src/org/antlr/v4/codegen/model/ListenerDispatchMethod.java index cc861f0..2a8a39f 100644 --- a/tool/src/org/antlr/v4/codegen/model/ListenerDispatchMethod.java +++ b/tool/src/org/antlr/v4/codegen/model/ListenerDispatchMethod.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/tool/src/org/antlr/v4/codegen/model/ListenerFile.java b/tool/src/org/antlr/v4/codegen/model/ListenerFile.java index 24efd74..2066c16 100644 --- a/tool/src/org/antlr/v4/codegen/model/ListenerFile.java +++ b/tool/src/org/antlr/v4/codegen/model/ListenerFile.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ @@ -23,6 +23,7 @@ import java.util.Set; */ public class ListenerFile extends OutputFile { public String genPackage; // from -package cmd-line + public String accessLevel; // from -DaccessLevel cmd-line public String exportMacro; // from -DexportMacro cmd-line public String grammarName; public String parserName; @@ -61,7 +62,8 @@ public class ListenerFile extends OutputFile { } ActionAST ast = g.namedActions.get("header"); if ( ast!=null ) header = new Action(factory, ast); - genPackage = factory.getGrammar().tool.genPackage; - exportMacro = factory.getGrammar().getOptionString("exportMacro"); + genPackage = g.tool.genPackage; + accessLevel = g.getOptionString("accessLevel"); + exportMacro = g.getOptionString("exportMacro"); } } diff --git a/tool/src/org/antlr/v4/codegen/model/Loop.java b/tool/src/org/antlr/v4/codegen/model/Loop.java index 695685e..38d29d6 100644 --- a/tool/src/org/antlr/v4/codegen/model/Loop.java +++ b/tool/src/org/antlr/v4/codegen/model/Loop.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/tool/src/org/antlr/v4/codegen/model/MatchNotSet.java b/tool/src/org/antlr/v4/codegen/model/MatchNotSet.java index 4b86f5e..68687ee 100644 --- a/tool/src/org/antlr/v4/codegen/model/MatchNotSet.java +++ b/tool/src/org/antlr/v4/codegen/model/MatchNotSet.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/tool/src/org/antlr/v4/codegen/model/MatchSet.java b/tool/src/org/antlr/v4/codegen/model/MatchSet.java index ab1f798..48e710c 100644 --- a/tool/src/org/antlr/v4/codegen/model/MatchSet.java +++ b/tool/src/org/antlr/v4/codegen/model/MatchSet.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/tool/src/org/antlr/v4/codegen/model/MatchToken.java b/tool/src/org/antlr/v4/codegen/model/MatchToken.java index 5399113..6756f82 100644 --- a/tool/src/org/antlr/v4/codegen/model/MatchToken.java +++ b/tool/src/org/antlr/v4/codegen/model/MatchToken.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/tool/src/org/antlr/v4/codegen/model/ModelElement.java b/tool/src/org/antlr/v4/codegen/model/ModelElement.java index a0401de..cff1d61 100644 --- a/tool/src/org/antlr/v4/codegen/model/ModelElement.java +++ b/tool/src/org/antlr/v4/codegen/model/ModelElement.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/tool/src/org/antlr/v4/codegen/model/OptionalBlock.java b/tool/src/org/antlr/v4/codegen/model/OptionalBlock.java index 50b8fe3..ade58a1 100644 --- a/tool/src/org/antlr/v4/codegen/model/OptionalBlock.java +++ b/tool/src/org/antlr/v4/codegen/model/OptionalBlock.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/tool/src/org/antlr/v4/codegen/model/OutputFile.java b/tool/src/org/antlr/v4/codegen/model/OutputFile.java index e238dae..cd23324 100644 --- a/tool/src/org/antlr/v4/codegen/model/OutputFile.java +++ b/tool/src/org/antlr/v4/codegen/model/OutputFile.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/tool/src/org/antlr/v4/codegen/model/OutputModelObject.java b/tool/src/org/antlr/v4/codegen/model/OutputModelObject.java index 5cfb1ee..96e44e4 100644 --- a/tool/src/org/antlr/v4/codegen/model/OutputModelObject.java +++ b/tool/src/org/antlr/v4/codegen/model/OutputModelObject.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/tool/src/org/antlr/v4/codegen/model/Parser.java b/tool/src/org/antlr/v4/codegen/model/Parser.java index 298bcff..4e5e135 100644 --- a/tool/src/org/antlr/v4/codegen/model/Parser.java +++ b/tool/src/org/antlr/v4/codegen/model/Parser.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/tool/src/org/antlr/v4/codegen/model/ParserFile.java b/tool/src/org/antlr/v4/codegen/model/ParserFile.java index 49181dd..4cd6b26 100644 --- a/tool/src/org/antlr/v4/codegen/model/ParserFile.java +++ b/tool/src/org/antlr/v4/codegen/model/ParserFile.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/tool/src/org/antlr/v4/codegen/model/PlusBlock.java b/tool/src/org/antlr/v4/codegen/model/PlusBlock.java index 088089c..424ae03 100644 --- a/tool/src/org/antlr/v4/codegen/model/PlusBlock.java +++ b/tool/src/org/antlr/v4/codegen/model/PlusBlock.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/tool/src/org/antlr/v4/codegen/model/Recognizer.java b/tool/src/org/antlr/v4/codegen/model/Recognizer.java index 723dd08..5d79390 100644 --- a/tool/src/org/antlr/v4/codegen/model/Recognizer.java +++ b/tool/src/org/antlr/v4/codegen/model/Recognizer.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ @@ -24,6 +24,7 @@ public abstract class Recognizer extends OutputModelObject { public String name; public String grammarName; public String grammarFileName; + public String accessLevel; public Map<String,Integer> tokens; /** @@ -51,6 +52,7 @@ public abstract class Recognizer extends OutputModelObject { grammarFileName = new File(g.fileName).getName(); grammarName = g.name; name = g.getRecognizerName(); + accessLevel = g.getOptionString("accessLevel"); tokens = new LinkedHashMap<String,Integer>(); for (Map.Entry<String, Integer> entry : g.tokenNameToTypeMap.entrySet()) { Integer ttype = entry.getValue(); diff --git a/tool/src/org/antlr/v4/codegen/model/RuleActionFunction.java b/tool/src/org/antlr/v4/codegen/model/RuleActionFunction.java index 624ffd4..fffd5d2 100644 --- a/tool/src/org/antlr/v4/codegen/model/RuleActionFunction.java +++ b/tool/src/org/antlr/v4/codegen/model/RuleActionFunction.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/tool/src/org/antlr/v4/codegen/model/RuleElement.java b/tool/src/org/antlr/v4/codegen/model/RuleElement.java index ba0c0c0..8643bb9 100644 --- a/tool/src/org/antlr/v4/codegen/model/RuleElement.java +++ b/tool/src/org/antlr/v4/codegen/model/RuleElement.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/tool/src/org/antlr/v4/codegen/model/RuleFunction.java b/tool/src/org/antlr/v4/codegen/model/RuleFunction.java index d6e165e..d8b69b5 100644 --- a/tool/src/org/antlr/v4/codegen/model/RuleFunction.java +++ b/tool/src/org/antlr/v4/codegen/model/RuleFunction.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ @@ -171,49 +171,66 @@ public class RuleFunction extends OutputModelObject { */ public Set<Decl> getDeclsForAllElements(List<AltAST> altASTs) { Set<String> needsList = new HashSet<String>(); + Set<String> nonOptional = new HashSet<String>(); List<GrammarAST> allRefs = new ArrayList<GrammarAST>(); + boolean firstAlt = true; for (AltAST ast : altASTs) { IntervalSet reftypes = new IntervalSet(RULE_REF, TOKEN_REF); List<GrammarAST> refs = ast.getNodesWithType(reftypes); allRefs.addAll(refs); - FrequencySet<String> altFreq = getElementFrequenciesForAlt(ast); + Pair<FrequencySet<String>, FrequencySet<String>> minAndAltFreq = getElementFrequenciesForAlt(ast); + FrequencySet<String> minFreq = minAndAltFreq.a; + FrequencySet<String> altFreq = minAndAltFreq.b; for (GrammarAST t : refs) { String refLabelName = t.getText(); if ( altFreq.count(refLabelName)>1 ) { needsList.add(refLabelName); } + + if (firstAlt && minFreq.count(refLabelName) != 0) { + nonOptional.add(refLabelName); + } + } + + for (String ref : nonOptional.toArray(new String[nonOptional.size()])) { + if (minFreq.count(ref) == 0) { + nonOptional.remove(ref); + } } + + firstAlt = false; } Set<Decl> decls = new LinkedHashSet<Decl>(); for (GrammarAST t : allRefs) { String refLabelName = t.getText(); List<Decl> d = getDeclForAltElement(t, refLabelName, - needsList.contains(refLabelName)); + needsList.contains(refLabelName), + !nonOptional.contains(refLabelName)); decls.addAll(d); } return decls; } /** Given list of X and r refs in alt, compute how many of each there are */ - protected FrequencySet<String> getElementFrequenciesForAlt(AltAST ast) { + protected Pair<FrequencySet<String>, FrequencySet<String>> getElementFrequenciesForAlt(AltAST ast) { try { ElementFrequenciesVisitor visitor = new ElementFrequenciesVisitor(new CommonTreeNodeStream(new GrammarASTAdaptor(), ast)); visitor.outerAlternative(); if (visitor.frequencies.size() != 1) { factory.getGrammar().tool.errMgr.toolError(ErrorType.INTERNAL_ERROR); - return new FrequencySet<String>(); + return new Pair<>(new FrequencySet<String>(), new FrequencySet<String>()); } - return visitor.frequencies.peek(); + return new Pair<>(visitor.getMinFrequencies(), visitor.frequencies.peek()); } catch (RecognitionException ex) { factory.getGrammar().tool.errMgr.toolError(ErrorType.INTERNAL_ERROR, ex); - return new FrequencySet<String>(); + return new Pair<>(new FrequencySet<String>(), new FrequencySet<String>()); } } - public List<Decl> getDeclForAltElement(GrammarAST t, String refLabelName, boolean needList) { + public List<Decl> getDeclForAltElement(GrammarAST t, String refLabelName, boolean needList, boolean optional) { List<Decl> decls = new ArrayList<Decl>(); if ( t.getType()==RULE_REF ) { Rule rref = factory.getGrammar().getRule(t.getText()); @@ -225,7 +242,7 @@ public class RuleFunction extends OutputModelObject { decls.add( new ContextRuleListIndexedGetterDecl(factory, refLabelName, ctxName) ); } else { - decls.add( new ContextRuleGetterDecl(factory, refLabelName, ctxName) ); + decls.add( new ContextRuleGetterDecl(factory, refLabelName, ctxName, optional) ); } } else { @@ -235,7 +252,7 @@ public class RuleFunction extends OutputModelObject { decls.add( new ContextTokenListIndexedGetterDecl(factory, refLabelName) ); } else { - decls.add( new ContextTokenGetterDecl(factory, refLabelName) ); + decls.add( new ContextTokenGetterDecl(factory, refLabelName, optional) ); } } return decls; diff --git a/tool/src/org/antlr/v4/codegen/model/RuleSempredFunction.java b/tool/src/org/antlr/v4/codegen/model/RuleSempredFunction.java index aba9d7e..75d10cb 100644 --- a/tool/src/org/antlr/v4/codegen/model/RuleSempredFunction.java +++ b/tool/src/org/antlr/v4/codegen/model/RuleSempredFunction.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/tool/src/org/antlr/v4/codegen/model/SemPred.java b/tool/src/org/antlr/v4/codegen/model/SemPred.java index e42649b..3496b44 100644 --- a/tool/src/org/antlr/v4/codegen/model/SemPred.java +++ b/tool/src/org/antlr/v4/codegen/model/SemPred.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/tool/src/org/antlr/v4/codegen/model/SerializedATN.java b/tool/src/org/antlr/v4/codegen/model/SerializedATN.java index 6a1094e..1ace766 100644 --- a/tool/src/org/antlr/v4/codegen/model/SerializedATN.java +++ b/tool/src/org/antlr/v4/codegen/model/SerializedATN.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/tool/src/org/antlr/v4/codegen/model/SrcOp.java b/tool/src/org/antlr/v4/codegen/model/SrcOp.java index ff9bd9d..ab51f88 100644 --- a/tool/src/org/antlr/v4/codegen/model/SrcOp.java +++ b/tool/src/org/antlr/v4/codegen/model/SrcOp.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/tool/src/org/antlr/v4/codegen/model/StarBlock.java b/tool/src/org/antlr/v4/codegen/model/StarBlock.java index 41b49f2..6c32293 100644 --- a/tool/src/org/antlr/v4/codegen/model/StarBlock.java +++ b/tool/src/org/antlr/v4/codegen/model/StarBlock.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/tool/src/org/antlr/v4/codegen/model/Sync.java b/tool/src/org/antlr/v4/codegen/model/Sync.java index db5287e..3c4f23a 100644 --- a/tool/src/org/antlr/v4/codegen/model/Sync.java +++ b/tool/src/org/antlr/v4/codegen/model/Sync.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/tool/src/org/antlr/v4/codegen/model/TestSetInline.java b/tool/src/org/antlr/v4/codegen/model/TestSetInline.java index d4c9381..dfbfd03 100644 --- a/tool/src/org/antlr/v4/codegen/model/TestSetInline.java +++ b/tool/src/org/antlr/v4/codegen/model/TestSetInline.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/tool/src/org/antlr/v4/codegen/model/ThrowEarlyExitException.java b/tool/src/org/antlr/v4/codegen/model/ThrowEarlyExitException.java index efadf44..870271f 100644 --- a/tool/src/org/antlr/v4/codegen/model/ThrowEarlyExitException.java +++ b/tool/src/org/antlr/v4/codegen/model/ThrowEarlyExitException.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/tool/src/org/antlr/v4/codegen/model/ThrowNoViableAlt.java b/tool/src/org/antlr/v4/codegen/model/ThrowNoViableAlt.java index 7bb6bbd..bc33dc8 100644 --- a/tool/src/org/antlr/v4/codegen/model/ThrowNoViableAlt.java +++ b/tool/src/org/antlr/v4/codegen/model/ThrowNoViableAlt.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/tool/src/org/antlr/v4/codegen/model/ThrowRecognitionException.java b/tool/src/org/antlr/v4/codegen/model/ThrowRecognitionException.java index 6653038..4f1d630 100644 --- a/tool/src/org/antlr/v4/codegen/model/ThrowRecognitionException.java +++ b/tool/src/org/antlr/v4/codegen/model/ThrowRecognitionException.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/tool/src/org/antlr/v4/codegen/model/VisitorDispatchMethod.java b/tool/src/org/antlr/v4/codegen/model/VisitorDispatchMethod.java index 90c6e82..3c62be8 100644 --- a/tool/src/org/antlr/v4/codegen/model/VisitorDispatchMethod.java +++ b/tool/src/org/antlr/v4/codegen/model/VisitorDispatchMethod.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/tool/src/org/antlr/v4/codegen/model/VisitorFile.java b/tool/src/org/antlr/v4/codegen/model/VisitorFile.java index 4abcdb1..3c746d6 100644 --- a/tool/src/org/antlr/v4/codegen/model/VisitorFile.java +++ b/tool/src/org/antlr/v4/codegen/model/VisitorFile.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ @@ -20,6 +20,7 @@ import java.util.Set; public class VisitorFile extends OutputFile { public String genPackage; // from -package cmd-line + public String accessLevel; // from -DaccessLevel cmd-line public String exportMacro; // from -DexportMacro cmd-line public String grammarName; public String parserName; @@ -58,7 +59,8 @@ public class VisitorFile extends OutputFile { } ActionAST ast = g.namedActions.get("header"); if ( ast!=null ) header = new Action(factory, ast); - genPackage = factory.getGrammar().tool.genPackage; - exportMacro = factory.getGrammar().getOptionString("exportMacro"); + genPackage = g.tool.genPackage; + accessLevel = g.getOptionString("accessLevel"); + exportMacro = g.getOptionString("exportMacro"); } } diff --git a/tool/src/org/antlr/v4/codegen/Wildcard.java b/tool/src/org/antlr/v4/codegen/model/Wildcard.java index 6537cc7..977d22c 100644 --- a/tool/src/org/antlr/v4/codegen/Wildcard.java +++ b/tool/src/org/antlr/v4/codegen/model/Wildcard.java @@ -1,12 +1,12 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ -package org.antlr.v4.codegen; +package org.antlr.v4.codegen.model; -import org.antlr.v4.codegen.model.MatchToken; +import org.antlr.v4.codegen.OutputModelFactory; import org.antlr.v4.tool.ast.GrammarAST; public class Wildcard extends MatchToken { diff --git a/tool/src/org/antlr/v4/codegen/model/chunk/ActionChunk.java b/tool/src/org/antlr/v4/codegen/model/chunk/ActionChunk.java index 36775b5..53d046a 100644 --- a/tool/src/org/antlr/v4/codegen/model/chunk/ActionChunk.java +++ b/tool/src/org/antlr/v4/codegen/model/chunk/ActionChunk.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/tool/src/org/antlr/v4/codegen/model/chunk/ActionTemplate.java b/tool/src/org/antlr/v4/codegen/model/chunk/ActionTemplate.java index 861fd8f..28dd97a 100644 --- a/tool/src/org/antlr/v4/codegen/model/chunk/ActionTemplate.java +++ b/tool/src/org/antlr/v4/codegen/model/chunk/ActionTemplate.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/tool/src/org/antlr/v4/codegen/model/chunk/ActionText.java b/tool/src/org/antlr/v4/codegen/model/chunk/ActionText.java index b1c64c2..4ed87a1 100644 --- a/tool/src/org/antlr/v4/codegen/model/chunk/ActionText.java +++ b/tool/src/org/antlr/v4/codegen/model/chunk/ActionText.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/tool/src/org/antlr/v4/codegen/model/chunk/ArgRef.java b/tool/src/org/antlr/v4/codegen/model/chunk/ArgRef.java index 8544692..4c10fb1 100644 --- a/tool/src/org/antlr/v4/codegen/model/chunk/ArgRef.java +++ b/tool/src/org/antlr/v4/codegen/model/chunk/ArgRef.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/tool/src/org/antlr/v4/codegen/model/chunk/LabelRef.java b/tool/src/org/antlr/v4/codegen/model/chunk/LabelRef.java index 5e5418b..3f703a8 100644 --- a/tool/src/org/antlr/v4/codegen/model/chunk/LabelRef.java +++ b/tool/src/org/antlr/v4/codegen/model/chunk/LabelRef.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/tool/src/org/antlr/v4/codegen/model/chunk/ListLabelRef.java b/tool/src/org/antlr/v4/codegen/model/chunk/ListLabelRef.java index 07dc911..dc941ba 100644 --- a/tool/src/org/antlr/v4/codegen/model/chunk/ListLabelRef.java +++ b/tool/src/org/antlr/v4/codegen/model/chunk/ListLabelRef.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/tool/src/org/antlr/v4/codegen/model/chunk/LocalRef.java b/tool/src/org/antlr/v4/codegen/model/chunk/LocalRef.java index 6cf25ab..6a797cf 100644 --- a/tool/src/org/antlr/v4/codegen/model/chunk/LocalRef.java +++ b/tool/src/org/antlr/v4/codegen/model/chunk/LocalRef.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/tool/src/org/antlr/v4/codegen/model/chunk/NonLocalAttrRef.java b/tool/src/org/antlr/v4/codegen/model/chunk/NonLocalAttrRef.java index db6763f..37698f7 100644 --- a/tool/src/org/antlr/v4/codegen/model/chunk/NonLocalAttrRef.java +++ b/tool/src/org/antlr/v4/codegen/model/chunk/NonLocalAttrRef.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/tool/src/org/antlr/v4/codegen/model/chunk/QRetValueRef.java b/tool/src/org/antlr/v4/codegen/model/chunk/QRetValueRef.java index 245e77a..c31618a 100644 --- a/tool/src/org/antlr/v4/codegen/model/chunk/QRetValueRef.java +++ b/tool/src/org/antlr/v4/codegen/model/chunk/QRetValueRef.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/tool/src/org/antlr/v4/codegen/model/chunk/RetValueRef.java b/tool/src/org/antlr/v4/codegen/model/chunk/RetValueRef.java index 07dfe64..ee67a15 100644 --- a/tool/src/org/antlr/v4/codegen/model/chunk/RetValueRef.java +++ b/tool/src/org/antlr/v4/codegen/model/chunk/RetValueRef.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/tool/src/org/antlr/v4/codegen/model/chunk/RulePropertyRef.java b/tool/src/org/antlr/v4/codegen/model/chunk/RulePropertyRef.java index ded613f..430917a 100644 --- a/tool/src/org/antlr/v4/codegen/model/chunk/RulePropertyRef.java +++ b/tool/src/org/antlr/v4/codegen/model/chunk/RulePropertyRef.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/tool/src/org/antlr/v4/codegen/model/chunk/RulePropertyRef_ctx.java b/tool/src/org/antlr/v4/codegen/model/chunk/RulePropertyRef_ctx.java index 26301a0..aaec94f 100644 --- a/tool/src/org/antlr/v4/codegen/model/chunk/RulePropertyRef_ctx.java +++ b/tool/src/org/antlr/v4/codegen/model/chunk/RulePropertyRef_ctx.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/tool/src/org/antlr/v4/codegen/model/chunk/RulePropertyRef_parser.java b/tool/src/org/antlr/v4/codegen/model/chunk/RulePropertyRef_parser.java index 16646b9..39b6690 100644 --- a/tool/src/org/antlr/v4/codegen/model/chunk/RulePropertyRef_parser.java +++ b/tool/src/org/antlr/v4/codegen/model/chunk/RulePropertyRef_parser.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/tool/src/org/antlr/v4/codegen/model/chunk/RulePropertyRef_start.java b/tool/src/org/antlr/v4/codegen/model/chunk/RulePropertyRef_start.java index ef485e5..8a0977b 100644 --- a/tool/src/org/antlr/v4/codegen/model/chunk/RulePropertyRef_start.java +++ b/tool/src/org/antlr/v4/codegen/model/chunk/RulePropertyRef_start.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/tool/src/org/antlr/v4/codegen/model/chunk/RulePropertyRef_stop.java b/tool/src/org/antlr/v4/codegen/model/chunk/RulePropertyRef_stop.java index d649967..f213ab2 100644 --- a/tool/src/org/antlr/v4/codegen/model/chunk/RulePropertyRef_stop.java +++ b/tool/src/org/antlr/v4/codegen/model/chunk/RulePropertyRef_stop.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/tool/src/org/antlr/v4/codegen/model/chunk/RulePropertyRef_text.java b/tool/src/org/antlr/v4/codegen/model/chunk/RulePropertyRef_text.java index f3d0a08..6ef2313 100644 --- a/tool/src/org/antlr/v4/codegen/model/chunk/RulePropertyRef_text.java +++ b/tool/src/org/antlr/v4/codegen/model/chunk/RulePropertyRef_text.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/tool/src/org/antlr/v4/codegen/model/chunk/SetAttr.java b/tool/src/org/antlr/v4/codegen/model/chunk/SetAttr.java index 6c5755c..dac369b 100644 --- a/tool/src/org/antlr/v4/codegen/model/chunk/SetAttr.java +++ b/tool/src/org/antlr/v4/codegen/model/chunk/SetAttr.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/tool/src/org/antlr/v4/codegen/model/chunk/SetNonLocalAttr.java b/tool/src/org/antlr/v4/codegen/model/chunk/SetNonLocalAttr.java index c2e7924..0345c29 100644 --- a/tool/src/org/antlr/v4/codegen/model/chunk/SetNonLocalAttr.java +++ b/tool/src/org/antlr/v4/codegen/model/chunk/SetNonLocalAttr.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/tool/src/org/antlr/v4/codegen/model/chunk/ThisRulePropertyRef_ctx.java b/tool/src/org/antlr/v4/codegen/model/chunk/ThisRulePropertyRef_ctx.java index 0293664..b14b452 100644 --- a/tool/src/org/antlr/v4/codegen/model/chunk/ThisRulePropertyRef_ctx.java +++ b/tool/src/org/antlr/v4/codegen/model/chunk/ThisRulePropertyRef_ctx.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/tool/src/org/antlr/v4/codegen/model/chunk/ThisRulePropertyRef_parser.java b/tool/src/org/antlr/v4/codegen/model/chunk/ThisRulePropertyRef_parser.java index 22fc377..f1d6541 100644 --- a/tool/src/org/antlr/v4/codegen/model/chunk/ThisRulePropertyRef_parser.java +++ b/tool/src/org/antlr/v4/codegen/model/chunk/ThisRulePropertyRef_parser.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/tool/src/org/antlr/v4/codegen/model/chunk/ThisRulePropertyRef_start.java b/tool/src/org/antlr/v4/codegen/model/chunk/ThisRulePropertyRef_start.java index c4cafec..bfb483c 100644 --- a/tool/src/org/antlr/v4/codegen/model/chunk/ThisRulePropertyRef_start.java +++ b/tool/src/org/antlr/v4/codegen/model/chunk/ThisRulePropertyRef_start.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/tool/src/org/antlr/v4/codegen/model/chunk/ThisRulePropertyRef_stop.java b/tool/src/org/antlr/v4/codegen/model/chunk/ThisRulePropertyRef_stop.java index ea51694..fa1f7c6 100644 --- a/tool/src/org/antlr/v4/codegen/model/chunk/ThisRulePropertyRef_stop.java +++ b/tool/src/org/antlr/v4/codegen/model/chunk/ThisRulePropertyRef_stop.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/tool/src/org/antlr/v4/codegen/model/chunk/ThisRulePropertyRef_text.java b/tool/src/org/antlr/v4/codegen/model/chunk/ThisRulePropertyRef_text.java index 696066a..504b1d4 100644 --- a/tool/src/org/antlr/v4/codegen/model/chunk/ThisRulePropertyRef_text.java +++ b/tool/src/org/antlr/v4/codegen/model/chunk/ThisRulePropertyRef_text.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/tool/src/org/antlr/v4/codegen/model/chunk/TokenPropertyRef.java b/tool/src/org/antlr/v4/codegen/model/chunk/TokenPropertyRef.java index 82c2f69..4a6433d 100644 --- a/tool/src/org/antlr/v4/codegen/model/chunk/TokenPropertyRef.java +++ b/tool/src/org/antlr/v4/codegen/model/chunk/TokenPropertyRef.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/tool/src/org/antlr/v4/codegen/model/chunk/TokenPropertyRef_channel.java b/tool/src/org/antlr/v4/codegen/model/chunk/TokenPropertyRef_channel.java index 1404bf9..b69e096 100644 --- a/tool/src/org/antlr/v4/codegen/model/chunk/TokenPropertyRef_channel.java +++ b/tool/src/org/antlr/v4/codegen/model/chunk/TokenPropertyRef_channel.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/tool/src/org/antlr/v4/codegen/model/chunk/TokenPropertyRef_index.java b/tool/src/org/antlr/v4/codegen/model/chunk/TokenPropertyRef_index.java index bb92a20..b92ea9f 100644 --- a/tool/src/org/antlr/v4/codegen/model/chunk/TokenPropertyRef_index.java +++ b/tool/src/org/antlr/v4/codegen/model/chunk/TokenPropertyRef_index.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/tool/src/org/antlr/v4/codegen/model/chunk/TokenPropertyRef_int.java b/tool/src/org/antlr/v4/codegen/model/chunk/TokenPropertyRef_int.java index b02053f..24b6e22 100644 --- a/tool/src/org/antlr/v4/codegen/model/chunk/TokenPropertyRef_int.java +++ b/tool/src/org/antlr/v4/codegen/model/chunk/TokenPropertyRef_int.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/tool/src/org/antlr/v4/codegen/model/chunk/TokenPropertyRef_line.java b/tool/src/org/antlr/v4/codegen/model/chunk/TokenPropertyRef_line.java index 0229204..f8dd162 100644 --- a/tool/src/org/antlr/v4/codegen/model/chunk/TokenPropertyRef_line.java +++ b/tool/src/org/antlr/v4/codegen/model/chunk/TokenPropertyRef_line.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/tool/src/org/antlr/v4/codegen/model/chunk/TokenPropertyRef_pos.java b/tool/src/org/antlr/v4/codegen/model/chunk/TokenPropertyRef_pos.java index 7badb92..f7e964f 100644 --- a/tool/src/org/antlr/v4/codegen/model/chunk/TokenPropertyRef_pos.java +++ b/tool/src/org/antlr/v4/codegen/model/chunk/TokenPropertyRef_pos.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/tool/src/org/antlr/v4/codegen/model/chunk/TokenPropertyRef_text.java b/tool/src/org/antlr/v4/codegen/model/chunk/TokenPropertyRef_text.java index d049c6e..05bfbab 100644 --- a/tool/src/org/antlr/v4/codegen/model/chunk/TokenPropertyRef_text.java +++ b/tool/src/org/antlr/v4/codegen/model/chunk/TokenPropertyRef_text.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/tool/src/org/antlr/v4/codegen/model/chunk/TokenPropertyRef_type.java b/tool/src/org/antlr/v4/codegen/model/chunk/TokenPropertyRef_type.java index a8b4222..747b676 100644 --- a/tool/src/org/antlr/v4/codegen/model/chunk/TokenPropertyRef_type.java +++ b/tool/src/org/antlr/v4/codegen/model/chunk/TokenPropertyRef_type.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/tool/src/org/antlr/v4/codegen/model/chunk/TokenRef.java b/tool/src/org/antlr/v4/codegen/model/chunk/TokenRef.java index d39115d..b2b6200 100644 --- a/tool/src/org/antlr/v4/codegen/model/chunk/TokenRef.java +++ b/tool/src/org/antlr/v4/codegen/model/chunk/TokenRef.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/tool/src/org/antlr/v4/codegen/model/dbg.java b/tool/src/org/antlr/v4/codegen/model/dbg.java index 0aaeebf..2f6aa07 100644 --- a/tool/src/org/antlr/v4/codegen/model/dbg.java +++ b/tool/src/org/antlr/v4/codegen/model/dbg.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/tool/src/org/antlr/v4/codegen/model/decl/AltLabelStructDecl.java b/tool/src/org/antlr/v4/codegen/model/decl/AltLabelStructDecl.java index b855b28..a0a8cc8 100644 --- a/tool/src/org/antlr/v4/codegen/model/decl/AltLabelStructDecl.java +++ b/tool/src/org/antlr/v4/codegen/model/decl/AltLabelStructDecl.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/tool/src/org/antlr/v4/codegen/model/decl/AttributeDecl.java b/tool/src/org/antlr/v4/codegen/model/decl/AttributeDecl.java index 80af232..0436752 100644 --- a/tool/src/org/antlr/v4/codegen/model/decl/AttributeDecl.java +++ b/tool/src/org/antlr/v4/codegen/model/decl/AttributeDecl.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/tool/src/org/antlr/v4/codegen/model/decl/CodeBlock.java b/tool/src/org/antlr/v4/codegen/model/decl/CodeBlock.java index 89e1f20..2e608a1 100644 --- a/tool/src/org/antlr/v4/codegen/model/decl/CodeBlock.java +++ b/tool/src/org/antlr/v4/codegen/model/decl/CodeBlock.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/tool/src/org/antlr/v4/codegen/model/decl/ContextGetterDecl.java b/tool/src/org/antlr/v4/codegen/model/decl/ContextGetterDecl.java index fc4b262..b708f8a 100644 --- a/tool/src/org/antlr/v4/codegen/model/decl/ContextGetterDecl.java +++ b/tool/src/org/antlr/v4/codegen/model/decl/ContextGetterDecl.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/tool/src/org/antlr/v4/codegen/model/decl/ContextRuleGetterDecl.java b/tool/src/org/antlr/v4/codegen/model/decl/ContextRuleGetterDecl.java index b763965..322054d 100644 --- a/tool/src/org/antlr/v4/codegen/model/decl/ContextRuleGetterDecl.java +++ b/tool/src/org/antlr/v4/codegen/model/decl/ContextRuleGetterDecl.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ @@ -11,8 +11,11 @@ import org.antlr.v4.codegen.OutputModelFactory; /** {@code public XContext X() { }} */ public class ContextRuleGetterDecl extends ContextGetterDecl { public String ctxName; - public ContextRuleGetterDecl(OutputModelFactory factory, String name, String ctxName) { + public boolean optional; + + public ContextRuleGetterDecl(OutputModelFactory factory, String name, String ctxName, boolean optional) { super(factory, name); this.ctxName = ctxName; + this.optional = optional; } } diff --git a/tool/src/org/antlr/v4/codegen/model/decl/ContextRuleListGetterDecl.java b/tool/src/org/antlr/v4/codegen/model/decl/ContextRuleListGetterDecl.java index 4daf182..43e5d3c 100644 --- a/tool/src/org/antlr/v4/codegen/model/decl/ContextRuleListGetterDecl.java +++ b/tool/src/org/antlr/v4/codegen/model/decl/ContextRuleListGetterDecl.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/tool/src/org/antlr/v4/codegen/model/decl/ContextRuleListIndexedGetterDecl.java b/tool/src/org/antlr/v4/codegen/model/decl/ContextRuleListIndexedGetterDecl.java index 175394f..4399c4c 100644 --- a/tool/src/org/antlr/v4/codegen/model/decl/ContextRuleListIndexedGetterDecl.java +++ b/tool/src/org/antlr/v4/codegen/model/decl/ContextRuleListIndexedGetterDecl.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/tool/src/org/antlr/v4/codegen/model/decl/ContextTokenGetterDecl.java b/tool/src/org/antlr/v4/codegen/model/decl/ContextTokenGetterDecl.java index d663181..2ca6478 100644 --- a/tool/src/org/antlr/v4/codegen/model/decl/ContextTokenGetterDecl.java +++ b/tool/src/org/antlr/v4/codegen/model/decl/ContextTokenGetterDecl.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ @@ -10,7 +10,10 @@ import org.antlr.v4.codegen.OutputModelFactory; /** {@code public Token X() { }} */ public class ContextTokenGetterDecl extends ContextGetterDecl { - public ContextTokenGetterDecl(OutputModelFactory factory, String name) { + public boolean optional; + + public ContextTokenGetterDecl(OutputModelFactory factory, String name, boolean optional) { super(factory, name); + this.optional = optional; } } diff --git a/tool/src/org/antlr/v4/codegen/model/decl/ContextTokenListGetterDecl.java b/tool/src/org/antlr/v4/codegen/model/decl/ContextTokenListGetterDecl.java index 7c8e2ff..0005d4b 100644 --- a/tool/src/org/antlr/v4/codegen/model/decl/ContextTokenListGetterDecl.java +++ b/tool/src/org/antlr/v4/codegen/model/decl/ContextTokenListGetterDecl.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/tool/src/org/antlr/v4/codegen/model/decl/ContextTokenListIndexedGetterDecl.java b/tool/src/org/antlr/v4/codegen/model/decl/ContextTokenListIndexedGetterDecl.java index bc722c9..4db542a 100644 --- a/tool/src/org/antlr/v4/codegen/model/decl/ContextTokenListIndexedGetterDecl.java +++ b/tool/src/org/antlr/v4/codegen/model/decl/ContextTokenListIndexedGetterDecl.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/tool/src/org/antlr/v4/codegen/model/decl/Decl.java b/tool/src/org/antlr/v4/codegen/model/decl/Decl.java index 2cbcf9c..b2b9742 100644 --- a/tool/src/org/antlr/v4/codegen/model/decl/Decl.java +++ b/tool/src/org/antlr/v4/codegen/model/decl/Decl.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/tool/src/org/antlr/v4/codegen/model/decl/ElementListDecl.java b/tool/src/org/antlr/v4/codegen/model/decl/ElementListDecl.java index f5fa1bb..e89db11 100644 --- a/tool/src/org/antlr/v4/codegen/model/decl/ElementListDecl.java +++ b/tool/src/org/antlr/v4/codegen/model/decl/ElementListDecl.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/tool/src/org/antlr/v4/codegen/model/decl/RuleContextDecl.java b/tool/src/org/antlr/v4/codegen/model/decl/RuleContextDecl.java index a6c7eff..2169618 100644 --- a/tool/src/org/antlr/v4/codegen/model/decl/RuleContextDecl.java +++ b/tool/src/org/antlr/v4/codegen/model/decl/RuleContextDecl.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/tool/src/org/antlr/v4/codegen/model/decl/RuleContextListDecl.java b/tool/src/org/antlr/v4/codegen/model/decl/RuleContextListDecl.java index 8418dbf..decf0f6 100644 --- a/tool/src/org/antlr/v4/codegen/model/decl/RuleContextListDecl.java +++ b/tool/src/org/antlr/v4/codegen/model/decl/RuleContextListDecl.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/tool/src/org/antlr/v4/codegen/model/decl/StructDecl.java b/tool/src/org/antlr/v4/codegen/model/decl/StructDecl.java index 14a764c..f7a48ee 100644 --- a/tool/src/org/antlr/v4/codegen/model/decl/StructDecl.java +++ b/tool/src/org/antlr/v4/codegen/model/decl/StructDecl.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/tool/src/org/antlr/v4/codegen/model/decl/TokenDecl.java b/tool/src/org/antlr/v4/codegen/model/decl/TokenDecl.java index 6545125..b42e9c7 100644 --- a/tool/src/org/antlr/v4/codegen/model/decl/TokenDecl.java +++ b/tool/src/org/antlr/v4/codegen/model/decl/TokenDecl.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/tool/src/org/antlr/v4/codegen/model/decl/TokenListDecl.java b/tool/src/org/antlr/v4/codegen/model/decl/TokenListDecl.java index ecbfac7..beb334d 100644 --- a/tool/src/org/antlr/v4/codegen/model/decl/TokenListDecl.java +++ b/tool/src/org/antlr/v4/codegen/model/decl/TokenListDecl.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/tool/src/org/antlr/v4/codegen/model/decl/TokenTypeDecl.java b/tool/src/org/antlr/v4/codegen/model/decl/TokenTypeDecl.java index c13ce63..da9e037 100644 --- a/tool/src/org/antlr/v4/codegen/model/decl/TokenTypeDecl.java +++ b/tool/src/org/antlr/v4/codegen/model/decl/TokenTypeDecl.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/tool/src/org/antlr/v4/codegen/target/CSharpTarget.java b/tool/src/org/antlr/v4/codegen/target/CSharpTarget.java index 88d1438..e7a2322 100644 --- a/tool/src/org/antlr/v4/codegen/target/CSharpTarget.java +++ b/tool/src/org/antlr/v4/codegen/target/CSharpTarget.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ @@ -7,6 +7,7 @@ package org.antlr.v4.codegen.target; import org.antlr.v4.codegen.CodeGenerator; import org.antlr.v4.codegen.Target; +import org.antlr.v4.codegen.UnicodeEscapes; import org.antlr.v4.tool.ErrorType; import org.antlr.v4.tool.ast.GrammarAST; import org.stringtemplate.v4.NumberRenderer; @@ -27,7 +28,7 @@ public class CSharpTarget extends Target { @Override public String getVersion() { - return "4.6"; + return "4.7.1"; } @Override @@ -36,78 +37,18 @@ public class CSharpTarget extends Target { throw new IllegalArgumentException(String.format("Cannot encode the specified value: %d", v)); } + String formatted; if (v >= 0 && v < targetCharValueEscape.length && targetCharValueEscape[v] != null) { - return targetCharValueEscape[v]; + formatted = targetCharValueEscape[v]; } - - if (v >= 0x20 && v < 127 && (v < '0' || v > '9') && (v < 'a' || v > 'f') && (v < 'A' || v > 'F')) { - return String.valueOf((char)v); + else if (v >= 0x20 && v < 127 && (v < '0' || v > '9') && (v < 'a' || v > 'f') && (v < 'A' || v > 'F')) { + formatted = Character.toString((char)v); } - - return String.format("\\x%X", v & 0xFFFF); - } - - @Override - public String getTargetStringLiteralFromANTLRStringLiteral( - CodeGenerator generator, - String literal, boolean addQuotes) - { - StringBuilder sb = new StringBuilder(); - String is = literal; - - if ( addQuotes ) sb.append('"'); - - for (int i = 1; i < is.length() -1; i++) { - if (is.charAt(i) == '\\') { - // Anything escaped is what it is! We assume that - // people know how to escape characters correctly. However - // we catch anything that does not need an escape in Java (which - // is what the default implementation is dealing with and remove - // the escape. The C target does this for instance. - // - switch (is.charAt(i+1)) { - // Pass through any escapes that Java also needs - // - case '"': - case 'n': - case 'r': - case 't': - case 'b': - case 'f': - case '\\': - // Pass the escape through - sb.append('\\'); - break; - - case 'u': // Assume unnnn - // Pass the escape through as double \\ - // so that Java leaves as \u0000 string not char - sb.append('\\'); - sb.append('\\'); - break; - - default: - // Remove the escape by virtue of not adding it here - // Thus \' becomes ' and so on - break; - } - - // Go past the \ character - i++; - } else { - // Characters that don't need \ in ANTLR 'strings' but do in Java - if (is.charAt(i) == '"') { - // We need to escape " in Java - sb.append('\\'); - } - } - // Add in the next character, which may have been escaped - sb.append(is.charAt(i)); + else { + formatted = String.format("\\x%X", v & 0xFFFF); } - if ( addQuotes ) sb.append('"'); - - return sb.toString(); + return "'" + formatted + "'"; } @Override @@ -150,4 +91,9 @@ public class CSharpTarget extends Target { return result; } + @Override + protected void appendUnicodeEscapedCodePoint(int codePoint, StringBuilder sb) { + // C# and Python share the same escaping style. + UnicodeEscapes.appendPythonStyleEscapedCodePoint(codePoint, sb); + } } diff --git a/tool/src/org/antlr/v4/codegen/target/CppTarget.java b/tool/src/org/antlr/v4/codegen/target/CppTarget.java index 14ac858..6ef32c2 100644 --- a/tool/src/org/antlr/v4/codegen/target/CppTarget.java +++ b/tool/src/org/antlr/v4/codegen/target/CppTarget.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ @@ -8,6 +8,7 @@ package org.antlr.v4.codegen.target; import org.antlr.v4.codegen.CodeGenerator; import org.antlr.v4.codegen.Target; +import org.antlr.v4.codegen.UnicodeEscapes; import org.antlr.v4.tool.ErrorType; import org.antlr.v4.tool.ast.GrammarAST; import org.stringtemplate.v4.NumberRenderer; @@ -49,7 +50,7 @@ public class CppTarget extends Target { } public String getVersion() { - return "4.6"; + return "4.7.1"; } public boolean needsHeader() { return true; } @@ -68,81 +69,6 @@ public class CppTarget extends Target { badWords.add("parserRule"); } - /** - * {@inheritDoc} - * <p/> - * For C++, this is the translation {@code 'a\n"'} → {@code "a\n\""}. - * Expect single quotes around the incoming literal. Just flip the quotes - * and replace double quotes with {@code \"}. - * <p/> - * Note that we have decided to allow people to use '\"' without penalty, so - * we must build the target string in a loop as {@link String#replace} - * cannot handle both {@code \"} and {@code "} without a lot of messing - * around. - */ - @Override - public String getTargetStringLiteralFromANTLRStringLiteral( - CodeGenerator generator, - String literal, boolean addQuotes) - { - StringBuilder sb = new StringBuilder(); - String is = literal; - - if ( addQuotes ) sb.append('"'); - - for (int i = 1; i < is.length() -1; i++) { - if (is.charAt(i) == '\\') { - // Anything escaped is what it is! We assume that - // people know how to escape characters correctly. However - // we catch anything that does not need an escape in Java (which - // is what the default implementation is dealing with and remove - // the escape. The C target does this for instance. - // - switch (is.charAt(i+1)) { - // Pass through any escapes that Java also needs - // - case '"': - case 'n': - case 'r': - case 't': - case 'b': - case 'f': - case '\\': - // Pass the escape through - sb.append('\\'); - break; - - case 'u': // Assume unnnn - // Pass the escape through as double \\ - // so that Java leaves as \u0000 string not char - sb.append('\\'); - sb.append('\\'); - break; - - default: - // Remove the escape by virtue of not adding it here - // Thus \' becomes ' and so on - break; - } - - // Go past the \ character - i++; - } else { - // Characters that don't need \ in ANTLR 'strings' but do in Java - if (is.charAt(i) == '"') { - // We need to escape " in Java - sb.append('\\'); - } - } - // Add in the next character, which may have been escaped - sb.append(is.charAt(i)); - } - - if ( addQuotes ) sb.append('"'); - - return sb.toString(); - } - @Override public String encodeIntAsCharEscape(int v) { return "0x" + Integer.toHexString(v) + ", "; @@ -232,4 +158,10 @@ public class CppTarget extends Target { return result; } + + @Override + protected void appendUnicodeEscapedCodePoint(int codePoint, StringBuilder sb) { + // C99 and Python share the same escaping style. + UnicodeEscapes.appendPythonStyleEscapedCodePoint(codePoint, sb); + } } diff --git a/tool/src/org/antlr/v4/codegen/target/GoTarget.java b/tool/src/org/antlr/v4/codegen/target/GoTarget.java index 678b532..be4e3cc 100644 --- a/tool/src/org/antlr/v4/codegen/target/GoTarget.java +++ b/tool/src/org/antlr/v4/codegen/target/GoTarget.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ @@ -8,6 +8,7 @@ package org.antlr.v4.codegen.target; import org.antlr.v4.codegen.CodeGenerator; import org.antlr.v4.codegen.Target; +import org.antlr.v4.codegen.UnicodeEscapes; import org.antlr.v4.parse.ANTLRParser; import org.antlr.v4.tool.Grammar; import org.antlr.v4.tool.ast.GrammarAST; @@ -48,8 +49,18 @@ public class GoTarget extends Target { "make", "new", "panic", "print", "println", "real", "recover" }; + // interface definition of RuleContext from runtime/Go/antlr/rule_context.go + private static final String[] goRuleContextInterfaceMethods = { + "Accept", "GetAltNumber", "GetBaseRuleContext", "GetChild", "GetChildCount", + "GetChildren", "GetInvokingState", "GetParent", "GetPayload", "GetRuleContext", + "GetRuleIndex", "GetSourceInterval", "GetText", "IsEmpty", "SetAltNumber", + "SetInvokingState", "SetParent", "String" + }; + /** Avoid grammar symbols in this set to prevent conflicts in gen'd code. */ - private final Set<String> badWords = new HashSet<String>(goKeywords.length + goPredeclaredIdentifiers.length + 2); + private final Set<String> badWords = new HashSet<String>( + goKeywords.length + goPredeclaredIdentifiers.length + goRuleContextInterfaceMethods.length + 3 + ); private static final boolean DO_GOFMT = !Boolean.parseBoolean(System.getenv("ANTLR_GO_DISABLE_GOFMT")) && !Boolean.parseBoolean(System.getProperty("antlr.go.disable-gofmt")); @@ -58,12 +69,12 @@ public class GoTarget extends Target { super(gen, "Go"); } - @Override - public String getVersion() { - return "4.6"; + @Override + public String getVersion() { + return "4.7.1"; } - public Set<String> getBadWords() { + public Set<String> getBadWords() { if (badWords.isEmpty()) { addBadWords(); } @@ -74,8 +85,10 @@ public class GoTarget extends Target { protected void addBadWords() { badWords.addAll(Arrays.asList(goKeywords)); badWords.addAll(Arrays.asList(goPredeclaredIdentifiers)); + badWords.addAll(Arrays.asList(goRuleContextInterfaceMethods)); badWords.add("rule"); badWords.add("parserRule"); + badWords.add("action"); } @Override @@ -214,5 +227,10 @@ public class GoTarget extends Target { } } -} + @Override + protected void appendUnicodeEscapedCodePoint(int codePoint, StringBuilder sb) { + // Go and Python share the same escaping style. + UnicodeEscapes.appendPythonStyleEscapedCodePoint(codePoint, sb); + } +} diff --git a/tool/src/org/antlr/v4/codegen/target/JavaScriptTarget.java b/tool/src/org/antlr/v4/codegen/target/JavaScriptTarget.java index 455c488..adbfc7c 100644 --- a/tool/src/org/antlr/v4/codegen/target/JavaScriptTarget.java +++ b/tool/src/org/antlr/v4/codegen/target/JavaScriptTarget.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ @@ -8,6 +8,7 @@ package org.antlr.v4.codegen.target; import org.antlr.v4.codegen.CodeGenerator; import org.antlr.v4.codegen.Target; +import org.antlr.v4.codegen.UnicodeEscapes; import org.antlr.v4.tool.ast.GrammarAST; import org.stringtemplate.v4.STGroup; import org.stringtemplate.v4.StringRenderer; @@ -50,7 +51,7 @@ public class JavaScriptTarget extends Target { @Override public String getVersion() { - return "4.6"; + return "4.7.1"; } public Set<String> getBadWords() { @@ -67,81 +68,6 @@ public class JavaScriptTarget extends Target { badWords.add("parserRule"); } - /** - * {@inheritDoc} - * <p> - * For Java, this is the translation {@code 'a\n"'} → {@code "a\n\""}. - * Expect single quotes around the incoming literal. Just flip the quotes - * and replace double quotes with {@code \"}. - * <p> - * Note that we have decided to allow people to use '\"' without penalty, so - * we must build the target string in a loop as {@link String#replace} - * cannot handle both {@code \"} and {@code "} without a lot of messing - * around. - */ - @Override - public String getTargetStringLiteralFromANTLRStringLiteral( - CodeGenerator generator, - String literal, boolean addQuotes) - { - StringBuilder sb = new StringBuilder(); - String is = literal; - - if ( addQuotes ) sb.append('"'); - - for (int i = 1; i < is.length() -1; i++) { - if (is.charAt(i) == '\\') { - // Anything escaped is what it is! We assume that - // people know how to escape characters correctly. However - // we catch anything that does not need an escape in Java (which - // is what the default implementation is dealing with and remove - // the escape. The C target does this for instance. - // - switch (is.charAt(i+1)) { - // Pass through any escapes that Java also needs - // - case '"': - case 'n': - case 'r': - case 't': - case 'b': - case 'f': - case '\\': - // Pass the escape through - sb.append('\\'); - break; - - case 'u': // Assume unnnn - // Pass the escape through as double \\ - // so that Java leaves as \u0000 string not char - sb.append('\\'); - sb.append('\\'); - break; - - default: - // Remove the escape by virtue of not adding it here - // Thus \' becomes ' and so on - break; - } - - // Go past the \ character - i++; - } else { - // Characters that don't need \ in ANTLR 'strings' but do in Java - if (is.charAt(i) == '"') { - // We need to escape " in Java - sb.append('\\'); - } - } - // Add in the next character, which may have been escaped - sb.append(is.charAt(i)); - } - - if ( addQuotes ) sb.append('"'); - - return sb.toString(); - } - @Override public String encodeIntAsCharEscape(int v) { if (v < Character.MIN_VALUE || v > Character.MAX_VALUE) { @@ -210,4 +136,10 @@ public class JavaScriptTarget extends Target { public boolean supportsOverloadedMethods() { return false; } + + @Override + protected void appendUnicodeEscapedCodePoint(int codePoint, StringBuilder sb) { + // JavaScript and Java share the same escaping style. + UnicodeEscapes.appendJavaStyleEscapedCodePoint(codePoint, sb); + } } diff --git a/tool/src/org/antlr/v4/codegen/target/JavaTarget.java b/tool/src/org/antlr/v4/codegen/target/JavaTarget.java index a7cb900..2482fef 100644 --- a/tool/src/org/antlr/v4/codegen/target/JavaTarget.java +++ b/tool/src/org/antlr/v4/codegen/target/JavaTarget.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ @@ -9,6 +9,7 @@ package org.antlr.v4.codegen.target; import org.antlr.v4.Tool; import org.antlr.v4.codegen.CodeGenerator; import org.antlr.v4.codegen.Target; +import org.antlr.v4.codegen.UnicodeEscapes; import org.antlr.v4.tool.ast.GrammarAST; import org.stringtemplate.v4.STGroup; import org.stringtemplate.v4.StringRenderer; @@ -99,4 +100,9 @@ public class JavaTarget extends Target { } } + + @Override + protected void appendUnicodeEscapedCodePoint(int codePoint, StringBuilder sb) { + UnicodeEscapes.appendJavaStyleEscapedCodePoint(codePoint, sb); + } } diff --git a/tool/src/org/antlr/v4/codegen/target/Python2Target.java b/tool/src/org/antlr/v4/codegen/target/Python2Target.java index 85cdb9c..3fc6a35 100644 --- a/tool/src/org/antlr/v4/codegen/target/Python2Target.java +++ b/tool/src/org/antlr/v4/codegen/target/Python2Target.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ @@ -8,6 +8,7 @@ package org.antlr.v4.codegen.target; import org.antlr.v4.codegen.CodeGenerator; import org.antlr.v4.codegen.Target; +import org.antlr.v4.codegen.UnicodeEscapes; import org.antlr.v4.tool.ast.GrammarAST; import org.stringtemplate.v4.STGroup; import org.stringtemplate.v4.StringRenderer; @@ -93,7 +94,7 @@ public class Python2Target extends Target { @Override public String getVersion() { - return "4.6"; + return "4.7.1"; } public Set<String> getBadWords() { @@ -109,4 +110,9 @@ public class Python2Target extends Target { badWords.add("rule"); badWords.add("parserRule"); } + + @Override + protected void appendUnicodeEscapedCodePoint(int codePoint, StringBuilder sb) { + UnicodeEscapes.appendPythonStyleEscapedCodePoint(codePoint, sb); + } } diff --git a/tool/src/org/antlr/v4/codegen/target/Python3Target.java b/tool/src/org/antlr/v4/codegen/target/Python3Target.java index e151ea0..388269a 100644 --- a/tool/src/org/antlr/v4/codegen/target/Python3Target.java +++ b/tool/src/org/antlr/v4/codegen/target/Python3Target.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ @@ -8,6 +8,7 @@ package org.antlr.v4.codegen.target; import org.antlr.v4.codegen.CodeGenerator; import org.antlr.v4.codegen.Target; +import org.antlr.v4.codegen.UnicodeEscapes; import org.antlr.v4.tool.ast.GrammarAST; import org.stringtemplate.v4.STGroup; import org.stringtemplate.v4.StringRenderer; @@ -55,7 +56,7 @@ public class Python3Target extends Target { @Override public int getSerializedATNSegmentLimit() { // set to something stupid to avoid segmentation - return 2 ^ 31; + return Integer.MAX_VALUE; } @Override @@ -95,7 +96,7 @@ public class Python3Target extends Target { @Override public String getVersion() { - return "4.6"; + return "4.7.1"; } /** Avoid grammar symbols in this set to prevent conflicts in gen'd code. */ @@ -115,5 +116,8 @@ public class Python3Target extends Target { badWords.add("parserRule"); } - + @Override + protected void appendUnicodeEscapedCodePoint(int codePoint, StringBuilder sb) { + UnicodeEscapes.appendPythonStyleEscapedCodePoint(codePoint, sb); + } } diff --git a/tool/src/org/antlr/v4/codegen/target/SwiftTarget.java b/tool/src/org/antlr/v4/codegen/target/SwiftTarget.java index b4db954..5e28506 100644 --- a/tool/src/org/antlr/v4/codegen/target/SwiftTarget.java +++ b/tool/src/org/antlr/v4/codegen/target/SwiftTarget.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ @@ -8,6 +8,7 @@ package org.antlr.v4.codegen.target; import org.antlr.v4.codegen.CodeGenerator; import org.antlr.v4.codegen.Target; +import org.antlr.v4.codegen.UnicodeEscapes; import org.antlr.v4.runtime.Token; import org.antlr.v4.runtime.atn.ATN; import org.antlr.v4.runtime.atn.ATNDeserializer; @@ -86,7 +87,7 @@ public class SwiftTarget extends Target { @Override public String getVersion() { - return "4.6"; // Java and tool versions move in lock step + return "4.7.1"; // Java and tool versions move in lock step } public Set<String> getBadWords() { @@ -123,7 +124,8 @@ public class SwiftTarget extends Target { if (g.isLexer() && lexerAtnJSON == null) { lexerAtnJSON = getLexerOrParserATNJson(g, fileName); - } else if (!g.isLexer() && parserAtnJSON == null && g.atn != null) { + } + else if (!g.isLexer() && parserAtnJSON == null && g.atn != null) { parserAtnJSON = getLexerOrParserATNJson(g, fileName); } @@ -550,4 +552,9 @@ public class SwiftTarget extends Target { } } + + @Override + protected void appendUnicodeEscapedCodePoint(int codePoint, StringBuilder sb) { + UnicodeEscapes.appendSwiftStyleEscapedCodePoint(codePoint, sb); + } } diff --git a/tool/src/org/antlr/v4/gui/BasicFontMetrics.java b/tool/src/org/antlr/v4/gui/BasicFontMetrics.java index 39ce6bd..f145e85 100644 --- a/tool/src/org/antlr/v4/gui/BasicFontMetrics.java +++ b/tool/src/org/antlr/v4/gui/BasicFontMetrics.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/tool/src/org/antlr/v4/gui/GraphicsSupport.java b/tool/src/org/antlr/v4/gui/GraphicsSupport.java index fbee7f4..74fa215 100644 --- a/tool/src/org/antlr/v4/gui/GraphicsSupport.java +++ b/tool/src/org/antlr/v4/gui/GraphicsSupport.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ @@ -93,7 +93,8 @@ public class GraphicsSupport { job.print(doc, attributes); out.close(); } - } else { + } + else { // parrt: works with [image/jpeg, image/png, image/x-png, image/vnd.wap.wbmp, image/bmp, image/gif] Rectangle rect = comp.getBounds(); BufferedImage image = new BufferedImage(rect.width, rect.height, diff --git a/tool/src/org/antlr/v4/gui/JFileChooserConfirmOverwrite.java b/tool/src/org/antlr/v4/gui/JFileChooserConfirmOverwrite.java index f30b555..e6788e4 100644 --- a/tool/src/org/antlr/v4/gui/JFileChooserConfirmOverwrite.java +++ b/tool/src/org/antlr/v4/gui/JFileChooserConfirmOverwrite.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/tool/src/org/antlr/v4/gui/PostScriptDocument.java b/tool/src/org/antlr/v4/gui/PostScriptDocument.java index 62e2216..e3f9c22 100644 --- a/tool/src/org/antlr/v4/gui/PostScriptDocument.java +++ b/tool/src/org/antlr/v4/gui/PostScriptDocument.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/tool/src/org/antlr/v4/gui/SystemFontMetrics.java b/tool/src/org/antlr/v4/gui/SystemFontMetrics.java index f1b4d91..ff40f1a 100644 --- a/tool/src/org/antlr/v4/gui/SystemFontMetrics.java +++ b/tool/src/org/antlr/v4/gui/SystemFontMetrics.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/tool/src/org/antlr/v4/gui/TestRig.java b/tool/src/org/antlr/v4/gui/TestRig.java index 1fc96f0..a466363 100644 --- a/tool/src/org/antlr/v4/gui/TestRig.java +++ b/tool/src/org/antlr/v4/gui/TestRig.java @@ -1,13 +1,13 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ package org.antlr.v4.gui; -import org.antlr.v4.runtime.ANTLRInputStream; import org.antlr.v4.runtime.CharStream; +import org.antlr.v4.runtime.CharStreams; import org.antlr.v4.runtime.CommonToken; import org.antlr.v4.runtime.CommonTokenStream; import org.antlr.v4.runtime.DiagnosticErrorListener; @@ -19,14 +19,12 @@ import org.antlr.v4.runtime.TokenStream; import org.antlr.v4.runtime.atn.PredictionMode; import javax.print.PrintException; -import java.io.FileInputStream; import java.io.IOException; -import java.io.InputStream; -import java.io.InputStreamReader; -import java.io.Reader; import java.lang.reflect.Constructor; import java.lang.reflect.InvocationTargetException; import java.lang.reflect.Method; +import java.nio.charset.Charset; +import java.nio.file.Paths; import java.util.ArrayList; import java.util.List; @@ -154,42 +152,22 @@ public class TestRig { parser = parserCtor.newInstance((TokenStream)null); } + Charset charset = ( encoding == null ? Charset.defaultCharset () : Charset.forName(encoding) ); if ( inputFiles.size()==0 ) { - InputStream is = System.in; - Reader r; - if ( encoding!=null ) { - r = new InputStreamReader(is, encoding); - } - else { - r = new InputStreamReader(is); - } - - process(lexer, parserClass, parser, is, r); + CharStream charStream = CharStreams.fromStream(System.in, charset); + process(lexer, parserClass, parser, charStream); return; } for (String inputFile : inputFiles) { - InputStream is = System.in; - if ( inputFile!=null ) { - is = new FileInputStream(inputFile); - } - Reader r; - if ( encoding!=null ) { - r = new InputStreamReader(is, encoding); - } - else { - r = new InputStreamReader(is); - } - + CharStream charStream = CharStreams.fromPath(Paths.get(inputFile), charset); if ( inputFiles.size()>1 ) { System.err.println(inputFile); } - process(lexer, parserClass, parser, is, r); + process(lexer, parserClass, parser, charStream); } } - protected void process(Lexer lexer, Class<? extends Parser> parserClass, Parser parser, InputStream is, Reader r) throws IOException, IllegalAccessException, InvocationTargetException, PrintException { - try { - ANTLRInputStream input = new ANTLRInputStream(r); + protected void process(Lexer lexer, Class<? extends Parser> parserClass, Parser parser, CharStream input) throws IOException, IllegalAccessException, InvocationTargetException, PrintException { lexer.setInputStream(input); CommonTokenStream tokens = new CommonTokenStream(lexer); @@ -242,9 +220,4 @@ public class TestRig { System.err.println("No method for rule "+startRuleName+" or it has arguments"); } } - finally { - if ( r!=null ) r.close(); - if ( is!=null ) is.close(); - } - } } diff --git a/tool/src/org/antlr/v4/gui/TreeLayoutAdaptor.java b/tool/src/org/antlr/v4/gui/TreeLayoutAdaptor.java index 295ab99..d821efb 100644 --- a/tool/src/org/antlr/v4/gui/TreeLayoutAdaptor.java +++ b/tool/src/org/antlr/v4/gui/TreeLayoutAdaptor.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/tool/src/org/antlr/v4/gui/TreePostScriptGenerator.java b/tool/src/org/antlr/v4/gui/TreePostScriptGenerator.java index 19f3512..d0411ee 100644 --- a/tool/src/org/antlr/v4/gui/TreePostScriptGenerator.java +++ b/tool/src/org/antlr/v4/gui/TreePostScriptGenerator.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/tool/src/org/antlr/v4/gui/TreeTextProvider.java b/tool/src/org/antlr/v4/gui/TreeTextProvider.java index ff3983e..59c6c38 100644 --- a/tool/src/org/antlr/v4/gui/TreeTextProvider.java +++ b/tool/src/org/antlr/v4/gui/TreeTextProvider.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/tool/src/org/antlr/v4/gui/TreeViewer.java b/tool/src/org/antlr/v4/gui/TreeViewer.java index 21d0f5c..2c000aa 100644 --- a/tool/src/org/antlr/v4/gui/TreeViewer.java +++ b/tool/src/org/antlr/v4/gui/TreeViewer.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ @@ -37,7 +37,10 @@ import java.awt.geom.CubicCurve2D; import java.awt.geom.Rectangle2D; import java.awt.image.BufferedImage; import java.io.File; +import java.io.FileWriter; +import java.io.BufferedWriter; import java.io.IOException; +import java.io.Writer; import java.util.ArrayList; import java.util.Collection; import java.util.List; @@ -159,7 +162,8 @@ public class TreeViewer extends JComponent { double ctrly2 = y1; c.setCurve(x1, y1, ctrlx1, ctrly1, ctrlx2, ctrly2, x2, y2); ((Graphics2D) g).draw(c); - } else { + } + else { g.drawLine((int) x1, (int) y1, (int) x2, (int) y2); } @@ -244,6 +248,69 @@ public class TreeViewer extends JComponent { } } + protected void generateEdges(Writer writer, Tree parent) throws IOException { + if (!getTree().isLeaf(parent)) { + Rectangle2D.Double b1 = getBoundsOfNode(parent); + double x1 = b1.getCenterX(); + double y1 = b1.getCenterY(); + + for (Tree child : getTree().getChildren(parent)) { + Rectangle2D.Double childBounds = getBoundsOfNode(child); + double x2 = childBounds.getCenterX(); + double y2 = childBounds.getMinY(); + writer.write(line(""+x1, ""+y1, ""+x2, ""+y2, + "stroke:black; stroke-width:1px;")); + generateEdges(writer, child); + } + } + } + + protected void generateBox(Writer writer, Tree parent) throws IOException { + + // draw the box in the background + Rectangle2D.Double box = getBoundsOfNode(parent); + writer.write(rect(""+box.x, ""+box.y, ""+box.width, ""+box.height, + "fill:orange; stroke:rgb(0,0,0);", "rx=\"1\"")); + + // draw the text on top of the box (possibly multiple lines) + String line = getText(parent).replace("<","<").replace(">",">"); + int fontSize = 10; + int x = (int) box.x + 2; + int y = (int) box.y + fontSize - 1; + String style = String.format("font-family:sans-serif;font-size:%dpx;", + fontSize); + writer.write(text(""+x, ""+y, style, line)); + } + + private static String line(String x1, String y1, String x2, String y2, + String style) { + return String + .format("<line x1=\"%s\" y1=\"%s\" x2=\"%s\" y2=\"%s\" style=\"%s\" />\n", + x1, y1, x2, y2, style); + } + + private static String rect(String x, String y, String width, String height, + String style, String extraAttributes) { + return String + .format("<rect x=\"%s\" y=\"%s\" width=\"%s\" height=\"%s\" style=\"%s\" %s/>\n", + x, y, width, height, style, extraAttributes); + } + + private static String text(String x, String y, String style, String text) { + return String.format( + "<text x=\"%s\" y=\"%s\" style=\"%s\">\n%s\n</text>\n", x, y, + style, text); + } + + private void paintSVG(Writer writer) throws IOException { + + generateEdges(writer, getTree().getRoot()); + + for (Tree tree : treeLayout.getNodeBounds().keySet()) { + generateBox(writer, tree); + } + } + @Override protected Graphics getComponentGraphics(Graphics g) { Graphics2D g2d=(Graphics2D)g; @@ -306,6 +373,18 @@ public class TreeViewer extends JComponent { ); wrapper.add(png); + // Add an export-to-png button right of the "OK" button + JButton svg = new JButton("Export as SVG"); + svg.addActionListener( + new ActionListener() { + @Override + public void actionPerformed(ActionEvent e) { + generateSVGFile(viewer, dialog); + } + } + ); + wrapper.add(svg); + bottomPanel.add(wrapper, BorderLayout.SOUTH); // Add scale slider @@ -417,29 +496,7 @@ public class TreeViewer extends JComponent { g.dispose(); try { - File suggestedFile = generateNonExistingPngFile(); - JFileChooser fileChooser = new JFileChooserConfirmOverwrite(); - fileChooser.setCurrentDirectory(suggestedFile.getParentFile()); - fileChooser.setSelectedFile(suggestedFile); - FileFilter pngFilter = new FileFilter() { - - @Override - public boolean accept(File pathname) { - if (pathname.isFile()) { - return pathname.getName().toLowerCase().endsWith(".png"); - } - - return true; - } - - @Override - public String getDescription() { - return "PNG Files (*.png)"; - } - }; - - fileChooser.addChoosableFileFilter(pngFilter); - fileChooser.setFileFilter(pngFilter); + JFileChooser fileChooser = getFileChooser(".png", "PNG files"); int returnValue = fileChooser.showSaveDialog(dialog); if (returnValue == JFileChooser.APPROVE_OPTION) { @@ -468,23 +525,85 @@ public class TreeViewer extends JComponent { } } - private static File generateNonExistingPngFile() { + private static JFileChooser getFileChooser(final String fileEnding, + final String description) { + File suggestedFile = generateNonExistingFile(fileEnding); + JFileChooser fileChooser = new JFileChooserConfirmOverwrite(); + fileChooser.setCurrentDirectory(suggestedFile.getParentFile()); + fileChooser.setSelectedFile(suggestedFile); + FileFilter filter = new FileFilter() { + + @Override + public boolean accept(File pathname) { + if (pathname.isFile()) { + return pathname.getName().toLowerCase().endsWith(fileEnding); + } + + return true; + } + + @Override + public String getDescription() { + return description+" (*"+fileEnding+")"; + } + }; + fileChooser.addChoosableFileFilter(filter); + fileChooser.setFileFilter(filter); + return fileChooser; + } + + private static void generateSVGFile(TreeViewer viewer, JFrame dialog) { + + try { + JFileChooser fileChooser = getFileChooser(".svg", "SVG files"); + + int returnValue = fileChooser.showSaveDialog(dialog); + if (returnValue == JFileChooser.APPROVE_OPTION) { + File svgFile = fileChooser.getSelectedFile(); + // save the new svg file here! + BufferedWriter writer = new BufferedWriter(new FileWriter(svgFile)); + // HACK: multiplying with 1.1 should be replaced wit an accurate number + writer.write("<svg width=\"" + viewer.getSize().getWidth() * 1.1 + "\" height=\"" + viewer.getSize().getHeight() * 1.1 + "\" xmlns=\"http://www.w3.org/2000/svg\" xmlns:xlink=\"http://www.w3.org/1999/xlink\">"); + viewer.paintSVG(writer); + writer.write("</svg>"); + writer.flush(); + writer.close(); + try { + // Try to open the parent folder using the OS' native file manager. + Desktop.getDesktop().open(svgFile.getParentFile()); + } catch (Exception ex) { + // We could not launch the file manager: just show a popup that we + // succeeded in saving the PNG file. + JOptionPane.showMessageDialog(dialog, "Saved SVG to: " + + svgFile.getAbsolutePath()); + ex.printStackTrace(); + } + } + } catch (Exception ex) { + JOptionPane.showMessageDialog(dialog, + "Could not export to SVG: " + ex.getMessage(), + "Error", + JOptionPane.ERROR_MESSAGE); + ex.printStackTrace(); + } + } + + private static File generateNonExistingFile(String extension) { final String parent = "."; final String name = "antlr4_parse_tree"; - final String extension = ".png"; - File pngFile = new File(parent, name + extension); + File file = new File(parent, name + extension); int counter = 1; // Keep looping until we create a File that does not yet exist. - while (pngFile.exists()) { - pngFile = new File(parent, name + "_"+ counter + extension); + while (file.exists()) { + file = new File(parent, name + "_" + counter + extension); counter++; } - return pngFile; + return file; } private static void fillTree(TreeNodeWrapper node, Tree tree, TreeViewer viewer) { diff --git a/tool/src/org/antlr/v4/gui/Trees.java b/tool/src/org/antlr/v4/gui/Trees.java index eca200a..5db80ca 100644 --- a/tool/src/org/antlr/v4/gui/Trees.java +++ b/tool/src/org/antlr/v4/gui/Trees.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/tool/src/org/antlr/v4/misc/CharSupport.java b/tool/src/org/antlr/v4/misc/CharSupport.java index 50db0d9..47e0033 100644 --- a/tool/src/org/antlr/v4/misc/CharSupport.java +++ b/tool/src/org/antlr/v4/misc/CharSupport.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ @@ -7,6 +7,10 @@ package org.antlr.v4.misc; import org.antlr.v4.runtime.Lexer; +import org.antlr.v4.runtime.misc.Interval; +import org.antlr.v4.runtime.misc.IntervalSet; + +import java.util.Iterator; /** */ public class CharSupport { @@ -26,47 +30,48 @@ public class CharSupport { ANTLRLiteralEscapedCharValue['b'] = '\b'; ANTLRLiteralEscapedCharValue['f'] = '\f'; ANTLRLiteralEscapedCharValue['\\'] = '\\'; - ANTLRLiteralEscapedCharValue['\''] = '\''; - ANTLRLiteralEscapedCharValue['"'] = '"'; - ANTLRLiteralEscapedCharValue['-'] = '-'; - ANTLRLiteralEscapedCharValue[']'] = ']'; ANTLRLiteralCharValueEscape['\n'] = "\\n"; ANTLRLiteralCharValueEscape['\r'] = "\\r"; ANTLRLiteralCharValueEscape['\t'] = "\\t"; ANTLRLiteralCharValueEscape['\b'] = "\\b"; ANTLRLiteralCharValueEscape['\f'] = "\\f"; ANTLRLiteralCharValueEscape['\\'] = "\\\\"; - ANTLRLiteralCharValueEscape['\''] = "\\'"; } /** Return a string representing the escaped char for code c. E.g., If c - * has value 0x100, you will get "\u0100". ASCII gets the usual - * char (non-hex) representation. Control characters are spit out - * as unicode. While this is specially set up for returning Java strings, - * it can be used by any language target that has the same syntax. :) + * has value 0x100, you will get "\\u0100". ASCII gets the usual + * char (non-hex) representation. Non-ASCII characters are spit out + * as \\uXXXX or \\u{XXXXXX} escapes. */ public static String getANTLRCharLiteralForChar(int c) { - if ( c< Lexer.MIN_CHAR_VALUE ) { - return "'<INVALID>'"; - } - if ( c<ANTLRLiteralCharValueEscape.length && ANTLRLiteralCharValueEscape[c]!=null ) { - return '\''+ANTLRLiteralCharValueEscape[c]+'\''; + String result; + if ( c < Lexer.MIN_CHAR_VALUE ) { + result = "<INVALID>"; } - if ( Character.UnicodeBlock.of((char)c)==Character.UnicodeBlock.BASIC_LATIN && - !Character.isISOControl((char)c) ) { - if ( c=='\\' ) { - return "'\\\\'"; + else { + String charValueEscape = c < ANTLRLiteralCharValueEscape.length ? ANTLRLiteralCharValueEscape[c] : null; + if (charValueEscape != null) { + result = charValueEscape; + } + else if (Character.UnicodeBlock.of((char) c) == Character.UnicodeBlock.BASIC_LATIN && + !Character.isISOControl((char) c)) { + if (c == '\\') { + result = "\\\\"; + } + else if (c == '\'') { + result = "\\'"; + } + else { + result = Character.toString((char) c); + } } - if ( c=='\'') { - return "'\\''"; + else if (c <= 0xFFFF) { + result = String.format("\\u%04X", c); + } else { + result = String.format("\\u{%06X}", c); } - return '\''+Character.toString((char)c)+'\''; } - // turn on the bit above max "\uFFFF" value so that we pad with zeros - // then only take last 4 digits - String hex = Integer.toHexString(c|0x10000).toUpperCase().substring(1,5); - String unicodeStr = "'\\u"+hex+"'"; - return unicodeStr; + return '\'' + result + '\''; } /** Given a literal like (the 3 char sequence with single quotes) 'a', @@ -87,11 +92,26 @@ public class CharSupport { if ( literal.charAt(i) == '\\' ) { end = i+2; if ( i+1 < n && literal.charAt(i+1) == 'u' ) { - for (end = i + 2; end < i + 6; end++) { - if ( end>n ) return null; // invalid escape sequence. - char charAt = literal.charAt(end); - if (!Character.isDigit(charAt) && !(charAt >= 'a' && charAt <= 'f') && !(charAt >= 'A' && charAt <= 'F')) { - return null; // invalid escape sequence. + if ( i+2 < n && literal.charAt(i+2) == '{' ) { // extended escape sequence + end = i + 3; + while (true) { + if ( end + 1 > n ) return null; // invalid escape sequence. + char charAt = literal.charAt(end++); + if (charAt == '}') { + break; + } + if (!Character.isDigit(charAt) && !(charAt >= 'a' && charAt <= 'f') && !(charAt >= 'A' && charAt <= 'F')) { + return null; // invalid escape sequence. + } + } + } + else { + for (end = i + 2; end < i + 6; end++) { + if ( end>n ) return null; // invalid escape sequence. + char charAt = literal.charAt(end); + if (!Character.isDigit(charAt) && !(charAt >= 'a' && charAt <= 'f') && !(charAt >= 'A' && charAt <= 'F')) { + return null; // invalid escape sequence. + } } } } @@ -102,13 +122,13 @@ public class CharSupport { if ( c==-1 ) { return null; // invalid escape sequence. } - else buf.append((char)c); + else buf.appendCodePoint(c); i = end; } return buf.toString(); } - /** Given char x or \t or \u1234 return the char value; + /** Given char x or \\t or \\u1234 return the char value; * Unnecessary escapes like '\{' yield -1. */ public static int getCharValueFromCharInGrammarLiteral(String cstr) { @@ -119,28 +139,67 @@ public class CharSupport { case 2: if ( cstr.charAt(0)!='\\' ) return -1; // '\x' (antlr lexer will catch invalid char) - if ( Character.isDigit(cstr.charAt(1)) ) return -1; - int escChar = cstr.charAt(1); + char escChar = cstr.charAt(1); + if (escChar == '\'') return escChar; // escape quote only in string literals. int charVal = ANTLRLiteralEscapedCharValue[escChar]; - if ( charVal==0 ) return -1; + if (charVal == 0) return -1; return charVal; case 6: - // '\u1234' + // '\\u1234' or '\\u{12}' if ( !cstr.startsWith("\\u") ) return -1; - String unicodeChars = cstr.substring(2, cstr.length()); - int result = -1; - try { - result = Integer.parseInt(unicodeChars, 16); + int startOff; + int endOff; + if ( cstr.charAt(2) == '{' ) { + startOff = 3; + endOff = cstr.indexOf('}'); } - catch (NumberFormatException e) { + else { + startOff = 2; + endOff = cstr.length(); } - return result; + return parseHexValue(cstr, startOff, endOff); default: + if ( cstr.startsWith("\\u{") ) { + return parseHexValue(cstr, 3, cstr.indexOf('}')); + } return -1; } } + public static int parseHexValue(String cstr, int startOff, int endOff) { + if (startOff < 0 || endOff < 0) { + return -1; + } + String unicodeChars = cstr.substring(startOff, endOff); + int result = -1; + try { + result = Integer.parseInt(unicodeChars, 16); + } + catch (NumberFormatException e) { + } + return result; + } + public static String capitalize(String s) { return Character.toUpperCase(s.charAt(0)) + s.substring(1); } + + public static String getIntervalSetEscapedString(IntervalSet intervalSet) { + StringBuilder buf = new StringBuilder(); + Iterator<Interval> iter = intervalSet.getIntervals().iterator(); + while (iter.hasNext()) { + Interval interval = iter.next(); + buf.append(getRangeEscapedString(interval.a, interval.b)); + if (iter.hasNext()) { + buf.append(" | "); + } + } + return buf.toString(); + } + + public static String getRangeEscapedString(int codePointStart, int codePointEnd) { + return codePointStart != codePointEnd + ? getANTLRCharLiteralForChar(codePointStart) + ".." + getANTLRCharLiteralForChar(codePointEnd) + : getANTLRCharLiteralForChar(codePointStart); + } } diff --git a/tool/src/org/antlr/v4/misc/EscapeSequenceParsing.java b/tool/src/org/antlr/v4/misc/EscapeSequenceParsing.java new file mode 100644 index 0000000..d34988d --- /dev/null +++ b/tool/src/org/antlr/v4/misc/EscapeSequenceParsing.java @@ -0,0 +1,186 @@ +/* + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. + * Use of this file is governed by the BSD 3-clause license that + * can be found in the LICENSE.txt file in the project root. + */ + +package org.antlr.v4.misc; + +import org.antlr.v4.runtime.misc.IntervalSet; +import org.antlr.v4.unicode.UnicodeData; + +import java.util.Objects; + +/** + * Utility class to parse escapes like: + * \\n + * \\uABCD + * \\u{10ABCD} + * \\p{Foo} + * \\P{Bar} + * \\p{Baz=Blech} + * \\P{Baz=Blech} + */ +public abstract class EscapeSequenceParsing { + public static class Result { + public enum Type { + INVALID, + CODE_POINT, + PROPERTY + }; + + public final Type type; + public final int codePoint; + public final IntervalSet propertyIntervalSet; + public final int startOffset; + public final int parseLength; + + public Result(Type type, int codePoint, IntervalSet propertyIntervalSet, int startOffset, int parseLength) { + this.type = type; + this.codePoint = codePoint; + this.propertyIntervalSet = propertyIntervalSet; + this.startOffset = startOffset; + this.parseLength = parseLength; + } + + @Override + public String toString() { + return String.format( + "%s type=%s codePoint=%d propertyIntervalSet=%s parseLength=%d", + super.toString(), + type, + codePoint, + propertyIntervalSet, + parseLength); + } + + @Override + public boolean equals(Object other) { + if (!(other instanceof Result)) { + return false; + } + Result that = (Result) other; + if (this == that) { + return true; + } + return Objects.equals(this.type, that.type) && + Objects.equals(this.codePoint, that.codePoint) && + Objects.equals(this.propertyIntervalSet, that.propertyIntervalSet) && + Objects.equals(this.parseLength, that.parseLength); + } + + @Override + public int hashCode() { + return Objects.hash(type, codePoint, propertyIntervalSet, parseLength); + } + } + + /** + * Parses a single escape sequence starting at {@code startOff}. + * + * Returns a type of INVALID if no valid escape sequence was found, a Result otherwise. + */ + public static Result parseEscape(String s, int startOff) { + int offset = startOff; + if (offset + 2 > s.length() || s.codePointAt(offset) != '\\') { + return invalid(startOff, s.length()-1); + } + // Move past backslash + offset++; + int escaped = s.codePointAt(offset); + // Move past escaped code point + offset += Character.charCount(escaped); + if (escaped == 'u') { + // \\u{1} is the shortest we support + if (offset + 3 > s.length()) { + return invalid(startOff, s.length()-1); + } + int hexStartOffset; + int hexEndOffset; // appears to be exclusive + if (s.codePointAt(offset) == '{') { + hexStartOffset = offset + 1; + hexEndOffset = s.indexOf('}', hexStartOffset); + if (hexEndOffset == -1) { + return invalid(startOff, s.length()-1); + } + offset = hexEndOffset + 1; + } + else { + if (offset + 4 > s.length()) { + return invalid(startOff, s.length()-1); + } + hexStartOffset = offset; + hexEndOffset = offset + 4; + offset = hexEndOffset; + } + int codePointValue = CharSupport.parseHexValue(s, hexStartOffset, hexEndOffset); + if (codePointValue == -1 || codePointValue > Character.MAX_CODE_POINT) { + return invalid(startOff, startOff+6-1); + } + return new Result( + Result.Type.CODE_POINT, + codePointValue, + IntervalSet.EMPTY_SET, + startOff, + offset - startOff); + } + else if (escaped == 'p' || escaped == 'P') { + // \p{L} is the shortest we support + if (offset + 3 > s.length()) { + return invalid(startOff, s.length()-1); + } + if (s.codePointAt(offset) != '{') { + return invalid(startOff, offset); + } + int openBraceOffset = offset; + int closeBraceOffset = s.indexOf('}', openBraceOffset); + if (closeBraceOffset == -1) { + return invalid(startOff, s.length()-1); + } + String propertyName = s.substring(openBraceOffset + 1, closeBraceOffset); + IntervalSet propertyIntervalSet = UnicodeData.getPropertyCodePoints(propertyName); + if (propertyIntervalSet == null) { + return invalid(startOff, closeBraceOffset); + } + offset = closeBraceOffset + 1; + if (escaped == 'P') { + propertyIntervalSet = propertyIntervalSet.complement(IntervalSet.COMPLETE_CHAR_SET); + } + return new Result( + Result.Type.PROPERTY, + -1, + propertyIntervalSet, + startOff, + offset - startOff); + } + else if (escaped < CharSupport.ANTLRLiteralEscapedCharValue.length) { + int codePoint = CharSupport.ANTLRLiteralEscapedCharValue[escaped]; + if (codePoint == 0) { + if (escaped != ']' && escaped != '-') { // escape ']' and '-' only in char sets. + return invalid(startOff, startOff+1); + } + else { + codePoint = escaped; + } + } + return new Result( + Result.Type.CODE_POINT, + codePoint, + IntervalSet.EMPTY_SET, + startOff, + offset - startOff); + } + else { + return invalid(startOff,s.length()-1); + } + } + + private static Result invalid(int start, int stop) { // start..stop is inclusive + return new Result( + Result.Type.INVALID, + 0, + IntervalSet.EMPTY_SET, + start, + stop - start + 1); + } +} diff --git a/tool/src/org/antlr/v4/misc/FrequencySet.java b/tool/src/org/antlr/v4/misc/FrequencySet.java index d3d4c73..343f12b 100644 --- a/tool/src/org/antlr/v4/misc/FrequencySet.java +++ b/tool/src/org/antlr/v4/misc/FrequencySet.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/tool/src/org/antlr/v4/misc/Graph.java b/tool/src/org/antlr/v4/misc/Graph.java index ec4c147..ea10cc0 100644 --- a/tool/src/org/antlr/v4/misc/Graph.java +++ b/tool/src/org/antlr/v4/misc/Graph.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/tool/src/org/antlr/v4/misc/MutableInt.java b/tool/src/org/antlr/v4/misc/MutableInt.java index 607b8f6..007f4c2 100644 --- a/tool/src/org/antlr/v4/misc/MutableInt.java +++ b/tool/src/org/antlr/v4/misc/MutableInt.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/tool/src/org/antlr/v4/misc/OrderedHashMap.java b/tool/src/org/antlr/v4/misc/OrderedHashMap.java index 67f9deb..7a988f9 100644 --- a/tool/src/org/antlr/v4/misc/OrderedHashMap.java +++ b/tool/src/org/antlr/v4/misc/OrderedHashMap.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/tool/src/org/antlr/v4/misc/Utils.java b/tool/src/org/antlr/v4/misc/Utils.java index d9a2473..abdedb7 100644 --- a/tool/src/org/antlr/v4/misc/Utils.java +++ b/tool/src/org/antlr/v4/misc/Utils.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ @@ -122,7 +122,8 @@ public class Utils { public static void setSize(List<?> list, int size) { if (size < list.size()) { list.subList(size, list.size()).clear(); - } else { + } + else { while (size > list.size()) { list.add(null); } diff --git a/tool/src/org/antlr/v4/parse/ANTLRLexer.g b/tool/src/org/antlr/v4/parse/ANTLRLexer.g index 7d61c11..4f113a7 100644 --- a/tool/src/org/antlr/v4/parse/ANTLRLexer.g +++ b/tool/src/org/antlr/v4/parse/ANTLRLexer.g @@ -146,6 +146,7 @@ tokens { SEMPRED; TOKEN_REF; RULE_REF; LEXER_CHAR_SET; ARG_ACTION; } */ package org.antlr.v4.parse; import org.antlr.v4.tool.*; +import org.antlr.v4.runtime.misc.Interval; } @@ -615,8 +616,8 @@ SRC : 'src' WSCHARS+ file=ACTION_STRING_LITERAL WSCHARS+ line=INT // // ANTLR makes no disticintion between a single character literal and a // multi-character string. All literals are single quote delimited and -// may contain unicode escape sequences of the form \uxxxx, where x -// is a valid hexadecimal number (as per Java basically). +// may contain unicode escape sequences of the form \uxxxx or \u{xxxxxx}, +// where x is a valid hexadecimal number. STRING_LITERAL : '\'' ( ( ESC_SEQ | ~('\\'|'\''|'\r'|'\n') ) )* ( '\'' @@ -643,30 +644,29 @@ fragment ESC_SEQ : '\\' ( - // The standard escaped character set such as tab, newline, - // etc. - // - 'b'|'t'|'n'|'f'|'r'|'\"'|'\''|'\\' + // The standard escaped character set such as tab, newline, etc... + 'b'|'t'|'n'|'f'|'r'|'\''|'\\' | // A Java style Unicode escape sequence - // UNICODE_ESC + | // A Swift/Hack style Unicode escape sequence + UNICODE_EXTENDED_ESC + | // An illegal escape seqeunce - // + ~('b'|'t'|'n'|'f'|'r'|'\''|'\\'|'u') // \x for any invalid x (make sure to match char here) { - Token t = new CommonToken(input, state.type, state.channel, getCharIndex()-1, getCharIndex()); + Token t = new CommonToken(input, state.type, state.channel, getCharIndex()-2, getCharIndex()-1); t.setText(t.getText()); t.setLine(input.getLine()); - t.setCharPositionInLine(input.getCharPositionInLine()-1); - grammarError(ErrorType.INVALID_ESCAPE_SEQUENCE, t); + t.setCharPositionInLine(input.getCharPositionInLine()-2); + grammarError(ErrorType.INVALID_ESCAPE_SEQUENCE, t, input.substring(getCharIndex()-2,getCharIndex()-1)); } ) ; fragment UNICODE_ESC - @init { // Flag to tell us whether we have a valid number of @@ -710,16 +710,42 @@ UNICODE_ESC // Now check the digit count and issue an error if we need to // { - if (hCount != 4) { - Token t = new CommonToken(input, state.type, state.channel, getCharIndex()-3-hCount, getCharIndex()-1); - t.setText(t.getText()); - t.setLine(input.getLine()); - t.setCharPositionInLine(input.getCharPositionInLine()-hCount-2); - grammarError(ErrorType.INVALID_ESCAPE_SEQUENCE, t); + if (hCount < 4) { + Interval badRange = Interval.of(getCharIndex()-2-hCount, getCharIndex()); + String lastChar = input.substring(badRange.b, badRange.b); + if ( lastChar.codePointAt(0)=='\'' ) { + badRange.b--; + } + String bad = input.substring(badRange.a, badRange.b); + Token t = new CommonToken(input, state.type, state.channel, badRange.a, badRange.b); + t.setLine(input.getLine()); + t.setCharPositionInLine(input.getCharPositionInLine()-hCount-2); + grammarError(ErrorType.INVALID_ESCAPE_SEQUENCE, t, bad); } } ; +fragment +UNICODE_EXTENDED_ESC + : 'u{' // Leadin for unicode extended escape sequence + + HEX_DIGIT+ // One or more hexadecimal digits + + '}' // Leadout for unicode extended escape sequence + + // Now check the digit count and issue an error if we need to + { + int numDigits = getCharIndex()-state.tokenStartCharIndex-6; + if (numDigits > 6) { + Token t = new CommonToken(input, state.type, state.channel, state.tokenStartCharIndex, getCharIndex()-1); + t.setText(t.getText()); + t.setLine(input.getLine()); + t.setCharPositionInLine(input.getCharPositionInLine()-numDigits); + grammarError(ErrorType.INVALID_ESCAPE_SEQUENCE, t, input.substring(state.tokenStartCharIndex,getCharIndex()-1)); + } + } + ; + // ---------- // Whitespace // diff --git a/tool/src/org/antlr/v4/parse/ATNBuilder.g b/tool/src/org/antlr/v4/parse/ATNBuilder.g index 32cdec5..dfd47dc 100644 --- a/tool/src/org/antlr/v4/parse/ATNBuilder.g +++ b/tool/src/org/antlr/v4/parse/ATNBuilder.g @@ -137,7 +137,7 @@ element returns [ATNFactory.Handle p] | ^(ACTION .) {$p = factory.action((ActionAST)$ACTION);} | ^(SEMPRED .) {$p = factory.sempred((PredAST)$SEMPRED);} | ^(NOT b=blockSet[true]) {$p = $b.p;} - | LEXER_CHAR_SET {$p = factory.charSetLiteral($start);} + | LEXER_CHAR_SET {$p = factory.charSetLiteral($start);} ; astOperand returns [ATNFactory.Handle p] diff --git a/tool/src/org/antlr/v4/parse/ActionSplitterListener.java b/tool/src/org/antlr/v4/parse/ActionSplitterListener.java index fe364f9..7ed8b87 100644 --- a/tool/src/org/antlr/v4/parse/ActionSplitterListener.java +++ b/tool/src/org/antlr/v4/parse/ActionSplitterListener.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/tool/src/org/antlr/v4/parse/GrammarASTAdaptor.java b/tool/src/org/antlr/v4/parse/GrammarASTAdaptor.java index 47d6e0a..fd54ec4 100644 --- a/tool/src/org/antlr/v4/parse/GrammarASTAdaptor.java +++ b/tool/src/org/antlr/v4/parse/GrammarASTAdaptor.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/tool/src/org/antlr/v4/parse/GrammarToken.java b/tool/src/org/antlr/v4/parse/GrammarToken.java index edf75b6..1a349ee 100644 --- a/tool/src/org/antlr/v4/parse/GrammarToken.java +++ b/tool/src/org/antlr/v4/parse/GrammarToken.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/tool/src/org/antlr/v4/parse/GrammarTreeVisitor.g b/tool/src/org/antlr/v4/parse/GrammarTreeVisitor.g index c407aa4..9bc3996 100644 --- a/tool/src/org/antlr/v4/parse/GrammarTreeVisitor.g +++ b/tool/src/org/antlr/v4/parse/GrammarTreeVisitor.g @@ -844,7 +844,7 @@ element | SEMPRED {sempredInAlt((PredAST)$SEMPRED);} | ^(ACTION elementOptions) {actionInAlt((ActionAST)$ACTION);} | ^(SEMPRED elementOptions) {sempredInAlt((PredAST)$SEMPRED);} - + | range | ^(NOT blockSet) | ^(NOT block) ; diff --git a/tool/src/org/antlr/v4/parse/ResyncToEndOfRuleBlock.java b/tool/src/org/antlr/v4/parse/ResyncToEndOfRuleBlock.java index 0564d38..802b5ea 100644 --- a/tool/src/org/antlr/v4/parse/ResyncToEndOfRuleBlock.java +++ b/tool/src/org/antlr/v4/parse/ResyncToEndOfRuleBlock.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/tool/src/org/antlr/v4/parse/ScopeParser.java b/tool/src/org/antlr/v4/parse/ScopeParser.java index 25a3f0a..c70ea47 100644 --- a/tool/src/org/antlr/v4/parse/ScopeParser.java +++ b/tool/src/org/antlr/v4/parse/ScopeParser.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ @@ -312,7 +312,8 @@ public class ScopeParser { // do we see a matching '>' ahead? if so, hope it's a generic // and not less followed by expr with greater than p = _splitArgumentList(actionText, p + 1, '>', separatorChar, args); - } else { + } + else { p++; // treat as normal char } break; diff --git a/tool/src/org/antlr/v4/parse/TokenVocabParser.java b/tool/src/org/antlr/v4/parse/TokenVocabParser.java index abfa71c..cf84a92 100644 --- a/tool/src/org/antlr/v4/parse/TokenVocabParser.java +++ b/tool/src/org/antlr/v4/parse/TokenVocabParser.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ @@ -146,6 +146,22 @@ public class TokenVocabParser { // files are generated (in the base, not relative to the input // location.) f = new File(g.tool.outputDirectory, vocabName + CodeGenerator.VOCAB_FILE_EXTENSION); - return f; + if ( f.exists() ) { + return f; + } + + // Still not found? Use the grammar's subfolder then. + String fileDirectory; + + if (g.fileName.lastIndexOf(File.separatorChar) == -1) { + // No path is included in the file name, so make the file + // directory the same as the parent grammar (which might still be just "" + // but when it is not, we will write the file in the correct place. + fileDirectory = "."; + } + else { + fileDirectory = g.fileName.substring(0, g.fileName.lastIndexOf(File.separatorChar)); + } + return new File(fileDirectory, vocabName + CodeGenerator.VOCAB_FILE_EXTENSION); } } diff --git a/tool/src/org/antlr/v4/parse/ToolANTLRLexer.java b/tool/src/org/antlr/v4/parse/ToolANTLRLexer.java index 613c06c..22a1a00 100644 --- a/tool/src/org/antlr/v4/parse/ToolANTLRLexer.java +++ b/tool/src/org/antlr/v4/parse/ToolANTLRLexer.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/tool/src/org/antlr/v4/parse/ToolANTLRParser.java b/tool/src/org/antlr/v4/parse/ToolANTLRParser.java index bbc4964..41d8352 100644 --- a/tool/src/org/antlr/v4/parse/ToolANTLRParser.java +++ b/tool/src/org/antlr/v4/parse/ToolANTLRParser.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/tool/src/org/antlr/v4/parse/v3TreeGrammarException.java b/tool/src/org/antlr/v4/parse/v3TreeGrammarException.java index 8c5c7b5..dad0d6e 100644 --- a/tool/src/org/antlr/v4/parse/v3TreeGrammarException.java +++ b/tool/src/org/antlr/v4/parse/v3TreeGrammarException.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/tool/src/org/antlr/v4/parse/v4ParserException.java b/tool/src/org/antlr/v4/parse/v4ParserException.java index cdcbe30..88fec58 100644 --- a/tool/src/org/antlr/v4/parse/v4ParserException.java +++ b/tool/src/org/antlr/v4/parse/v4ParserException.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/tool/src/org/antlr/v4/semantics/ActionSniffer.java b/tool/src/org/antlr/v4/semantics/ActionSniffer.java index ce25233..506589c 100644 --- a/tool/src/org/antlr/v4/semantics/ActionSniffer.java +++ b/tool/src/org/antlr/v4/semantics/ActionSniffer.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/tool/src/org/antlr/v4/semantics/AttributeChecks.java b/tool/src/org/antlr/v4/semantics/AttributeChecks.java index 5e35e8c..84bbd41 100644 --- a/tool/src/org/antlr/v4/semantics/AttributeChecks.java +++ b/tool/src/org/antlr/v4/semantics/AttributeChecks.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/tool/src/org/antlr/v4/semantics/BasicSemanticChecks.java b/tool/src/org/antlr/v4/semantics/BasicSemanticChecks.java index 4f5e792..4f76b5b 100644 --- a/tool/src/org/antlr/v4/semantics/BasicSemanticChecks.java +++ b/tool/src/org/antlr/v4/semantics/BasicSemanticChecks.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/tool/src/org/antlr/v4/semantics/BlankActionSplitterListener.java b/tool/src/org/antlr/v4/semantics/BlankActionSplitterListener.java index b0c05e7..2e18e67 100644 --- a/tool/src/org/antlr/v4/semantics/BlankActionSplitterListener.java +++ b/tool/src/org/antlr/v4/semantics/BlankActionSplitterListener.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/tool/src/org/antlr/v4/semantics/RuleCollector.java b/tool/src/org/antlr/v4/semantics/RuleCollector.java index 94b68d2..0931986 100644 --- a/tool/src/org/antlr/v4/semantics/RuleCollector.java +++ b/tool/src/org/antlr/v4/semantics/RuleCollector.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/tool/src/org/antlr/v4/semantics/SemanticPipeline.java b/tool/src/org/antlr/v4/semantics/SemanticPipeline.java index 6f95558..dc3cb2f 100644 --- a/tool/src/org/antlr/v4/semantics/SemanticPipeline.java +++ b/tool/src/org/antlr/v4/semantics/SemanticPipeline.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ @@ -108,6 +108,7 @@ public class SemanticPipeline { } symcheck.checkForModeConflicts(g); + symcheck.checkForUnreachableTokens(g); assignChannelTypes(g, collector.channelDefs); diff --git a/tool/src/org/antlr/v4/semantics/SymbolChecks.java b/tool/src/org/antlr/v4/semantics/SymbolChecks.java index 5eccef5..abbf22d 100644 --- a/tool/src/org/antlr/v4/semantics/SymbolChecks.java +++ b/tool/src/org/antlr/v4/semantics/SymbolChecks.java @@ -1,12 +1,15 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ package org.antlr.v4.semantics; +import org.antlr.runtime.tree.CommonTree; +import org.antlr.runtime.tree.Tree; import org.antlr.v4.automata.LexerATNFactory; +import org.antlr.v4.parse.ANTLRLexer; import org.antlr.v4.parse.ANTLRParser; import org.antlr.v4.runtime.Token; import org.antlr.v4.tool.Alternative; @@ -16,11 +19,15 @@ import org.antlr.v4.tool.ErrorManager; import org.antlr.v4.tool.ErrorType; import org.antlr.v4.tool.Grammar; import org.antlr.v4.tool.LabelElementPair; +import org.antlr.v4.tool.LabelType; +import org.antlr.v4.tool.LeftRecursiveRule; import org.antlr.v4.tool.LexerGrammar; import org.antlr.v4.tool.Rule; +import org.antlr.v4.tool.ast.AltAST; import org.antlr.v4.tool.ast.GrammarAST; -import org.antlr.v4.tool.LabelType; +import org.antlr.v4.tool.ast.TerminalAST; +import java.util.ArrayList; import java.util.Collection; import java.util.HashMap; import java.util.HashSet; @@ -35,42 +42,31 @@ import java.util.Set; * Side-effect: strip away redef'd rules. */ public class SymbolChecks { - Grammar g; - SymbolCollector collector; - Map<String, Rule> nameToRuleMap = new HashMap<String, Rule>(); + Grammar g; + SymbolCollector collector; + Map<String, Rule> nameToRuleMap = new HashMap<String, Rule>(); Set<String> tokenIDs = new HashSet<String>(); - Map<String, Set<String>> actionScopeToActionNames = new HashMap<String, Set<String>>(); -// DoubleKeyMap<String, String, GrammarAST> namedActions = -// new DoubleKeyMap<String, String, GrammarAST>(); + Map<String, Set<String>> actionScopeToActionNames = new HashMap<String, Set<String>>(); public ErrorManager errMgr; protected final Set<String> reservedNames = new HashSet<String>(); + { reservedNames.addAll(LexerATNFactory.getCommonConstants()); } - public SymbolChecks(Grammar g, SymbolCollector collector) { - this.g = g; - this.collector = collector; + public SymbolChecks(Grammar g, SymbolCollector collector) { + this.g = g; + this.collector = collector; this.errMgr = g.tool.errMgr; - for (GrammarAST tokenId : collector.tokenIDRefs) { - tokenIDs.add(tokenId.getText()); - } - /* - System.out.println("rules="+collector.rules); - System.out.println("rulerefs="+collector.rulerefs); - System.out.println("tokenIDRefs="+collector.tokenIDRefs); - System.out.println("terminals="+collector.terminals); - System.out.println("strings="+collector.strings); - System.out.println("tokensDef="+collector.tokensDefs); - System.out.println("actions="+collector.actions); - System.out.println("scopes="+collector.scopes); - */ - } - - public void process() { + for (GrammarAST tokenId : collector.tokenIDRefs) { + tokenIDs.add(tokenId.getText()); + } + } + + public void process() { // methods affect fields, but no side-effects outside this object // So, call order sensitive // First collect all rules for later use in checkForLabelConflict() @@ -79,7 +75,6 @@ public class SymbolChecks { } checkReservedNames(g.rules.values()); checkActionRedefinitions(collector.namedActions); - checkForTokenConflicts(collector.tokenIDRefs); // sets tokenIDs checkForLabelConflicts(g.rules.values()); } @@ -92,7 +87,8 @@ public class SymbolChecks { nameNode = (GrammarAST) ampersandAST.getChild(0); if (ampersandAST.getChildCount() == 2) { name = nameNode.getText(); - } else { + } + else { scope = nameNode.getText(); name = ampersandAST.getChild(1).getText(); } @@ -103,72 +99,126 @@ public class SymbolChecks { } if (!scopeActions.contains(name)) { scopeActions.add(name); - } else { + } + else { errMgr.grammarError(ErrorType.ACTION_REDEFINITION, g.fileName, nameNode.token, name); } } } - public void checkForTokenConflicts(List<GrammarAST> tokenIDRefs) { -// for (GrammarAST a : tokenIDRefs) { -// Token t = a.token; -// String ID = t.getText(); -// tokenIDs.add(ID); -// } - } - - /** Make sure a label doesn't conflict with another symbol. - * Labels must not conflict with: rules, tokens, scope names, - * return values, parameters, and rule-scope dynamic attributes - * defined in surrounding rule. Also they must have same type - * for repeated defs. - */ - public void checkForLabelConflicts(Collection<Rule> rules) { + /** + * Make sure a label doesn't conflict with another symbol. + * Labels must not conflict with: rules, tokens, scope names, + * return values, parameters, and rule-scope dynamic attributes + * defined in surrounding rule. Also they must have same type + * for repeated defs. + */ + public void checkForLabelConflicts(Collection<Rule> rules) { for (Rule r : rules) { checkForAttributeConflicts(r); - Map<String, LabelElementPair> labelNameSpace = - new HashMap<String, LabelElementPair>(); - for (int i = 1; i <= r.numberOfAlts; i++) { - if (r.hasAltSpecificContexts()) { - labelNameSpace.clear(); - } + Map<String, LabelElementPair> labelNameSpace = new HashMap<>(); + for (int i = 1; i <= r.numberOfAlts; i++) { Alternative a = r.alt[i]; for (List<LabelElementPair> pairs : a.labelDefs.values()) { - for (LabelElementPair p : pairs) { - checkForLabelConflict(r, p.label); - String name = p.label.getText(); - LabelElementPair prev = labelNameSpace.get(name); - if (prev == null) labelNameSpace.put(name, p); - else checkForTypeMismatch(prev, p); + if (r.hasAltSpecificContexts()) { + // Collect labelName-labeledRules map for rule with alternative labels. + Map<String, List<LabelElementPair>> labelPairs = new HashMap<>(); + for (LabelElementPair p : pairs) { + String labelName = findAltLabelName(p.label); + if (labelName != null) { + List<LabelElementPair> list; + if (labelPairs.containsKey(labelName)) { + list = labelPairs.get(labelName); + } + else { + list = new ArrayList<>(); + labelPairs.put(labelName, list); + } + list.add(p); + } + } + + for (List<LabelElementPair> internalPairs : labelPairs.values()) { + labelNameSpace.clear(); + checkLabelPairs(r, labelNameSpace, internalPairs); + } + } + else { + checkLabelPairs(r, labelNameSpace, pairs); } } } } } - void checkForTypeMismatch(LabelElementPair prevLabelPair, LabelElementPair labelPair) { + private void checkLabelPairs(Rule r, Map<String, LabelElementPair> labelNameSpace, List<LabelElementPair> pairs) { + for (LabelElementPair p : pairs) { + checkForLabelConflict(r, p.label); + String name = p.label.getText(); + LabelElementPair prev = labelNameSpace.get(name); + if (prev == null) { + labelNameSpace.put(name, p); + } + else { + checkForTypeMismatch(r, prev, p); + } + } + } + + private String findAltLabelName(CommonTree label) { + if (label == null) { + return null; + } + else if (label instanceof AltAST) { + AltAST altAST = (AltAST) label; + if (altAST.altLabel != null) { + return altAST.altLabel.toString(); + } + else if (altAST.leftRecursiveAltInfo != null) { + return altAST.leftRecursiveAltInfo.altLabel.toString(); + } + else { + return findAltLabelName(label.parent); + } + } + else { + return findAltLabelName(label.parent); + } + } + + private void checkForTypeMismatch(Rule r, LabelElementPair prevLabelPair, LabelElementPair labelPair) { // label already defined; if same type, no problem if (prevLabelPair.type != labelPair.type) { - String typeMismatchExpr = labelPair.type + "!=" + prevLabelPair.type; + // Current behavior: take a token of rule declaration in case of left-recursive rule + // Desired behavior: take a token of proper label declaration in case of left-recursive rule + // See https://github.com/antlr/antlr4/pull/1585 + // Such behavior is referring to the fact that the warning is typically reported on the actual label redefinition, + // but for left-recursive rules the warning is reported on the enclosing rule. + org.antlr.runtime.Token token = r instanceof LeftRecursiveRule + ? ((GrammarAST) r.ast.getChild(0)).getToken() + : labelPair.label.token; errMgr.grammarError( ErrorType.LABEL_TYPE_CONFLICT, g.fileName, - labelPair.label.token, + token, labelPair.label.getText(), - typeMismatchExpr); + labelPair.type + "!=" + prevLabelPair.type); } if (!prevLabelPair.element.getText().equals(labelPair.element.getText()) && (prevLabelPair.type.equals(LabelType.RULE_LABEL) || prevLabelPair.type.equals(LabelType.RULE_LIST_LABEL)) && (labelPair.type.equals(LabelType.RULE_LABEL) || labelPair.type.equals(LabelType.RULE_LIST_LABEL))) { + org.antlr.runtime.Token token = r instanceof LeftRecursiveRule + ? ((GrammarAST) r.ast.getChild(0)).getToken() + : labelPair.label.token; String prevLabelOp = prevLabelPair.type.equals(LabelType.RULE_LIST_LABEL) ? "+=" : "="; String labelOp = labelPair.type.equals(LabelType.RULE_LIST_LABEL) ? "+=" : "="; errMgr.grammarError( ErrorType.LABEL_TYPE_CONFLICT, g.fileName, - labelPair.label.token, + token, labelPair.label.getText() + labelOp + labelPair.element.getText(), prevLabelPair.label.getText() + prevLabelOp + prevLabelPair.element.getText()); } @@ -225,11 +275,11 @@ public class SymbolChecks { for (Attribute attribute : attributes.attributes.values()) { if (ruleNames.contains(attribute.name)) { errMgr.grammarError( - errorType, - g.fileName, - attribute.token != null ? attribute.token : ((GrammarAST)r.ast.getChild(0)).token, - attribute.name, - r.name); + errorType, + g.fileName, + attribute.token != null ? attribute.token : ((GrammarAST) r.ast.getChild(0)).token, + attribute.name, + r.name); } } } @@ -242,11 +292,11 @@ public class SymbolChecks { Set<String> conflictingKeys = attributes.intersection(referenceAttributes); for (String key : conflictingKeys) { errMgr.grammarError( - errorType, - g.fileName, - attributes.get(key).token != null ? attributes.get(key).token : ((GrammarAST) r.ast.getChild(0)).token, - key, - r.name); + errorType, + g.fileName, + attributes.get(key).token != null ? attributes.get(key).token : ((GrammarAST)r.ast.getChild(0)).token, + key, + r.name); } } @@ -275,8 +325,122 @@ public class SymbolChecks { } } - // CAN ONLY CALL THE TWO NEXT METHODS AFTER GRAMMAR HAS RULE DEFS (see semanticpipeline) + /** + * Algorithm steps: + * 1. Collect all simple string literals (i.e. 'asdf', 'as' 'df', but not [a-z]+, 'a'..'z') + * for all lexer rules in each mode except of autogenerated tokens ({@link #getSingleTokenValues(Rule) getSingleTokenValues}) + * 2. Compare every string literal with each other ({@link #checkForOverlap(Grammar, Rule, Rule, List<String>, List<String>) checkForOverlap}) + * and throw TOKEN_UNREACHABLE warning if the same string found. + * Complexity: O(m * n^2 / 2), approximately equals to O(n^2) + * where m - number of modes, n - average number of lexer rules per mode. + * See also testUnreachableTokens unit test for details. + */ + public void checkForUnreachableTokens(Grammar g) { + if (g.isLexer()) { + LexerGrammar lexerGrammar = (LexerGrammar)g; + for (List<Rule> rules : lexerGrammar.modes.values()) { + // Collect string literal lexer rules for each mode + List<Rule> stringLiteralRules = new ArrayList<>(); + List<List<String>> stringLiteralValues = new ArrayList<>(); + for (int i = 0; i < rules.size(); i++) { + Rule rule = rules.get(i); + + List<String> ruleStringAlts = getSingleTokenValues(rule); + if (ruleStringAlts != null && ruleStringAlts.size() > 0) { + stringLiteralRules.add(rule); + stringLiteralValues.add(ruleStringAlts); + } + } + + // Check string sets intersection + for (int i = 0; i < stringLiteralRules.size(); i++) { + List<String> firstTokenStringValues = stringLiteralValues.get(i); + Rule rule1 = stringLiteralRules.get(i); + checkForOverlap(g, rule1, rule1, firstTokenStringValues, stringLiteralValues.get(i)); + + // Check fragment rules only with themself + if (!rule1.isFragment()) { + for (int j = i + 1; j < stringLiteralRules.size(); j++) { + Rule rule2 = stringLiteralRules.get(j); + if (!rule2.isFragment()) { + checkForOverlap(g, rule1, stringLiteralRules.get(j), firstTokenStringValues, stringLiteralValues.get(j)); + } + } + } + } + } + } + } + + /** + * {@return} list of simple string literals for rule {@param rule} + */ + private List<String> getSingleTokenValues(Rule rule) + { + List<String> values = new ArrayList<>(); + for (Alternative alt : rule.alt) { + if (alt != null) { + // select first alt if token has a command + Tree rootNode = alt.ast.getChildCount() == 2 && + alt.ast.getChild(0) instanceof AltAST && alt.ast.getChild(1) instanceof GrammarAST + ? alt.ast.getChild(0) + : alt.ast; + + if (rootNode.getTokenStartIndex() == -1) { + continue; // ignore autogenerated tokens from combined grammars that start with T__ + } + // Ignore alt if contains not only string literals (repetition, optional) + boolean ignore = false; + StringBuilder currentValue = new StringBuilder(); + for (int i = 0; i < rootNode.getChildCount(); i++) { + Tree child = rootNode.getChild(i); + if (!(child instanceof TerminalAST)) { + ignore = true; + break; + } + + TerminalAST terminalAST = (TerminalAST)child; + if (terminalAST.token.getType() != ANTLRLexer.STRING_LITERAL) { + ignore = true; + break; + } + else { + String text = terminalAST.token.getText(); + currentValue.append(text.substring(1, text.length() - 1)); + } + } + + if (!ignore) { + values.add(currentValue.toString()); + } + } + } + return values; + } + + /** + * For same rule compare values from next index: + * TOKEN_WITH_SAME_VALUES: 'asdf' | 'asdf'; + * For different rules compare from start value: + * TOKEN1: 'asdf'; + * TOKEN2: 'asdf'; + */ + private void checkForOverlap(Grammar g, Rule rule1, Rule rule2, List<String> firstTokenStringValues, List<String> secondTokenStringValues) { + for (int i = 0; i < firstTokenStringValues.size(); i++) { + int secondTokenInd = rule1 == rule2 ? i + 1 : 0; + String str1 = firstTokenStringValues.get(i); + for (int j = secondTokenInd; j < secondTokenStringValues.size(); j++) { + String str2 = secondTokenStringValues.get(j); + if (str1.equals(str2)) { + errMgr.grammarError(ErrorType.TOKEN_UNREACHABLE, g.fileName, + ((GrammarAST) rule2.ast.getChild(0)).token, rule2.name, str2, rule1.name); + } + } + } + } + + // CAN ONLY CALL THE TWO NEXT METHODS AFTER GRAMMAR HAS RULE DEFS (see semanticpipeline) public void checkRuleArgs(Grammar g, List<GrammarAST> rulerefs) { if ( rulerefs==null ) return; for (GrammarAST ref : rulerefs) { @@ -285,12 +449,12 @@ public class SymbolChecks { GrammarAST arg = (GrammarAST)ref.getFirstChildWithType(ANTLRParser.ARG_ACTION); if ( arg!=null && (r==null || r.args==null) ) { errMgr.grammarError(ErrorType.RULE_HAS_NO_ARGS, - g.fileName, ref.token, ruleName); + g.fileName, ref.token, ruleName); } - else if ( arg==null && (r!=null&&r.args!=null) ) { + else if ( arg==null && (r!=null && r.args!=null) ) { errMgr.grammarError(ErrorType.MISSING_RULE_ARGS, - g.fileName, ref.token, ruleName); + g.fileName, ref.token, ruleName); } } } @@ -299,18 +463,18 @@ public class SymbolChecks { for (GrammarAST dot : qualifiedRuleRefs) { GrammarAST grammar = (GrammarAST)dot.getChild(0); GrammarAST rule = (GrammarAST)dot.getChild(1); - g.tool.log("semantics", grammar.getText()+"."+rule.getText()); + g.tool.log("semantics", grammar.getText()+"."+rule.getText()); Grammar delegate = g.getImportedGrammar(grammar.getText()); if ( delegate==null ) { errMgr.grammarError(ErrorType.NO_SUCH_GRAMMAR_SCOPE, - g.fileName, grammar.token, grammar.getText(), - rule.getText()); + g.fileName, grammar.token, grammar.getText(), + rule.getText()); } else { if ( g.getRule(grammar.getText(), rule.getText())==null ) { errMgr.grammarError(ErrorType.NO_SUCH_RULE_IN_SCOPE, - g.fileName, rule.token, grammar.getText(), - rule.getText()); + g.fileName, rule.token, grammar.getText(), + rule.getText()); } } } diff --git a/tool/src/org/antlr/v4/semantics/SymbolCollector.java b/tool/src/org/antlr/v4/semantics/SymbolCollector.java index 2bcf9ca..7c41eef 100644 --- a/tool/src/org/antlr/v4/semantics/SymbolCollector.java +++ b/tool/src/org/antlr/v4/semantics/SymbolCollector.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/tool/src/org/antlr/v4/semantics/UseDefAnalyzer.java b/tool/src/org/antlr/v4/semantics/UseDefAnalyzer.java index 9c6f444..4644198 100644 --- a/tool/src/org/antlr/v4/semantics/UseDefAnalyzer.java +++ b/tool/src/org/antlr/v4/semantics/UseDefAnalyzer.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/tool/src/org/antlr/v4/tool/ANTLRMessage.java b/tool/src/org/antlr/v4/tool/ANTLRMessage.java index 0ed03d6..cf585cb 100644 --- a/tool/src/org/antlr/v4/tool/ANTLRMessage.java +++ b/tool/src/org/antlr/v4/tool/ANTLRMessage.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/tool/src/org/antlr/v4/tool/ANTLRToolListener.java b/tool/src/org/antlr/v4/tool/ANTLRToolListener.java index 5662b34..3aa89e4 100644 --- a/tool/src/org/antlr/v4/tool/ANTLRToolListener.java +++ b/tool/src/org/antlr/v4/tool/ANTLRToolListener.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/tool/src/org/antlr/v4/tool/Alternative.java b/tool/src/org/antlr/v4/tool/Alternative.java index f47543c..2a993c2 100644 --- a/tool/src/org/antlr/v4/tool/Alternative.java +++ b/tool/src/org/antlr/v4/tool/Alternative.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/tool/src/org/antlr/v4/tool/Attribute.java b/tool/src/org/antlr/v4/tool/Attribute.java index 94baaa6..5e15c01 100644 --- a/tool/src/org/antlr/v4/tool/Attribute.java +++ b/tool/src/org/antlr/v4/tool/Attribute.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/tool/src/org/antlr/v4/tool/AttributeDict.java b/tool/src/org/antlr/v4/tool/AttributeDict.java index 6513f9b..2af27f6 100644 --- a/tool/src/org/antlr/v4/tool/AttributeDict.java +++ b/tool/src/org/antlr/v4/tool/AttributeDict.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/tool/src/org/antlr/v4/tool/AttributeResolver.java b/tool/src/org/antlr/v4/tool/AttributeResolver.java index e581f23..18dd9ef 100644 --- a/tool/src/org/antlr/v4/tool/AttributeResolver.java +++ b/tool/src/org/antlr/v4/tool/AttributeResolver.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/tool/src/org/antlr/v4/tool/BuildDependencyGenerator.java b/tool/src/org/antlr/v4/tool/BuildDependencyGenerator.java index be48bfa..c33aa4d 100644 --- a/tool/src/org/antlr/v4/tool/BuildDependencyGenerator.java +++ b/tool/src/org/antlr/v4/tool/BuildDependencyGenerator.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/tool/src/org/antlr/v4/tool/DOTGenerator.java b/tool/src/org/antlr/v4/tool/DOTGenerator.java index 6f7aaea..062f2da 100644 --- a/tool/src/org/antlr/v4/tool/DOTGenerator.java +++ b/tool/src/org/antlr/v4/tool/DOTGenerator.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ @@ -92,7 +92,7 @@ public class DOTGenerator { if ( target.stateNumber == Integer.MAX_VALUE ) continue; int ttype = i-1; // we shift up for EOF as -1 for parser String label = String.valueOf(ttype); - if ( isLexer ) label = "'"+getEdgeLabel(String.valueOf((char) i))+"'"; + if ( isLexer ) label = "'"+getEdgeLabel(new StringBuilder().appendCodePoint(i).toString())+"'"; else if ( grammar!=null ) label = grammar.getTokenDisplayName(ttype); ST st = stlib.getInstanceOf("edge"); st.add("label", label); @@ -259,7 +259,7 @@ public class DOTGenerator { edgeST = stlib.getInstanceOf("edge"); AtomTransition atom = (AtomTransition)edge; String label = String.valueOf(atom.label); - if ( isLexer ) label = "'"+getEdgeLabel(String.valueOf((char)atom.label))+"'"; + if ( isLexer ) label = "'"+getEdgeLabel(new StringBuilder().appendCodePoint(atom.label).toString())+"'"; else if ( grammar!=null ) label = grammar.getTokenDisplayName(atom.label); edgeST.add("label", getEdgeLabel(label)); } @@ -289,7 +289,8 @@ public class DOTGenerator { edgeST.add("arrowhead", arrowhead); if (s.getNumberOfTransitions() > 1) { edgeST.add("transitionIndex", i); - } else { + } + else { edgeST.add("transitionIndex", false); } dot.add("edges", edgeST); diff --git a/tool/src/org/antlr/v4/tool/DefaultToolListener.java b/tool/src/org/antlr/v4/tool/DefaultToolListener.java index 0f64cd0..a2e810b 100644 --- a/tool/src/org/antlr/v4/tool/DefaultToolListener.java +++ b/tool/src/org/antlr/v4/tool/DefaultToolListener.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/tool/src/org/antlr/v4/tool/ErrorManager.java b/tool/src/org/antlr/v4/tool/ErrorManager.java index 34f0d50..0230d2b 100644 --- a/tool/src/org/antlr/v4/tool/ErrorManager.java +++ b/tool/src/org/antlr/v4/tool/ErrorManager.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ @@ -16,7 +16,6 @@ import java.io.File; import java.net.URL; import java.util.Collection; import java.util.EnumSet; -import java.util.Locale; import java.util.Set; public class ErrorManager { @@ -32,8 +31,6 @@ public class ErrorManager { /** The group of templates that represent the current message format. */ STGroup format; - /** Messages should be sensitive to the locale. */ - Locale locale; String formatName; ErrorBuffer initSTListener = new ErrorBuffer(); @@ -63,11 +60,20 @@ public class ErrorManager { locationValid = true; } if (msg.fileName != null) { - File f = new File(msg.fileName); - // Don't show path to file in messages; too long. String displayFileName = msg.fileName; - if ( f.exists() ) { - displayFileName = f.getName(); + if (format.equals("antlr")) { + // Don't show path to file in messages in ANTLR format; + // they're too long. + File f = new File(msg.fileName); + if ( f.exists() ) { + displayFileName = f.getName(); + } + } + else { + // For other message formats, use the full filename in the + // message. This assumes that these formats are intended to + // be parsed by IDEs, and so they need the full path to + // resolve correctly. } locationST.add("file", displayFileName); locationValid = true; diff --git a/tool/src/org/antlr/v4/tool/ErrorSeverity.java b/tool/src/org/antlr/v4/tool/ErrorSeverity.java index 1c527b1..3bc89db 100644 --- a/tool/src/org/antlr/v4/tool/ErrorSeverity.java +++ b/tool/src/org/antlr/v4/tool/ErrorSeverity.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/tool/src/org/antlr/v4/tool/ErrorType.java b/tool/src/org/antlr/v4/tool/ErrorType.java index c4a3861..a034803 100644 --- a/tool/src/org/antlr/v4/tool/ErrorType.java +++ b/tool/src/org/antlr/v4/tool/ErrorType.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ @@ -394,11 +394,11 @@ public enum ErrorType { */ IMPORT_NAME_CLASH(113, "<arg.typeString> grammar <arg.name> and imported <arg2.typeString> grammar <arg2.name> both generate <arg2.recognizerName>", ErrorSeverity.ERROR), /** - * Compiler Error 160. + * Compiler Error 114. * * <p>cannot find tokens file <em>filename</em></p> */ - CANNOT_FIND_TOKENS_FILE_REFD_IN_GRAMMAR(160, "cannot find tokens file <arg>", ErrorSeverity.ERROR), + CANNOT_FIND_TOKENS_FILE_REFD_IN_GRAMMAR(114, "cannot find tokens file <arg>", ErrorSeverity.ERROR), /** * Compiler Warning 118. * @@ -522,7 +522,7 @@ public enum ErrorType { */ USE_OF_BAD_WORD(134, "symbol <arg> conflicts with generated code in target language or runtime", ErrorSeverity.ERROR), /** - * Compiler Error 134. + * Compiler Error 183. * * <p>rule reference <em>rule</em> is not currently supported in a set</p> * @@ -530,7 +530,7 @@ public enum ErrorType { * Note: This error has the same number as the unrelated error * {@link #USE_OF_BAD_WORD}.</p> */ - UNSUPPORTED_REFERENCE_IN_LEXER_SET(134, "rule reference <arg> is not currently supported in a set", ErrorSeverity.ERROR), + UNSUPPORTED_REFERENCE_IN_LEXER_SET(183, "rule reference <arg> is not currently supported in a set", ErrorSeverity.ERROR), /** * Compiler Error 135. * @@ -824,7 +824,7 @@ public enum ErrorType { * * @since 4.2.1 */ - INVALID_ESCAPE_SEQUENCE(156, "invalid escape sequence", ErrorSeverity.ERROR), + INVALID_ESCAPE_SEQUENCE(156, "invalid escape sequence <arg>", ErrorSeverity.WARNING), /** * Compiler Warning 157. * @@ -1043,7 +1043,51 @@ public enum ErrorType { * * TODO: Does not work with fragment rules. */ - CHARACTERS_COLLISION_IN_SET(180, "chars \"<arg>\" used multiple times in set <arg2>", ErrorSeverity.WARNING), + CHARACTERS_COLLISION_IN_SET(180, "chars <arg> used multiple times in set <arg2>", ErrorSeverity.WARNING), + + /** + * Compiler Warning 181 + * + * <p>The token range operator makes no sense in the parser as token types + * are not ordered (except in implementation). + * </p> + * + * <pre> + * grammar T; + * a : 'A'..'Z' ; + * </pre> + * + */ + TOKEN_RANGE_IN_PARSER(181, "token ranges not allowed in parser: <arg>..<arg2>", ErrorSeverity.ERROR), + + /** + * Compiler Error 182. + * + * <p>Unicode properties cannot be part of a lexer charset range</p> + * + * <pre> + * A: [\\p{Letter}-\\p{Number}]; + * </pre> + */ + UNICODE_PROPERTY_NOT_ALLOWED_IN_RANGE( + 182, + "unicode property escapes not allowed in lexer charset range: <arg>", + ErrorSeverity.ERROR), + + /** + * Compiler Warning 184. + * + * <p>The token value overlapped by another token or self</p> + * + * <pre> + * TOKEN1: 'value'; + * TOKEN2: 'value'; // warning + * </pre> + */ + TOKEN_UNREACHABLE( + 184, + "One of the token <arg> values unreachable. <arg2> is always overlapped by token <arg3>", + ErrorSeverity.WARNING), /* * Backward incompatibility errors diff --git a/tool/src/org/antlr/v4/tool/Grammar.java b/tool/src/org/antlr/v4/tool/Grammar.java index ea249f8..b86d622 100644 --- a/tool/src/org/antlr/v4/tool/Grammar.java +++ b/tool/src/org/antlr/v4/tool/Grammar.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ @@ -81,6 +81,7 @@ public class Grammar implements AttributeResolver { parserOptions.add("TokenLabelType"); parserOptions.add("tokenVocab"); parserOptions.add("language"); + parserOptions.add("accessLevel"); parserOptions.add("exportMacro"); } @@ -1301,7 +1302,11 @@ public class Grammar implements AttributeResolver { char[] serializedAtn = ATNSerializer.getSerializedAsChars(atn); ATN deserialized = new ATNDeserializer().deserialize(serializedAtn); - return new LexerInterpreter(fileName, getVocabulary(), Arrays.asList(getRuleNames()), ((LexerGrammar)this).modes.keySet(), deserialized, input); + List<String> allChannels = new ArrayList<String>(); + allChannels.add("DEFAULT_TOKEN_CHANNEL"); + allChannels.add("HIDDEN"); + allChannels.addAll(channelValueToNameList); + return new LexerInterpreter(fileName, getVocabulary(), Arrays.asList(getRuleNames()), allChannels, ((LexerGrammar)this).modes.keySet(), deserialized, input); } /** @since 4.5.1 */ diff --git a/tool/src/org/antlr/v4/tool/GrammarInterpreterRuleContext.java b/tool/src/org/antlr/v4/tool/GrammarInterpreterRuleContext.java index 78d1731..1e7db8b 100644 --- a/tool/src/org/antlr/v4/tool/GrammarInterpreterRuleContext.java +++ b/tool/src/org/antlr/v4/tool/GrammarInterpreterRuleContext.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/tool/src/org/antlr/v4/tool/GrammarParserInterpreter.java b/tool/src/org/antlr/v4/tool/GrammarParserInterpreter.java index b8cbb77..7123b43 100644 --- a/tool/src/org/antlr/v4/tool/GrammarParserInterpreter.java +++ b/tool/src/org/antlr/v4/tool/GrammarParserInterpreter.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/tool/src/org/antlr/v4/tool/GrammarSemanticsMessage.java b/tool/src/org/antlr/v4/tool/GrammarSemanticsMessage.java index 3be7c72..802aff7 100644 --- a/tool/src/org/antlr/v4/tool/GrammarSemanticsMessage.java +++ b/tool/src/org/antlr/v4/tool/GrammarSemanticsMessage.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/tool/src/org/antlr/v4/tool/GrammarSyntaxMessage.java b/tool/src/org/antlr/v4/tool/GrammarSyntaxMessage.java index 6de8861..e3fb5b6 100644 --- a/tool/src/org/antlr/v4/tool/GrammarSyntaxMessage.java +++ b/tool/src/org/antlr/v4/tool/GrammarSyntaxMessage.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/tool/src/org/antlr/v4/tool/GrammarTransformPipeline.java b/tool/src/org/antlr/v4/tool/GrammarTransformPipeline.java index fa27e7f..f6e4cdf 100644 --- a/tool/src/org/antlr/v4/tool/GrammarTransformPipeline.java +++ b/tool/src/org/antlr/v4/tool/GrammarTransformPipeline.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ @@ -161,6 +161,7 @@ public class GrammarTransformPipeline { GrammarAST id = (GrammarAST) root.getChild(0); GrammarASTAdaptor adaptor = new GrammarASTAdaptor(id.token.getInputStream()); + GrammarAST channelsRoot = (GrammarAST)root.getFirstChildWithType(ANTLRParser.CHANNELS); GrammarAST tokensRoot = (GrammarAST)root.getFirstChildWithType(ANTLRParser.TOKENS_SPEC); List<GrammarAST> actionRoots = root.getNodesWithType(ANTLRParser.AT); @@ -172,7 +173,39 @@ public class GrammarTransformPipeline { List<GrammarAST> rootRules = RULES.getNodesWithType(ANTLRParser.RULE); for (GrammarAST r : rootRules) rootRuleNames.add(r.getChild(0).getText()); + // make list of modes we have in root grammar + List<GrammarAST> rootModes = root.getNodesWithType(ANTLRParser.MODE); + Set<String> rootModeNames = new HashSet<String>(); + for (GrammarAST m : rootModes) rootModeNames.add(m.getChild(0).getText()); + List<GrammarAST> addedModes = new ArrayList<GrammarAST>(); + for (Grammar imp : imports) { + // COPY CHANNELS + GrammarAST imp_channelRoot = (GrammarAST)imp.ast.getFirstChildWithType(ANTLRParser.CHANNELS); + if ( imp_channelRoot != null) { + rootGrammar.tool.log("grammar", "imported channels: "+imp_channelRoot.getChildren()); + if (channelsRoot==null) { + channelsRoot = imp_channelRoot.dupTree(); + channelsRoot.g = rootGrammar; + root.insertChild(1, channelsRoot); // ^(GRAMMAR ID TOKENS...) + } else { + for (int c = 0; c < imp_channelRoot.getChildCount(); ++c) { + String channel = imp_channelRoot.getChild(c).getText(); + boolean channelIsInRootGrammar = false; + for (int rc = 0; rc < channelsRoot.getChildCount(); ++rc) { + String rootChannel = channelsRoot.getChild(rc).getText(); + if (rootChannel.equals(channel)) { + channelIsInRootGrammar = true; + break; + } + } + if (!channelIsInRootGrammar) { + channelsRoot.addChild(imp_channelRoot.getChild(c).dupNode()); + } + } + } + } + // COPY TOKENS GrammarAST imp_tokensRoot = (GrammarAST)imp.ast.getFirstChildWithType(ANTLRParser.TOKENS_SPEC); if ( imp_tokensRoot!=null ) { @@ -242,7 +275,54 @@ public class GrammarTransformPipeline { } } + // COPY MODES + // The strategy is to copy all the mode sections rules across to any + // mode section in the new grammar with the same name or a new + // mode section if no matching mode is resolved. Rules which are + // already in the new grammar are ignored for copy. If the mode + // section being added ends up empty it is not added to the merged + // grammar. + List<GrammarAST> modes = imp.ast.getNodesWithType(ANTLRParser.MODE); + if (modes != null) { + for (GrammarAST m : modes) { + rootGrammar.tool.log("grammar", "imported mode: " + m.toStringTree()); + String name = m.getChild(0).getText(); + boolean rootAlreadyHasMode = rootModeNames.contains(name); + GrammarAST destinationAST = null; + if (rootAlreadyHasMode) { + for (GrammarAST m2 : rootModes) { + if (m2.getChild(0).getText().equals(name)) { + destinationAST = m2; + break; + } + } + } else { + destinationAST = m.dupNode(); + destinationAST.addChild(m.getChild(0).dupNode()); + } + + int addedRules = 0; + List<GrammarAST> modeRules = m.getAllChildrenWithType(ANTLRParser.RULE); + for (GrammarAST r : modeRules) { + rootGrammar.tool.log("grammar", "imported rule: "+r.toStringTree()); + String ruleName = r.getChild(0).getText(); + boolean rootAlreadyHasRule = rootRuleNames.contains(ruleName); + if (!rootAlreadyHasRule) { + destinationAST.addChild(r); + addedRules++; + rootRuleNames.add(ruleName); + } + } + if (!rootAlreadyHasMode && addedRules > 0) { + rootGrammar.ast.addChild(destinationAST); + rootModeNames.add(name); + rootModes.add(destinationAST); + } + } + } + // COPY RULES + // Rules copied in the mode copy phase are not copied again. List<GrammarAST> rules = imp.ast.getNodesWithType(ANTLRParser.RULE); if ( rules!=null ) { for (GrammarAST r : rules) { diff --git a/tool/src/org/antlr/v4/tool/LabelElementPair.java b/tool/src/org/antlr/v4/tool/LabelElementPair.java index aa2b292..a415ef2 100644 --- a/tool/src/org/antlr/v4/tool/LabelElementPair.java +++ b/tool/src/org/antlr/v4/tool/LabelElementPair.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/tool/src/org/antlr/v4/tool/LabelType.java b/tool/src/org/antlr/v4/tool/LabelType.java index b28cee2..f28f9c5 100644 --- a/tool/src/org/antlr/v4/tool/LabelType.java +++ b/tool/src/org/antlr/v4/tool/LabelType.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/tool/src/org/antlr/v4/tool/LeftRecursionCyclesMessage.java b/tool/src/org/antlr/v4/tool/LeftRecursionCyclesMessage.java index ca1fbc3..9b17abc 100644 --- a/tool/src/org/antlr/v4/tool/LeftRecursionCyclesMessage.java +++ b/tool/src/org/antlr/v4/tool/LeftRecursionCyclesMessage.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/tool/src/org/antlr/v4/tool/LeftRecursiveRule.java b/tool/src/org/antlr/v4/tool/LeftRecursiveRule.java index fb07c5b..d47ddcb 100644 --- a/tool/src/org/antlr/v4/tool/LeftRecursiveRule.java +++ b/tool/src/org/antlr/v4/tool/LeftRecursiveRule.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/tool/src/org/antlr/v4/tool/LexerGrammar.java b/tool/src/org/antlr/v4/tool/LexerGrammar.java index 5c89ab8..1713a33 100644 --- a/tool/src/org/antlr/v4/tool/LexerGrammar.java +++ b/tool/src/org/antlr/v4/tool/LexerGrammar.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/tool/src/org/antlr/v4/tool/Rule.java b/tool/src/org/antlr/v4/tool/Rule.java index 986ea7f..401dfaa 100644 --- a/tool/src/org/antlr/v4/tool/Rule.java +++ b/tool/src/org/antlr/v4/tool/Rule.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/tool/src/org/antlr/v4/tool/ToolMessage.java b/tool/src/org/antlr/v4/tool/ToolMessage.java index b340466..7af97e2 100644 --- a/tool/src/org/antlr/v4/tool/ToolMessage.java +++ b/tool/src/org/antlr/v4/tool/ToolMessage.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/tool/src/org/antlr/v4/tool/ast/ActionAST.java b/tool/src/org/antlr/v4/tool/ast/ActionAST.java index 9ac97da..3067606 100644 --- a/tool/src/org/antlr/v4/tool/ast/ActionAST.java +++ b/tool/src/org/antlr/v4/tool/ast/ActionAST.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/tool/src/org/antlr/v4/tool/ast/AltAST.java b/tool/src/org/antlr/v4/tool/ast/AltAST.java index 38f5a6b..77e15d4 100644 --- a/tool/src/org/antlr/v4/tool/ast/AltAST.java +++ b/tool/src/org/antlr/v4/tool/ast/AltAST.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/tool/src/org/antlr/v4/tool/ast/BlockAST.java b/tool/src/org/antlr/v4/tool/ast/BlockAST.java index 9ddf349..dba507c 100644 --- a/tool/src/org/antlr/v4/tool/ast/BlockAST.java +++ b/tool/src/org/antlr/v4/tool/ast/BlockAST.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/tool/src/org/antlr/v4/tool/ast/GrammarAST.java b/tool/src/org/antlr/v4/tool/ast/GrammarAST.java index cb59b83..adad395 100644 --- a/tool/src/org/antlr/v4/tool/ast/GrammarAST.java +++ b/tool/src/org/antlr/v4/tool/ast/GrammarAST.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/tool/src/org/antlr/v4/tool/ast/GrammarASTErrorNode.java b/tool/src/org/antlr/v4/tool/ast/GrammarASTErrorNode.java index 472dcf5..dc64413 100644 --- a/tool/src/org/antlr/v4/tool/ast/GrammarASTErrorNode.java +++ b/tool/src/org/antlr/v4/tool/ast/GrammarASTErrorNode.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/tool/src/org/antlr/v4/tool/ast/GrammarASTVisitor.java b/tool/src/org/antlr/v4/tool/ast/GrammarASTVisitor.java index be92f85..c005c8f 100644 --- a/tool/src/org/antlr/v4/tool/ast/GrammarASTVisitor.java +++ b/tool/src/org/antlr/v4/tool/ast/GrammarASTVisitor.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/tool/src/org/antlr/v4/tool/ast/GrammarASTWithOptions.java b/tool/src/org/antlr/v4/tool/ast/GrammarASTWithOptions.java index f6b5bca..19e696f 100644 --- a/tool/src/org/antlr/v4/tool/ast/GrammarASTWithOptions.java +++ b/tool/src/org/antlr/v4/tool/ast/GrammarASTWithOptions.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ @@ -43,7 +43,7 @@ public abstract class GrammarASTWithOptions extends GrammarAST { if ( v.startsWith("'") || v.startsWith("\"") ) { v = CharSupport.getStringFromGrammarStringLiteral(v); if (v == null) { - g.tool.errMgr.grammarError(ErrorType.INVALID_ESCAPE_SEQUENCE, g.fileName, value.getToken()); + g.tool.errMgr.grammarError(ErrorType.INVALID_ESCAPE_SEQUENCE, g.fileName, value.getToken(), value.getText()); v = ""; } } diff --git a/tool/src/org/antlr/v4/tool/ast/GrammarRootAST.java b/tool/src/org/antlr/v4/tool/ast/GrammarRootAST.java index 95f5128..5392145 100644 --- a/tool/src/org/antlr/v4/tool/ast/GrammarRootAST.java +++ b/tool/src/org/antlr/v4/tool/ast/GrammarRootAST.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/tool/src/org/antlr/v4/tool/ast/NotAST.java b/tool/src/org/antlr/v4/tool/ast/NotAST.java index ecd1d5a..620ec6e 100644 --- a/tool/src/org/antlr/v4/tool/ast/NotAST.java +++ b/tool/src/org/antlr/v4/tool/ast/NotAST.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/tool/src/org/antlr/v4/tool/ast/OptionalBlockAST.java b/tool/src/org/antlr/v4/tool/ast/OptionalBlockAST.java index 3a05d89..1b99249 100644 --- a/tool/src/org/antlr/v4/tool/ast/OptionalBlockAST.java +++ b/tool/src/org/antlr/v4/tool/ast/OptionalBlockAST.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/tool/src/org/antlr/v4/tool/ast/PlusBlockAST.java b/tool/src/org/antlr/v4/tool/ast/PlusBlockAST.java index ad0ae26..78881f5 100644 --- a/tool/src/org/antlr/v4/tool/ast/PlusBlockAST.java +++ b/tool/src/org/antlr/v4/tool/ast/PlusBlockAST.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/tool/src/org/antlr/v4/tool/ast/PredAST.java b/tool/src/org/antlr/v4/tool/ast/PredAST.java index 8069916..6cfe9ca 100644 --- a/tool/src/org/antlr/v4/tool/ast/PredAST.java +++ b/tool/src/org/antlr/v4/tool/ast/PredAST.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/tool/src/org/antlr/v4/tool/ast/QuantifierAST.java b/tool/src/org/antlr/v4/tool/ast/QuantifierAST.java index acfa942..3abb879 100644 --- a/tool/src/org/antlr/v4/tool/ast/QuantifierAST.java +++ b/tool/src/org/antlr/v4/tool/ast/QuantifierAST.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/tool/src/org/antlr/v4/tool/ast/RangeAST.java b/tool/src/org/antlr/v4/tool/ast/RangeAST.java index e285ca6..4630e8b 100644 --- a/tool/src/org/antlr/v4/tool/ast/RangeAST.java +++ b/tool/src/org/antlr/v4/tool/ast/RangeAST.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/tool/src/org/antlr/v4/tool/ast/RuleAST.java b/tool/src/org/antlr/v4/tool/ast/RuleAST.java index 3600971..5041ad2 100644 --- a/tool/src/org/antlr/v4/tool/ast/RuleAST.java +++ b/tool/src/org/antlr/v4/tool/ast/RuleAST.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/tool/src/org/antlr/v4/tool/ast/RuleElementAST.java b/tool/src/org/antlr/v4/tool/ast/RuleElementAST.java index e355604..6815d48 100644 --- a/tool/src/org/antlr/v4/tool/ast/RuleElementAST.java +++ b/tool/src/org/antlr/v4/tool/ast/RuleElementAST.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/tool/src/org/antlr/v4/tool/ast/RuleRefAST.java b/tool/src/org/antlr/v4/tool/ast/RuleRefAST.java index be07e58..bdde62e 100644 --- a/tool/src/org/antlr/v4/tool/ast/RuleRefAST.java +++ b/tool/src/org/antlr/v4/tool/ast/RuleRefAST.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/tool/src/org/antlr/v4/tool/ast/SetAST.java b/tool/src/org/antlr/v4/tool/ast/SetAST.java index 53c4c36..6c1fa78 100644 --- a/tool/src/org/antlr/v4/tool/ast/SetAST.java +++ b/tool/src/org/antlr/v4/tool/ast/SetAST.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/tool/src/org/antlr/v4/tool/ast/StarBlockAST.java b/tool/src/org/antlr/v4/tool/ast/StarBlockAST.java index 20b8793..9db14bb 100644 --- a/tool/src/org/antlr/v4/tool/ast/StarBlockAST.java +++ b/tool/src/org/antlr/v4/tool/ast/StarBlockAST.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/tool/src/org/antlr/v4/tool/ast/TerminalAST.java b/tool/src/org/antlr/v4/tool/ast/TerminalAST.java index fcac32d..234dd85 100644 --- a/tool/src/org/antlr/v4/tool/ast/TerminalAST.java +++ b/tool/src/org/antlr/v4/tool/ast/TerminalAST.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/tool/src/org/antlr/v4/unicode/UnicodeDataTemplateController.java b/tool/src/org/antlr/v4/unicode/UnicodeDataTemplateController.java new file mode 100644 index 0000000..da244a3 --- /dev/null +++ b/tool/src/org/antlr/v4/unicode/UnicodeDataTemplateController.java @@ -0,0 +1,418 @@ +/* + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. + * Use of this file is governed by the BSD 3-clause license that + * can be found in the LICENSE.txt file in the project root. + */ + +package org.antlr.v4.unicode; + +import com.ibm.icu.lang.UCharacter; +import com.ibm.icu.lang.UCharacterCategory; +import com.ibm.icu.lang.UProperty; +import com.ibm.icu.lang.UScript; +import com.ibm.icu.text.UnicodeSet; +import com.ibm.icu.util.RangeValueIterator; + +import org.antlr.v4.runtime.misc.IntervalSet; + +import java.util.LinkedHashMap; +import java.util.Locale; +import java.util.Map; + +/** + * StringTemplate controller used to generate parameters to feed + * to {@code unicodedata.st} to code-generate {@code UnicodeData.java}, + * used by the tool for Unicode property escapes like {@code \\p\{Lu\}}. + * + * Uses ICU to iterate over Unicode character categories, properties, + * and script codes, as well as aliases for those codes. + * + * This class exists in its own Maven module to avoid adding a + * dependency from the tool onto the (large) ICU runtime. + */ +public abstract class UnicodeDataTemplateController { + private static void addIntervalForCategory( + Map<String, IntervalSet> categoryMap, + String categoryName, + int start, + int finish) { + IntervalSet intervalSet = categoryMap.get(categoryName); + if (intervalSet == null) { + intervalSet = new IntervalSet(); + categoryMap.put(categoryName, intervalSet); + } + intervalSet.add(start, finish); + } + + private static void addPropertyAliases( + Map<String, String> propertyAliases, + String propertyName, + int property) { + int nameChoice = UProperty.NameChoice.LONG; + while (true) { + String alias; + try { + alias = UCharacter.getPropertyName(property, nameChoice); + } catch (IllegalArgumentException e) { + // No more aliases. + break; + } + assert alias != null; + addPropertyAlias(propertyAliases, alias, propertyName); + nameChoice++; + } + } + + private static void addPropertyAlias( + Map<String, String> propertyAliases, + String alias, + String propertyName) { + propertyAliases.put(alias, propertyName); + } + + public static Map<String, Object> getProperties() { + Map<String, IntervalSet> propertyCodePointRanges = new LinkedHashMap<>(); + addUnicodeCategoryCodesToCodePointRanges(propertyCodePointRanges); + addUnicodeBinaryPropertyCodesToCodePointRanges(propertyCodePointRanges); + addUnicodeIntPropertyCodesToCodePointRanges(propertyCodePointRanges); + addTR35ExtendedPictographicPropertyCodesToCodePointRanges(propertyCodePointRanges); + addEmojiPresentationPropertyCodesToCodePointRanges(propertyCodePointRanges); + + Map<String, String> propertyAliases = new LinkedHashMap<>(); + addUnicodeCategoryCodesToNames(propertyAliases); + addUnicodeBinaryPropertyCodesToNames(propertyAliases); + addUnicodeScriptCodesToNames(propertyAliases); + addUnicodeBlocksToNames(propertyAliases); + addUnicodeIntPropertyCodesToNames(propertyAliases); + propertyAliases.put("EP", "Extended_Pictographic"); + + Map<String, Object> properties = new LinkedHashMap<>(); + properties.put("propertyCodePointRanges", propertyCodePointRanges); + properties.put("propertyAliases", propertyAliases); + return properties; + } + + private static String getShortPropertyName(int property) { + String propertyName = UCharacter.getPropertyName(property, UProperty.NameChoice.SHORT); + // For some reason, a few properties only have long names. + if (propertyName == null) { + propertyName = UCharacter.getPropertyName(property, UProperty.NameChoice.LONG); + } + return propertyName; + } + + private static void addUnicodeCategoryCodesToCodePointRanges(Map<String, IntervalSet> propertyCodePointRanges) { + RangeValueIterator iter = UCharacter.getTypeIterator(); + RangeValueIterator.Element element = new RangeValueIterator.Element(); + while (iter.next(element)) { + String categoryName = UCharacter.getPropertyValueName( + UProperty.GENERAL_CATEGORY_MASK, + 1 << element.value, + UProperty.NameChoice.SHORT); + addIntervalForCategory(propertyCodePointRanges, categoryName, element.start, element.limit - 1); + // Add short category so Ll, Lu, Lo, etc. all show up under L + String shortCategoryName = categoryName.substring(0, 1); + addIntervalForCategory(propertyCodePointRanges, shortCategoryName, element.start, element.limit - 1); + } + } + + private static void addUnicodeCategoryCodesToNames(Map<String, String> propertyAliases) { + RangeValueIterator iter = UCharacter.getTypeIterator(); + RangeValueIterator.Element element = new RangeValueIterator.Element(); + while (iter.next(element)) { + int generalCategoryMask = 1 << element.value; + String categoryName = UCharacter.getPropertyValueName( + UProperty.GENERAL_CATEGORY_MASK, + generalCategoryMask, + UProperty.NameChoice.SHORT); + int nameChoice = UProperty.NameChoice.LONG; + while (true) { + String alias; + try { + alias = UCharacter.getPropertyValueName( + UProperty.GENERAL_CATEGORY_MASK, + generalCategoryMask, + nameChoice); + } catch (IllegalArgumentException e) { + // No more aliases. + break; + } + assert alias != null; + addPropertyAlias(propertyAliases, alias, categoryName); + nameChoice++; + } + } + // Add short categories + addPropertyAlias(propertyAliases, "Control", "C"); + addPropertyAlias(propertyAliases, "Letter", "L"); + addPropertyAlias(propertyAliases, "Number", "N"); + addPropertyAlias(propertyAliases, "Mark", "M"); + addPropertyAlias(propertyAliases, "Punctuation", "P"); + addPropertyAlias(propertyAliases, "Symbol", "S"); + addPropertyAlias(propertyAliases, "Space", "Z"); + } + + private static void addUnicodeBinaryPropertyCodesToCodePointRanges(Map<String, IntervalSet> propertyCodePointRanges) { + for (int property = UProperty.BINARY_START; + property < UProperty.BINARY_LIMIT; + property++) { + String propertyName = getShortPropertyName(property); + IntervalSet intervalSet = new IntervalSet(); + UnicodeSet unicodeSet = new UnicodeSet(); + unicodeSet.applyIntPropertyValue(property, 1); + for (UnicodeSet.EntryRange range : unicodeSet.ranges()) { + intervalSet.add(range.codepoint, range.codepointEnd); + } + propertyCodePointRanges.put(propertyName, intervalSet); + } + } + + private static void addUnicodeBinaryPropertyCodesToNames(Map<String, String> propertyAliases) { + for (int property = UProperty.BINARY_START; + property < UProperty.BINARY_LIMIT; + property++) { + String propertyName = getShortPropertyName(property); + addPropertyAliases(propertyAliases, propertyName, property); + } + } + + private static void addIntPropertyRanges(int property, String namePrefix, Map<String, IntervalSet> propertyCodePointRanges) { + for (int propertyValue = UCharacter.getIntPropertyMinValue(property); + propertyValue <= UCharacter.getIntPropertyMaxValue(property); + propertyValue++) { + UnicodeSet set = new UnicodeSet(); + set.applyIntPropertyValue(property, propertyValue); + String propertyName = namePrefix + UCharacter.getPropertyValueName(property, propertyValue, UProperty.NameChoice.SHORT); + IntervalSet intervalSet = propertyCodePointRanges.get(propertyName); + if (intervalSet == null) { + intervalSet = new IntervalSet(); + propertyCodePointRanges.put(propertyName, intervalSet); + } + addUnicodeSetToIntervalSet(set, intervalSet); + } + } + + private static void addUnicodeSetToIntervalSet(UnicodeSet unicodeSet, IntervalSet intervalSet) { + for (UnicodeSet.EntryRange range : unicodeSet.ranges()) { + intervalSet.add(range.codepoint, range.codepointEnd); + } + } + + private static void addUnicodeIntPropertyCodesToCodePointRanges(Map<String, IntervalSet> propertyCodePointRanges) { + for (int property = UProperty.INT_START; + property < UProperty.INT_LIMIT; + property++) { + String propertyName = getShortPropertyName(property); + addIntPropertyRanges(property, propertyName + "=", propertyCodePointRanges); + } + } + + private static void addTR35ExtendedPictographicPropertyCodesToCodePointRanges(Map<String, IntervalSet> propertyCodePointRanges) { + IntervalSet set = new IntervalSet(); + // Generated using scripts/parse-extended-pictographic/parse.py + set.add(0x1F774, 0x1F77F); + set.add(0x2700, 0x2701); + set.add(0x2703, 0x2704); + set.add(0x270E); + set.add(0x2710, 0x2711); + set.add(0x2765, 0x2767); + set.add(0x1F030, 0x1F093); + set.add(0x1F094, 0x1F09F); + set.add(0x1F10D, 0x1F10F); + set.add(0x1F12F); + set.add(0x1F16C, 0x1F16F); + set.add(0x1F1AD, 0x1F1E5); + set.add(0x1F260, 0x1F265); + set.add(0x1F203, 0x1F20F); + set.add(0x1F23C, 0x1F23F); + set.add(0x1F249, 0x1F24F); + set.add(0x1F252, 0x1F25F); + set.add(0x1F266, 0x1F2FF); + set.add(0x1F7D5, 0x1F7FF); + set.add(0x1F000, 0x1F003); + set.add(0x1F005, 0x1F02B); + set.add(0x1F02C, 0x1F02F); + set.add(0x1F322, 0x1F323); + set.add(0x1F394, 0x1F395); + set.add(0x1F398); + set.add(0x1F39C, 0x1F39D); + set.add(0x1F3F1, 0x1F3F2); + set.add(0x1F3F6); + set.add(0x1F4FE); + set.add(0x1F53E, 0x1F548); + set.add(0x1F54F); + set.add(0x1F568, 0x1F56E); + set.add(0x1F571, 0x1F572); + set.add(0x1F57B, 0x1F586); + set.add(0x1F588, 0x1F589); + set.add(0x1F58E, 0x1F58F); + set.add(0x1F591, 0x1F594); + set.add(0x1F597, 0x1F5A3); + set.add(0x1F5A6, 0x1F5A7); + set.add(0x1F5A9, 0x1F5B0); + set.add(0x1F5B3, 0x1F5BB); + set.add(0x1F5BD, 0x1F5C1); + set.add(0x1F5C5, 0x1F5D0); + set.add(0x1F5D4, 0x1F5DB); + set.add(0x1F5DF, 0x1F5E0); + set.add(0x1F5E2); + set.add(0x1F5E4, 0x1F5E7); + set.add(0x1F5E9, 0x1F5EE); + set.add(0x1F5F0, 0x1F5F2); + set.add(0x1F5F4, 0x1F5F9); + set.add(0x2605); + set.add(0x2607, 0x260D); + set.add(0x260F, 0x2610); + set.add(0x2612); + set.add(0x2616, 0x2617); + set.add(0x2619, 0x261C); + set.add(0x261E, 0x261F); + set.add(0x2621); + set.add(0x2624, 0x2625); + set.add(0x2627, 0x2629); + set.add(0x262B, 0x262D); + set.add(0x2630, 0x2637); + set.add(0x263B, 0x2647); + set.add(0x2654, 0x265F); + set.add(0x2661, 0x2662); + set.add(0x2664); + set.add(0x2667); + set.add(0x2669, 0x267A); + set.add(0x267C, 0x267E); + set.add(0x2680, 0x2691); + set.add(0x2695); + set.add(0x2698); + set.add(0x269A); + set.add(0x269D, 0x269F); + set.add(0x26A2, 0x26A9); + set.add(0x26AC, 0x26AF); + set.add(0x26B2, 0x26BC); + set.add(0x26BF, 0x26C3); + set.add(0x26C6, 0x26C7); + set.add(0x26C9, 0x26CD); + set.add(0x26D0); + set.add(0x26D2); + set.add(0x26D5, 0x26E8); + set.add(0x26EB, 0x26EF); + set.add(0x26F6); + set.add(0x26FB, 0x26FC); + set.add(0x26FE, 0x26FF); + set.add(0x2388); + set.add(0x1FA00, 0x1FFFD); + set.add(0x1F0A0, 0x1F0AE); + set.add(0x1F0B1, 0x1F0BF); + set.add(0x1F0C1, 0x1F0CF); + set.add(0x1F0D1, 0x1F0F5); + set.add(0x1F0AF, 0x1F0B0); + set.add(0x1F0C0); + set.add(0x1F0D0); + set.add(0x1F0F6, 0x1F0FF); + set.add(0x1F80C, 0x1F80F); + set.add(0x1F848, 0x1F84F); + set.add(0x1F85A, 0x1F85F); + set.add(0x1F888, 0x1F88F); + set.add(0x1F8AE, 0x1F8FF); + set.add(0x1F900, 0x1F90B); + set.add(0x1F91F); + set.add(0x1F928, 0x1F92F); + set.add(0x1F931, 0x1F932); + set.add(0x1F94C); + set.add(0x1F95F, 0x1F96B); + set.add(0x1F992, 0x1F997); + set.add(0x1F9D0, 0x1F9E6); + set.add(0x1F90C, 0x1F90F); + set.add(0x1F93F); + set.add(0x1F94D, 0x1F94F); + set.add(0x1F96C, 0x1F97F); + set.add(0x1F998, 0x1F9BF); + set.add(0x1F9C1, 0x1F9CF); + set.add(0x1F9E7, 0x1F9FF); + set.add(0x1F6C6, 0x1F6CA); + set.add(0x1F6D3, 0x1F6D4); + set.add(0x1F6E6, 0x1F6E8); + set.add(0x1F6EA); + set.add(0x1F6F1, 0x1F6F2); + set.add(0x1F6F7, 0x1F6F8); + set.add(0x1F6D5, 0x1F6DF); + set.add(0x1F6ED, 0x1F6EF); + set.add(0x1F6F9, 0x1F6FF); + propertyCodePointRanges.put("Extended_Pictographic", set); + + UnicodeSet emojiRKUnicodeSet = new UnicodeSet("[\\p{GCB=Regional_Indicator}\\*#0-9\\u00a9\\u00ae\\u2122\\u3030\\u303d]"); + IntervalSet emojiRKIntervalSet = new IntervalSet(); + addUnicodeSetToIntervalSet(emojiRKUnicodeSet, emojiRKIntervalSet); + propertyCodePointRanges.put("EmojiRK", emojiRKIntervalSet); + + UnicodeSet emojiNRKUnicodeSet = new UnicodeSet("[\\p{Emoji=Yes}]"); + emojiNRKUnicodeSet.removeAll(emojiRKUnicodeSet); + IntervalSet emojiNRKIntervalSet = new IntervalSet(); + addUnicodeSetToIntervalSet(emojiNRKUnicodeSet, emojiNRKIntervalSet); + propertyCodePointRanges.put("EmojiNRK", emojiNRKIntervalSet); + } + + private static void addEmojiPresentationPropertyCodesToCodePointRanges(Map<String, IntervalSet> propertyCodePointRanges) { + UnicodeSet emojiDefaultUnicodeSet = new UnicodeSet("[[\\p{Emoji=Yes}]&[\\p{Emoji_Presentation=Yes}]]"); + IntervalSet emojiDefaultIntervalSet = new IntervalSet(); + addUnicodeSetToIntervalSet(emojiDefaultUnicodeSet, emojiDefaultIntervalSet); + propertyCodePointRanges.put("EmojiPresentation=EmojiDefault", emojiDefaultIntervalSet); + + UnicodeSet textDefaultUnicodeSet = new UnicodeSet("[[\\p{Emoji=Yes}]&[\\p{Emoji_Presentation=No}]]"); + IntervalSet textDefaultIntervalSet = new IntervalSet(); + addUnicodeSetToIntervalSet(textDefaultUnicodeSet, textDefaultIntervalSet); + propertyCodePointRanges.put("EmojiPresentation=TextDefault", textDefaultIntervalSet); + + UnicodeSet textUnicodeSet = new UnicodeSet("[\\p{Emoji=No}]"); + IntervalSet textIntervalSet = new IntervalSet(); + addUnicodeSetToIntervalSet(textUnicodeSet, textIntervalSet); + propertyCodePointRanges.put("EmojiPresentation=Text", textIntervalSet); + } + + private static void addIntPropertyAliases(int property, String namePrefix, Map<String, String> propertyAliases) { + String propertyName = getShortPropertyName(property); + for (int propertyValue = UCharacter.getIntPropertyMinValue(property); + propertyValue <= UCharacter.getIntPropertyMaxValue(property); + propertyValue++) { + String aliasTarget = propertyName + "=" + UCharacter.getPropertyValueName(property, propertyValue, UProperty.NameChoice.SHORT); + int nameChoice = UProperty.NameChoice.SHORT; + String alias; + while (true) { + try { + alias = namePrefix + UCharacter.getPropertyValueName(property, propertyValue, nameChoice); + } catch (IllegalArgumentException e) { + // No more aliases. + break; + } + assert alias != null; + addPropertyAlias(propertyAliases, alias, aliasTarget); + nameChoice++; + } + } + } + + private static void addUnicodeScriptCodesToNames(Map<String, String> propertyAliases) { + addIntPropertyAliases(UProperty.SCRIPT, "", propertyAliases); + } + + private static void addUnicodeBlocksToNames(Map<String, String> propertyAliases) { + addIntPropertyAliases(UProperty.BLOCK, "In", propertyAliases); + } + + private static void addUnicodeIntPropertyCodesToNames(Map<String, String> propertyAliases) { + for (int property = UProperty.INT_START; + property < UProperty.INT_LIMIT; + property++) { + int nameChoice = UProperty.NameChoice.SHORT + 1; + while (true) { + String propertyNameAlias; + try { + propertyNameAlias = UCharacter.getPropertyName(property, nameChoice); + } catch (IllegalArgumentException e) { + // No more aliases. + break; + } + addIntPropertyAliases(property, propertyNameAlias + "=", propertyAliases); + nameChoice++; + } + } + } +} |