diff options
| author | Andrius Merkys <andrius.merkys@gmail.com> | 2018-11-23 04:52:08 -0500 |
|---|---|---|
| committer | Andrius Merkys <andrius.merkys@gmail.com> | 2018-11-23 04:52:08 -0500 |
| commit | 7b19e9be5be41c69c451b63c526bee059881f9b1 (patch) | |
| tree | 699bf0523df6868d15843981ea9914ac096ee270 | |
| parent | 1d0464db4ec5e5c20b2ae62bb3c4eceaa6840bde (diff) | |
New upstream version 4.7.1
524 files changed, 8684 insertions, 2239 deletions
diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md deleted file mode 100644 index 61171f1..0000000 --- a/.github/ISSUE_TEMPLATE.md +++ /dev/null @@ -1,8 +0,0 @@ -Before submitting an issue to ANTLR, please check off these boxes: - -- [ ] I am not submitting a question on how to use ANTLR; instead, go to [antlr4-discussion google group](https://groups.google.com/forum/#!forum/antlr-discussion) or ask at [stackoverflow](http://stackoverflow.com/questions/tagged/antlr4) -- [ ] I have done a search of the existing issues to make sure I'm not sending in a duplicate - -[cut] -Please include information about the expected behavior, actual behavior, and the smallest grammar or code that reproduces the behavior. If appropriate, please indicate the code generation targets such as Java, C#, ... Pointers into offending code regions are also very welcome. -[/cut] diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md deleted file mode 100644 index e2af226..0000000 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ /dev/null @@ -1,5 +0,0 @@ -[cut] -Thank you for proposing a contribution to the ANTLR project. In order to accept changes from the outside world, all contributors must "sign" the [contributors.txt](https://github.com/antlr/antlr4/blob/master/contributors.txt) contributors certificate of origin. It's an unfortunate reality of today's fuzzy and bizarre world of open-source ownership. - -Make sure you are already in the contributors.txt file or add a commit to this pull request with the appropriate change. Thanks! -[/cut] diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 0000000..989acfe --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,10 @@ +# Contributing to ANTLR 4 + +1. [Fork](https://help.github.com/articles/fork-a-repo) the [antlr/antlr4 repo](https://github.com/antlr/antlr4) +2. Install and configure [EditorConfig](http://editorconfig.org/) so your text editor or IDE uses the ANTLR 4 coding style +3. [Build ANTLR 4](doc/building-antlr.md) +4. [Run the ANTLR project unit tests](doc/antlr-project-testing.md) +5. Create a [pull request](https://help.github.com/articles/using-pull-requests/) including your change + + +**Note:** You must sign the `contributors.txt` certificate of origin with your pull request if you've not done so before. diff --git a/LICENSE.txt b/LICENSE.txt index 5496b67..2042d1b 100644 --- a/LICENSE.txt +++ b/LICENSE.txt @@ -1,5 +1,5 @@ [The "BSD 3-clause license"] -Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. +Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions @@ -24,3 +24,29 @@ DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +===== + +MIT License for codepointat.js from https://git.io/codepointat +MIT License for fromcodepoint.js from https://git.io/vDW1m + +Copyright Mathias Bynens <https://mathiasbynens.be/> + +Permission is hereby granted, free of charge, to any person obtaining +a copy of this software and associated documentation files (the +"Software"), to deal in the Software without restriction, including +without limitation the rights to use, copy, modify, merge, publish, +distribute, sublicense, and/or sell copies of the Software, and to +permit persons to whom the Software is furnished to do so, subject to +the following conditions: + +The above copyright notice and this permission notice shall be +included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. @@ -21,6 +21,7 @@ ANTLR project lead and supreme dictator for life * David Sisson (C++ initial target and test) * [Janyou](https://github.com/janyou) (Swift target) * [Ewan Mellor](https://github.com/ewanmellor), [Hanzhou Shi](https://github.com/hanjoes) (Swift target merging) +* [Ben Hamilton](https://github.com/bhamiltoncx) (Full Unicode support in serialized ATN and all languages' runtimes for code points > U+FFFF) ## Useful information diff --git a/antlr4-maven-plugin/pom.xml b/antlr4-maven-plugin/pom.xml index 319418a..c148583 100644 --- a/antlr4-maven-plugin/pom.xml +++ b/antlr4-maven-plugin/pom.xml @@ -1,5 +1,5 @@ <!-- - ~ Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + ~ Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. ~ Use of this file is governed by the BSD 3-clause license that ~ can be found in the LICENSE.txt file in the project root. --> @@ -8,7 +8,7 @@ <parent> <groupId>org.antlr</groupId> <artifactId>antlr4-master</artifactId> - <version>4.6</version> + <version>4.7.1</version> </parent> <artifactId>antlr4-maven-plugin</artifactId> <packaging>maven-plugin</packaging> diff --git a/antlr4-maven-plugin/resources/META-INF/m2e/lifecycle-mapping-metadata.xml b/antlr4-maven-plugin/resources/META-INF/m2e/lifecycle-mapping-metadata.xml index 84a4919..e4ea6e7 100644 --- a/antlr4-maven-plugin/resources/META-INF/m2e/lifecycle-mapping-metadata.xml +++ b/antlr4-maven-plugin/resources/META-INF/m2e/lifecycle-mapping-metadata.xml @@ -1,6 +1,6 @@ <?xml version="1.0" encoding="utf-8"?> <!-- - ~ Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + ~ Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. ~ Use of this file is governed by the BSD 3-clause license that ~ can be found in the LICENSE.txt file in the project root. --> diff --git a/antlr4-maven-plugin/src/main/java/org/antlr/mojo/antlr4/Antlr4ErrorLog.java b/antlr4-maven-plugin/src/main/java/org/antlr/mojo/antlr4/Antlr4ErrorLog.java index 92d2377..d8196a8 100644 --- a/antlr4-maven-plugin/src/main/java/org/antlr/mojo/antlr4/Antlr4ErrorLog.java +++ b/antlr4-maven-plugin/src/main/java/org/antlr/mojo/antlr4/Antlr4ErrorLog.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/antlr4-maven-plugin/src/main/java/org/antlr/mojo/antlr4/Antlr4Mojo.java b/antlr4-maven-plugin/src/main/java/org/antlr/mojo/antlr4/Antlr4Mojo.java index 0afe54f..c0926fe 100644 --- a/antlr4-maven-plugin/src/main/java/org/antlr/mojo/antlr4/Antlr4Mojo.java +++ b/antlr4-maven-plugin/src/main/java/org/antlr/mojo/antlr4/Antlr4Mojo.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ @@ -150,6 +150,13 @@ public class Antlr4Mojo extends AbstractMojo { @Parameter(property = "project", required = true, readonly = true) protected MavenProject project; + /** + * Specifies whether sources are added to the {@code compile} or + * {@code test} scope. + */ + @Parameter(property = "antlr4.generateTestSources", defaultValue = "false") + private boolean generateTestSources; + /** * The directory where the ANTLR grammar files ({@code *.g4}) are located. */ @@ -190,7 +197,12 @@ public class Antlr4Mojo extends AbstractMojo { } void addSourceRoot(File outputDir) { - project.addCompileSourceRoot(outputDir.getPath()); + if (generateTestSources) { + project.addTestCompileSourceRoot(outputDir.getPath()); + } + else { + project.addCompileSourceRoot(outputDir.getPath()); + } } /** @@ -383,7 +395,7 @@ public class Antlr4Mojo extends AbstractMojo { String tokensFileName = grammarFile.getName().split("\\.")[0] + ".tokens"; File outputFile = new File(outputDirectory, tokensFileName); if ( (! outputFile.exists()) || - outputFile.lastModified() < grammarFile.lastModified() || + outputFile.lastModified() <= grammarFile.lastModified() || dependencies.isDependencyChanged(grammarFile)) { grammarFilesToProcess.add(grammarFile); } @@ -400,10 +412,7 @@ public class Antlr4Mojo extends AbstractMojo { // Iterate each grammar file we were given and add it into the tool's list of // grammars to process. for (File grammarFile : grammarFiles) { - if (!buildContext.hasDelta(grammarFile)) { - continue; - } - + buildContext.refresh(grammarFile); buildContext.removeMessages(grammarFile); getLog().debug("Grammar file '" + grammarFile.getPath() + "' detected."); diff --git a/antlr4-maven-plugin/src/main/java/org/antlr/mojo/antlr4/GrammarDependencies.java b/antlr4-maven-plugin/src/main/java/org/antlr/mojo/antlr4/GrammarDependencies.java index f8ebb1e..d21d1ab 100644 --- a/antlr4-maven-plugin/src/main/java/org/antlr/mojo/antlr4/GrammarDependencies.java +++ b/antlr4-maven-plugin/src/main/java/org/antlr/mojo/antlr4/GrammarDependencies.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ @@ -216,14 +216,14 @@ class GrammarDependencies { return; for (GrammarAST importDecl : grammar.getAllChildrenWithType(ANTLRParser.IMPORT)) { - Tree id = importDecl.getFirstChildWithType(ANTLRParser.ID); + for (Tree id: importDecl.getAllChildrenWithType(ANTLRParser.ID)) { + // missing id is not valid, but we don't want to prevent the root cause from + // being reported by the ANTLR tool + if (id != null) { + String grammarPath = getRelativePath(grammarFile); - // missing id is not valid, but we don't want to prevent the root cause from - // being reported by the ANTLR tool - if (id != null) { - String grammarPath = getRelativePath(grammarFile); - - graph.addEdge(id.getText() + ".g4", grammarPath); + graph.addEdge(id.getText() + ".g4", grammarPath); + } } } diff --git a/antlr4-maven-plugin/src/main/java/org/antlr/mojo/antlr4/MojoUtils.java b/antlr4-maven-plugin/src/main/java/org/antlr/mojo/antlr4/MojoUtils.java index 56da59b..116055e 100644 --- a/antlr4-maven-plugin/src/main/java/org/antlr/mojo/antlr4/MojoUtils.java +++ b/antlr4-maven-plugin/src/main/java/org/antlr/mojo/antlr4/MojoUtils.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/antlr4-maven-plugin/src/site/site.xml b/antlr4-maven-plugin/src/site/site.xml index 23cd29c..609b0af 100644 --- a/antlr4-maven-plugin/src/site/site.xml +++ b/antlr4-maven-plugin/src/site/site.xml @@ -1,7 +1,7 @@ <?xml version="1.0" encoding="UTF-8"?> <!-- - ~ Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + ~ Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. ~ Use of this file is governed by the BSD 3-clause license that ~ can be found in the LICENSE.txt file in the project root. --> diff --git a/antlr4-maven-plugin/src/test/java/org/antlr/mojo/antlr4/Antlr4MojoTest.java b/antlr4-maven-plugin/src/test/java/org/antlr/mojo/antlr4/Antlr4MojoTest.java index 12c45cd..da38c58 100644 --- a/antlr4-maven-plugin/src/test/java/org/antlr/mojo/antlr4/Antlr4MojoTest.java +++ b/antlr4-maven-plugin/src/test/java/org/antlr/mojo/antlr4/Antlr4MojoTest.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ @@ -202,6 +202,7 @@ public class Antlr4MojoTest { Path genHello = generatedSources.resolve("test/HelloParser.java"); Path baseGrammar = antlrDir.resolve("imports/TestBaseLexer.g4"); + Path baseGrammar2 = antlrDir.resolve("imports/TestBaseLexer2.g4"); Path lexerGrammar = antlrDir.resolve("test/TestLexer.g4"); Path parserGrammar = antlrDir.resolve("test/TestParser.g4"); @@ -222,21 +223,20 @@ public class Antlr4MojoTest { assertTrue(Files.exists(genHello)); assertTrue(Files.exists(genTestParser)); assertTrue(Files.exists(genTestLexer)); + byte[] origTestLexerSum = checksum(genTestLexer); + byte[] origTestParserSum = checksum(genTestParser); + byte[] origHelloSum = checksum(genHello); //////////////////////////////////////////////////////////////////////// // 2nd - nothing has been modified, no grammars have to be processed //////////////////////////////////////////////////////////////////////// { - byte[] testLexerSum = checksum(genTestLexer); - byte[] testParserSum = checksum(genTestParser); - byte[] helloSum = checksum(genHello); - maven.executeMojo(session, project, exec); - assertTrue(Arrays.equals(testLexerSum, checksum(genTestLexer))); - assertTrue(Arrays.equals(testParserSum, checksum(genTestParser))); - assertTrue(Arrays.equals(helloSum, checksum(genHello))); + assertTrue(Arrays.equals(origTestLexerSum, checksum(genTestLexer))); + assertTrue(Arrays.equals(origTestParserSum, checksum(genTestParser))); + assertTrue(Arrays.equals(origHelloSum, checksum(genHello))); } //////////////////////////////////////////////////////////////////////// @@ -245,50 +245,71 @@ public class Antlr4MojoTest { // modify the grammar to make checksum comparison detect a change try(Change change = Change.of(baseGrammar, "DOT: '.' ;")) { - byte[] testLexerSum = checksum(genTestLexer); - byte[] testParserSum = checksum(genTestParser); - byte[] helloSum = checksum(genHello); - maven.executeMojo(session, project, exec); - assertFalse(Arrays.equals(testLexerSum, checksum(genTestLexer))); - assertFalse(Arrays.equals(testParserSum, checksum(genTestParser))); - assertTrue(Arrays.equals(helloSum, checksum(genHello))); + assertFalse(Arrays.equals(origTestLexerSum, checksum(genTestLexer))); + assertFalse(Arrays.equals(origTestParserSum, checksum(genTestParser))); + assertTrue(Arrays.equals(origHelloSum, checksum(genHello))); } + // Restore file and confirm it was restored. + maven.executeMojo(session, project, exec); + assertTrue(Arrays.equals(origTestLexerSum, checksum(genTestLexer))); + assertTrue(Arrays.equals(origTestParserSum, checksum(genTestParser))); + assertTrue(Arrays.equals(origHelloSum, checksum(genHello))); //////////////////////////////////////////////////////////////////////// - // 4th - the lexer grammar changed, the parser grammar has to be processed as well + // 4th - the second imported grammar changed, every dependency has to be processed //////////////////////////////////////////////////////////////////////// // modify the grammar to make checksum comparison detect a change - try(Change change = Change.of(lexerGrammar)) { - byte[] testLexerSum = checksum(genTestLexer); - byte[] testParserSum = checksum(genTestParser); - byte[] helloSum = checksum(genHello); + try(Change change = Change.of(baseGrammar2, "BANG: '!' ;")) { + maven.executeMojo(session, project, exec); + assertFalse(Arrays.equals(origTestLexerSum, checksum(genTestLexer))); + assertFalse(Arrays.equals(origTestParserSum, checksum(genTestParser))); + assertTrue(Arrays.equals(origHelloSum, checksum(genHello))); + } + // Restore file and confirm it was restored. + maven.executeMojo(session, project, exec); + assertTrue(Arrays.equals(origTestLexerSum, checksum(genTestLexer))); + assertTrue(Arrays.equals(origTestParserSum, checksum(genTestParser))); + assertTrue(Arrays.equals(origHelloSum, checksum(genHello))); + + //////////////////////////////////////////////////////////////////////// + // 5th - the lexer grammar changed, the parser grammar has to be processed as well + //////////////////////////////////////////////////////////////////////// + + // modify the grammar to make checksum comparison detect a change + try(Change change = Change.of(lexerGrammar, "FOO: 'foo' ;")) { maven.executeMojo(session, project, exec); - assertFalse(Arrays.equals(testLexerSum, checksum(genTestLexer))); - assertFalse(Arrays.equals(testParserSum, checksum(genTestParser))); - assertTrue(Arrays.equals(helloSum, checksum(genHello))); + assertFalse(Arrays.equals(origTestLexerSum, checksum(genTestLexer))); + assertFalse(Arrays.equals(origTestParserSum, checksum(genTestParser))); + assertTrue(Arrays.equals(origHelloSum, checksum(genHello))); } + // Restore file and confirm it was restored. + maven.executeMojo(session, project, exec); + assertTrue(Arrays.equals(origTestLexerSum, checksum(genTestLexer))); + assertTrue(Arrays.equals(origTestParserSum, checksum(genTestParser))); + assertTrue(Arrays.equals(origHelloSum, checksum(genHello))); //////////////////////////////////////////////////////////////////////// - // 5th - the parser grammar changed, no other grammars have to be processed + // 6th - the parser grammar changed, no other grammars have to be processed //////////////////////////////////////////////////////////////////////// // modify the grammar to make checksum comparison detect a change try(Change change = Change.of(parserGrammar, " t : WS* ;")) { - byte[] testLexerSum = checksum(genTestLexer); - byte[] testParserSum = checksum(genTestParser); - byte[] helloSum = checksum(genHello); - maven.executeMojo(session, project, exec); - assertTrue(Arrays.equals(testLexerSum, checksum(genTestLexer))); - assertFalse(Arrays.equals(testParserSum, checksum(genTestParser))); - assertTrue(Arrays.equals(helloSum, checksum(genHello))); + assertTrue(Arrays.equals(origTestLexerSum, checksum(genTestLexer))); + assertFalse(Arrays.equals(origTestParserSum, checksum(genTestParser))); + assertTrue(Arrays.equals(origHelloSum, checksum(genHello))); } + // Restore file and confirm it was restored. + maven.executeMojo(session, project, exec); + assertTrue(Arrays.equals(origTestLexerSum, checksum(genTestLexer))); + assertTrue(Arrays.equals(origTestParserSum, checksum(genTestParser))); + assertTrue(Arrays.equals(origHelloSum, checksum(genHello))); } @Test diff --git a/antlr4-maven-plugin/src/test/projects/dependencyRemoved/pom.xml b/antlr4-maven-plugin/src/test/projects/dependencyRemoved/pom.xml index 3163b81..35be721 100644 --- a/antlr4-maven-plugin/src/test/projects/dependencyRemoved/pom.xml +++ b/antlr4-maven-plugin/src/test/projects/dependencyRemoved/pom.xml @@ -1,5 +1,5 @@ <!-- - ~ Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + ~ Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. ~ Use of this file is governed by the BSD 3-clause license that ~ can be found in the LICENSE.txt file in the project root. --> diff --git a/antlr4-maven-plugin/src/test/projects/importTokens/pom.xml b/antlr4-maven-plugin/src/test/projects/importTokens/pom.xml index 478b517..7f8c657 100644 --- a/antlr4-maven-plugin/src/test/projects/importTokens/pom.xml +++ b/antlr4-maven-plugin/src/test/projects/importTokens/pom.xml @@ -1,5 +1,5 @@ <!-- - ~ Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + ~ Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. ~ Use of this file is governed by the BSD 3-clause license that ~ can be found in the LICENSE.txt file in the project root. --> diff --git a/antlr4-maven-plugin/src/test/projects/importsCustom/pom.xml b/antlr4-maven-plugin/src/test/projects/importsCustom/pom.xml index a5826bc..4ab0de8 100644 --- a/antlr4-maven-plugin/src/test/projects/importsCustom/pom.xml +++ b/antlr4-maven-plugin/src/test/projects/importsCustom/pom.xml @@ -1,5 +1,5 @@ <!-- - ~ Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + ~ Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. ~ Use of this file is governed by the BSD 3-clause license that ~ can be found in the LICENSE.txt file in the project root. --> diff --git a/antlr4-maven-plugin/src/test/projects/importsStandard/pom.xml b/antlr4-maven-plugin/src/test/projects/importsStandard/pom.xml index 77f63a2..77dcf00 100644 --- a/antlr4-maven-plugin/src/test/projects/importsStandard/pom.xml +++ b/antlr4-maven-plugin/src/test/projects/importsStandard/pom.xml @@ -1,5 +1,5 @@ <!-- - ~ Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + ~ Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. ~ Use of this file is governed by the BSD 3-clause license that ~ can be found in the LICENSE.txt file in the project root. --> diff --git a/antlr4-maven-plugin/src/test/projects/importsStandard/src/main/antlr4/imports/TestBaseLexer.g4 b/antlr4-maven-plugin/src/test/projects/importsStandard/src/main/antlr4/imports/TestBaseLexer.g4 index 5fcc6d3..6c3164d 100644 --- a/antlr4-maven-plugin/src/test/projects/importsStandard/src/main/antlr4/imports/TestBaseLexer.g4 +++ b/antlr4-maven-plugin/src/test/projects/importsStandard/src/main/antlr4/imports/TestBaseLexer.g4 @@ -10,7 +10,4 @@ fragment Whitespace : ' ' | '\n' | '\t' | '\r' ; fragment -Hexdigit : [a-fA-F0-9] ; - -fragment -Digit : [0-9] ; +Hexdigit : [a-fA-F0-9] ;
\ No newline at end of file diff --git a/antlr4-maven-plugin/src/test/projects/importsStandard/src/main/antlr4/imports/TestBaseLexer2.g4 b/antlr4-maven-plugin/src/test/projects/importsStandard/src/main/antlr4/imports/TestBaseLexer2.g4 new file mode 100644 index 0000000..18aa0c4 --- /dev/null +++ b/antlr4-maven-plugin/src/test/projects/importsStandard/src/main/antlr4/imports/TestBaseLexer2.g4 @@ -0,0 +1,4 @@ +lexer grammar TestBaseLexer2; + +fragment +Digit : [0-9] ; diff --git a/antlr4-maven-plugin/src/test/projects/importsStandard/src/main/antlr4/test/TestLexer.g4 b/antlr4-maven-plugin/src/test/projects/importsStandard/src/main/antlr4/test/TestLexer.g4 index 668b764..b9c07b3 100644 --- a/antlr4-maven-plugin/src/test/projects/importsStandard/src/main/antlr4/test/TestLexer.g4 +++ b/antlr4-maven-plugin/src/test/projects/importsStandard/src/main/antlr4/test/TestLexer.g4 @@ -1,6 +1,6 @@ lexer grammar TestLexer; -import TestBaseLexer; +import TestBaseLexer, TestBaseLexer2; WS : Whitespace+ -> skip; -TEXT : ~[<&]+ ; // match any 16 bit char other than < and &
\ No newline at end of file +TEXT : ~[<&]+ ; // match any 16 bit char other than < and & diff --git a/appveyor.yml b/appveyor.yml index 5718455..266eeeb 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -1,8 +1,17 @@ -version: '4.6-SNAPSHOT+AppVeyor.{build}' -os: Windows Server 2012 +version: '4.7.1-SNAPSHOT+AppVeyor.{build}' +cache: + - '%USERPROFILE%\.m2' + - '%USERPROFILE%\.nuget\packages -> **\project.json' +image: Visual Studio 2017 +build: off build_script: - - mvn -DskipTests install -q --batch-mode + - mvn -DskipTests install --batch-mode + - msbuild /target:restore /target:rebuild /property:Configuration=Release /logger:"C:\Program Files\AppVeyor\BuildAgent\Appveyor.MSBuildLogger.dll" /verbosity:detailed runtime/CSharp/runtime/CSharp/Antlr4.dotnet.sln + - msbuild ./runtime-testsuite/target/classes/CSharp/runtime/CSharp/Antlr4.vs2013.sln /logger:"C:\Program Files\AppVeyor\BuildAgent\Appveyor.MSBuildLogger.dll" /verbosity:detailed +after_build: + - msbuild /target:pack /property:Configuration=Release /verbosity:detailed runtime/CSharp/runtime/CSharp/Antlr4.dotnet.sln test_script: - - mvn install -q -Dantlr-python2-python="C:\Python27\python.exe" -Dantlr-python3-python="C:\Python35\python.exe" -Dantlr-javascript-nodejs="C:\Program Files (x86)\nodejs\node.exe" --batch-mode -build: - verbosity: minimal + - mvn install -Dantlr-python2-python="C:\Python27\python.exe" -Dantlr-python3-python="C:\Python35\python.exe" -Dantlr-javascript-nodejs="C:\Program Files (x86)\nodejs\node.exe" --batch-mode +artifacts: +- path: 'runtime\**\*.nupkg' + name: NuGet
\ No newline at end of file diff --git a/contributors.txt b/contributors.txt index db7d4df..b85abf0 100644 --- a/contributors.txt +++ b/contributors.txt @@ -1,4 +1,4 @@ -ANTLR Project Contributors Certification of Origin and Rights +ANTLR Project Contributors Certification of Origin and Rights All contributors to ANTLR v4 must formally agree to abide by this certificate of origin by signing on the bottom with their github @@ -127,3 +127,54 @@ YYYY/MM/DD, github id, Full name, email 2016/12/01, samtatasurya, Samuel Tatasurya, xemradiant@gmail.com 2016/12/03, redxdev, Samuel Bloomberg, sam@redxdev.com 2016/12/11, Gaulouis, Gaulouis, gaulouis.com@gmail.com +2016/12/22, akosthekiss, Akos Kiss, akiss@inf.u-szeged.hu +2016/12/24, adrpo, Adrian Pop, adrian.pop@liu.se +2017/01/11, robertbrignull, Robert Brignull, robertbrignull@gmail.com +2017/01/13, marcelo-rocha, Marcelo Rocha, mcrocha@gmail.com +2017/01/23, bhamiltoncx, Ben Hamilton, bhamiltoncx+antlr@gmail.com +2017/01/18, mshockwave, Bekket McClane, yihshyng223@gmail.com +2017/02/10, lionelplessis, Lionel Plessis, lionelplessis@users.noreply.github.com +2017/02/14, lecode-official, David Neumann, david.neumann@lecode.de +2017/02/14, xied75, Dong Xie, xied75@gmail.com +2017/02/20, Thomasb81, Thomas Burg, thomasb81@gmail.com +2017/02/26, jvasileff, John Vasileff, john@vasileff.com +2017/03/08, harry-tallbelt, Igor Vysokopoyasny, harry.tallbelt@gmail.com +2017/03/09, teverett, Tom Everett, tom@khubla.com +2017/03/03, chund, Christian Hund, christian.hund@gmail.com +2017/03/15, robertvanderhulst, Robert van der Hulst, robert@xsharp.eu +2017/03/28, cmd-johnson, Jonas Auer, jonas.auer.94@gmail.com +2017/04/12, lys0716, Yishuang Lu, luyscmu@gmail.com +2017/04/30, shravanrn, Shravan Narayan, shravanrn@gmail.com +2017/05/11, jimallman, Jim Allman, jim@ibang.com +2017/05/26, waf, Will Fuqua, wafuqua@gmail.com +2017/05/29, kosak, Corey Kosak, kosak@kosak.com +2017/06/11, erikbra, Erik A. Brandstadmoen, erik@brandstadmoen.net +2017/06/10, jm-mikkelsen, Jan Martin Mikkelsen, janm@transactionware.com +2017/06/25, alimg, Alim Gökkaya, alim.gokkaya@gmail.com +2017/06/28, jBugman, Sergey Parshukov, codedby@bugman.me +2017/07/09, neatnerd, Mike Arshinskiy, neatnerd@users.noreply.github.com +2017/07/11, dhalperi, Daniel Halperin, daniel@halper.in +2017/07/17, vaibhavaingankar09, Vaibhav Vaingankar, vbhvvaingankar9@gmail.com +2017/07/23, venkatperi, Venkat Peri, venkatperi@gmail.com +2017/07/27, shirou, WAKAYAMA Shirou, shirou.faw@gmail.com +2017/07/09, neatnerd, Mike Arshinskiy, neatnerd@users.noreply.github.com +2017/07/27, matthauck, Matt Hauck, matthauck@gmail.com +2017/07/27, shirou, WAKAYAMA Shirou, shirou.faw@gmail.com +2017/08/20, tiagomazzutti, Tiago Mazzutti, tiagomzt@gmail.com +2017/08/20, milanaleksic, Milan Aleksic, milanaleksic@gmail.com +2017/08/29, Eddy Reyes, eddy@mindsight.io +2017/09/09, brauliobz, Bráulio Bezerra, brauliobezerra@gmail.com +2017/09/11, sachinjain024, Sachin Jain, sachinjain024@gmail.com +2017/09/25, kaedvann, Rostislav Listerenko, r.listerenko@gmail.com +2017/10/06, bramp, Andrew Brampton, brampton@gmail.com +2017/10/15, simkimsia, Sim Kim Sia, kimcity@gmail.com +2017/10/27, Griffon26, Maurice van der Pot, griffon26@kfk4ever.com +2017/05/29, rlfnb, Ralf Neeb, rlfnb@rlfnb.de +2017/10/29, gendalph, Максим Прохоренко, Maxim\dotProhorenko@gm@il.com +2017/11/02, jasonmoo, Jason Mooberry, jason.mooberry@gmail.com +2017/11/05, ajaypanyala, Ajay Panyala, ajay.panyala@gmail.com +2017/11/24, zqlu.cn, Zhiqiang Lu, zqlu.cn@gmail.com +2017/11/28, niccroad, Nicolas Croad, nic.croad@gmail.com +2017/12/01, DavidMoraisFerreira, David Morais Ferreira, david.moraisferreira@gmail.com +2017/12/01, SebastianLng, Sebastian Lang, sebastian.lang@outlook.com +2017/12/03, oranoran, Oran Epelbaum, oran / epelbaum me diff --git a/doc/ace-javascript-target.md b/doc/ace-javascript-target.md index fbba9e9..e3f33ba 100644 --- a/doc/ace-javascript-target.md +++ b/doc/ace-javascript-target.md @@ -174,13 +174,20 @@ At this point, you should have an editor which displays an error icon next to th What remains to be done is have our validate function actually validate the input. Finally ANTLR comes in the picture! -To start with, let's load ANTLR and your parser, listener etc.. Easy, since you could write: +To start with, let's load ANTLR and your parser, listener etc.. + +The preferred approach for loading parser code is to bundle your parser, [as described here](javascript-target.md). +You can then load it as part of the importScripts instruction at the start of your worker code. + +Another approach is to load it using 'require'. Easy, since you could write: ```js var antlr4 = require('antlr4/index'); ``` -This may work, but it's actually unreliable. The reason is that the require function used by ANTLR, which exactly mimics the NodeJS require function, uses a different syntax than the require function that comes with ACE. So we need to bring in a require function that conforms to the NodeJS syntax. I personally use one that comes from Torben Haase's Honey project, which you can find here. But hey, now we're going to have 2 'require' functions not compatible with each other! Indeed, this is why you need to take special care, as follows: +This may work, but it's actually unreliable. The reason is that the 'require' function that comes with ACE uses a different syntax than the 'require' function used by ANTLR, which follows the NodeJS 'require' convention. +So we need to bring in a NodeJS compatible 'require' function that conforms to the NodeJS syntax. I personally use one that comes from Torben Haase's Honey project, which you can find in li/require.js. +But hey, now we're going to have 2 'require' functions not compatible with each other! Indeed, this is why you need to take special care, as follows: ```js // load nodejs compatible require @@ -191,7 +198,8 @@ importScripts("../lib/require.js"); var antlr4_require = require; require = ace_require; ``` -Now it's safe to load antlr, and the parsers generated for your language. Assuming that your language files (generated or hand-built) are in a folder with an index.js file that calls require for each file, your parser loading code can be as simple as follows: +Now it's safe to load antlr and the parsers generated for your language. +Assuming that your language files (generated or hand-built) are in a folder with an index.js file that calls require for each file, your parser loading code can be as simple as follows: ```js // load antlr4 and myLanguage var antlr4, mylanguage; @@ -204,6 +212,7 @@ try { } ``` Please note the try-finally construct. ANTLR uses 'require' synchronously so it's perfectly safe to ignore the ACE 'require' while running ANTLR code. ACE itself does not guarantee synchronous execution, so you are much safer always switching 'require' back to 'ace_require'. + Now detecting deep syntax errors in your code is a task for your ANTLR listener or visitor or whatever piece of code you've delegated this to. We're not going to describe this here, since it would require some knowledge of your language. However, detecting grammar syntax errors is something ANTLR does beautifully (isn't that why you went for ANTLR in the first place?). So what we will illustrate here is how to report grammar syntax errors. I have no doubt that from there, you will be able to extend the validator to suit your specific needs. Whenever ANTLR encounters an unexpected token, it fires an error. By default, the error is routed to an error listener which simply writes to the console. What we need to do is replace this listener by our own listener, se we can route errors to the ACE editor. First, let's create such a listener: @@ -230,7 +239,7 @@ AnnotatingErrorListener.prototype.syntaxError = function(recognizer, offendingSy With this, all that remains to be done is plug the listener in when we parse the code. Here is how I do it: ```js var validate = function(input) { - var stream = new antlr4.InputStream(input); + var stream = CharStreams.fromString(input); var lexer = new mylanguage.MyLexer(stream); var tokens = new antlr4.CommonTokenStream(lexer); var parser = new mylanguage.MyParser(tokens); @@ -243,5 +252,4 @@ var validate = function(input) { }; ``` You know what? That's it! You now have an ACE editor that does syntax validation using ANTLR! I hope you find this useful, and simple enough to get started. -What I did not address here is packaging, not something I'm an expert at. The good news is that it makes development simple, since I don't have to run any compilation process. I just edit my code, reload my editor page, and check how it goes. -Now wait, hey! How do you debug this? Well, as usual, using Chrome, since neither Firefox or Safari are able to debug worker code. What a shame... +Now wait, hey! How do you debug this? Well, as usual, using Chrome, since no other browser is able to debug worker code. What a shame... diff --git a/doc/antlr-project-testing.md b/doc/antlr-project-testing.md index e29b00c..335f2da 100644 --- a/doc/antlr-project-testing.md +++ b/doc/antlr-project-testing.md @@ -59,8 +59,7 @@ Now, make sure C# runtime is built and installed locally. ```bash cd ~/antlr/code/antlr4/runtime/CSharp/runtime/CSharp # kill previous ones manually as "xbuild /t:Clean" didn't seem to do it -rm Antlr4.Runtime/bin/net20/Release/Antlr4.Runtime.dll -rm Antlr4.Runtime/obj/net20/Release/Antlr4.Runtime.dll +find . -name '*.dll' -exec rm {} \; # build xbuild /p:Configuration=Release Antlr4.Runtime/Antlr4.Runtime.mono.csproj ``` @@ -256,6 +255,14 @@ where `<TARGET>` is replaced with Java, Cpp, CSharp, Python2, ... in the various In order to turn off a test for a particular target, we need to use the `ignore` method. Given a target name, a descriptor object can decide whether to ignore the test. This is not always convenient but it is fully general and works well for the one case we have now where we have to ignore `Visitor` tests in all targets except JavaScript. +### Target API/library testing + +Some parts of the runtime API need to be tested with code written specifically in the target language. For example, you can see all of the Java runtime API tests here: + +[https://github.com/antlr/antlr4/tree/master/runtime-testsuite/test/org/antlr/v4/test/runtime/java/api](https://github.com/antlr/antlr4/tree/master/runtime-testsuite/test/org/antlr/v4/test/runtime/java/api) + +Notice that it is under an `api` dir. The directory above is where all of the `Test*` files go. + ### Cross-language actions embedded within grammars To get: diff --git a/doc/building-antlr.md b/doc/building-antlr.md index ac220bc..06f1ace 100644 --- a/doc/building-antlr.md +++ b/doc/building-antlr.md @@ -39,6 +39,7 @@ Resolving deltas: 100% (31898/31898), done. Checking connectivity... done. $ cd antlr4 $ export MAVEN_OPTS="-Xmx1G" # don't forget this on linux +$ mvn clean # must be separate, not part of install/compile $ mvn -DskipTests install ... [INFO] ------------------------------------------------------------------------ @@ -61,7 +62,7 @@ $ mvn -DskipTests install [INFO] ------------------------------------------------------------------------ ``` -We do `install` not `compile` as tool tests and such refer to modules that must be pulled from the maven install local cache. +**NOTE:** We do `install` not `compile` as tool tests and such refer to modules that must be pulled from the maven install local cache. # Installing libs to mvn cache locally diff --git a/doc/case-insensitive-lexing.md b/doc/case-insensitive-lexing.md new file mode 100644 index 0000000..8e9a4f7 --- /dev/null +++ b/doc/case-insensitive-lexing.md @@ -0,0 +1,78 @@ +# Case-Insensitive Lexing + +In some languages, keywords are case insensitive meaning that `BeGiN` means the same thing as `begin` or `BEGIN`. ANTLR has two mechanisms to support building grammars for such languages: + +1. Build lexical rules that match either upper or lower case. + * **Advantage**: no changes required to ANTLR, makes it clear in the grammar that the language in this case insensitive. + * **Disadvantage**: might have a small efficiency cost and grammar is a more verbose and more of a hassle to write. + +2. Build lexical rules that match keywords in all uppercase and then parse with a custom [character stream](https://github.com/antlr/antlr4/blob/master/runtime/Java/src/org/antlr/v4/runtime/CharStream.java) that converts all characters to uppercase before sending them to the lexer (via the `LA()` method). Care must be taken not to convert all characters in the stream to uppercase because characters within strings and comments should be unaffected. All we really want is to trick the lexer into thinking the input is all uppercase. + * **Advantage**: Could have a speed advantage depending on implementation, no change required to the grammar. + * **Disadvantage**: Requires that the case-insensitive stream and grammar are used in correctly in conjunction with each other, makes all characters appear as uppercase/lowercase to the lexer but some grammars are case sensitive outside of keywords, errors new case insensitive streams and language output targets (java, C#, C++, ...). + +For the 4.7.1 release, we discussed both approaches in [detail](https://github.com/antlr/antlr4/pull/2046) and even possibly altering the ANTLR metalanguage to directly support case-insensitive lexing. We discussed including the case insensitive streams into the runtime but not all would be immediately supported. I decided to simply make documentation that clearly states how to handle this and include the appropriate snippets that people can cut-and-paste into their grammars. + +## Case-insensitive grammars + +As a prime example of a grammar that specifically describes case insensitive keywords, see the +[SQLite grammar](https://github.com/antlr/grammars-v4/blob/master/sqlite/SQLite.g4). To match a case insensitive keyword, there are rules such as + +``` +K_UPDATE : U P D A T E; +``` + +that will match `UpdaTE` and `upDATE` etc... as the `update` keyword. This rule makes use of some generically useful fragment rules that you can cut-and-paste into your grammars: + +``` +fragment A : [aA]; // match either an 'a' or 'A' +fragment B : [bB]; +fragment C : [cC]; +fragment D : [dD]; +fragment E : [eE]; +fragment F : [fF]; +fragment G : [gG]; +fragment H : [hH]; +fragment I : [iI]; +fragment J : [jJ]; +fragment K : [kK]; +fragment L : [lL]; +fragment M : [mM]; +fragment N : [nN]; +fragment O : [oO]; +fragment P : [pP]; +fragment Q : [qQ]; +fragment R : [rR]; +fragment S : [sS]; +fragment T : [tT]; +fragment U : [uU]; +fragment V : [vV]; +fragment W : [wW]; +fragment X : [xX]; +fragment Y : [yY]; +fragment Z : [zZ]; +``` + +No special streams are required to use this mechanism for case insensitivity. + +## Custom character streams approach + +The other approach is to use lexical rules that match either all uppercase or all lowercase, such as: + +``` +K_UPDATE : 'UPDATE'; +``` + +Then, when creating the character stream to parse from, we need a custom class that overrides methods used by the lexer. Below you will find custom character streams for a number of the targets that you can copy into your projects, but here is how to use the streams in Java as an example: + +```java +CharStream s = CharStreams.fromPath(Paths.get('test.sql')); +CaseChangingCharStream upper = new CaseChangingCharStream(s, true); +Lexer lexer = new SomeSQLLexer(upper); +``` + +Here are implementations of `CaseChangingCharStream` in various target languages: + +* [Java](https://github.com/parrt/antlr4/blob/case-insensitivity-doc/doc/resources/CaseChangingCharStream.java) +* [JavaScript](https://github.com/parrt/antlr4/blob/case-insensitivity-doc/doc/resources/CaseInsensitiveInputStream.js) +* [Go](https://github.com/parrt/antlr4/blob/case-insensitivity-doc/doc/resources/case_changing_stream.go) +* [C#](https://github.com/parrt/antlr4/blob/case-insensitivity-doc/doc/resources/CaseChangingCharStream.cs)
\ No newline at end of file diff --git a/doc/cpp-target.md b/doc/cpp-target.md index 31283e8..2933d60 100644 --- a/doc/cpp-target.md +++ b/doc/cpp-target.md @@ -1,6 +1,6 @@ # C++ -The C++ target supports all platforms that can either run MS Visual Studio 2013 (or newer), XCode 7 (or newer) or CMake (C++11 required). All build tools can either create static or dynamic libraries, both as 64bit or 32bit arch. Additionally, XCode can create an iOS library. +The C++ target supports all platforms that can either run MS Visual Studio 2013 (or newer), XCode 7 (or newer) or CMake (C++11 required). All build tools can either create static or dynamic libraries, both as 64bit or 32bit arch. Additionally, XCode can create an iOS library. Also see [Antlr4 for C++ with CMake: A practical example](http://blorente.me//Antlr-,-C++-and-CMake-Wait-what.html). ## How to create a C++ lexer or parser? This is pretty much the same as creating a Java lexer or parser, except you need to specify the language target, for example: @@ -49,7 +49,7 @@ using namespace org::antlr::v4::runtime; class TreeShapeListener : public MyGrammarBaseListener { public: - void enterKey(Ref<ParserRuleContext> ctx) { + void enterKey(ParserRuleContext *ctx) override { // Do something when entering the key rule. } }; @@ -63,31 +63,26 @@ int main(int argc, const char* argv[]) { CommonTokenStream tokens(&lexer); MyGrammarParser parser(&tokens); - Ref<tree::ParseTree> tree = parser.key(); - Ref<TreeShapeListener> listener(new TreeShapeListener()); - tree::ParseTreeWalker::DEFAULT->walk(listener, tree); + tree::ParseTree *tree = parser.key(); + TreeShapeListener listener; + tree::ParseTreeWalker::DEFAULT->walk(&listener, tree); return 0; } ``` -This example assumes your grammar contains a parser rule named `key` for which the enterKey function was generated. The `Ref<>` template is an alias for `std::shared_ptr<>` to simplify the runtime source code which often makes use of smart pointers. +This example assumes your grammar contains a parser rule named `key` for which the enterKey function was generated. ## Specialities of this ANTLR target There are a couple of things that only the C++ ANTLR target has to deal with. They are described here. ### Build Aspects - -The code generation (by running the ANTLR4 jar) allows to specify 2 values you -might find useful for better integration of the generated files into your -application (both are optional): +The code generation (by running the ANTLR4 jar) allows to specify 2 values you might find useful for better integration of the generated files into your application (both are optional): * A **namespace**: use the **`-package`** parameter to specify the namespace you want. -* An **export macro**: especially in VC++ extra work is required to export your classes from a DLL. -This is usually accomplished by a macro that has different values depending on whether -you are creating the DLL or import it. The ANTLR4 runtime itself also uses one for its classes: +* An **export macro**: especially in VC++ extra work is required to export your classes from a DLL. This is usually accomplished by a macro that has different values depending on whether you are creating the DLL or import it. The ANTLR4 runtime itself also uses one for its classes: ```c++ #ifdef ANTLR4CPP_EXPORTS @@ -100,41 +95,23 @@ you are creating the DLL or import it. The ANTLR4 runtime itself also uses one f #endif #endif ``` - -Just like the `ANTLR4CPP_PUBLIC` macro here you can specify your own one for -the generated classes using the **`-DexportMacro=...`** command-line parameter or +Just like the `ANTLR4CPP_PUBLIC` macro here you can specify your own one for the generated classes using the **`-DexportMacro=...`** command-line parameter or grammar option `options {exportMacro='...';}` in your grammar file. -In order to create a static lib in Visual Studio define the `ANTLR4CPP_STATIC` macro -in addition to the project settings that must be set for a static library -(if you compile the runtime yourself). +In order to create a static lib in Visual Studio define the `ANTLR4CPP_STATIC` macro in addition to the project settings that must be set for a static library (if you compile the runtime yourself). -For gcc and clang it is possible to use the `-fvisibility=hidden` setting to -hide all symbols except those that are made default-visible (which has been -defined for all public classes in the runtime). +For gcc and clang it is possible to use the `-fvisibility=hidden` setting to hide all symbols except those that are made default-visible (which has been defined for all public classes in the runtime). ### Memory Management - -Since C++ has no built-in memory management we need to take extra care. -For that we rely mostly on smart pointers, which however might cause time -penalties or memory side effects (like cyclic references) if not used with care. -Currently however the memory household looks very stable. Generally, when you -see a raw pointer in code consider this as being managed elsewehere. You -should never try to manage such a pointer (delete, assign to smart pointer etc.). +Since C++ has no built-in memory management we need to take extra care. For that we rely mostly on smart pointers, which however might cause time penalties or memory side effects (like cyclic references) if not used with care. Currently however the memory household looks very stable. Generally, when you see a raw pointer in code consider this as being managed elsewehere. You should never try to manage such a pointer (delete, assign to smart pointer etc.). ### Unicode Support +Encoding is mostly an input issue, i.e. when the lexer converts text input into lexer tokens. The parser is completely encoding unaware. -Encoding is mostly an input issue, i.e. when the lexer converts text input into lexer tokens. The parser is completely encoding unaware. However, lexer input in the grammar is defined by character ranges with either a single member (e.g. 'a' or [a] or [abc]), an explicit range (e.g. 'a'..'z' or [a-z]), the full Unicode range (for a wildcard) and the full Unicode range minus a sub range (for negated ranges, e.g. ~[a]). The explicit ranges (including single member ranges) are encoded in the serialized ATN by 16bit numbers, hence cannot reach beyond 0xFFFF (the Unicode BMP), while the implicit ranges can include any value (and hence support the full Unicode set, up to 0x10FFFF). - -> An interesting side note here is that the Java target fully supports Unicode as well, despite the inherent limitations from the serialized ATN. That's possible because the Java String class represents characters beyond the BMP as surrogate pairs (two 16bit values) and even reads them as 2 separate input characters. To make this work a character range for an identifier in a grammar must include the surrogate pairs area (for a Java parser). - -The C++ target however always expects UTF-8 input (either in a string or via a wide stream) which is then converted to UTF-32 (a char32_t array) and fed to the lexer. ANTLR, when parsing your grammar, limits character ranges explicitly to the BMP currently. So, in order to allow specifying the full Unicode set the C++ target uses a little trick: whenever an explicit character range includes the (unused) codepoint 0xFFFF in a grammar it is silently extended to the full Unicode range. It's clear that this is an all-or-nothing solution. You cannot define a subset of Unicode codepoints > 0xFFFF that way. This can only be solved if ANTLR supports larger character intervals. - -The differences in handling characters beyond the BMP leads to a difference between Java and C++ lexers: the character offsets may not concur. This is because Java reads two 16bit values per Unicode char (if that falls into the surrogate area) while a C++ parser only reads one 32bit value. That usually doesn't have practical consequences, but might confuse people when comparing token positions. +The C++ target always expects UTF-8 input (either in a string or stream) which is then converted to UTF-32 (a char32_t array) and fed to the lexer. ### Named Actions - -In order to help customizing the generated files there are a number of additional socalled **named actions**. These actions are tight to specific areas in the generated code and allow to add custom (target specific) code. All targets support these actions +In order to help customizing the generated files there are a number of additional socalled **named actions**. These actions are tight to specific areas in the generated code and allow to add custom (target specific) code. All targets support these actions * @parser::header * @parser::members diff --git a/doc/csharp-target.md b/doc/csharp-target.md index 18e6075..9158b02 100644 --- a/doc/csharp-target.md +++ b/doc/csharp-target.md @@ -24,9 +24,9 @@ Let's suppose that your grammar is named `MyGrammar`. The tool will generate for * MyGrammarLexer.cs * MyGrammarParser.cs * MyGrammarListener.cs (if you have not activated the -no-listener option) -* MyGrammarBaseListener.js (if you have not activated the -no-listener option) -* MyGrammarVisitor.js (if you have activated the -visitor option) -* MyGrammarBaseVisitor.js (if you have activated the -visitor option) +* MyGrammarBaseListener.cs (if you have not activated the -no-listener option) +* MyGrammarVisitor.cs (if you have activated the -visitor option) +* MyGrammarBaseVisitor.cs (if you have activated the -visitor option) Now a fully functioning code might look like the following for start rule `StartRule`: @@ -35,7 +35,7 @@ using Antlr4.Runtime; public void MyParseMethod() { String input = "your text to parse here"; - AntlrInputStream stream = new InputStream(input); + ICharStream stream = CharStreams.fromstring(input); ITokenSource lexer = new MyGrammarLexer(stream); ITokenStream tokens = new CommonTokenStream(lexer); MyGrammarParser parser = new MyGrammarParser(tokens); diff --git a/doc/faq/general.md b/doc/faq/general.md index 9189ac9..b7d8dd8 100644 --- a/doc/faq/general.md +++ b/doc/faq/general.md @@ -82,7 +82,7 @@ Make sure to use two-stage parsing. See example in [bug report](https://github.c ```Java -CharStream input = new ANTLRFileStream(args[0]); +CharStream input = CharStreams.fromPath(Paths.get(args[0])); ExprLexer lexer = new ExprLexer(input); CommonTokenStream tokens = new CommonTokenStream(lexer); ExprParser parser = new ExprParser(tokens); diff --git a/doc/getting-started.md b/doc/getting-started.md index 056f0fa..dabd8c1 100644 --- a/doc/getting-started.md +++ b/doc/getting-started.md @@ -6,7 +6,7 @@ Hi and welcome to the version 4 release of ANTLR! It's named after the fearless ANTLR is really two things: a tool that translates your grammar to a parser/lexer in Java (or other target language) and the runtime needed by the generated parsers/lexers. Even if you are using the ANTLR Intellij plug-in or ANTLRWorks to run the ANTLR tool, the generated code will still need the runtime library. -The first thing you should do is probably download and install a development tool plug-in. Even if you only use such tools for editing, they are great. Then, follow the instructions below to get the runtime environment available to your system to run generated parsers/lexers. In what follows, I talk about antlr-4.5.3-complete.jar, which has the tool and the runtime and any other support libraries (e.g., ANTLR v4 is written in v3). +The first thing you should do is probably download and install a development tool plug-in. Even if you only use such tools for editing, they are great. Then, follow the instructions below to get the runtime environment available to your system to run generated parsers/lexers. In what follows, I talk about antlr-4.7.1-complete.jar, which has the tool and the runtime and any other support libraries (e.g., ANTLR v4 is written in v3). If you are going to integrate ANTLR into your existing build system using mvn, ant, or want to get ANTLR into your IDE such as eclipse or intellij, see Integrating ANTLR into Development Systems. @@ -16,19 +16,21 @@ If you are going to integrate ANTLR into your existing build system using mvn, a 1. Download ``` $ cd /usr/local/lib -$ curl -O http://www.antlr.org/download/antlr-4.5.3-complete.jar +$ curl -O http://www.antlr.org/download/antlr-4.7.1-complete.jar ``` Or just download in browser from website: [http://www.antlr.org/download.html](http://www.antlr.org/download.html) and put it somewhere rational like `/usr/local/lib`. -2. Add `antlr-4.5.3-complete.jar` to your `CLASSPATH`: + +2. Add `antlr-4.7.1-complete.jar` to your `CLASSPATH`: ``` -$ export CLASSPATH=".:/usr/local/lib/antlr-4.5.3-complete.jar:$CLASSPATH" +$ export CLASSPATH=".:/usr/local/lib/antlr-4.7.1-complete.jar:$CLASSPATH" ``` It's also a good idea to put this in your `.bash_profile` or whatever your startup script is. + 3. Create aliases for the ANTLR Tool, and `TestRig`. ``` -$ alias antlr4='java -Xmx500M -cp "/usr/local/lib/antlr-4.5.3-complete.jar:$CLASSPATH" org.antlr.v4.Tool' +$ alias antlr4='java -Xmx500M -cp "/usr/local/lib/antlr-4.7.1-complete.jar:$CLASSPATH" org.antlr.v4.Tool' $ alias grun='java org.antlr.v4.gui.TestRig' ``` @@ -39,11 +41,11 @@ $ alias grun='java org.antlr.v4.gui.TestRig' 0. Install Java (version 1.6 or higher) 1. Download antlr-4.5.3-complete.jar (or whatever version) from [http://www.antlr.org/download/](http://www.antlr.org/download/) Save to your directory for 3rd party Java libraries, say `C:\Javalib` -2. Add `antlr-4.5-complete.jar` to CLASSPATH, either: +2. Add `antlr-4.5.3-complete.jar` to CLASSPATH, either: * Permanently: Using System Properties dialog > Environment variables > Create or append to `CLASSPATH` variable * Temporarily, at command line: ``` -SET CLASSPATH=.;C:\Javalib\antlr-4.5.3-complete.jar;%CLASSPATH% +SET CLASSPATH=.;C:\Javalib\antlr-4.7.1-complete.jar;%CLASSPATH% ``` 3. Create short convenient commands for the ANTLR Tool, and TestRig, using batch files or doskey commands: * Batch files (in directory in system PATH) antlr4.bat and grun.bat @@ -65,7 +67,7 @@ Either launch org.antlr.v4.Tool directly: ``` $ java org.antlr.v4.Tool -ANTLR Parser Generator Version 4.5.3 +ANTLR Parser Generator Version 4.7.1 -o ___ specify output directory where all output is generated -lib ___ specify location of .tokens files ... @@ -74,8 +76,8 @@ ANTLR Parser Generator Version 4.5.3 or use -jar option on java: ``` -$ java -jar /usr/local/lib/antlr-4.5.3-complete.jar -ANTLR Parser Generator Version 4.5.3 +$ java -jar /usr/local/lib/antlr-4.7.1-complete.jar +ANTLR Parser Generator Version 4.7.1 -o ___ specify output directory where all output is generated -lib ___ specify location of .tokens files ... @@ -122,7 +124,7 @@ That pops up a dialog box showing that rule `r` matched keyword `hello` followed ## Book source code -The book has lots and lots of examples that should be useful to. You can download them here for free: +The book has lots and lots of examples that should be useful too. You can download them here for free: [http://pragprog.com/titles/tpantlr2/source_code](http://pragprog.com/titles/tpantlr2/source_code) diff --git a/doc/go-target.md b/doc/go-target.md index 6de6db9..4f7e64e 100644 --- a/doc/go-target.md +++ b/doc/go-target.md @@ -88,7 +88,7 @@ func (this *TreeShapeListener) EnterEveryRule(ctx antlr.ParserRuleContext) { } func main() { - input := antlr.NewFileStream(os.Args[1]) + input, _ := antlr.NewFileStream(os.Args[1]) lexer := parser.NewJSONLexer(input) stream := antlr.NewCommonTokenStream(lexer,0) p := parser.NewJSONParser(stream) diff --git a/doc/grammars.md b/doc/grammars.md index c40d974..5f88bf2 100644 --- a/doc/grammars.md +++ b/doc/grammars.md @@ -81,7 +81,10 @@ $ grun MyELang stat <= line 3:0 extraneous input ';' expecting {INT, ID} ``` -If there were any `tokens` specifications, the main grammar would merge the token sets. Any named actions such as `@members` would be merged. In general, you should avoid named actions and actions within rules in imported grammars since that limits their reuse. ANTLR also ignores any options in imported grammars. +If there are modes in the main grammar or any of the imported grammars then the import process will import those modes and merge their rules where they are not overridden. In the event any mode becomes empty as all its +rules have been overridden by rules outside the mode this mode will be discarded. + +If there were any `tokens` specifications, the main grammar would merge the token sets. If there were any `channel` specifications, the main grammar would merge the channel sets. Any named actions such as `@members` would be merged. In general, you should avoid named actions and actions within rules in imported grammars since that limits their reuse. ANTLR also ignores any options in imported grammars. Imported grammars can also import other grammars. ANTLR pursues all imported grammars in a depth-first fashion. If two or more imported grammars define rule `r`, ANTLR chooses the first version of `r` it finds. In the following diagram, ANTLR examines grammars in the following order `Nested`, `G1`, `G3`, `G2`. @@ -91,9 +94,9 @@ Imported grammars can also import other grammars. ANTLR pursues all imported gra Not every kind of grammar can import every other kind of grammar: -* Lexer grammars can import lexers. +* Lexer grammars can import lexers, including lexers containing modes. * Parsers can import parsers. -* Combined grammars can import lexers or parsers. +* Combined grammars can import parsers or lexers without modes. ANTLR adds imported rules to the end of the rule list in a main lexer grammar. That means lexer rules in the main grammar get precedence over imported rules. For example, if a main grammar defines rule `IF : ’if’ ;` and an imported grammar defines rule `ID : [a-z]+ ;` (which also recognizes `if`), the imported `ID` won’t hide the main grammar’s `IF` token definition. diff --git a/doc/images/gen_spm_module.png b/doc/images/gen_spm_module.png Binary files differnew file mode 100644 index 0000000..0798c37 --- /dev/null +++ b/doc/images/gen_spm_module.png diff --git a/doc/index.md b/doc/index.md index d5c5551..60bd634 100644 --- a/doc/index.md +++ b/doc/index.md @@ -8,7 +8,7 @@ Notes: <li>Copyright © 2012, The Pragmatic Bookshelf. Pragmatic Bookshelf grants a nonexclusive, irrevocable, royalty-free, worldwide license to reproduce, distribute, prepare derivative works, and otherwise use this contribution as part of the ANTLR project and associated documentation.</li> -<li>This text was copied with permission from the <a href=http://pragprog.com/book/tpantlr2/the-definitive-antlr-4-reference>The Definitive ANTLR 4 Reference</a>, though it is being morphed over time as the tool changes.</li> +<li>Much of this text was copied with permission from the <a href=http://pragprog.com/book/tpantlr2/the-definitive-antlr-4-reference>The Definitive ANTLR 4 Reference</a>, though it is being morphed over time as the tool changes.</li> </ul> Links in the documentation refer to various sections of the book but have been redirected to the general book page on the publisher's site. There are two excerpts on the publisher's website that might be useful to you without having to purchase the book: [Let's get Meta](http://media.pragprog.com/titles/tpantlr2/picture.pdf) and [Building a Translator with a Listener](http://media.pragprog.com/titles/tpantlr2/listener.pdf). You should also consider reading the following books (the vid describes the reference book): @@ -16,7 +16,7 @@ Links in the documentation refer to various sections of the book but have been r <a href=""><img src=images/tpantlr2.png width=120></a> <a href=""><img src=images/tpdsl.png width=120></a> <a href="https://www.youtube.com/watch?v=OAoA3E-cyug"><img src=images/teronbook.png width=250></a> - + This documentation is a reference and summarizes grammar syntax and the key semantics of ANTLR grammars. The source code for all examples in the book, not just this chapter, are free at the publisher's website. The following video is a general tour of ANTLR 4 and includes a description of how to use parse tree listeners to process Java files easily: <a href="https://vimeo.com/59285751"><img src=images/tertalk.png width=200></a> @@ -51,8 +51,12 @@ This documentation is a reference and summarizes grammar syntax and the key sema * [Runtime Libraries and Code Generation Targets](targets.md) +* [Unicode U+FFFF, U+10FFFF character streams](unicode.md) + * [Parsing binary streams](parsing-binary-files.md) - + +* [Case-Insensitive Lexing](case-insensitive-lexing.md) + * [Parser and lexer interpreters](interpreters.md) * [Resources](resources.md) @@ -61,6 +65,8 @@ This documentation is a reference and summarizes grammar syntax and the key sema * [Building ANTLR itself](building-antlr.md) +* [Contributing to ANTLR](/CONTRIBUTING.md) + * [Cutting an ANTLR Release](releasing-antlr.md) * [ANTLR project unit tests](antlr-project-testing.md) diff --git a/doc/interpreters.md b/doc/interpreters.md index c99e6d5..3be57f1 100644 --- a/doc/interpreters.md +++ b/doc/interpreters.md @@ -1,8 +1,14 @@ -# Parser and lexer interpreters +# Parser and Lexer Interpreters *Since ANTLR 4.2* -For small parsing tasks it is sometimes convenient to use ANTLR in interpreted mode, rather than generating a parser in a particular target, compiling it and running it as part of your application. Here's some sample code that creates lexer and parser Grammar objects and then creates interpreters. Once we have a ParserInterpreter, we can use it to parse starting in any rule we like, given a rule index (which the Grammar can provide). +For small parsing tasks it is sometimes convenient to use ANTLR in interpreted mode, rather than generating a parser in a particular target, compiling it and running it as part of your application. Here's some sample code that creates lexer and parser Grammar objects and then creates interpreters. Once we have a ParserInterpreter, we can use it to parse starting in any rule we like, given a rule index (which the grammar + the parser can provide). + +## Action Code + +Since interpreters don't use generated parsers + lexers they cannot execute any action code (including predicates). That means the interpreter runs as if there were no predicates at all. If your grammar requires action code in order to parse correctly you will not be able to test it using this approach. + +## Java Target Interpreter Setup ```java LexerGrammar lg = new LexerGrammar( @@ -30,7 +36,7 @@ public static ParseTree parse(String fileName, throws IOException { final Grammar g = Grammar.load(combinedGrammarFileName); - LexerInterpreter lexEngine = g.createLexerInterpreter(new ANTLRFileStream(fileName)); + LexerInterpreter lexEngine = g.createLexerInterpreter(CharStreams.fromPath(Paths.get(fileName))); CommonTokenStream tokens = new CommonTokenStream(lexEngine); ParserInterpreter parser = g.createParserInterpreter(tokens); ParseTree t = parser.parse(g.getRule(startRule).index); @@ -58,7 +64,7 @@ public static ParseTree parse(String fileNameToParse, { final LexerGrammar lg = (LexerGrammar) Grammar.load(lexerGrammarFileName); final Grammar pg = Grammar.load(parserGrammarFileName, lg); - ANTLRFileStream input = new ANTLRFileStream(fileNameToParse); + CharStream input = CharStreams.fromPath(Paths.get(fileNameToParse)); LexerInterpreter lexEngine = lg.createLexerInterpreter(input); CommonTokenStream tokens = new CommonTokenStream(lexEngine); ParserInterpreter parser = pg.createParserInterpreter(tokens); @@ -76,4 +82,49 @@ ParseTree t = parse(fileName, XMLLexerGrammar, XMLParserGrammar, "document"); This is also how we will integrate instantaneous parsing into ANTLRWorks2 and development environment plug-ins. -See [TestParserInterpreter.java](https://github.com/antlr/antlr4/blob/master/tool-testsuite/test/org/antlr/v4/test/tool/TestParserInterpreter.java). +See [TestParserInterpreter.java](../tool-testsuite/test/org/antlr/v4/test/tool/TestParserInterpreter.java). + +## Non-Java Target Interpreter Setup +The ANTLR4 runtimes do not contain any grammar parsing classes (they are in the ANTLR4 tool jar). Hence we cannot use `LexerGrammar` and `Grammar` to parse grammars for the interpreter. Instead we directly instantiate `LexerInterpreter` and `ParserInterpreter` objects. They require some data (namely symbol information and the ATNs) which only the ANTLR4 tool can give us. However, on each generation run ANTLR not only produces your parser + lexer files but also interpreter data files (*.interp) which contain all you need to feed the interpreters. + +A support class (`InterpreterDataReader`) is used to load the data for your convenience, which makes this very easy to use. Btw. even the Java target go this route instead of using the non-runtime classes `Grammar` and `LexerGrammar`. Sometimes it might not be feasible to use the tool jar for whatever reason. + +Here's how the setup looks like (C++ example): + +```cpp +/** + * sourceFileName - name of the file with content to parse + * lexerName - the name of your lexer (arbitrary, that's what is used in error messages) + * parserName - ditto for the parser + * lexerDataFileName - the lexer interpeter data file name (e.g. `<path>/ExprLexer.interp`) + * parserDataFileName - ditto for the parser (e.g. `<path>/Expr.interp`) + * startRule - the name of the rule to start parsing at + */ +void parse(std::string const& sourceFileName, + std::string const& lexerName, std::string const& parserName, + std::string const& lexerDataFileName, std::string const& parserDataFileName, + std::string const& startRule) { + + InterpreterData lexerData = InterpreterDataReader::parseFile(lexerDataFileName); + InterpreterData parserData = InterpreterDataReader::parseFile(parserDataFileName); + + ANTLRFileStream input(sourceFileName); + LexerInterpreter lexEngine(lexerName, lexerData.vocabulary, lexerData.ruleNames, + lexerData.channels, lexerData.modes, lexerData.atn, &input); + CommonTokenStream tokens(&lexEngine); + + /* Remove comment to print the tokens. + tokens.fill(); + std::cout << "INPUT:" << std::endl; + for (auto token : tokens.getTokens()) { + std::cout << token->toString() << std::endl; + } + */ + + ParserInterpreter parser(parserName, parserData.vocabulary, parserData.ruleNames, + parserData.atn, &tokens); + tree::ParseTree *tree = parser.parse(parser.getRuleIndex(startRule)); + + std::cout << "parse tree: " << tree->toStringTree(&parser) << std::endl; +} +``` diff --git a/doc/javascript-target.md b/doc/javascript-target.md index e00f3b7..3473e95 100644 --- a/doc/javascript-target.md +++ b/doc/javascript-target.md @@ -15,7 +15,7 @@ The tests were conducted using Selenium. No issue was found, so you should find ## Is NodeJS supported? -The runtime has also been extensively tested against Node.js 0.10.33. No issue was found. +The runtime has also been extensively tested against Node.js 0.12.7. No issue was found. ## How to create a JavaScript lexer or parser? @@ -31,7 +31,9 @@ For a full list of antlr4 tool options, please visit the [tool documentation pag Once you've generated the lexer and/or parser code, you need to download the runtime. -The JavaScript runtime is available from the ANTLR web site [download section](http://www.antlr.org/download/index.html). The runtime is provided in the form of source code, so no additional installation is required. +The JavaScript runtime is [available from npm](https://www.npmjs.com/package/antlr4). + +If you can't use npm, the JavaScript runtime is also available from the ANTLR web site [download section](http://www.antlr.org/download/index.html). The runtime is provided in the form of source code, so no additional installation is required. We will not document here how to refer to the runtime from your project, since this would differ a lot depending on your project type and IDE. @@ -47,11 +49,28 @@ However, it would be a bit of a problem when it comes to get it into a browser. <script src='lib/myscript.js'> ``` -In order to avoid having to do this, and also to have the exact same code for browsers and Node.js, we rely on a script which provides the equivalent of the Node.js 'require' function. +To avoid having doing this, the preferred approach is to bundle antlr4 with your parser code, using webpack. + +You can get [information on webpack here](https://webpack.github.io). -This script is provided by Torben Haase, and is NOT part of ANTLR JavaScript runtime, although the runtime heavily relies on it. Please note that syntax for 'require' in NodeJS is different from the one implemented by RequireJS and similar frameworks. +The steps to create your parsing code are the following: + - generate your lexer, parser, listener and visitor using the antlr tool + - write your parse tree handling code by providig your custom listener or visitor, and associated code, using 'require' to load antlr. + - create an index.js file with the entry point to your parsing code (or several if required). + - test your parsing logic thoroughly using node.js + +You are now ready to bundle your parsing code as follows: + - following webpack specs, create a webpack.config file + - in the webpack.config file, exclude node.js only modules using: node: { module: "empty", net: "empty", fs: "empty" } + - from the cmd line, nag-vigate to the directory containing webpack.config and type: webpack + +This will produce a single js file containing all your parsing code. Easy to include in your web pages! -So in short, assuming you have at the root of your web site, both the 'antlr4' directory and a 'lib' directory with 'require.js' inside it, all you need to put in your HTML header is the following: +If you can't use webpack, you can use the lib/require.js script which implements the Node.js 'require' function in brwsers. + +This script is provided by Torben Haase, and is NOT part of ANTLR JavaScript runtime. + +Assuming you have, at the root of your web site, both the 'antlr4' directory and a 'lib' directory with 'require.js' inside it, all you need to put in your HTML header is the following: ```xml <script src='lib/require.js'> @@ -62,16 +81,6 @@ So in short, assuming you have at the root of your web site, both the 'antlr4' d This will load the runtime asynchronously. -## How do I get the runtime in Node.js? - -Right now, there is no npm package available, so you need to register a link instead. This can be done by running the following command from the antlr4 directory: - -```bash -$ npm link antlr4 -``` - -This will install antlr4 using the package.json descriptor that comes with the script. - ## How do I run the generated lexer and/or parser? Let's suppose that your grammar is named, as above, "MyGrammar". Let's suppose this parser comprises a rule named "StartRule". The tool will have generated for you the following files: @@ -145,6 +154,10 @@ In order to execute this listener, you would simply add the following lines to t antlr4.tree.ParseTreeWalker.DEFAULT.walk(printer, tree); ``` +## What about TypeScript? + +We currently do not have a TypeScript target, but Sam Harwell is [working on a port](https://github.com/tunnelvisionlabs/antlr4ts). [Here](https://github.com/ChuckJonas/extract-interface-ts) is Section 4.3 of [ANTLR 4 book](http://a.co/5jUJYmh) converted to typescript. + ## How do I integrate my parser with ACE editor? This specific task is described in this [dedicated page](ace-javascript-target.md). diff --git a/doc/lexer-rules.md b/doc/lexer-rules.md index adda9e8..5070f47 100644 --- a/doc/lexer-rules.md +++ b/doc/lexer-rules.md @@ -58,13 +58,37 @@ Match that character or sequence of characters. E.g., ’while’ or ’=’.</t <tr> <td>[char set]</td><td> -Match one of the characters specified in the character set. Interpret x-y as set of characters between range x and y, inclusively. The following escaped characters are interpreted as single special characters: \n, \r, \b, \t, and \f. To get ], \, or - you must escape them with \. You can also use Unicode character specifications: \uXXXX. Here are a few examples: +<p>Match one of the characters specified in the character set. Interpret <tt>x-y</tt> as the set of characters between range <tt>x</tt> and <tt>y</tt>, inclusively. The following escaped characters are interpreted as single special characters: <tt>\n</tt>, <tt>\r</tt>, <tt>\b</tt>, <tt>\t</tt>, <tt>\f</tt>, <tt>\uXXXX</tt>, and <tt>\u{XXXXXX}</tt>. To get <tt>]</tt>, <tt>\</tt>, or <tt>-</tt> you must escape them with <tt>\</tt>.</p> + +<p>You can also include all characters matching Unicode properties (general category, boolean, or enumerated including scripts and blocks) with <tt>\p{PropertyName}</tt> or <tt>\p{EnumProperty=Value}</tt>. (You can invert the test with <tt>\P{PropertyName}</tt> or <tt>\P{EnumProperty=Value}</tt>).</p> + +<p>For a list of valid Unicode property names, see <a href="http://unicode.org/reports/tr44/#Properties">Unicode Standard Annex #44</a>. (ANTLR also supports <a href="http://unicode.org/reports/tr44/#General_Category_Values">short and long Unicode general category names and values</a> like <tt>\p{Lu}</tt>, <tt>\p{Z}</tt>, <tt>\p{Symbol}</tt>, <tt>\p{Blk=Latin_1_Sup}</tt>, and <tt>\p{Block=Latin_1_Supplement}</tt>.)</p> + +<p>As a shortcut for <tt>\p{Block=Latin_1_Supplement}</tt>, you can refer to blocks using <a href="http://www.unicode.org/Public/UCD/latest/ucd/Blocks.txt">Unicode block names</a> prefixed with <tt>In</tt> and with spaces changed to <tt>_</tt>. For example: <tt>\p{InLatin_1_Supplement}</tt>, <tt>\p{InYijing_Hexagram_Symbols}</tt>, and <tt>\p{InAncient_Greek_Numbers}</tt>.</p> + +<p>A few extra properties are supported:</p> +<ul> +<li><tt>\p{Extended_Pictographic}</tt> (see <a href="http://unicode.org/reports/tr35/">UTS #35</a>)</li> +<li><tt>\p{EmojiPresentation=EmojiDefault}</tt> (code points which have colorful emoji-style presentation by default but which can also be displayed text-style)</li> +<li><tt>\p{EmojiPresentation=TextDefault}</tt> (code points which have black-and-white text-style presentation by default but which can also be displayed emoji-style)</li> +<li><tt>\p{EmojiPresentation=Text}</tt> (code points which have only black-and-white text-style and lack a colorful emoji-style presentation)</li> +</ul> + +<p>Property names are <b>case-insensitive</b>, and <tt>_</tt> and <tt>-</tt> are treated identically</p> + +<p>Here are a few examples:</p> <pre> WS : [ \n\u000D] -> skip ; // same as [ \n\r] - + +UNICODE_WS : [\p{White_Space}] -> skip; // match all Unicode whitespace + ID : [a-zA-Z] [a-zA-Z0-9]* ; // match usual identifier spec - + +UNICODE_ID : [\p{Alpha}\p{General_Category=Other_Letter}] [\p{Alnum}\p{General_Category=Other_Letter}]* ; // match full Unicode alphabetic ids + +EMOJI : [\u{1F4A9}\u{1F926}] ; // note Unicode code points > U+FFFF + DASHBRACK : [\-\]]+ ; // match - or ] one or more times </pre> </td> diff --git a/doc/lexicon.md b/doc/lexicon.md index 804a745..078dc3e 100644 --- a/doc/lexicon.md +++ b/doc/lexicon.md @@ -79,16 +79,20 @@ These more or less correspond to `isJavaIdentifierPart` and `isJavaIdentifierSta ## Literals -ANTLR does not distinguish between character and string literals as most languages do. All literal strings one or more characters in length are enclosed in single quotes such as `’;’`, `’if’`, `’>=’`, and `’\’'` (refers to the one-character string containing the single quote character). Literals never contain regular expressions. +ANTLR does not distinguish between character and string literals as most languages do. All literal strings one or more characters in length are enclosed in single quotes such as `’;’`, `’if’`, `’>=’`, and `’\’` (refers to the one-character string containing the single quote character). Literals never contain regular expressions. -Literals can contain Unicode escape sequences of the form `\uXXXX`, where XXXX is the hexadecimal Unicode character value. For example, `’\u00E8’` is the French letter with a grave accent: `’è’`. ANTLR also understands the usual special escape sequences: `’\n’` (newline), `’\r’` (carriage return), `’\t’` (tab), `’\b’` (backspace), and `’\f’` (form feed). You can use Unicode characters directly within literals or use the Unicode escape sequences: +Literals can contain Unicode escape sequences of the form `’\uXXXX’` (for Unicode code points up to `’U+FFFF’`) or `’\u{XXXXXX}’` (for all Unicode code points), where `’XXXX’` is the hexadecimal Unicode code point value. + +For example, `’\u00E8’` is the French letter with a grave accent: `’è’`, and `’\u{1F4A9}’` is the famous emoji: `’💩’`. + +ANTLR also understands the usual special escape sequences: `’\n’` (newline), `’\r’` (carriage return), `’\t’` (tab), `’\b’` (backspace), and `’\f’` (form feed). You can use Unicode code points directly within literals or use the Unicode escape sequences: ``` grammar Foreign; a : '外' ; ``` -The recognizers that ANTLR generates assume a character vocabulary containing all Unicode characters. The input file encoding assumed by the runtime library depends on the target language. For the Java target, the runtime library assumes files are in UTF-8. Using the constructors, you can specify a different encoding. See, for example, ANTLR’s `ANTLRFileStream`. +The recognizers that ANTLR generates assume a character vocabulary containing all Unicode characters. The input file encoding assumed by the runtime library depends on the target language. For the Java target, the runtime library assumes files are in UTF-8. Using the factory methods in `CharStreams`, you can specify a different encoding. ## Actions diff --git a/doc/parsing-binary-files.md b/doc/parsing-binary-files.md index 0014691..d7e9017 100644 --- a/doc/parsing-binary-files.md +++ b/doc/parsing-binary-files.md @@ -15,7 +15,7 @@ grammar IP; file : ip+ (MARKER ip)* ; -ip : BYTE '.' BYTE '.' BYTE '.' BYTE ; +ip : BYTE BYTE BYTE BYTE ; MARKER : '\u00CA' '\u00FE' ; BYTE : '\u0000'..'\u00FF' ; @@ -42,7 +42,7 @@ public class WriteBinaryFile { }; public static void main(String[] args) throws IOException { - Files.write(new File("resources/ips").toPath(), bytes); + Files.write(new File("/tmp/ips").toPath(), bytes); } } ``` @@ -50,14 +50,15 @@ public class WriteBinaryFile { Now we need to create a stream of bytes satisfactory to ANTLR, which is as simple as: ```java -ANTLRFileStream bytesAsChar = new ANTLRFileStream("resources/ips", "ISO-8859-1"); +CharStream bytesAsChar = CharStreams.fromFileName("/tmp/ips", StandardCharsets.ISO_8859_1); ``` The `ISO-8859-1` encoding is just the 8-bit char encoding for LATIN-1, which effectively tells the stream to treat each byte as a character. That's what we want. Then we have the usual test rig: ```java -ANTLRFileStream bytesAsChar = new ANTLRFileStream("resources/ips", "ISO-8859-1"); +//ANTLRFileStream bytesAsChar = new ANTLRFileStream("/tmp/ips", "ISO-8859-1"); DEPRECATED in 4.7 +CharStream bytesAsChar = CharStreams.fromFileName("/tmp/ips", StandardCharsets.ISO_8859_1); IPLexer lexer = new IPLexer(bytesAsChar); CommonTokenStream tokens = new CommonTokenStream(lexer); IPParser parser = new IPParser(tokens); @@ -93,6 +94,8 @@ We can't just print out the text because we are not reading in text. We need to ## Custom stream +(*ANTLRFileStream was deprecated in 4.7*) + If you want to play around with the stream, you can. Here's an example that alters how "text" is computed from the byte stream (which changes how tokens print out their text as well): ```java @@ -124,7 +127,7 @@ class BinaryANTLRFileStream extends ANTLRFileStream { The new test code starts out like this: ```java -ANTLRFileStream bytesAsChar = new BinaryANTLRFileStream("resources/ips"); +ANTLRFileStream bytesAsChar = new BinaryANTLRFileStream("/tmp/ips"); IPLexer lexer = new IPLexer(bytesAsChar); ... ``` @@ -155,9 +158,9 @@ Error handling proceeds exactly like any other parser. For example, let's alter ```java public static final byte[] bytes = { - (byte)172, '.', 0, '.', '.', 1, (byte)0xCA, (byte)0xFE, // OOOPS - (byte)10, '.', 10, '.', 10, '.', 1, (byte)0xCA, (byte)0xFE, - (byte)10, '.', 10, '.', 10, '.', 99 + (byte)172, 0, 1, (byte)0xCA, (byte)0xFE, // OOOPS + (byte)10, 10, 10, 1, (byte)0xCA, (byte)0xFE, + (byte)10, 10, 10, 99 }; ``` diff --git a/doc/releasing-antlr.md b/doc/releasing-antlr.md index ad0df08..49bcd18 100644 --- a/doc/releasing-antlr.md +++ b/doc/releasing-antlr.md @@ -9,17 +9,17 @@ Create a pre-release or full release at github; [Example 4.5-rc-1](https://githu Wack any existing tag as mvn will create one and it fails if already there. ``` -$ git tag -d 4.6 -$ git push origin :refs/tags/4.6 -$ git push upstream :refs/tags/4.6 +$ git tag -d 4.7 +$ git push origin :refs/tags/4.7 +$ git push upstream :refs/tags/4.7 ``` ### Create release candidate tag ```bash -$ git tag -a 4.6-rc1 -m 'heading towards 4.6' -$ git push origin 4.6-rc1 -$ git push upstream 4.6-rc1 +$ git tag -a 4.7-rc1 -m 'heading towards 4.7' +$ git push origin 4.7-rc1 +$ git push upstream 4.7-rc1 ``` ## Bump version @@ -32,22 +32,29 @@ Edit the repository looking for 4.5 or whatever and update it. Bump version in t * runtime/Python3/setup.py * runtime/Python3/src/antlr4/Recognizer.py * runtime/CSharp/runtime/CSharp/Antlr4.Runtime/Properties/AssemblyInfo.cs - * runtime/CSharp/build/version.ps1 + * runtime/CSharp/runtime/CSharp/Antlr4.Runtime/Antlr4.Runtime.dotnet.csproj * runtime/JavaScript/src/antlr4/package.json * runtime/JavaScript/src/antlr4/Recognizer.js * runtime/Cpp/VERSION * runtime/Cpp/runtime/src/RuntimeMetaData.cpp * runtime/Cpp/cmake/ExternalAntlr4Cpp.cmake + * runtime/Cpp/demo/generate.cmd + * runtime/Go/antlr/recognizer.go + * runtime/Swift/Antlr4/org/antlr/v4/runtime/RuntimeMetaData.swift + * tool/src/org/antlr/v4/codegen/target/GoTarget.java * tool/src/org/antlr/v4/codegen/target/CppTarget.java * tool/src/org/antlr/v4/codegen/target/CSharpTarget.java * tool/src/org/antlr/v4/codegen/target/JavaScriptTarget.java * tool/src/org/antlr/v4/codegen/target/Python2Target.java * tool/src/org/antlr/v4/codegen/target/Python3Target.java + * tool/src/org/antlr/v4/codegen/target/SwiftTarget.java + * tool/src/org/antlr/v4/codegen/Target.java + * tool/resources/org/antlr/v4/tool/templates/codegen/Swift/Swift.stg Here is a simple script to display any line from the critical files with, say, `4.5` in it: ```bash -find /tmp/antlr4 -type f -exec grep -l '4\.5' {} \; +find tool runtime -type f -exec grep -l '4\.6' {} \; ``` Commit to repository. @@ -99,7 +106,7 @@ Here is the file template ## Maven deploy snapshot -The goal is to get a snapshot, such as `4.6-SNAPSHOT`, to the staging server: [antlr4 tool](https://oss.sonatype.org/content/repositories/snapshots/org/antlr/antlr4) and [antlr4 java runtime](https://oss.sonatype.org/content/repositories/snapshots/org/antlr/antlr4-runtime). +The goal is to get a snapshot, such as `4.7-SNAPSHOT`, to the staging server: [antlr4 tool](https://oss.sonatype.org/content/repositories/snapshots/org/antlr/antlr4) and [antlr4 java runtime](https://oss.sonatype.org/content/repositories/snapshots/org/antlr/antlr4-runtime). Do this: @@ -107,15 +114,15 @@ Do this: $ mvn deploy -DskipTests ... [INFO] --- maven-deploy-plugin:2.7:deploy (default-deploy) @ antlr4-tool-testsuite --- -Downloading: https://oss.sonatype.org/content/repositories/snapshots/org/antlr/antlr4-tool-testsuite/4.6-SNAPSHOT/maven-metadata.xml -Uploading: https://oss.sonatype.org/content/repositories/snapshots/org/antlr/antlr4-tool-testsuite/4.6-SNAPSHOT/antlr4-tool-testsuite-4.6-20161211.173752-1.jar -Uploaded: https://oss.sonatype.org/content/repositories/snapshots/org/antlr/antlr4-tool-testsuite/4.6-SNAPSHOT/antlr4-tool-testsuite-4.6-20161211.173752-1.jar (3 KB at 3.4 KB/sec) -Uploading: https://oss.sonatype.org/content/repositories/snapshots/org/antlr/antlr4-tool-testsuite/4.6-SNAPSHOT/antlr4-tool-testsuite-4.6-20161211.173752-1.pom -Uploaded: https://oss.sonatype.org/content/repositories/snapshots/org/antlr/antlr4-tool-testsuite/4.6-SNAPSHOT/antlr4-tool-testsuite-4.6-20161211.173752-1.pom (3 KB at 6.5 KB/sec) +Downloading: https://oss.sonatype.org/content/repositories/snapshots/org/antlr/antlr4-tool-testsuite/4.7-SNAPSHOT/maven-metadata.xml +Uploading: https://oss.sonatype.org/content/repositories/snapshots/org/antlr/antlr4-tool-testsuite/4.7-SNAPSHOT/antlr4-tool-testsuite-4.7-20161211.173752-1.jar +Uploaded: https://oss.sonatype.org/content/repositories/snapshots/org/antlr/antlr4-tool-testsuite/4.7-SNAPSHOT/antlr4-tool-testsuite-4.7-20161211.173752-1.jar (3 KB at 3.4 KB/sec) +Uploading: https://oss.sonatype.org/content/repositories/snapshots/org/antlr/antlr4-tool-testsuite/4.7-SNAPSHOT/antlr4-tool-testsuite-4.7-20161211.173752-1.pom +Uploaded: https://oss.sonatype.org/content/repositories/snapshots/org/antlr/antlr4-tool-testsuite/4.7-SNAPSHOT/antlr4-tool-testsuite-4.7-20161211.173752-1.pom (3 KB at 6.5 KB/sec) Downloading: https://oss.sonatype.org/content/repositories/snapshots/org/antlr/antlr4-tool-testsuite/maven-metadata.xml Downloaded: https://oss.sonatype.org/content/repositories/snapshots/org/antlr/antlr4-tool-testsuite/maven-metadata.xml (371 B at 1.4 KB/sec) -Uploading: https://oss.sonatype.org/content/repositories/snapshots/org/antlr/antlr4-tool-testsuite/4.6-SNAPSHOT/maven-metadata.xml -Uploaded: https://oss.sonatype.org/content/repositories/snapshots/org/antlr/antlr4-tool-testsuite/4.6-SNAPSHOT/maven-metadata.xml (774 B at 1.8 KB/sec) +Uploading: https://oss.sonatype.org/content/repositories/snapshots/org/antlr/antlr4-tool-testsuite/4.7-SNAPSHOT/maven-metadata.xml +Uploaded: https://oss.sonatype.org/content/repositories/snapshots/org/antlr/antlr4-tool-testsuite/4.7-SNAPSHOT/maven-metadata.xml (774 B at 1.8 KB/sec) Uploading: https://oss.sonatype.org/content/repositories/snapshots/org/antlr/antlr4-tool-testsuite/maven-metadata.xml Uploaded: https://oss.sonatype.org/content/repositories/snapshots/org/antlr/antlr4-tool-testsuite/maven-metadata.xml (388 B at 0.9 KB/sec) [INFO] ------------------------------------------------------------------------ @@ -152,22 +159,38 @@ With JDK 1.7 (not 6 or 8), do this: mvn release:prepare -Darguments="-DskipTests" ``` +Side note to set jdk 1.7 on os x: + +```bash +alias java='/Library/Java/JavaVirtualMachines/jdk1.7.0_21.jdk/Contents/Home/bin/java' +alias javac='/Library/Java/JavaVirtualMachines/jdk1.7.0_21.jdk/Contents/Home/bin/javac' +alias javadoc='/Library/Java/JavaVirtualMachines/jdk1.7.0_21.jdk/Contents/Home/bin/javadoc' +alias jar='/Library/Java/JavaVirtualMachines/jdk1.7.0_21.jdk/Contents/Home/bin/jar' +``` + +You should see 0x33 in generated .class files after 0xCAFEBABE; see [Java SE 7 = 51 (0x33 hex)](https://en.wikipedia.org/wiki/Java_class_file): + +```bash +beast:/tmp/org/antlr/v4 $ od -h Tool.class |head -1 +0000000 feca beba 0000 3300 fa04 0207 0ab8 0100 +``` + It will start out by asking you the version number: ``` ... -What is the release version for "ANTLR 4"? (org.antlr:antlr4-master) 4.6: : 4.6 -What is the release version for "ANTLR 4 Runtime"? (org.antlr:antlr4-runtime) 4.6: : -What is the release version for "ANTLR 4 Tool"? (org.antlr:antlr4) 4.6: : -What is the release version for "ANTLR 4 Maven plugin"? (org.antlr:antlr4-maven-plugin) 4.6: : -What is the release version for "ANTLR 4 Runtime Test Generator"? (org.antlr:antlr4-runtime-testsuite) 4.6: : -What is the release version for "ANTLR 4 Tool Tests"? (org.antlr:antlr4-tool-testsuite) 4.6: : -What is SCM release tag or label for "ANTLR 4"? (org.antlr:antlr4-master) antlr4-master-4.6: : 4.6 -What is the new development version for "ANTLR 4"? (org.antlr:antlr4-master) 4.6.1-SNAPSHOT: +What is the release version for "ANTLR 4"? (org.antlr:antlr4-master) 4.7: : 4.7 +What is the release version for "ANTLR 4 Runtime"? (org.antlr:antlr4-runtime) 4.7: : +What is the release version for "ANTLR 4 Tool"? (org.antlr:antlr4) 4.7: : +What is the release version for "ANTLR 4 Maven plugin"? (org.antlr:antlr4-maven-plugin) 4.7: : +What is the release version for "ANTLR 4 Runtime Test Generator"? (org.antlr:antlr4-runtime-testsuite) 4.7: : +What is the release version for "ANTLR 4 Tool Tests"? (org.antlr:antlr4-tool-testsuite) 4.7: : +What is SCM release tag or label for "ANTLR 4"? (org.antlr:antlr4-master) antlr4-master-4.7: : 4.7 +What is the new development version for "ANTLR 4"? (org.antlr:antlr4-master) 4.7.1-SNAPSHOT: ... ``` -Maven will go through your pom.xml files to update versions from 4.6-SNAPSHOT to 4.6 for release and then to 4.6.1-SNAPSHOT after release, which is done with: +Maven will go through your pom.xml files to update versions from 4.7-SNAPSHOT to 4.7 for release and then to 4.7.1-SNAPSHOT after release, which is done with: ```bash mvn release:perform -Darguments="-DskipTests" @@ -186,11 +209,11 @@ and on the left click "Staging Repositories". You click the staging repo and clo Copy the jars to antlr.org site and update download/index.html ```bash -cp ~/.m2/repository/org/antlr/antlr4-runtime/4.6/antlr4-runtime-4.6.jar ~/antlr/sites/website-antlr4/download/antlr-runtime-4.6.jar -cp ~/.m2/repository/org/antlr/antlr4/4.6/antlr4-4.6-complete.jar ~/antlr/sites/website-antlr4/download/antlr-4.6-complete.jar +cp ~/.m2/repository/org/antlr/antlr4-runtime/4.7/antlr4-runtime-4.7.jar ~/antlr/sites/website-antlr4/download/antlr-runtime-4.7.jar +cp ~/.m2/repository/org/antlr/antlr4/4.7/antlr4-4.7-complete.jar ~/antlr/sites/website-antlr4/download/antlr-4.7-complete.jar cd ~/antlr/sites/website-antlr4/download -git add antlr-4.6-complete.jar -git add antlr-runtime-4.6.jar +git add antlr-4.7-complete.jar +git add antlr-runtime-4.7.jar ``` Update on site: @@ -202,7 +225,7 @@ Update on site: * scripts/topnav.js ``` -git commit -a -m 'add 4.6 jars' +git commit -a -m 'add 4.7 jars' git push origin gh-pages ``` @@ -212,16 +235,24 @@ git push origin gh-pages ```bash cd runtime/JavaScript/src -zip -r /tmp/antlr-javascript-runtime-4.6.zip antlr4 -cp /tmp/antlr-javascript-runtime-4.6.zip ~/antlr/sites/website-antlr4/download +zip -r /tmp/antlr-javascript-runtime-4.7.zip antlr4 +cp /tmp/antlr-javascript-runtime-4.7.zip ~/antlr/sites/website-antlr4/download # git add, commit, push ``` +**Push to npm** + +```bash +cd runtime/JavaScript/src +npm login +npm publish antlr4 +``` + Move target to website ```bash pushd ~/antlr/sites/website-antlr4/download -git add antlr-javascript-runtime-4.6.zip +git add antlr-javascript-runtime-4.7.zip git commit -a -m 'update JS runtime' git push origin gh-pages popd @@ -229,91 +260,55 @@ popd ### CSharp -*Publishing to Nuget from Linux/MacOSX* +*Publishing to Nuget from Windows* -**Getting ready to run Nuget** +**Install the pre-requisites** Of course you need Mono and `nuget` to be installed. On mac: -```bash -brew install mono -brew install nuget -``` +- .NET build tools - can be loaded from [here](https://www.visualstudio.com/downloads/) +- nuget - download [nuget.exe](https://www.nuget.org/downloads) +- dotnet - follow [the instructions here](https://www.microsoft.com/net/core) -Or, you can [download nuget.exe](https://dist.nuget.org/win-x86-commandline/latest/nuget.exe). - -From the shell on mac, you can check all is ok by typing - -```bash -nuget -``` +Alternatively, you can install Visual Studio 2017 and make sure to check boxes with .NET Core SDK. -This should display the nuget help. +You also need to enable .NET Framework 3.5 support in Windows "Programs and Features". -**Creating the assembly** +If everything is ok, the following command will restore nuget packages, build Antlr for .NET Standard and .NET 3.5 and create nuget package: -```bash -$ cd runtime/CSharp/runtime/CSharp/Antlr4.Runtime -$ xbuild /p:Configuration=Release Antlr4.Runtime.mono.csproj -... - Copying file from '/Users/parrt/antlr/code/antlr4/runtime/CSharp/runtime/CSharp/Antlr4.Runtime/obj/net20/Release/Antlr4.Runtime.Standard.dll' to '/Users/parrt/antlr/code/antlr4/runtime/CSharp/runtime/CSharp/Antlr4.Runtime/lib/Release/Antlr4.Runtime.Standard.dll' -Done building project "/Users/parrt/antlr/code/antlr4/runtime/CSharp/runtime/CSharp/Antlr4.Runtime/Antlr4.Runtime.mono.csproj". +```PS +msbuild /target:restore /target:rebuild /target:pack /property:Configuration=Release .\Antlr4.dotnet.sln /verbosity:minimal ``` -Alternately, you may want to build ANTLR using Xamarin Studio Community (free). +This should display something like this: -**Packaging for NuGet** +**Creating and packaging the assembly** -```bash -cd runtime/CSharp/runtime/CSharp ``` +Microsoft (R) Build Engine version 15.4.8.50001 for .NET Framework +Copyright (C) Microsoft Corporation. All rights reserved. -which is where the `Package.nuspec` file resides. - -Type the following command: - -```bash -$ nuget pack Package.nuspec -Attempting to build package from 'Package.nuspec'. -Successfully created package '/Users/parrt/antlr/code/antlr4/runtime/CSharp/runtime/CSharp/Antlr4.Runtime.Standard.4.6.0.nupkg'. + Restoring packages for C:\Code\antlr4-fork\runtime\CSharp\runtime\CSharp\Antlr4.Runtime\Antlr4.Runtime.dotnet.csproj... + Generating MSBuild file C:\Code\antlr4-fork\runtime\CSharp\runtime\CSharp\Antlr4.Runtime\obj\Antlr4.Runtime.dotnet.csproj.nuget.g.props. + Generating MSBuild file C:\Code\antlr4-fork\runtime\CSharp\runtime\CSharp\Antlr4.Runtime\obj\Antlr4.Runtime.dotnet.csproj.nuget.g.targets. + Restore completed in 427.62 ms for C:\Code\antlr4-fork\runtime\CSharp\runtime\CSharp\Antlr4.Runtime\Antlr4.Runtime.dotnet.csproj. + Antlr4.Runtime.dotnet -> C:\Code\antlr4-fork\runtime\CSharp\runtime\CSharp\Antlr4.Runtime\lib\Release\netstandard1.3\Antlr4.Runtime.Standard.dll + Antlr4.Runtime.dotnet -> C:\Code\antlr4-fork\runtime\CSharp\runtime\CSharp\Antlr4.Runtime\lib\Release\net35\Antlr4.Runtime.Standard.dll + Successfully created package 'C:\Code\antlr4-fork\runtime\CSharp\runtime\CSharp\Antlr4.Runtime\lib\Release\Antlr4.Runtime.Standard.4.7.2.nupkg'. ``` -This should display: Successfully created package *<package-path>* - **Publishing to NuGet** You need to be a NuGet owner for "ANTLR 4 Standard Runtime" -As a registered NuGet user, you can then manually upload the package spec here (`runtime/CSharp/runtime/CSharp/Package.nuspec`): [https://www.nuget.org/packages/manage/upload](https://www.nuget.org/packages/manage/upload) +As a registered NuGet user, you can then manually upload the package here: [https://www.nuget.org/packages/manage/upload](https://www.nuget.org/packages/manage/upload) Alternately, you can publish from the cmd line. You need to get your NuGet key from [https://www.nuget.org/account#](https://www.nuget.org/account#) and then from the cmd line, you can then type: -```bash +```cmd nuget push Antlr4.Runtime.Standard.<version>.nupkg <your-key> -Source https://www.nuget.org/api/v2/package ``` -**Creating DLLs** - -```bash -cd ~/antlr/code/antlr4/runtime/CSharp/runtime/CSharp -# kill previous ones manually as "xbuild /t:Clean" didn't seem to do it -rm Antlr4.Runtime/bin/net20/Release/Antlr4.Runtime.dll -rm Antlr4.Runtime/obj/net20/Release/Antlr4.Runtime.dll -# build -xbuild /p:Configuration=Release Antlr4.Runtime/Antlr4.Runtime.mono.csproj -# zip it up to get a version number on zip filename -zip --junk-paths /tmp/antlr-csharp-runtime-4.6.zip Antlr4.Runtime/bin/net35/Release/Antlr4.Runtime.dll -cp /tmp/antlr-csharp-runtime-4.6.zip ~/antlr/sites/website-antlr4/download -``` - -Move target to website - -```bash -pushd ~/antlr/sites/website-antlr4/download -git add antlr-csharp-runtime-4.6.zip -git commit -a -m 'update C# runtime' -git push origin gh-pages -popd -``` +Nuget packages are also accessible as artifacts of [AppVeyor builds](https://ci.appveyor.com/project/parrt/antlr4/build/artifacts). ### Python @@ -358,7 +353,7 @@ python setup.py register -r pypi python setup.py sdist bdist_wininst upload -r pypi ``` -Add links to the artifacts from download.html +There are links to the artifacts in [download.html](http://www.antlr.org/download.html) already. ### C++ @@ -378,6 +373,7 @@ On a Mac (with XCode 7+ installed): ```bash cd runtime/Cpp ./deploy-macos.sh +cp antlr4-cpp-runtime-macos.zip ~/antlr/sites/website-antlr4/download/antlr4-cpp-runtime-4.7-macos.zip ``` On any Mac or Linux machine: @@ -385,6 +381,7 @@ On any Mac or Linux machine: ```bash cd runtime/Cpp ./deploy-source.sh +cp antlr4-cpp-runtime-source.zip ~/antlr/sites/website-antlr4/download/antlr4-cpp-runtime-4.7-source.zip ``` On a Windows machine the build scripts checks if VS 2013 and/or VS 2015 are installed and builds binaries for each, if found. This script requires 7z to be installed (http://7-zip.org). @@ -392,15 +389,17 @@ On a Windows machine the build scripts checks if VS 2013 and/or VS 2015 are inst ```bash cd runtime/Cpp deploy-windows.cmd +cp antlr4-cpp-runtime-vs2015.zip ~/antlr/sites/website-antlr4/download/antlr4-cpp-runtime-4.7-vs2015.zip ``` Move target to website (**_rename to a specific ANTLR version first if needed_**): ```bash pushd ~/antlr/sites/website-antlr4/download -git add antlr4cpp-runtime-macos.zip -git add antlr4cpp-runtime-windows.zip -git add antlr4cpp-runtime-source.zip +# vi index.html +git add antlr4cpp-runtime-4.7-macos.zip +git add antlr4cpp-runtime-4.7-windows.zip +git add antlr4cpp-runtime-4.7-source.zip git commit -a -m 'update C++ runtime' git push origin gh-pages popd @@ -408,7 +407,7 @@ popd ## Update javadoc for runtime and tool -First gen javadoc: +First, gen javadoc: ```bash $ cd antlr4 @@ -422,9 +421,9 @@ cd ~/antlr/sites/website-antlr4/api git checkout gh-pages git pull origin gh-pages cd Java -jar xvf ~/.m2/repository/org/antlr/antlr4-runtime/4.6/antlr4-runtime-4.6-javadoc.jar +jar xvf ~/.m2/repository/org/antlr/antlr4-runtime/4.7/antlr4-runtime-4.7-javadoc.jar cd ../JavaTool -jar xvf ~/.m2/repository/org/antlr/antlr4/4.6/antlr4-4.6-javadoc.jar +jar xvf ~/.m2/repository/org/antlr/antlr4/4.7/antlr4-4.7-javadoc.jar git commit -a -m 'freshen api doc' git push origin gh-pages ``` diff --git a/doc/resources/CaseChangingCharStream.cs b/doc/resources/CaseChangingCharStream.cs new file mode 100644 index 0000000..9f73a03 --- /dev/null +++ b/doc/resources/CaseChangingCharStream.cs @@ -0,0 +1,105 @@ +/* Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. + * Use of this file is governed by the BSD 3-clause license that + * can be found in the LICENSE.txt file in the project root. + */ +using System; +using Antlr4.Runtime.Misc; + +namespace Antlr4.Runtime +{ + /// <summary> + /// This class supports case-insensitive lexing by wrapping an existing + /// <see cref="ICharStream"/> and forcing the lexer to see either upper or + /// lowercase characters. Grammar literals should then be either upper or + /// lower case such as 'BEGIN' or 'begin'. The text of the character + /// stream is unaffected. Example: input 'BeGiN' would match lexer rule + /// 'BEGIN' if constructor parameter upper=true but getText() would return + /// 'BeGiN'. + /// </summary> + public class CaseChangingCharStream : ICharStream + { + private ICharStream stream; + private bool upper; + + /// <summary> + /// Constructs a new CaseChangingCharStream wrapping the given <paramref name="stream"/> forcing + /// all characters to upper case or lower case. + /// </summary> + /// <param name="stream">The stream to wrap.</param> + /// <param name="upper">If true force each symbol to upper case, otherwise force to lower.</param> + public CaseChangingCharStream(ICharStream stream, bool upper) + { + this.stream = stream; + this.upper = upper; + } + + public int Index + { + get + { + return stream.Index; + } + } + + public int Size + { + get + { + return stream.Size; + } + } + + public string SourceName + { + get + { + return stream.SourceName; + } + } + + public void Consume() + { + stream.Consume(); + } + + [return: NotNull] + public string GetText(Interval interval) + { + return stream.GetText(interval); + } + + public int LA(int i) + { + int c = stream.LA(i); + + if (c <= 0) + { + return c; + } + + char o = (char)c; + + if (upper) + { + return (int)char.ToUpperInvariant(o); + } + + return (int)char.ToLowerInvariant(o); + } + + public int Mark() + { + return stream.Mark(); + } + + public void Release(int marker) + { + stream.Release(marker); + } + + public void Seek(int index) + { + stream.Seek(index); + } + } +} diff --git a/doc/resources/CaseChangingCharStream.java b/doc/resources/CaseChangingCharStream.java new file mode 100644 index 0000000..d069d01 --- /dev/null +++ b/doc/resources/CaseChangingCharStream.java @@ -0,0 +1,81 @@ +package org.antlr.v4.runtime; + +import org.antlr.v4.runtime.misc.Interval; + +/** + * This class supports case-insensitive lexing by wrapping an existing + * {@link CharStream} and forcing the lexer to see either upper or + * lowercase characters. Grammar literals should then be either upper or + * lower case such as 'BEGIN' or 'begin'. The text of the character + * stream is unaffected. Example: input 'BeGiN' would match lexer rule + * 'BEGIN' if constructor parameter upper=true but getText() would return + * 'BeGiN'. + */ +public class CaseChangingCharStream implements CharStream { + + final CharStream stream; + final boolean upper; + + /** + * Constructs a new CaseChangingCharStream wrapping the given {@link CharStream} forcing + * all characters to upper case or lower case. + * @param stream The stream to wrap. + * @param upper If true force each symbol to upper case, otherwise force to lower. + */ + public CaseChangingCharStream(CharStream stream, boolean upper) { + this.stream = stream; + this.upper = upper; + } + + @Override + public String getText(Interval interval) { + return stream.getText(interval); + } + + @Override + public void consume() { + stream.consume(); + } + + @Override + public int LA(int i) { + int c = stream.LA(i); + if (c <= 0) { + return c; + } + if (upper) { + return Character.toUpperCase(c); + } + return Character.toLowerCase(c); + } + + @Override + public int mark() { + return stream.mark(); + } + + @Override + public void release(int marker) { + stream.release(marker); + } + + @Override + public int index() { + return stream.index(); + } + + @Override + public void seek(int index) { + stream.seek(index); + } + + @Override + public int size() { + return stream.size(); + } + + @Override + public String getSourceName() { + return stream.getSourceName(); + } +} diff --git a/doc/resources/CaseInsensitiveInputStream.js b/doc/resources/CaseInsensitiveInputStream.js new file mode 100644 index 0000000..5ec762d --- /dev/null +++ b/doc/resources/CaseInsensitiveInputStream.js @@ -0,0 +1,54 @@ +// +/* Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. + * Use of this file is governed by the BSD 3-clause license that + * can be found in the LICENSE.txt file in the project root. + */ +// + +function CaseInsensitiveInputStream(stream, upper) { + this._stream = stream; + this._case = upper ? String.toUpperCase : String.toLowerCase; + return this; +} + +CaseInsensitiveInputStream.prototype.LA = function (offset) { + c = this._stream.LA(i); + if (c <= 0) { + return c; + } + return this._case.call(String.fromCodePoint(c)) +}; + +CaseInsensitiveInputStream.prototype.reset = function() { + return this._stream.reset(); +}; + +CaseInsensitiveInputStream.prototype.consume = function() { + return this._stream.consume(); +}; + +CaseInsensitiveInputStream.prototype.LT = function(offset) { + return this._stream.LT(offset); +}; + +CaseInsensitiveInputStream.prototype.mark = function() { + return this._stream.mark(); +}; + +CaseInsensitiveInputStream.prototype.release = function(marker) { + return this._stream.release(marker); +}; + +CaseInsensitiveInputStream.prototype.seek = function(_index) { + return this._stream.getText(start, stop); +}; + +CaseInsensitiveInputStream.prototype.getText = function(start, stop) { + return this._stream.getText(start, stop); +}; + +CaseInsensitiveInputStream.prototype.toString = function() { + return this._stream.toString(); +}; + +exports.CaseInsensitiveInputStream = CaseInsensitiveInputStream; diff --git a/doc/resources/case_changing_stream.go b/doc/resources/case_changing_stream.go new file mode 100644 index 0000000..2963acf --- /dev/null +++ b/doc/resources/case_changing_stream.go @@ -0,0 +1,37 @@ +package antlr + +import ( + "unicode" +) + +// CaseChangingStream wraps an existing CharStream, but upper cases, or +// lower cases the input before it is tokenized. +type CaseChangingStream struct { + CharStream + + upper bool +} + +// NewCaseChangingStream returns a new CaseChangingStream that forces +// all tokens read from the underlying stream to be either upper case +// or lower case based on the upper argument. +func NewCaseChangingStream(in CharStream, upper bool) *CaseChangingStream { + return &CaseChangingStream{ + in, upper, + } +} + +// LA gets the value of the symbol at offset from the current position +// from the underlying CharStream and converts it to either upper case +// or lower case. +func (is *CaseChangingStream) LA(offset int) int { + in := is.CharStream.LA(offset) + if in < 0 { + // Such as antlr.TokenEOF which is -1 + return in + } + if is.upper { + return int(unicode.ToUpper(rune(in))) + } + return int(unicode.ToLower(rune(in))) +} diff --git a/doc/runtimetests-overview.md b/doc/runtimetests-overview.md deleted file mode 100644 index 1308586..0000000 --- a/doc/runtimetests-overview.md +++ /dev/null @@ -1,41 +0,0 @@ -# Overview of the ANTLR Runtime Test Suite - -An important part of ANTLR4 is its runtime test suite, which consist of 2 subparts: - -* Tests for the tool itself -* Tests for the ANTLR runtime - -Usually the tests are executed while compiling and installing an ANTLR4 jar from source code. The command for that is simply: - -```bash -$ mvn install -``` - -to be executed in the root of the ANTLR4 repository. More details about this can be found in [Building ANTLR](building-antlr.md). - -However, you don't need to run the installation again and again just to run the tests. Instead use - -```bash -$ mvn test -``` - -to only trigger testing. You can find all runtime tests in the [runtime-testsuite/resources/org/antlr/v4/test/runtime](../runtime-testsuite/resources/org/antlr/v4/test/runtime) subfolder (tool tests under [tool-testsuite/test/org/antlr/v4/test/tool](../tool-testsuite/test/org/antlr/v4/test/tool)). The tool tests are just a bunch of Java test cases that test the tool's internal behavior (e.g. for code generation). We focus on the runtime tests here. - -The underlying process of running the tests is quite a complicated setup to cater especially for a flexible test specification that can run with different target runtimes. Everything runs in Java, except for the actual target runtime tests. Runtime tests run first, followed by the tool tests. If there was a test failure in the first step, the tool tests are not executed, however. These are the steps involved when running the runtime tests: - -* Generate Java JUnit test cases from the test templates, once for each target (C++, C#, Python, Java, Javascript atm.). -* These test cases generate grammar files when executed and run the target specific parser generation step, including compiling a binary, if necessary (e.g. for C++ and C#). -* Finally run the compiled test module using the input specified in the test template. The output (usually a token or parse tree dump) is then compared against the expected output, specified in the test template as well. This also includes any error messages written to the console. - -## Generating JUnit Tests - -The test specification part makes heavy use of the StringTemplate engine to allow defining target language agnostic tests. For that all tests are described in template (`stg`) files. You can find them in the [templates](../runtime-testsuite/resources/org/antlr/v4/test/runtime/templates) subfolder of the runtime tests folder. Read more about the folder structure in the [antlr-project-testing.md](antlr-project-testing.md) file. As lined out there you have to run - -```bash -$ mvn -Pgen generate-test-sources -``` -everytime you change any of the test templates or your target language specific template (which is used to translate certain text to your specific language). And a small hint: this command can be executed from the ANTLR source root as well. No need to dig into a subfolder. - -## Running the Generated Tests - -After generation you can run the tests as written above (`mvn install` or `mvn test`, both recompile ANTLR if necessary), which takes about 40 minutes for the full set (of which 30 mins are alone consumed by the C++ target tests). Which tests actually run is controlled by the [runtime tests pom.xml file](../runtime-testsuite/pom.xml). Look for the `maven-surefire-plugin` plugin entry and especially its includes. If you ever want to run tests only for a specific target, comment out all other `<include>` elements. For a specific test change the wildcard to that specific test name. This is especially helpful when debugging a test (e.g. when it fails) or when creating/changing tests. Additionally, some targets require to install additional dependencies you may not want to add to your box (e.g. mono, python 3.5) just to run e.g. the Java or C++ tests. diff --git a/doc/swift-target.md b/doc/swift-target.md index 69eb88e..e438908 100644 --- a/doc/swift-target.md +++ b/doc/swift-target.md @@ -1,9 +1,15 @@ # ANTLR4 Language Target, Runtime for Swift +## Performance Note + +To use ANTLR4 Swift target in production environment, make sure to turn on compiler optimizations by following [these instructions](https://github.com/apple/swift-package-manager/blob/master/Documentation/Usage.md#build-configurations) if you use SwiftPM to build your project. If you are using Xcode to build your project, it's unlikely you will not use `release` build for production build. + +Conclusion is, you need to turn on `release` mode (which will have all the optimization pre configured for you) so the ANTLR4 Swift target can have reasonable parsing speed. + ## Install ANTLR4 Make sure you have the ANTLR -installed.[The getting started guide](getting-started.md) should get +installed. [The getting started guide](getting-started.md) should get you started. ## Create a Swift lexer or parser @@ -13,87 +19,175 @@ except you need to specify the language target, for example: ``` $ antlr4 -Dlanguage=Swift MyGrammar.g4 ``` + +If you integrate this as a build step inside Xcode, then you should use the +"gnu" message format to have any error messages parsed by Xcode. You may +also want to use the `-o` option to put the autogenerated files in a +separate subdirectory. + +``` +antlr4 -Dlanguage=Swift -message-format gnu -o Autogen MyGrammar.g4 +``` + For a full list of antlr4 tool options, please visit the [tool documentation page](tool-options.md). ## Build your Swift project with ANTLR runtime -The following instructions are assuming Xcode as the IDE: +### Note + +We use __boot.py__ script located at the root of the Swift runtime folder +`antlr4/runtime/Swift` to provide additional support for both Xcode-based +projects and SPM-based projects. Below sections are organized for both of +the flavors. If you want to quickly get started, try: + +``` +python boot.py --help +``` + +for information about this script. + +### Xcode Projects + +Note that even if you are otherwise using ANTLR from a binary distribution, +you should compile the ANTLR Swift runtime from source, because the Swift +language does not yet have a stable ABI. + +ANTLR uses Swift Package Manager to generate Xcode project files. Note that +Swift Package Manager does not currently support iOS, watchOS, or tvOS, so +if you wish to use those platforms, you will need to alter the project build +settings manually as appropriate. + +#### Download source code for ANTLR + +``` +git clone https://github.com/antlr/antlr4 +``` + +#### Generate Xcode project for ANTLR runtime + +The `boot.py` script includes a wrapper around `swift package +generate-xcodeproj`. Use this to generate `Antlr4.xcodeproj` for the ANTLR +Swift runtime. (using _swift package generate-xcodeproj_ is not recommended) +since the project is dependent on some parser files generated by _boot.py_. + +``` +cd antlr4/runtime/Swift +python boot.py --gen-xcodeproj +``` + +#### Import ANTLR Swift runtime into your project -* __Add parser/lexer to project__. Make sure the parsers/lexers +Open your own project in Xcode. + +Open Finder in the `runtime/Swift` directory: + +``` +# From antlr4/runtime/Swift +open . +``` + +Drag `Antlr4.xcodeproj` into your project. + +After this is done, your Xcode project navigator will be something like the +screenshot below. In this example, your own project is "Smalltalk", and you +will be able to see `Antlr4.xcodeproj` shown as a contained project. + +<img src=images/xcodenav.png width="300"> + +#### Edit the build settings if necessary + +Swift Package Manager currently does not support iOS, watchOS, or tvOS. If +you wish to build for those platforms, you will need to alter the project +build settings manually. + +#### Add generated parser and lexer to project + +Make sure the parsers/lexers generated in __step 2__ are added to the project. To do this, you can drag the generated files from Finder to the Xcode IDE. Remember to check __Copy items if needed__ to make sure the files are actually moved into the project folder instead of symbolic links (see the screenshot below). After moving you will be able to see your files in -the project navigator. But when you open one of the files, you will -see Xcode complaining the module "Antlr4" could not be found at the -import statement. This is expected, since we still need the ANTLR -Swift runtime for those missing symbols. +the project navigator. Make sure that the Target Membership settings +are correct for your project. <img src=images/dragfile.png width="500"> -* __Download ANTLR runtime__. Due to unstable ABI of Swift language, -there will not be a single "library" for the Swift ANTLR runtime for -now. To get Swift ANTLR runtime, clone the ANTLR repository. Open it -in finder. From the root directory of the repo, go to runtime/Swift -folder. You will see the Xcode project manifest file: -__Antlr4.xcodeproj__. - -* __Import ANTLR Swift runtime into project__. Drag Antlr4.xcodeproj -into your project, after this is done, your Xcode project navigator -will be something like the screenshot below. In this case, your own -project is "Smalltalk", and you will be able to see the -Antlr4.xcodeproj shown as a contained project. The error message will -still be there, that's because we still need to tell Xcode how to find -the runtime. +#### Add the ANTLR Swift runtime as a dependency -<img src=images/xcodenav.png width="300"> - -* __Build ANTLR runtime__. By expanding the "Products" folder in the -inner project (Antlr4.xcodeproj), you will see two Antlr4.framework -files. ".framework" file is the swift version of ".jar", ".a" as in -JAVA, C/C++ Initially those two files should be red, that's because -they are not built. To build, click the "target selection" button -right next to your Xcode run button. And in the drop down select the -target you want to build. And you will see the two Antlr4.framework -files are for iOS and OSX, as shown below. After target selection, -press "CMD+B", and Xcode will build the framework for you. Then you -will see one of the frameworks become black. - -<img src=images/targetselection.png width="500"> - -* __Add dependencies__. Simply adding ANTLR Swift runtime and build -the artifact is not enough. You still need to specify -dependencies. Click your own project (Smalltalk), and you will see -project setting page. Go to "Build Phase", and inside it make sure -your ANTLR Swift runtime framework is added to both "__Target -Dependencies__" and "__Link Binary With Libraries__" sections, as -shown below. After correctly added dependencies, the error message for -importing library will be gone. +Select your own project in Xcode and go to the Build Phases settings panel. +Add the ANTLR runtime under __Target Dependencies__ and __Link Binary With +Libraries__. <img src=images/xcodedep.png width="800"> -## Example playground +#### Build your project -The Swift runtime includes an Xcode playground to get started with. +The runtime and generated grammar should now build correctly. -First go to the ANTLR4 repository, and open -`runtime/Swift/Antlr4.xcworkspace` in Xcode. Select "Antlr4 OSX > My -Mac" as the build target, and build the project as normal. The -playground should then be active. +### Swift Package Manager Projects -The playground includes a simple grammar called "Hello", and an -example for walking the parse tree. You should see in the playground -output that it is printing messages for each node in the parse tree as -it walks. +Since we cannot have a separate repository for Swift target (see issue [#1774](https://github.com/antlr/antlr4/issues/1774)), +and Swift is currently not ABI stable. We currently support support SPM-based +projects by creating temporary local repository. -The grammar is defined in the playground's `Resources/Hello.g4`. The -parser was generated from the grammar using ANTLR like this: +For people using [Swift Package Manager](https://swift.org/package-manager/), +the __boot.py__ script supports generating local repository that can be used +as a dependency to your project. Simply run: -``` -antlr4 -Dlanguage=Swift -visitor -o ../Sources/Autogen Hello.g4 +``` +python boot.py --gen-spm-module +``` + +The prompt will show something like below: + +<img src=images/gen_spm_module.png width="800"> + +Put the SPM directive that contains the url to temporary repository to your +project's Package.swift. And run `swift build` in your project. + +The project is generated in your system's `/tmp/` directory, if you find it +inconvenient, consider copy that generated ANTLR repository to some place +that won't be cleaned automatically and update `url` parameter in your +`Package.swift` file. + +## Swift access levels + +You may use the `accessLevel` option to control the access levels on generated +code. This option can either be specified with `-DaccessLevel=value` on +the `antlr4` command line, or inside your `.g4` file like this: + +``` +options { + accessLevel = 'value'; +} ``` -The example tree walker is in Sources/HelloWalker.swift. - +By default (with the `accessLevel` option unspecified) the generated code +uses the following access levels: + +* `open` for anything that you can feasibly extend with subclassing: +the generated parser, lexer, and context classes, the the listener and +visitor base classes, and all their accessor and setter functions. +* `public` for anything that should not be subclassed, but otherwise is +useful to client code: protocols, initializers, and static definitions such +as the lexer tokens, symbol names, and so on. +* `internal` or `private` for anything that should not be accessed directly. + +If you specify `accessLevel = 'public'` then all items that are `open` by +default will use `public` instead. Otherwise, the behavior is the same as +the default. + +If you specify `accessLevel = ''` or `accessLevel='internal'` then all items +that are `open` or `public` by default will use Swift's default (internal) +access level instead. + +Those are the only supported values for `accessLevel` when using the Swift +code-generator. + +We recommend using `accessLevel = ''`. Even if you are creating a parser +as part of a library, you would usually want to wrap it in an API of your +own and keep the ANTLR-generated parser internal to your module. You +only need to use the less restrictive access levels if you need to expose +the parser directly as part of your own module's API. diff --git a/doc/tool-options.md b/doc/tool-options.md index fe5d4a3..1944542 100644 --- a/doc/tool-options.md +++ b/doc/tool-options.md @@ -4,7 +4,7 @@ If you invoke the ANTLR tool without command line arguments, you’ll get a help ```bash $ antlr4 -ANTLR Parser Generator Version 4.5 +ANTLR Parser Generator Version 4.7.1 -o ___ specify output directory where all output is generated -lib ___ specify location of grammars, tokens files -atn generate rule augmented transition network diagrams @@ -23,6 +23,7 @@ ANTLR Parser Generator Version 4.5 -XdbgSTWait wait for STViz to close before continuing -Xforce-atn use the ATN simulator for all predictions -Xlog dump lots of logging info to antlr-timestamp.log + -Xexact-output-dir all output goes into -o dir regardless of paths/package ``` Here are more details on the options: @@ -159,3 +160,175 @@ This option creates a log file containing lots of information messages from ANTL $ antlr4 -Xlog T.g4 wrote ./antlr-2012-09-06-17.56.19.log ``` + +## `-Xexact-output-dir` + +(*See the [discussion](https://github.com/antlr/antlr4/pull/2065)*). + +All output goes into `-o` dir regardless of paths/package. + +* Output `-o` directory specifier is the exact directory containing the output. Previously it would include the relative path specified on the grammar itself for the purposes of packages. + +**new**: `-o /tmp subdir/T.g4` => `/tmp/subdir/T.java` +**old**: `-o /tmp subdir/T.g4` => `/tmp/T.java` + +* Previously we looked for the tokens vocab file in the `-lib` dir or in the output dir. **New**: also look in the directory containing the grammar, particularly if it it is specified with a path. + +### Example for the output directory (4.7) + +Here is the existing 4.7 functionality. + +(For these examples, assume a4.7 and a4.7.1 are aliases to the right version of ANTLR's `org.antlr.v4.Tool`.) + +```bash +$ cd /tmp/parrt +$ tree +. +├── B.g4 +└── src + └── pkg + └── A.g4 +$ a4.7 -o /tmp/build src/pkg/A.g4 +$ tree /tmp/build +/tmp/build/ +└── src + └── pkg + ├── A.tokens + ├── ABaseListener.java + ├── ALexer.java + ├── ALexer.tokens + ├── AListener.java + └── AParser.java +``` + +Now, let's build a grammar that sits in the current directory: + +```bash +$ a4.7 -o /tmp/build B.g4 +$ tree /tmp/build +/tmp/build +├── B.tokens +├── BBaseListener.java +├── BLexer.java +├── BLexer.tokens +├── BListener.java +├── BParser.java +└── src + └── pkg + ├── A.tokens + ├── ABaseListener.java + ├── ALexer.java + ├── ALexer.tokens + ├── AListener.java + └── AParser.java +``` + +Finally, if we don't specify the output directory, it paid attention to the relative path specified on the input grammar: + +```bash +$ a4.7 src/pkg/A.g4 +$ tree +. +├── B.g4 +└── src + └── pkg + ├── A.g4 + ├── A.tokens + ├── ABaseListener.java + ├── ALexer.java + ├── ALexer.tokens + ├── AListener.java + └── AParser.java +``` + +### Example for the output directory (4.7.1 with -Xexact-output-dir) + +Now, the output directory is the exact directory where output is generated regardless of relative paths on the grammar + +```bash +$ cd /tmp/parrt +$ a4.7.1 -Xexact-output-dir -o /tmp/build src/pkg/A.g4 +$ tree /tmp/build +/tmp/build +├── A.tokens +├── ABaseListener.java +├── ALexer.java +├── ALexer.tokens +├── AListener.java +└── AParser.java +``` + +If you use the package option, it still does not change where the output is generated if you use `-o` + +```bash +$ a4.7.1 -Xexact-output-dir -package pkg -o /tmp/build src/pkg/A.g4 +$ tree /tmp/build +/tmp/build +├── A.tokens +├── ABaseListener.java +├── ALexer.java +├── ALexer.tokens +├── AListener.java +└── AParser.java +``` + +4.7.1 does however add the package specification into the generated files: + +```bash +$ grep package /tmp/build/A*.java +/tmp/build/ABaseListener.java:package pkg; +/tmp/build/ALexer.java:package pkg; +/tmp/build/AListener.java:package pkg; +/tmp/build/AParser.java:package pkg; +``` + +Compare this to 4.7: + +```bash +$ a4.7 -package pkg -o /tmp/build src/pkg/A.g4 +beast:/tmp/parrt $ tree /tmp/build +/tmp/build +└── src + └── pkg + ├── A.tokens + ├── ABaseListener.java + ├── ALexer.java + ├── ALexer.tokens + ├── AListener.java + └── AParser.java +``` + +### Example of where it looks for tokens vocab + +In 4.7, we got an error for an obvious case that should work: + +```bash +$ cd /tmp/parrt +$ tree +. +└── src + └── pkg + ├── L.g4 + └── P.g4 +$ a4.7 -o /tmp/build src/pkg/*.g4 +error(160): P.g4:2:21: cannot find tokens file /tmp/build/L.tokens +warning(125): P.g4:3:4: implicit definition of token A in parser +``` + +In 4.7.1 it looks in the directory containing the grammars as well: + +```bash +$ a4.7.1 -o /tmp/build src/pkg/*.g4 +$ tree /tmp/build +/tmp/build +├── L.java +├── L.tokens +├── P.java +├── P.tokens +├── PBaseListener.java +├── PListener.java +└── src + └── pkg + ├── L.java + └── L.tokens +``` diff --git a/doc/unicode.md b/doc/unicode.md new file mode 100644 index 0000000..45bf90f --- /dev/null +++ b/doc/unicode.md @@ -0,0 +1,168 @@ +# Lexers and Unicode text + +Prior to ANTLR 4.7, generated lexers in most targets only supported part of the Unicode standard (code points up to `U+FFFF`). As of ANTLR 4.7, the lexers in all language runtimes support the full range of Unicode code points up to `U+10FFFF`. + +C++, Python, Go, and Swift APIs didn't need any API changes to support Unicode code points, so we decided to leave those class interfaces as-is. + +Java, C#, and JavaScript runtimes required changes and, rather than break the previous interface, we deprecated them. (The *Java-target* deprecated `ANTLRInputStream` and `ANTLRFileStream` APIs only support Unicode code points up to `U+FFFF`.) Now, those targets must create `CharStream`s from input using `CharStreams.fromPath()`, `CharStreams.fromFileName()`, etc... + +A big shout out to Ben Hamilton (github bhamiltoncx) for his superhuman +efforts across all targets to get true support for U+10FFFF code points. + +## Example + +The Java, C#, and JavaScript runtimes use the new factory style stream creation interface. For example, here is some sample Java code that uses `CharStreams.fromPath()`: + +```java +public static void main(String[] args) { + CharStream charStream = CharStreams.fromPath(Paths.get(args[0])); + Lexer lexer = new UnicodeLexer(charStream); + CommonTokenStream tokens = new CommonTokenStream(lexer); + tokens.fill(); + for (Token token : tokens.getTokens()) { + System.out.println("Got token: " + token.toString()); + } +} +``` + +# Unicode Code Points in Lexer Grammars + +To refer to Unicode [code points](https://en.wikipedia.org/wiki/Code_point) +in lexer grammars, use the `\u` string escape plus up to 4 hex digits. For example, to create +a lexer rule for a single Cyrillic character by creating a range from +`U+0400` to `U+04FF`: + +```ANTLR +CYRILLIC : '\u0400'..'\u04FF' ; // or [\u0400-\u04FF] without quotes +``` + +Unicode literals larger than U+FFFF must use the extended `\u{12345}` syntax. For example, to create a lexer rule for a selection of smiley faces +from the [Emoticons Unicode block](http://www.unicode.org/charts/PDF/U1F600.pdf): + +```ANTLR +EMOTICONS : ('\u{1F600}' | '\u{1F602}' | '\u{1F615}') ; // or [\u{1F600}\u{1F602}\u{1F615}] +``` + +Finally, lexer char sets can include Unicode properties. Each Unicode code point has at least one property that describes the type group to which it belongs (e.g. alpha, number, punctuation). Other properties can be the language script or special binary properties and Unicode code blocks. That means however, that a property specifies a group of code points, hence they are only allowed in lexer char sets. + +```ANTLR +EMOJI : [\p{Emoji}] ; +JAPANESE : [\p{Script=Hiragana}\p{Script=Katakana}\p{Script=Han}] ; +NOT_CYRILLIC : [\P{Script=Cyrillic}] ; +``` + +See [lexer-rules.md](lexer-rules.md#lexer-rule-elements) for more detail on Unicode +escapes in lexer rules. + +## Migration + + +Code for **4.6** looked like this: + + +```java +CharStream input = new ANTLRFileStream("myinputfile"); +JavaLexer lexer = new JavaLexer(input); +CommonTokenStream tokens = new CommonTokenStream(lexer); +``` + +(It didn't use UTF-8 by default, despite the documentation saying so previously; it actually depended on the calling environments default.) + +Code for **4.7** assumes UTF-8 by default and looks like this: + +```java +CharStream input = CharStreams.fromFileName("inputfile"); +JavaLexer lexer = new JavaLexer(input); +CommonTokenStream tokens = new CommonTokenStream(lexer); +``` + +Or, if you'd like to specify the file encoding: + +```java +CharStream input = CharStreams.fromFileName("inputfile", Charset.forName("windows-1252")); +``` + +### Motivation + +After a [lively discussion](https://github.com/antlr/antlr4/pull/1771), I (parrt) decided not to simply gut the 4.6 `ANTLRFileStream` and `ANTLRInputStream` to incorporate the new U+10FFFF functionality. I decided to *deprecate* the old interface and recommend use of the new interface to prevent confusion. My reasoning is summarized as: + +* I didn't like the idea of breaking all 4.6 code. To get the previous streams to properly support > 16 bit Unicode would require a lot of changes to the method signatures. +* Using `int` buffer element types would double the size of memory required to hold streams in memory, given that we buffer everything (and I didn't want to change that aspect of the streams). +* The new factory-style interface supports creation of the smallest possible code point buffer element size according to the Unicode code points found in the input stream. This means using half as much memory +as the old {@link ANTLRFileStream}, which assumed 16-bit characters, for ASCII text. +* Through some [serious testing and performance tweaking](https://github.com/antlr/antlr4/pull/1781), the new streams perform as fast or faster than the 4.6 streams. + +**WARNING**. *You should avoid using both the deprecated and the new streams* in the same application because you will see +a nontrivial performance degradation. This speed hit is because the +`Lexer`'s internal code goes from a monomorphic to megamorphic +dynamic dispatch to get characters from the input stream. Java's +on-the-fly compiler (JIT) is unable to perform the same optimizations +so stick with either the old or the new streams, if performance is +a primary concern. See the [extreme debugging and spelunking](https://github.com/antlr/antlr4/pull/1781) needed to identify this issue in our timing rig. + +### Legacy grammar using surrogate code units + +Legacy grammars that did their own UTF-16 surrogate code unit matching will need to continue to use `ANTLRInputStream` (Java target) until the parser-application code can upgrade to `CharStreams` interface. Then the surrogate code unit matching should be removed from the grammar in favor of letting the new streams do the decoding. + +Prior to 4.7, application code could directly pass `Token.getStartIndex()` and `Token.getStopIndex()` to Java and C# String APIs (because both used UTF-16 code units as the fundamental unit of length). With the new streams, clients will have to convert from code point indices to UTF-16 code unit indices. Here is some (Java) code to show you the necessary logic: + +```java +public final class CodePointCounter { + private final String input; + public int inputIndex = 0; + public int codePointIndex = 0; + + public int advanceToIndex(int newCodePointIndex) { + assert newCodePointIndex >= codePointIndex; + while (codePointIndex < newCodePointOffset) { + int codePoint = Character.codePointAt(input, inputIndex); + inputIndex += Character.charCount(codePoint); + codePointIndex++; + } + return inputIndex; + } +} +``` + +### Character Buffering, Unbuffered streams + +The ANTLR character streams still buffer all the input when you create +the stream, as they have done for ~20 years. + +If you need unbuffered +access, please note that it becomes challenging to create +parse trees. The parse tree has to point to tokens which will either +point into a stale location in an unbuffered stream or you have to copy +the characters out of the buffer into the token. That defeats the purpose +of unbuffered input. See the [ANTLR 4 book](https://www.amazon.com/Definitive-ANTLR-4-Reference/dp/1934356999) "13.8 Unbuffered Character and Token Streams". Unbuffered streams are primarily +useful for processing infinite streams *during the parse* and require that you manually buffer characters. Use `UnbufferedCharStream` and `UnbufferedTokenStream`. + +```java +CharStream input = new UnbufferedCharStream(is); +CSVLexer lex = new CSVLexer(input); // copy text out of sliding buffer and store in tokens +lex.setTokenFactory(new CommonTokenFactory(true)); +TokenStream tokens = new UnbufferedTokenStream<CommonToken>(lex); +CSVParser parser = new CSVParser(tokens); +parser.setBuildParseTree(false); +parser.file(); +``` + +Your grammar that needs to have embedded actions that access the tokens as they are created, but before they disappear and are garbage collected. For example, + +``` +data : a=INT {int x = Integer.parseInt($a.text);} ; +``` + +From the code comments of `CommonTokenFactory`: + +> That `true` in `new CommonTokenFactory(true)` indicates whether `CommonToken.setText` should be called after +constructing tokens to explicitly set the text. This is useful for cases +where the input stream might not be able to provide arbitrary substrings +of text from the input after the lexer creates a token (e.g. the +implementation of `CharStream.getText` in +`UnbufferedCharStream` throws an +`UnsupportedOperationException`). Explicitly setting the token text +allows `Token.getText` to be called at any time regardless of the +input stream implementation. + +*Currently, only Java, C++, and C# have these unbuffered streams implemented*. @@ -1,145 +1,165 @@ <!-- - ~ Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + ~ Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. ~ Use of this file is governed by the BSD 3-clause license that ~ can be found in the LICENSE.txt file in the project root. --> <project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd"> - <modelVersion>4.0.0</modelVersion> - <parent> - <groupId>org.sonatype.oss</groupId> - <artifactId>oss-parent</artifactId> - <version>9</version> - </parent> - <groupId>org.antlr</groupId> - <artifactId>antlr4-master</artifactId> - <version>4.6</version> - <packaging>pom</packaging> + <modelVersion>4.0.0</modelVersion> + <parent> + <groupId>org.sonatype.oss</groupId> + <artifactId>oss-parent</artifactId> + <version>9</version> + </parent> + <groupId>org.antlr</groupId> + <artifactId>antlr4-master</artifactId> + <version>4.7.1</version> + <packaging>pom</packaging> - <name>ANTLR 4</name> - <description>ANTLR 4 Master Build POM</description> - <url>http://www.antlr.org</url> - <inceptionYear>1992</inceptionYear> - <organization> - <name>ANTLR</name> - <url>http://www.antlr.org</url> - </organization> + <name>ANTLR 4</name> + <description>ANTLR 4 Master Build POM</description> + <url>http://www.antlr.org</url> + <inceptionYear>1992</inceptionYear> + <organization> + <name>ANTLR</name> + <url>http://www.antlr.org</url> + </organization> - <licenses> - <license> - <name>The BSD License</name> - <url>http://www.antlr.org/license.html</url> - <distribution>repo</distribution> - </license> - </licenses> + <licenses> + <license> + <name>The BSD License</name> + <url>http://www.antlr.org/license.html</url> + <distribution>repo</distribution> + </license> + </licenses> - <developers> - <developer> - <name>Terence Parr</name> - <url>http://parrt.cs.usfca.edu</url> - <roles> - <role>Project lead - ANTLR</role> - </roles> - </developer> - <developer> - <name>Sam Harwell</name> - <url>http://tunnelvisionlabs.com</url> - <roles> - <role>Developer</role> - </roles> - </developer> - <developer> - <name>Eric Vergnaud</name> - <roles> - <role>Developer - JavaScript, C#, Python 2, Python 3</role> - </roles> - </developer> - <developer> - <name>Peter Boyer</name> - <roles> - <role>Developer - Go</role> - </roles> - </developer> - <developer> - <name>Jim Idle</name> - <email>jimi@idle.ws</email> - <url>http://www.linkedin.com/in/jimidle</url> - <roles> - <role>Developer - Maven Plugin</role> - </roles> - </developer> - <developer> - <name>Mike Lischke</name> - <roles> - <role>Developer - C++ Target</role> - </roles> - </developer> - </developers> + <developers> + <developer> + <name>Terence Parr</name> + <url>http://parrt.cs.usfca.edu</url> + <roles> + <role>Project lead - ANTLR</role> + </roles> + </developer> + <developer> + <name>Sam Harwell</name> + <url>http://tunnelvisionlabs.com</url> + <roles> + <role>Developer</role> + </roles> + </developer> + <developer> + <name>Eric Vergnaud</name> + <roles> + <role>Developer - JavaScript, C#, Python 2, Python 3</role> + </roles> + </developer> + <developer> + <name>Peter Boyer</name> + <roles> + <role>Developer - Go</role> + </roles> + </developer> + <developer> + <name>Jim Idle</name> + <email>jimi@idle.ws</email> + <url>http://www.linkedin.com/in/jimidle</url> + <roles> + <role>Developer - Maven Plugin</role> + </roles> + </developer> + <developer> + <name>Mike Lischke</name> + <roles> + <role>Developer - C++ Target</role> + </roles> + </developer> + <developer> + <name>Tom Everett</name> + <roles> + <role>Developer</role> + </roles> + </developer> + </developers> - <modules> - <module>runtime/Java</module> - <module>tool</module> - <module>antlr4-maven-plugin</module> - <module>tool-testsuite</module> - <module>runtime-testsuite/annotations</module> - <module>runtime-testsuite/processors</module> - <module>runtime-testsuite</module> - </modules> + <modules> + <module>runtime/Java</module> + <module>tool</module> + <module>antlr4-maven-plugin</module> + <module>tool-testsuite</module> + <module>runtime-testsuite/annotations</module> + <module>runtime-testsuite/processors</module> + <module>runtime-testsuite</module> + </modules> - <properties> - <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding> - <project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding> - <antlr.testinprocess>true</antlr.testinprocess> - <maven.compiler.source>1.7</maven.compiler.source> - <maven.compiler.target>1.7</maven.compiler.target> - </properties> + <properties> + <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding> + <project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding> + <antlr.testinprocess>true</antlr.testinprocess> + <maven.compiler.source>1.7</maven.compiler.source> + <maven.compiler.target>1.7</maven.compiler.target> + </properties> - <mailingLists> - <mailingList> - <name>antlr-discussion</name> - <archive>https://groups.google.com/forum/?fromgroups#!forum/antlr-discussion</archive> - </mailingList> - </mailingLists> + <mailingLists> + <mailingList> + <name>antlr-discussion</name> + <archive>https://groups.google.com/forum/?fromgroups#!forum/antlr-discussion</archive> + </mailingList> + </mailingLists> - <issueManagement> - <system>GitHub Issues</system> - <url>https://github.com/antlr/antlr4/issues</url> - </issueManagement> + <issueManagement> + <system>GitHub Issues</system> + <url>https://github.com/antlr/antlr4/issues</url> + </issueManagement> - <scm> - <url>https://github.com/antlr/antlr4/tree/master</url> - <connection>scm:git:git://github.com/antlr/antlr4.git</connection> - <developerConnection>scm:git:git@github.com:antlr/antlr4.git</developerConnection> - <tag>HEAD</tag> - </scm> + <scm> + <url>https://github.com/antlr/antlr4/tree/master</url> + <connection>scm:git:git://github.com/antlr/antlr4.git</connection> + <developerConnection>scm:git:git@github.com:antlr/antlr4.git</developerConnection> + <tag>HEAD</tag> + </scm> - <build> - <resources> - <resource> - <directory>resources</directory> - </resource> - </resources> - <testResources> - <testResource> - <directory>test</directory> - </testResource> - </testResources> - <pluginManagement> - <plugins> - <plugin> - <groupId>org.apache.maven.plugins</groupId> - <artifactId>maven-compiler-plugin</artifactId> - <version>3.6.0</version> - <configuration> - <source>${maven.compiler.source}</source> - <target>${maven.compiler.target}</target> - </configuration> - </plugin> - <plugin> - <groupId>org.apache.maven.plugins</groupId> - <artifactId>maven-javadoc-plugin</artifactId> - </plugin> - </plugins> - </pluginManagement> - </build> + <build> + <resources> + <resource> + <directory>resources</directory> + </resource> + </resources> + <testResources> + <testResource> + <directory>test</directory> + </testResource> + </testResources> + <plugins> + <plugin> + <artifactId>maven-clean-plugin</artifactId> + <version>3.0.0</version> + <configuration> + <filesets> + <fileset> + <directory>runtime/Swift/.build</directory> + <directory>runtime/Swift/Tests/Antlr4Tests/gen</directory> + </fileset> + </filesets> + </configuration> + </plugin> + </plugins> + <pluginManagement> + <plugins> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-compiler-plugin</artifactId> + <version>3.6.0</version> + <configuration> + <source>${maven.compiler.source}</source> + <target>${maven.compiler.target}</target> + </configuration> + </plugin> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-javadoc-plugin</artifactId> + </plugin> + </plugins> + </pluginManagement> + </build> </project> diff --git a/runtime-testsuite/annotations/pom.xml b/runtime-testsuite/annotations/pom.xml index 87b617b..b476c0a 100644 --- a/runtime-testsuite/annotations/pom.xml +++ b/runtime-testsuite/annotations/pom.xml @@ -1,5 +1,5 @@ <!-- - ~ Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + ~ Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. ~ Use of this file is governed by the BSD 3-clause license that ~ can be found in the LICENSE.txt file in the project root. --> @@ -9,7 +9,7 @@ <parent> <groupId>org.antlr</groupId> <artifactId>antlr4-master</artifactId> - <version>4.6</version> + <version>4.7.1</version> <relativePath>../../pom.xml</relativePath> </parent> <artifactId>antlr4-runtime-test-annotations</artifactId> diff --git a/runtime-testsuite/annotations/src/org/antlr/v4/test/runtime/CommentHasStringValue.java b/runtime-testsuite/annotations/src/org/antlr/v4/test/runtime/CommentHasStringValue.java index b2bf486..8b48eb1 100644 --- a/runtime-testsuite/annotations/src/org/antlr/v4/test/runtime/CommentHasStringValue.java +++ b/runtime-testsuite/annotations/src/org/antlr/v4/test/runtime/CommentHasStringValue.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/runtime-testsuite/pom.xml b/runtime-testsuite/pom.xml index 824a4c0..7015303 100644 --- a/runtime-testsuite/pom.xml +++ b/runtime-testsuite/pom.xml @@ -1,5 +1,5 @@ <!-- - ~ Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + ~ Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. ~ Use of this file is governed by the BSD 3-clause license that ~ can be found in the LICENSE.txt file in the project root. --> @@ -10,7 +10,7 @@ <parent> <groupId>org.antlr</groupId> <artifactId>antlr4-master</artifactId> - <version>4.6</version> + <version>4.7.1</version> </parent> <artifactId>antlr4-runtime-testsuite</artifactId> <name>ANTLR 4 Runtime Tests (2nd generation)</name> @@ -77,6 +77,11 @@ <version>1.0.4</version> <scope>test</scope> </dependency> + <dependency> + <groupId>org.openjdk.jol</groupId> + <artifactId>jol-core</artifactId> + <version>0.8</version> + </dependency> </dependencies> <build> @@ -95,6 +100,8 @@ <artifactId>maven-surefire-plugin</artifactId> <version>2.19.1</version> <configuration> + <!-- SUREFIRE-951: file.encoding cannot be set via systemPropertyVariables --> + <argLine>-Dfile.encoding=UTF-8</argLine> <includes> <include>**/csharp/Test*.java</include> <include>**/java/Test*.java</include> @@ -118,6 +125,24 @@ </execution> </executions> </plugin> + <plugin> + <groupId>org.antlr</groupId> + <artifactId>antlr4-maven-plugin</artifactId> + <version>${project.version}</version> + <executions> + <execution> + <goals> + <goal>antlr4</goal> + </goals> + <configuration> + <sourceDirectory>${basedir}/test</sourceDirectory> + <outputDirectory>${project.build.directory}/generated-test-sources/antlr4</outputDirectory> + <visitor>true</visitor> + <generateTestSources>true</generateTestSources> + </configuration> + </execution> + </executions> + </plugin> </plugins> </build> diff --git a/runtime-testsuite/processors/pom.xml b/runtime-testsuite/processors/pom.xml index 69fd940..deccaef 100644 --- a/runtime-testsuite/processors/pom.xml +++ b/runtime-testsuite/processors/pom.xml @@ -1,5 +1,5 @@ <!-- - ~ Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + ~ Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. ~ Use of this file is governed by the BSD 3-clause license that ~ can be found in the LICENSE.txt file in the project root. --> @@ -9,7 +9,7 @@ <parent> <groupId>org.antlr</groupId> <artifactId>antlr4-master</artifactId> - <version>4.6</version> + <version>4.7.1</version> <relativePath>../../pom.xml</relativePath> </parent> <artifactId>antlr4-runtime-test-annotation-processors</artifactId> diff --git a/runtime-testsuite/processors/resources/META-INF/services/javax.annotation.processing.Processor b/runtime-testsuite/processors/resources/META-INF/services/javax.annotation.processing.Processor index 08311da..95b9a3a 100644 --- a/runtime-testsuite/processors/resources/META-INF/services/javax.annotation.processing.Processor +++ b/runtime-testsuite/processors/resources/META-INF/services/javax.annotation.processing.Processor @@ -1 +1,7 @@ +# +# Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. +# Use of this file is governed by the BSD 3-clause license that +# can be found in the LICENSE.txt file in the project root. +# + org.antlr.v4.test.runtime.CommentHasStringValueProcessor diff --git a/runtime-testsuite/processors/src/org/antlr/v4/test/runtime/CommentHasStringValueProcessor.java b/runtime-testsuite/processors/src/org/antlr/v4/test/runtime/CommentHasStringValueProcessor.java index b4fcb06..9c902c0 100644 --- a/runtime-testsuite/processors/src/org/antlr/v4/test/runtime/CommentHasStringValueProcessor.java +++ b/runtime-testsuite/processors/src/org/antlr/v4/test/runtime/CommentHasStringValueProcessor.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/runtime/Java/pom.xml b/runtime/Java/pom.xml index 29627bc..c700407 100644 --- a/runtime/Java/pom.xml +++ b/runtime/Java/pom.xml @@ -1,5 +1,5 @@ <!-- - ~ Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + ~ Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. ~ Use of this file is governed by the BSD 3-clause license that ~ can be found in the LICENSE.txt file in the project root. --> @@ -9,7 +9,7 @@ <parent> <groupId>org.antlr</groupId> <artifactId>antlr4-master</artifactId> - <version>4.6</version> + <version>4.7.1</version> <relativePath>../../pom.xml</relativePath> </parent> <artifactId>antlr4-runtime</artifactId> @@ -27,6 +27,7 @@ <plugin> <!-- create src jar --> <groupId>org.apache.maven.plugins</groupId> <artifactId>maven-source-plugin</artifactId> + <version>3.0.1</version> <executions> <execution> <goals> diff --git a/runtime/Java/src/org/antlr/v4/runtime/ANTLRErrorListener.java b/runtime/Java/src/org/antlr/v4/runtime/ANTLRErrorListener.java index 013f1cf..d7c8bb6 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/ANTLRErrorListener.java +++ b/runtime/Java/src/org/antlr/v4/runtime/ANTLRErrorListener.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/runtime/Java/src/org/antlr/v4/runtime/ANTLRErrorStrategy.java b/runtime/Java/src/org/antlr/v4/runtime/ANTLRErrorStrategy.java index 90c5528..7604791 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/ANTLRErrorStrategy.java +++ b/runtime/Java/src/org/antlr/v4/runtime/ANTLRErrorStrategy.java @@ -1,11 +1,13 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ package org.antlr.v4.runtime; +import org.antlr.v4.runtime.tree.ErrorNode; + /** * The interface for defining strategies to deal with syntax errors encountered * during a parse by ANTLR-generated parsers. We distinguish between three @@ -89,8 +91,9 @@ public interface ANTLRErrorStrategy { * Tests whether or not {@code recognizer} is in the process of recovering * from an error. In error recovery mode, {@link Parser#consume} adds * symbols to the parse tree by calling - * {@link ParserRuleContext#addErrorNode(Token)} instead of - * {@link ParserRuleContext#addChild(Token)}. + * {@link Parser#createErrorNode(ParserRuleContext, Token)} then + * {@link ParserRuleContext#addErrorNode(ErrorNode)} instead of + * {@link Parser#createTerminalNode(ParserRuleContext, Token)}. * * @param recognizer the parser instance * @return {@code true} if the parser is currently recovering from a parse diff --git a/runtime/Java/src/org/antlr/v4/runtime/ANTLRFileStream.java b/runtime/Java/src/org/antlr/v4/runtime/ANTLRFileStream.java index e31ff32..ede83eb 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/ANTLRFileStream.java +++ b/runtime/Java/src/org/antlr/v4/runtime/ANTLRFileStream.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ @@ -12,6 +12,8 @@ import java.io.IOException; /** * This is an {@link ANTLRInputStream} that is loaded from a file all at once * when you construct the object. + * + * @deprecated as of 4.7 Please use {@link CharStreams} interface. */ public class ANTLRFileStream extends ANTLRInputStream { protected String fileName; diff --git a/runtime/Java/src/org/antlr/v4/runtime/ANTLRInputStream.java b/runtime/Java/src/org/antlr/v4/runtime/ANTLRInputStream.java index 7ee3cca..f71a62d 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/ANTLRInputStream.java +++ b/runtime/Java/src/org/antlr/v4/runtime/ANTLRInputStream.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ @@ -19,6 +19,8 @@ import java.util.Arrays; * {@code char[]} to use. * * <p>If you need encoding, pass in stream/reader with correct encoding.</p> + * + * @deprecated as of 4.7 Please use {@link CharStreams} interface. */ public class ANTLRInputStream implements CharStream { public static final int READ_BUFFER_SIZE = 1024; diff --git a/runtime/Java/src/org/antlr/v4/runtime/BailErrorStrategy.java b/runtime/Java/src/org/antlr/v4/runtime/BailErrorStrategy.java index a7f08c9..d6830bb 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/BailErrorStrategy.java +++ b/runtime/Java/src/org/antlr/v4/runtime/BailErrorStrategy.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/runtime/Java/src/org/antlr/v4/runtime/BaseErrorListener.java b/runtime/Java/src/org/antlr/v4/runtime/BaseErrorListener.java index 7f6ea8e..8453422 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/BaseErrorListener.java +++ b/runtime/Java/src/org/antlr/v4/runtime/BaseErrorListener.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/runtime/Java/src/org/antlr/v4/runtime/BufferedTokenStream.java b/runtime/Java/src/org/antlr/v4/runtime/BufferedTokenStream.java index 8ca3928..e1dc4b1 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/BufferedTokenStream.java +++ b/runtime/Java/src/org/antlr/v4/runtime/BufferedTokenStream.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ @@ -89,6 +89,14 @@ public class BufferedTokenStream implements TokenStream { // no resources to release } + /** + * This method resets the token stream back to the first token in the + * buffer. It is equivalent to calling {@link #seek}{@code (0)}. + * + * @see #setTokenSource(TokenSource) + * @deprecated Use {@code seek(0)} instead. + */ + @Deprecated public void reset() { seek(0); } @@ -252,6 +260,7 @@ public class BufferedTokenStream implements TokenStream { this.tokenSource = tokenSource; tokens.clear(); p = -1; + fetchedEOF = false; } public List<Token> getTokens() { return tokens; } @@ -372,7 +381,7 @@ public class BufferedTokenStream implements TokenStream { /** Collect all hidden tokens (any off-default channel) to the right of * the current token up until we see a token on DEFAULT_TOKEN_CHANNEL - * of EOF. + * or EOF. */ public List<Token> getHiddenTokensToRight(int tokenIndex) { return getHiddenTokensToRight(tokenIndex, -1); diff --git a/runtime/Java/src/org/antlr/v4/runtime/CharStream.java b/runtime/Java/src/org/antlr/v4/runtime/CharStream.java index fcd6bfb..57dbfa9 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/CharStream.java +++ b/runtime/Java/src/org/antlr/v4/runtime/CharStream.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/runtime/Java/src/org/antlr/v4/runtime/CharStreams.java b/runtime/Java/src/org/antlr/v4/runtime/CharStreams.java new file mode 100644 index 0000000..73fb875 --- /dev/null +++ b/runtime/Java/src/org/antlr/v4/runtime/CharStreams.java @@ -0,0 +1,306 @@ +/* + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. + * Use of this file is governed by the BSD 3-clause license that + * can be found in the LICENSE.txt file in the project root. + */ + +package org.antlr.v4.runtime; + +import java.io.IOException; +import java.io.InputStream; +import java.io.Reader; +import java.nio.ByteBuffer; +import java.nio.CharBuffer; +import java.nio.channels.Channels; +import java.nio.channels.ReadableByteChannel; +import java.nio.charset.Charset; +import java.nio.charset.CharsetDecoder; +import java.nio.charset.CoderResult; +import java.nio.charset.CodingErrorAction; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; + +/** This class represents the primary interface for creating {@link CharStream}s + * from a variety of sources as of 4.7. The motivation was to support + * Unicode code points > U+FFFF. {@link ANTLRInputStream} and + * {@link ANTLRFileStream} are now deprecated in favor of the streams created + * by this interface. + * + * DEPRECATED: {@code new ANTLRFileStream("myinputfile")} + * NEW: {@code CharStreams.fromFileName("myinputfile")} + * + * WARNING: If you use both the deprecated and the new streams, you will see + * a nontrivial performance degradation. This speed hit is because the + * {@link Lexer}'s internal code goes from a monomorphic to megamorphic + * dynamic dispatch to get characters from the input stream. Java's + * on-the-fly compiler (JIT) is unable to perform the same optimizations + * so stick with either the old or the new streams, if performance is + * a primary concern. See the extreme debugging and spelunking + * needed to identify this issue in our timing rig: + * + * https://github.com/antlr/antlr4/pull/1781 + * + * The ANTLR character streams still buffer all the input when you create + * the stream, as they have done for ~20 years. If you need unbuffered + * access, please note that it becomes challenging to create + * parse trees. The parse tree has to point to tokens which will either + * point into a stale location in an unbuffered stream or you have to copy + * the characters out of the buffer into the token. That defeats the purpose + * of unbuffered input. Per the ANTLR book, unbuffered streams are primarily + * useful for processing infinite streams *during the parse.* + * + * The new streams also use 8-bit buffers when possible so this new + * interface supports character streams that use half as much memory + * as the old {@link ANTLRFileStream}, which assumed 16-bit characters. + * + * A big shout out to Ben Hamilton (github bhamiltoncx) for his superhuman + * efforts across all targets to get true Unicode 3.1 support for U+10FFFF. + * + * @since 4.7 + */ +public final class CharStreams { + private static final int DEFAULT_BUFFER_SIZE = 4096; + + // Utility class; do not construct. + private CharStreams() { } + + /** + * Creates a {@link CharStream} given a path to a UTF-8 + * encoded file on disk. + * + * Reads the entire contents of the file into the result before returning. + */ + public static CharStream fromPath(Path path) throws IOException { + return fromPath(path, StandardCharsets.UTF_8); + } + + /** + * Creates a {@link CharStream} given a path to a file on disk and the + * charset of the bytes contained in the file. + * + * Reads the entire contents of the file into the result before returning. + */ + public static CharStream fromPath(Path path, Charset charset) throws IOException { + long size = Files.size(path); + try (ReadableByteChannel channel = Files.newByteChannel(path)) { + return fromChannel( + channel, + charset, + DEFAULT_BUFFER_SIZE, + CodingErrorAction.REPLACE, + path.toString(), + size); + } + } + + /** + * Creates a {@link CharStream} given a string containing a + * path to a UTF-8 file on disk. + * + * Reads the entire contents of the file into the result before returning. + */ + public static CharStream fromFileName(String fileName) throws IOException { + return fromPath(Paths.get(fileName), StandardCharsets.UTF_8); + } + + /** + * Creates a {@link CharStream} given a string containing a + * path to a file on disk and the charset of the bytes + * contained in the file. + * + * Reads the entire contents of the file into the result before returning. + */ + public static CharStream fromFileName(String fileName, Charset charset) throws IOException { + return fromPath(Paths.get(fileName), charset); + } + + + /** + * Creates a {@link CharStream} given an opened {@link InputStream} + * containing UTF-8 bytes. + * + * Reads the entire contents of the {@code InputStream} into + * the result before returning, then closes the {@code InputStream}. + */ + public static CharStream fromStream(InputStream is) throws IOException { + return fromStream(is, StandardCharsets.UTF_8); + } + + /** + * Creates a {@link CharStream} given an opened {@link InputStream} and the + * charset of the bytes contained in the stream. + * + * Reads the entire contents of the {@code InputStream} into + * the result before returning, then closes the {@code InputStream}. + */ + public static CharStream fromStream(InputStream is, Charset charset) throws IOException { + return fromStream(is, charset, -1); + } + + public static CharStream fromStream(InputStream is, Charset charset, long inputSize) throws IOException { + try (ReadableByteChannel channel = Channels.newChannel(is)) { + return fromChannel( + channel, + charset, + DEFAULT_BUFFER_SIZE, + CodingErrorAction.REPLACE, + IntStream.UNKNOWN_SOURCE_NAME, + inputSize); + } + } + + /** + * Creates a {@link CharStream} given an opened {@link ReadableByteChannel} + * containing UTF-8 bytes. + * + * Reads the entire contents of the {@code channel} into + * the result before returning, then closes the {@code channel}. + */ + public static CharStream fromChannel(ReadableByteChannel channel) throws IOException { + return fromChannel(channel, StandardCharsets.UTF_8); + } + + /** + * Creates a {@link CharStream} given an opened {@link ReadableByteChannel} and the + * charset of the bytes contained in the channel. + * + * Reads the entire contents of the {@code channel} into + * the result before returning, then closes the {@code channel}. + */ + public static CharStream fromChannel(ReadableByteChannel channel, Charset charset) throws IOException { + return fromChannel( + channel, + DEFAULT_BUFFER_SIZE, + CodingErrorAction.REPLACE, + IntStream.UNKNOWN_SOURCE_NAME); + } + + /** + * Creates a {@link CharStream} given a {@link Reader}. Closes + * the reader before returning. + */ + public static CodePointCharStream fromReader(Reader r) throws IOException { + return fromReader(r, IntStream.UNKNOWN_SOURCE_NAME); + } + + /** + * Creates a {@link CharStream} given a {@link Reader} and its + * source name. Closes the reader before returning. + */ + public static CodePointCharStream fromReader(Reader r, String sourceName) throws IOException { + try { + CodePointBuffer.Builder codePointBufferBuilder = CodePointBuffer.builder(DEFAULT_BUFFER_SIZE); + CharBuffer charBuffer = CharBuffer.allocate(DEFAULT_BUFFER_SIZE); + while ((r.read(charBuffer)) != -1) { + charBuffer.flip(); + codePointBufferBuilder.append(charBuffer); + charBuffer.compact(); + } + return CodePointCharStream.fromBuffer(codePointBufferBuilder.build(), sourceName); + } + finally { + r.close(); + } + } + + /** + * Creates a {@link CharStream} given a {@link String}. + */ + public static CodePointCharStream fromString(String s) { + return fromString(s, IntStream.UNKNOWN_SOURCE_NAME); + } + + /** + * Creates a {@link CharStream} given a {@link String} and the {@code sourceName} + * from which it came. + */ + public static CodePointCharStream fromString(String s, String sourceName) { + // Initial guess assumes no code points > U+FFFF: one code + // point for each code unit in the string + CodePointBuffer.Builder codePointBufferBuilder = CodePointBuffer.builder(s.length()); + // TODO: CharBuffer.wrap(String) rightfully returns a read-only buffer + // which doesn't expose its array, so we make a copy. + CharBuffer cb = CharBuffer.allocate(s.length()); + cb.put(s); + cb.flip(); + codePointBufferBuilder.append(cb); + return CodePointCharStream.fromBuffer(codePointBufferBuilder.build(), sourceName); + } + + /** + * Creates a {@link CharStream} given an opened {@link ReadableByteChannel} + * containing UTF-8 bytes. + * + * Reads the entire contents of the {@code channel} into + * the result before returning, then closes the {@code channel}. + */ + public static CodePointCharStream fromChannel( + ReadableByteChannel channel, + int bufferSize, + CodingErrorAction decodingErrorAction, + String sourceName) + throws IOException + { + return fromChannel(channel, StandardCharsets.UTF_8, bufferSize, decodingErrorAction, sourceName, -1); + } + + public static CodePointCharStream fromChannel( + ReadableByteChannel channel, + Charset charset, + int bufferSize, + CodingErrorAction decodingErrorAction, + String sourceName, + long inputSize) + throws IOException + { + try { + ByteBuffer utf8BytesIn = ByteBuffer.allocate(bufferSize); + CharBuffer utf16CodeUnitsOut = CharBuffer.allocate(bufferSize); + if (inputSize == -1) { + inputSize = bufferSize; + } else if (inputSize > Integer.MAX_VALUE) { + // ByteBuffer et al don't support long sizes + throw new IOException(String.format("inputSize %d larger than max %d", inputSize, Integer.MAX_VALUE)); + } + CodePointBuffer.Builder codePointBufferBuilder = CodePointBuffer.builder((int) inputSize); + CharsetDecoder decoder = charset + .newDecoder() + .onMalformedInput(decodingErrorAction) + .onUnmappableCharacter(decodingErrorAction); + + boolean endOfInput = false; + while (!endOfInput) { + int bytesRead = channel.read(utf8BytesIn); + endOfInput = (bytesRead == -1); + utf8BytesIn.flip(); + CoderResult result = decoder.decode( + utf8BytesIn, + utf16CodeUnitsOut, + endOfInput); + if (result.isError() && decodingErrorAction.equals(CodingErrorAction.REPORT)) { + result.throwException(); + } + utf16CodeUnitsOut.flip(); + codePointBufferBuilder.append(utf16CodeUnitsOut); + utf8BytesIn.compact(); + utf16CodeUnitsOut.compact(); + } + // Handle any bytes at the end of the file which need to + // be represented as errors or substitution characters. + CoderResult flushResult = decoder.flush(utf16CodeUnitsOut); + if (flushResult.isError() && decodingErrorAction.equals(CodingErrorAction.REPORT)) { + flushResult.throwException(); + } + utf16CodeUnitsOut.flip(); + codePointBufferBuilder.append(utf16CodeUnitsOut); + + CodePointBuffer codePointBuffer = codePointBufferBuilder.build(); + return CodePointCharStream.fromBuffer(codePointBuffer, sourceName); + } + finally { + channel.close(); + } + } +} diff --git a/runtime/Java/src/org/antlr/v4/runtime/CodePointBuffer.java b/runtime/Java/src/org/antlr/v4/runtime/CodePointBuffer.java new file mode 100644 index 0000000..58035a0 --- /dev/null +++ b/runtime/Java/src/org/antlr/v4/runtime/CodePointBuffer.java @@ -0,0 +1,389 @@ +/* + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. + * Use of this file is governed by the BSD 3-clause license that + * can be found in the LICENSE.txt file in the project root. + */ + +package org.antlr.v4.runtime; + +import java.nio.ByteBuffer; +import java.nio.CharBuffer; +import java.nio.IntBuffer; + +/** + * Wrapper for {@link ByteBuffer} / {@link CharBuffer} / {@link IntBuffer}. + * + * Because Java lacks generics on primitive types, these three types + * do not share an interface, so we have to write one manually. + */ +public class CodePointBuffer { + public enum Type { + BYTE, + CHAR, + INT + } + private final Type type; + private final ByteBuffer byteBuffer; + private final CharBuffer charBuffer; + private final IntBuffer intBuffer; + + private CodePointBuffer(Type type, ByteBuffer byteBuffer, CharBuffer charBuffer, IntBuffer intBuffer) { + this.type = type; + this.byteBuffer = byteBuffer; + this.charBuffer = charBuffer; + this.intBuffer = intBuffer; + } + + public static CodePointBuffer withBytes(ByteBuffer byteBuffer) { + return new CodePointBuffer(Type.BYTE, byteBuffer, null, null); + } + + public static CodePointBuffer withChars(CharBuffer charBuffer) { + return new CodePointBuffer(Type.CHAR, null, charBuffer, null); + } + + public static CodePointBuffer withInts(IntBuffer intBuffer) { + return new CodePointBuffer(Type.INT, null, null, intBuffer); + } + + public int position() { + switch (type) { + case BYTE: + return byteBuffer.position(); + case CHAR: + return charBuffer.position(); + case INT: + return intBuffer.position(); + } + throw new UnsupportedOperationException("Not reached"); + } + + public void position(int newPosition) { + switch (type) { + case BYTE: + byteBuffer.position(newPosition); + break; + case CHAR: + charBuffer.position(newPosition); + break; + case INT: + intBuffer.position(newPosition); + break; + } + } + + public int remaining() { + switch (type) { + case BYTE: + return byteBuffer.remaining(); + case CHAR: + return charBuffer.remaining(); + case INT: + return intBuffer.remaining(); + } + throw new UnsupportedOperationException("Not reached"); + } + + public int get(int offset) { + switch (type) { + case BYTE: + return byteBuffer.get(offset); + case CHAR: + return charBuffer.get(offset); + case INT: + return intBuffer.get(offset); + } + throw new UnsupportedOperationException("Not reached"); + } + + Type getType() { + return type; + } + + int arrayOffset() { + switch (type) { + case BYTE: + return byteBuffer.arrayOffset(); + case CHAR: + return charBuffer.arrayOffset(); + case INT: + return intBuffer.arrayOffset(); + } + throw new UnsupportedOperationException("Not reached"); + } + + byte[] byteArray() { + assert type == Type.BYTE; + return byteBuffer.array(); + } + + char[] charArray() { + assert type == Type.CHAR; + return charBuffer.array(); + } + + int[] intArray() { + assert type == Type.INT; + return intBuffer.array(); + } + + public static Builder builder(int initialBufferSize) { + return new Builder(initialBufferSize); + } + + public static class Builder { + private Type type; + private ByteBuffer byteBuffer; + private CharBuffer charBuffer; + private IntBuffer intBuffer; + private int prevHighSurrogate; + + private Builder(int initialBufferSize) { + type = Type.BYTE; + byteBuffer = ByteBuffer.allocate(initialBufferSize); + charBuffer = null; + intBuffer = null; + prevHighSurrogate = -1; + } + + Type getType() { + return type; + } + + ByteBuffer getByteBuffer() { + return byteBuffer; + } + + CharBuffer getCharBuffer() { + return charBuffer; + } + + IntBuffer getIntBuffer() { + return intBuffer; + } + + public CodePointBuffer build() { + switch (type) { + case BYTE: + byteBuffer.flip(); + break; + case CHAR: + charBuffer.flip(); + break; + case INT: + intBuffer.flip(); + break; + } + return new CodePointBuffer(type, byteBuffer, charBuffer, intBuffer); + } + + private static int roundUpToNextPowerOfTwo(int i) { + int nextPowerOfTwo = 32 - Integer.numberOfLeadingZeros(i - 1); + return (int) Math.pow(2, nextPowerOfTwo); + } + + public void ensureRemaining(int remainingNeeded) { + switch (type) { + case BYTE: + if (byteBuffer.remaining() < remainingNeeded) { + int newCapacity = roundUpToNextPowerOfTwo(byteBuffer.capacity() + remainingNeeded); + ByteBuffer newBuffer = ByteBuffer.allocate(newCapacity); + byteBuffer.flip(); + newBuffer.put(byteBuffer); + byteBuffer = newBuffer; + } + break; + case CHAR: + if (charBuffer.remaining() < remainingNeeded) { + int newCapacity = roundUpToNextPowerOfTwo(charBuffer.capacity() + remainingNeeded); + CharBuffer newBuffer = CharBuffer.allocate(newCapacity); + charBuffer.flip(); + newBuffer.put(charBuffer); + charBuffer = newBuffer; + } + break; + case INT: + if (intBuffer.remaining() < remainingNeeded) { + int newCapacity = roundUpToNextPowerOfTwo(intBuffer.capacity() + remainingNeeded); + IntBuffer newBuffer = IntBuffer.allocate(newCapacity); + intBuffer.flip(); + newBuffer.put(intBuffer); + intBuffer = newBuffer; + } + break; + } + } + + public void append(CharBuffer utf16In) { + ensureRemaining(utf16In.remaining()); + if (utf16In.hasArray()) { + appendArray(utf16In); + } else { + // TODO + throw new UnsupportedOperationException("TODO"); + } + } + + private void appendArray(CharBuffer utf16In) { + assert utf16In.hasArray(); + + switch (type) { + case BYTE: + appendArrayByte(utf16In); + break; + case CHAR: + appendArrayChar(utf16In); + break; + case INT: + appendArrayInt(utf16In); + break; + } + } + + private void appendArrayByte(CharBuffer utf16In) { + assert prevHighSurrogate == -1; + + char[] in = utf16In.array(); + int inOffset = utf16In.arrayOffset() + utf16In.position(); + int inLimit = utf16In.arrayOffset() + utf16In.limit(); + + byte[] outByte = byteBuffer.array(); + int outOffset = byteBuffer.arrayOffset() + byteBuffer.position(); + + while (inOffset < inLimit) { + char c = in[inOffset]; + if (c <= 0xFF) { + outByte[outOffset] = (byte)(c & 0xFF); + } else { + utf16In.position(inOffset - utf16In.arrayOffset()); + byteBuffer.position(outOffset - byteBuffer.arrayOffset()); + if (!Character.isHighSurrogate(c)) { + byteToCharBuffer(utf16In.remaining()); + appendArrayChar(utf16In); + return; + } else { + byteToIntBuffer(utf16In.remaining()); + appendArrayInt(utf16In); + return; + } + } + inOffset++; + outOffset++; + } + + utf16In.position(inOffset - utf16In.arrayOffset()); + byteBuffer.position(outOffset - byteBuffer.arrayOffset()); + } + + private void appendArrayChar(CharBuffer utf16In) { + assert prevHighSurrogate == -1; + + char[] in = utf16In.array(); + int inOffset = utf16In.arrayOffset() + utf16In.position(); + int inLimit = utf16In.arrayOffset() + utf16In.limit(); + + char[] outChar = charBuffer.array(); + int outOffset = charBuffer.arrayOffset() + charBuffer.position(); + + while (inOffset < inLimit) { + char c = in[inOffset]; + if (!Character.isHighSurrogate(c)) { + outChar[outOffset] = c; + } else { + utf16In.position(inOffset - utf16In.arrayOffset()); + charBuffer.position(outOffset - charBuffer.arrayOffset()); + charToIntBuffer(utf16In.remaining()); + appendArrayInt(utf16In); + return; + } + inOffset++; + outOffset++; + } + + utf16In.position(inOffset - utf16In.arrayOffset()); + charBuffer.position(outOffset - charBuffer.arrayOffset()); + } + + private void appendArrayInt(CharBuffer utf16In) { + char[] in = utf16In.array(); + int inOffset = utf16In.arrayOffset() + utf16In.position(); + int inLimit = utf16In.arrayOffset() + utf16In.limit(); + + int[] outInt = intBuffer.array(); + int outOffset = intBuffer.arrayOffset() + intBuffer.position(); + + while (inOffset < inLimit) { + char c = in[inOffset]; + inOffset++; + if (prevHighSurrogate != -1) { + if (Character.isLowSurrogate(c)) { + outInt[outOffset] = Character.toCodePoint((char) prevHighSurrogate, c); + outOffset++; + prevHighSurrogate = -1; + } else { + // Dangling high surrogate + outInt[outOffset] = prevHighSurrogate; + outOffset++; + if (Character.isHighSurrogate(c)) { + prevHighSurrogate = c & 0xFFFF; + } else { + outInt[outOffset] = c & 0xFFFF; + outOffset++; + prevHighSurrogate = -1; + } + } + } else if (Character.isHighSurrogate(c)) { + prevHighSurrogate = c & 0xFFFF; + } else { + outInt[outOffset] = c & 0xFFFF; + outOffset++; + } + } + + if (prevHighSurrogate != -1) { + // Dangling high surrogate + outInt[outOffset] = prevHighSurrogate & 0xFFFF; + outOffset++; + } + + utf16In.position(inOffset - utf16In.arrayOffset()); + intBuffer.position(outOffset - intBuffer.arrayOffset()); + } + + private void byteToCharBuffer(int toAppend) { + byteBuffer.flip(); + // CharBuffers hold twice as much per unit as ByteBuffers, so start with half the capacity. + CharBuffer newBuffer = CharBuffer.allocate(Math.max(byteBuffer.remaining() + toAppend, byteBuffer.capacity() / 2)); + while (byteBuffer.hasRemaining()) { + newBuffer.put((char) (byteBuffer.get() & 0xFF)); + } + type = Type.CHAR; + byteBuffer = null; + charBuffer = newBuffer; + } + + private void byteToIntBuffer(int toAppend) { + byteBuffer.flip(); + // IntBuffers hold four times as much per unit as ByteBuffers, so start with one quarter the capacity. + IntBuffer newBuffer = IntBuffer.allocate(Math.max(byteBuffer.remaining() + toAppend, byteBuffer.capacity() / 4)); + while (byteBuffer.hasRemaining()) { + newBuffer.put(byteBuffer.get() & 0xFF); + } + type = Type.INT; + byteBuffer = null; + intBuffer = newBuffer; + } + + private void charToIntBuffer(int toAppend) { + charBuffer.flip(); + // IntBuffers hold two times as much per unit as ByteBuffers, so start with one half the capacity. + IntBuffer newBuffer = IntBuffer.allocate(Math.max(charBuffer.remaining() + toAppend, charBuffer.capacity() / 2)); + while (charBuffer.hasRemaining()) { + newBuffer.put(charBuffer.get() & 0xFFFF); + } + type = Type.INT; + charBuffer = null; + intBuffer = newBuffer; + } + } +} diff --git a/runtime/Java/src/org/antlr/v4/runtime/CodePointCharStream.java b/runtime/Java/src/org/antlr/v4/runtime/CodePointCharStream.java new file mode 100644 index 0000000..107faa7 --- /dev/null +++ b/runtime/Java/src/org/antlr/v4/runtime/CodePointCharStream.java @@ -0,0 +1,297 @@ +/* + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. + * Use of this file is governed by the BSD 3-clause license that + * can be found in the LICENSE.txt file in the project root. + */ + +package org.antlr.v4.runtime; + +import org.antlr.v4.runtime.misc.Interval; + +import java.nio.charset.StandardCharsets; + +/** + * Alternative to {@link ANTLRInputStream} which treats the input + * as a series of Unicode code points, instead of a series of UTF-16 + * code units. + * + * Use this if you need to parse input which potentially contains + * Unicode values > U+FFFF. + */ +public abstract class CodePointCharStream implements CharStream { + protected final int size; + protected final String name; + + // To avoid lots of virtual method calls, we directly access + // the state of the underlying code points in the + // CodePointBuffer. + protected int position; + + // Use the factory method {@link #fromBuffer(CodePointBuffer)} to + // construct instances of this type. + private CodePointCharStream(int position, int remaining, String name) { + // TODO + assert position == 0; + this.size = remaining; + this.name = name; + this.position = 0; + } + + // Visible for testing. + abstract Object getInternalStorage(); + + /** + * Constructs a {@link CodePointCharStream} which provides access + * to the Unicode code points stored in {@code codePointBuffer}. + */ + public static CodePointCharStream fromBuffer(CodePointBuffer codePointBuffer) { + return fromBuffer(codePointBuffer, UNKNOWN_SOURCE_NAME); + } + + /** + * Constructs a named {@link CodePointCharStream} which provides access + * to the Unicode code points stored in {@code codePointBuffer}. + */ + public static CodePointCharStream fromBuffer(CodePointBuffer codePointBuffer, String name) { + // Java lacks generics on primitive types. + // + // To avoid lots of calls to virtual methods in the + // very hot codepath of LA() below, we construct one + // of three concrete subclasses. + // + // The concrete subclasses directly access the code + // points stored in the underlying array (byte[], + // char[], or int[]), so we can avoid lots of virtual + // method calls to ByteBuffer.get(offset). + switch (codePointBuffer.getType()) { + case BYTE: + return new CodePoint8BitCharStream( + codePointBuffer.position(), + codePointBuffer.remaining(), + name, + codePointBuffer.byteArray(), + codePointBuffer.arrayOffset()); + case CHAR: + return new CodePoint16BitCharStream( + codePointBuffer.position(), + codePointBuffer.remaining(), + name, + codePointBuffer.charArray(), + codePointBuffer.arrayOffset()); + case INT: + return new CodePoint32BitCharStream( + codePointBuffer.position(), + codePointBuffer.remaining(), + name, + codePointBuffer.intArray(), + codePointBuffer.arrayOffset()); + } + throw new UnsupportedOperationException("Not reached"); + } + + @Override + public final void consume() { + if (size - position == 0) { + assert LA(1) == IntStream.EOF; + throw new IllegalStateException("cannot consume EOF"); + } + position = position + 1; + } + + @Override + public final int index() { + return position; + } + + @Override + public final int size() { + return size; + } + + /** mark/release do nothing; we have entire buffer */ + @Override + public final int mark() { + return -1; + } + + @Override + public final void release(int marker) { + } + + @Override + public final void seek(int index) { + position = index; + } + + @Override + public final String getSourceName() { + if (name == null || name.isEmpty()) { + return UNKNOWN_SOURCE_NAME; + } + + return name; + } + + @Override + public final String toString() { + return getText(Interval.of(0, size - 1)); + } + + // 8-bit storage for code points <= U+00FF. + private static final class CodePoint8BitCharStream extends CodePointCharStream { + private final byte[] byteArray; + + private CodePoint8BitCharStream(int position, int remaining, String name, byte[] byteArray, int arrayOffset) { + super(position, remaining, name); + // TODO + assert arrayOffset == 0; + this.byteArray = byteArray; + } + + /** Return the UTF-16 encoded string for the given interval */ + @Override + public String getText(Interval interval) { + int startIdx = Math.min(interval.a, size); + int len = Math.min(interval.b - interval.a + 1, size - startIdx); + + // We know the maximum code point in byteArray is U+00FF, + // so we can treat this as if it were ISO-8859-1, aka Latin-1, + // which shares the same code points up to 0xFF. + return new String(byteArray, startIdx, len, StandardCharsets.ISO_8859_1); + } + + @Override + public int LA(int i) { + int offset; + switch (Integer.signum(i)) { + case -1: + offset = position + i; + if (offset < 0) { + return IntStream.EOF; + } + return byteArray[offset] & 0xFF; + case 0: + // Undefined + return 0; + case 1: + offset = position + i - 1; + if (offset >= size) { + return IntStream.EOF; + } + return byteArray[offset] & 0xFF; + } + throw new UnsupportedOperationException("Not reached"); + } + + @Override + Object getInternalStorage() { + return byteArray; + } + } + + // 16-bit internal storage for code points between U+0100 and U+FFFF. + private static final class CodePoint16BitCharStream extends CodePointCharStream { + private final char[] charArray; + + private CodePoint16BitCharStream(int position, int remaining, String name, char[] charArray, int arrayOffset) { + super(position, remaining, name); + this.charArray = charArray; + // TODO + assert arrayOffset == 0; + } + + /** Return the UTF-16 encoded string for the given interval */ + @Override + public String getText(Interval interval) { + int startIdx = Math.min(interval.a, size - 1); + int len = Math.min(interval.b - interval.a + 1, size); + + // We know there are no surrogates in this + // array, since otherwise we would be given a + // 32-bit int[] array. + // + // So, it's safe to treat this as if it were + // UTF-16. + return new String(charArray, startIdx, len); + } + + @Override + public int LA(int i) { + int offset; + switch (Integer.signum(i)) { + case -1: + offset = position + i; + if (offset < 0) { + return IntStream.EOF; + } + return charArray[offset] & 0xFFFF; + case 0: + // Undefined + return 0; + case 1: + offset = position + i - 1; + if (offset >= size) { + return IntStream.EOF; + } + return charArray[offset] & 0xFFFF; + } + throw new UnsupportedOperationException("Not reached"); + } + + @Override + Object getInternalStorage() { + return charArray; + } + } + + // 32-bit internal storage for code points between U+10000 and U+10FFFF. + private static final class CodePoint32BitCharStream extends CodePointCharStream { + private final int[] intArray; + + private CodePoint32BitCharStream(int position, int remaining, String name, int[] intArray, int arrayOffset) { + super(position, remaining, name); + this.intArray = intArray; + // TODO + assert arrayOffset == 0; + } + + /** Return the UTF-16 encoded string for the given interval */ + @Override + public String getText(Interval interval) { + int startIdx = Math.min(interval.a, size - 1); + int len = Math.min(interval.b - interval.a + 1, size); + + // Note that we pass the int[] code points to the String constructor -- + // this is supported, and the constructor will convert to UTF-16 internally. + return new String(intArray, startIdx, len); + } + + @Override + public int LA(int i) { + int offset; + switch (Integer.signum(i)) { + case -1: + offset = position + i; + if (offset < 0) { + return IntStream.EOF; + } + return intArray[offset]; + case 0: + // Undefined + return 0; + case 1: + offset = position + i - 1; + if (offset >= size) { + return IntStream.EOF; + } + return intArray[offset]; + } + throw new UnsupportedOperationException("Not reached"); + } + + @Override + Object getInternalStorage() { + return intArray; + } + } +} diff --git a/runtime/Java/src/org/antlr/v4/runtime/CommonToken.java b/runtime/Java/src/org/antlr/v4/runtime/CommonToken.java index 015df7f..a34020f 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/CommonToken.java +++ b/runtime/Java/src/org/antlr/v4/runtime/CommonToken.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/runtime/Java/src/org/antlr/v4/runtime/CommonTokenFactory.java b/runtime/Java/src/org/antlr/v4/runtime/CommonTokenFactory.java index da7bace..d52cbae 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/CommonTokenFactory.java +++ b/runtime/Java/src/org/antlr/v4/runtime/CommonTokenFactory.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/runtime/Java/src/org/antlr/v4/runtime/CommonTokenStream.java b/runtime/Java/src/org/antlr/v4/runtime/CommonTokenStream.java index fd6a8eb..153f421 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/CommonTokenStream.java +++ b/runtime/Java/src/org/antlr/v4/runtime/CommonTokenStream.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/runtime/Java/src/org/antlr/v4/runtime/ConsoleErrorListener.java b/runtime/Java/src/org/antlr/v4/runtime/ConsoleErrorListener.java index f7d36b5..aa2e20d 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/ConsoleErrorListener.java +++ b/runtime/Java/src/org/antlr/v4/runtime/ConsoleErrorListener.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/runtime/Java/src/org/antlr/v4/runtime/DefaultErrorStrategy.java b/runtime/Java/src/org/antlr/v4/runtime/DefaultErrorStrategy.java index db5061c..02b5ee5 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/DefaultErrorStrategy.java +++ b/runtime/Java/src/org/antlr/v4/runtime/DefaultErrorStrategy.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ @@ -37,6 +37,21 @@ public class DefaultErrorStrategy implements ANTLRErrorStrategy { protected IntervalSet lastErrorStates; /** + * This field is used to propagate information about the lookahead following + * the previous match. Since prediction prefers completing the current rule + * to error recovery efforts, error reporting may occur later than the + * original point where it was discoverable. The original context is used to + * compute the true expected sets as though the reporting occurred as early + * as possible. + */ + protected ParserRuleContext nextTokensContext; + + /** + * @see #nextTokensContext + */ + protected int nextTokensState; + + /** * {@inheritDoc} * * <p>The default implementation simply calls {@link #endErrorCondition} to @@ -224,10 +239,21 @@ public class DefaultErrorStrategy implements ANTLRErrorStrategy { int la = tokens.LA(1); // try cheaper subset first; might get lucky. seems to shave a wee bit off - if ( recognizer.getATN().nextTokens(s).contains(la) || la==Token.EOF ) return; + IntervalSet nextTokens = recognizer.getATN().nextTokens(s); + if (nextTokens.contains(la)) { + // We are sure the token matches + nextTokensContext = null; + nextTokensState = ATNState.INVALID_STATE_NUMBER; + return; + } - // Return but don't end recovery. only do that upon valid token match - if (recognizer.isExpectedToken(la)) { + if (nextTokens.contains(Token.EPSILON)) { + if (nextTokensContext == null) { + // It's possible the next token won't match; information tracked + // by sync is restricted for performance. + nextTokensContext = recognizer.getContext(); + nextTokensState = recognizer.getState(); + } return; } @@ -452,7 +478,14 @@ public class DefaultErrorStrategy implements ANTLRErrorStrategy { } // even that didn't work; must throw the exception - throw new InputMismatchException(recognizer); + InputMismatchException e; + if (nextTokensContext == null) { + e = new InputMismatchException(recognizer); + } else { + e = new InputMismatchException(recognizer, nextTokensState, nextTokensContext); + } + + throw e; } /** @@ -550,7 +583,10 @@ public class DefaultErrorStrategy implements ANTLRErrorStrategy { protected Token getMissingSymbol(Parser recognizer) { Token currentSymbol = recognizer.getCurrentToken(); IntervalSet expecting = getExpectedTokens(recognizer); - int expectedTokenType = expecting.getMinElement(); // get any element + int expectedTokenType = Token.INVALID_TYPE; + if ( !expecting.isNil() ) { + expectedTokenType = expecting.getMinElement(); // get any element + } String tokenText; if ( expectedTokenType== Token.EOF ) tokenText = "<missing EOF>"; else tokenText = "<missing "+recognizer.getVocabulary().getDisplayName(expectedTokenType)+">"; diff --git a/runtime/Java/src/org/antlr/v4/runtime/DiagnosticErrorListener.java b/runtime/Java/src/org/antlr/v4/runtime/DiagnosticErrorListener.java index 94a8e9b..03fd0aa 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/DiagnosticErrorListener.java +++ b/runtime/Java/src/org/antlr/v4/runtime/DiagnosticErrorListener.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/runtime/Java/src/org/antlr/v4/runtime/FailedPredicateException.java b/runtime/Java/src/org/antlr/v4/runtime/FailedPredicateException.java index 422a7a9..d9f0fec 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/FailedPredicateException.java +++ b/runtime/Java/src/org/antlr/v4/runtime/FailedPredicateException.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/runtime/Java/src/org/antlr/v4/runtime/InputMismatchException.java b/runtime/Java/src/org/antlr/v4/runtime/InputMismatchException.java index 1f45d7c..08ef67c 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/InputMismatchException.java +++ b/runtime/Java/src/org/antlr/v4/runtime/InputMismatchException.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ @@ -13,4 +13,10 @@ public class InputMismatchException extends RecognitionException { super(recognizer, recognizer.getInputStream(), recognizer._ctx); this.setOffendingToken(recognizer.getCurrentToken()); } + + public InputMismatchException(Parser recognizer, int state, ParserRuleContext ctx) { + super(recognizer, recognizer.getInputStream(), ctx); + this.setOffendingState(state); + this.setOffendingToken(recognizer.getCurrentToken()); + } } diff --git a/runtime/Java/src/org/antlr/v4/runtime/IntStream.java b/runtime/Java/src/org/antlr/v4/runtime/IntStream.java index 4fd59dc..07ccf6d 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/IntStream.java +++ b/runtime/Java/src/org/antlr/v4/runtime/IntStream.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ @@ -53,7 +53,7 @@ public interface IntStream { * filtering streams (e.g. {@link CommonTokenStream} which distinguishes * between "on-channel" and "off-channel" tokens). * - * @throws IllegalStateException if an attempt is made to consume the the + * @throws IllegalStateException if an attempt is made to consume the * end of the stream (i.e. if {@code LA(1)==}{@link #EOF EOF} before calling * {@code consume}). */ diff --git a/runtime/Java/src/org/antlr/v4/runtime/InterpreterRuleContext.java b/runtime/Java/src/org/antlr/v4/runtime/InterpreterRuleContext.java index 893a5a8..862f107 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/InterpreterRuleContext.java +++ b/runtime/Java/src/org/antlr/v4/runtime/InterpreterRuleContext.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/runtime/Java/src/org/antlr/v4/runtime/Lexer.java b/runtime/Java/src/org/antlr/v4/runtime/Lexer.java index bb082aa..bba3668 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/Lexer.java +++ b/runtime/Java/src/org/antlr/v4/runtime/Lexer.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ @@ -28,8 +28,8 @@ public abstract class Lexer extends Recognizer<Integer, LexerATNSimulator> public static final int DEFAULT_TOKEN_CHANNEL = Token.DEFAULT_CHANNEL; public static final int HIDDEN = Token.HIDDEN_CHANNEL; - public static final int MIN_CHAR_VALUE = '\u0000'; - public static final int MAX_CHAR_VALUE = '\uFFFE'; + public static final int MIN_CHAR_VALUE = 0x0000; + public static final int MAX_CHAR_VALUE = 0x10FFFF; public CharStream _input; protected Pair<TokenSource, CharStream> _tokenFactorySourcePair; @@ -320,6 +320,8 @@ public abstract class Lexer extends Recognizer<Integer, LexerATNSimulator> return _channel; } + public String[] getChannelNames() { return null; } + public String[] getModeNames() { return null; } diff --git a/runtime/Java/src/org/antlr/v4/runtime/LexerInterpreter.java b/runtime/Java/src/org/antlr/v4/runtime/LexerInterpreter.java index bb50c53..8a8dfb4 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/LexerInterpreter.java +++ b/runtime/Java/src/org/antlr/v4/runtime/LexerInterpreter.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ @@ -12,6 +12,7 @@ import org.antlr.v4.runtime.atn.LexerATNSimulator; import org.antlr.v4.runtime.atn.PredictionContextCache; import org.antlr.v4.runtime.dfa.DFA; +import java.util.ArrayList; import java.util.Collection; public class LexerInterpreter extends Lexer { @@ -21,6 +22,7 @@ public class LexerInterpreter extends Lexer { @Deprecated protected final String[] tokenNames; protected final String[] ruleNames; + protected final String[] channelNames; protected final String[] modeNames; @@ -32,10 +34,15 @@ public class LexerInterpreter extends Lexer { @Deprecated public LexerInterpreter(String grammarFileName, Collection<String> tokenNames, Collection<String> ruleNames, Collection<String> modeNames, ATN atn, CharStream input) { - this(grammarFileName, VocabularyImpl.fromTokenNames(tokenNames.toArray(new String[tokenNames.size()])), ruleNames, modeNames, atn, input); + this(grammarFileName, VocabularyImpl.fromTokenNames(tokenNames.toArray(new String[tokenNames.size()])), ruleNames, new ArrayList<String>(), modeNames, atn, input); } + @Deprecated public LexerInterpreter(String grammarFileName, Vocabulary vocabulary, Collection<String> ruleNames, Collection<String> modeNames, ATN atn, CharStream input) { + this(grammarFileName, vocabulary, ruleNames, new ArrayList<String>(), modeNames, atn, input); + } + + public LexerInterpreter(String grammarFileName, Vocabulary vocabulary, Collection<String> ruleNames, Collection<String> channelNames, Collection<String> modeNames, ATN atn, CharStream input) { super(input); if (atn.grammarType != ATNType.LEXER) { @@ -50,6 +57,7 @@ public class LexerInterpreter extends Lexer { } this.ruleNames = ruleNames.toArray(new String[ruleNames.size()]); + this.channelNames = channelNames.toArray(new String[channelNames.size()]); this.modeNames = modeNames.toArray(new String[modeNames.size()]); this.vocabulary = vocabulary; @@ -82,6 +90,11 @@ public class LexerInterpreter extends Lexer { } @Override + public String[] getChannelNames() { + return channelNames; + } + + @Override public String[] getModeNames() { return modeNames; } diff --git a/runtime/Java/src/org/antlr/v4/runtime/LexerNoViableAltException.java b/runtime/Java/src/org/antlr/v4/runtime/LexerNoViableAltException.java index 2540d10..8b33196 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/LexerNoViableAltException.java +++ b/runtime/Java/src/org/antlr/v4/runtime/LexerNoViableAltException.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/runtime/Java/src/org/antlr/v4/runtime/ListTokenSource.java b/runtime/Java/src/org/antlr/v4/runtime/ListTokenSource.java index 592c8d9..317027e 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/ListTokenSource.java +++ b/runtime/Java/src/org/antlr/v4/runtime/ListTokenSource.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/runtime/Java/src/org/antlr/v4/runtime/NoViableAltException.java b/runtime/Java/src/org/antlr/v4/runtime/NoViableAltException.java index a805e80..4e20343 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/NoViableAltException.java +++ b/runtime/Java/src/org/antlr/v4/runtime/NoViableAltException.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/runtime/Java/src/org/antlr/v4/runtime/Parser.java b/runtime/Java/src/org/antlr/v4/runtime/Parser.java index 1edc457..566aea0 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/Parser.java +++ b/runtime/Java/src/org/antlr/v4/runtime/Parser.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ @@ -19,9 +19,11 @@ import org.antlr.v4.runtime.dfa.DFA; import org.antlr.v4.runtime.misc.IntegerStack; import org.antlr.v4.runtime.misc.IntervalSet; import org.antlr.v4.runtime.tree.ErrorNode; +import org.antlr.v4.runtime.tree.ErrorNodeImpl; import org.antlr.v4.runtime.tree.ParseTreeListener; import org.antlr.v4.runtime.tree.ParseTreeWalker; import org.antlr.v4.runtime.tree.TerminalNode; +import org.antlr.v4.runtime.tree.TerminalNodeImpl; import org.antlr.v4.runtime.tree.pattern.ParseTreePattern; import org.antlr.v4.runtime.tree.pattern.ParseTreePatternMatcher; @@ -182,7 +184,8 @@ public abstract class Parser extends Recognizer<Token, ParserATNSimulator> { * strategy to attempt recovery. If {@link #getBuildParseTree} is * {@code true} and the token index of the symbol returned by * {@link ANTLRErrorStrategy#recoverInline} is -1, the symbol is added to - * the parse tree by calling {@link ParserRuleContext#addErrorNode}.</p> + * the parse tree by calling {@link #createErrorNode(ParserRuleContext, Token)} then + * {@link ParserRuleContext#addErrorNode(ErrorNode)}.</p> * * @param ttype the token type to match * @return the matched symbol @@ -204,7 +207,7 @@ public abstract class Parser extends Recognizer<Token, ParserATNSimulator> { if ( _buildParseTrees && t.getTokenIndex()==-1 ) { // we must have conjured up a new token during single token insertion // if it's not the current symbol - _ctx.addErrorNode(t); + _ctx.addErrorNode(createErrorNode(_ctx,t)); } } return t; @@ -220,7 +223,8 @@ public abstract class Parser extends Recognizer<Token, ParserATNSimulator> { * strategy to attempt recovery. If {@link #getBuildParseTree} is * {@code true} and the token index of the symbol returned by * {@link ANTLRErrorStrategy#recoverInline} is -1, the symbol is added to - * the parse tree by calling {@link ParserRuleContext#addErrorNode}.</p> + * the parse tree by calling {@link Parser#createErrorNode(ParserRuleContext, Token)}. then + * {@link ParserRuleContext#addErrorNode(ErrorNode)}</p> * * @return the matched symbol * @throws RecognitionException if the current input symbol did not match @@ -238,7 +242,7 @@ public abstract class Parser extends Recognizer<Token, ParserATNSimulator> { if (_buildParseTrees && t.getTokenIndex() == -1) { // we must have conjured up a new token during single token insertion // if it's not the current symbol - _ctx.addErrorNode(t); + _ctx.addErrorNode(createErrorNode(_ctx,t)); } } @@ -553,11 +557,11 @@ public abstract class Parser extends Recognizer<Token, ParserATNSimulator> { * </pre> * * If the parser is not in error recovery mode, the consumed symbol is added - * to the parse tree using {@link ParserRuleContext#addChild(Token)}, and + * to the parse tree using {@link ParserRuleContext#addChild(TerminalNode)}, and * {@link ParseTreeListener#visitTerminal} is called on any parse listeners. * If the parser <em>is</em> in error recovery mode, the consumed symbol is - * added to the parse tree using - * {@link ParserRuleContext#addErrorNode(Token)}, and + * added to the parse tree using {@link #createErrorNode(ParserRuleContext, Token)} then + * {@link ParserRuleContext#addErrorNode(ErrorNode)} and * {@link ParseTreeListener#visitErrorNode} is called on any parse * listeners. */ @@ -569,7 +573,7 @@ public abstract class Parser extends Recognizer<Token, ParserATNSimulator> { boolean hasListener = _parseListeners != null && !_parseListeners.isEmpty(); if (_buildParseTrees || hasListener) { if ( _errHandler.inErrorRecoveryMode(this) ) { - ErrorNode node = _ctx.addErrorNode(o); + ErrorNode node = _ctx.addErrorNode(createErrorNode(_ctx,o)); if (_parseListeners != null) { for (ParseTreeListener listener : _parseListeners) { listener.visitErrorNode(node); @@ -577,7 +581,7 @@ public abstract class Parser extends Recognizer<Token, ParserATNSimulator> { } } else { - TerminalNode node = _ctx.addChild(o); + TerminalNode node = _ctx.addChild(createTerminalNode(_ctx,o)); if (_parseListeners != null) { for (ParseTreeListener listener : _parseListeners) { listener.visitTerminal(node); @@ -588,6 +592,24 @@ public abstract class Parser extends Recognizer<Token, ParserATNSimulator> { return o; } + /** How to create a token leaf node associated with a parent. + * Typically, the terminal node to create is not a function of the parent. + * + * @since 4.7 + */ + public TerminalNode createTerminalNode(ParserRuleContext parent, Token t) { + return new TerminalNodeImpl(t); + } + + /** How to create an error node, given a token, associated with a parent. + * Typically, the error node to create is not a function of the parent. + * + * @since 4.7 + */ + public ErrorNode createErrorNode(ParserRuleContext parent, Token t) { + return new ErrorNodeImpl(t); + } + protected void addContextToParseTree() { ParserRuleContext parent = (ParserRuleContext)_ctx.parent; // add current context to parent if we have a parent @@ -925,3 +947,4 @@ public abstract class Parser extends Recognizer<Token, ParserATNSimulator> { return _tracer != null; } } + diff --git a/runtime/Java/src/org/antlr/v4/runtime/ParserInterpreter.java b/runtime/Java/src/org/antlr/v4/runtime/ParserInterpreter.java index 568495b..6045b27 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/ParserInterpreter.java +++ b/runtime/Java/src/org/antlr/v4/runtime/ParserInterpreter.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ @@ -407,14 +407,17 @@ public class ParserInterpreter extends Parser { if ( e instanceof InputMismatchException ) { InputMismatchException ime = (InputMismatchException)e; Token tok = e.getOffendingToken(); - int expectedTokenType = ime.getExpectedTokens().getMinElement(); // get any element + int expectedTokenType = Token.INVALID_TYPE; + if ( !ime.getExpectedTokens().isNil() ) { + expectedTokenType = ime.getExpectedTokens().getMinElement(); // get any element + } Token errToken = getTokenFactory().create(new Pair<TokenSource, CharStream>(tok.getTokenSource(), tok.getTokenSource().getInputStream()), expectedTokenType, tok.getText(), Token.DEFAULT_CHANNEL, -1, -1, // invalid start/stop tok.getLine(), tok.getCharPositionInLine()); - _ctx.addErrorNode(errToken); + _ctx.addErrorNode(createErrorNode(_ctx,errToken)); } else { // NoViableAlt Token tok = e.getOffendingToken(); @@ -424,7 +427,7 @@ public class ParserInterpreter extends Parser { Token.DEFAULT_CHANNEL, -1, -1, // invalid start/stop tok.getLine(), tok.getCharPositionInLine()); - _ctx.addErrorNode(errToken); + _ctx.addErrorNode(createErrorNode(_ctx,errToken)); } } } @@ -445,3 +448,4 @@ public class ParserInterpreter extends Parser { return rootContext; } } + diff --git a/runtime/Java/src/org/antlr/v4/runtime/ParserRuleContext.java b/runtime/Java/src/org/antlr/v4/runtime/ParserRuleContext.java index 9f3b97f..4513b09 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/ParserRuleContext.java +++ b/runtime/Java/src/org/antlr/v4/runtime/ParserRuleContext.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ @@ -63,8 +63,8 @@ public class ParserRuleContext extends RuleContext { * * The parser setState() method updates field s and adds it to this list * if we are debugging/tracing. - * - * This does not trace states visited during prediction. + * + * This does not trace states visited during prediction. */ // public List<Integer> states; @@ -79,7 +79,8 @@ public class ParserRuleContext extends RuleContext { public ParserRuleContext() { } /** COPY a ctx (I'm deliberately not using copy constructor) to avoid - * confusion with creating node with parent. Does not copy children. + * confusion with creating node with parent. Does not copy children + * (except error leaves). * * This is used in the generated parser code to flip a generic XContext * node for rule X to a YContext for alt label Y. In that sense, it is @@ -101,9 +102,8 @@ public class ParserRuleContext extends RuleContext { this.children = new ArrayList<>(); // reset parent pointer for any error nodes for (ParseTree child : ctx.children) { - if ( child instanceof ErrorNodeImpl ) { - this.children.add(child); - ((ErrorNodeImpl) child).parent = this; + if ( child instanceof ErrorNode ) { + addChild((ErrorNode)child); } } } @@ -118,48 +118,83 @@ public class ParserRuleContext extends RuleContext { public void enterRule(ParseTreeListener listener) { } public void exitRule(ParseTreeListener listener) { } - /** Does not set parent link; other add methods do that */ - public TerminalNode addChild(TerminalNode t) { - if ( children==null ) children = new ArrayList<ParseTree>(); + /** Add a parse tree node to this as a child. Works for + * internal and leaf nodes. Does not set parent link; + * other add methods must do that. Other addChild methods + * call this. + * + * We cannot set the parent pointer of the incoming node + * because the existing interfaces do not have a setParent() + * method and I don't want to break backward compatibility for this. + * + * @since 4.7 + */ + public <T extends ParseTree> T addAnyChild(T t) { + if ( children==null ) children = new ArrayList<>(); children.add(t); return t; } public RuleContext addChild(RuleContext ruleInvocation) { - if ( children==null ) children = new ArrayList<ParseTree>(); - children.add(ruleInvocation); - return ruleInvocation; + return addAnyChild(ruleInvocation); } - /** Used by enterOuterAlt to toss out a RuleContext previously added as - * we entered a rule. If we have # label, we will need to remove - * generic ruleContext object. - */ - public void removeLastChild() { - if ( children!=null ) { - children.remove(children.size()-1); - } + /** Add a token leaf node child and force its parent to be this node. */ + public TerminalNode addChild(TerminalNode t) { + t.setParent(this); + return addAnyChild(t); } -// public void trace(int s) { -// if ( states==null ) states = new ArrayList<Integer>(); -// states.add(s); -// } + /** Add an error node child and force its parent to be this node. + * + * @since 4.7 + */ + public ErrorNode addErrorNode(ErrorNode errorNode) { + errorNode.setParent(this); + return addAnyChild(errorNode); + } + /** Add a child to this node based upon matchedToken. It + * creates a TerminalNodeImpl rather than using + * {@link Parser#createTerminalNode(ParserRuleContext, Token)}. I'm leaving this + * in for compatibility but the parser doesn't use this anymore. + */ + @Deprecated public TerminalNode addChild(Token matchedToken) { TerminalNodeImpl t = new TerminalNodeImpl(matchedToken); - addChild(t); - t.parent = this; + addAnyChild(t); + t.setParent(this); return t; } + /** Add a child to this node based upon badToken. It + * creates a ErrorNodeImpl rather than using + * {@link Parser#createErrorNode(ParserRuleContext, Token)}. I'm leaving this + * in for compatibility but the parser doesn't use this anymore. + */ + @Deprecated public ErrorNode addErrorNode(Token badToken) { ErrorNodeImpl t = new ErrorNodeImpl(badToken); - addChild(t); - t.parent = this; + addAnyChild(t); + t.setParent(this); return t; } +// public void trace(int s) { +// if ( states==null ) states = new ArrayList<Integer>(); +// states.add(s); +// } + + /** Used by enterOuterAlt to toss out a RuleContext previously added as + * we entered a rule. If we have # label, we will need to remove + * generic ruleContext object. + */ + public void removeLastChild() { + if ( children!=null ) { + children.remove(children.size()-1); + } + } + @Override /** Override to make type more specific */ public ParserRuleContext getParent() { @@ -290,13 +325,14 @@ public class ParserRuleContext extends RuleContext { */ public Token getStop() { return stop; } - /** Used for rule context info debugging during parse-time, not so much for ATN debugging */ - public String toInfoString(Parser recognizer) { - List<String> rules = recognizer.getRuleInvocationStack(this); - Collections.reverse(rules); - return "ParserRuleContext"+rules+"{" + - "start=" + start + - ", stop=" + stop + - '}'; - } + /** Used for rule context info debugging during parse-time, not so much for ATN debugging */ + public String toInfoString(Parser recognizer) { + List<String> rules = recognizer.getRuleInvocationStack(this); + Collections.reverse(rules); + return "ParserRuleContext"+rules+"{" + + "start=" + start + + ", stop=" + stop + + '}'; + } } + diff --git a/runtime/Java/src/org/antlr/v4/runtime/ProxyErrorListener.java b/runtime/Java/src/org/antlr/v4/runtime/ProxyErrorListener.java index 35d7546..703fbb6 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/ProxyErrorListener.java +++ b/runtime/Java/src/org/antlr/v4/runtime/ProxyErrorListener.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/runtime/Java/src/org/antlr/v4/runtime/RecognitionException.java b/runtime/Java/src/org/antlr/v4/runtime/RecognitionException.java index 5157421..f5d0399 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/RecognitionException.java +++ b/runtime/Java/src/org/antlr/v4/runtime/RecognitionException.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/runtime/Java/src/org/antlr/v4/runtime/Recognizer.java b/runtime/Java/src/org/antlr/v4/runtime/Recognizer.java index 6b407bc..3e09d15 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/Recognizer.java +++ b/runtime/Java/src/org/antlr/v4/runtime/Recognizer.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/runtime/Java/src/org/antlr/v4/runtime/RuleContext.java b/runtime/Java/src/org/antlr/v4/runtime/RuleContext.java index 91eafd0..8b7f168 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/RuleContext.java +++ b/runtime/Java/src/org/antlr/v4/runtime/RuleContext.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ @@ -163,6 +163,12 @@ public class RuleContext implements RuleNode { */ public void setAltNumber(int altNumber) { } + /** @since 4.7. {@see ParseTree#setParent} comment */ + @Override + public void setParent(RuleContext parent) { + this.parent = parent; + } + @Override public ParseTree getChild(int i) { return null; diff --git a/runtime/Java/src/org/antlr/v4/runtime/RuleContextWithAltNum.java b/runtime/Java/src/org/antlr/v4/runtime/RuleContextWithAltNum.java index 81fe476..38ec190 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/RuleContextWithAltNum.java +++ b/runtime/Java/src/org/antlr/v4/runtime/RuleContextWithAltNum.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/runtime/Java/src/org/antlr/v4/runtime/RuntimeMetaData.java b/runtime/Java/src/org/antlr/v4/runtime/RuntimeMetaData.java index 9355d31..bc2e478 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/RuntimeMetaData.java +++ b/runtime/Java/src/org/antlr/v4/runtime/RuntimeMetaData.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ @@ -67,7 +67,7 @@ public class RuntimeMetaData { * omitted.</li> * </ul> */ - public static final String VERSION = "4.6"; + public static final String VERSION = "4.7.1"; /** * Gets the currently executing version of the ANTLR 4 runtime library. diff --git a/runtime/Java/src/org/antlr/v4/runtime/Token.java b/runtime/Java/src/org/antlr/v4/runtime/Token.java index f25bed5..b6432ca 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/Token.java +++ b/runtime/Java/src/org/antlr/v4/runtime/Token.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/runtime/Java/src/org/antlr/v4/runtime/TokenFactory.java b/runtime/Java/src/org/antlr/v4/runtime/TokenFactory.java index 0053537..ad21f42 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/TokenFactory.java +++ b/runtime/Java/src/org/antlr/v4/runtime/TokenFactory.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ @@ -10,7 +10,7 @@ import org.antlr.v4.runtime.misc.Pair; /** The default mechanism for creating tokens. It's used by default in Lexer and * the error handling strategy (to create missing tokens). Notifying the parser - * of a new factory means that it notifies it's token source and error strategy. + * of a new factory means that it notifies its token source and error strategy. */ public interface TokenFactory<Symbol extends Token> { /** This is the method used to create tokens in the lexer and in the diff --git a/runtime/Java/src/org/antlr/v4/runtime/TokenSource.java b/runtime/Java/src/org/antlr/v4/runtime/TokenSource.java index 1e01bf8..3eb2c99 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/TokenSource.java +++ b/runtime/Java/src/org/antlr/v4/runtime/TokenSource.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/runtime/Java/src/org/antlr/v4/runtime/TokenStream.java b/runtime/Java/src/org/antlr/v4/runtime/TokenStream.java index 18b06f7..4e9887d 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/TokenStream.java +++ b/runtime/Java/src/org/antlr/v4/runtime/TokenStream.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/runtime/Java/src/org/antlr/v4/runtime/TokenStreamRewriter.java b/runtime/Java/src/org/antlr/v4/runtime/TokenStreamRewriter.java index 8b8abca..7195e39 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/TokenStreamRewriter.java +++ b/runtime/Java/src/org/antlr/v4/runtime/TokenStreamRewriter.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/runtime/Java/src/org/antlr/v4/runtime/UnbufferedCharStream.java b/runtime/Java/src/org/antlr/v4/runtime/UnbufferedCharStream.java index 712dc85..3db4970 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/UnbufferedCharStream.java +++ b/runtime/Java/src/org/antlr/v4/runtime/UnbufferedCharStream.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ @@ -12,12 +12,20 @@ import java.io.IOException; import java.io.InputStream; import java.io.InputStreamReader; import java.io.Reader; +import java.nio.charset.Charset; +import java.nio.charset.StandardCharsets; import java.util.Arrays; /** Do not buffer up the entire char stream. It does keep a small buffer * for efficiency and also buffers while a mark exists (set by the * lookahead prediction in parser). "Unbuffered" here refers to fact * that it doesn't buffer all data, not that's it's on demand loading of char. + * + * Before 4.7, this class used the default environment encoding to convert + * bytes to UTF-16, and held the UTF-16 bytes in the buffer as chars. + * + * As of 4.7, the class uses UTF-8 by default, and the buffer holds Unicode + * code points in the buffer as ints. */ public class UnbufferedCharStream implements CharStream { /** @@ -25,7 +33,7 @@ public class UnbufferedCharStream implements CharStream { * we keep adding to buffer. Otherwise, {@link #consume consume()} resets so * we start filling at index 0 again. */ - protected char[] data; + protected int[] data; /** * The number of characters currently in {@link #data data}. @@ -82,7 +90,7 @@ public class UnbufferedCharStream implements CharStream { /** Useful for subclasses that pull char from other than this.input. */ public UnbufferedCharStream(int bufferSize) { n = 0; - data = new char[bufferSize]; + data = new int[bufferSize]; } public UnbufferedCharStream(InputStream input) { @@ -94,8 +102,12 @@ public class UnbufferedCharStream implements CharStream { } public UnbufferedCharStream(InputStream input, int bufferSize) { + this(input, bufferSize, StandardCharsets.UTF_8); + } + + public UnbufferedCharStream(InputStream input, int bufferSize, Charset charset) { this(bufferSize); - this.input = new InputStreamReader(input); + this.input = new InputStreamReader(input, charset); fill(1); // prime } @@ -145,13 +157,42 @@ public class UnbufferedCharStream implements CharStream { */ protected int fill(int n) { for (int i=0; i<n; i++) { - if (this.n > 0 && data[this.n - 1] == (char)IntStream.EOF) { + if (this.n > 0 && data[this.n - 1] == IntStream.EOF) { return i; } try { int c = nextChar(); - add(c); + if (c > Character.MAX_VALUE || c == IntStream.EOF) { + add(c); + } + else { + char ch = (char) c; + if (Character.isLowSurrogate(ch)) { + throw new RuntimeException("Invalid UTF-16 (low surrogate with no preceding high surrogate)"); + } + else if (Character.isHighSurrogate(ch)) { + int lowSurrogate = nextChar(); + if (lowSurrogate > Character.MAX_VALUE) { + throw new RuntimeException("Invalid UTF-16 (high surrogate followed by code point > U+FFFF"); + } + else if (lowSurrogate == IntStream.EOF) { + throw new RuntimeException("Invalid UTF-16 (dangling high surrogate at end of file)"); + } + else { + char lowSurrogateChar = (char) lowSurrogate; + if (Character.isLowSurrogate(lowSurrogateChar)) { + add(Character.toCodePoint(ch, lowSurrogateChar)); + } + else { + throw new RuntimeException("Invalid UTF-16 (dangling high surrogate"); + } + } + } + else { + add(c); + } + } } catch (IOException ioe) { throw new RuntimeException(ioe); @@ -173,7 +214,7 @@ public class UnbufferedCharStream implements CharStream { if ( n>=data.length ) { data = Arrays.copyOf(data, data.length * 2); } - data[n++] = (char)c; + data[n++] = c; } @Override @@ -183,9 +224,7 @@ public class UnbufferedCharStream implements CharStream { int index = p + i - 1; if ( index < 0 ) throw new IndexOutOfBoundsException(); if ( index >= n ) return IntStream.EOF; - char c = data[index]; - if ( c==(char)IntStream.EOF ) return IntStream.EOF; - return c; + return data[index]; } /** diff --git a/runtime/Java/src/org/antlr/v4/runtime/UnbufferedTokenStream.java b/runtime/Java/src/org/antlr/v4/runtime/UnbufferedTokenStream.java index 046d821..f44b1c9 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/UnbufferedTokenStream.java +++ b/runtime/Java/src/org/antlr/v4/runtime/UnbufferedTokenStream.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/runtime/Java/src/org/antlr/v4/runtime/Vocabulary.java b/runtime/Java/src/org/antlr/v4/runtime/Vocabulary.java index c3662ea..83febb2 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/Vocabulary.java +++ b/runtime/Java/src/org/antlr/v4/runtime/Vocabulary.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/runtime/Java/src/org/antlr/v4/runtime/VocabularyImpl.java b/runtime/Java/src/org/antlr/v4/runtime/VocabularyImpl.java index 75b450e..00593e3 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/VocabularyImpl.java +++ b/runtime/Java/src/org/antlr/v4/runtime/VocabularyImpl.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/runtime/Java/src/org/antlr/v4/runtime/WritableToken.java b/runtime/Java/src/org/antlr/v4/runtime/WritableToken.java index bdaaac3..dc3aafd 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/WritableToken.java +++ b/runtime/Java/src/org/antlr/v4/runtime/WritableToken.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/runtime/Java/src/org/antlr/v4/runtime/atn/ATN.java b/runtime/Java/src/org/antlr/v4/runtime/atn/ATN.java index 1e67bea..96f71f8 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/atn/ATN.java +++ b/runtime/Java/src/org/antlr/v4/runtime/atn/ATN.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/runtime/Java/src/org/antlr/v4/runtime/atn/ATNConfig.java b/runtime/Java/src/org/antlr/v4/runtime/atn/ATNConfig.java index 6cededb..a3e4d1d 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/atn/ATNConfig.java +++ b/runtime/Java/src/org/antlr/v4/runtime/atn/ATNConfig.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ @@ -161,7 +161,8 @@ public class ATNConfig { public boolean equals(ATNConfig other) { if (this == other) { return true; - } else if (other == null) { + } + else if (other == null) { return false; } diff --git a/runtime/Java/src/org/antlr/v4/runtime/atn/ATNConfigSet.java b/runtime/Java/src/org/antlr/v4/runtime/atn/ATNConfigSet.java index 423c5d3..d40dfb4 100755 --- a/runtime/Java/src/org/antlr/v4/runtime/atn/ATNConfigSet.java +++ b/runtime/Java/src/org/antlr/v4/runtime/atn/ATNConfigSet.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/runtime/Java/src/org/antlr/v4/runtime/atn/ATNDeserializationOptions.java b/runtime/Java/src/org/antlr/v4/runtime/atn/ATNDeserializationOptions.java index 2191eb1..9aee1f2 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/atn/ATNDeserializationOptions.java +++ b/runtime/Java/src/org/antlr/v4/runtime/atn/ATNDeserializationOptions.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/runtime/Java/src/org/antlr/v4/runtime/atn/ATNDeserializer.java b/runtime/Java/src/org/antlr/v4/runtime/atn/ATNDeserializer.java index 9821b8b..0ebcdeb 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/atn/ATNDeserializer.java +++ b/runtime/Java/src/org/antlr/v4/runtime/atn/ATNDeserializer.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ @@ -45,6 +45,12 @@ public class ATNDeserializer { */ private static final UUID ADDED_LEXER_ACTIONS; /** + * This UUID indicates the serialized ATN contains two sets of + * IntervalSets, where the second set's values are encoded as + * 32-bit integers to support the full Unicode SMP range up to U+10FFFF. + */ + private static final UUID ADDED_UNICODE_SMP; + /** * This list contains all of the currently supported UUIDs, ordered by when * the feature first appeared in this branch. */ @@ -61,15 +67,59 @@ public class ATNDeserializer { BASE_SERIALIZED_UUID = UUID.fromString("33761B2D-78BB-4A43-8B0B-4F5BEE8AACF3"); ADDED_PRECEDENCE_TRANSITIONS = UUID.fromString("1DA0C57D-6C06-438A-9B27-10BCB3CE0F61"); ADDED_LEXER_ACTIONS = UUID.fromString("AADB8D7E-AEEF-4415-AD2B-8204D6CF042E"); + ADDED_UNICODE_SMP = UUID.fromString("59627784-3BE5-417A-B9EB-8131A7286089"); SUPPORTED_UUIDS = new ArrayList<UUID>(); SUPPORTED_UUIDS.add(BASE_SERIALIZED_UUID); SUPPORTED_UUIDS.add(ADDED_PRECEDENCE_TRANSITIONS); SUPPORTED_UUIDS.add(ADDED_LEXER_ACTIONS); + SUPPORTED_UUIDS.add(ADDED_UNICODE_SMP); + + SERIALIZED_UUID = ADDED_UNICODE_SMP; + } + + interface UnicodeDeserializer { + // Wrapper for readInt() or readInt32() + int readUnicode(char[] data, int p); + + // Work around Java not allowing mutation of captured variables + // by returning amount by which to increment p after each read + int size(); + } - SERIALIZED_UUID = ADDED_LEXER_ACTIONS; + enum UnicodeDeserializingMode { + UNICODE_BMP, + UNICODE_SMP } + static UnicodeDeserializer getUnicodeDeserializer(UnicodeDeserializingMode mode) { + if (mode == UnicodeDeserializingMode.UNICODE_BMP) { + return new UnicodeDeserializer() { + @Override + public int readUnicode(char[] data, int p) { + return toInt(data[p]); + } + + @Override + public int size() { + return 1; + } + }; + } + else { + return new UnicodeDeserializer() { + @Override + public int readUnicode(char[] data, int p) { + return toInt32(data, p); + } + + @Override + public int size() { + return 2; + } + }; + } + } private final ATNDeserializationOptions deserializationOptions; @@ -98,7 +148,7 @@ public class ATNDeserializer { * serialized ATN at or after the feature identified by {@code feature} was * introduced; otherwise, {@code false}. */ - protected boolean isFeatureSupported(UUID feature, UUID actualUuid) { + static protected boolean isFeatureSupported(UUID feature, UUID actualUuid) { int featureIndex = SUPPORTED_UUIDS.indexOf(feature); if (featureIndex < 0) { return false; @@ -110,7 +160,22 @@ public class ATNDeserializer { @SuppressWarnings("deprecation") public ATN deserialize(char[] data) { data = data.clone(); - // don't adjust the first value since that's the version number + + // Each char value in data is shifted by +2 at the entry to this method. + // This is an encoding optimization targeting the serialized values 0 + // and -1 (serialized to 0xFFFF), each of which are very common in the + // serialized form of the ATN. In the modified UTF-8 that Java uses for + // compiled string literals, these two character values have multi-byte + // forms. By shifting each value by +2, they become characters 2 and 1 + // prior to writing the string, each of which have single-byte + // representations. Since the shift occurs in the tool during ATN + // serialization, each target is responsible for adjusting the values + // during deserialization. + // + // As a special case, note that the first element of data is not + // adjusted because it contains the major version number of the + // serialized ATN, which was fixed at 3 at the time the value shifting + // was implemented. for (int i = 1; i < data.length; i++) { data[i] = (char)(data[i] - 2); } @@ -243,22 +308,14 @@ public class ATNDeserializer { // SETS // List<IntervalSet> sets = new ArrayList<IntervalSet>(); - int nsets = toInt(data[p++]); - for (int i=0; i<nsets; i++) { - int nintervals = toInt(data[p]); - p++; - IntervalSet set = new IntervalSet(); - sets.add(set); - boolean containsEof = toInt(data[p++]) != 0; - if (containsEof) { - set.add(-1); - } + // First, read all sets with 16-bit Unicode code points <= U+FFFF. + p = deserializeSets(data, p, sets, getUnicodeDeserializer(UnicodeDeserializingMode.UNICODE_BMP)); - for (int j=0; j<nintervals; j++) { - set.add(toInt(data[p]), toInt(data[p + 1])); - p += 2; - } + // Next, if the ATN was serialized with the Unicode SMP feature, + // deserialize sets with 32-bit arguments <= U+10FFFF. + if (isFeatureSupported(ADDED_UNICODE_SMP, uuid)) { + p = deserializeSets(data, p, sets, getUnicodeDeserializer(UnicodeDeserializingMode.UNICODE_SMP)); } // @@ -495,6 +552,30 @@ public class ATNDeserializer { return atn; } + private int deserializeSets(char[] data, int p, List<IntervalSet> sets, UnicodeDeserializer unicodeDeserializer) { + int nsets = toInt(data[p++]); + for (int i=0; i<nsets; i++) { + int nintervals = toInt(data[p]); + p++; + IntervalSet set = new IntervalSet(); + sets.add(set); + + boolean containsEof = toInt(data[p++]) != 0; + if (containsEof) { + set.add(-1); + } + + for (int j=0; j<nintervals; j++) { + int a = unicodeDeserializer.readUnicode(data, p); + p += unicodeDeserializer.size(); + int b = unicodeDeserializer.readUnicode(data, p); + p += unicodeDeserializer.size(); + set.add(a, b); + } + } + return p; + } + /** * Analyze the {@link StarLoopEntryState} states in the specified ATN to set * the {@link StarLoopEntryState#isPrecedenceDecision} field to the diff --git a/runtime/Java/src/org/antlr/v4/runtime/atn/ATNSerializer.java b/runtime/Java/src/org/antlr/v4/runtime/atn/ATNSerializer.java index e5f0344..b1efd14 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/atn/ATNSerializer.java +++ b/runtime/Java/src/org/antlr/v4/runtime/atn/ATNSerializer.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ @@ -14,7 +14,9 @@ import org.antlr.v4.runtime.misc.Utils; import java.io.InvalidClassException; import java.util.ArrayList; +import java.util.Collection; import java.util.HashMap; +import java.util.LinkedHashMap; import java.util.List; import java.util.Locale; import java.util.Map; @@ -24,6 +26,10 @@ public class ATNSerializer { public ATN atn; private List<String> tokenNames; + private interface CodePointSerializer { + void serializeCodePoint(IntegerList data, int cp); + } + public ATNSerializer(ATN atn) { assert atn.grammarType != null; this.atn = atn; @@ -47,9 +53,11 @@ public class ATNSerializer { * (args are token type,actionIndex in lexer else 0,0) * num modes, * mode-0-start-state, mode-1-start-state, ... (parser has 0 modes) - * num sets - * set-0-interval-count intervals, set-1-interval-count intervals, ... - * num total edges, + * num unicode-bmp-sets + * bmp-set-0-interval-count intervals, bmp-set-1-interval-count intervals, ... + * num unicode-smp-sets + * smp-set-0-interval-count intervals, smp-set-1-interval-count intervals, ... + * num total edges, * src, trg, edge-type, edge arg1, optional edge arg2 (present always), ... * num decisions, * decision-0-start-state, decision-1-start-state, ... @@ -66,8 +74,10 @@ public class ATNSerializer { data.add(atn.maxTokenType); int nedges = 0; - Map<IntervalSet, Integer> setIndices = new HashMap<IntervalSet, Integer>(); - List<IntervalSet> sets = new ArrayList<IntervalSet>(); + // Note that we use a LinkedHashMap as a set to + // maintain insertion order while deduplicating + // entries with the same key. + Map<IntervalSet, Boolean> sets = new LinkedHashMap<>(); // dump states, count edges and collect sets while doing so IntegerList nonGreedyStates = new IntegerList(); @@ -114,10 +124,7 @@ public class ATNSerializer { int edgeType = Transition.serializationTypes.get(t.getClass()); if ( edgeType == Transition.SET || edgeType == Transition.NOT_SET ) { SetTransition st = (SetTransition)t; - if (!setIndices.containsKey(st.set)) { - sets.add(st.set); - setIndices.put(st.set, sets.size() - 1); - } + sets.put(st.set, true); } } } @@ -156,34 +163,41 @@ public class ATNSerializer { data.add(modeStartState.stateNumber); } } - - int nsets = sets.size(); - data.add(nsets); - for (IntervalSet set : sets) { - boolean containsEof = set.contains(Token.EOF); - if (containsEof && set.getIntervals().get(0).b == Token.EOF) { - data.add(set.getIntervals().size() - 1); + List<IntervalSet> bmpSets = new ArrayList<>(); + List<IntervalSet> smpSets = new ArrayList<>(); + for (IntervalSet set : sets.keySet()) { + if (set.getMaxElement() <= Character.MAX_VALUE) { + bmpSets.add(set); } else { - data.add(set.getIntervals().size()); + smpSets.add(set); } - - data.add(containsEof ? 1 : 0); - for (Interval I : set.getIntervals()) { - if (I.a == Token.EOF) { - if (I.b == Token.EOF) { - continue; - } - else { - data.add(0); - } + } + serializeSets( + data, + bmpSets, + new CodePointSerializer() { + @Override + public void serializeCodePoint(IntegerList data, int cp) { + data.add(cp); } - else { - data.add(I.a); + }); + serializeSets( + data, + smpSets, + new CodePointSerializer() { + @Override + public void serializeCodePoint(IntegerList data, int cp) { + serializeInt(data, cp); } - - data.add(I.b); - } + }); + Map<IntervalSet, Integer> setIndices = new HashMap<>(); + int setIndex = 0; + for (IntervalSet bmpSet : bmpSets) { + setIndices.put(bmpSet, setIndex++); + } + for (IntervalSet smpSet : smpSets) { + setIndices.put(smpSet, setIndex++); } data.add(nedges); @@ -340,6 +354,7 @@ public class ATNSerializer { } } + // Note: This value shifting loop is documented in ATNDeserializer. // don't adjust the first value since that's the version number for (int i = 1; i < data.size(); i++) { if (data.get(i) < Character.MIN_VALUE || data.get(i) > Character.MAX_VALUE) { @@ -358,6 +373,42 @@ public class ATNSerializer { return data; } + private static void serializeSets( + IntegerList data, + Collection<IntervalSet> sets, + CodePointSerializer codePointSerializer) + { + int nSets = sets.size(); + data.add(nSets); + + for (IntervalSet set : sets) { + boolean containsEof = set.contains(Token.EOF); + if (containsEof && set.getIntervals().get(0).b == Token.EOF) { + data.add(set.getIntervals().size() - 1); + } + else { + data.add(set.getIntervals().size()); + } + + data.add(containsEof ? 1 : 0); + for (Interval I : set.getIntervals()) { + if (I.a == Token.EOF) { + if (I.b == Token.EOF) { + continue; + } + else { + codePointSerializer.serializeCodePoint(data, 0); + } + } + else { + codePointSerializer.serializeCodePoint(data, I.a); + } + + codePointSerializer.serializeCodePoint(data, I.b); + } + } + } + public String decode(char[] data) { data = data.clone(); // don't adjust the first value since that's the version number @@ -436,25 +487,10 @@ public class ATNSerializer { int s = ATNDeserializer.toInt(data[p++]); buf.append("mode ").append(i).append(":").append(s).append('\n'); } - int nsets = ATNDeserializer.toInt(data[p++]); - for (int i=0; i<nsets; i++) { - int nintervals = ATNDeserializer.toInt(data[p++]); - buf.append(i).append(":"); - boolean containsEof = data[p++] != 0; - if (containsEof) { - buf.append(getTokenName(Token.EOF)); - } - - for (int j=0; j<nintervals; j++) { - if ( containsEof || j>0 ) { - buf.append(", "); - } - - buf.append(getTokenName(ATNDeserializer.toInt(data[p]))).append("..").append(getTokenName(ATNDeserializer.toInt(data[p + 1]))); - p += 2; - } - buf.append("\n"); - } + int numBMPSets = ATNDeserializer.toInt(data[p++]); + p = appendSets(buf, data, p, numBMPSets, 0, ATNDeserializer.getUnicodeDeserializer(ATNDeserializer.UnicodeDeserializingMode.UNICODE_BMP)); + int numSMPSets = ATNDeserializer.toInt(data[p++]); + p = appendSets(buf, data, p, numSMPSets, numBMPSets, ATNDeserializer.getUnicodeDeserializer(ATNDeserializer.UnicodeDeserializingMode.UNICODE_SMP)); int nedges = ATNDeserializer.toInt(data[p++]); for (int i=0; i<nedges; i++) { int src = ATNDeserializer.toInt(data[p]); @@ -490,6 +526,31 @@ public class ATNSerializer { return buf.toString(); } + private int appendSets(StringBuilder buf, char[] data, int p, int nsets, int setIndexOffset, ATNDeserializer.UnicodeDeserializer unicodeDeserializer) { + for (int i=0; i<nsets; i++) { + int nintervals = ATNDeserializer.toInt(data[p++]); + buf.append(i+setIndexOffset).append(":"); + boolean containsEof = data[p++] != 0; + if (containsEof) { + buf.append(getTokenName(Token.EOF)); + } + + for (int j=0; j<nintervals; j++) { + if ( containsEof || j>0 ) { + buf.append(", "); + } + + int a = unicodeDeserializer.readUnicode(data, p); + p += unicodeDeserializer.size(); + int b = unicodeDeserializer.readUnicode(data, p); + p += unicodeDeserializer.size(); + buf.append(getTokenName(a)).append("..").append(getTokenName(b)); + } + buf.append("\n"); + } + return p; + } + public String getTokenName(int t) { if ( t==-1 ) return "EOF"; diff --git a/runtime/Java/src/org/antlr/v4/runtime/atn/ATNSimulator.java b/runtime/Java/src/org/antlr/v4/runtime/atn/ATNSimulator.java index 5a1abe0..752a9a7 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/atn/ATNSimulator.java +++ b/runtime/Java/src/org/antlr/v4/runtime/atn/ATNSimulator.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/runtime/Java/src/org/antlr/v4/runtime/atn/ATNState.java b/runtime/Java/src/org/antlr/v4/runtime/atn/ATNState.java index deb845d..955a25d 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/atn/ATNState.java +++ b/runtime/Java/src/org/antlr/v4/runtime/atn/ATNState.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/runtime/Java/src/org/antlr/v4/runtime/atn/ATNType.java b/runtime/Java/src/org/antlr/v4/runtime/atn/ATNType.java index 2ac99b1..613b566 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/atn/ATNType.java +++ b/runtime/Java/src/org/antlr/v4/runtime/atn/ATNType.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/runtime/Java/src/org/antlr/v4/runtime/atn/AbstractPredicateTransition.java b/runtime/Java/src/org/antlr/v4/runtime/atn/AbstractPredicateTransition.java index d00a793..b5aee23 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/atn/AbstractPredicateTransition.java +++ b/runtime/Java/src/org/antlr/v4/runtime/atn/AbstractPredicateTransition.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/runtime/Java/src/org/antlr/v4/runtime/atn/ActionTransition.java b/runtime/Java/src/org/antlr/v4/runtime/atn/ActionTransition.java index e89f5c2..1d35b21 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/atn/ActionTransition.java +++ b/runtime/Java/src/org/antlr/v4/runtime/atn/ActionTransition.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/runtime/Java/src/org/antlr/v4/runtime/atn/AmbiguityInfo.java b/runtime/Java/src/org/antlr/v4/runtime/atn/AmbiguityInfo.java index 647f4b1..8bc2323 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/atn/AmbiguityInfo.java +++ b/runtime/Java/src/org/antlr/v4/runtime/atn/AmbiguityInfo.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/runtime/Java/src/org/antlr/v4/runtime/atn/ArrayPredictionContext.java b/runtime/Java/src/org/antlr/v4/runtime/atn/ArrayPredictionContext.java index 178b085..9e9dcdd 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/atn/ArrayPredictionContext.java +++ b/runtime/Java/src/org/antlr/v4/runtime/atn/ArrayPredictionContext.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/runtime/Java/src/org/antlr/v4/runtime/atn/AtomTransition.java b/runtime/Java/src/org/antlr/v4/runtime/atn/AtomTransition.java index 446f5b3..56cf6e5 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/atn/AtomTransition.java +++ b/runtime/Java/src/org/antlr/v4/runtime/atn/AtomTransition.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/runtime/Java/src/org/antlr/v4/runtime/atn/BasicBlockStartState.java b/runtime/Java/src/org/antlr/v4/runtime/atn/BasicBlockStartState.java index 84216ae..d706162 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/atn/BasicBlockStartState.java +++ b/runtime/Java/src/org/antlr/v4/runtime/atn/BasicBlockStartState.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/runtime/Java/src/org/antlr/v4/runtime/atn/BasicState.java b/runtime/Java/src/org/antlr/v4/runtime/atn/BasicState.java index b139cea..d7b20bc 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/atn/BasicState.java +++ b/runtime/Java/src/org/antlr/v4/runtime/atn/BasicState.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/runtime/Java/src/org/antlr/v4/runtime/atn/BlockEndState.java b/runtime/Java/src/org/antlr/v4/runtime/atn/BlockEndState.java index c8cc871..fae31cc 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/atn/BlockEndState.java +++ b/runtime/Java/src/org/antlr/v4/runtime/atn/BlockEndState.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/runtime/Java/src/org/antlr/v4/runtime/atn/BlockStartState.java b/runtime/Java/src/org/antlr/v4/runtime/atn/BlockStartState.java index 0fb8380..68f03d5 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/atn/BlockStartState.java +++ b/runtime/Java/src/org/antlr/v4/runtime/atn/BlockStartState.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/runtime/Java/src/org/antlr/v4/runtime/atn/CodePointTransitions.java b/runtime/Java/src/org/antlr/v4/runtime/atn/CodePointTransitions.java new file mode 100644 index 0000000..7aedfc4 --- /dev/null +++ b/runtime/Java/src/org/antlr/v4/runtime/atn/CodePointTransitions.java @@ -0,0 +1,52 @@ +/* + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. + * Use of this file is governed by the BSD 3-clause license that + * can be found in the LICENSE.txt file in the project root. + */ + +package org.antlr.v4.runtime.atn; + +import org.antlr.v4.runtime.misc.IntervalSet; + +/** + * Utility class to create {@link AtomTransition}, {@link RangeTransition}, + * and {@link SetTransition} appropriately based on the range of the input. + * + * To keep the serialized ATN size small, we only inline atom and + * range transitions for Unicode code points <= U+FFFF. + * + * Whenever we encounter a Unicode code point > U+FFFF, we represent that + * as a set transition (even if it is logically an atom or a range). + */ +public abstract class CodePointTransitions { + /** + * If {@code codePoint} is <= U+FFFF, returns a new {@link AtomTransition}. + * Otherwise, returns a new {@link SetTransition}. + */ + public static Transition createWithCodePoint(ATNState target, int codePoint) { + if (Character.isSupplementaryCodePoint(codePoint)) { + return new SetTransition(target, IntervalSet.of(codePoint)); + } + else { + return new AtomTransition(target, codePoint); + } + } + + /** + * If {@code codePointFrom} and {@code codePointTo} are both + * <= U+FFFF, returns a new {@link RangeTransition}. + * Otherwise, returns a new {@link SetTransition}. + */ + public static Transition createWithCodePointRange( + ATNState target, + int codePointFrom, + int codePointTo) { + if (Character.isSupplementaryCodePoint(codePointFrom) || + Character.isSupplementaryCodePoint(codePointTo)) { + return new SetTransition(target, IntervalSet.of(codePointFrom, codePointTo)); + } + else { + return new RangeTransition(target, codePointFrom, codePointTo); + } + } +} diff --git a/runtime/Java/src/org/antlr/v4/runtime/atn/ContextSensitivityInfo.java b/runtime/Java/src/org/antlr/v4/runtime/atn/ContextSensitivityInfo.java index e825a74..f6e8413 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/atn/ContextSensitivityInfo.java +++ b/runtime/Java/src/org/antlr/v4/runtime/atn/ContextSensitivityInfo.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/runtime/Java/src/org/antlr/v4/runtime/atn/DecisionEventInfo.java b/runtime/Java/src/org/antlr/v4/runtime/atn/DecisionEventInfo.java index a0a117e..e82ca7a 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/atn/DecisionEventInfo.java +++ b/runtime/Java/src/org/antlr/v4/runtime/atn/DecisionEventInfo.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/runtime/Java/src/org/antlr/v4/runtime/atn/DecisionInfo.java b/runtime/Java/src/org/antlr/v4/runtime/atn/DecisionInfo.java index 98ff1c3..7f0bf66 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/atn/DecisionInfo.java +++ b/runtime/Java/src/org/antlr/v4/runtime/atn/DecisionInfo.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/runtime/Java/src/org/antlr/v4/runtime/atn/DecisionState.java b/runtime/Java/src/org/antlr/v4/runtime/atn/DecisionState.java index e4c710e..3ef93d2 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/atn/DecisionState.java +++ b/runtime/Java/src/org/antlr/v4/runtime/atn/DecisionState.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/runtime/Java/src/org/antlr/v4/runtime/atn/EmptyPredictionContext.java b/runtime/Java/src/org/antlr/v4/runtime/atn/EmptyPredictionContext.java index a7c66e9..a11ff15 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/atn/EmptyPredictionContext.java +++ b/runtime/Java/src/org/antlr/v4/runtime/atn/EmptyPredictionContext.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/runtime/Java/src/org/antlr/v4/runtime/atn/EpsilonTransition.java b/runtime/Java/src/org/antlr/v4/runtime/atn/EpsilonTransition.java index ea1b639..ab0b2db 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/atn/EpsilonTransition.java +++ b/runtime/Java/src/org/antlr/v4/runtime/atn/EpsilonTransition.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/runtime/Java/src/org/antlr/v4/runtime/atn/ErrorInfo.java b/runtime/Java/src/org/antlr/v4/runtime/atn/ErrorInfo.java index 35d9277..7b78e83 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/atn/ErrorInfo.java +++ b/runtime/Java/src/org/antlr/v4/runtime/atn/ErrorInfo.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/runtime/Java/src/org/antlr/v4/runtime/atn/LL1Analyzer.java b/runtime/Java/src/org/antlr/v4/runtime/atn/LL1Analyzer.java index 2df50c3..8d5117b 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/atn/LL1Analyzer.java +++ b/runtime/Java/src/org/antlr/v4/runtime/atn/LL1Analyzer.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/runtime/Java/src/org/antlr/v4/runtime/atn/LexerATNConfig.java b/runtime/Java/src/org/antlr/v4/runtime/atn/LexerATNConfig.java index db7b8e8..ad8766c 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/atn/LexerATNConfig.java +++ b/runtime/Java/src/org/antlr/v4/runtime/atn/LexerATNConfig.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/runtime/Java/src/org/antlr/v4/runtime/atn/LexerATNSimulator.java b/runtime/Java/src/org/antlr/v4/runtime/atn/LexerATNSimulator.java index 81f7f64..ec74ce5 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/atn/LexerATNSimulator.java +++ b/runtime/Java/src/org/antlr/v4/runtime/atn/LexerATNSimulator.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ @@ -372,7 +372,7 @@ public class LexerATNSimulator extends ATNSimulator { protected ATNState getReachableTarget(Transition trans, int t) { - if (trans.matches(t, Character.MIN_VALUE, Character.MAX_VALUE)) { + if (trans.matches(t, Lexer.MIN_CHAR_VALUE, Lexer.MAX_CHAR_VALUE)) { return trans.target; } @@ -544,7 +544,7 @@ public class LexerATNSimulator extends ATNSimulator { case Transition.RANGE: case Transition.SET: if (treatEofAsEpsilon) { - if (t.matches(CharStream.EOF, Character.MIN_VALUE, Character.MAX_VALUE)) { + if (t.matches(CharStream.EOF, Lexer.MIN_CHAR_VALUE, Lexer.MAX_CHAR_VALUE)) { c = new LexerATNConfig(config, t.target); break; } @@ -738,7 +738,8 @@ public class LexerATNSimulator extends ATNSimulator { if ( curChar=='\n' ) { line++; charPositionInLine=0; - } else { + } + else { charPositionInLine++; } input.consume(); diff --git a/runtime/Java/src/org/antlr/v4/runtime/atn/LexerAction.java b/runtime/Java/src/org/antlr/v4/runtime/atn/LexerAction.java index 9657f63..eb10ff4 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/atn/LexerAction.java +++ b/runtime/Java/src/org/antlr/v4/runtime/atn/LexerAction.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/runtime/Java/src/org/antlr/v4/runtime/atn/LexerActionExecutor.java b/runtime/Java/src/org/antlr/v4/runtime/atn/LexerActionExecutor.java index f8c214b..3ec58f7 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/atn/LexerActionExecutor.java +++ b/runtime/Java/src/org/antlr/v4/runtime/atn/LexerActionExecutor.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/runtime/Java/src/org/antlr/v4/runtime/atn/LexerActionType.java b/runtime/Java/src/org/antlr/v4/runtime/atn/LexerActionType.java index e1413d4..4042fd3 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/atn/LexerActionType.java +++ b/runtime/Java/src/org/antlr/v4/runtime/atn/LexerActionType.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/runtime/Java/src/org/antlr/v4/runtime/atn/LexerChannelAction.java b/runtime/Java/src/org/antlr/v4/runtime/atn/LexerChannelAction.java index b37f773..af4419f 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/atn/LexerChannelAction.java +++ b/runtime/Java/src/org/antlr/v4/runtime/atn/LexerChannelAction.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/runtime/Java/src/org/antlr/v4/runtime/atn/LexerCustomAction.java b/runtime/Java/src/org/antlr/v4/runtime/atn/LexerCustomAction.java index 04bec9f..8752b1d 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/atn/LexerCustomAction.java +++ b/runtime/Java/src/org/antlr/v4/runtime/atn/LexerCustomAction.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/runtime/Java/src/org/antlr/v4/runtime/atn/LexerIndexedCustomAction.java b/runtime/Java/src/org/antlr/v4/runtime/atn/LexerIndexedCustomAction.java index 9fcd797..a78c823 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/atn/LexerIndexedCustomAction.java +++ b/runtime/Java/src/org/antlr/v4/runtime/atn/LexerIndexedCustomAction.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/runtime/Java/src/org/antlr/v4/runtime/atn/LexerModeAction.java b/runtime/Java/src/org/antlr/v4/runtime/atn/LexerModeAction.java index a7afcca..1663d6c 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/atn/LexerModeAction.java +++ b/runtime/Java/src/org/antlr/v4/runtime/atn/LexerModeAction.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/runtime/Java/src/org/antlr/v4/runtime/atn/LexerMoreAction.java b/runtime/Java/src/org/antlr/v4/runtime/atn/LexerMoreAction.java index ee1543c..5e0e386 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/atn/LexerMoreAction.java +++ b/runtime/Java/src/org/antlr/v4/runtime/atn/LexerMoreAction.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/runtime/Java/src/org/antlr/v4/runtime/atn/LexerPopModeAction.java b/runtime/Java/src/org/antlr/v4/runtime/atn/LexerPopModeAction.java index 36e7445..afdd2bc 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/atn/LexerPopModeAction.java +++ b/runtime/Java/src/org/antlr/v4/runtime/atn/LexerPopModeAction.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/runtime/Java/src/org/antlr/v4/runtime/atn/LexerPushModeAction.java b/runtime/Java/src/org/antlr/v4/runtime/atn/LexerPushModeAction.java index eb1c138..844c620 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/atn/LexerPushModeAction.java +++ b/runtime/Java/src/org/antlr/v4/runtime/atn/LexerPushModeAction.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/runtime/Java/src/org/antlr/v4/runtime/atn/LexerSkipAction.java b/runtime/Java/src/org/antlr/v4/runtime/atn/LexerSkipAction.java index 762477b..72b7bb6 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/atn/LexerSkipAction.java +++ b/runtime/Java/src/org/antlr/v4/runtime/atn/LexerSkipAction.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/runtime/Java/src/org/antlr/v4/runtime/atn/LexerTypeAction.java b/runtime/Java/src/org/antlr/v4/runtime/atn/LexerTypeAction.java index 9f30ecc..4151d2e 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/atn/LexerTypeAction.java +++ b/runtime/Java/src/org/antlr/v4/runtime/atn/LexerTypeAction.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/runtime/Java/src/org/antlr/v4/runtime/atn/LookaheadEventInfo.java b/runtime/Java/src/org/antlr/v4/runtime/atn/LookaheadEventInfo.java index 4f929fd..90e275e 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/atn/LookaheadEventInfo.java +++ b/runtime/Java/src/org/antlr/v4/runtime/atn/LookaheadEventInfo.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/runtime/Java/src/org/antlr/v4/runtime/atn/LoopEndState.java b/runtime/Java/src/org/antlr/v4/runtime/atn/LoopEndState.java index e06fcbf..d8213a9 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/atn/LoopEndState.java +++ b/runtime/Java/src/org/antlr/v4/runtime/atn/LoopEndState.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/runtime/Java/src/org/antlr/v4/runtime/atn/NotSetTransition.java b/runtime/Java/src/org/antlr/v4/runtime/atn/NotSetTransition.java index 36f0cbb..4599e7b 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/atn/NotSetTransition.java +++ b/runtime/Java/src/org/antlr/v4/runtime/atn/NotSetTransition.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/runtime/Java/src/org/antlr/v4/runtime/atn/OrderedATNConfigSet.java b/runtime/Java/src/org/antlr/v4/runtime/atn/OrderedATNConfigSet.java index 5b3dc3d..dbaa2ca 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/atn/OrderedATNConfigSet.java +++ b/runtime/Java/src/org/antlr/v4/runtime/atn/OrderedATNConfigSet.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/runtime/Java/src/org/antlr/v4/runtime/atn/ParseInfo.java b/runtime/Java/src/org/antlr/v4/runtime/atn/ParseInfo.java index 3f8bd88..1e83ace 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/atn/ParseInfo.java +++ b/runtime/Java/src/org/antlr/v4/runtime/atn/ParseInfo.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/runtime/Java/src/org/antlr/v4/runtime/atn/ParserATNSimulator.java b/runtime/Java/src/org/antlr/v4/runtime/atn/ParserATNSimulator.java index 36965ad..76524eb 100755 --- a/runtime/Java/src/org/antlr/v4/runtime/atn/ParserATNSimulator.java +++ b/runtime/Java/src/org/antlr/v4/runtime/atn/ParserATNSimulator.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ @@ -270,7 +270,7 @@ public class ParserATNSimulator extends ATNSimulator { public static final boolean retry_debug = false; /** Just in case this optimization is bad, add an ENV variable to turn it off */ - public static final boolean TURN_OFF_LR_LOOP_ENTRY_BRANCH_OPT = Boolean.parseBoolean(System.getenv("TURN_OFF_LR_LOOP_ENTRY_BRANCH_OPT")); + public static final boolean TURN_OFF_LR_LOOP_ENTRY_BRANCH_OPT = Boolean.parseBoolean(getSafeEnv("TURN_OFF_LR_LOOP_ENTRY_BRANCH_OPT")); protected final Parser parser; @@ -1541,11 +1541,6 @@ public class ParserATNSimulator extends ATNSimulator { ATNConfig c = getEpsilonTarget(config, t, continueCollecting, depth == 0, fullCtx, treatEofAsEpsilon); if ( c!=null ) { - if (!t.isEpsilon() && !closureBusy.add(c)) { - // avoid infinite recursion for EOF* and EOF+ - continue; - } - int newDepth = depth; if ( config.state instanceof RuleStopState) { assert !fullCtx; @@ -1555,11 +1550,6 @@ public class ParserATNSimulator extends ATNSimulator { // come in handy and we avoid evaluating context dependent // preds if this is > 0. - if (!closureBusy.add(c)) { - // avoid infinite recursion for right-recursive rules - continue; - } - if (_dfa != null && _dfa.isPrecedenceDfa()) { int outermostPrecedenceReturn = ((EpsilonTransition)t).outermostPrecedenceReturn(); if (outermostPrecedenceReturn == _dfa.atnStartState.ruleIndex) { @@ -1568,15 +1558,28 @@ public class ParserATNSimulator extends ATNSimulator { } c.reachesIntoOuterContext++; + + if (!closureBusy.add(c)) { + // avoid infinite recursion for right-recursive rules + continue; + } + configs.dipsIntoOuterContext = true; // TODO: can remove? only care when we add to set per middle of this method assert newDepth > Integer.MIN_VALUE; newDepth--; if ( debug ) System.out.println("dips into outer ctx: "+c); } - else if (t instanceof RuleTransition) { - // latch when newDepth goes negative - once we step out of the entry context we can't return - if (newDepth >= 0) { - newDepth++; + else { + if (!t.isEpsilon() && !closureBusy.add(c)) { + // avoid infinite recursion for EOF* and EOF+ + continue; + } + + if (t instanceof RuleTransition) { + // latch when newDepth goes negative - once we step out of the entry context we can't return + if (newDepth >= 0) { + newDepth++; + } } } @@ -2178,4 +2181,14 @@ public class ParserATNSimulator extends ATNSimulator { public Parser getParser() { return parser; } + + public static String getSafeEnv(String envName) { + try { + return System.getenv(envName); + } + catch(SecurityException e) { + // use the default value + } + return null; + } } diff --git a/runtime/Java/src/org/antlr/v4/runtime/atn/PlusBlockStartState.java b/runtime/Java/src/org/antlr/v4/runtime/atn/PlusBlockStartState.java index 095a48a..025ae46 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/atn/PlusBlockStartState.java +++ b/runtime/Java/src/org/antlr/v4/runtime/atn/PlusBlockStartState.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/runtime/Java/src/org/antlr/v4/runtime/atn/PlusLoopbackState.java b/runtime/Java/src/org/antlr/v4/runtime/atn/PlusLoopbackState.java index 96b5d16..77baf36 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/atn/PlusLoopbackState.java +++ b/runtime/Java/src/org/antlr/v4/runtime/atn/PlusLoopbackState.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/runtime/Java/src/org/antlr/v4/runtime/atn/PrecedencePredicateTransition.java b/runtime/Java/src/org/antlr/v4/runtime/atn/PrecedencePredicateTransition.java index 546bafd..9823c63 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/atn/PrecedencePredicateTransition.java +++ b/runtime/Java/src/org/antlr/v4/runtime/atn/PrecedencePredicateTransition.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/runtime/Java/src/org/antlr/v4/runtime/atn/PredicateEvalInfo.java b/runtime/Java/src/org/antlr/v4/runtime/atn/PredicateEvalInfo.java index d2a73b4..3ee83e1 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/atn/PredicateEvalInfo.java +++ b/runtime/Java/src/org/antlr/v4/runtime/atn/PredicateEvalInfo.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/runtime/Java/src/org/antlr/v4/runtime/atn/PredicateTransition.java b/runtime/Java/src/org/antlr/v4/runtime/atn/PredicateTransition.java index c4381bd..cc4058e 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/atn/PredicateTransition.java +++ b/runtime/Java/src/org/antlr/v4/runtime/atn/PredicateTransition.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/runtime/Java/src/org/antlr/v4/runtime/atn/PredictionContext.java b/runtime/Java/src/org/antlr/v4/runtime/atn/PredictionContext.java index 8a23d34..c55ba09 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/atn/PredictionContext.java +++ b/runtime/Java/src/org/antlr/v4/runtime/atn/PredictionContext.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/runtime/Java/src/org/antlr/v4/runtime/atn/PredictionContextCache.java b/runtime/Java/src/org/antlr/v4/runtime/atn/PredictionContextCache.java index 62c9196..11711ee 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/atn/PredictionContextCache.java +++ b/runtime/Java/src/org/antlr/v4/runtime/atn/PredictionContextCache.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/runtime/Java/src/org/antlr/v4/runtime/atn/PredictionMode.java b/runtime/Java/src/org/antlr/v4/runtime/atn/PredictionMode.java index b0ca601..51a51d2 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/atn/PredictionMode.java +++ b/runtime/Java/src/org/antlr/v4/runtime/atn/PredictionMode.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/runtime/Java/src/org/antlr/v4/runtime/atn/ProfilingATNSimulator.java b/runtime/Java/src/org/antlr/v4/runtime/atn/ProfilingATNSimulator.java index 51ded0a..dd29f02 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/atn/ProfilingATNSimulator.java +++ b/runtime/Java/src/org/antlr/v4/runtime/atn/ProfilingATNSimulator.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/runtime/Java/src/org/antlr/v4/runtime/atn/RangeTransition.java b/runtime/Java/src/org/antlr/v4/runtime/atn/RangeTransition.java index 92001cc..4eee283 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/atn/RangeTransition.java +++ b/runtime/Java/src/org/antlr/v4/runtime/atn/RangeTransition.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ @@ -34,6 +34,11 @@ public final class RangeTransition extends Transition { @Override public String toString() { - return "'"+(char)from+"'..'"+(char)to+"'"; + return new StringBuilder("'") + .appendCodePoint(from) + .append("'..'") + .appendCodePoint(to) + .append("'") + .toString(); } } diff --git a/runtime/Java/src/org/antlr/v4/runtime/atn/RuleStartState.java b/runtime/Java/src/org/antlr/v4/runtime/atn/RuleStartState.java index e7c9629..9265ef5 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/atn/RuleStartState.java +++ b/runtime/Java/src/org/antlr/v4/runtime/atn/RuleStartState.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/runtime/Java/src/org/antlr/v4/runtime/atn/RuleStopState.java b/runtime/Java/src/org/antlr/v4/runtime/atn/RuleStopState.java index b72ee31..fdaf9b3 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/atn/RuleStopState.java +++ b/runtime/Java/src/org/antlr/v4/runtime/atn/RuleStopState.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/runtime/Java/src/org/antlr/v4/runtime/atn/RuleTransition.java b/runtime/Java/src/org/antlr/v4/runtime/atn/RuleTransition.java index ff855d7..c3c0cee 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/atn/RuleTransition.java +++ b/runtime/Java/src/org/antlr/v4/runtime/atn/RuleTransition.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/runtime/Java/src/org/antlr/v4/runtime/atn/SemanticContext.java b/runtime/Java/src/org/antlr/v4/runtime/atn/SemanticContext.java index 922e931..3b888e7 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/atn/SemanticContext.java +++ b/runtime/Java/src/org/antlr/v4/runtime/atn/SemanticContext.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/runtime/Java/src/org/antlr/v4/runtime/atn/SetTransition.java b/runtime/Java/src/org/antlr/v4/runtime/atn/SetTransition.java index 86904bd..a72708a 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/atn/SetTransition.java +++ b/runtime/Java/src/org/antlr/v4/runtime/atn/SetTransition.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/runtime/Java/src/org/antlr/v4/runtime/atn/SingletonPredictionContext.java b/runtime/Java/src/org/antlr/v4/runtime/atn/SingletonPredictionContext.java index 582c6a6..ca5ea66 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/atn/SingletonPredictionContext.java +++ b/runtime/Java/src/org/antlr/v4/runtime/atn/SingletonPredictionContext.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/runtime/Java/src/org/antlr/v4/runtime/atn/StarBlockStartState.java b/runtime/Java/src/org/antlr/v4/runtime/atn/StarBlockStartState.java index 3321e0a..5996806 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/atn/StarBlockStartState.java +++ b/runtime/Java/src/org/antlr/v4/runtime/atn/StarBlockStartState.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/runtime/Java/src/org/antlr/v4/runtime/atn/StarLoopEntryState.java b/runtime/Java/src/org/antlr/v4/runtime/atn/StarLoopEntryState.java index aa1fe02..5951831 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/atn/StarLoopEntryState.java +++ b/runtime/Java/src/org/antlr/v4/runtime/atn/StarLoopEntryState.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/runtime/Java/src/org/antlr/v4/runtime/atn/StarLoopbackState.java b/runtime/Java/src/org/antlr/v4/runtime/atn/StarLoopbackState.java index f7ad420..4e2b449 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/atn/StarLoopbackState.java +++ b/runtime/Java/src/org/antlr/v4/runtime/atn/StarLoopbackState.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/runtime/Java/src/org/antlr/v4/runtime/atn/TokensStartState.java b/runtime/Java/src/org/antlr/v4/runtime/atn/TokensStartState.java index d1be204..846b6ec 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/atn/TokensStartState.java +++ b/runtime/Java/src/org/antlr/v4/runtime/atn/TokensStartState.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/runtime/Java/src/org/antlr/v4/runtime/atn/Transition.java b/runtime/Java/src/org/antlr/v4/runtime/atn/Transition.java index 04b368f..8c0fbc5 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/atn/Transition.java +++ b/runtime/Java/src/org/antlr/v4/runtime/atn/Transition.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/runtime/Java/src/org/antlr/v4/runtime/atn/WildcardTransition.java b/runtime/Java/src/org/antlr/v4/runtime/atn/WildcardTransition.java index 1634196..2707065 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/atn/WildcardTransition.java +++ b/runtime/Java/src/org/antlr/v4/runtime/atn/WildcardTransition.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/runtime/Java/src/org/antlr/v4/runtime/dfa/DFA.java b/runtime/Java/src/org/antlr/v4/runtime/dfa/DFA.java index b508f8b..3c91dab 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/dfa/DFA.java +++ b/runtime/Java/src/org/antlr/v4/runtime/dfa/DFA.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/runtime/Java/src/org/antlr/v4/runtime/dfa/DFASerializer.java b/runtime/Java/src/org/antlr/v4/runtime/dfa/DFASerializer.java index 54b0fab..6504fc6 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/dfa/DFASerializer.java +++ b/runtime/Java/src/org/antlr/v4/runtime/dfa/DFASerializer.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/runtime/Java/src/org/antlr/v4/runtime/dfa/DFAState.java b/runtime/Java/src/org/antlr/v4/runtime/dfa/DFAState.java index 5b74f6c..709094a 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/dfa/DFAState.java +++ b/runtime/Java/src/org/antlr/v4/runtime/dfa/DFAState.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/runtime/Java/src/org/antlr/v4/runtime/dfa/LexerDFASerializer.java b/runtime/Java/src/org/antlr/v4/runtime/dfa/LexerDFASerializer.java index d1d0cca..83d4f8c 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/dfa/LexerDFASerializer.java +++ b/runtime/Java/src/org/antlr/v4/runtime/dfa/LexerDFASerializer.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ @@ -16,6 +16,9 @@ public class LexerDFASerializer extends DFASerializer { @Override protected String getEdgeLabel(int i) { - return "'"+(char)i+"'"; + return new StringBuilder("'") + .appendCodePoint(i) + .append("'") + .toString(); } } diff --git a/runtime/Java/src/org/antlr/v4/runtime/misc/AbstractEqualityComparator.java b/runtime/Java/src/org/antlr/v4/runtime/misc/AbstractEqualityComparator.java index c149d9e..a2bae78 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/misc/AbstractEqualityComparator.java +++ b/runtime/Java/src/org/antlr/v4/runtime/misc/AbstractEqualityComparator.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/runtime/Java/src/org/antlr/v4/runtime/misc/Array2DHashSet.java b/runtime/Java/src/org/antlr/v4/runtime/misc/Array2DHashSet.java index 4595275..9ec8566 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/misc/Array2DHashSet.java +++ b/runtime/Java/src/org/antlr/v4/runtime/misc/Array2DHashSet.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/runtime/Java/src/org/antlr/v4/runtime/misc/DoubleKeyMap.java b/runtime/Java/src/org/antlr/v4/runtime/misc/DoubleKeyMap.java index a612347..a88d17d 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/misc/DoubleKeyMap.java +++ b/runtime/Java/src/org/antlr/v4/runtime/misc/DoubleKeyMap.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/runtime/Java/src/org/antlr/v4/runtime/misc/EqualityComparator.java b/runtime/Java/src/org/antlr/v4/runtime/misc/EqualityComparator.java index 0fee89b..679c6a6 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/misc/EqualityComparator.java +++ b/runtime/Java/src/org/antlr/v4/runtime/misc/EqualityComparator.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/runtime/Java/src/org/antlr/v4/runtime/misc/FlexibleHashMap.java b/runtime/Java/src/org/antlr/v4/runtime/misc/FlexibleHashMap.java index a05008e..3c49eb2 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/misc/FlexibleHashMap.java +++ b/runtime/Java/src/org/antlr/v4/runtime/misc/FlexibleHashMap.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/runtime/Java/src/org/antlr/v4/runtime/misc/IntSet.java b/runtime/Java/src/org/antlr/v4/runtime/misc/IntSet.java index 80d908b..53db074 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/misc/IntSet.java +++ b/runtime/Java/src/org/antlr/v4/runtime/misc/IntSet.java @@ -1,13 +1,11 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ package org.antlr.v4.runtime.misc; -import org.antlr.v4.runtime.Token; - import java.util.List; /** @@ -131,15 +129,6 @@ public interface IntSet { boolean equals(Object obj); /** - * Returns the single value contained in the set, if {@link #size} is 1; - * otherwise, returns {@link Token#INVALID_TYPE}. - * - * @return the single value contained in the set, if {@link #size} is 1; - * otherwise, returns {@link Token#INVALID_TYPE}. - */ - int getSingleElement(); - - /** * Returns {@code true} if the set contains the specified element. * * @param el The element to check for. diff --git a/runtime/Java/src/org/antlr/v4/runtime/misc/IntegerList.java b/runtime/Java/src/org/antlr/v4/runtime/misc/IntegerList.java index 21d5bcd..d6af911 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/misc/IntegerList.java +++ b/runtime/Java/src/org/antlr/v4/runtime/misc/IntegerList.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ @@ -36,7 +36,8 @@ public class IntegerList { if (capacity == 0) { _data = EMPTY_DATA; - } else { + } + else { _data = new int[capacity]; } } @@ -258,7 +259,8 @@ public class IntegerList { int newLength; if (_data.length == 0) { newLength = INITIAL_SIZE; - } else { + } + else { newLength = _data.length; } @@ -272,4 +274,39 @@ public class IntegerList { _data = Arrays.copyOf(_data, newLength); } + /** Convert the list to a UTF-16 encoded char array. If all values are less + * than the 0xFFFF 16-bit code point limit then this is just a char array + * of 16-bit char as usual. For values in the supplementary range, encode + * them as two UTF-16 code units. + */ + public final char[] toCharArray() { + // Optimize for the common case (all data values are + // < 0xFFFF) to avoid an extra scan + char[] resultArray = new char[_size]; + int resultIdx = 0; + boolean calculatedPreciseResultSize = false; + for (int i = 0; i < _size; i++) { + int codePoint = _data[i]; + // Calculate the precise result size if we encounter + // a code point > 0xFFFF + if (!calculatedPreciseResultSize && + Character.isSupplementaryCodePoint(codePoint)) { + resultArray = Arrays.copyOf(resultArray, charArraySize()); + calculatedPreciseResultSize = true; + } + // This will throw IllegalArgumentException if + // the code point is not a valid Unicode code point + int charsWritten = Character.toChars(codePoint, resultArray, resultIdx); + resultIdx += charsWritten; + } + return resultArray; + } + + private int charArraySize() { + int result = 0; + for (int i = 0; i < _size; i++) { + result += Character.charCount(_data[i]); + } + return result; + } } diff --git a/runtime/Java/src/org/antlr/v4/runtime/misc/IntegerStack.java b/runtime/Java/src/org/antlr/v4/runtime/misc/IntegerStack.java index 5250a72..a8f65d9 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/misc/IntegerStack.java +++ b/runtime/Java/src/org/antlr/v4/runtime/misc/IntegerStack.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/runtime/Java/src/org/antlr/v4/runtime/misc/InterpreterDataReader.java b/runtime/Java/src/org/antlr/v4/runtime/misc/InterpreterDataReader.java new file mode 100644 index 0000000..221fa55 --- /dev/null +++ b/runtime/Java/src/org/antlr/v4/runtime/misc/InterpreterDataReader.java @@ -0,0 +1,141 @@ +/* + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. + * Use of this file is governed by the BSD 3-clause license that + * can be found in the LICENSE.txt file in the project root. + */ + +package org.antlr.v4.runtime.misc; + +import org.antlr.v4.runtime.Vocabulary; +import org.antlr.v4.runtime.VocabularyImpl; +import org.antlr.v4.runtime.atn.ATN; +import org.antlr.v4.runtime.atn.ATNDeserializer; +import org.antlr.v4.runtime.dfa.DFA; + +import java.util.Collection; +import java.util.List; +import java.util.ArrayList; +import java.io.BufferedReader; +import java.io.FileReader; + +// A class to read plain text interpreter data produced by ANTLR. +public class InterpreterDataReader { + + public static class InterpreterData { + ATN atn; + Vocabulary vocabulary; + List<String> ruleNames; + List<String> channels; // Only valid for lexer grammars. + List<String> modes; // ditto + }; + + /** + * The structure of the data file is very simple. Everything is line based with empty lines + * separating the different parts. For lexers the layout is: + * token literal names: + * ... + * + * token symbolic names: + * ... + * + * rule names: + * ... + * + * channel names: + * ... + * + * mode names: + * ... + * + * atn: + * <a single line with comma separated int values> enclosed in a pair of squared brackets. + * + * Data for a parser does not contain channel and mode names. + */ + public static InterpreterData parseFile(String fileName) { + InterpreterData result = new InterpreterData(); + result.ruleNames = new ArrayList<String>(); + + try (BufferedReader br = new BufferedReader(new FileReader(fileName))) { + String line; + List<String> literalNames = new ArrayList<String>(); + List<String> symbolicNames = new ArrayList<String>(); + + line = br.readLine(); + if ( !line.equals("token literal names:") ) + throw new RuntimeException("Unexpected data entry"); + while ((line = br.readLine()) != null) { + if ( line.isEmpty() ) + break; + literalNames.add(line.equals("null") ? "" : line); + } + + line = br.readLine(); + if ( !line.equals("token symbolic names:") ) + throw new RuntimeException("Unexpected data entry"); + while ((line = br.readLine()) != null) { + if ( line.isEmpty() ) + break; + symbolicNames.add(line.equals("null") ? "" : line); + } + + result.vocabulary = new VocabularyImpl(literalNames.toArray(new String[0]), symbolicNames.toArray(new String[0])); + + line = br.readLine(); + if ( !line.equals("rule names:") ) + throw new RuntimeException("Unexpected data entry"); + while ((line = br.readLine()) != null) { + if ( line.isEmpty() ) + break; + result.ruleNames.add(line); + } + + if ( line.equals("channel names:") ) { // Additional lexer data. + result.channels = new ArrayList<String>(); + while ((line = br.readLine()) != null) { + if ( line.isEmpty() ) + break; + result.channels.add(line); + } + + line = br.readLine(); + if ( !line.equals("mode names:") ) + throw new RuntimeException("Unexpected data entry"); + result.modes = new ArrayList<String>(); + while ((line = br.readLine()) != null) { + if ( line.isEmpty() ) + break; + result.modes.add(line); + } + } + + line = br.readLine(); + if ( !line.equals("atn:") ) + throw new RuntimeException("Unexpected data entry"); + line = br.readLine(); + String[] elements = line.split(","); + char[] serializedATN = new char[elements.length]; + + for (int i = 0; i < elements.length; ++i) { + int value; + String element = elements[i]; + if ( element.startsWith("[") ) + value = Integer.parseInt(element.substring(1).trim()); + else if ( element.endsWith("]") ) + value = Integer.parseInt(element.substring(0, element.length() - 1).trim()); + else + value = Integer.parseInt(element.trim()); + serializedATN[i] = (char)value; + } + + ATNDeserializer deserializer = new ATNDeserializer(); + result.atn = deserializer.deserialize(serializedATN); + } + catch (java.io.IOException e) { + // We just swallow the error and return empty objects instead. + } + + return result; + } + +} diff --git a/runtime/Java/src/org/antlr/v4/runtime/misc/Interval.java b/runtime/Java/src/org/antlr/v4/runtime/misc/Interval.java index b5d7d1d..ab3f46b 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/misc/Interval.java +++ b/runtime/Java/src/org/antlr/v4/runtime/misc/Interval.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/runtime/Java/src/org/antlr/v4/runtime/misc/IntervalSet.java b/runtime/Java/src/org/antlr/v4/runtime/misc/IntervalSet.java index 27fb574..3dcb0f3 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/misc/IntervalSet.java +++ b/runtime/Java/src/org/antlr/v4/runtime/misc/IntervalSet.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ @@ -384,30 +384,24 @@ public class IntervalSet implements IntSet { @Override public boolean contains(int el) { int n = intervals.size(); - for (int i = 0; i < n; i++) { - Interval I = intervals.get(i); + int l = 0; + int r = n - 1; + // Binary search for the element in the (sorted, + // disjoint) array of intervals. + while (l <= r) { + int m = (l + r) / 2; + Interval I = intervals.get(m); int a = I.a; int b = I.b; - if ( el<a ) { - break; // list is sorted and el is before this interval; not here - } - if ( el>=a && el<=b ) { - return true; // found in this interval + if ( b<el ) { + l = m + 1; + } else if ( a>el ) { + r = m - 1; + } else { // el >= a && el <= b + return true; } } return false; -/* - for (ListIterator iter = intervals.listIterator(); iter.hasNext();) { - Interval I = (Interval) iter.next(); - if ( el<I.a ) { - break; // list is sorted and el is before this interval; not here - } - if ( el>=I.a && el<=I.b ) { - return true; // found in this interval - } - } - return false; - */ } /** {@inheritDoc} */ @@ -416,41 +410,29 @@ public class IntervalSet implements IntSet { return intervals==null || intervals.isEmpty(); } - /** {@inheritDoc} */ - @Override - public int getSingleElement() { - if ( intervals!=null && intervals.size()==1 ) { - Interval I = intervals.get(0); - if ( I.a == I.b ) { - return I.a; - } - } - return Token.INVALID_TYPE; - } - /** - * Returns the maximum value contained in the set. + * Returns the maximum value contained in the set if not isNil(). * - * @return the maximum value contained in the set. If the set is empty, this - * method returns {@link Token#INVALID_TYPE}. + * @return the maximum value contained in the set. + * @throws RuntimeException if set is empty */ public int getMaxElement() { if ( isNil() ) { - return Token.INVALID_TYPE; + throw new RuntimeException("set is empty"); } Interval last = intervals.get(intervals.size()-1); return last.b; } /** - * Returns the minimum value contained in the set. + * Returns the minimum value contained in the set if not isNil(). * - * @return the minimum value contained in the set. If the set is empty, this - * method returns {@link Token#INVALID_TYPE}. + * @return the minimum value contained in the set. + * @throws RuntimeException if set is empty */ public int getMinElement() { if ( isNil() ) { - return Token.INVALID_TYPE; + throw new RuntimeException("set is empty"); } return intervals.get(0).a; @@ -505,11 +487,11 @@ public class IntervalSet implements IntSet { int b = I.b; if ( a==b ) { if ( a==Token.EOF ) buf.append("<EOF>"); - else if ( elemAreChar ) buf.append("'").append((char)a).append("'"); + else if ( elemAreChar ) buf.append("'").appendCodePoint(a).append("'"); else buf.append(a); } else { - if ( elemAreChar ) buf.append("'").append((char)a).append("'..'").append((char)b).append("'"); + if ( elemAreChar ) buf.append("'").appendCodePoint(a).append("'..'").appendCodePoint(b).append("'"); else buf.append(a).append("..").append(b); } if ( iter.hasNext() ) { diff --git a/runtime/Java/src/org/antlr/v4/runtime/misc/LogManager.java b/runtime/Java/src/org/antlr/v4/runtime/misc/LogManager.java index d47b5a2..174a060 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/misc/LogManager.java +++ b/runtime/Java/src/org/antlr/v4/runtime/misc/LogManager.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/runtime/Java/src/org/antlr/v4/runtime/misc/MultiMap.java b/runtime/Java/src/org/antlr/v4/runtime/misc/MultiMap.java index ce7972e..dcf802c 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/misc/MultiMap.java +++ b/runtime/Java/src/org/antlr/v4/runtime/misc/MultiMap.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/runtime/Java/src/org/antlr/v4/runtime/misc/MurmurHash.java b/runtime/Java/src/org/antlr/v4/runtime/misc/MurmurHash.java index c498b06..79e251f 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/misc/MurmurHash.java +++ b/runtime/Java/src/org/antlr/v4/runtime/misc/MurmurHash.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/runtime/Java/src/org/antlr/v4/runtime/misc/NotNull.java b/runtime/Java/src/org/antlr/v4/runtime/misc/NotNull.java index 25c59e4..7805291 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/misc/NotNull.java +++ b/runtime/Java/src/org/antlr/v4/runtime/misc/NotNull.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/runtime/Java/src/org/antlr/v4/runtime/misc/ObjectEqualityComparator.java b/runtime/Java/src/org/antlr/v4/runtime/misc/ObjectEqualityComparator.java index 256a17b..d9c0022 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/misc/ObjectEqualityComparator.java +++ b/runtime/Java/src/org/antlr/v4/runtime/misc/ObjectEqualityComparator.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/runtime/Java/src/org/antlr/v4/runtime/misc/OrderedHashSet.java b/runtime/Java/src/org/antlr/v4/runtime/misc/OrderedHashSet.java index d8f9d1b..7c4c526 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/misc/OrderedHashSet.java +++ b/runtime/Java/src/org/antlr/v4/runtime/misc/OrderedHashSet.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/runtime/Java/src/org/antlr/v4/runtime/misc/Pair.java b/runtime/Java/src/org/antlr/v4/runtime/misc/Pair.java index c01c907..289966f 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/misc/Pair.java +++ b/runtime/Java/src/org/antlr/v4/runtime/misc/Pair.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/runtime/Java/src/org/antlr/v4/runtime/misc/ParseCancellationException.java b/runtime/Java/src/org/antlr/v4/runtime/misc/ParseCancellationException.java index 4e1ebdb..b53d518 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/misc/ParseCancellationException.java +++ b/runtime/Java/src/org/antlr/v4/runtime/misc/ParseCancellationException.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/runtime/Java/src/org/antlr/v4/runtime/misc/Predicate.java b/runtime/Java/src/org/antlr/v4/runtime/misc/Predicate.java index 1c297a5..3107eb7 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/misc/Predicate.java +++ b/runtime/Java/src/org/antlr/v4/runtime/misc/Predicate.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/runtime/Java/src/org/antlr/v4/runtime/misc/TestRig.java b/runtime/Java/src/org/antlr/v4/runtime/misc/TestRig.java index de6f630..2be9784 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/misc/TestRig.java +++ b/runtime/Java/src/org/antlr/v4/runtime/misc/TestRig.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/runtime/Java/src/org/antlr/v4/runtime/misc/Triple.java b/runtime/Java/src/org/antlr/v4/runtime/misc/Triple.java index fdbf1c0..a7960cb 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/misc/Triple.java +++ b/runtime/Java/src/org/antlr/v4/runtime/misc/Triple.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/runtime/Java/src/org/antlr/v4/runtime/misc/Utils.java b/runtime/Java/src/org/antlr/v4/runtime/misc/Utils.java index 450c603..e4d91a1 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/misc/Utils.java +++ b/runtime/Java/src/org/antlr/v4/runtime/misc/Utils.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ @@ -137,11 +137,7 @@ public class Utils { public static char[] toCharArray(IntegerList data) { if ( data==null ) return null; - char[] cdata = new char[data.size()]; - for (int i=0; i<data.size(); i++) { - cdata[i] = (char)data.get(i); - } - return cdata; + return data.toCharArray(); } public static IntervalSet toSet(BitSet bits) { diff --git a/runtime/Java/src/org/antlr/v4/runtime/tree/AbstractParseTreeVisitor.java b/runtime/Java/src/org/antlr/v4/runtime/tree/AbstractParseTreeVisitor.java index f565fbf..93eaa46 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/tree/AbstractParseTreeVisitor.java +++ b/runtime/Java/src/org/antlr/v4/runtime/tree/AbstractParseTreeVisitor.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/runtime/Java/src/org/antlr/v4/runtime/tree/ErrorNode.java b/runtime/Java/src/org/antlr/v4/runtime/tree/ErrorNode.java index 87bf2fa..a54fce5 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/tree/ErrorNode.java +++ b/runtime/Java/src/org/antlr/v4/runtime/tree/ErrorNode.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/runtime/Java/src/org/antlr/v4/runtime/tree/ErrorNodeImpl.java b/runtime/Java/src/org/antlr/v4/runtime/tree/ErrorNodeImpl.java index 9295282..769763a 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/tree/ErrorNodeImpl.java +++ b/runtime/Java/src/org/antlr/v4/runtime/tree/ErrorNodeImpl.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/runtime/Java/src/org/antlr/v4/runtime/tree/IterativeParseTreeWalker.java b/runtime/Java/src/org/antlr/v4/runtime/tree/IterativeParseTreeWalker.java index edb0746..915869f 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/tree/IterativeParseTreeWalker.java +++ b/runtime/Java/src/org/antlr/v4/runtime/tree/IterativeParseTreeWalker.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ @@ -32,9 +32,11 @@ public class IterativeParseTreeWalker extends ParseTreeWalker { // pre-order visit if (currentNode instanceof ErrorNode) { listener.visitErrorNode((ErrorNode) currentNode); - } else if (currentNode instanceof TerminalNode) { + } + else if (currentNode instanceof TerminalNode) { listener.visitTerminal((TerminalNode) currentNode); - } else { + } + else { final RuleNode r = (RuleNode) currentNode; enterRule(listener, r); } diff --git a/runtime/Java/src/org/antlr/v4/runtime/tree/ParseTree.java b/runtime/Java/src/org/antlr/v4/runtime/tree/ParseTree.java index ebb20db..2417068 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/tree/ParseTree.java +++ b/runtime/Java/src/org/antlr/v4/runtime/tree/ParseTree.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ @@ -24,6 +24,24 @@ public interface ParseTree extends SyntaxTree { @Override ParseTree getChild(int i); + + /** Set the parent for this node. + * + * This is not backward compatible as it changes + * the interface but no one was able to create custom + * nodes anyway so I'm adding as it improves internal + * code quality. + * + * One could argue for a restructuring of + * the class/interface hierarchy so that + * setParent, addChild are moved up to Tree + * but that's a major change. So I'll do the + * minimal change, which is to add this method. + * + * @since 4.7 + */ + void setParent(RuleContext parent); + /** The {@link ParseTreeVisitor} needs a double dispatch method. */ <T> T accept(ParseTreeVisitor<? extends T> visitor); diff --git a/runtime/Java/src/org/antlr/v4/runtime/tree/ParseTreeListener.java b/runtime/Java/src/org/antlr/v4/runtime/tree/ParseTreeListener.java index 4fa245e..b9fe5aa 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/tree/ParseTreeListener.java +++ b/runtime/Java/src/org/antlr/v4/runtime/tree/ParseTreeListener.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/runtime/Java/src/org/antlr/v4/runtime/tree/ParseTreeProperty.java b/runtime/Java/src/org/antlr/v4/runtime/tree/ParseTreeProperty.java index ca16b18..22fbddd 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/tree/ParseTreeProperty.java +++ b/runtime/Java/src/org/antlr/v4/runtime/tree/ParseTreeProperty.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/runtime/Java/src/org/antlr/v4/runtime/tree/ParseTreeVisitor.java b/runtime/Java/src/org/antlr/v4/runtime/tree/ParseTreeVisitor.java index f9519da..862ba05 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/tree/ParseTreeVisitor.java +++ b/runtime/Java/src/org/antlr/v4/runtime/tree/ParseTreeVisitor.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/runtime/Java/src/org/antlr/v4/runtime/tree/ParseTreeWalker.java b/runtime/Java/src/org/antlr/v4/runtime/tree/ParseTreeWalker.java index 3d0546d..ede5feb 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/tree/ParseTreeWalker.java +++ b/runtime/Java/src/org/antlr/v4/runtime/tree/ParseTreeWalker.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/runtime/Java/src/org/antlr/v4/runtime/tree/RuleNode.java b/runtime/Java/src/org/antlr/v4/runtime/tree/RuleNode.java index 5e31437..f4a1235 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/tree/RuleNode.java +++ b/runtime/Java/src/org/antlr/v4/runtime/tree/RuleNode.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/runtime/Java/src/org/antlr/v4/runtime/tree/SyntaxTree.java b/runtime/Java/src/org/antlr/v4/runtime/tree/SyntaxTree.java index cedfa19..1502e3d 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/tree/SyntaxTree.java +++ b/runtime/Java/src/org/antlr/v4/runtime/tree/SyntaxTree.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/runtime/Java/src/org/antlr/v4/runtime/tree/TerminalNode.java b/runtime/Java/src/org/antlr/v4/runtime/tree/TerminalNode.java index d8a3e65..c7cfe64 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/tree/TerminalNode.java +++ b/runtime/Java/src/org/antlr/v4/runtime/tree/TerminalNode.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/runtime/Java/src/org/antlr/v4/runtime/tree/TerminalNodeImpl.java b/runtime/Java/src/org/antlr/v4/runtime/tree/TerminalNodeImpl.java index 1882d16..9869cb2 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/tree/TerminalNodeImpl.java +++ b/runtime/Java/src/org/antlr/v4/runtime/tree/TerminalNodeImpl.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ @@ -7,6 +7,7 @@ package org.antlr.v4.runtime.tree; import org.antlr.v4.runtime.Parser; +import org.antlr.v4.runtime.RuleContext; import org.antlr.v4.runtime.Token; import org.antlr.v4.runtime.misc.Interval; @@ -26,6 +27,11 @@ public class TerminalNodeImpl implements TerminalNode { public ParseTree getParent() { return parent; } @Override + public void setParent(RuleContext parent) { + this.parent = parent; + } + + @Override public Token getPayload() { return symbol; } @Override diff --git a/runtime/Java/src/org/antlr/v4/runtime/tree/Tree.java b/runtime/Java/src/org/antlr/v4/runtime/tree/Tree.java index 78f2b75..0a1bed6 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/tree/Tree.java +++ b/runtime/Java/src/org/antlr/v4/runtime/tree/Tree.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/runtime/Java/src/org/antlr/v4/runtime/tree/Trees.java b/runtime/Java/src/org/antlr/v4/runtime/tree/Trees.java index 9c07298..406d2af 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/tree/Trees.java +++ b/runtime/Java/src/org/antlr/v4/runtime/tree/Trees.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ @@ -249,6 +249,8 @@ public class Trees { public static Tree findNodeSuchThat(Tree t, Predicate<Tree> pred) { if ( pred.test(t) ) return t; + if ( t==null ) return null; + int n = t.getChildCount(); for (int i = 0 ; i < n ; i++){ Tree u = findNodeSuchThat(t.getChild(i), pred); diff --git a/runtime/Java/src/org/antlr/v4/runtime/tree/pattern/Chunk.java b/runtime/Java/src/org/antlr/v4/runtime/tree/pattern/Chunk.java index fc33fed..241478f 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/tree/pattern/Chunk.java +++ b/runtime/Java/src/org/antlr/v4/runtime/tree/pattern/Chunk.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/runtime/Java/src/org/antlr/v4/runtime/tree/pattern/ParseTreeMatch.java b/runtime/Java/src/org/antlr/v4/runtime/tree/pattern/ParseTreeMatch.java index 3101916..11204a7 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/tree/pattern/ParseTreeMatch.java +++ b/runtime/Java/src/org/antlr/v4/runtime/tree/pattern/ParseTreeMatch.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/runtime/Java/src/org/antlr/v4/runtime/tree/pattern/ParseTreePattern.java b/runtime/Java/src/org/antlr/v4/runtime/tree/pattern/ParseTreePattern.java index a8f5d14..de7b82a 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/tree/pattern/ParseTreePattern.java +++ b/runtime/Java/src/org/antlr/v4/runtime/tree/pattern/ParseTreePattern.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/runtime/Java/src/org/antlr/v4/runtime/tree/pattern/ParseTreePatternMatcher.java b/runtime/Java/src/org/antlr/v4/runtime/tree/pattern/ParseTreePatternMatcher.java index 7c2c954..0ff53f1 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/tree/pattern/ParseTreePatternMatcher.java +++ b/runtime/Java/src/org/antlr/v4/runtime/tree/pattern/ParseTreePatternMatcher.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/runtime/Java/src/org/antlr/v4/runtime/tree/pattern/RuleTagToken.java b/runtime/Java/src/org/antlr/v4/runtime/tree/pattern/RuleTagToken.java index 1fcb0c6..5531e56 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/tree/pattern/RuleTagToken.java +++ b/runtime/Java/src/org/antlr/v4/runtime/tree/pattern/RuleTagToken.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/runtime/Java/src/org/antlr/v4/runtime/tree/pattern/TagChunk.java b/runtime/Java/src/org/antlr/v4/runtime/tree/pattern/TagChunk.java index 420c892..c55ff0f 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/tree/pattern/TagChunk.java +++ b/runtime/Java/src/org/antlr/v4/runtime/tree/pattern/TagChunk.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/runtime/Java/src/org/antlr/v4/runtime/tree/pattern/TextChunk.java b/runtime/Java/src/org/antlr/v4/runtime/tree/pattern/TextChunk.java index 0ef75a3..da55460 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/tree/pattern/TextChunk.java +++ b/runtime/Java/src/org/antlr/v4/runtime/tree/pattern/TextChunk.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/runtime/Java/src/org/antlr/v4/runtime/tree/pattern/TokenTagToken.java b/runtime/Java/src/org/antlr/v4/runtime/tree/pattern/TokenTagToken.java index 740a898..93251fa 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/tree/pattern/TokenTagToken.java +++ b/runtime/Java/src/org/antlr/v4/runtime/tree/pattern/TokenTagToken.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/runtime/Java/src/org/antlr/v4/runtime/tree/xpath/XPath.java b/runtime/Java/src/org/antlr/v4/runtime/tree/xpath/XPath.java index a4a91bb..1e4416d 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/tree/xpath/XPath.java +++ b/runtime/Java/src/org/antlr/v4/runtime/tree/xpath/XPath.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/runtime/Java/src/org/antlr/v4/runtime/tree/xpath/XPathElement.java b/runtime/Java/src/org/antlr/v4/runtime/tree/xpath/XPathElement.java index 221a729..6db5b9a 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/tree/xpath/XPathElement.java +++ b/runtime/Java/src/org/antlr/v4/runtime/tree/xpath/XPathElement.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/runtime/Java/src/org/antlr/v4/runtime/tree/xpath/XPathLexer.g4 b/runtime/Java/src/org/antlr/v4/runtime/tree/xpath/XPathLexer.g4 deleted file mode 100644 index 1598977..0000000 --- a/runtime/Java/src/org/antlr/v4/runtime/tree/xpath/XPathLexer.g4 +++ /dev/null @@ -1,63 +0,0 @@ -lexer grammar XPathLexer; - -tokens { TOKEN_REF, RULE_REF } - -/* -path : separator? word (separator word)* EOF ; - -separator - : '/' '!' - | '//' '!' - | '/' - | '//' - ; - -word: TOKEN_REF - | RULE_REF - | STRING - | '*' - ; -*/ - -ANYWHERE : '//' ; -ROOT : '/' ; -WILDCARD : '*' ; -BANG : '!' ; - -ID : NameStartChar NameChar* - { - String text = getText(); - if ( Character.isUpperCase(text.charAt(0)) ) setType(TOKEN_REF); - else setType(RULE_REF); - } - ; - -fragment -NameChar : NameStartChar - | '0'..'9' - | '_' - | '\u00B7' - | '\u0300'..'\u036F' - | '\u203F'..'\u2040' - ; - -fragment -NameStartChar - : 'A'..'Z' | 'a'..'z' - | '\u00C0'..'\u00D6' - | '\u00D8'..'\u00F6' - | '\u00F8'..'\u02FF' - | '\u0370'..'\u037D' - | '\u037F'..'\u1FFF' - | '\u200C'..'\u200D' - | '\u2070'..'\u218F' - | '\u2C00'..'\u2FEF' - | '\u3001'..'\uD7FF' - | '\uF900'..'\uFDCF' - | '\uFDF0'..'\uFFFD' - ; // ignores | ['\u10000-'\uEFFFF] ; - -STRING : '\'' .*? '\'' ; - -//WS : [ \t\r\n]+ -> skip ; - diff --git a/runtime/Java/src/org/antlr/v4/runtime/tree/xpath/XPathLexer.java b/runtime/Java/src/org/antlr/v4/runtime/tree/xpath/XPathLexer.java new file mode 100644 index 0000000..4d120a4 --- /dev/null +++ b/runtime/Java/src/org/antlr/v4/runtime/tree/xpath/XPathLexer.java @@ -0,0 +1,179 @@ +/* + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. + * Use of this file is governed by the BSD 3-clause license that + * can be found in the LICENSE.txt file in the project root. + */ +package org.antlr.v4.runtime.tree.xpath; + +import org.antlr.v4.runtime.CharStream; +import org.antlr.v4.runtime.CommonToken; +import org.antlr.v4.runtime.Lexer; +import org.antlr.v4.runtime.LexerNoViableAltException; +import org.antlr.v4.runtime.Token; +import org.antlr.v4.runtime.Vocabulary; +import org.antlr.v4.runtime.VocabularyImpl; +import org.antlr.v4.runtime.atn.ATN; +import org.antlr.v4.runtime.misc.Interval; + +/** Mimic the old XPathLexer from .g4 file */ +public class XPathLexer extends Lexer { + public static final int + TOKEN_REF=1, RULE_REF=2, ANYWHERE=3, ROOT=4, WILDCARD=5, BANG=6, ID=7, + STRING=8; + public static String[] modeNames = { + "DEFAULT_MODE" + }; + + public static final String[] ruleNames = { + "ANYWHERE", "ROOT", "WILDCARD", "BANG", "ID", "NameChar", "NameStartChar", + "STRING" + }; + + private static final String[] _LITERAL_NAMES = { + null, null, null, "'//'", "'/'", "'*'", "'!'" + }; + private static final String[] _SYMBOLIC_NAMES = { + null, "TOKEN_REF", "RULE_REF", "ANYWHERE", "ROOT", "WILDCARD", "BANG", + "ID", "STRING" + }; + public static final Vocabulary VOCABULARY = new VocabularyImpl(_LITERAL_NAMES, _SYMBOLIC_NAMES); + + /** + * @deprecated Use {@link #VOCABULARY} instead. + */ + @Deprecated + public static final String[] tokenNames; + static { + tokenNames = new String[_SYMBOLIC_NAMES.length]; + for (int i = 0; i < tokenNames.length; i++) { + tokenNames[i] = VOCABULARY.getLiteralName(i); + if (tokenNames[i] == null) { + tokenNames[i] = VOCABULARY.getSymbolicName(i); + } + + if (tokenNames[i] == null) { + tokenNames[i] = "<INVALID>"; + } + } + } + + @Override + public String getGrammarFileName() { return "XPathLexer.g4"; } + + @Override + public String[] getRuleNames() { return ruleNames; } + + @Override + public String[] getModeNames() { return modeNames; } + + @Override + @Deprecated + public String[] getTokenNames() { + return tokenNames; + } + + @Override + public Vocabulary getVocabulary() { + return VOCABULARY; + } + + @Override + public ATN getATN() { + return null; + } + + protected int line = 1; + protected int charPositionInLine = 0; + + public XPathLexer(CharStream input) { + super(input); + } + + @Override + public Token nextToken() { + _tokenStartCharIndex = _input.index(); + CommonToken t = null; + while ( t==null ) { + switch ( _input.LA(1) ) { + case '/': + consume(); + if ( _input.LA(1)=='/' ) { + consume(); + t = new CommonToken(ANYWHERE, "//"); + } + else { + t = new CommonToken(ROOT, "/"); + } + break; + case '*': + consume(); + t = new CommonToken(WILDCARD, "*"); + break; + case '!': + consume(); + t = new CommonToken(BANG, "!"); + break; + case '\'': + String s = matchString(); + t = new CommonToken(STRING, s); + break; + case CharStream.EOF : + return new CommonToken(EOF, "<EOF>"); + default: + if ( isNameStartChar(_input.LA(1)) ) { + String id = matchID(); + if ( Character.isUpperCase(id.charAt(0)) ) t = new CommonToken(TOKEN_REF, id); + else t = new CommonToken(RULE_REF, id); + } + else { + throw new LexerNoViableAltException(this, _input, _tokenStartCharIndex, null); + } + break; + } + } + t.setStartIndex(_tokenStartCharIndex); + t.setCharPositionInLine(_tokenStartCharIndex); + t.setLine(line); + return t; + } + + public void consume() { + int curChar = _input.LA(1); + if ( curChar=='\n' ) { + line++; + charPositionInLine=0; + } + else { + charPositionInLine++; + } + _input.consume(); + } + + @Override + public int getCharPositionInLine() { + return charPositionInLine; + } + + public String matchID() { + int start = _input.index(); + consume(); // drop start char + while ( isNameChar(_input.LA(1)) ) { + consume(); + } + return _input.getText(Interval.of(start,_input.index()-1)); + } + + public String matchString() { + int start = _input.index(); + consume(); // drop first quote + while ( _input.LA(1)!='\'' ) { + consume(); + } + consume(); // drop last quote + return _input.getText(Interval.of(start,_input.index()-1)); + } + + public boolean isNameChar(int c) { return Character.isUnicodeIdentifierPart(c); } + + public boolean isNameStartChar(int c) { return Character.isUnicodeIdentifierStart(c); } +} diff --git a/runtime/Java/src/org/antlr/v4/runtime/tree/xpath/XPathLexerErrorListener.java b/runtime/Java/src/org/antlr/v4/runtime/tree/xpath/XPathLexerErrorListener.java index a669850..21bcb54 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/tree/xpath/XPathLexerErrorListener.java +++ b/runtime/Java/src/org/antlr/v4/runtime/tree/xpath/XPathLexerErrorListener.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/runtime/Java/src/org/antlr/v4/runtime/tree/xpath/XPathRuleAnywhereElement.java b/runtime/Java/src/org/antlr/v4/runtime/tree/xpath/XPathRuleAnywhereElement.java index 66684ae..e5a4bfe 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/tree/xpath/XPathRuleAnywhereElement.java +++ b/runtime/Java/src/org/antlr/v4/runtime/tree/xpath/XPathRuleAnywhereElement.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/runtime/Java/src/org/antlr/v4/runtime/tree/xpath/XPathRuleElement.java b/runtime/Java/src/org/antlr/v4/runtime/tree/xpath/XPathRuleElement.java index 2827914..d966666 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/tree/xpath/XPathRuleElement.java +++ b/runtime/Java/src/org/antlr/v4/runtime/tree/xpath/XPathRuleElement.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/runtime/Java/src/org/antlr/v4/runtime/tree/xpath/XPathTokenAnywhereElement.java b/runtime/Java/src/org/antlr/v4/runtime/tree/xpath/XPathTokenAnywhereElement.java index af08ec9..41d34b0 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/tree/xpath/XPathTokenAnywhereElement.java +++ b/runtime/Java/src/org/antlr/v4/runtime/tree/xpath/XPathTokenAnywhereElement.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/runtime/Java/src/org/antlr/v4/runtime/tree/xpath/XPathTokenElement.java b/runtime/Java/src/org/antlr/v4/runtime/tree/xpath/XPathTokenElement.java index 9560af1..4a0e36d 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/tree/xpath/XPathTokenElement.java +++ b/runtime/Java/src/org/antlr/v4/runtime/tree/xpath/XPathTokenElement.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/runtime/Java/src/org/antlr/v4/runtime/tree/xpath/XPathWildcardAnywhereElement.java b/runtime/Java/src/org/antlr/v4/runtime/tree/xpath/XPathWildcardAnywhereElement.java index 43c63ed..4ab3a10 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/tree/xpath/XPathWildcardAnywhereElement.java +++ b/runtime/Java/src/org/antlr/v4/runtime/tree/xpath/XPathWildcardAnywhereElement.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/runtime/Java/src/org/antlr/v4/runtime/tree/xpath/XPathWildcardElement.java b/runtime/Java/src/org/antlr/v4/runtime/tree/xpath/XPathWildcardElement.java index e027299..0de1483 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/tree/xpath/XPathWildcardElement.java +++ b/runtime/Java/src/org/antlr/v4/runtime/tree/xpath/XPathWildcardElement.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/scripts/github_release_notes.py b/scripts/github_release_notes.py new file mode 100644 index 0000000..a0b12d2 --- /dev/null +++ b/scripts/github_release_notes.py @@ -0,0 +1,64 @@ +# Get github issues / PR for a release +# Exec with "python github_release_notes.py YOUR_GITHUB_API_ACCESS_TOKEN 4.7.1" + +from github import Github +from collections import Counter +import sys + +TARGETS = ['csharp', 'cpp', 'go', 'java', 'javascript', 'python2', 'python3', 'swift'] + +TOKEN=sys.argv[1] +MILESTONE=sys.argv[2] +g = Github(login_or_token=TOKEN) + +# Then play with your Github objects: +org = g.get_organization("antlr") +repo = org.get_repo("antlr4") +milestone = [x for x in repo.get_milestones() if x.title==MILESTONE] +milestone = milestone[0] + +issues = repo.get_issues(state="closed", milestone=milestone, sort="created", direction="desc") + +# # dump bugs fixed +# print() +# print("## Issues fixed") +# for x in issues: +# labels = [l.name for l in x.labels] +# if x.pull_request is None and not ("type:improvement" in labels or "type:feature" in labels): +# print("* [%s](%s) (%s)" % (x.title, x.html_url, ", ".join([l.name for l in x.labels]))) +# +# +# print() +# # dump improvements closed for this release (issues or pulls) +# print("## Improvements, features") +# for x in issues: +# labels = [l.name for l in x.labels] +# if ("type:improvement" in labels or "type:feature" in labels): +# print("* [%s](%s) (%s)" % (x.title, x.html_url, ", ".join(labels))) +# +# print() +# +# +# # dump PRs closed for this release by target +# print("## Pull requests grouped by target") +# for target in TARGETS: +# print() +# print(f"### {target} target") +# for x in issues: +# labels = [l.name for l in x.labels] +# if x.pull_request is not None and f"target:{target}" in labels: +# print("* [%s](%s) (%s)" % (x.title, x.html_url, ", ".join(labels))) +# + +# dump contributors +print() +print("## Contributors") +user_counts = Counter([x.user.login for x in issues]) +users = {x.user.login:x.user for x in issues} +for login,count in user_counts.most_common(10000): + name = users[login].name + logins = f" ({users[login].login})" + if name is None: + name = users[login].login + logins = "" + print(f"* {count:3d} items: [{name}]({users[login].html_url}){logins}") diff --git a/scripts/parse-extended-pictographic/ExtendedPictographic-Parsed.txt b/scripts/parse-extended-pictographic/ExtendedPictographic-Parsed.txt new file mode 100644 index 0000000..5195cfc --- /dev/null +++ b/scripts/parse-extended-pictographic/ExtendedPictographic-Parsed.txt @@ -0,0 +1,126 @@ +set.add(0x1F774, 0x1F77F); +set.add(0x2700, 0x2701); +set.add(0x2703, 0x2704); +set.add(0x270E); +set.add(0x2710, 0x2711); +set.add(0x2765, 0x2767); +set.add(0x1F030, 0x1F093); +set.add(0x1F094, 0x1F09F); +set.add(0x1F10D, 0x1F10F); +set.add(0x1F12F); +set.add(0x1F16C, 0x1F16F); +set.add(0x1F1AD, 0x1F1E5); +set.add(0x1F260, 0x1F265); +set.add(0x1F203, 0x1F20F); +set.add(0x1F23C, 0x1F23F); +set.add(0x1F249, 0x1F24F); +set.add(0x1F252, 0x1F25F); +set.add(0x1F266, 0x1F2FF); +set.add(0x1F7D5, 0x1F7FF); +set.add(0x1F000, 0x1F003); +set.add(0x1F005, 0x1F02B); +set.add(0x1F02C, 0x1F02F); +set.add(0x1F322, 0x1F323); +set.add(0x1F394, 0x1F395); +set.add(0x1F398); +set.add(0x1F39C, 0x1F39D); +set.add(0x1F3F1, 0x1F3F2); +set.add(0x1F3F6); +set.add(0x1F4FE); +set.add(0x1F53E, 0x1F548); +set.add(0x1F54F); +set.add(0x1F568, 0x1F56E); +set.add(0x1F571, 0x1F572); +set.add(0x1F57B, 0x1F586); +set.add(0x1F588, 0x1F589); +set.add(0x1F58E, 0x1F58F); +set.add(0x1F591, 0x1F594); +set.add(0x1F597, 0x1F5A3); +set.add(0x1F5A6, 0x1F5A7); +set.add(0x1F5A9, 0x1F5B0); +set.add(0x1F5B3, 0x1F5BB); +set.add(0x1F5BD, 0x1F5C1); +set.add(0x1F5C5, 0x1F5D0); +set.add(0x1F5D4, 0x1F5DB); +set.add(0x1F5DF, 0x1F5E0); +set.add(0x1F5E2); +set.add(0x1F5E4, 0x1F5E7); +set.add(0x1F5E9, 0x1F5EE); +set.add(0x1F5F0, 0x1F5F2); +set.add(0x1F5F4, 0x1F5F9); +set.add(0x2605); +set.add(0x2607, 0x260D); +set.add(0x260F, 0x2610); +set.add(0x2612); +set.add(0x2616, 0x2617); +set.add(0x2619, 0x261C); +set.add(0x261E, 0x261F); +set.add(0x2621); +set.add(0x2624, 0x2625); +set.add(0x2627, 0x2629); +set.add(0x262B, 0x262D); +set.add(0x2630, 0x2637); +set.add(0x263B, 0x2647); +set.add(0x2654, 0x265F); +set.add(0x2661, 0x2662); +set.add(0x2664); +set.add(0x2667); +set.add(0x2669, 0x267A); +set.add(0x267C, 0x267E); +set.add(0x2680, 0x2691); +set.add(0x2695); +set.add(0x2698); +set.add(0x269A); +set.add(0x269D, 0x269F); +set.add(0x26A2, 0x26A9); +set.add(0x26AC, 0x26AF); +set.add(0x26B2, 0x26BC); +set.add(0x26BF, 0x26C3); +set.add(0x26C6, 0x26C7); +set.add(0x26C9, 0x26CD); +set.add(0x26D0); +set.add(0x26D2); +set.add(0x26D5, 0x26E8); +set.add(0x26EB, 0x26EF); +set.add(0x26F6); +set.add(0x26FB, 0x26FC); +set.add(0x26FE, 0x26FF); +set.add(0x2388); +set.add(0x1FA00, 0x1FFFD); +set.add(0x1F0A0, 0x1F0AE); +set.add(0x1F0B1, 0x1F0BF); +set.add(0x1F0C1, 0x1F0CF); +set.add(0x1F0D1, 0x1F0F5); +set.add(0x1F0AF, 0x1F0B0); +set.add(0x1F0C0); +set.add(0x1F0D0); +set.add(0x1F0F6, 0x1F0FF); +set.add(0x1F80C, 0x1F80F); +set.add(0x1F848, 0x1F84F); +set.add(0x1F85A, 0x1F85F); +set.add(0x1F888, 0x1F88F); +set.add(0x1F8AE, 0x1F8FF); +set.add(0x1F900, 0x1F90B); +set.add(0x1F91F); +set.add(0x1F928, 0x1F92F); +set.add(0x1F931, 0x1F932); +set.add(0x1F94C); +set.add(0x1F95F, 0x1F96B); +set.add(0x1F992, 0x1F997); +set.add(0x1F9D0, 0x1F9E6); +set.add(0x1F90C, 0x1F90F); +set.add(0x1F93F); +set.add(0x1F94D, 0x1F94F); +set.add(0x1F96C, 0x1F97F); +set.add(0x1F998, 0x1F9BF); +set.add(0x1F9C1, 0x1F9CF); +set.add(0x1F9E7, 0x1F9FF); +set.add(0x1F6C6, 0x1F6CA); +set.add(0x1F6D3, 0x1F6D4); +set.add(0x1F6E6, 0x1F6E8); +set.add(0x1F6EA); +set.add(0x1F6F1, 0x1F6F2); +set.add(0x1F6F7, 0x1F6F8); +set.add(0x1F6D5, 0x1F6DF); +set.add(0x1F6ED, 0x1F6EF); +set.add(0x1F6F9, 0x1F6FF); diff --git a/scripts/parse-extended-pictographic/ExtendedPictographic.txt b/scripts/parse-extended-pictographic/ExtendedPictographic.txt new file mode 100644 index 0000000..a84c87e --- /dev/null +++ b/scripts/parse-extended-pictographic/ExtendedPictographic.txt @@ -0,0 +1,309 @@ +# ExtendedPictographic.txt +# Date: 2016-10-12 +# © 2016 Unicode®, Inc. +# For terms of use, see http://www.unicode.org/terms_of_use.html +# +# Extended_Pictographic (EP) is a binary code point property for characters that are pictographic +# (or otherwise similar in kind to characters with the Emoji property) +# and used to customize segmentation (UAX #29 and UAX #14) so that possible future emoji zwj sequences +# will not break grapheme clusters, words, or lines. It also includes unassigned codepoints that +# are in blocks intended for use for emoji characters, added to the Unicode 9.0 Linebreak property. +# +# For usage information, see http://unicode.org/reports/tr35/#Property_Data +# +# Format +# +# This file follows the format used for UCD files. +# Field 1 indicates that the character(s) listed in Field 0 have the value ExtendedPictographic=Yes +# All other characters have the value ExtendedPictographic=No +# +# The files contain additional comments to aid in understanding the context of the property values. +# +# 1. Blocks are included if they contain any emoji characters, even if there are no EP characters. +# +# 2. EP code points may include unassigned values. Those are based on the Line_Break assignments that +# were added in Unicode 9.0 to future-proof ZWJ linebreak behavior. +# +# 3. Comment lines are included to describe the content of the blocks for comparison. For example: +# emoji=33 : [✂✅✈-✍✏✒✔✖✝✡✨✳✴❄❇❌❎❓-❕❗❣❤➕-➗➡➰➿] +# EP=10 : [✀✁✃✄✎✐✑❥-❧] +# other=149 : [✆✇✓✕✗-✜✞-✠✢-✧✩-✲✵-❃❅❆❈-❋❍❏-❒❖❘-❢❨-➔➘-➠➢-➯➱-➾] +# otherCn=2 +# +# Meaning +# • emoji: the number of Emoji=Yes characters in that block, +# • EP: the number of ExtendedPictographic in the block (with details below), and +# • other: the number of other characters in the block (non-Emoji, non-EP), +# • otherCn: the number of other code points in the block (non-Emoji, non-EP, not characters). +# +# The characters can be listed out in detail by using any implementation that supports UnicodeSet, +# such as http://unicode.org/cldr/utility/list-unicodeset.jsp. +# The lines are omitted if empty: for example, Transport_And_Map_Symbols have no "other" characters. +# +# ===== + +# Alchemical_Symbols +# EP=12 : [🝴-🝿] +# other=116 : [🜀-🝳] + +U+1F774..U+1F77F ; ExtendedPictographic # GC=Cn +# count=12 + +# Arrows +# emoji=8 : [↔-↙↩↪] +# other=104 : [←-↓↚-↨↫-⇿] + +# CJK_Symbols_And_Punctuation +# emoji=2 : [〰〽] +# other=62 : [ -〯〱-〼〾〿] + +# Dingbats +# emoji=33 : [✂✅✈-✍✏✒✔✖✝✡✨✳✴❄❇❌❎❓-❕❗❣❤➕-➗➡➰➿] +# EP=10 : [✀✁✃✄✎✐✑❥-❧] +# other=149 : [✆✇✓✕✗-✜✞-✠✢-✧✩-✲✵-❃❅❆❈-❋❍❏-❒❖❘-❢❨-➔➘-➠➢-➯➱-➾] + +U+2700..U+2701 ; ExtendedPictographic # [✀✁] BLACK SAFETY SCISSORS .. UPPER BLADE SCISSORS +U+2703..U+2704 ; ExtendedPictographic # [✃✄] LOWER BLADE SCISSORS .. WHITE SCISSORS +U+270E ; ExtendedPictographic # [✎] LOWER RIGHT PENCIL +U+2710..U+2711 ; ExtendedPictographic # [✐✑] UPPER RIGHT PENCIL .. WHITE NIB +U+2765..U+2767 ; ExtendedPictographic # [❥-❧] ROTATED HEAVY BLACK HEART BULLET .. ROTATED FLORAL HEART BULLET +# count=10 + +# Domino_Tiles +# EP=112 : [🀰-] + +U+1F030..U+1F093 ; ExtendedPictographic # [🀰-🂓] DOMINO TILE HORIZONTAL BACK .. DOMINO TILE VERTICAL-06-06 +U+1F094..U+1F09F ; ExtendedPictographic # GC=Cn +# count=112 + +# Emoticons +# emoji=80 : [😀-🙏] + +# Enclosed_Alphanumerics +# emoji=1 : [Ⓜ] +# other=159 : [①-ⓁⓃ-⓿] + +# Enclosed_Alphanumeric_Supplement +# emoji=15 : [🅰🅱🅾🅿🆎🆑-🆚] +# EP=65 : [🄍-🄏🄯🅬-🅯🆭-] +# other=176 : [🄀-🄌🄐-🄮🄰-🅫🅲-🅽🆀-🆍🆏🆐🆛-🆬🇦-🇿] + +U+1F10D..U+1F10F ; ExtendedPictographic # GC=Cn +U+1F12F ; ExtendedPictographic # GC=Cn +U+1F16C..U+1F16F ; ExtendedPictographic # GC=Cn +U+1F1AD..U+1F1E5 ; ExtendedPictographic # GC=Cn +# count=65 + +# Enclosed_CJK_Letters_And_Months +# emoji=2 : [㊗㊙] +# other=252 : [㈀-㈞㈠-㊖㊘㊚-㋾] + +# Enclosed_Ideographic_Supplement +# emoji=15 : [🈁🈂🈚🈯🈲-🈺🉐🉑] +# EP=198 : [----] +# other=43 : [🈀🈐-🈙🈛-🈮🈰🈱🈻🉀-🉈] + +U+1F260..U+1F265 ; ExtendedPictographic # [🉠-🉥] ROUNDED SYMBOL FOR FU .. ROUNDED SYMBOL FOR CAI +U+1F203..U+1F20F ; ExtendedPictographic # GC=Cn +U+1F23C..U+1F23F ; ExtendedPictographic # GC=Cn +U+1F249..U+1F24F ; ExtendedPictographic # GC=Cn +U+1F252..U+1F25F ; ExtendedPictographic # GC=Cn +U+1F266..U+1F2FF ; ExtendedPictographic # GC=Cn +# count=198 + +# Geometric_Shapes +# emoji=8 : [▪▫▶◀◻-◾] +# other=88 : [■-▩▬-▵▷-▿◁-◺◿] + +# Geometric_Shapes_Extended +# EP=43 : [🟕-] +# other=85 : [🞀-🟔] + +U+1F7D5..U+1F7FF ; ExtendedPictographic # GC=Cn +# count=43 + +# Latin_1_Supplement +# emoji=2 : [©®] +# other=126 : [-¨ª-¯-ÿ] + +# Letterlike_Symbols +# emoji=2 : [™ℹ] +# other=78 : [℀-℡℣-ℸ℺-⅏] + +# Mahjong_Tiles +# emoji=1 : [🀄] +# EP=47 : [🀀-🀃🀅-] + +U+1F000..U+1F003 ; ExtendedPictographic # [🀀-🀃] MAHJONG TILE EAST WIND .. MAHJONG TILE NORTH WIND +U+1F005..U+1F02B ; ExtendedPictographic # [🀅-🀫] MAHJONG TILE GREEN DRAGON .. MAHJONG TILE BACK +U+1F02C..U+1F02F ; ExtendedPictographic # GC=Cn +# count=47 + +# Miscellaneous_Symbols_And_Arrows +# emoji=7 : [⬅-⬇⬛⬜⭐⭕] +# other=200 : [⬀-⬄⬈-⬚⬝-⭏⭑-⭔⭖-⭳⭶-⮕⮘-⮹⮽-⯈⯊-⯒⯬-⯯] + +# Miscellaneous_Symbols_And_Pictographs +# emoji=637 : [🌀-🌡🌤-🎓🎖🎗🎙-🎛🎞-🏰🏳-🏵🏷-📽📿-🔽🕉-🕎🕐-🕧🕯🕰🕳-🕺🖇🖊-🖍🖐🖕🖖🖤🖥🖨🖱🖲🖼🗂-🗄🗑-🗓🗜-🗞🗡🗣🗨🗯🗳🗺-🗿] +# EP=131 : [🌢🌣🎔🎕🎘🎜🎝🏱🏲🏶📾🔾-🕈🕏🕨-🕮🕱🕲🕻-🖆🖈🖉🖎🖏🖑-🖔🖗-🖣🖦🖧🖩-🖰🖳-🖻🖽-🗁🗅-🗐🗔-🗛🗟🗠🗢🗤-🗧🗩-🗮🗰-🗲🗴-🗹] + +U+1F322..U+1F323 ; ExtendedPictographic # [🌢🌣] BLACK DROPLET .. WHITE SUN +U+1F394..U+1F395 ; ExtendedPictographic # [🎔🎕] HEART WITH TIP ON THE LEFT .. BOUQUET OF FLOWERS +U+1F398 ; ExtendedPictographic # [🎘] MUSICAL KEYBOARD WITH JACKS +U+1F39C..U+1F39D ; ExtendedPictographic # [🎜🎝] BEAMED ASCENDING MUSICAL NOTES .. BEAMED DESCENDING MUSICAL NOTES +U+1F3F1..U+1F3F2 ; ExtendedPictographic # [🏱🏲] WHITE PENNANT .. BLACK PENNANT +U+1F3F6 ; ExtendedPictographic # [🏶] BLACK ROSETTE +U+1F4FE ; ExtendedPictographic # [📾] PORTABLE STEREO +U+1F53E..U+1F548 ; ExtendedPictographic # [🔾-🕈] LOWER RIGHT SHADOWED WHITE CIRCLE .. CELTIC CROSS +U+1F54F ; ExtendedPictographic # [🕏] BOWL OF HYGIEIA +U+1F568..U+1F56E ; ExtendedPictographic # [🕨-🕮] RIGHT SPEAKER .. BOOK +U+1F571..U+1F572 ; ExtendedPictographic # [🕱🕲] BLACK SKULL AND CROSSBONES .. NO PIRACY +U+1F57B..U+1F586 ; ExtendedPictographic # [🕻-🖆] LEFT HAND TELEPHONE RECEIVER .. PEN OVER STAMPED ENVELOPE +U+1F588..U+1F589 ; ExtendedPictographic # [🖈🖉] BLACK PUSHPIN .. LOWER LEFT PENCIL +U+1F58E..U+1F58F ; ExtendedPictographic # [🖎🖏] LEFT WRITING HAND .. TURNED OK HAND SIGN +U+1F591..U+1F594 ; ExtendedPictographic # [🖑-🖔] REVERSED RAISED HAND WITH FINGERS SPLAYED .. REVERSED VICTORY HAND +U+1F597..U+1F5A3 ; ExtendedPictographic # [🖗-🖣] WHITE DOWN POINTING LEFT HAND INDEX .. BLACK DOWN POINTING BACKHAND INDEX +U+1F5A6..U+1F5A7 ; ExtendedPictographic # [🖦🖧] KEYBOARD AND MOUSE .. THREE NETWORKED COMPUTERS +U+1F5A9..U+1F5B0 ; ExtendedPictographic # [🖩-🖰] POCKET CALCULATOR .. TWO BUTTON MOUSE +U+1F5B3..U+1F5BB ; ExtendedPictographic # [🖳-🖻] OLD PERSONAL COMPUTER .. DOCUMENT WITH PICTURE +U+1F5BD..U+1F5C1 ; ExtendedPictographic # [🖽-🗁] FRAME WITH TILES .. OPEN FOLDER +U+1F5C5..U+1F5D0 ; ExtendedPictographic # [🗅-🗐] EMPTY NOTE .. PAGES +U+1F5D4..U+1F5DB ; ExtendedPictographic # [🗔-🗛] DESKTOP WINDOW .. DECREASE FONT SIZE SYMBOL +U+1F5DF..U+1F5E0 ; ExtendedPictographic # [🗟🗠] PAGE WITH CIRCLED TEXT .. STOCK CHART +U+1F5E2 ; ExtendedPictographic # [🗢] LIPS +U+1F5E4..U+1F5E7 ; ExtendedPictographic # [🗤-🗧] THREE RAYS ABOVE .. THREE RAYS RIGHT +U+1F5E9..U+1F5EE ; ExtendedPictographic # [🗩-🗮] RIGHT SPEECH BUBBLE .. LEFT ANGER BUBBLE +U+1F5F0..U+1F5F2 ; ExtendedPictographic # [🗰-🗲] MOOD BUBBLE .. LIGHTNING MOOD +U+1F5F4..U+1F5F9 ; ExtendedPictographic # [🗴-🗹] BALLOT SCRIPT X .. BALLOT BOX WITH BOLD CHECK +# count=131 + +# Miscellaneous_Symbols +# emoji=80 : [☀-☄☎☑☔☕☘☝☠☢☣☦☪☮☯☸-☺♀♂♈-♓♠♣♥♦♨♻♿⚒-⚗⚙⚛⚜⚠⚡⚪⚫⚰⚱⚽⚾⛄⛅⛈⛎⛏⛑⛓⛔⛩⛪⛰-⛵⛷-⛺⛽] +# EP=177 : [★☇-☍☏☐☒☖☗☙-☜☞☟☡☤☥☧-☩☫-☭☰-☷☻-♇♔-♟♡♢♤♧♩-♺♼-♾⚀-⚑⚕⚘⚚⚝-⚟⚢-⚩⚬-⚯⚲-⚼⚿-⛃⛆⛇⛉-⛍⛐⛒⛕-⛨⛫-⛯⛶⛻⛼⛾⛿] +# other=2 : [☆☓] + +U+2605 ; ExtendedPictographic # [★] BLACK STAR +U+2607..U+260D ; ExtendedPictographic # [☇-☍] LIGHTNING .. OPPOSITION +U+260F..U+2610 ; ExtendedPictographic # [☏☐] WHITE TELEPHONE .. BALLOT BOX +U+2612 ; ExtendedPictographic # [☒] BALLOT BOX WITH X +U+2616..U+2617 ; ExtendedPictographic # [☖☗] WHITE SHOGI PIECE .. BLACK SHOGI PIECE +U+2619..U+261C ; ExtendedPictographic # [☙-☜] REVERSED ROTATED FLORAL HEART BULLET .. WHITE LEFT POINTING INDEX +U+261E..U+261F ; ExtendedPictographic # [☞☟] WHITE RIGHT POINTING INDEX .. WHITE DOWN POINTING INDEX +U+2621 ; ExtendedPictographic # [☡] CAUTION SIGN +U+2624..U+2625 ; ExtendedPictographic # [☤☥] CADUCEUS .. ANKH +U+2627..U+2629 ; ExtendedPictographic # [☧-☩] CHI RHO .. CROSS OF JERUSALEM +U+262B..U+262D ; ExtendedPictographic # [☫-☭] FARSI SYMBOL .. HAMMER AND SICKLE +U+2630..U+2637 ; ExtendedPictographic # [☰-☷] TRIGRAM FOR HEAVEN .. TRIGRAM FOR EARTH +U+263B..U+2647 ; ExtendedPictographic # [☻-♇] BLACK SMILING FACE .. PLUTO +U+2654..U+265F ; ExtendedPictographic # [♔-♟] WHITE CHESS KING .. BLACK CHESS PAWN +U+2661..U+2662 ; ExtendedPictographic # [♡♢] WHITE HEART SUIT .. WHITE DIAMOND SUIT +U+2664 ; ExtendedPictographic # [♤] WHITE SPADE SUIT +U+2667 ; ExtendedPictographic # [♧] WHITE CLUB SUIT +U+2669..U+267A ; ExtendedPictographic # [♩-♺] QUARTER NOTE .. RECYCLING SYMBOL FOR GENERIC MATERIALS +U+267C..U+267E ; ExtendedPictographic # [♼-♾] RECYCLED PAPER SYMBOL .. PERMANENT PAPER SIGN +U+2680..U+2691 ; ExtendedPictographic # [⚀-⚑] DIE FACE-1 .. BLACK FLAG +U+2695 ; ExtendedPictographic # [⚕] STAFF OF AESCULAPIUS +U+2698 ; ExtendedPictographic # [⚘] FLOWER +U+269A ; ExtendedPictographic # [⚚] STAFF OF HERMES +U+269D..U+269F ; ExtendedPictographic # [⚝-⚟] OUTLINED WHITE STAR .. THREE LINES CONVERGING LEFT +U+26A2..U+26A9 ; ExtendedPictographic # [⚢-⚩] DOUBLED FEMALE SIGN .. HORIZONTAL MALE WITH STROKE SIGN +U+26AC..U+26AF ; ExtendedPictographic # [⚬-⚯] MEDIUM SMALL WHITE CIRCLE .. UNMARRIED PARTNERSHIP SYMBOL +U+26B2..U+26BC ; ExtendedPictographic # [⚲-⚼] NEUTER .. SESQUIQUADRATE +U+26BF..U+26C3 ; ExtendedPictographic # [⚿-⛃] SQUARED KEY .. BLACK DRAUGHTS KING +U+26C6..U+26C7 ; ExtendedPictographic # [⛆⛇] RAIN .. BLACK SNOWMAN +U+26C9..U+26CD ; ExtendedPictographic # [⛉-⛍] TURNED WHITE SHOGI PIECE .. DISABLED CAR +U+26D0 ; ExtendedPictographic # [⛐] CAR SLIDING +U+26D2 ; ExtendedPictographic # [⛒] CIRCLED CROSSING LANES +U+26D5..U+26E8 ; ExtendedPictographic # [⛕-⛨] ALTERNATE ONE-WAY LEFT WAY TRAFFIC .. BLACK CROSS ON SHIELD +U+26EB..U+26EF ; ExtendedPictographic # [⛫-⛯] CASTLE .. MAP SYMBOL FOR LIGHTHOUSE +U+26F6 ; ExtendedPictographic # [⛶] SQUARE FOUR CORNERS +U+26FB..U+26FC ; ExtendedPictographic # [⛻⛼] JAPANESE BANK SYMBOL .. HEADSTONE GRAVEYARD SYMBOL +U+26FE..U+26FF ; ExtendedPictographic # [⛾⛿] CUP ON BLACK SQUARE .. WHITE FLAG WITH HORIZONTAL MIDDLE BLACK STRIPE +# count=177 + +# Miscellaneous_Technical +# emoji=18 : [⌚⌛⌨⏏⏩-⏳⏸-⏺] +# EP=1 : [⎈] +# other=237 : [⌀-⌙⌜-⌧〈-⎇⎉-⏎⏐-⏨⏴-⏷⏻-⏿] + +U+2388 ; ExtendedPictographic # [⎈] HELM SYMBOL +# count=1 + +# No_Block +# EP=1534 : [🨀-] + +U+1FA00..U+1FFFD ; ExtendedPictographic # GC=Cn +# count=1534 + +# Playing_Cards +# emoji=1 : [🃏] +# EP=96 : [🂠-] + +U+1F0A0..U+1F0AE ; ExtendedPictographic # [🂠-🂮] PLAYING CARD BACK .. PLAYING CARD KING OF SPADES +U+1F0B1..U+1F0BF ; ExtendedPictographic # [🂱-🂿] PLAYING CARD ACE OF HEARTS .. PLAYING CARD RED JOKER +U+1F0C1..U+1F0CF ; ExtendedPictographic # [🃁-🃏] PLAYING CARD ACE OF DIAMONDS .. PLAYING CARD BLACK JOKER +U+1F0D1..U+1F0F5 ; ExtendedPictographic # [🃑-🃵] PLAYING CARD ACE OF CLUBS .. PLAYING CARD TRUMP-21 +U+1F0AF..U+1F0B0 ; ExtendedPictographic # GC=Cn +U+1F0C0 ; ExtendedPictographic # GC=Cn +U+1F0D0 ; ExtendedPictographic # GC=Cn +U+1F0F6..U+1F0FF ; ExtendedPictographic # GC=Cn +# count=96 + +# General_Punctuation +# emoji=2 : [‼⁉] +# other=109 : [ -※‽-⁈⁊--] + +# Supplemental_Arrows_B +# emoji=2 : [⤴⤵] +# other=126 : [⤀-⤳⤶-⥿] + +# Supplemental_Arrows_C +# EP=108 : [-----] +# other=148 : [🠀-🠋🠐-🡇🡐-🡙🡠-🢇🢐-🢭] + +U+1F80C..U+1F80F ; ExtendedPictographic # GC=Cn +U+1F848..U+1F84F ; ExtendedPictographic # GC=Cn +U+1F85A..U+1F85F ; ExtendedPictographic # GC=Cn +U+1F888..U+1F88F ; ExtendedPictographic # GC=Cn +U+1F8AE..U+1F8FF ; ExtendedPictographic # GC=Cn +# count=108 + +# Supplemental_Symbols_And_Pictographs +# emoji=134 : [🤐-🤺🤼-🤾🥀-🥅🥇-🥌🥐-🥫🦀-🦗🧀🧐-🧦] +# EP=174 : [🤀-🤏🤟🤨-🤯🤱🤲🤿🥌-🥏🥟-🥿🦒-🦿🧁-🧿] +# other=2 : [🤻🥆] + +U+1F900..U+1F90B ; ExtendedPictographic # [🤀-🤋] CIRCLED CROSS FORMEE WITH FOUR DOTS .. DOWNWARD FACING NOTCHED HOOK WITH DOT +U+1F91F ; ExtendedPictographic # [🤟] I LOVE YOU HAND SIGN +U+1F928..U+1F92F ; ExtendedPictographic # [🤨-🤯] FACE WITH ONE EYEBROW RAISED .. SHOCKED FACE WITH EXPLODING HEAD +U+1F931..U+1F932 ; ExtendedPictographic # [🤱🤲] BREAST-FEEDING .. PALMS UP TOGETHER +U+1F94C ; ExtendedPictographic # [🥌] CURLING STONE +U+1F95F..U+1F96B ; ExtendedPictographic # [🥟-🥫] DUMPLING .. CANNED FOOD +U+1F992..U+1F997 ; ExtendedPictographic # [🦒-🦗] GIRAFFE FACE .. CRICKET +U+1F9D0..U+1F9E6 ; ExtendedPictographic # [🧐-🧦] FACE WITH MONOCLE .. SOCKS +U+1F90C..U+1F90F ; ExtendedPictographic # GC=Cn +U+1F93F ; ExtendedPictographic # GC=Cn +U+1F94D..U+1F94F ; ExtendedPictographic # GC=Cn +U+1F96C..U+1F97F ; ExtendedPictographic # GC=Cn +U+1F998..U+1F9BF ; ExtendedPictographic # GC=Cn +U+1F9C1..U+1F9CF ; ExtendedPictographic # GC=Cn +U+1F9E7..U+1F9FF ; ExtendedPictographic # GC=Cn +# count=174 + +# Transport_And_Map_Symbols +# emoji=94 : [🚀-🛅🛋-🛒🛠-🛥🛩🛫🛬🛰🛳-🛸] +# EP=36 : [🛆-🛊🛓-🛟🛦-🛨🛪-🛱🛲🛷-] + +U+1F6C6..U+1F6CA ; ExtendedPictographic # [🛆-🛊] TRIANGLE WITH ROUNDED CORNERS .. GIRLS SYMBOL +U+1F6D3..U+1F6D4 ; ExtendedPictographic # [🛓🛔] STUPA .. PAGODA +U+1F6E6..U+1F6E8 ; ExtendedPictographic # [🛦-🛨] UP-POINTING MILITARY AIRPLANE .. UP-POINTING SMALL AIRPLANE +U+1F6EA ; ExtendedPictographic # [🛪] NORTHEAST-POINTING AIRPLANE +U+1F6F1..U+1F6F2 ; ExtendedPictographic # [🛱🛲] ONCOMING FIRE ENGINE .. DIESEL LOCOMOTIVE +U+1F6F7..U+1F6F8 ; ExtendedPictographic # [🛷🛸] SLED .. FLYING SAUCER +U+1F6D5..U+1F6DF ; ExtendedPictographic # GC=Cn +U+1F6ED..U+1F6EF ; ExtendedPictographic # GC=Cn +U+1F6F9..U+1F6FF ; ExtendedPictographic # GC=Cn +# count=36 + +# total_count=2744 +# EOF diff --git a/scripts/parse-extended-pictographic/README.md b/scripts/parse-extended-pictographic/README.md new file mode 100644 index 0000000..10be275 --- /dev/null +++ b/scripts/parse-extended-pictographic/README.md @@ -0,0 +1,8 @@ +README for scripts/extended-pictographic +=========== + +This directory contains the Unicode UTS #35 `ExtendedPictographic.txt` data file, +intended to be parsed by the script `parse.py` to produce `ExtendedPictographic-Parsed.txt`. + +This produces a series of `IntervalSet` entries to be consumed by +`UnicodeDataTemplateController`. diff --git a/scripts/parse-extended-pictographic/parse.py b/scripts/parse-extended-pictographic/parse.py new file mode 100755 index 0000000..751695e --- /dev/null +++ b/scripts/parse-extended-pictographic/parse.py @@ -0,0 +1,21 @@ +from __future__ import print_function +import codecs +import re +import sys + +def main(input, output): + code_point_re = re.compile(r'^U\+([0-9a-fA-F]+)\s*;\s*ExtendedPictographic.*$') + code_point_range_re = re.compile(r'^U\+([0-9a-fA-F]+)\.\.U\+([0-9a-fA-F]+)\s*;\s*ExtendedPictographic.*$') + + for line in input: + m = code_point_re.match(line) + if m: + print('set.add(0x' + m.group(1) + ');', file=output) + else: + m = code_point_range_re.match(line) + if m: + print('set.add(0x' + m.group(1) + ', 0x' + m.group(2) + ');', file=output) + +if __name__ == '__main__': + with codecs.open(sys.argv[1], 'r', 'utf-8') as f: + main(f, sys.stdout) diff --git a/tool-testsuite/pom.xml b/tool-testsuite/pom.xml index ba6bcdf..5ad476e 100644 --- a/tool-testsuite/pom.xml +++ b/tool-testsuite/pom.xml @@ -1,5 +1,5 @@ <!-- - ~ Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + ~ Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. ~ Use of this file is governed by the BSD 3-clause license that ~ can be found in the LICENSE.txt file in the project root. --> @@ -10,7 +10,7 @@ <parent> <groupId>org.antlr</groupId> <artifactId>antlr4-master</artifactId> - <version>4.6</version> + <version>4.7.1</version> </parent> <artifactId>antlr4-tool-testsuite</artifactId> <name>ANTLR 4 Tool Tests</name> @@ -59,6 +59,8 @@ <artifactId>maven-surefire-plugin</artifactId> <version>2.12.4</version> <configuration> + <!-- SUREFIRE-951: file.encoding cannot be set via systemPropertyVariables --> + <argLine>-Dfile.encoding=UTF-8</argLine> <includes> <include>**/Test*.java</include> </includes> diff --git a/tool-testsuite/test/org/antlr/v4/test/tool/BaseJavaToolTest.java b/tool-testsuite/test/org/antlr/v4/test/tool/BaseJavaToolTest.java index d13af2d..6c2dd46 100644 --- a/tool-testsuite/test/org/antlr/v4/test/tool/BaseJavaToolTest.java +++ b/tool-testsuite/test/org/antlr/v4/test/tool/BaseJavaToolTest.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/tool-testsuite/test/org/antlr/v4/test/tool/InterpreterTreeTextProvider.java b/tool-testsuite/test/org/antlr/v4/test/tool/InterpreterTreeTextProvider.java index d973939..adaf598 100644 --- a/tool-testsuite/test/org/antlr/v4/test/tool/InterpreterTreeTextProvider.java +++ b/tool-testsuite/test/org/antlr/v4/test/tool/InterpreterTreeTextProvider.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/tool-testsuite/test/org/antlr/v4/test/tool/Java.g4 b/tool-testsuite/test/org/antlr/v4/test/tool/Java.g4 index e3e39f6..7db38ef 100644 --- a/tool-testsuite/test/org/antlr/v4/test/tool/Java.g4 +++ b/tool-testsuite/test/org/antlr/v4/test/tool/Java.g4 @@ -1140,7 +1140,7 @@ SignedInteger fragment Sign - : [+-] + : [+\-] ; fragment diff --git a/tool-testsuite/test/org/antlr/v4/test/tool/JavaUnicodeInputStream.java b/tool-testsuite/test/org/antlr/v4/test/tool/JavaUnicodeInputStream.java index cdbf9c3..0496992 100644 --- a/tool-testsuite/test/org/antlr/v4/test/tool/JavaUnicodeInputStream.java +++ b/tool-testsuite/test/org/antlr/v4/test/tool/JavaUnicodeInputStream.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/tool-testsuite/test/org/antlr/v4/test/tool/ParserInterpreterForTesting.java b/tool-testsuite/test/org/antlr/v4/test/tool/ParserInterpreterForTesting.java index 2f5ac4d..6fbee43 100644 --- a/tool-testsuite/test/org/antlr/v4/test/tool/ParserInterpreterForTesting.java +++ b/tool-testsuite/test/org/antlr/v4/test/tool/ParserInterpreterForTesting.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/tool-testsuite/test/org/antlr/v4/test/tool/TestASTStructure.java b/tool-testsuite/test/org/antlr/v4/test/tool/TestASTStructure.java index 7211efd..c1d6238 100644 --- a/tool-testsuite/test/org/antlr/v4/test/tool/TestASTStructure.java +++ b/tool-testsuite/test/org/antlr/v4/test/tool/TestASTStructure.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/tool-testsuite/test/org/antlr/v4/test/tool/TestATNConstruction.java b/tool-testsuite/test/org/antlr/v4/test/tool/TestATNConstruction.java index 5764175..0452b2b 100644 --- a/tool-testsuite/test/org/antlr/v4/test/tool/TestATNConstruction.java +++ b/tool-testsuite/test/org/antlr/v4/test/tool/TestATNConstruction.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ @@ -115,6 +115,129 @@ public class TestATNConstruction extends BaseJavaToolTest { "s4->RuleStop_A_2\n"; checkTokensRule(g, null, expecting); } + @Test public void testCharSet() throws Exception { + LexerGrammar g = new LexerGrammar( + "lexer grammar P;\n"+ + "A : [abc] ;" + ); + String expecting = + "s0->RuleStart_A_1\n" + + "RuleStart_A_1->s3\n" + + "s3-{97..99}->s4\n" + + "s4->RuleStop_A_2\n"; + checkTokensRule(g, null, expecting); + } + @Test public void testCharSetRange() throws Exception { + LexerGrammar g = new LexerGrammar( + "lexer grammar P;\n"+ + "A : [a-c] ;" + ); + String expecting = + "s0->RuleStart_A_1\n" + + "RuleStart_A_1->s3\n" + + "s3-{97..99}->s4\n" + + "s4->RuleStop_A_2\n"; + checkTokensRule(g, null, expecting); + } + @Test public void testCharSetUnicodeBMPEscape() throws Exception { + LexerGrammar g = new LexerGrammar( + "lexer grammar P;\n"+ + "A : [\\uABCD] ;" + ); + String expecting = + "s0->RuleStart_A_1\n" + + "RuleStart_A_1->s3\n" + + "s3-43981->s4\n" + + "s4->RuleStop_A_2\n"; + checkTokensRule(g, null, expecting); + } + @Test public void testCharSetUnicodeBMPEscapeRange() throws Exception { + LexerGrammar g = new LexerGrammar( + "lexer grammar P;\n"+ + "A : [a-c\\uABCD-\\uABFF] ;" + ); + String expecting = + "s0->RuleStart_A_1\n" + + "RuleStart_A_1->s3\n" + + "s3-{97..99, 43981..44031}->s4\n" + + "s4->RuleStop_A_2\n"; + checkTokensRule(g, null, expecting); + } + @Test public void testCharSetUnicodeSMPEscape() throws Exception { + LexerGrammar g = new LexerGrammar( + "lexer grammar P;\n"+ + "A : [\\u{10ABCD}] ;" + ); + String expecting = + "s0->RuleStart_A_1\n" + + "RuleStart_A_1->s3\n" + + "s3-1092557->s4\n" + + "s4->RuleStop_A_2\n"; + checkTokensRule(g, null, expecting); + } + @Test public void testCharSetUnicodeSMPEscapeRange() throws Exception { + LexerGrammar g = new LexerGrammar( + "lexer grammar P;\n"+ + "A : [a-c\\u{10ABCD}-\\u{10ABFF}] ;" + ); + String expecting = + "s0->RuleStart_A_1\n" + + "RuleStart_A_1->s3\n" + + "s3-{97..99, 1092557..1092607}->s4\n" + + "s4->RuleStop_A_2\n"; + checkTokensRule(g, null, expecting); + } + @Test public void testCharSetUnicodePropertyEscape() throws Exception { + // The Gothic script is long dead and unlikely to change (which would + // cause this test to fail) + LexerGrammar g = new LexerGrammar( + "lexer grammar P;\n"+ + "A : [\\p{Gothic}] ;" + ); + String expecting = + "s0->RuleStart_A_1\n" + + "RuleStart_A_1->s3\n" + + "s3-{66352..66378}->s4\n" + + "s4->RuleStop_A_2\n"; + checkTokensRule(g, null, expecting); + } + @Test public void testCharSetUnicodePropertyInvertEscape() throws Exception { + LexerGrammar g = new LexerGrammar( + "lexer grammar P;\n"+ + "A : [\\P{Gothic}] ;" + ); + String expecting = + "s0->RuleStart_A_1\n" + + "RuleStart_A_1->s3\n" + + "s3-{0..66351, 66379..1114111}->s4\n" + + "s4->RuleStop_A_2\n"; + checkTokensRule(g, null, expecting); + } + @Test public void testCharSetUnicodeMultiplePropertyEscape() throws Exception { + // Ditto the Mahajani script. Not going to change soon. I hope. + LexerGrammar g = new LexerGrammar( + "lexer grammar P;\n"+ + "A : [\\p{Gothic}\\p{Mahajani}] ;" + ); + String expecting = + "s0->RuleStart_A_1\n" + + "RuleStart_A_1->s3\n" + + "s3-{66352..66378, 69968..70006}->s4\n" + + "s4->RuleStop_A_2\n"; + checkTokensRule(g, null, expecting); + } + @Test public void testCharSetUnicodePropertyOverlap() throws Exception { + LexerGrammar g = new LexerGrammar( + "lexer grammar P;\n"+ + "A : [\\p{ASCII_Hex_Digit}\\p{Hex_Digit}] ;" + ); + String expecting = + "s0->RuleStart_A_1\n" + + "RuleStart_A_1->s3\n" + + "s3-{48..57, 65..70, 97..102, 65296..65305, 65313..65318, 65345..65350}->s4\n" + + "s4->RuleStop_A_2\n"; + checkTokensRule(g, null, expecting); + } @Test public void testRangeOrRange() throws Exception { LexerGrammar g = new LexerGrammar( "lexer grammar P;\n"+ diff --git a/tool-testsuite/test/org/antlr/v4/test/tool/TestATNDeserialization.java b/tool-testsuite/test/org/antlr/v4/test/tool/TestATNDeserialization.java index 5f19024..bb4b15a 100644 --- a/tool-testsuite/test/org/antlr/v4/test/tool/TestATNDeserialization.java +++ b/tool-testsuite/test/org/antlr/v4/test/tool/TestATNDeserialization.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/tool-testsuite/test/org/antlr/v4/test/tool/TestATNInterpreter.java b/tool-testsuite/test/org/antlr/v4/test/tool/TestATNInterpreter.java index 14e82ae..201463e 100644 --- a/tool-testsuite/test/org/antlr/v4/test/tool/TestATNInterpreter.java +++ b/tool-testsuite/test/org/antlr/v4/test/tool/TestATNInterpreter.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/tool-testsuite/test/org/antlr/v4/test/tool/TestATNLexerInterpreter.java b/tool-testsuite/test/org/antlr/v4/test/tool/TestATNLexerInterpreter.java index 0324d39..0d06a03 100644 --- a/tool-testsuite/test/org/antlr/v4/test/tool/TestATNLexerInterpreter.java +++ b/tool-testsuite/test/org/antlr/v4/test/tool/TestATNLexerInterpreter.java @@ -1,13 +1,13 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ package org.antlr.v4.test.tool; -import org.antlr.v4.runtime.ANTLRInputStream; import org.antlr.v4.runtime.CharStream; +import org.antlr.v4.runtime.CharStreams; import org.antlr.v4.runtime.atn.ATN; import org.antlr.v4.runtime.atn.ATNState; import org.antlr.v4.runtime.misc.Utils; @@ -121,6 +121,94 @@ public class TestATNLexerInterpreter extends BaseJavaToolTest { checkLexerMatches(lg, "c", expecting); } + @Test public void testLexerSetUnicodeBMP() throws Exception { + LexerGrammar lg = new LexerGrammar( + "lexer grammar L;\n"+ + "ID : ('\u611B'|'\u611C')\n ;"); + String expecting = "ID, EOF"; + checkLexerMatches(lg, "\u611B", expecting); + } + + @Test public void testLexerNotSetUnicodeBMP() throws Exception { + LexerGrammar lg = new LexerGrammar( + "lexer grammar L;\n"+ + "ID : ~('\u611B'|'\u611C')\n ;"); + String expecting = "ID, EOF"; + checkLexerMatches(lg, "\u611D", expecting); + } + + @Test public void testLexerNotSetUnicodeBMPMatchesSMP() throws Exception { + LexerGrammar lg = new LexerGrammar( + "lexer grammar L;\n"+ + "ID : ~('\u611B'|'\u611C')\n ;"); + String expecting = "ID, EOF"; + checkLexerMatches(lg, new StringBuilder().appendCodePoint(0x1F4A9).toString(), expecting); + } + + @Test public void testLexerSetUnicodeSMP() throws Exception { + LexerGrammar lg = new LexerGrammar( + "lexer grammar L;\n"+ + "ID : ('\\u{1F4A9}'|'\\u{1F4AA}')\n ;"); + String expecting = "ID, EOF"; + checkLexerMatches(lg, new StringBuilder().appendCodePoint(0x1F4A9).toString(), expecting); + } + + @Test public void testLexerNotBMPSetMatchesUnicodeSMP() throws Exception { + LexerGrammar lg = new LexerGrammar( + "lexer grammar L;\n"+ + "ID : ~('a'|'b')\n ;"); + String expecting = "ID, EOF"; + checkLexerMatches(lg, new StringBuilder().appendCodePoint(0x1F4A9).toString(), expecting); + } + + @Test public void testLexerNotBMPSetMatchesBMP() throws Exception { + LexerGrammar lg = new LexerGrammar( + "lexer grammar L;\n"+ + "ID : ~('a'|'b')\n ;"); + String expecting = "ID, EOF"; + checkLexerMatches(lg, "\u611B", expecting); + } + + @Test public void testLexerNotBMPSetMatchesSMP() throws Exception { + LexerGrammar lg = new LexerGrammar( + "lexer grammar L;\n"+ + "ID : ~('a'|'b')\n ;"); + String expecting = "ID, EOF"; + checkLexerMatches(lg, new StringBuilder().appendCodePoint(0x1F4A9).toString(), expecting); + } + + @Test public void testLexerNotSMPSetMatchesBMP() throws Exception { + LexerGrammar lg = new LexerGrammar( + "lexer grammar L;\n"+ + "ID : ~('\\u{1F4A9}'|'\\u{1F4AA}')\n ;"); + String expecting = "ID, EOF"; + checkLexerMatches(lg, "\u611B", expecting); + } + + @Test public void testLexerNotSMPSetMatchesSMP() throws Exception { + LexerGrammar lg = new LexerGrammar( + "lexer grammar L;\n"+ + "ID : ~('\\u{1F4A9}'|'\\u{1F4AA}')\n ;"); + String expecting = "ID, EOF"; + checkLexerMatches(lg, new StringBuilder().appendCodePoint(0x1D7C0).toString(), expecting); + } + + @Test public void testLexerRangeUnicodeSMP() throws Exception { + LexerGrammar lg = new LexerGrammar( + "lexer grammar L;\n"+ + "ID : ('\\u{1F4A9}'..'\\u{1F4B0}')\n ;"); + String expecting = "ID, EOF"; + checkLexerMatches(lg, new StringBuilder().appendCodePoint(0x1F4AF).toString(), expecting); + } + + @Test public void testLexerRangeUnicodeBMPToSMP() throws Exception { + LexerGrammar lg = new LexerGrammar( + "lexer grammar L;\n"+ + "ID : ('\\u611B'..'\\u{1F4B0}')\n ;"); + String expecting = "ID, EOF"; + checkLexerMatches(lg, new StringBuilder().appendCodePoint(0x12001).toString(), expecting); + } + @Test public void testLexerKeywordIDAmbiguity() throws Exception { LexerGrammar lg = new LexerGrammar( "lexer grammar L;\n"+ @@ -293,7 +381,7 @@ public class TestATNLexerInterpreter extends BaseJavaToolTest { protected void checkLexerMatches(LexerGrammar lg, String inputString, String expecting) { ATN atn = createATN(lg, true); - CharStream input = new ANTLRInputStream(inputString); + CharStream input = CharStreams.fromString(inputString); ATNState startState = atn.modeNameToStartState.get("DEFAULT_MODE"); DOTGenerator dot = new DOTGenerator(lg); // System.out.println(dot.getDOT(startState, true)); diff --git a/tool-testsuite/test/org/antlr/v4/test/tool/TestATNParserPrediction.java b/tool-testsuite/test/org/antlr/v4/test/tool/TestATNParserPrediction.java index 3049fd8..9e58fe9 100644 --- a/tool-testsuite/test/org/antlr/v4/test/tool/TestATNParserPrediction.java +++ b/tool-testsuite/test/org/antlr/v4/test/tool/TestATNParserPrediction.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/tool-testsuite/test/org/antlr/v4/test/tool/TestATNSerialization.java b/tool-testsuite/test/org/antlr/v4/test/tool/TestATNSerialization.java index 5e2b57e..d8afa5c 100644 --- a/tool-testsuite/test/org/antlr/v4/test/tool/TestATNSerialization.java +++ b/tool-testsuite/test/org/antlr/v4/test/tool/TestATNSerialization.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ @@ -291,6 +291,113 @@ public class TestATNSerialization extends BaseJavaToolTest { assertEquals(expecting, result); } + @Test public void testLexerUnicodeSMPLiteralSerializedToSet() throws Exception { + LexerGrammar lg = new LexerGrammar( + "lexer grammar L;\n"+ + "INT : '\\u{1F4A9}' ;"); + String expecting = + "max type 1\n" + + "0:TOKEN_START -1\n" + + "1:RULE_START 0\n" + + "2:RULE_STOP 0\n" + + "3:BASIC 0\n" + + "4:BASIC 0\n" + + "rule 0:1 1\n" + + "mode 0:0\n" + + "0:128169..128169\n" + + "0->1 EPSILON 0,0,0\n" + + "1->3 EPSILON 0,0,0\n" + + "3->4 SET 0,0,0\n" + + "4->2 EPSILON 0,0,0\n" + + "0:0\n"; + ATN atn = createATN(lg, true); + String result = ATNSerializer.getDecoded(atn, Arrays.asList(lg.getTokenNames())); + assertEquals(expecting, result); + } + + @Test public void testLexerUnicodeSMPRangeSerializedToSet() throws Exception { + LexerGrammar lg = new LexerGrammar( + "lexer grammar L;\n"+ + "INT : ('a'..'\\u{1F4A9}') ;"); + String expecting = + "max type 1\n" + + "0:TOKEN_START -1\n" + + "1:RULE_START 0\n" + + "2:RULE_STOP 0\n" + + "3:BASIC 0\n" + + "4:BASIC 0\n" + + "rule 0:1 1\n" + + "mode 0:0\n" + + "0:'a'..128169\n" + + "0->1 EPSILON 0,0,0\n" + + "1->3 EPSILON 0,0,0\n" + + "3->4 SET 0,0,0\n" + + "4->2 EPSILON 0,0,0\n" + + "0:0\n"; + ATN atn = createATN(lg, true); + String result = ATNSerializer.getDecoded(atn, Arrays.asList(lg.getTokenNames())); + assertEquals(expecting, result); + } + + @Test public void testLexerUnicodeSMPSetSerializedAfterBMPSet() throws Exception { + LexerGrammar lg = new LexerGrammar( + "lexer grammar L;\n"+ + "SMP : ('\\u{1F4A9}' | '\\u{1F4AF}') ;\n"+ + "BMP : ('a' | 'x') ;"); + String expecting = + "max type 2\n" + + "0:TOKEN_START -1\n" + + "1:RULE_START 0\n" + + "2:RULE_STOP 0\n" + + "3:RULE_START 1\n" + + "4:RULE_STOP 1\n" + + "5:BASIC 0\n" + + "6:BASIC 0\n" + + "7:BASIC 1\n" + + "8:BASIC 1\n" + + "rule 0:1 1\n" + + "rule 1:3 2\n" + + "mode 0:0\n" + + "0:'a'..'a', 'x'..'x'\n" + + "1:128169..128169, 128175..128175\n" + + "0->1 EPSILON 0,0,0\n" + + "0->3 EPSILON 0,0,0\n" + + "1->5 EPSILON 0,0,0\n" + + "3->7 EPSILON 0,0,0\n" + + "5->6 SET 1,0,0\n" + + "6->2 EPSILON 0,0,0\n" + + "7->8 SET 0,0,0\n" + + "8->4 EPSILON 0,0,0\n" + + "0:0\n"; + ATN atn = createATN(lg, true); + String result = ATNSerializer.getDecoded(atn, Arrays.asList(lg.getTokenNames())); + assertEquals(expecting, result); + } + + @Test public void testLexerNotLiteral() throws Exception { + LexerGrammar lg = new LexerGrammar( + "lexer grammar L;\n"+ + "INT : ~'a' ;"); + String expecting = + "max type 1\n" + + "0:TOKEN_START -1\n" + + "1:RULE_START 0\n" + + "2:RULE_STOP 0\n" + + "3:BASIC 0\n" + + "4:BASIC 0\n" + + "rule 0:1 1\n" + + "mode 0:0\n" + + "0:'a'..'a'\n" + + "0->1 EPSILON 0,0,0\n" + + "1->3 EPSILON 0,0,0\n" + + "3->4 NOT_SET 0,0,0\n" + + "4->2 EPSILON 0,0,0\n" + + "0:0\n"; + ATN atn = createATN(lg, true); + String result = ATNSerializer.getDecoded(atn, Arrays.asList(lg.getTokenNames())); + assertEquals(expecting, result); + } + @Test public void testLexerRange() throws Exception { LexerGrammar lg = new LexerGrammar( "lexer grammar L;\n"+ @@ -518,6 +625,222 @@ public class TestATNSerialization extends BaseJavaToolTest { assertEquals(expecting, result); } + @Test public void testLexerUnicodeUnescapedBMPNotSet() throws Exception { + LexerGrammar lg = new LexerGrammar( + "lexer grammar L;\n"+ + "ID : ~('\u4E9C'|'\u4E9D')\n ;"); + String expecting = + "max type 1\n" + + "0:TOKEN_START -1\n" + + "1:RULE_START 0\n" + + "2:RULE_STOP 0\n" + + "3:BASIC 0\n" + + "4:BASIC 0\n" + + "rule 0:1 1\n" + + "mode 0:0\n" + + "0:'\\u4E9C'..'\\u4E9D'\n" + + "0->1 EPSILON 0,0,0\n" + + "1->3 EPSILON 0,0,0\n" + + "3->4 NOT_SET 0,0,0\n" + + "4->2 EPSILON 0,0,0\n" + + "0:0\n"; + ATN atn = createATN(lg, true); + String result = ATNSerializer.getDecoded(atn, Arrays.asList(lg.getTokenNames())); + assertEquals(expecting, result); + } + + @Test public void testLexerUnicodeUnescapedBMPSetWithRange() throws Exception { + LexerGrammar lg = new LexerGrammar( + "lexer grammar L;\n"+ + "ID : ('\u4E9C'|'\u4E9D'|'\u6C5F'|'\u305F'..'\u307B')\n ;"); + String expecting = + "max type 1\n" + + "0:TOKEN_START -1\n" + + "1:RULE_START 0\n" + + "2:RULE_STOP 0\n" + + "3:BASIC 0\n" + + "4:BASIC 0\n" + + "rule 0:1 1\n" + + "mode 0:0\n" + + "0:'\\u305F'..'\\u307B', '\\u4E9C'..'\\u4E9D', '\\u6C5F'..'\\u6C5F'\n" + + "0->1 EPSILON 0,0,0\n" + + "1->3 EPSILON 0,0,0\n" + + "3->4 SET 0,0,0\n" + + "4->2 EPSILON 0,0,0\n" + + "0:0\n"; + ATN atn = createATN(lg, true); + String result = ATNSerializer.getDecoded(atn, Arrays.asList(lg.getTokenNames())); + assertEquals(expecting, result); + } + + @Test public void testLexerUnicodeUnescapedBMPNotSetWithRange() throws Exception { + LexerGrammar lg = new LexerGrammar( + "lexer grammar L;\n"+ + "ID : ~('\u4E9C'|'\u4E9D'|'\u6C5F'|'\u305F'..'\u307B')\n ;"); + String expecting = + "max type 1\n" + + "0:TOKEN_START -1\n" + + "1:RULE_START 0\n" + + "2:RULE_STOP 0\n" + + "3:BASIC 0\n" + + "4:BASIC 0\n" + + "rule 0:1 1\n" + + "mode 0:0\n" + + "0:'\\u305F'..'\\u307B', '\\u4E9C'..'\\u4E9D', '\\u6C5F'..'\\u6C5F'\n" + + "0->1 EPSILON 0,0,0\n" + + "1->3 EPSILON 0,0,0\n" + + "3->4 NOT_SET 0,0,0\n" + + "4->2 EPSILON 0,0,0\n" + + "0:0\n"; + ATN atn = createATN(lg, true); + String result = ATNSerializer.getDecoded(atn, Arrays.asList(lg.getTokenNames())); + assertEquals(expecting, result); + } + + @Test public void testLexerUnicodeEscapedBMPNotSet() throws Exception { + LexerGrammar lg = new LexerGrammar( + "lexer grammar L;\n"+ + "ID : ~('\\u4E9C'|'\\u4E9D')\n ;"); + String expecting = + "max type 1\n" + + "0:TOKEN_START -1\n" + + "1:RULE_START 0\n" + + "2:RULE_STOP 0\n" + + "3:BASIC 0\n" + + "4:BASIC 0\n" + + "rule 0:1 1\n" + + "mode 0:0\n" + + "0:'\\u4E9C'..'\\u4E9D'\n" + + "0->1 EPSILON 0,0,0\n" + + "1->3 EPSILON 0,0,0\n" + + "3->4 NOT_SET 0,0,0\n" + + "4->2 EPSILON 0,0,0\n" + + "0:0\n"; + ATN atn = createATN(lg, true); + String result = ATNSerializer.getDecoded(atn, Arrays.asList(lg.getTokenNames())); + assertEquals(expecting, result); + } + + @Test public void testLexerUnicodeEscapedBMPSetWithRange() throws Exception { + LexerGrammar lg = new LexerGrammar( + "lexer grammar L;\n"+ + "ID : ('\\u4E9C'|'\\u4E9D'|'\\u6C5F'|'\\u305F'..'\\u307B')\n ;"); + String expecting = + "max type 1\n" + + "0:TOKEN_START -1\n" + + "1:RULE_START 0\n" + + "2:RULE_STOP 0\n" + + "3:BASIC 0\n" + + "4:BASIC 0\n" + + "rule 0:1 1\n" + + "mode 0:0\n" + + "0:'\\u305F'..'\\u307B', '\\u4E9C'..'\\u4E9D', '\\u6C5F'..'\\u6C5F'\n" + + "0->1 EPSILON 0,0,0\n" + + "1->3 EPSILON 0,0,0\n" + + "3->4 SET 0,0,0\n" + + "4->2 EPSILON 0,0,0\n" + + "0:0\n"; + ATN atn = createATN(lg, true); + String result = ATNSerializer.getDecoded(atn, Arrays.asList(lg.getTokenNames())); + assertEquals(expecting, result); + } + + @Test public void testLexerUnicodeEscapedBMPNotSetWithRange() throws Exception { + LexerGrammar lg = new LexerGrammar( + "lexer grammar L;\n"+ + "ID : ~('\\u4E9C'|'\\u4E9D'|'\\u6C5F'|'\\u305F'..'\\u307B')\n ;"); + String expecting = + "max type 1\n" + + "0:TOKEN_START -1\n" + + "1:RULE_START 0\n" + + "2:RULE_STOP 0\n" + + "3:BASIC 0\n" + + "4:BASIC 0\n" + + "rule 0:1 1\n" + + "mode 0:0\n" + + "0:'\\u305F'..'\\u307B', '\\u4E9C'..'\\u4E9D', '\\u6C5F'..'\\u6C5F'\n" + + "0->1 EPSILON 0,0,0\n" + + "1->3 EPSILON 0,0,0\n" + + "3->4 NOT_SET 0,0,0\n" + + "4->2 EPSILON 0,0,0\n" + + "0:0\n"; + ATN atn = createATN(lg, true); + String result = ATNSerializer.getDecoded(atn, Arrays.asList(lg.getTokenNames())); + assertEquals(expecting, result); + } + + @Test public void testLexerUnicodeEscapedSMPNotSet() throws Exception { + LexerGrammar lg = new LexerGrammar( + "lexer grammar L;\n"+ + "ID : ~('\\u{1F4A9}'|'\\u{1F4AA}')\n ;"); + String expecting = + "max type 1\n" + + "0:TOKEN_START -1\n" + + "1:RULE_START 0\n" + + "2:RULE_STOP 0\n" + + "3:BASIC 0\n" + + "4:BASIC 0\n" + + "rule 0:1 1\n" + + "mode 0:0\n" + + "0:128169..128170\n" + + "0->1 EPSILON 0,0,0\n" + + "1->3 EPSILON 0,0,0\n" + + "3->4 NOT_SET 0,0,0\n" + + "4->2 EPSILON 0,0,0\n" + + "0:0\n"; + ATN atn = createATN(lg, true); + String result = ATNSerializer.getDecoded(atn, Arrays.asList(lg.getTokenNames())); + assertEquals(expecting, result); + } + + @Test public void testLexerUnicodeEscapedSMPSetWithRange() throws Exception { + LexerGrammar lg = new LexerGrammar( + "lexer grammar L;\n"+ + "ID : ('\\u{1F4A9}'|'\\u{1F4AA}'|'\\u{1F441}'|'\\u{1D40F}'..'\\u{1D413}')\n ;"); + String expecting = + "max type 1\n" + + "0:TOKEN_START -1\n" + + "1:RULE_START 0\n" + + "2:RULE_STOP 0\n" + + "3:BASIC 0\n" + + "4:BASIC 0\n" + + "rule 0:1 1\n" + + "mode 0:0\n" + + "0:119823..119827, 128065..128065, 128169..128170\n" + + "0->1 EPSILON 0,0,0\n" + + "1->3 EPSILON 0,0,0\n" + + "3->4 SET 0,0,0\n" + + "4->2 EPSILON 0,0,0\n" + + "0:0\n"; + ATN atn = createATN(lg, true); + String result = ATNSerializer.getDecoded(atn, Arrays.asList(lg.getTokenNames())); + assertEquals(expecting, result); + } + + @Test public void testLexerUnicodeEscapedSMPNotSetWithRange() throws Exception { + LexerGrammar lg = new LexerGrammar( + "lexer grammar L;\n"+ + "ID : ~('\\u{1F4A9}'|'\\u{1F4AA}'|'\\u{1F441}'|'\\u{1D40F}'..'\\u{1D413}')\n ;"); + String expecting = + "max type 1\n" + + "0:TOKEN_START -1\n" + + "1:RULE_START 0\n" + + "2:RULE_STOP 0\n" + + "3:BASIC 0\n" + + "4:BASIC 0\n" + + "rule 0:1 1\n" + + "mode 0:0\n" + + "0:119823..119827, 128065..128065, 128169..128170\n" + + "0->1 EPSILON 0,0,0\n" + + "1->3 EPSILON 0,0,0\n" + + "3->4 NOT_SET 0,0,0\n" + + "4->2 EPSILON 0,0,0\n" + + "0:0\n"; + ATN atn = createATN(lg, true); + String result = ATNSerializer.getDecoded(atn, Arrays.asList(lg.getTokenNames())); + assertEquals(expecting, result); + } + @Test public void testLexerWildcardWithMode() throws Exception { LexerGrammar lg = new LexerGrammar( "lexer grammar L;\n"+ diff --git a/tool-testsuite/test/org/antlr/v4/test/tool/TestActionSplitter.java b/tool-testsuite/test/org/antlr/v4/test/tool/TestActionSplitter.java index 391da8e..d376c7b 100644 --- a/tool-testsuite/test/org/antlr/v4/test/tool/TestActionSplitter.java +++ b/tool-testsuite/test/org/antlr/v4/test/tool/TestActionSplitter.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/tool-testsuite/test/org/antlr/v4/test/tool/TestActionTranslation.java b/tool-testsuite/test/org/antlr/v4/test/tool/TestActionTranslation.java index db73a81..e601cec 100644 --- a/tool-testsuite/test/org/antlr/v4/test/tool/TestActionTranslation.java +++ b/tool-testsuite/test/org/antlr/v4/test/tool/TestActionTranslation.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/tool-testsuite/test/org/antlr/v4/test/tool/TestAmbigParseTrees.java b/tool-testsuite/test/org/antlr/v4/test/tool/TestAmbigParseTrees.java index 546517e..a91ade8 100644 --- a/tool-testsuite/test/org/antlr/v4/test/tool/TestAmbigParseTrees.java +++ b/tool-testsuite/test/org/antlr/v4/test/tool/TestAmbigParseTrees.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/tool-testsuite/test/org/antlr/v4/test/tool/TestAttributeChecks.java b/tool-testsuite/test/org/antlr/v4/test/tool/TestAttributeChecks.java index 33ca19b..f36511e 100644 --- a/tool-testsuite/test/org/antlr/v4/test/tool/TestAttributeChecks.java +++ b/tool-testsuite/test/org/antlr/v4/test/tool/TestAttributeChecks.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/tool-testsuite/test/org/antlr/v4/test/tool/TestBasicSemanticErrors.java b/tool-testsuite/test/org/antlr/v4/test/tool/TestBasicSemanticErrors.java index 661506d..f6e2ad2 100644 --- a/tool-testsuite/test/org/antlr/v4/test/tool/TestBasicSemanticErrors.java +++ b/tool-testsuite/test/org/antlr/v4/test/tool/TestBasicSemanticErrors.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/tool-testsuite/test/org/antlr/v4/test/tool/TestBufferedTokenStream.java b/tool-testsuite/test/org/antlr/v4/test/tool/TestBufferedTokenStream.java index d9cbab5..b5548fa 100644 --- a/tool-testsuite/test/org/antlr/v4/test/tool/TestBufferedTokenStream.java +++ b/tool-testsuite/test/org/antlr/v4/test/tool/TestBufferedTokenStream.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/tool-testsuite/test/org/antlr/v4/test/tool/TestCodeGeneration.java b/tool-testsuite/test/org/antlr/v4/test/tool/TestCodeGeneration.java index 7574c7c..8fb6fb7 100644 --- a/tool-testsuite/test/org/antlr/v4/test/tool/TestCodeGeneration.java +++ b/tool-testsuite/test/org/antlr/v4/test/tool/TestCodeGeneration.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/tool-testsuite/test/org/antlr/v4/test/tool/TestCommonTokenStream.java b/tool-testsuite/test/org/antlr/v4/test/tool/TestCommonTokenStream.java index 2453add..6098ae5 100644 --- a/tool-testsuite/test/org/antlr/v4/test/tool/TestCommonTokenStream.java +++ b/tool-testsuite/test/org/antlr/v4/test/tool/TestCommonTokenStream.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/tool-testsuite/test/org/antlr/v4/test/tool/TestCompositeGrammars.java b/tool-testsuite/test/org/antlr/v4/test/tool/TestCompositeGrammars.java index 9b89b0e..0336878 100644 --- a/tool-testsuite/test/org/antlr/v4/test/tool/TestCompositeGrammars.java +++ b/tool-testsuite/test/org/antlr/v4/test/tool/TestCompositeGrammars.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ @@ -17,6 +17,7 @@ import org.junit.Test; import java.io.File; +import static org.antlr.v4.test.runtime.BaseRuntimeTest.writeFile; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNull; @@ -60,6 +61,181 @@ public class TestCompositeGrammars extends BaseJavaToolTest { assertEquals(0, equeue.size()); } + @Test public void testImportIntoLexerGrammar() throws Exception { + BaseRuntimeTest.mkdir(tmpdir); + + String master = + "lexer grammar M;\n" + + "import S;\n" + + "A : 'a';\n" + + "B : 'b';\n"; + writeFile(tmpdir, "M.g4", master); + + String slave = + "lexer grammar S;\n" + + "C : 'c';\n"; + writeFile(tmpdir, "S.g4", slave); + + ErrorQueue equeue = BaseRuntimeTest.antlrOnString(tmpdir, "Java", "M.g4", false, "-lib", tmpdir); + assertEquals(0, equeue.errors.size()); + } + + @Test public void testImportModesIntoLexerGrammar() throws Exception { + BaseRuntimeTest.mkdir(tmpdir); + + String master = + "lexer grammar M;\n" + + "import S;\n" + + "A : 'a' -> pushMode(X);\n" + + "B : 'b';\n"; + writeFile(tmpdir, "M.g4", master); + + String slave = + "lexer grammar S;\n" + + "D : 'd';\n" + + "mode X;\n" + + "C : 'c' -> popMode;\n"; + writeFile(tmpdir, "S.g4", slave); + + ErrorQueue equeue = BaseRuntimeTest.antlrOnString(tmpdir, "Java", "M.g4", false, "-lib", tmpdir); + assertEquals(0, equeue.errors.size()); + } + + @Test public void testImportChannelsIntoLexerGrammar() throws Exception { + BaseRuntimeTest.mkdir(tmpdir); + + String master = + "lexer grammar M;\n" + + "import S;\n" + + "channels {CH_A, CH_B}\n" + + "A : 'a' -> channel(CH_A);\n" + + "B : 'b' -> channel(CH_B);\n"; + writeFile(tmpdir, "M.g4", master); + + String slave = + "lexer grammar S;\n" + + "C : 'c';\n"; + writeFile(tmpdir, "S.g4", slave); + + ErrorQueue equeue = BaseRuntimeTest.antlrOnString(tmpdir, "Java", "M.g4", false, "-lib", tmpdir); + assertEquals(0, equeue.errors.size()); + } + + @Test public void testImportMixedChannelsIntoLexerGrammar() throws Exception { + BaseRuntimeTest.mkdir(tmpdir); + + String master = + "lexer grammar M;\n" + + "import S;\n" + + "channels {CH_A, CH_B}\n" + + "A : 'a' -> channel(CH_A);\n" + + "B : 'b' -> channel(CH_B);\n"; + writeFile(tmpdir, "M.g4", master); + + String slave = + "lexer grammar S;\n" + + "channels {CH_C}\n" + + "C : 'c' -> channel(CH_C);\n"; + writeFile(tmpdir, "S.g4", slave); + + ErrorQueue equeue = BaseRuntimeTest.antlrOnString(tmpdir, "Java", "M.g4", false, "-lib", tmpdir); + assertEquals(0, equeue.errors.size()); + } + + @Test public void testImportClashingChannelsIntoLexerGrammar() throws Exception { + BaseRuntimeTest.mkdir(tmpdir); + + String master = + "lexer grammar M;\n" + + "import S;\n" + + "channels {CH_A, CH_B, CH_C}\n" + + "A : 'a' -> channel(CH_A);\n" + + "B : 'b' -> channel(CH_B);\n" + + "C : 'C' -> channel(CH_C);\n"; + writeFile(tmpdir, "M.g4", master); + + String slave = + "lexer grammar S;\n" + + "channels {CH_C}\n" + + "C : 'c' -> channel(CH_C);\n"; + writeFile(tmpdir, "S.g4", slave); + + ErrorQueue equeue = BaseRuntimeTest.antlrOnString(tmpdir, "Java", "M.g4", false, "-lib", tmpdir); + assertEquals(0, equeue.errors.size()); + } + + @Test public void testMergeModesIntoLexerGrammar() throws Exception { + BaseRuntimeTest.mkdir(tmpdir); + + String master = + "lexer grammar M;\n" + + "import S;\n" + + "A : 'a' -> pushMode(X);\n" + + "mode X;\n" + + "B : 'b';\n"; + writeFile(tmpdir, "M.g4", master); + + String slave = + "lexer grammar S;\n" + + "D : 'd';\n" + + "mode X;\n" + + "C : 'c' -> popMode;\n"; + writeFile(tmpdir, "S.g4", slave); + + ErrorQueue equeue = BaseRuntimeTest.antlrOnString(tmpdir, "Java", "M.g4", false, "-lib", tmpdir); + assertEquals(0, equeue.errors.size()); + } + + @Test public void testEmptyModesInLexerGrammar() throws Exception { + BaseRuntimeTest.mkdir(tmpdir); + + String master = + "lexer grammar M;\n" + + "import S;\n" + + "A : 'a';\n" + + "C : 'e';\n" + + "B : 'b';\n"; + writeFile(tmpdir, "M.g4", master); + + String slave = + "lexer grammar S;\n" + + "D : 'd';\n" + + "mode X;\n" + + "C : 'c' -> popMode;\n"; + writeFile(tmpdir, "S.g4", slave); + + ErrorQueue equeue = BaseRuntimeTest.antlrOnString(tmpdir, "Java", "M.g4", false, "-lib", tmpdir); + assertEquals(0, equeue.errors.size()); + } + + @Test public void testCombinedGrammarImportsModalLexerGrammar() throws Exception { + BaseRuntimeTest.mkdir(tmpdir); + + String master = + "grammar M;\n" + + "import S;\n" + + "A : 'a';\n" + + "B : 'b';\n" + + "r : A B;\n"; + writeFile(tmpdir, "M.g4", master); + + String slave = + "lexer grammar S;\n" + + "D : 'd';\n" + + "mode X;\n" + + "C : 'c' -> popMode;\n"; + writeFile(tmpdir, "S.g4", slave); + + ErrorQueue equeue = BaseRuntimeTest.antlrOnString(tmpdir, "Java", "M.g4", false, "-lib", tmpdir); + assertEquals(1, equeue.errors.size()); + ANTLRMessage msg = equeue.errors.get(0); + assertEquals(ErrorType.MODE_NOT_IN_LEXER, msg.getErrorType()); + assertEquals("X", msg.getArgs()[0]); + assertEquals(3, msg.line); + assertEquals(5, msg.charPosition); + assertEquals("M.g4", new File(msg.fileName).getName()); + } + @Test public void testDelegatesSeeSameTokenType() throws Exception { String slaveS = "parser grammar S;\n"+ @@ -460,7 +636,7 @@ public class TestCompositeGrammars extends BaseJavaToolTest { BaseRuntimeTest.mkdir(tmpdir); writeFile(tmpdir, "Java.g4", slave); String found = execParser("NewJava.g4", master, "NewJavaParser", "NewJavaLexer", - null, null, "compilationUnit", "package Foo;", debug); + null, null, "compilationUnit", "package Foo;", debug); assertEquals(null, found); assertNull(stderrDuringParse); } @@ -488,7 +664,7 @@ public class TestCompositeGrammars extends BaseJavaToolTest { BaseRuntimeTest.mkdir(tmpdir); writeFile(tmpdir, "Java.g4", slave); String found = execParser("T.g4", master, "TParser", "TLexer", - null, null, "s", "a=b", debug); + null, null, "s", "a=b", debug); assertEquals(null, found); assertNull(stderrDuringParse); } diff --git a/tool-testsuite/test/org/antlr/v4/test/tool/TestDollarParser.java b/tool-testsuite/test/org/antlr/v4/test/tool/TestDollarParser.java index 03f0f0e..6103e44 100644 --- a/tool-testsuite/test/org/antlr/v4/test/tool/TestDollarParser.java +++ b/tool-testsuite/test/org/antlr/v4/test/tool/TestDollarParser.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/tool-testsuite/test/org/antlr/v4/test/tool/TestErrorSets.java b/tool-testsuite/test/org/antlr/v4/test/tool/TestErrorSets.java index 7071500..a0bdc18 100644 --- a/tool-testsuite/test/org/antlr/v4/test/tool/TestErrorSets.java +++ b/tool-testsuite/test/org/antlr/v4/test/tool/TestErrorSets.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/tool-testsuite/test/org/antlr/v4/test/tool/TestEscapeSequenceParsing.java b/tool-testsuite/test/org/antlr/v4/test/tool/TestEscapeSequenceParsing.java new file mode 100644 index 0000000..e4b3c37 --- /dev/null +++ b/tool-testsuite/test/org/antlr/v4/test/tool/TestEscapeSequenceParsing.java @@ -0,0 +1,137 @@ +/* + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. + * Use of this file is governed by the BSD 3-clause license that + * can be found in the LICENSE.txt file in the project root. + */ + +package org.antlr.v4.test.tool; + +import org.antlr.v4.misc.EscapeSequenceParsing; +import org.antlr.v4.runtime.misc.IntervalSet; +import org.junit.Test; + +import static org.antlr.v4.misc.EscapeSequenceParsing.Result; +import static org.junit.Assert.assertEquals; + +public class TestEscapeSequenceParsing { + @Test + public void testParseEmpty() { + assertEquals( + EscapeSequenceParsing.Result.Type.INVALID, + EscapeSequenceParsing.parseEscape("", 0).type); + } + + @Test + public void testParseJustBackslash() { + assertEquals( + EscapeSequenceParsing.Result.Type.INVALID, + EscapeSequenceParsing.parseEscape("\\", 0).type); + } + + @Test + public void testParseInvalidEscape() { + assertEquals( + EscapeSequenceParsing.Result.Type.INVALID, + EscapeSequenceParsing.parseEscape("\\z", 0).type); + } + + @Test + public void testParseNewline() { + assertEquals( + new Result(Result.Type.CODE_POINT, '\n', IntervalSet.EMPTY_SET, 0,2), + EscapeSequenceParsing.parseEscape("\\n", 0)); + } + + @Test + public void testParseTab() { + assertEquals( + new Result(Result.Type.CODE_POINT, '\t', IntervalSet.EMPTY_SET, 0,2), + EscapeSequenceParsing.parseEscape("\\t", 0)); + } + + @Test + public void testParseUnicodeTooShort() { + assertEquals( + EscapeSequenceParsing.Result.Type.INVALID, + EscapeSequenceParsing.parseEscape("\\uABC", 0).type); + } + + @Test + public void testParseUnicodeBMP() { + assertEquals( + new Result(Result.Type.CODE_POINT, 0xABCD, IntervalSet.EMPTY_SET, 0,6), + EscapeSequenceParsing.parseEscape("\\uABCD", 0)); + } + + @Test + public void testParseUnicodeSMPTooShort() { + assertEquals( + EscapeSequenceParsing.Result.Type.INVALID, + EscapeSequenceParsing.parseEscape("\\u{}", 0).type); + } + + @Test + public void testParseUnicodeSMPMissingCloseBrace() { + assertEquals( + EscapeSequenceParsing.Result.Type.INVALID, + EscapeSequenceParsing.parseEscape("\\u{12345", 0).type); + } + + @Test + public void testParseUnicodeTooBig() { + assertEquals( + EscapeSequenceParsing.Result.Type.INVALID, + EscapeSequenceParsing.parseEscape("\\u{110000}", 0).type); + } + + @Test + public void testParseUnicodeSMP() { + assertEquals( + new Result(Result.Type.CODE_POINT, 0x10ABCD, IntervalSet.EMPTY_SET, 0,10), + EscapeSequenceParsing.parseEscape("\\u{10ABCD}", 0)); + } + + @Test + public void testParseUnicodePropertyTooShort() { + assertEquals( + EscapeSequenceParsing.Result.Type.INVALID, + EscapeSequenceParsing.parseEscape("\\p{}", 0).type); + } + + @Test + public void testParseUnicodePropertyMissingCloseBrace() { + assertEquals( + EscapeSequenceParsing.Result.Type.INVALID, + EscapeSequenceParsing.parseEscape("\\p{1234", 0).type); + } + + @Test + public void testParseUnicodeProperty() { + assertEquals( + new Result(Result.Type.PROPERTY, -1, IntervalSet.of(66560, 66639), 0,11), + EscapeSequenceParsing.parseEscape("\\p{Deseret}", 0)); + } + + @Test + public void testParseUnicodePropertyInvertedTooShort() { + assertEquals( + EscapeSequenceParsing.Result.Type.INVALID, + EscapeSequenceParsing.parseEscape("\\P{}", 0).type); + } + + @Test + public void testParseUnicodePropertyInvertedMissingCloseBrace() { + assertEquals( + EscapeSequenceParsing.Result.Type.INVALID, + EscapeSequenceParsing.parseEscape("\\P{Deseret", 0).type); + } + + @Test + public void testParseUnicodePropertyInverted() { + IntervalSet expected = IntervalSet.of(0, 66559); + expected.add(66640, Character.MAX_CODE_POINT); + assertEquals( + new Result(Result.Type.PROPERTY, -1, expected, 0, 11), + EscapeSequenceParsing.parseEscape("\\P{Deseret}", 0)); + } +} diff --git a/tool-testsuite/test/org/antlr/v4/test/tool/TestFastQueue.java b/tool-testsuite/test/org/antlr/v4/test/tool/TestFastQueue.java index 926be09..3bf4f4a 100644 --- a/tool-testsuite/test/org/antlr/v4/test/tool/TestFastQueue.java +++ b/tool-testsuite/test/org/antlr/v4/test/tool/TestFastQueue.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/tool-testsuite/test/org/antlr/v4/test/tool/TestGrammarParserInterpreter.java b/tool-testsuite/test/org/antlr/v4/test/tool/TestGrammarParserInterpreter.java index b15ed09..def2642 100644 --- a/tool-testsuite/test/org/antlr/v4/test/tool/TestGrammarParserInterpreter.java +++ b/tool-testsuite/test/org/antlr/v4/test/tool/TestGrammarParserInterpreter.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/tool-testsuite/test/org/antlr/v4/test/tool/TestGraphNodes.java b/tool-testsuite/test/org/antlr/v4/test/tool/TestGraphNodes.java index fd7b781..cb2cfcb 100644 --- a/tool-testsuite/test/org/antlr/v4/test/tool/TestGraphNodes.java +++ b/tool-testsuite/test/org/antlr/v4/test/tool/TestGraphNodes.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/tool-testsuite/test/org/antlr/v4/test/tool/TestIntervalSet.java b/tool-testsuite/test/org/antlr/v4/test/tool/TestIntervalSet.java index 1d3157b..59bcba3 100644 --- a/tool-testsuite/test/org/antlr/v4/test/tool/TestIntervalSet.java +++ b/tool-testsuite/test/org/antlr/v4/test/tool/TestIntervalSet.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/tool-testsuite/test/org/antlr/v4/test/tool/TestLeftRecursionToolIssues.java b/tool-testsuite/test/org/antlr/v4/test/tool/TestLeftRecursionToolIssues.java index 1ac210c..04159db 100644 --- a/tool-testsuite/test/org/antlr/v4/test/tool/TestLeftRecursionToolIssues.java +++ b/tool-testsuite/test/org/antlr/v4/test/tool/TestLeftRecursionToolIssues.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/tool-testsuite/test/org/antlr/v4/test/tool/TestLexerActions.java b/tool-testsuite/test/org/antlr/v4/test/tool/TestLexerActions.java index 526efed..069c6d6 100644 --- a/tool-testsuite/test/org/antlr/v4/test/tool/TestLexerActions.java +++ b/tool-testsuite/test/org/antlr/v4/test/tool/TestLexerActions.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/tool-testsuite/test/org/antlr/v4/test/tool/TestLookaheadTrees.java b/tool-testsuite/test/org/antlr/v4/test/tool/TestLookaheadTrees.java index 3fdad47..e560fd5 100644 --- a/tool-testsuite/test/org/antlr/v4/test/tool/TestLookaheadTrees.java +++ b/tool-testsuite/test/org/antlr/v4/test/tool/TestLookaheadTrees.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/tool-testsuite/test/org/antlr/v4/test/tool/TestParseTreeMatcher.java b/tool-testsuite/test/org/antlr/v4/test/tool/TestParseTreeMatcher.java index a750880..c5af500 100644 --- a/tool-testsuite/test/org/antlr/v4/test/tool/TestParseTreeMatcher.java +++ b/tool-testsuite/test/org/antlr/v4/test/tool/TestParseTreeMatcher.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/tool-testsuite/test/org/antlr/v4/test/tool/TestParserExec.java b/tool-testsuite/test/org/antlr/v4/test/tool/TestParserExec.java index fc9a62f..14005c3 100644 --- a/tool-testsuite/test/org/antlr/v4/test/tool/TestParserExec.java +++ b/tool-testsuite/test/org/antlr/v4/test/tool/TestParserExec.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/tool-testsuite/test/org/antlr/v4/test/tool/TestParserInterpreter.java b/tool-testsuite/test/org/antlr/v4/test/tool/TestParserInterpreter.java index 2a8952f..b424fc0 100644 --- a/tool-testsuite/test/org/antlr/v4/test/tool/TestParserInterpreter.java +++ b/tool-testsuite/test/org/antlr/v4/test/tool/TestParserInterpreter.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/tool-testsuite/test/org/antlr/v4/test/tool/TestParserProfiler.java b/tool-testsuite/test/org/antlr/v4/test/tool/TestParserProfiler.java index e18aa75..a5cd58a 100644 --- a/tool-testsuite/test/org/antlr/v4/test/tool/TestParserProfiler.java +++ b/tool-testsuite/test/org/antlr/v4/test/tool/TestParserProfiler.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/tool-testsuite/test/org/antlr/v4/test/tool/TestPerformance.java b/tool-testsuite/test/org/antlr/v4/test/tool/TestPerformance.java index 70ee764..b5bd606 100644 --- a/tool-testsuite/test/org/antlr/v4/test/tool/TestPerformance.java +++ b/tool-testsuite/test/org/antlr/v4/test/tool/TestPerformance.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ @@ -33,6 +33,7 @@ import org.antlr.v4.runtime.atn.PredictionMode; import org.antlr.v4.runtime.dfa.DFA; import org.antlr.v4.runtime.dfa.DFAState; import org.antlr.v4.runtime.misc.Interval; +import org.antlr.v4.runtime.misc.MurmurHash; import org.antlr.v4.runtime.misc.ParseCancellationException; import org.antlr.v4.runtime.tree.ErrorNode; import org.antlr.v4.runtime.tree.ParseTree; @@ -76,9 +77,8 @@ import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicIntegerArray; import java.util.logging.Level; import java.util.logging.Logger; -import java.util.zip.CRC32; -import java.util.zip.Checksum; +import static org.antlr.v4.test.runtime.BaseRuntimeTest.writeFile; import static org.hamcrest.CoreMatchers.instanceOf; import static org.junit.Assert.assertThat; import static org.junit.Assert.assertTrue; @@ -825,7 +825,7 @@ public class TestPerformance extends BaseJavaToolTest { results.add(futureChecksum); } - Checksum checksum = new CRC32(); + MurmurHashChecksum checksum = new MurmurHashChecksum(); int currentIndex = -1; for (Future<FileParseResult> future : results) { currentIndex++; @@ -1115,14 +1115,11 @@ public class TestPerformance extends BaseJavaToolTest { assertTrue(success); } - private static void updateChecksum(Checksum checksum, int value) { - checksum.update((value) & 0xFF); - checksum.update((value >>> 8) & 0xFF); - checksum.update((value >>> 16) & 0xFF); - checksum.update((value >>> 24) & 0xFF); + private static void updateChecksum(MurmurHashChecksum checksum, int value) { + checksum.update(value); } - private static void updateChecksum(Checksum checksum, Token token) { + private static void updateChecksum(MurmurHashChecksum checksum, Token token) { if (token == null) { checksum.update(0); return; @@ -1154,7 +1151,7 @@ public class TestPerformance extends BaseJavaToolTest { @Override public FileParseResult parseFile(CharStream input, int currentPass, int thread) { - final Checksum checksum = new CRC32(); + final MurmurHashChecksum checksum = new MurmurHashChecksum(); final long startTime = System.nanoTime(); assert thread >= 0 && thread < NUMBER_OF_THREADS; @@ -1292,7 +1289,7 @@ public class TestPerformance extends BaseJavaToolTest { throw ex; } - tokens.reset(); + tokens.seek(0); if (REUSE_PARSER && parser != null) { parser.setInputStream(tokens); } else { @@ -1835,9 +1832,9 @@ public class TestPerformance extends BaseJavaToolTest { private static final int ENTER_RULE = 3; private static final int EXIT_RULE = 4; - private final Checksum checksum; + private final MurmurHashChecksum checksum; - public ChecksumParseTreeListener(Checksum checksum) { + public ChecksumParseTreeListener(MurmurHashChecksum checksum) { this.checksum = checksum; } @@ -1928,6 +1925,24 @@ public class TestPerformance extends BaseJavaToolTest { } } + private static class MurmurHashChecksum { + private int value; + private int count; + + public MurmurHashChecksum() { + this.value = MurmurHash.initialize(); + } + + public void update(int value) { + this.value = MurmurHash.update(this.value, value); + this.count++; + } + + public int getValue() { + return MurmurHash.finish(value, count); + } + } + @Test(timeout = 20000) public void testExponentialInclude() { String grammarFormat = diff --git a/tool-testsuite/test/org/antlr/v4/test/tool/TestScopeParsing.java b/tool-testsuite/test/org/antlr/v4/test/tool/TestScopeParsing.java index 6c8f589..dda7574 100644 --- a/tool-testsuite/test/org/antlr/v4/test/tool/TestScopeParsing.java +++ b/tool-testsuite/test/org/antlr/v4/test/tool/TestScopeParsing.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/tool-testsuite/test/org/antlr/v4/test/tool/TestSymbolIssues.java b/tool-testsuite/test/org/antlr/v4/test/tool/TestSymbolIssues.java index 6d56775..ad53c03 100644 --- a/tool-testsuite/test/org/antlr/v4/test/tool/TestSymbolIssues.java +++ b/tool-testsuite/test/org/antlr/v4/test/tool/TestSymbolIssues.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ @@ -291,21 +291,36 @@ public class TestSymbolIssues extends BaseJavaToolTest { String[] test = { "grammar L;\n" + "\n" + - "rule1 // Correct (Alternatives)\n" + - " : t1 = a #aLabel\n" + - " | t1 = b #bLabel\n" + + "rule1 // Correct (Alternatives)\n" + + " : t1=a #aLabel\n" + + " | t1=b #bLabel\n" + " ;\n" + "rule2 //Incorrect type casting in generated code (RULE_LABEL)\n" + - " : t2 = a | t2 = b\n" + + " : t2=a | t2=b\n" + " ;\n" + "rule3\n" + - " : t3 += a+ b t3 += c+ //Incorrect type casting in generated code (RULE_LIST_LABEL)\n" + + " : t3+=a+ b t3+=c+ //Incorrect type casting in generated code (RULE_LIST_LABEL)\n" + " ;\n" + "rule4\n" + - " : a t4 = A b t4 = B c // Correct (TOKEN_LABEL)\n" + + " : a t4=A b t4=B c // Correct (TOKEN_LABEL)\n" + " ;\n" + "rule5\n" + - " : a t5 += A b t5 += B c // Correct (TOKEN_LIST_LABEL)\n" + + " : a t5+=A b t5+=B c // Correct (TOKEN_LIST_LABEL)\n" + + " ;\n" + + "rule6 // Correct (https://github.com/antlr/antlr4/issues/1543)\n" + + " : t6=a #t6_1_Label\n" + + " | t6=rule6 b (t61=c)? t62=rule6 #t6_2_Label\n" + + " | t6=A a (t61=B)? t62=A #t6_3_Label\n" + + " ;\n" + + "rule7 // Incorrect (https://github.com/antlr/antlr4/issues/1543)\n" + + " : a\n" + + " | t7=rule7 b (t71=c)? t72=rule7 \n" + + " | t7=A a (t71=B)? t72=A \n" + + " ;\n" + + "rule8 // Correct (https://github.com/antlr/antlr4/issues/1543)\n" + + " : a\n" + + " | t8=rule8 a t8=rule8\n" + + " | t8=rule8 b t8=rule8\n" + " ;\n" + "a: A;\n" + "b: B;\n" + @@ -314,8 +329,54 @@ public class TestSymbolIssues extends BaseJavaToolTest { "B: 'b';\n" + "C: 'c';\n", - "error(" + ErrorType.LABEL_TYPE_CONFLICT.code + "): L.g4:8:15: label t2=b type mismatch with previous definition: t2=a\n" + - "error(" + ErrorType.LABEL_TYPE_CONFLICT.code + "): L.g4:11:17: label t3+=c type mismatch with previous definition: t3+=a\n" + "error(" + ErrorType.LABEL_TYPE_CONFLICT.code + "): L.g4:8:13: label t2=b type mismatch with previous definition: t2=a\n" + + "error(" + ErrorType.LABEL_TYPE_CONFLICT.code + "): L.g4:11:15: label t3+=c type mismatch with previous definition: t3+=a\n" + + + "error(" + ErrorType.LABEL_TYPE_CONFLICT.code + "): L.g4:24:0: label t7 type mismatch with previous definition: TOKEN_LABEL!=RULE_LABEL\n" + + "error(" + ErrorType.LABEL_TYPE_CONFLICT.code + "): L.g4:24:0: label t71 type mismatch with previous definition: RULE_LABEL!=TOKEN_LABEL\n" + + "error(" + ErrorType.LABEL_TYPE_CONFLICT.code + "): L.g4:24:0: label t72 type mismatch with previous definition: RULE_LABEL!=TOKEN_LABEL\n" + }; + + testErrors(test, false); + } + + // https://github.com/antlr/antlr4/issues/1543 + @Test public void testLabelsForTokensWithMixedTypesLRWithLabels() { + String[] test = { + "grammar L;\n" + + "\n" + + "expr\n" + + " : left=A '+' right=A #primary\n" + + " | left=expr '-' right=expr #sub\n" + + " ;\n" + + "\n" + + "A: 'a';\n" + + "B: 'b';\n" + + "C: 'c';\n", + + "" + }; + + testErrors(test, false); + } + + // https://github.com/antlr/antlr4/issues/1543 + @Test + public void testLabelsForTokensWithMixedTypesLRWithoutLabels() { + String[] test = { + "grammar L;\n" + + "\n" + + "expr\n" + + " : left=A '+' right=A\n" + + " | left=expr '-' right=expr\n" + + " ;\n" + + "\n" + + "A: 'a';\n" + + "B: 'b';\n" + + "C: 'c';\n", + + "error(" + ErrorType.LABEL_TYPE_CONFLICT.code + "): L.g4:3:0: label left type mismatch with previous definition: TOKEN_LABEL!=RULE_LABEL\n" + + "error(" + ErrorType.LABEL_TYPE_CONFLICT.code + "): L.g4:3:0: label right type mismatch with previous definition: RULE_LABEL!=TOKEN_LABEL\n" }; testErrors(test, false); @@ -327,12 +388,50 @@ public class TestSymbolIssues extends BaseJavaToolTest { "TOKEN_RANGE: [aa-f];\n" + "TOKEN_RANGE_2: [A-FD-J];\n" + "TOKEN_RANGE_3: 'Z' | 'K'..'R' | 'O'..'V';\n" + - "TOKEN_RANGE_4: 'g'..'l' | [g-l];\n", // Handling in ATNOptimizer. + "TOKEN_RANGE_4: 'g'..'l' | [g-l];\n" + + "TOKEN_RANGE_WITHOUT_COLLISION: '_' | [a-zA-Z];\n" + + "TOKEN_RANGE_WITH_ESCAPED_CHARS: [\\n-\\r] | '\\n'..'\\r';", + + "warning(" + ErrorType.CHARACTERS_COLLISION_IN_SET.code + "): L.g4:2:18: chars 'a'..'f' used multiple times in set [aa-f]\n" + + "warning(" + ErrorType.CHARACTERS_COLLISION_IN_SET.code + "): L.g4:3:18: chars 'D'..'J' used multiple times in set [A-FD-J]\n" + + "warning(" + ErrorType.CHARACTERS_COLLISION_IN_SET.code + "): L.g4:4:13: chars 'O'..'V' used multiple times in set 'Z' | 'K'..'R' | 'O'..'V'\n" + + "warning(" + ErrorType.CHARACTERS_COLLISION_IN_SET.code + "): L.g4::: chars 'g' used multiple times in set 'g'..'l'\n" + + "warning(" + ErrorType.CHARACTERS_COLLISION_IN_SET.code + "): L.g4::: chars '\\n' used multiple times in set '\\n'..'\\r'\n" + }; - "warning(" + ErrorType.CHARACTERS_COLLISION_IN_SET.code + "): L.g4:2:18: chars \"a-f\" used multiple times in set [aa-f]\n" + - "warning(" + ErrorType.CHARACTERS_COLLISION_IN_SET.code + "): L.g4:3:18: chars \"D-J\" used multiple times in set [A-FD-J]\n" + - "warning(" + ErrorType.CHARACTERS_COLLISION_IN_SET.code + "): L.g4:4:13: chars \"O-V\" used multiple times in set 'Z' | 'K'..'R' | 'O'..'V'\n" + - "warning(" + ErrorType.CHARACTERS_COLLISION_IN_SET.code + "): L.g4::: chars \"g-l\" used multiple times in set [g-l]\n" + testErrors(test, false); + } + + @Test public void testUnreachableTokens() { + String[] test = { + "lexer grammar Test;\n" + + "TOKEN1: 'as' 'df' | 'qwer';\n" + + "TOKEN2: [0-9];\n" + + "TOKEN3: 'asdf';\n" + + "TOKEN4: 'q' 'w' 'e' 'r' | A;\n" + + "TOKEN5: 'aaaa';\n" + + "TOKEN6: 'asdf';\n" + + "TOKEN7: 'qwer'+;\n" + + "TOKEN8: 'a' 'b' | 'b' | 'a' 'b';\n" + + "fragment\n" + + "TOKEN9: 'asdf' | 'qwer' | 'qwer';\n" + + "TOKEN10: '\\r\\n' | '\\r\\n';\n" + + "TOKEN11: '\\r\\n';\n" + + "\n" + + "mode MODE1;\n" + + "TOKEN12: 'asdf';\n" + + "\n" + + "fragment A: 'A';", + + "warning(" + ErrorType.TOKEN_UNREACHABLE.code + "): Test.g4:4:0: One of the token TOKEN3 values unreachable. asdf is always overlapped by token TOKEN1\n" + + "warning(" + ErrorType.TOKEN_UNREACHABLE.code + "): Test.g4:5:0: One of the token TOKEN4 values unreachable. qwer is always overlapped by token TOKEN1\n" + + "warning(" + ErrorType.TOKEN_UNREACHABLE.code + "): Test.g4:7:0: One of the token TOKEN6 values unreachable. asdf is always overlapped by token TOKEN1\n" + + "warning(" + ErrorType.TOKEN_UNREACHABLE.code + "): Test.g4:7:0: One of the token TOKEN6 values unreachable. asdf is always overlapped by token TOKEN3\n" + + "warning(" + ErrorType.TOKEN_UNREACHABLE.code + "): Test.g4:9:0: One of the token TOKEN8 values unreachable. ab is always overlapped by token TOKEN8\n" + + "warning(" + ErrorType.TOKEN_UNREACHABLE.code + "): Test.g4:11:0: One of the token TOKEN9 values unreachable. qwer is always overlapped by token TOKEN9\n" + + "warning(" + ErrorType.TOKEN_UNREACHABLE.code + "): Test.g4:12:0: One of the token TOKEN10 values unreachable. \\r\\n is always overlapped by token TOKEN10\n" + + "warning(" + ErrorType.TOKEN_UNREACHABLE.code + "): Test.g4:13:0: One of the token TOKEN11 values unreachable. \\r\\n is always overlapped by token TOKEN10\n" + + "warning(" + ErrorType.TOKEN_UNREACHABLE.code + "): Test.g4:13:0: One of the token TOKEN11 values unreachable. \\r\\n is always overlapped by token TOKEN10\n" }; testErrors(test, false); diff --git a/tool-testsuite/test/org/antlr/v4/test/tool/TestTokenPositionOptions.java b/tool-testsuite/test/org/antlr/v4/test/tool/TestTokenPositionOptions.java index c6af3ee..a7527d7 100644 --- a/tool-testsuite/test/org/antlr/v4/test/tool/TestTokenPositionOptions.java +++ b/tool-testsuite/test/org/antlr/v4/test/tool/TestTokenPositionOptions.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/tool-testsuite/test/org/antlr/v4/test/tool/TestTokenTypeAssignment.java b/tool-testsuite/test/org/antlr/v4/test/tool/TestTokenTypeAssignment.java index c338eec..4c8d7fe 100644 --- a/tool-testsuite/test/org/antlr/v4/test/tool/TestTokenTypeAssignment.java +++ b/tool-testsuite/test/org/antlr/v4/test/tool/TestTokenTypeAssignment.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ @@ -141,6 +141,24 @@ public class TestTokenTypeAssignment extends BaseJavaToolTest { assertEquals("'\\n'", literals.toArray()[0]); } + @Test public void testParserCharLiteralWithBasicUnicodeEscape() throws Exception { + Grammar g = new Grammar( + "grammar t;\n"+ + "a : '\\uABCD';\n"); + Set<?> literals = g.stringLiteralToTypeMap.keySet(); + // must store literals how they appear in the antlr grammar + assertEquals("'\\uABCD'", literals.toArray()[0]); + } + + @Test public void testParserCharLiteralWithExtendedUnicodeEscape() throws Exception { + Grammar g = new Grammar( + "grammar t;\n"+ + "a : '\\u{1ABCD}';\n"); + Set<?> literals = g.stringLiteralToTypeMap.keySet(); + // must store literals how they appear in the antlr grammar + assertEquals("'\\u{1ABCD}'", literals.toArray()[0]); + } + protected void checkSymbols(Grammar g, String rulesStr, String allValidTokensStr) diff --git a/tool-testsuite/test/org/antlr/v4/test/tool/TestToolSyntaxErrors.java b/tool-testsuite/test/org/antlr/v4/test/tool/TestToolSyntaxErrors.java index 639c638..a8bb864 100644 --- a/tool-testsuite/test/org/antlr/v4/test/tool/TestToolSyntaxErrors.java +++ b/tool-testsuite/test/org/antlr/v4/test/tool/TestToolSyntaxErrors.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ @@ -8,6 +8,7 @@ package org.antlr.v4.test.tool; import org.antlr.v4.Tool; import org.antlr.v4.tool.ErrorType; +import org.junit.Assert; import org.junit.Before; import org.junit.Test; @@ -60,6 +61,16 @@ public class TestToolSyntaxErrors extends BaseJavaToolTest { super.testSetUp(); } + @Test + public void AllErrorCodesDistinct() { + ErrorType[] errorTypes = ErrorType.class.getEnumConstants(); + for (int i = 0; i < errorTypes.length; i++) { + for (int j = i + 1; j < errorTypes.length; j++) { + Assert.assertNotEquals(errorTypes[i].code, errorTypes[j].code); + } + } + } + @Test public void testA() { super.testErrors(A, true); } @Test public void testExtraColon() { @@ -258,11 +269,11 @@ public class TestToolSyntaxErrors extends BaseJavaToolTest { "grammar A;\n" + "tokens{Foo}\n" + "b : Foo ;\n" + - "X : 'foo' -> popmode;\n" + // "meant" to use -> popMode - "Y : 'foo' -> token(Foo);", // "meant" to use -> type(Foo) + "X : 'foo1' -> popmode;\n" + // "meant" to use -> popMode + "Y : 'foo2' -> token(Foo);", // "meant" to use -> type(Foo) - "error(" + ErrorType.INVALID_LEXER_COMMAND.code + "): A.g4:4:13: lexer command popmode does not exist or is not supported by the current target\n" + - "error(" + ErrorType.INVALID_LEXER_COMMAND.code + "): A.g4:5:13: lexer command token does not exist or is not supported by the current target\n" + "error(" + ErrorType.INVALID_LEXER_COMMAND.code + "): A.g4:4:14: lexer command popmode does not exist or is not supported by the current target\n" + + "error(" + ErrorType.INVALID_LEXER_COMMAND.code + "): A.g4:5:14: lexer command token does not exist or is not supported by the current target\n" }; super.testErrors(pair, true); } @@ -272,11 +283,11 @@ public class TestToolSyntaxErrors extends BaseJavaToolTest { "grammar A;\n" + "tokens{Foo}\n" + "b : Foo ;\n" + - "X : 'foo' -> popMode(Foo);\n" + // "meant" to use -> popMode - "Y : 'foo' -> type;", // "meant" to use -> type(Foo) + "X : 'foo1' -> popMode(Foo);\n" + // "meant" to use -> popMode + "Y : 'foo2' -> type;", // "meant" to use -> type(Foo) - "error(" + ErrorType.UNWANTED_LEXER_COMMAND_ARGUMENT.code + "): A.g4:4:13: lexer command popMode does not take any arguments\n" + - "error(" + ErrorType.MISSING_LEXER_COMMAND_ARGUMENT.code + "): A.g4:5:13: missing argument for lexer command type\n" + "error(" + ErrorType.UNWANTED_LEXER_COMMAND_ARGUMENT.code + "): A.g4:4:14: lexer command popMode does not take any arguments\n" + + "error(" + ErrorType.MISSING_LEXER_COMMAND_ARGUMENT.code + "): A.g4:5:14: missing argument for lexer command type\n" }; super.testErrors(pair, true); } @@ -433,13 +444,33 @@ public class TestToolSyntaxErrors extends BaseJavaToolTest { } /** + * This is a regression test for https://github.com/antlr/antlr4/issues/1815 + * "Null ptr exception in SqlBase.g4" + */ + @Test public void testDoubleQuoteInTwoStringLiterals() { + String grammar = + "lexer grammar A;\n" + + "STRING : '\\\"' '\\\"' 'x' ;"; + String expected = + "warning(" + ErrorType.INVALID_ESCAPE_SEQUENCE.code + "): A.g4:2:10: invalid escape sequence \\\"\n"+ + "warning(" + ErrorType.INVALID_ESCAPE_SEQUENCE.code + "): A.g4:2:15: invalid escape sequence \\\"\n"; + + String[] pair = new String[] { + grammar, + expected + }; + + super.testErrors(pair, true); + } + + /** * This test ensures that the {@link ErrorType#INVALID_ESCAPE_SEQUENCE} * error is not reported for escape sequences that are known to be valid. */ @Test public void testValidEscapeSequences() { String grammar = "lexer grammar A;\n" + - "NORMAL_ESCAPE : '\\b \\t \\n \\f \\r \\\" \\' \\\\';\n" + + "NORMAL_ESCAPE : '\\b \\t \\n \\f \\r \\' \\\\';\n" + "UNICODE_ESCAPE : '\\u0001 \\u00A1 \\u00a1 \\uaaaa \\uAAAA';\n"; String expected = ""; @@ -462,9 +493,10 @@ public class TestToolSyntaxErrors extends BaseJavaToolTest { "lexer grammar A;\n" + "RULE : 'Foo \\uAABG \\x \\u';\n"; String expected = - "error(" + ErrorType.INVALID_ESCAPE_SEQUENCE.code + "): A.g4:2:12: invalid escape sequence\n" + - "error(" + ErrorType.INVALID_ESCAPE_SEQUENCE.code + "): A.g4:2:19: invalid escape sequence\n" + - "error(" + ErrorType.INVALID_ESCAPE_SEQUENCE.code + "): A.g4:2:22: invalid escape sequence\n"; + "warning(" + ErrorType.INVALID_ESCAPE_SEQUENCE.code + "): A.g4:2:12: invalid escape sequence \\uAABG\n" + + "warning(" + ErrorType.INVALID_ESCAPE_SEQUENCE.code + "): A.g4:2:19: invalid escape sequence \\x\n" + + "warning(" + ErrorType.INVALID_ESCAPE_SEQUENCE.code + "): A.g4:2:22: invalid escape sequence \\u\n" + + "warning("+ErrorType.EPSILON_TOKEN.code+"): A.g4:2:0: non-fragment lexer rule RULE can match the empty string\n"; String[] pair = new String[] { grammar, @@ -501,25 +533,70 @@ public class TestToolSyntaxErrors extends BaseJavaToolTest { super.testErrors(pair, true); } - @Test public void testInvalidCharSetAndRange() { + @Test public void testInvalidCharSetsAndStringLiterals() { String grammar = "lexer grammar Test;\n" + - "INVALID_RANGE: 'GH'..'LM';\n" + - "INVALID_RANGE_2: 'F'..'A' | 'Z';\n" + - "VALID_STRING_LITERALS: '\\u1234' | '\\t' | [\\-\\]];\n" + - "INVALID_CHAR_SET: [f-az][];\n" + - "INVALID_CHAR_SET_2: [\\u24\\uA2][\\u24];\n" + //https://github.com/antlr/antlr4/issues/1077 - "INVALID_CHAR_SET_3: [\\t\\{];"; + "INVALID_STRING_LITERAL: '\\\"' | '\\]' | '\\u24';\n" + + "INVALID_STRING_LITERAL_RANGE: 'GH'..'LM';\n" + + "INVALID_CHAR_SET: [\\u24\\uA2][\\{];\n" + //https://github.com/antlr/antlr4/issues/1077 + "EMPTY_STRING_LITERAL_RANGE: 'F'..'A' | 'Z';\n" + + "EMPTY_CHAR_SET: [f-az][];\n" + + "START_HYPHEN_IN_CHAR_SET: [-z];\n" + + "END_HYPHEN_IN_CHAR_SET: [a-];\n" + + "SINGLE_HYPHEN_IN_CHAR_SET: [-];\n" + + "VALID_STRING_LITERALS: '\\u1234' | '\\t' | '\\'';\n" + + "VALID_CHAR_SET: [`\\-=\\]];"; String expected = - "error(" + ErrorType.INVALID_LITERAL_IN_LEXER_SET.code + "): Test.g4:2:23: multi-character literals are not allowed in lexer sets: 'GH'\n" + - "error(" + ErrorType.INVALID_LITERAL_IN_LEXER_SET.code + "): Test.g4:2:29: multi-character literals are not allowed in lexer sets: 'LM'\n" + - "error(" + ErrorType.EMPTY_STRINGS_AND_SETS_NOT_ALLOWED.code + "): Test.g4:3:26: string literals and sets cannot be empty: 'F'..'A'\n" + - "error(" + ErrorType.EMPTY_STRINGS_AND_SETS_NOT_ALLOWED.code + "): Test.g4:5:23: string literals and sets cannot be empty: [f-a]\n" + - "error(" + ErrorType.EMPTY_STRINGS_AND_SETS_NOT_ALLOWED.code + "): Test.g4:5:29: string literals and sets cannot be empty: []\n" + - "error(" + ErrorType.INVALID_ESCAPE_SEQUENCE.code + "): Test.g4:6:23: invalid escape sequence\n" + - "error(" + ErrorType.INVALID_ESCAPE_SEQUENCE.code + "): Test.g4:6:33: invalid escape sequence\n" + - "error(" + ErrorType.INVALID_ESCAPE_SEQUENCE.code + "): Test.g4:7:23: invalid escape sequence\n"; + "warning(" + ErrorType.INVALID_ESCAPE_SEQUENCE.code + "): Test.g4:2:31: invalid escape sequence \\\"\n" + + "warning(" + ErrorType.INVALID_ESCAPE_SEQUENCE.code + "): Test.g4:2:38: invalid escape sequence \\]\n" + + "warning(" + ErrorType.INVALID_ESCAPE_SEQUENCE.code + "): Test.g4:2:45: invalid escape sequence \\u24\n" + + "error(" + ErrorType.INVALID_LITERAL_IN_LEXER_SET.code + "): Test.g4:3:30: multi-character literals are not allowed in lexer sets: 'GH'\n" + + "error(" + ErrorType.INVALID_LITERAL_IN_LEXER_SET.code + "): Test.g4:3:36: multi-character literals are not allowed in lexer sets: 'LM'\n" + + "warning(" + ErrorType.INVALID_ESCAPE_SEQUENCE.code + "): Test.g4:4:30: invalid escape sequence \\u24\\u\n" + + "warning(" + ErrorType.INVALID_ESCAPE_SEQUENCE.code + "): Test.g4:4:40: invalid escape sequence \\{\n" + + "error(" + ErrorType.EMPTY_STRINGS_AND_SETS_NOT_ALLOWED.code + "): Test.g4:5:33: string literals and sets cannot be empty: 'F'..'A'\n" + + "error(" + ErrorType.EMPTY_STRINGS_AND_SETS_NOT_ALLOWED.code + "): Test.g4:6:30: string literals and sets cannot be empty: 'f'..'a'\n" + + "error(" + ErrorType.EMPTY_STRINGS_AND_SETS_NOT_ALLOWED.code + "): Test.g4:6:36: string literals and sets cannot be empty: []\n" + + "warning("+ ErrorType.EPSILON_TOKEN.code + "): Test.g4:2:0: non-fragment lexer rule INVALID_STRING_LITERAL can match the empty string\n"; + + String[] pair = new String[] { + grammar, + expected + }; + + super.testErrors(pair, true); + } + + @Test public void testInvalidUnicodeEscapesInCharSet() { + String grammar = + "lexer grammar Test;\n" + + "INVALID_EXTENDED_UNICODE_EMPTY: [\\u{}];\n" + + "INVALID_EXTENDED_UNICODE_NOT_TERMINATED: [\\u{];\n" + + "INVALID_EXTENDED_UNICODE_TOO_LONG: [\\u{110000}];\n" + + "INVALID_UNICODE_PROPERTY_EMPTY: [\\p{}];\n" + + "INVALID_UNICODE_PROPERTY_NOT_TERMINATED: [\\p{];\n" + + "INVALID_INVERTED_UNICODE_PROPERTY_EMPTY: [\\P{}];\n" + + "INVALID_UNICODE_PROPERTY_UNKNOWN: [\\p{NotAProperty}];\n" + + "INVALID_INVERTED_UNICODE_PROPERTY_UNKNOWN: [\\P{NotAProperty}];\n" + + "UNICODE_PROPERTY_NOT_ALLOWED_IN_RANGE: [\\p{Uppercase_Letter}-\\p{Lowercase_Letter}];\n" + + "UNICODE_PROPERTY_NOT_ALLOWED_IN_RANGE_2: [\\p{Letter}-Z];\n" + + "UNICODE_PROPERTY_NOT_ALLOWED_IN_RANGE_3: [A-\\p{Number}];\n" + + "INVERTED_UNICODE_PROPERTY_NOT_ALLOWED_IN_RANGE: [\\P{Uppercase_Letter}-\\P{Number}];\n"; + + String expected = + "warning(" + ErrorType.INVALID_ESCAPE_SEQUENCE.code + "): Test.g4:2:32: invalid escape sequence \\u{}\n"+ + "warning(" + ErrorType.INVALID_ESCAPE_SEQUENCE.code + "): Test.g4:3:41: invalid escape sequence \\u{\n"+ + "warning(" + ErrorType.INVALID_ESCAPE_SEQUENCE.code + "): Test.g4:4:35: invalid escape sequence \\u{110\n"+ + "warning(" + ErrorType.INVALID_ESCAPE_SEQUENCE.code + "): Test.g4:5:32: invalid escape sequence \\p{}\n"+ + "warning(" + ErrorType.INVALID_ESCAPE_SEQUENCE.code + "): Test.g4:6:41: invalid escape sequence \\p{\n"+ + "warning(" + ErrorType.INVALID_ESCAPE_SEQUENCE.code + "): Test.g4:7:41: invalid escape sequence \\P{}\n"+ + "warning(" + ErrorType.INVALID_ESCAPE_SEQUENCE.code + "): Test.g4:8:34: invalid escape sequence \\p{NotAProperty}\n"+ + "warning(" + ErrorType.INVALID_ESCAPE_SEQUENCE.code + "): Test.g4:9:43: invalid escape sequence \\P{NotAProperty}\n"+ + "error(" + ErrorType.UNICODE_PROPERTY_NOT_ALLOWED_IN_RANGE.code + "): Test.g4:10:39: unicode property escapes not allowed in lexer charset range: [\\p{Uppercase_Letter}-\\p{Lowercase_Letter}]\n" + + "error(" + ErrorType.UNICODE_PROPERTY_NOT_ALLOWED_IN_RANGE.code + "): Test.g4:11:41: unicode property escapes not allowed in lexer charset range: [\\p{Letter}-Z]\n" + + "error(" + ErrorType.UNICODE_PROPERTY_NOT_ALLOWED_IN_RANGE.code + "): Test.g4:12:41: unicode property escapes not allowed in lexer charset range: [A-\\p{Number}]\n" + + "error(" + ErrorType.UNICODE_PROPERTY_NOT_ALLOWED_IN_RANGE.code + "): Test.g4:13:48: unicode property escapes not allowed in lexer charset range: [\\P{Uppercase_Letter}-\\P{Number}]\n"; String[] pair = new String[] { grammar, @@ -729,4 +806,20 @@ public class TestToolSyntaxErrors extends BaseJavaToolTest { String[] pair = { grammar, expected }; super.testErrors(pair, true); } + + // Test for https://github.com/antlr/antlr4/issues/1556 + @Test public void testRangeInParserGrammar() { + String grammar = + "grammar T;\n"+ + "a: 'A'..'Z' ;\n"; + String expected = + "error(" + ErrorType.TOKEN_RANGE_IN_PARSER.code + "): T.g4:2:4: token ranges not allowed in parser: 'A'..'Z'\n"; + + String[] pair = new String[] { + grammar, + expected + }; + + super.testErrors(pair, true); + } } diff --git a/tool-testsuite/test/org/antlr/v4/test/tool/TestTopologicalSort.java b/tool-testsuite/test/org/antlr/v4/test/tool/TestTopologicalSort.java index 2c0b596..7e3dfef 100644 --- a/tool-testsuite/test/org/antlr/v4/test/tool/TestTopologicalSort.java +++ b/tool-testsuite/test/org/antlr/v4/test/tool/TestTopologicalSort.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/tool-testsuite/test/org/antlr/v4/test/tool/TestUnbufferedCharStream.java b/tool-testsuite/test/org/antlr/v4/test/tool/TestUnbufferedCharStream.java index fec0936..11f781d 100644 --- a/tool-testsuite/test/org/antlr/v4/test/tool/TestUnbufferedCharStream.java +++ b/tool-testsuite/test/org/antlr/v4/test/tool/TestUnbufferedCharStream.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ @@ -313,6 +313,30 @@ public class TestUnbufferedCharStream extends BaseJavaToolTest { assertEquals(expecting, tokens.getTokens().toString()); } + @Test public void testUnicodeSMP() throws Exception { + TestingUnbufferedCharStream input = createStream("\uD83C\uDF0E"); + assertEquals(0x1F30E, input.LA(1)); + assertEquals("\uD83C\uDF0E", input.getBuffer()); + input.consume(); + assertEquals(IntStream.EOF, input.LA(1)); + assertEquals("\uFFFF", input.getBuffer()); + } + + @Test(expected = RuntimeException.class) + public void testDanglingHighSurrogateAtEOFThrows() throws Exception { + createStream("\uD83C"); + } + + @Test(expected = RuntimeException.class) + public void testDanglingHighSurrogateThrows() throws Exception { + createStream("\uD83C\u0123"); + } + + @Test(expected = RuntimeException.class) + public void testDanglingLowSurrogateThrows() throws Exception { + createStream("\uDF0E"); + } + protected static TestingUnbufferedCharStream createStream(String text) { return new TestingUnbufferedCharStream(new StringReader(text)); } @@ -336,7 +360,13 @@ public class TestUnbufferedCharStream extends BaseJavaToolTest { */ public String getRemainingBuffer() { if ( n==0 ) return ""; - return new String(data,p,n-p); + int len = n; + if (data[len-1] == IntStream.EOF) { + // Don't pass -1 to new String(). + return new String(data,p,len-p-1) + "\uFFFF"; + } else { + return new String(data,p,len-p); + } } /** For testing. What's in moving window buffer into data stream. @@ -344,7 +374,14 @@ public class TestUnbufferedCharStream extends BaseJavaToolTest { */ public String getBuffer() { if ( n==0 ) return ""; - return new String(data,0,n); + int len = n; + // Don't pass -1 to new String(). + if (data[len-1] == IntStream.EOF) { + // Don't pass -1 to new String(). + return new String(data,0,len-1) + "\uFFFF"; + } else { + return new String(data,0,len); + } } } diff --git a/tool-testsuite/test/org/antlr/v4/test/tool/TestUnbufferedTokenStream.java b/tool-testsuite/test/org/antlr/v4/test/tool/TestUnbufferedTokenStream.java index 5d9795d..700fe74 100644 --- a/tool-testsuite/test/org/antlr/v4/test/tool/TestUnbufferedTokenStream.java +++ b/tool-testsuite/test/org/antlr/v4/test/tool/TestUnbufferedTokenStream.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/tool-testsuite/test/org/antlr/v4/test/tool/TestUnicodeData.java b/tool-testsuite/test/org/antlr/v4/test/tool/TestUnicodeData.java new file mode 100644 index 0000000..231c5af --- /dev/null +++ b/tool-testsuite/test/org/antlr/v4/test/tool/TestUnicodeData.java @@ -0,0 +1,220 @@ +/* + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. + * Use of this file is governed by the BSD 3-clause license that + * can be found in the LICENSE.txt file in the project root. + */ + +package org.antlr.v4.test.tool; + +import java.util.Map; + +import org.antlr.v4.unicode.UnicodeData; +import org.antlr.v4.runtime.misc.IntervalSet; + +import org.junit.Test; +import org.junit.Rule; +import org.junit.rules.ExpectedException; + +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +public class TestUnicodeData { + @Rule + public ExpectedException thrown = ExpectedException.none(); + + @Test + public void testUnicodeGeneralCategoriesLatin() { + assertTrue(UnicodeData.getPropertyCodePoints("Lu").contains('X')); + assertFalse(UnicodeData.getPropertyCodePoints("Lu").contains('x')); + assertTrue(UnicodeData.getPropertyCodePoints("Ll").contains('x')); + assertFalse(UnicodeData.getPropertyCodePoints("Ll").contains('X')); + assertTrue(UnicodeData.getPropertyCodePoints("L").contains('X')); + assertTrue(UnicodeData.getPropertyCodePoints("L").contains('x')); + assertTrue(UnicodeData.getPropertyCodePoints("N").contains('0')); + assertTrue(UnicodeData.getPropertyCodePoints("Z").contains(' ')); + } + + @Test + public void testUnicodeGeneralCategoriesBMP() { + assertTrue(UnicodeData.getPropertyCodePoints("Lu").contains('\u1E3A')); + assertFalse(UnicodeData.getPropertyCodePoints("Lu").contains('\u1E3B')); + assertTrue(UnicodeData.getPropertyCodePoints("Ll").contains('\u1E3B')); + assertFalse(UnicodeData.getPropertyCodePoints("Ll").contains('\u1E3A')); + assertTrue(UnicodeData.getPropertyCodePoints("L").contains('\u1E3A')); + assertTrue(UnicodeData.getPropertyCodePoints("L").contains('\u1E3B')); + assertTrue(UnicodeData.getPropertyCodePoints("N").contains('\u1BB0')); + assertFalse(UnicodeData.getPropertyCodePoints("N").contains('\u1E3A')); + assertTrue(UnicodeData.getPropertyCodePoints("Z").contains('\u2028')); + assertFalse(UnicodeData.getPropertyCodePoints("Z").contains('\u1E3A')); + } + + @Test + public void testUnicodeGeneralCategoriesSMP() { + assertTrue(UnicodeData.getPropertyCodePoints("Lu").contains(0x1D5D4)); + assertFalse(UnicodeData.getPropertyCodePoints("Lu").contains(0x1D770)); + assertTrue(UnicodeData.getPropertyCodePoints("Ll").contains(0x1D770)); + assertFalse(UnicodeData.getPropertyCodePoints("Ll").contains(0x1D5D4)); + assertTrue(UnicodeData.getPropertyCodePoints("L").contains(0x1D5D4)); + assertTrue(UnicodeData.getPropertyCodePoints("L").contains(0x1D770)); + assertTrue(UnicodeData.getPropertyCodePoints("N").contains(0x11C50)); + assertFalse(UnicodeData.getPropertyCodePoints("N").contains(0x1D5D4)); + } + + @Test + public void testUnicodeCategoryAliases() { + assertTrue(UnicodeData.getPropertyCodePoints("Lowercase_Letter").contains('x')); + assertFalse(UnicodeData.getPropertyCodePoints("Lowercase_Letter").contains('X')); + assertTrue(UnicodeData.getPropertyCodePoints("Letter").contains('x')); + assertFalse(UnicodeData.getPropertyCodePoints("Letter").contains('0')); + assertTrue(UnicodeData.getPropertyCodePoints("Enclosing_Mark").contains(0x20E2)); + assertFalse(UnicodeData.getPropertyCodePoints("Enclosing_Mark").contains('x')); + } + + @Test + public void testUnicodeBinaryProperties() { + assertTrue(UnicodeData.getPropertyCodePoints("Emoji").contains(0x1F4A9)); + assertFalse(UnicodeData.getPropertyCodePoints("Emoji").contains('X')); + assertTrue(UnicodeData.getPropertyCodePoints("alnum").contains('9')); + assertFalse(UnicodeData.getPropertyCodePoints("alnum").contains(0x1F4A9)); + assertTrue(UnicodeData.getPropertyCodePoints("Dash").contains('-')); + assertTrue(UnicodeData.getPropertyCodePoints("Hex").contains('D')); + assertFalse(UnicodeData.getPropertyCodePoints("Hex").contains('Q')); + } + + @Test + public void testUnicodeBinaryPropertyAliases() { + assertTrue(UnicodeData.getPropertyCodePoints("Ideo").contains('\u611B')); + assertFalse(UnicodeData.getPropertyCodePoints("Ideo").contains('X')); + assertTrue(UnicodeData.getPropertyCodePoints("Soft_Dotted").contains('\u0456')); + assertFalse(UnicodeData.getPropertyCodePoints("Soft_Dotted").contains('X')); + assertTrue(UnicodeData.getPropertyCodePoints("Noncharacter_Code_Point").contains('\uFFFF')); + assertFalse(UnicodeData.getPropertyCodePoints("Noncharacter_Code_Point").contains('X')); + } + + @Test + public void testUnicodeScripts() { + assertTrue(UnicodeData.getPropertyCodePoints("Zyyy").contains('0')); + assertTrue(UnicodeData.getPropertyCodePoints("Latn").contains('X')); + assertTrue(UnicodeData.getPropertyCodePoints("Hani").contains(0x4E04)); + assertTrue(UnicodeData.getPropertyCodePoints("Cyrl").contains(0x0404)); + } + + @Test + public void testUnicodeScriptEquals() { + assertTrue(UnicodeData.getPropertyCodePoints("Script=Zyyy").contains('0')); + assertTrue(UnicodeData.getPropertyCodePoints("Script=Latn").contains('X')); + assertTrue(UnicodeData.getPropertyCodePoints("Script=Hani").contains(0x4E04)); + assertTrue(UnicodeData.getPropertyCodePoints("Script=Cyrl").contains(0x0404)); + } + + @Test + public void testUnicodeScriptAliases() { + assertTrue(UnicodeData.getPropertyCodePoints("Common").contains('0')); + assertTrue(UnicodeData.getPropertyCodePoints("Latin").contains('X')); + assertTrue(UnicodeData.getPropertyCodePoints("Han").contains(0x4E04)); + assertTrue(UnicodeData.getPropertyCodePoints("Cyrillic").contains(0x0404)); + } + + @Test + public void testUnicodeBlocks() { + assertTrue(UnicodeData.getPropertyCodePoints("InASCII").contains('0')); + assertTrue(UnicodeData.getPropertyCodePoints("InCJK").contains(0x4E04)); + assertTrue(UnicodeData.getPropertyCodePoints("InCyrillic").contains(0x0404)); + assertTrue(UnicodeData.getPropertyCodePoints("InMisc_Pictographs").contains(0x1F4A9)); + } + + @Test + public void testUnicodeBlockEquals() { + assertTrue(UnicodeData.getPropertyCodePoints("Block=ASCII").contains('0')); + assertTrue(UnicodeData.getPropertyCodePoints("Block=CJK").contains(0x4E04)); + assertTrue(UnicodeData.getPropertyCodePoints("Block=Cyrillic").contains(0x0404)); + assertTrue(UnicodeData.getPropertyCodePoints("Block=Misc_Pictographs").contains(0x1F4A9)); + } + + @Test + public void testUnicodeBlockAliases() { + assertTrue(UnicodeData.getPropertyCodePoints("InBasic_Latin").contains('0')); + assertTrue(UnicodeData.getPropertyCodePoints("InMiscellaneous_Mathematical_Symbols_B").contains(0x29BE)); + } + + @Test + public void testEnumeratedPropertyEquals() { + assertTrue( + "U+1F481 INFORMATION DESK PERSON is an emoji modifier base", + UnicodeData.getPropertyCodePoints("Grapheme_Cluster_Break=E_Base").contains(0x1F481)); + + assertFalse( + "U+1F47E ALIEN MONSTER is not an emoji modifier", + UnicodeData.getPropertyCodePoints("Grapheme_Cluster_Break=E_Base").contains(0x1F47E)); + + assertTrue( + "U+0E33 THAI CHARACTER SARA AM is a spacing mark", + UnicodeData.getPropertyCodePoints("Grapheme_Cluster_Break=E_Base").contains(0x1F481)); + + assertFalse( + "U+1038 MYANMAR SIGN VISARGA is not a spacing mark", + UnicodeData.getPropertyCodePoints("Grapheme_Cluster_Break=E_Base").contains(0x1038)); + + assertTrue( + "U+00A1 INVERTED EXCLAMATION MARK has ambiguous East Asian Width", + UnicodeData.getPropertyCodePoints("East_Asian_Width=Ambiguous").contains(0x00A1)); + + assertFalse( + "U+00A2 CENT SIGN does not have ambiguous East Asian Width", + UnicodeData.getPropertyCodePoints("East_Asian_Width=Ambiguous").contains(0x00A2)); + + } + + @Test + public void extendedPictographic() { + assertTrue( + "U+1F588 BLACK PUSHPIN is in Extended Pictographic", + UnicodeData.getPropertyCodePoints("Extended_Pictographic").contains(0x1F588)); + assertFalse( + "0 is not in Extended Pictographic", + UnicodeData.getPropertyCodePoints("Extended_Pictographic").contains('0')); + } + + @Test + public void emojiPresentation() { + assertTrue( + "U+1F4A9 PILE OF POO is in EmojiPresentation=EmojiDefault", + UnicodeData.getPropertyCodePoints("EmojiPresentation=EmojiDefault").contains(0x1F4A9)); + assertFalse( + "0 is not in EmojiPresentation=EmojiDefault", + UnicodeData.getPropertyCodePoints("EmojiPresentation=EmojiDefault").contains('0')); + assertFalse( + "A is not in EmojiPresentation=EmojiDefault", + UnicodeData.getPropertyCodePoints("EmojiPresentation=EmojiDefault").contains('A')); + assertFalse( + "U+1F4A9 PILE OF POO is not in EmojiPresentation=TextDefault", + UnicodeData.getPropertyCodePoints("EmojiPresentation=TextDefault").contains(0x1F4A9)); + assertTrue( + "0 is in EmojiPresentation=TextDefault", + UnicodeData.getPropertyCodePoints("EmojiPresentation=TextDefault").contains('0')); + assertFalse( + "A is not in EmojiPresentation=TextDefault", + UnicodeData.getPropertyCodePoints("EmojiPresentation=TextDefault").contains('A')); + } + + @Test + public void testPropertyCaseInsensitivity() { + assertTrue(UnicodeData.getPropertyCodePoints("l").contains('x')); + assertFalse(UnicodeData.getPropertyCodePoints("l").contains('0')); + assertTrue(UnicodeData.getPropertyCodePoints("common").contains('0')); + assertTrue(UnicodeData.getPropertyCodePoints("Alnum").contains('0')); + } + + @Test + public void testPropertyDashSameAsUnderscore() { + assertTrue(UnicodeData.getPropertyCodePoints("InLatin-1").contains('\u00F0')); + } + + @Test + public void modifyingUnicodeDataShouldThrow() { + thrown.expect(IllegalStateException.class); + thrown.expectMessage("can't alter readonly IntervalSet"); + UnicodeData.getPropertyCodePoints("L").add(0x12345); + } +} diff --git a/tool-testsuite/test/org/antlr/v4/test/tool/TestUnicodeEscapes.java b/tool-testsuite/test/org/antlr/v4/test/tool/TestUnicodeEscapes.java new file mode 100644 index 0000000..2792462 --- /dev/null +++ b/tool-testsuite/test/org/antlr/v4/test/tool/TestUnicodeEscapes.java @@ -0,0 +1,78 @@ +/* + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. + * Use of this file is governed by the BSD 3-clause license that + * can be found in the LICENSE.txt file in the project root. + */ + +package org.antlr.v4.test.tool; + +import org.antlr.v4.codegen.UnicodeEscapes; + +import org.junit.Test; + +import static org.junit.Assert.assertEquals; + +public class TestUnicodeEscapes { + @Test + public void latinJavaEscape() { + StringBuilder sb = new StringBuilder(); + UnicodeEscapes.appendJavaStyleEscapedCodePoint(0x0061, sb); + assertEquals("\\u0061", sb.toString()); + } + + @Test + public void latinPythonEscape() { + StringBuilder sb = new StringBuilder(); + UnicodeEscapes.appendPythonStyleEscapedCodePoint(0x0061, sb); + assertEquals("\\u0061", sb.toString()); + } + + @Test + public void latinSwiftEscape() { + StringBuilder sb = new StringBuilder(); + UnicodeEscapes.appendSwiftStyleEscapedCodePoint(0x0061, sb); + assertEquals("\\u{0061}", sb.toString()); + } + + @Test + public void bmpJavaEscape() { + StringBuilder sb = new StringBuilder(); + UnicodeEscapes.appendJavaStyleEscapedCodePoint(0xABCD, sb); + assertEquals("\\uABCD", sb.toString()); + } + + @Test + public void bmpPythonEscape() { + StringBuilder sb = new StringBuilder(); + UnicodeEscapes.appendPythonStyleEscapedCodePoint(0xABCD, sb); + assertEquals("\\uABCD", sb.toString()); + } + + @Test + public void bmpSwiftEscape() { + StringBuilder sb = new StringBuilder(); + UnicodeEscapes.appendSwiftStyleEscapedCodePoint(0xABCD, sb); + assertEquals("\\u{ABCD}", sb.toString()); + } + + @Test + public void smpJavaEscape() { + StringBuilder sb = new StringBuilder(); + UnicodeEscapes.appendJavaStyleEscapedCodePoint(0x1F4A9, sb); + assertEquals("\\uD83D\\uDCA9", sb.toString()); + } + + @Test + public void smpPythonEscape() { + StringBuilder sb = new StringBuilder(); + UnicodeEscapes.appendPythonStyleEscapedCodePoint(0x1F4A9, sb); + assertEquals("\\U0001F4A9", sb.toString()); + } + + @Test + public void smpSwiftEscape() { + StringBuilder sb = new StringBuilder(); + UnicodeEscapes.appendSwiftStyleEscapedCodePoint(0x1F4A9, sb); + assertEquals("\\u{1F4A9}", sb.toString()); + } +} diff --git a/tool-testsuite/test/org/antlr/v4/test/tool/TestUnicodeGrammar.java b/tool-testsuite/test/org/antlr/v4/test/tool/TestUnicodeGrammar.java new file mode 100644 index 0000000..fb19531 --- /dev/null +++ b/tool-testsuite/test/org/antlr/v4/test/tool/TestUnicodeGrammar.java @@ -0,0 +1,170 @@ +/* + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. + * Use of this file is governed by the BSD 3-clause license that + * can be found in the LICENSE.txt file in the project root. + */ + +package org.antlr.v4.test.tool; + +import org.antlr.v4.gui.Trees; +import org.antlr.v4.runtime.ANTLRInputStream; +import org.antlr.v4.runtime.CharStream; +import org.antlr.v4.runtime.CharStreams; +import org.antlr.v4.runtime.CommonTokenStream; +import org.antlr.v4.runtime.LexerInterpreter; +import org.antlr.v4.runtime.tree.ParseTree; +import org.antlr.v4.tool.Grammar; +import org.antlr.v4.tool.GrammarParserInterpreter; +import org.junit.Test; + +import java.io.ByteArrayInputStream; +import java.io.InputStreamReader; +import java.nio.charset.StandardCharsets; + +import static org.junit.Assert.assertEquals; + +public class TestUnicodeGrammar extends BaseJavaToolTest { + @Test + public void unicodeBMPLiteralInGrammar() throws Exception { + String grammarText = + "grammar Unicode;\n" + + "r : 'hello' WORLD;\n" + + "WORLD : ('world' | '\\u4E16\\u754C' | '\\u1000\\u1019\\u1039\\u1018\\u102C' );\n" + + "WS : [ \\t\\r\\n]+ -> skip;\n"; + String inputText = "hello \u4E16\u754C"; + assertEquals( + "(r:1 " + inputText + ")", + parseTreeForGrammarWithInput( + grammarText, + "r", + inputText)); + } + + // TODO: This test cannot pass unless we change either the grammar + // parser to decode surrogate pair literals to code points (which + // would break existing clients) or to treat them as an + // alternative: + // + // '\\uD83C\\uDF0D' -> ('\\u{1F30E}' | '\\uD83C\\uDF0D') + // + // but I worry that might cause parse ambiguity if we're not careful. + //@Test + public void unicodeSurrogatePairLiteralInGrammar() throws Exception { + String grammarText = + "grammar Unicode;\n" + + "r : 'hello' WORLD;\n" + + "WORLD : ('\\uD83C\\uDF0D' | '\\uD83C\\uDF0E' | '\\uD83C\\uDF0F' );\n" + + "WS : [ \\t\\r\\n]+ -> skip;\n"; + String inputText = new StringBuilder("hello ") + .appendCodePoint(0x1F30E) + .toString(); + assertEquals( + "(r:1 " + inputText + ")", + parseTreeForGrammarWithInput( + grammarText, + "r", + inputText)); + } + + @Test + public void unicodeSMPLiteralInGrammar() throws Exception { + String grammarText = + "grammar Unicode;\n" + + "r : 'hello' WORLD;\n" + + "WORLD : ('\\u{1F30D}' | '\\u{1F30E}' | '\\u{1F30F}' );\n" + + "WS : [ \\t\\r\\n]+ -> skip;\n"; + String inputText = new StringBuilder("hello ") + .appendCodePoint(0x1F30E) + .toString(); + assertEquals( + "(r:1 " + inputText + ")", + parseTreeForGrammarWithInput( + grammarText, + "r", + inputText)); + } + + @Test + public void unicodeSMPRangeInGrammar() throws Exception { + String grammarText = + "grammar Unicode;\n" + + "r : 'hello' WORLD;\n" + + "WORLD : ('\\u{1F30D}'..'\\u{1F30F}' );\n" + + "WS : [ \\t\\r\\n]+ -> skip;\n"; + String inputText = new StringBuilder("hello ") + .appendCodePoint(0x1F30E) + .toString(); + assertEquals( + "(r:1 " + inputText + ")", + parseTreeForGrammarWithInput( + grammarText, + "r", + inputText)); + } + + @Test + public void matchingDanglingSurrogateInInput() throws Exception { + String grammarText = + "grammar Unicode;\n" + + "r : 'hello' WORLD;\n" + + "WORLD : ('\\uD83C' | '\\uD83D' | '\\uD83E' );\n" + + "WS : [ \\t\\r\\n]+ -> skip;\n"; + String inputText = "hello \uD83C"; + assertEquals( + "(r:1 " + inputText + ")", + parseTreeForGrammarWithInput( + grammarText, + "r", + inputText)); + } + + @Test + public void binaryGrammar() throws Exception { + String grammarText = + "grammar Binary;\n" + + "r : HEADER PACKET+ FOOTER;\n" + + "HEADER : '\\u0002\\u0000\\u0001\\u0007';\n" + + "PACKET : '\\u00D0' ('\\u00D1' | '\\u00D2' | '\\u00D3') +;\n" + + "FOOTER : '\\u00FF';\n"; + byte[] toParse = new byte[] { + (byte)0x02, (byte)0x00, (byte)0x01, (byte)0x07, + (byte)0xD0, (byte)0xD2, (byte)0xD2, (byte)0xD3, (byte)0xD3, (byte)0xD3, + (byte)0xD0, (byte)0xD3, (byte)0xD3, (byte)0xD1, + (byte)0xFF + }; + CharStream charStream; + try (ByteArrayInputStream is = new ByteArrayInputStream(toParse); + // Note we use ISO_8859_1 to treat all byte values as Unicode "characters" from + // U+0000 to U+00FF. + InputStreamReader isr = new InputStreamReader(is, StandardCharsets.ISO_8859_1)) { + charStream = new ANTLRInputStream(isr); + } + Grammar grammar = new Grammar(grammarText); + LexerInterpreter lexEngine = grammar.createLexerInterpreter(charStream); + CommonTokenStream tokens = new CommonTokenStream(lexEngine); + GrammarParserInterpreter parser = grammar.createGrammarParserInterpreter(tokens); + ParseTree parseTree = parser.parse(grammar.rules.get("r").index); + InterpreterTreeTextProvider nodeTextProvider = + new InterpreterTreeTextProvider(grammar.getRuleNames()); + String result = Trees.toStringTree(parseTree, nodeTextProvider); + + assertEquals( + "(r:1 \u0002\u0000\u0001\u0007 \u00D0\u00D2\u00D2\u00D3\u00D3\u00D3 \u00D0\u00D3\u00D3\u00D1 \u00FF)", + result); + } + + private static String parseTreeForGrammarWithInput( + String grammarText, + String rootRule, + String inputText) throws Exception { + Grammar grammar = new Grammar(grammarText); + LexerInterpreter lexEngine = grammar.createLexerInterpreter( + CharStreams.fromString(inputText)); + CommonTokenStream tokens = new CommonTokenStream(lexEngine); + GrammarParserInterpreter parser = grammar.createGrammarParserInterpreter(tokens); + ParseTree parseTree = parser.parse(grammar.rules.get(rootRule).index); + InterpreterTreeTextProvider nodeTextProvider = + new InterpreterTreeTextProvider(grammar.getRuleNames()); + return Trees.toStringTree(parseTree, nodeTextProvider); + } +} diff --git a/tool-testsuite/test/org/antlr/v4/test/tool/TestVocabulary.java b/tool-testsuite/test/org/antlr/v4/test/tool/TestVocabulary.java index b956494..6a880c7 100644 --- a/tool-testsuite/test/org/antlr/v4/test/tool/TestVocabulary.java +++ b/tool-testsuite/test/org/antlr/v4/test/tool/TestVocabulary.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/tool-testsuite/test/org/antlr/v4/test/tool/TestXPath.java b/tool-testsuite/test/org/antlr/v4/test/tool/TestXPath.java index 7dbd741..d3b0b0a 100644 --- a/tool-testsuite/test/org/antlr/v4/test/tool/TestXPath.java +++ b/tool-testsuite/test/org/antlr/v4/test/tool/TestXPath.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/tool/pom.xml b/tool/pom.xml index b708168..72f632d 100644 --- a/tool/pom.xml +++ b/tool/pom.xml @@ -1,21 +1,20 @@ <!-- - ~ Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + ~ Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. ~ Use of this file is governed by the BSD 3-clause license that ~ can be found in the LICENSE.txt file in the project root. --> <project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd"> - <modelVersion>4.0.0</modelVersion> - <parent> - <groupId>org.antlr</groupId> - <artifactId>antlr4-master</artifactId> - <version>4.6</version> - </parent> - <artifactId>antlr4</artifactId> - <name>ANTLR 4 Tool</name> - <url>http://www.antlr.org</url> - <description>The ANTLR 4 grammar compiler.</description> - + <modelVersion>4.0.0</modelVersion> + <parent> + <groupId>org.antlr</groupId> + <artifactId>antlr4-master</artifactId> + <version>4.7.1</version> + </parent> + <artifactId>antlr4</artifactId> + <name>ANTLR 4 Tool</name> + <url>http://www.antlr.org</url> + <description>The ANTLR 4 grammar compiler.</description> <dependencies> <dependency> <groupId>org.antlr</groupId> @@ -42,9 +41,13 @@ <artifactId>javax.json</artifactId> <version>1.0.4</version> </dependency> + <dependency> + <groupId>com.ibm.icu</groupId> + <artifactId>icu4j</artifactId> + <version>58.2</version> + </dependency> </dependencies> - - <build> + <build> <sourceDirectory>src</sourceDirectory> <testResources> <testResource> @@ -55,6 +58,7 @@ <plugin> <!-- create src jar --> <groupId>org.apache.maven.plugins</groupId> <artifactId>maven-source-plugin</artifactId> + <version>3.0.1</version> <executions> <execution> <goals> @@ -80,74 +84,113 @@ </execution> </executions> </plugin> + <plugin> <!-- include code-generated sources --> + <groupId>org.codehaus.mojo</groupId> + <artifactId>build-helper-maven-plugin</artifactId> + <version>3.0.0</version> + <executions> + <execution> + <phase>generate-sources</phase> + <goals> + <goal>add-source</goal> + </goals> + <configuration> + <sources> + <source>${project.build.directory}/generated-sources/antlr4-tool-codegen</source> + </sources> + </configuration> + </execution> + </executions> + </plugin> <plugin> <!-- this makes a fat jar with all dependencies --> <groupId>org.apache.maven.plugins</groupId> <artifactId>maven-shade-plugin</artifactId> <version>2.3</version> <executions> <execution> - <phase>package</phase> - <configuration> + <phase>package</phase> + <configuration> <minimizeJar>false</minimizeJar> <createDependencyReducedPom>false</createDependencyReducedPom> <shadedArtifactAttached>true</shadedArtifactAttached> - <shadedClassifierName>complete</shadedClassifierName> - </configuration> - <goals> - <goal>shade</goal> - </goals> - </execution> - </executions> - </plugin> - <!-- can't get this to add bundle items - <plugin> - <groupId>org.apache.felix</groupId> - <artifactId>maven-bundle-plugin</artifactId> - <version>2.5.4</version> - <executions> - <execution> - <id>bundle-manifest</id> - <phase>process-classes</phase> - <configuration> - <instructions> - <Bundle-SymbolicName>org.antlr.antlr4-tool</Bundle-SymbolicName> - </instructions> - </configuration> - <goals> - <goal>manifest</goal> - </goals> - </execution> - </executions> - </plugin> - --> - <plugin> <!-- this just jars up tool stuff and sets main class --> - <artifactId>maven-jar-plugin</artifactId> - <version>3.0.0</version> - <configuration> - <archive> - <manifest> - <addDefaultImplementationEntries>true</addDefaultImplementationEntries> - <mainClass>org.antlr.v4.Tool</mainClass> - </manifest> - </archive> - </configuration> - </plugin> - <plugin> - <groupId>org.antlr</groupId> - <artifactId>antlr3-maven-plugin</artifactId> - <version>3.5.2</version> - <configuration> - <sourceDirectory>src</sourceDirectory> - <verbose>true</verbose> - </configuration> - <executions> - <execution> - <goals> - <goal>antlr</goal> - </goals> - </execution> - </executions> - </plugin> - </plugins> - </build> + <shadedClassifierName>complete</shadedClassifierName> + <artifactSet> + <excludes> + <exclude>com.ibm.icu:*</exclude> + </excludes> + </artifactSet> + </configuration> + <goals> + <goal>shade</goal> + </goals> + </execution> + </executions> + </plugin> + <!-- can't get this to add bundle items <plugin> <groupId>org.apache.felix</groupId> <artifactId>maven-bundle-plugin</artifactId> <version>2.5.4</version> <executions> <execution> <id>bundle-manifest</id> <phase>process-classes</phase> <configuration> <instructions> <Bundle-SymbolicName>org.antlr.antlr4-tool</Bundle-SymbolicName> </instructions> </configuration> <goals> <goal>manifest</goal> </goals> </execution> </executions> </plugin> --> + <plugin> <!-- this just jars up tool stuff and sets main class --> + <artifactId>maven-jar-plugin</artifactId> + <version>3.0.0</version> + <configuration> + <archive> + <manifest> + <addDefaultImplementationEntries>true</addDefaultImplementationEntries> + <mainClass>org.antlr.v4.Tool</mainClass> + </manifest> + </archive> + </configuration> + </plugin> + <plugin> + <groupId>org.antlr</groupId> + <artifactId>antlr3-maven-plugin</artifactId> + <version>3.5.2</version> + <configuration> + <sourceDirectory>src</sourceDirectory> + <verbose>true</verbose> + </configuration> + <executions> + <execution> + <goals> + <goal>antlr</goal> + </goals> + </execution> + </executions> + </plugin> + <plugin> + <groupId>com.webguys</groupId> + <artifactId>string-template-maven-plugin</artifactId> + <version>1.1</version> + <!-- this nonsense is for some reason needed to make this plugin work with maven 3 --> + <dependencies> + <dependency> + <groupId>org.twdata.maven</groupId> + <artifactId>mojo-executor</artifactId> + <version>2.1.0</version> + </dependency> + </dependencies> + <configuration> + <templates> + <template> + <directory>${basedir}/resources/org/antlr/v4/tool/templates</directory> + <name>unicodedata</name> + <target>${basedir}/target/generated-sources/tool/src/org/antlr/v4/unicode/UnicodeData.java</target> + <controller> + <className>org.antlr.v4.unicode.UnicodeDataTemplateController</className> + <sourceVersion>1.7</sourceVersion> + <targetVersion>1.7</targetVersion> + <method>getProperties</method> + </controller> + </template> + </templates> + </configuration> + <executions> + <execution> + <phase>generate-sources</phase> + <goals> + <goal>render</goal> + </goals> + </execution> + </executions> + </plugin> + </plugins> + </build> </project> diff --git a/tool/resources/org/antlr/v4/tool/templates/codegen/CSharp/CSharp.stg b/tool/resources/org/antlr/v4/tool/templates/codegen/CSharp/CSharp.stg index 07b9f8b..a665ba6 100644 --- a/tool/resources/org/antlr/v4/tool/templates/codegen/CSharp/CSharp.stg +++ b/tool/resources/org/antlr/v4/tool/templates/codegen/CSharp/CSharp.stg @@ -37,6 +37,7 @@ namespace <file.genPackage> { <endif> <namedActions.header> using System; +using System.IO; using System.Text; using System.Diagnostics; using System.Collections.Generic; @@ -287,7 +288,7 @@ public partial class <csIdentifier.(parser.name)> : <superClass; null="Parser"> public override string[] RuleNames { get { return ruleNames; } } - public override string SerializedAtn { get { return _serializedATN; } } + public override string SerializedAtn { get { return new string(_serializedATN); } } static <csIdentifier.(parser.name)>() { decisionToDFA = new DFA[_ATN.NumberOfDecisions]; @@ -357,8 +358,10 @@ case <f.ruleIndex> : return <f.name>_sempred(<if(!recog.modes)>(<f.ctxType>)<end >> parser_ctor(parser) ::= << -public <csIdentifier.(parser.name)>(ITokenStream input) - : base(input) + public <csIdentifier.(parser.name)>(ITokenStream input) : this(input, Console.Out, Console.Error) { } + + public <csIdentifier.(parser.name)>(ITokenStream input, TextWriter output, TextWriter errorOutput) + : base(input, output, errorOutput) { Interpreter = new ParserATNSimulator(this, _ATN, decisionToDFA, sharedContextCache); } @@ -566,7 +569,7 @@ switch (TokenStream.LA(1)) { <alt> break;}; separator="\n"> default: - <error> + break; } >> @@ -958,6 +961,7 @@ namespace <file.genPackage> { <endif> <namedActions.header> using System; +using System.IO; using System.Text; using Antlr4.Runtime; using Antlr4.Runtime.Atn; @@ -982,7 +986,14 @@ public partial class <csIdentifier.(lexer.name)> : <superClass; null="Lexer"> { public const int <lexer.channels:{k | <csIdentifier.(k)>=<lexer.channels.(k)>}; separator=", ", wrap, anchor>; <endif> - <rest(lexer.modes):{m| public const int <modeName.(m)> = <i>;}; separator="\n"> + <if(rest(lexer.modes))> + public const int + <rest(lexer.modes):{m | <m>=<i>}; separator=", ", wrap, anchor>; + <endif> + public static string[] channelNames = { + "DEFAULT_TOKEN_CHANNEL", "HIDDEN"<if (lexer.channels)>, <lexer.channels:{c| "<c>"}; separator=", ", wrap, anchor><endif> + }; + public static string[] modeNames = { <lexer.modes:{m| "<m>"}; separator=", ", wrap, anchor> }; @@ -994,7 +1005,10 @@ public partial class <csIdentifier.(lexer.name)> : <superClass; null="Lexer"> { <namedActions.members> public <csIdentifier.(lexer.name)>(ICharStream input) - : base(input) + : this(input, Console.Out, Console.Error) { } + + public <csIdentifier.(lexer.name)>(ICharStream input, TextWriter output, TextWriter errorOutput) + : base(input, output, errorOutput) { Interpreter = new LexerATNSimulator(this, _ATN, decisionToDFA, sharedContextCache); } @@ -1005,9 +1019,11 @@ public partial class <csIdentifier.(lexer.name)> : <superClass; null="Lexer"> { public override string[] RuleNames { get { return ruleNames; } } + public override string[] ChannelNames { get { return channelNames; } } + public override string[] ModeNames { get { return modeNames; } } - public override string SerializedAtn { get { return _serializedATN; } } + public override string SerializedAtn { get { return new string(_serializedATN); } } static <csIdentifier.(lexer.name)>() { decisionToDFA = new DFA[_ATN.NumberOfDecisions]; @@ -1022,16 +1038,12 @@ public partial class <csIdentifier.(lexer.name)> : <superClass; null="Lexer"> { SerializedATN(model) ::= << -private static string _serializedATN = _serializeATN(); -private static string _serializeATN() -{ - StringBuilder sb = new StringBuilder(); - sb.Append("<model.serialized; wrap={");<\n><\t>sb.Append("}>"); - return sb.ToString(); -} +private static char[] _serializedATN = { + <model.serialized; separator=", ", wrap>, +}; public static readonly ATN _ATN = - new ATNDeserializer().Deserialize(_serializedATN.ToCharArray()); + new ATNDeserializer().Deserialize(_serializedATN); >> @@ -1133,6 +1145,7 @@ csIdentifier ::= [ "ushort" : "@ushort", "using" : "@using", "virtual" : "@virtual", + "values" : "@values", "void" : "@void", "volatile" : "@volatile", "while" : "@while", diff --git a/tool/resources/org/antlr/v4/tool/templates/codegen/Cpp/Cpp.stg b/tool/resources/org/antlr/v4/tool/templates/codegen/Cpp/Cpp.stg index 78f9610..6342b66 100644 --- a/tool/resources/org/antlr/v4/tool/templates/codegen/Cpp/Cpp.stg +++ b/tool/resources/org/antlr/v4/tool/templates/codegen/Cpp/Cpp.stg @@ -59,7 +59,7 @@ public: <if (rest(lexer.modes))> enum { - <rest(lexer.modes): {m | <m> = <i>,}; separator="\n", anchor> + <rest(lexer.modes): {m | <m> = <i>}; separator=", ", wrap, anchor> }; <endif> @@ -70,6 +70,7 @@ public: virtual std::string getGrammarFileName() const override; virtual const std::vector\<std::string>& getRuleNames() const override; + virtual const std::vector\<std::string>& getChannelNames() const override; virtual const std::vector\<std::string>& getModeNames() const override; virtual const std::vector\<std::string>& getTokenNames() const override; // deprecated, use vocabulary instead virtual antlr4::dfa::Vocabulary& getVocabulary() const override; @@ -89,6 +90,7 @@ private: static antlr4::atn::PredictionContextCache _sharedContextCache; static std::vector\<std::string> _ruleNames; static std::vector\<std::string> _tokenNames; + static std::vector\<std::string> _channelNames; static std::vector\<std::string> _modeNames; static std::vector\<std::string> _literalNames; @@ -128,6 +130,10 @@ const std::vector\<std::string>& <lexer.name>::getRuleNames() const { return _ruleNames; } +const std::vector\<std::string>& <lexer.name>::getChannelNames() const { + return _channelNames; +} + const std::vector\<std::string>& <lexer.name>::getModeNames() const { return _modeNames; } @@ -186,19 +192,23 @@ atn::ATN <lexer.name>::_atn; std::vector\<uint16_t> <lexer.name>::_serializedATN; std::vector\<std::string> <lexer.name>::_ruleNames = { - <lexer.ruleNames: {r | "<r>"}; separator = ", ", wrap, anchor> + <lexer.ruleNames: {r | u8"<r>"}; separator = ", ", wrap, anchor> +}; + +std::vector\<std::string> <lexer.name>::_channelNames = { + "DEFAULT_TOKEN_CHANNEL", "HIDDEN"<if (lexer.channels)>, <lexer.channels: {c | u8"<c>"}; separator = ", ", wrap, anchor><endif> }; std::vector\<std::string> <lexer.name>::_modeNames = { - <lexer.modes: {m | "<m>"}; separator = ", ", wrap, anchor> + <lexer.modes: {m | u8"<m>"}; separator = ", ", wrap, anchor> }; std::vector\<std::string> <lexer.name>::_literalNames = { - <lexer.literalNames: {t | <t>}; null = "\"\"", separator = ", ", wrap, anchor> + <lexer.literalNames: {t | u8<t>}; null = "\"\"", separator = ", ", wrap, anchor> }; std::vector\<std::string> <lexer.name>::_symbolicNames = { - <lexer.symbolicNames: {t | <t>}; null = "\"\"", separator = ", ", wrap, anchor> + <lexer.symbolicNames: {t | u8<t>}; null = "\"\"", separator = ", ", wrap, anchor> }; dfa::Vocabulary <lexer.name>::_vocabulary(_literalNames, _symbolicNames); @@ -644,7 +654,7 @@ switch (_input->LA(1)) { \} }; separator="\n"> default: - <error> + break; } >> diff --git a/tool/resources/org/antlr/v4/tool/templates/codegen/Go/Go.stg b/tool/resources/org/antlr/v4/tool/templates/codegen/Go/Go.stg index 7559f01..81d0048 100644 --- a/tool/resources/org/antlr/v4/tool/templates/codegen/Go/Go.stg +++ b/tool/resources/org/antlr/v4/tool/templates/codegen/Go/Go.stg @@ -1,5 +1,5 @@ fileHeader(grammarFileName, ANTLRVersion) ::= << -// Generated from <grammarFileName; format="java-escape"> by ANTLR <ANTLRVersion>. +// Code generated from <grammarFileName; format="java-escape"> by ANTLR <ANTLRVersion>. DO NOT EDIT. >> ParserFile(file, parser, namedActions, contextSuperClass) ::= << @@ -145,19 +145,13 @@ func (v *Base<file.grammarName>Visitor) Visit<lname; format="cap">(ctx *<lname; >> Parser(parser, funcs, atn, sempredFuncs, superClass) ::= << -<if(superClass)> -import "./<superClass>" - -<endif> <if(atn)> var parserATN = <atn> <else> var parserATN []uint16 <endif> - var deserializer = antlr.NewATNDeserializer(nil) - var deserializedATN = deserializer.DeserializeFromUInt16(parserATN) <if(parser.literalNames)> @@ -168,7 +162,6 @@ var literalNames = []string{ var literalNames []string <endif> - <if(parser.symbolicNames)> var symbolicNames = []string{ <parser.symbolicNames; null="\"\"", separator=", ", wrap>, @@ -186,24 +179,24 @@ var ruleNames = []string{ var ruleNames []string <endif> +var decisionToDFA = make([]*antlr.DFA, len(deserializedATN.DecisionToState)) + +func init() { + for index, ds := range deserializedATN.DecisionToState { + decisionToDFA[index] = antlr.NewDFA(ds, index) + } +} type <parser.name> struct { <superClass; null="*antlr.BaseParser"> } func New<parser.name>(input antlr.TokenStream) *<parser.name> { - var decisionToDFA = make([]*antlr.DFA, len(deserializedATN.DecisionToState)) - var sharedContextCache = antlr.NewPredictionContextCache() - - for index, ds := range deserializedATN.DecisionToState { - decisionToDFA[index] = antlr.NewDFA(ds, index) - } - this := new(<parser.name>) this.BaseParser = antlr.NewBaseParser(input) - this.Interpreter = antlr.NewParserATNSimulator(this, deserializedATN, decisionToDFA, sharedContextCache) + this.Interpreter = antlr.NewParserATNSimulator(this, deserializedATN, decisionToDFA, antlr.NewPredictionContextCache()) this.RuleNames = ruleNames this.LiteralNames = literalNames this.SymbolicNames = symbolicNames @@ -563,9 +556,6 @@ switch p.GetTokenStream().LA(1) { <endif> default: - <if(error)> - <error> - <endif> } >> @@ -787,29 +777,31 @@ MatchSet(m, expr, capture) ::= "<CommonSetStuff(m, expr, capture, false)>" MatchNotSet(m, expr, capture) ::= "<CommonSetStuff(m, expr, capture, true)>" CommonSetStuff(m, expr, capture, invert) ::= << -p.SetState(<m.stateNumber>) -<if(m.labels)> - -var _lt = p.GetTokenStream().LT(1)<! TODO: Should LT be called always like InvokeRule and MatchToken? !> +{ + p.SetState(<m.stateNumber>) + <if(m.labels)> -<m.labels:{l | <labelref(l)> = _lt}; separator="\n"> + var _lt = p.GetTokenStream().LT(1)<! TODO: Should LT be called always like InvokeRule and MatchToken? !> -<endif> -<if(capture)> -<capture> + <m.labels:{l | <labelref(l)> = _lt}; separator="\n"> -<endif> -<if(invert)>if <m.varName> \<= 0 || <expr> <else>if !(<expr>)<endif> { - <if(m.labels)> - var _ri = p.GetErrorHandler().RecoverInline(p) + <endif> + <if(capture)> + <capture> - <m.labels:{l | <labelref(l)> = _ri}; separator="\n"> - <else> - p.GetErrorHandler().RecoverInline(p) <endif> -} else { - p.GetErrorHandler().ReportMatch(p) - p.Consume() + <if(invert)>if <m.varName> \<= 0 || <expr> <else>if !(<expr>)<endif> { + <if(m.labels)> + var _ri = p.GetErrorHandler().RecoverInline(p) + + <m.labels:{l | <labelref(l)> = _ri}; separator="\n"> + <else> + p.GetErrorHandler().RecoverInline(p) + <endif> + } else { + p.GetErrorHandler().ReportMatch(p) + p.Consume() + } } >> @@ -905,7 +897,7 @@ TokenListDecl(t) ::= "<t.name> []antlr.Token" RuleContextDecl(r) ::= "<r.name> I<r.ctxName> " RuleContextListDecl(rdecl) ::= "<rdecl.name> []I<rdecl.ctxName>" -AttributeDecl(d) ::= "<d.name> <d.type;format={lower}><if(d.initValue)>// TODO = <d.initValue><endif>" +AttributeDecl(d) ::= "<d.name> <d.type><if(d.initValue)>// TODO = <d.initValue><endif>" ContextTokenGetterDecl(t) ::= << <t.name; format="cap">() antlr.TerminalNode { @@ -1049,13 +1041,13 @@ Set<a.name; format="cap">([]I<a.ctxName>) }; separator="\n\n"> <if(struct.attributeDecls)> <struct.attributeDecls:{a | // Get<a.name; format="cap"> returns the <a.name> attribute. -Get<a.name; format="cap">() <a.type;format="lower">}; separator="\n\n"> +Get<a.name; format="cap">() <a.type>}; separator="\n\n"> <endif> <if(struct.attributeDecls)> <struct.attributeDecls:{a | // Set<a.name; format="cap"> sets the <a.name> attribute. -Set<a.name; format="cap">(<a.type;format="lower">)}; separator="\n\n"> +Set<a.name; format="cap">(<a.type>)}; separator="\n\n"> <endif> @@ -1080,7 +1072,7 @@ func NewEmpty<struct.name>() *<struct.name> { func (*<struct.name>) Is<struct.name>() {} -func New<struct.name>(parser antlr.Parser, parent antlr.ParserRuleContext, invokingState int<struct.ctorAttrs:{a | , <a.name> <a.type;format="lower">}>) *<struct.name> { +func New<struct.name>(parser antlr.Parser, parent antlr.ParserRuleContext, invokingState int<struct.ctorAttrs:{a | , <a.name> <a.type>}>) *<struct.name> { var p = new(<struct.name>) p.<if(contextSuperClass)><contextSuperClass><else>BaseParserRuleContext<endif> = <if(contextSuperClass)>New<contextSuperClass><else>antlr.NewBaseParserRuleContext<endif>(parent, invokingState) @@ -1148,12 +1140,12 @@ func (s *<struct.name>) GetParser() antlr.Parser { return s.parser } <if(struct.attributeDecls)> -<struct.attributeDecls:{a | func (s *<struct.name>) Get<a.name; format="cap">() <a.type;format="lower"> { return s.<a.name> \}}; separator="\n\n"> +<struct.attributeDecls:{a | func (s *<struct.name>) Get<a.name; format="cap">() <a.type> { return s.<a.name> \}}; separator="\n\n"> <endif> <if(struct.attributeDecls)> -<struct.attributeDecls:{a | func (s *<struct.name>) Set<a.name; format="cap">(v <a.type;format="lower">) { s.<a.name> = v \}}; separator="\n\n"> +<struct.attributeDecls:{a | func (s *<struct.name>) Set<a.name; format="cap">(v <a.type>) { s.<a.name> = v \}}; separator="\n\n"> <endif> <if(getters)> @@ -1258,12 +1250,12 @@ func New<struct.name>(parser antlr.Parser, ctx antlr.ParserRuleContext) *<struct <if(struct.attributeDecls)> -<struct.attributeDecls:{a | func (s *<struct.name>) Get<a.name; format="cap">() <a.type;format="lower"> { return s.<a.name> \}}; separator="\n\n"> +<struct.attributeDecls:{a | func (s *<struct.name>) Get<a.name; format="cap">() <a.type> { return s.<a.name> \}}; separator="\n\n"> <endif> <if(struct.attributeDecls)> -<struct.attributeDecls:{a | func (s *<struct.name>) Set<a.name; format="cap">(v <a.type;format="lower">) { s.<a.name> = v \}}; separator="\n\n"> +<struct.attributeDecls:{a | func (s *<struct.name>) Set<a.name; format="cap">(v <a.type>) { s.<a.name> = v \}}; separator="\n\n"> <endif> func (s *<struct.name>) GetRuleContext() antlr.RuleContext { @@ -1386,14 +1378,13 @@ var serializedLexerAtn []uint16 var lexerDeserializer = antlr.NewATNDeserializer(nil) var lexerAtn = lexerDeserializer.DeserializeFromUInt16(serializedLexerAtn) -<if(lexer.modes)> +var lexerChannelNames = []string{ + "DEFAULT_TOKEN_CHANNEL", "HIDDEN"<if (lexer.channels)>, <lexer.channels:{c | "<c>"}; separator=", ", wrap><endif>, +} + var lexerModeNames = []string{ <lexer.modes:{m | "<m>"}; separator=", ", wrap>, } -<else> -var lexerModeNames []string -<endif> - <if(lexer.literalNames)> var lexerLiteralNames = []string{ @@ -1424,22 +1415,27 @@ var lexerRuleNames []string type <lexer.name> struct { *<if(superClass)><superClass><else>antlr.BaseLexer<endif> + channelNames []string modeNames []string // TODO: EOF string } -func New<lexer.name>(input antlr.CharStream) *<lexer.name> { - var lexerDecisionToDFA = make([]*antlr.DFA, len(lexerAtn.DecisionToState)) +var lexerDecisionToDFA = make([]*antlr.DFA, len(lexerAtn.DecisionToState)) +func init() { for index, ds := range lexerAtn.DecisionToState { lexerDecisionToDFA[index] = antlr.NewDFA(ds, index) } +} + +func New<lexer.name>(input antlr.CharStream) *<lexer.name> { l := new(<lexer.name>) l.BaseLexer = antlr.NewBaseLexer(input) l.Interpreter = antlr.NewLexerATNSimulator(l, lexerAtn, lexerDecisionToDFA, antlr.NewPredictionContextCache()) + l.channelNames = lexerChannelNames l.modeNames = lexerModeNames l.RuleNames = lexerRuleNames l.LiteralNames = lexerLiteralNames @@ -1461,9 +1457,21 @@ const ( const <lexer.name><first(lexer.tokens)> = <lexer.tokens.(first(lexer.tokens))> <endif> +<if(rest(lexer.channels))> + +// <lexer.name> channels. +const ( + <lexer.channels:{c | <lexer.name><c> = <lexer.channels.(c)>}; separator="\n"> +) +<elseif(lexer.channels)> + +// <lexer.name><first(lexer.channels)> is the <lexer.name> channel. +const <lexer.name><first(lexer.channels)> = <lexer.channels.(first(lexer.channels))> +<endif> + <if(rest(rest(lexer.modes)))> -// <lexer.name> modes +// <lexer.name> modes. const ( <first(rest(lexer.modes)):{m | <lexer.name><m> = iota + 1}> <rest(rest(lexer.modes)):{m | <lexer.name><m>}; separator="\n"> diff --git a/tool/resources/org/antlr/v4/tool/templates/codegen/Java/Java.stg b/tool/resources/org/antlr/v4/tool/templates/codegen/Java/Java.stg index a1b18fc..492c566 100644 --- a/tool/resources/org/antlr/v4/tool/templates/codegen/Java/Java.stg +++ b/tool/resources/org/antlr/v4/tool/templates/codegen/Java/Java.stg @@ -489,7 +489,7 @@ switch (_input.LA(1)) { <alt> break;}; separator="\n"> default: - <error> + break; } >> @@ -894,13 +894,22 @@ public class <lexer.name> extends <superClass; null="Lexer"> { protected static final DFA[] _decisionToDFA; protected static final PredictionContextCache _sharedContextCache = new PredictionContextCache(); + <if(lexer.tokens)> public static final int <lexer.tokens:{k | <k>=<lexer.tokens.(k)>}; separator=", ", wrap, anchor>; + <endif> <if(lexer.channels)> public static final int - <lexer.channels:{k | <k>=<lexer.channels.(k)>}; separator=", ", wrap, anchor>; + <lexer.channels:{c | <c>=<lexer.channels.(c)>}; separator=", ", wrap, anchor>; + <endif> + <if(rest(lexer.modes))> + public static final int + <rest(lexer.modes):{m | <m>=<i>}; separator=", ", wrap, anchor>; <endif> - <rest(lexer.modes):{m| public static final int <m> = <i>;}; separator="\n"> + public static String[] channelNames = { + "DEFAULT_TOKEN_CHANNEL", "HIDDEN"<if (lexer.channels)>, <lexer.channels:{c| "<c>"}; separator=", ", wrap, anchor><endif> + }; + public static String[] modeNames = { <lexer.modes:{m| "<m>"}; separator=", ", wrap, anchor> }; @@ -928,6 +937,9 @@ public class <lexer.name> extends <superClass; null="Lexer"> { public String getSerializedATN() { return _serializedATN; } @Override + public String[] getChannelNames() { return channelNames; } + + @Override public String[] getModeNames() { return modeNames; } @Override diff --git a/tool/resources/org/antlr/v4/tool/templates/codegen/JavaScript/JavaScript.stg b/tool/resources/org/antlr/v4/tool/templates/codegen/JavaScript/JavaScript.stg index 2e10728..38c9ad8 100644 --- a/tool/resources/org/antlr/v4/tool/templates/codegen/JavaScript/JavaScript.stg +++ b/tool/resources/org/antlr/v4/tool/templates/codegen/JavaScript/JavaScript.stg @@ -36,12 +36,12 @@ * REQUIRED. */ -pythonTypeInitMap ::= [ - "bool":"False", +javascriptTypeInitMap ::= [ + "bool":"false", "int":"0", "float":"0.0", "str":"", - default:"None" // anything other than a primitive type is an object + default:"{}" // anything other than a primitive type is an object ] // args must be <object-model-object>, <fields-resulting-in-STs> @@ -360,7 +360,7 @@ switch (this._input.LA(1)) { <alt> break;}; separator="\n"> default: - <error> + break; } >> @@ -530,7 +530,7 @@ else { Wildcard(w) ::= << this.state = <w.stateNumber>; -<if(w.labels)><w.labels:{l | <labelref(l)> = }><endif>matchWildcard(); +<if(w.labels)><w.labels:{l | <labelref(l)> = }><endif>this.matchWildcard(); >> // ACTION STUFF @@ -802,6 +802,9 @@ var antlr4 = require('antlr4/index'); >> Lexer(lexer, atn, actionFuncs, sempredFuncs, superClass) ::= << +<if(superClass)> +var <superClass> = require('./<superClass>').<superClass>; +<endif> <atn> @@ -818,11 +821,25 @@ function <lexer.name>(input) { <lexer.name>.prototype = Object.create(<if(superClass)><superClass><else>antlr4.Lexer<endif>.prototype); <lexer.name>.prototype.constructor = <lexer.name>; +Object.defineProperty(<lexer.name>.prototype, "atn", { + get : function() { + return atn; + } +}); + <lexer.name>.EOF = antlr4.Token.EOF; <lexer.tokens:{k | <lexer.name>.<k> = <lexer.tokens.(k)>;}; separator="\n", wrap, anchor> +<if(lexer.channels)> +<lexer.channels:{c| <lexer.name>.<c> = <lexer.channels.(c)>;}; separator="\n"> + +<endif> +<if(rest(lexer.modes))> <rest(lexer.modes):{m| <lexer.name>.<m> = <i>;}; separator="\n"> +<endif> +<lexer.name>.prototype.channelNames = [ "DEFAULT_TOKEN_CHANNEL", "HIDDEN"<if (lexer.channels)>, <lexer.channels:{c| "<c>"}; separator=", ", wrap, anchor><endif> ]; + <lexer.name>.prototype.modeNames = [ <lexer.modes:{m| "<m>"}; separator=", ", wrap, anchor> ]; <lexer.name>.prototype.literalNames = [ <lexer.literalNames:{t | <t>}; null="null", separator=", ", wrap, anchor> ]; @@ -852,7 +869,7 @@ var serializedATN = ["<model.serialized; wrap={",<\n> "}>"].join(""); * must be an object, default value is "null". */ initValue(typeName) ::= << -<javaTypeInitMap.(typeName)> +<javacriptTypeInitMap.(typeName)> >> codeFileExtension() ::= ".js" diff --git a/tool/resources/org/antlr/v4/tool/templates/codegen/Python2/Python2.stg b/tool/resources/org/antlr/v4/tool/templates/codegen/Python2/Python2.stg index 287f366..570f165 100644 --- a/tool/resources/org/antlr/v4/tool/templates/codegen/Python2/Python2.stg +++ b/tool/resources/org/antlr/v4/tool/templates/codegen/Python2/Python2.stg @@ -52,6 +52,7 @@ ParserFile(file, parser, namedActions, contextSuperClass) ::= << from __future__ import print_function from antlr4 import * from io import StringIO +import sys <namedActions.header> <parser> @@ -166,16 +167,16 @@ class <parser.name> ( <if(superClass)><superClass><else>Parser<endif> ): dumpActions(recog, argFuncs, actionFuncs, sempredFuncs) ::= << <if(actionFuncs)> def action(self, localctx, ruleIndex, actionIndex): - if self._actions is None: - actions = dict() + if self._actions is None: + actions = dict() <recog.actionFuncs.values:{f| - actions[<f.ruleIndex>] = self.<f.name>_action }; separator="\n"> - self._actions = actions - action = self._actions.get(ruleIndex, None) - if action is not None: - action(localctx, actionIndex) - else: - raise Exception("No registered action for:" + str(ruleIndex)) + actions[<f.ruleIndex>] = self.<f.name>_action }; separator="\n"> + self._actions = actions + action = self._actions.get(ruleIndex, None) + if action is not None: + action(localctx, actionIndex) + else: + raise Exception("No registered action for:" + str(ruleIndex)) <actionFuncs.values; separator="\n"> @@ -198,8 +199,8 @@ def sempred(self, localctx, ruleIndex, predIndex): >> parser_ctor(p) ::= << -def __init__(self, input): - super(<parser.name>, self).__init__(input) +def __init__(self, input, output=sys.stdout): + super(<parser.name>, self).__init__(input, output=output) self.checkVersion("<file.ANTLRVersion>") self._interp = ParserATNSimulator(self, self.atn, self.decisionsToDFA, self.sharedContextCache) self._predicates = None @@ -334,7 +335,7 @@ token = self._input.LA(1) <alt> pass}; separator="\nel"> else: - <error> + pass >> LL1OptionalBlockSingleAlt(choice, expr, alts, preamble, error, followExpr) ::= << @@ -517,7 +518,7 @@ if not <chunks>: ExceptionClause(e, catchArg, catchAction) ::= << catch (<catchArg>) { - <catchAction> + <catchAction> } >> @@ -622,7 +623,7 @@ def <r.name>(self, i=None): LexerRuleContext() ::= "RuleContext" /** The rule context name is the rule followed by a suffix; e.g., - * r becomes rContext. + * r becomes rContext. */ RuleContextNameSuffix() ::= "Context" @@ -740,6 +741,7 @@ LexerFile(lexerFile, lexer, namedActions) ::= << from __future__ import print_function from antlr4 import * from io import StringIO +import sys <namedActions.header> @@ -756,10 +758,18 @@ class <lexer.name>(<if(superClass)><superClass><else>Lexer<endif>): decisionsToDFA = [ DFA(ds, i) for i, ds in enumerate(atn.decisionToState) ] +<if(lexer.channels)> + <lexer.channels:{c| <c> = <lexer.channels.(c)>}; separator="\n"> + +<endif> +<if(rest(lexer.modes))> <rest(lexer.modes):{m| <m> = <i>}; separator="\n"> +<endif> <lexer.tokens:{k | <k> = <lexer.tokens.(k)>}; separator="\n", wrap, anchor> + channelNames = [ u"DEFAULT_TOKEN_CHANNEL", u"HIDDEN"<if (lexer.channels)>, <lexer.channels:{c| u"<c>"}; separator=", ", wrap, anchor><endif> ] + modeNames = [ <lexer.modes:{m| u"<m>"}; separator=", ", wrap, anchor> ] literalNames = [ u"\<INVALID>", @@ -772,8 +782,8 @@ class <lexer.name>(<if(superClass)><superClass><else>Lexer<endif>): grammarFileName = u"<lexer.grammarFileName>" - def __init__(self, input=None): - super(<lexer.name>, self).__init__(input) + def __init__(self, input=None, output=sys.stdout): + super(<lexer.name>, self).__init__(input, output=output) self.checkVersion("<lexerFile.ANTLRVersion>") self._interp = LexerATNSimulator(self, self.atn, self.decisionsToDFA, PredictionContextCache()) self._actions = None @@ -796,10 +806,10 @@ def serializedATN(): >> /** Using a type to init value map, try to init a type; if not in table - * must be an object, default value is "null". + * must be an object, default value is "null". */ initValue(typeName) ::= << -<javaTypeInitMap.(typeName)> +<pythonTypeInitMap.(typeName)> >> codeFileExtension() ::= ".py" diff --git a/tool/resources/org/antlr/v4/tool/templates/codegen/Python3/Python3.stg b/tool/resources/org/antlr/v4/tool/templates/codegen/Python3/Python3.stg index 643cec7..34e525b 100644 --- a/tool/resources/org/antlr/v4/tool/templates/codegen/Python3/Python3.stg +++ b/tool/resources/org/antlr/v4/tool/templates/codegen/Python3/Python3.stg @@ -51,6 +51,8 @@ ParserFile(file, parser, namedActions, contextSuperClass) ::= << # encoding: utf-8 from antlr4 import * from io import StringIO +from typing.io import TextIO +import sys <namedActions.header> <parser> @@ -174,16 +176,16 @@ class <parser.name> ( <if(superClass)><superClass><else>Parser<endif> ): dumpActions(recog, argFuncs, actionFuncs, sempredFuncs) ::= << <if(actionFuncs)> def action(self, localctx:RuleContext, ruleIndex:int, actionIndex:int): - if self._actions is None: - actions = dict() + if self._actions is None: + actions = dict() <recog.actionFuncs.values:{f| - actions[<f.ruleIndex>] = self.<f.name>_action }; separator="\n"> - self._actions = actions - action = self._actions.get(ruleIndex, None) - if action is not None: - action(localctx, actionIndex) - else: - raise Exception("No registered action for:" + str(ruleIndex)) + actions[<f.ruleIndex>] = self.<f.name>_action }; separator="\n"> + self._actions = actions + action = self._actions.get(ruleIndex, None) + if action is not None: + action(localctx, actionIndex) + else: + raise Exception("No registered action for:" + str(ruleIndex)) <actionFuncs.values; separator="\n"> @@ -206,8 +208,8 @@ def sempred(self, localctx:RuleContext, ruleIndex:int, predIndex:int): >> parser_ctor(p) ::= << -def __init__(self, input:TokenStream): - super().__init__(input) +def __init__(self, input:TokenStream, output:TextIO = sys.stdout): + super().__init__(input, output) self.checkVersion("<file.ANTLRVersion>") self._interp = ParserATNSimulator(self, self.atn, self.decisionsToDFA, self.sharedContextCache) self._predicates = None @@ -342,7 +344,7 @@ token = self._input.LA(1) <alt> pass}; separator="\nel"> else: - <error> + pass >> LL1OptionalBlockSingleAlt(choice, expr, alts, preamble, error, followExpr) ::= << @@ -525,7 +527,7 @@ if not <chunks>: ExceptionClause(e, catchArg, catchAction) ::= << catch (<catchArg>) { - <catchAction> + <catchAction> } >> @@ -630,7 +632,7 @@ def <r.name>(self, i:int=None): LexerRuleContext() ::= "RuleContext" /** The rule context name is the rule followed by a suffix; e.g., - * r becomes rContext. + * r becomes rContext. */ RuleContextNameSuffix() ::= "Context" @@ -745,6 +747,8 @@ LexerFile(lexerFile, lexer, namedActions) ::= << <fileHeader(lexerFile.grammarFileName, lexerFile.ANTLRVersion)> from antlr4 import * from io import StringIO +from typing.io import TextIO +import sys <namedActions.header> @@ -761,10 +765,18 @@ class <lexer.name>(<if(superClass)><superClass><else>Lexer<endif>): decisionsToDFA = [ DFA(ds, i) for i, ds in enumerate(atn.decisionToState) ] +<if(lexer.channels)> + <lexer.channels:{c| <c> = <lexer.channels.(c)>}; separator="\n"> + +<endif> +<if(rest(lexer.modes))> <rest(lexer.modes):{m| <m> = <i>}; separator="\n"> +<endif> <lexer.tokens:{k | <k> = <lexer.tokens.(k)>}; separator="\n", wrap, anchor> + channelNames = [ u"DEFAULT_TOKEN_CHANNEL", u"HIDDEN"<if (lexer.channels)>, <lexer.channels:{c| u"<c>"}; separator=", ", wrap, anchor><endif> ] + modeNames = [ <lexer.modes:{m| "<m>"}; separator=", ", wrap, anchor> ] literalNames = [ "\<INVALID>", @@ -777,8 +789,8 @@ class <lexer.name>(<if(superClass)><superClass><else>Lexer<endif>): grammarFileName = "<lexer.grammarFileName>" - def __init__(self, input=None): - super().__init__(input) + def __init__(self, input=None, output:TextIO = sys.stdout): + super().__init__(input, output) self.checkVersion("<lexerFile.ANTLRVersion>") self._interp = LexerATNSimulator(self, self.atn, self.decisionsToDFA, PredictionContextCache()) self._actions = None @@ -801,10 +813,10 @@ def serializedATN(): >> /** Using a type to init value map, try to init a type; if not in table - * must be an object, default value is "null". + * must be an object, default value is "null". */ initValue(typeName) ::= << -<javaTypeInitMap.(typeName)> +<pythonTypeInitMap.(typeName)> >> codeFileExtension() ::= ".py" diff --git a/tool/resources/org/antlr/v4/tool/templates/codegen/Swift/Swift.stg b/tool/resources/org/antlr/v4/tool/templates/codegen/Swift/Swift.stg index 3fcdf38..088cf0c 100755 --- a/tool/resources/org/antlr/v4/tool/templates/codegen/Swift/Swift.stg +++ b/tool/resources/org/antlr/v4/tool/templates/codegen/Swift/Swift.stg @@ -48,6 +48,9 @@ SwiftTypeMap ::= [ ] // args must be <object-model-object>, <fields-resulting-in-STs> +accessLevelOpenOK(obj) ::= "<obj.accessLevel; null=\"open\">" +accessLevelNotOpen(obj) ::= "<obj.accessLevel; null=\"public\">" + ParserFile(file, parser, namedActions,contextSuperClass) ::= << <fileHeader(file.grammarFileName, file.ANTLRVersion)> <if(file.genPackage)> @@ -71,7 +74,7 @@ import Antlr4 * This interface defines a complete listener for a parse tree produced by * {@link <file.parserName>}. */ -public protocol <file.grammarName>Listener: ParseTreeListener { +<accessLevelNotOpen(file)> protocol <file.grammarName>Listener: ParseTreeListener { <file.listenerNames:{lname | /** <if(file.listenerLabelRuleNames.(lname))> @@ -113,8 +116,8 @@ import Antlr4 * which can be extended to create a listener which only needs to handle a subset * of the available methods. */ -open class <file.grammarName>BaseListener: <file.grammarName>Listener { - public init() { \} +<accessLevelOpenOK(file)> class <file.grammarName>BaseListener: <file.grammarName>Listener { + <accessLevelNotOpen(file)> init() { \} <file.listenerNames:{lname | /** @@ -122,38 +125,38 @@ open class <file.grammarName>BaseListener: <file.grammarName>Listener { * * \<p>The default implementation does nothing.\</p> */ -open func enter<lname; format="cap">(_ ctx: <file.parserName>.<lname; format="cap">Context) { \} +<accessLevelOpenOK(file)> func enter<lname; format="cap">(_ ctx: <file.parserName>.<lname; format="cap">Context) { \} /** * {@inheritDoc\} * * \<p>The default implementation does nothing.\</p> */ -open func exit<lname; format="cap">(_ ctx: <file.parserName>.<lname; format="cap">Context) { \}}; separator="\n"> +<accessLevelOpenOK(file)> func exit<lname; format="cap">(_ ctx: <file.parserName>.<lname; format="cap">Context) { \}}; separator="\n"> /** * {@inheritDoc\} * * \<p>The default implementation does nothing.\</p> */ - open func enterEveryRule(_ ctx: ParserRuleContext) { } + <accessLevelOpenOK(file)> func enterEveryRule(_ ctx: ParserRuleContext) { } /** * {@inheritDoc\} * * \<p>The default implementation does nothing.\</p> */ - open func exitEveryRule(_ ctx: ParserRuleContext) { } + <accessLevelOpenOK(file)> func exitEveryRule(_ ctx: ParserRuleContext) { } /** * {@inheritDoc\} * * \<p>The default implementation does nothing.\</p> */ - open func visitTerminal(_ node: TerminalNode) { } + <accessLevelOpenOK(file)> func visitTerminal(_ node: TerminalNode) { } /** * {@inheritDoc\} * * \<p>The default implementation does nothing.\</p> */ - open func visitErrorNode(_ node: ErrorNode) { } + <accessLevelOpenOK(file)> func visitErrorNode(_ node: ErrorNode) { } } >> @@ -172,7 +175,7 @@ import Antlr4 * @param \<T> The return type of the visit operation. Use {@link Void} for * operations with no return type. */ -open class <file.grammarName>Visitor\<T>: ParseTreeVisitor\<T> { +<accessLevelOpenOK(file)> class <file.grammarName>Visitor\<T>: ParseTreeVisitor\<T> { <file.visitorNames:{lname | /** <if(file.visitorLabelRuleNames.(lname))> @@ -185,7 +188,7 @@ open class <file.grammarName>Visitor\<T>: ParseTreeVisitor\<T> { - ctx: the parse tree - returns: the visitor result */ -open func visit<lname; format="cap">(_ ctx: <file.parserName>.<lname; format="cap">Context) -> T{ +<accessLevelOpenOK(file)> func visit<lname; format="cap">(_ ctx: <file.parserName>.<lname; format="cap">Context) -> T { fatalError(#function + " must be overridden") \} }; separator="\n"> @@ -208,7 +211,7 @@ import Antlr4 * @param \<T> The return type of the visit operation. Use {@link Void} for * operations with no return type. */ -open class <file.grammarName>BaseVisitor\<T>: AbstractParseTreeVisitor\<T> { +<accessLevelOpenOK(file)> class <file.grammarName>BaseVisitor\<T>: AbstractParseTreeVisitor\<T> { <file.visitorNames:{lname | /** * {@inheritDoc\} @@ -216,7 +219,7 @@ open class <file.grammarName>BaseVisitor\<T>: AbstractParseTreeVisitor\<T> { * \<p>The default implementation returns the result of calling * {@link #visitChildren\} on {@code ctx\}.\</p> */ -open func visit<lname; format="cap">(_ ctx: <file.parserName>.<lname; format="cap">Context) -> T? { return visitChildren(ctx) \}}; separator="\n"> +<accessLevelOpenOK(file)> func visit<lname; format="cap">(_ ctx: <file.parserName>.<lname; format="cap">Context) -> T? { return visitChildren(ctx) \}}; separator="\n"> } >> @@ -230,7 +233,7 @@ Parser(parser, funcs, atn, sempredFuncs, superClass) ::= << Parser_(parser, funcs, atn, sempredFuncs, ctor, superClass) ::= << <!//@SuppressWarnings({"all", "warnings", "unchecked", "unused", "cast"})!> -open class <parser.name>: <superClass; null="Parser"> { +<accessLevelOpenOK(parser)> class <parser.name>: <superClass; null="Parser"> { internal static var _decisionToDFA: [DFA] = { var decisionToDFA = [DFA]() @@ -241,38 +244,46 @@ open class <parser.name>: <superClass; null="Parser"> { } return decisionToDFA }() - internal static let _sharedContextCache: PredictionContextCache = PredictionContextCache() + + internal static let _sharedContextCache = PredictionContextCache() + <if(parser.tokens)> - public enum Tokens: Int { + <accessLevelNotOpen(parser)> + enum Tokens: Int { case EOF = -1, <parser.tokens:{k | <k> = <parser.tokens.(k)>}; separator=", ", wrap, anchor> } <endif> - public static let <parser.rules:{r | RULE_<r.name> = <r.index>}; separator=", ", wrap, anchor> - public static let ruleNames: [String] = [ + + <accessLevelNotOpen(parser)> + static let <parser.rules:{r | RULE_<r.name> = <r.index>}; separator=", ", wrap, anchor> + + <accessLevelNotOpen(parser)> + static let ruleNames: [String] = [ <parser.ruleNames:{r | "<r>"}; separator=", ", wrap, anchor> ] - <vocabulary(parser.literalNames, parser.symbolicNames)> + <vocabulary(parser.literalNames, parser.symbolicNames, + accessLevelNotOpen(parser))> - override - open func getGrammarFileName() -> String { return "<parser.grammarFileName; format="java-escape">" } + override <accessLevelOpenOK(parser)> + func getGrammarFileName() -> String { return "<parser.grammarFileName; format="java-escape">" } - override - open func getRuleNames() -> [String] { return <parser.name>.ruleNames } + override <accessLevelOpenOK(parser)> + func getRuleNames() -> [String] { return <parser.name>.ruleNames } - override - open func getSerializedATN() -> String { return <parser.name>._serializedATN } + override <accessLevelOpenOK(parser)> + func getSerializedATN() -> String { return <parser.name>._serializedATN } - override - open func getATN() -> ATN { return <parser.name>._ATN } + override <accessLevelOpenOK(parser)> + func getATN() -> ATN { return <parser.name>._ATN } <namedActions.members> <parser:(ctor)()> <funcs; separator="\n"> <if(sempredFuncs)> - override - open func sempred(_ _localctx: RuleContext?, _ ruleIndex: Int, _ predIndex: Int)throws -> Bool { + override <accessLevelOpenOK(parser)> + func sempred(_ _localctx: RuleContext?, _ ruleIndex: Int, _ predIndex: Int)throws -> Bool { switch (ruleIndex) { <parser.sempredFuncs.values:{f| case <f.ruleIndex>: @@ -285,51 +296,30 @@ case <f.ruleIndex>: <endif> <atn> - public static let _serializedATN : String = <parser.name>ATN().jsonString - public static let _ATN: ATN = ATNDeserializer().deserializeFromJson(_serializedATN) + + <accessLevelNotOpen(parser)> + static let _serializedATN = <parser.name>ATN().jsonString + + <accessLevelNotOpen(parser)> + static let _ATN = ATNDeserializer().deserializeFromJson(_serializedATN) } >> -vocabulary(literalNames, symbolicNames) ::= << +vocabulary(literalNames, symbolicNames, accessLevel) ::= << private static let _LITERAL_NAMES: [String?] = [ <literalNames:{t | <t>}; null="nil", separator=", ", wrap, anchor> ] private static let _SYMBOLIC_NAMES: [String?] = [ <symbolicNames:{t | <t>}; null="nil", separator=", ", wrap, anchor> ] -public static let VOCABULARY: Vocabulary = Vocabulary(_LITERAL_NAMES, _SYMBOLIC_NAMES) - -/** - * @deprecated Use {@link #VOCABULARY} instead. - */ -//@Deprecated -public let tokenNames: [String?]? = { - let length = _SYMBOLIC_NAMES.count - var tokenNames = [String?](repeating: nil, count: length) - for i in 0..\<length { - var name = VOCABULARY.getLiteralName(i) - if name == nil { - name = VOCABULARY.getSymbolicName(i) - } - if name == nil { - name = "\<INVALID>" - } - tokenNames[i] = name - } - return tokenNames -}() - -override -<!//@Deprecated!> -open func getTokenNames() -> [String?]? { - return tokenNames -} +<accessLevel> +static let VOCABULARY = Vocabulary(_LITERAL_NAMES, _SYMBOLIC_NAMES) >> dumpActions(recog, argFuncs, actionFuncs, sempredFuncs) ::= << <if(actionFuncs)> -override -open func action(_ _localctx: RuleContext?, _ ruleIndex: Int, _ actionIndex: Int) throws { +override <accessLevelOpenOK(parser)> +func action(_ _localctx: RuleContext?, _ ruleIndex: Int, _ actionIndex: Int) throws { switch (ruleIndex) { <recog.actionFuncs.values:{f| case <f.ruleIndex>: @@ -341,8 +331,8 @@ case <f.ruleIndex>: <actionFuncs.values; separator="\n"> <endif> <if(sempredFuncs)> -override -open func sempred(_ _localctx: RuleContext?, _ ruleIndex: Int,_ predIndex: Int) throws -> Bool{ +override <accessLevelOpenOK(parser)> +func sempred(_ _localctx: RuleContext?, _ ruleIndex: Int,_ predIndex: Int) throws -> Bool { switch (ruleIndex) { <recog.sempredFuncs.values:{f| case <f.ruleIndex>: @@ -357,15 +347,18 @@ case <f.ruleIndex>: parser_ctor(p) ::= << -open override func getVocabulary() -> Vocabulary { +override <accessLevelOpenOK(parser)> +func getVocabulary() -> Vocabulary { return <p.name>.VOCABULARY } -public override init(_ input:TokenStream)throws { - RuntimeMetaData.checkVersion("4.6", RuntimeMetaData.VERSION) +override <accessLevelNotOpen(parser)> +init(_ input:TokenStream) throws { + RuntimeMetaData.checkVersion("4.7.1", RuntimeMetaData.VERSION) try super.init(input) _interp = ParserATNSimulator(self,<p.name>._ATN,<p.name>._decisionToDFA, <parser.name>._sharedContextCache) } + >> /* This generates a private method since the actionIndex is generated, making an @@ -401,7 +394,7 @@ RuleFunction(currentRule,args,code,locals,ruleCtx,altLabelCtxs,namedActions,fina <ruleCtx> <altLabelCtxs:{l | <altLabelCtxs.(l)>}; separator="\n"> @discardableResult -<if(currentRule.modifiers)><currentRule.modifiers:{f | <f> }><else>open func <endif><currentRule.name>(<if(first(args))>_ <endif><args; separator=", _">) throws -> <currentRule.ctxType> { +<if(currentRule.modifiers)><currentRule.modifiers:{f | <f> }><else> <accessLevelOpenOK(parser)> func <endif><currentRule.name>(<if(first(args))>_ <endif><args; separator=", _">) throws -> <currentRule.ctxType> { var _localctx: <currentRule.ctxType> = <currentRule.ctxType>(_ctx, getState()<currentRule.args:{a | , <a.name>}>) try enterRule(_localctx, <currentRule.startState>, <parser.name>.RULE_<currentRule.name>) <namedActions.init> @@ -439,7 +432,7 @@ LeftRecursiveRuleFunction(currentRule,args,code,locals,ruleCtx,altLabelCtxs, <ruleCtx> <altLabelCtxs:{l | <altLabelCtxs.(l)>}; separator="\n"> -<if(currentRule.modifiers)><currentRule.modifiers:{f | <f> }><else>public final <endif> func <currentRule.name>( <if(first(args))>_ <endif><args; separator=", _">) throws -> <currentRule.ctxType> { +<if(currentRule.modifiers)><currentRule.modifiers:{f | <f> }><else> <accessLevelNotOpen(parser)> final <endif> func <currentRule.name>( <if(first(args))>_ <endif><args; separator=", _">) throws -> <currentRule.ctxType> { return try <currentRule.name>(0<currentRule.args:{a | , <a.name>}>) } @discardableResult @@ -510,7 +503,7 @@ switch (<parser.name>.Tokens(rawValue: try _input.LA(1))!) { <alt> break}; separator="\n"> default: - <error> + break } >> @@ -612,7 +605,7 @@ case <i><if(!choice.ast.greedy)>+1<endif>: Sync(s) ::= "sync(<s.expecting.name>);" -ThrowNoViableAlt(t) ::= "throw try ANTLRException.recognition(e: NoViableAltException(self))" +ThrowNoViableAlt(t) ::= "throw ANTLRException.recognition(e: NoViableAltException(self))" TestSetInline(s) ::= << <!<s.bitsets:{bits | <if(rest(rest(bits.ttypes)))><bitsetBitfieldComparison(s, bits)><else><bitsetInlineComparison(s, bits)><endif>}; separator=" || ">!> @@ -717,7 +710,7 @@ ArgAction(a, chunks) ::= "<chunks>" SemPred(p, chunks, failChunks) ::= << setState(<p.stateNumber>) if (!(<chunks>)) { - throw try ANTLRException.recognition(e:FailedPredicateException(self, <p.predicate><if(failChunks)>, <failChunks><elseif(p.msg)>, <p.msg><endif>)) + throw ANTLRException.recognition(e:FailedPredicateException(self, <p.predicate><if(failChunks)>, <failChunks><elseif(p.msg)>, <p.msg><endif>)) } >> @@ -781,33 +774,45 @@ AddToLabelList(a) ::= "<ctx(a.label)>.<a.listName>.append(<labelref(a.label)>)" TokenDecl(t) ::= "<t.name>: <SwiftTypeMap.(TokenLabelType())>!" TokenTypeDecl(t) ::= "var <t.name>: Int = 0" -TokenListDecl(t) ::= "<t.name>: Array\<Token> = Array\<Token>()" +TokenListDecl(t) ::= "<t.name>: [Token] = [Token]()" RuleContextDecl(r) ::= "<r.name>: <r.ctxName>!" -RuleContextListDecl(rdecl) ::= "<rdecl.name>:Array\<<rdecl.ctxName>> = Array\<<rdecl.ctxName>>()" - -ContextTokenGetterDecl(t) ::= - "open func <t.name>() -> TerminalNode? { return getToken(<parser.name>.Tokens.<t.name>.rawValue, 0) }" -ContextTokenListGetterDecl(t) ::= - "open func <t.name>() -> Array\<TerminalNode> { return getTokens(<parser.name>.Tokens.<t.name>.rawValue) }" -ContextTokenListIndexedGetterDecl(t) ::= << -open func <t.name>(_ i:Int) -> TerminalNode?{ - return getToken(<parser.name>.Tokens.<t.name>.rawValue, i) -} +RuleContextListDecl(rdecl) ::= "<rdecl.name>: [<rdecl.ctxName>] = [<rdecl.ctxName>]()" + +ContextTokenGetterDecl(t) ::= << + <accessLevelOpenOK(parser)> + func <t.name>() -> TerminalNode? { + return getToken(<parser.name>.Tokens.<t.name>.rawValue, 0) + } >> -ContextRuleGetterDecl(r) ::= << -open func <r.name>() -> <r.ctxName>? { - return getRuleContext(<r.ctxName>.self,0) -} +ContextTokenListGetterDecl(t) ::= << + <accessLevelOpenOK(parser)> + func <t.name>() -> [TerminalNode] { + return getTokens(<parser.name>.Tokens.<t.name>.rawValue) + } >> -ContextRuleListGetterDecl(r) ::= << -open func <r.name>() -> Array\<<r.ctxName>\> { - return getRuleContexts(<r.ctxName>.self) -} +ContextTokenListIndexedGetterDecl(t) ::= << + <accessLevelOpenOK(parser)> + func <t.name>(_ i:Int) -> TerminalNode? { + return getToken(<parser.name>.Tokens.<t.name>.rawValue, i) + } +>> +ContextRuleGetterDecl(r) ::= << + <accessLevelOpenOK(parser)> + func <r.name>() -> <r.ctxName>? { + return getRuleContext(<r.ctxName>.self, 0) + } +>> +ContextRuleListGetterDecl(r) ::= << + <accessLevelOpenOK(parser)> + func <r.name>() -> [<r.ctxName>] { + return getRuleContexts(<r.ctxName>.self) + } >> ContextRuleListIndexedGetterDecl(r) ::= << -open func <r.name>(_ i: Int) -> <r.ctxName>? { - return getRuleContext(<r.ctxName>.self,i) -} + <accessLevelOpenOK(parser)> + func <r.name>(_ i: Int) -> <r.ctxName>? { + return getRuleContext(<r.ctxName>.self, i) + } >> LexerRuleContext() ::= "RuleContext" @@ -827,20 +832,26 @@ CaptureNextTokenType(d) ::= "<d.varName> = try _input.LA(1)" StructDecl(struct,ctorAttrs,attrs,getters,dispatchMethods,interfaces,extensionMembers, superClass={ParserRuleContext}) ::= << -open class <struct.name>:<if(contextSuperClass)><contextSuperClass><else>ParserRuleContext<endif><if(interfaces)>, <interfaces; separator=", "><endif> { - <attrs:{a | public var <a>}; separator="\n"> + +<accessLevelNotOpen(parser)> class <struct.name>: <if(contextSuperClass)><contextSuperClass><else>ParserRuleContext<endif><if(interfaces)>, <interfaces; separator=", "><endif> { + <attrs:{a | <accessLevelOpenOK(parser)> var <a>}; separator="\n"> <getters:{g | <g>}; separator="\n"> - <! <if(ctorAttrs)>public init(_ parent: ParserRuleContext,_ invokingState: Int) { super.init(parent, invokingState) }<endif> !> - <if(ctorAttrs)> - public convenience init(_ parent: ParserRuleContext?, _ invokingState: Int<ctorAttrs:{a | , _ <a>}>) { + <! <if(ctorAttrs)> <accessLevelNotOpen(parser)> init(_ parent: ParserRuleContext,_ invokingState: Int) { super.init(parent, invokingState) }<endif> !> + <if(ctorAttrs)> + <accessLevelNotOpen(parser)> convenience init(_ parent: ParserRuleContext?, _ invokingState: Int<ctorAttrs:{a | , _ <a>}>) { self.init(parent, invokingState) <struct.ctorAttrs:{a | self.<a.name> = <a.name>;}; separator="\n"> - } + } <endif> - open override func getRuleIndex() -> Int { return <parser.name>.RULE_<struct.derivedFromName> } + + override <accessLevelOpenOK(parser)> + func getRuleIndex() -> Int { + return <parser.name>.RULE_<struct.derivedFromName> + } <if(struct.provideCopyFrom)> <! don't need copy unless we have subclasses !> - <!public init() { }!> - public func copyFrom(_ ctx: <struct.name>) { + <! <accessLevelNotOpen(parser)> init() { }!> + <accessLevelOpenOK(parser)> + func copyFrom(_ ctx: <struct.name>) { super.copyFrom(ctx) <struct.attrs:{a | self.<a.name> = ctx.<a.name>;}; separator="\n"> } @@ -851,10 +862,12 @@ open class <struct.name>:<if(contextSuperClass)><contextSuperClass><else>ParserR >> AltLabelStructDecl(struct,attrs,getters,dispatchMethods) ::= << -public final class <struct.name>: <currentRule.name; format="cap">Context { - <attrs:{a | public var <a>}; separator="\n"> +<accessLevelNotOpen(parser)> class <struct.name>: <currentRule.name; format="cap">Context { + <attrs:{a | <accessLevelNotOpen(parser)> var <a>}; separator="\n"> <getters:{g | <g>}; separator="\n"> - public init(_ ctx: <currentRule.name; format="cap">Context) { + + <accessLevelNotOpen(parser)> + init(_ ctx: <currentRule.name; format="cap">Context) { super.init() copyFrom(ctx) } @@ -863,22 +876,23 @@ public final class <struct.name>: <currentRule.name; format="cap">Context { >> ListenerDispatchMethod(method) ::= << -override -open func <if(method.isEnter)>enter<else>exit<endif>Rule(_ listener: ParseTreeListener) { - if listener is <parser.grammarName>Listener { - (listener as! <parser.grammarName>Listener).<if(method.isEnter)>enter<else>exit<endif><struct.derivedFromName; format="cap">(self) +override <accessLevelOpenOK(parser)> +func <if(method.isEnter)>enter<else>exit<endif>Rule(_ listener: ParseTreeListener) { + if let listener = listener as? <parser.grammarName>Listener { + listener.<if(method.isEnter)>enter<else>exit<endif><struct.derivedFromName; format="cap">(self) } } >> VisitorDispatchMethod(method) ::= << -override -open func accept\<T>(_ visitor: ParseTreeVisitor\<T>) -> T? { - if visitor is <parser.grammarName>Visitor { - return (visitor as! <parser.grammarName>Visitor\<T>).visit<struct.derivedFromName; format="cap">(self) - }else if visitor is <parser.grammarName>BaseVisitor { - return (visitor as! <parser.grammarName>BaseVisitor\<T>).visit<struct.derivedFromName; format="cap">(self) - } +override <accessLevelOpenOK(parser)> +func accept\<T>(_ visitor: ParseTreeVisitor\<T>) -> T? { + if let visitor = visitor as? <parser.grammarName>Visitor { + return visitor.visit<struct.derivedFromName; format="cap">(self) + } + else if let visitor = visitor as? <parser.grammarName>BaseVisitor { + return visitor.visit<struct.derivedFromName; format="cap">(self) + } else { return visitor.visitChildren(self) } @@ -948,7 +962,8 @@ import Antlr4 <lexer> >> Lexer(lexer, atn, actionFuncs, sempredFuncs, superClass) ::= << -open class <lexer.name>: <superClass; null="Lexer"> { +<accessLevelOpenOK(lexer)> class <lexer.name>: <superClass; null="Lexer"> { + internal static var _decisionToDFA: [DFA] = { var decisionToDFA = [DFA]() let length = <lexer.name>._ATN.getNumberOfDecisions() @@ -959,52 +974,77 @@ open class <lexer.name>: <superClass; null="Lexer"> { return decisionToDFA }() - internal static let _sharedContextCache:PredictionContextCache = PredictionContextCache() - public static let <lexer.tokens:{k | <k>=<lexer.tokens.(k)>}; separator=", ", wrap, anchor> + internal static let _sharedContextCache = PredictionContextCache() + + <accessLevelNotOpen(lexer)> + static let <lexer.tokens:{k | <k>=<lexer.tokens.(k)>}; separator=", ", wrap, anchor> + <if(lexer.channels)> - public let <lexer.channels:{k | <k>=<lexer.channels.(k)>}; separator=", ", wrap, anchor> + <accessLevelNotOpen(lexer)> + static let <lexer.channels:{k | <k>=<lexer.channels.(k)>}; separator=", ", wrap, anchor> <endif> - <rest(lexer.modes):{m| public static let <m>: Int = <i>;}; separator="\n"> - public static let modeNames: [String] = [ + <if(rest(lexer.modes))> + <accessLevelNotOpen(lexer)> + static let <rest(lexer.modes):{m| <m>=<i>}; separator=", ", wrap, anchor> + <endif> + <accessLevelNotOpen(lexer)> + static let channelNames: [String] = [ + "DEFAULT_TOKEN_CHANNEL", "HIDDEN"<if (lexer.channels)>, <lexer.channels:{c| "<c>"}; separator=", ", wrap, anchor><endif> + ] + + <accessLevelNotOpen(lexer)> + static let modeNames: [String] = [ <lexer.modes:{m| "<m>"}; separator=", ", wrap, anchor> ] - public static let ruleNames: [String] = [ + <accessLevelNotOpen(lexer)> + static let ruleNames: [String] = [ <lexer.ruleNames:{r | "<r>"}; separator=", ", wrap, anchor> ] - <vocabulary(lexer.literalNames, lexer.symbolicNames)> + <vocabulary(lexer.literalNames, lexer.symbolicNames, + accessLevelNotOpen(lexer))> <namedActions.members> - open override func getVocabulary() -> Vocabulary { - return <lexer.name>.VOCABULARY - } - public override init(_ input: CharStream) { + override <accessLevelOpenOK(lexer)> + func getVocabulary() -> Vocabulary { + return <lexer.name>.VOCABULARY + } + + <accessLevelNotOpen(lexer)> + required init(_ input: CharStream) { RuntimeMetaData.checkVersion("<lexerFile.ANTLRVersion>", RuntimeMetaData.VERSION) super.init(input) _interp = LexerATNSimulator(self, <lexer.name>._ATN, <lexer.name>._decisionToDFA, <lexer.name>._sharedContextCache) } - override - open func getGrammarFileName() -> String { return "<lexer.grammarFileName>" } + override <accessLevelOpenOK(lexer)> + func getGrammarFileName() -> String { return "<lexer.grammarFileName>" } - override - open func getRuleNames() -> [String] { return <lexer.name>.ruleNames } + override <accessLevelOpenOK(lexer)> + func getRuleNames() -> [String] { return <lexer.name>.ruleNames } - override - open func getSerializedATN() -> String { return <lexer.name>._serializedATN } + override <accessLevelOpenOK(lexer)> + func getSerializedATN() -> String { return <lexer.name>._serializedATN } - override - open func getModeNames() -> [String] { return <lexer.name>.modeNames } + override <accessLevelOpenOK(lexer)> + func getChannelNames() -> [String] { return <lexer.name>.channelNames } - override - open func getATN() -> ATN { return <lexer.name>._ATN } + override <accessLevelOpenOK(lexer)> + func getModeNames() -> [String] { return <lexer.name>.modeNames } + + override <accessLevelOpenOK(lexer)> + func getATN() -> ATN { return <lexer.name>._ATN } <dumpActions(lexer, "", actionFuncs, sempredFuncs)> <atn> - public static let _serializedATN: String = <lexer.name>ATN().jsonString - public static let _ATN: ATN = ATNDeserializer().deserializeFromJson(_serializedATN) + + <accessLevelNotOpen(lexer)> + static let _serializedATN: String = <lexer.name>ATN().jsonString + + <accessLevelNotOpen(lexer)> + static let _ATN: ATN = ATNDeserializer().deserializeFromJson(_serializedATN) } >> diff --git a/tool/resources/org/antlr/v4/tool/templates/messages/formats/gnu.stg b/tool/resources/org/antlr/v4/tool/templates/messages/formats/gnu.stg index f243ab7..c7d06d9 100644 --- a/tool/resources/org/antlr/v4/tool/templates/messages/formats/gnu.stg +++ b/tool/resources/org/antlr/v4/tool/templates/messages/formats/gnu.stg @@ -31,9 +31,9 @@ This file contains the actual layout of the messages emitted by ANTLR. This file contains the format that mimicks GCC output. */ -location(file, line, column) ::= "<file>:<line>:" +location(file, line, column) ::= "<file>:<line>:<column>:" -message(id, text) ::= "<text> (<id>)" +message(id, text) ::= "<text> [error <id>]" report(location, message, type) ::= "<location> <type>: <message>" diff --git a/tool/resources/org/antlr/v4/tool/templates/unicodedata.st b/tool/resources/org/antlr/v4/tool/templates/unicodedata.st new file mode 100644 index 0000000..0f22c73 --- /dev/null +++ b/tool/resources/org/antlr/v4/tool/templates/unicodedata.st @@ -0,0 +1,62 @@ +unicodedata(propertyCodePointRanges, propertyAliases) ::= << +package org.antlr.v4.unicode; + +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; +import java.util.Locale; +import java.util.Map; + +import org.antlr.v4.runtime.misc.IntervalSet; +import org.antlr.v4.runtime.misc.Interval; + +/** + * Code-generated utility class mapping Unicode properties to Unicode code point ranges. + */ +public abstract class UnicodeData { + private static final Map\<String, IntervalSet\> propertyCodePointRanges = new HashMap\<\>(<length(propertyCodePointRanges)>); + private static final Map\<String, String\> propertyAliases = new HashMap\<\>(<length(propertyAliases)>); + + // Work around Java 64k bytecode method limit by splitting up static + // initialization into one method per Unicode property + + <propertyCodePointRanges.keys:{ k | // Unicode code points with property "<k>" +static private void addProperty<i>() { + List\<Interval\> intervals = Arrays.asList( + <propertyCodePointRanges.(k).intervals:{ interval | Interval.of(<interval.a>, <interval.b>)}; separator=",\n"> + ); + IntervalSet codePointRanges = new IntervalSet(intervals); + codePointRanges.setReadonly(true); + propertyCodePointRanges.put("<k>".toLowerCase(Locale.US), codePointRanges); +\}}; separator="\n\n"> + + // Property aliases + static private void addPropertyAliases() { + <propertyAliases.keys:{ k | propertyAliases.put("<k>".toLowerCase(Locale.US), "<propertyAliases.(k)>".toLowerCase(Locale.US)); }; separator="\n"> + } + + // Put it all together + static { + <propertyCodePointRanges.keys:{ k | addProperty<i>(); }; separator="\n"> + addPropertyAliases(); + } + + private static String normalize(String propertyCodeOrAlias) { + return propertyCodeOrAlias.toLowerCase(Locale.US).replace('-', '_'); + } + + /** + * Given a Unicode property (general category code, binary property name, or script name), + * returns the {@link IntervalSet} of Unicode code point ranges which have that property. + */ + public static IntervalSet getPropertyCodePoints(String propertyCodeOrAlias) { + String normalizedPropertyCodeOrAlias = normalize(propertyCodeOrAlias); + IntervalSet result = propertyCodePointRanges.get(normalizedPropertyCodeOrAlias); + if (result == null) { + String propertyCode = propertyAliases.get(normalizedPropertyCodeOrAlias); + result = propertyCodePointRanges.get(propertyCode); + } + return result; + } +} +>> diff --git a/tool/src/org/antlr/v4/Tool.java b/tool/src/org/antlr/v4/Tool.java index 93a59ff..86c49e1 100644 --- a/tool/src/org/antlr/v4/Tool.java +++ b/tool/src/org/antlr/v4/Tool.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ @@ -27,6 +27,8 @@ import org.antlr.v4.parse.ToolANTLRParser; import org.antlr.v4.parse.v3TreeGrammarException; import org.antlr.v4.runtime.RuntimeMetaData; import org.antlr.v4.runtime.misc.LogManager; +import org.antlr.v4.runtime.misc.IntegerList; +import org.antlr.v4.runtime.atn.ATNSerializer; import org.antlr.v4.semantics.SemanticPipeline; import org.antlr.v4.tool.ANTLRMessage; import org.antlr.v4.tool.ANTLRToolListener; @@ -115,26 +117,28 @@ public class Tool { public Map<String, String> grammarOptions = null; public boolean warnings_are_errors = false; public boolean longMessages = false; + public boolean exact_output_dir = false; public static Option[] optionDefs = { - new Option("outputDirectory", "-o", OptionArgType.STRING, "specify output directory where all output is generated"), - new Option("libDirectory", "-lib", OptionArgType.STRING, "specify location of grammars, tokens files"), - new Option("generate_ATN_dot", "-atn", "generate rule augmented transition network diagrams"), - new Option("grammarEncoding", "-encoding", OptionArgType.STRING, "specify grammar file encoding; e.g., euc-jp"), - new Option("msgFormat", "-message-format", OptionArgType.STRING, "specify output style for messages in antlr, gnu, vs2005"), - new Option("longMessages", "-long-messages", "show exception details when available for errors and warnings"), - new Option("gen_listener", "-listener", "generate parse tree listener (default)"), - new Option("gen_listener", "-no-listener", "don't generate parse tree listener"), - new Option("gen_visitor", "-visitor", "generate parse tree visitor"), - new Option("gen_visitor", "-no-visitor", "don't generate parse tree visitor (default)"), - new Option("genPackage", "-package", OptionArgType.STRING, "specify a package/namespace for the generated code"), - new Option("gen_dependencies", "-depend", "generate file dependencies"), - new Option("", "-D<option>=value", "set/override a grammar-level option"), - new Option("warnings_are_errors", "-Werror", "treat warnings as errors"), - new Option("launch_ST_inspector", "-XdbgST", "launch StringTemplate visualizer on generated code"), + new Option("outputDirectory", "-o", OptionArgType.STRING, "specify output directory where all output is generated"), + new Option("libDirectory", "-lib", OptionArgType.STRING, "specify location of grammars, tokens files"), + new Option("generate_ATN_dot", "-atn", "generate rule augmented transition network diagrams"), + new Option("grammarEncoding", "-encoding", OptionArgType.STRING, "specify grammar file encoding; e.g., euc-jp"), + new Option("msgFormat", "-message-format", OptionArgType.STRING, "specify output style for messages in antlr, gnu, vs2005"), + new Option("longMessages", "-long-messages", "show exception details when available for errors and warnings"), + new Option("gen_listener", "-listener", "generate parse tree listener (default)"), + new Option("gen_listener", "-no-listener", "don't generate parse tree listener"), + new Option("gen_visitor", "-visitor", "generate parse tree visitor"), + new Option("gen_visitor", "-no-visitor", "don't generate parse tree visitor (default)"), + new Option("genPackage", "-package", OptionArgType.STRING, "specify a package/namespace for the generated code"), + new Option("gen_dependencies", "-depend", "generate file dependencies"), + new Option("", "-D<option>=value", "set/override a grammar-level option"), + new Option("warnings_are_errors", "-Werror", "treat warnings as errors"), + new Option("launch_ST_inspector", "-XdbgST", "launch StringTemplate visualizer on generated code"), new Option("ST_inspector_wait_for_close", "-XdbgSTWait", "wait for STViz to close before continuing"), - new Option("force_atn", "-Xforce-atn", "use the ATN simulator for all predictions"), - new Option("log", "-Xlog", "dump lots of logging info to antlr-timestamp.log"), + new Option("force_atn", "-Xforce-atn", "use the ATN simulator for all predictions"), + new Option("log", "-Xlog", "dump lots of logging info to antlr-timestamp.log"), + new Option("exact_output_dir", "-Xexact-output-dir", "all output goes into -o dir regardless of paths/package"), }; // helper vars for option management @@ -191,8 +195,11 @@ public class Tool { public Tool(String[] args) { this.args = args; errMgr = new ErrorManager(this); - errMgr.setFormat(msgFormat); + // We have to use the default message format until we have + // parsed the -message-format command line option. + errMgr.setFormat("antlr"); handleArgs(); + errMgr.setFormat(msgFormat); } protected void handleArgs() { @@ -246,7 +253,7 @@ public class Tool { haveOutputDir = true; if (outDir.exists() && !outDir.isDirectory()) { errMgr.toolError(ErrorType.OUTPUT_DIR_IS_FILE, outputDirectory); - libDirectory = "."; + outputDirectory = "."; } } else { @@ -390,6 +397,8 @@ public class Tool { if ( generate_ATN_dot ) generateATNs(g); + if ( g.tool.getNumErrors()==0 ) generateInterpreterData(g); + // PERFORM GRAMMAR ANALYSIS ON ATN: BUILD DECISION DFAs AnalysisPipeline anal = new AnalysisPipeline(g); anal.process(); @@ -693,6 +702,64 @@ public class Tool { } } + private void generateInterpreterData(Grammar g) { + StringBuilder content = new StringBuilder(); + + content.append("token literal names:\n"); + String[] names = g.getTokenLiteralNames(); + for (String name : names) { + content.append(name + "\n"); + } + content.append("\n"); + + content.append("token symbolic names:\n"); + names = g.getTokenSymbolicNames(); + for (String name : names) { + content.append(name + "\n"); + } + content.append("\n"); + + content.append("rule names:\n"); + names = g.getRuleNames(); + for (String name : names) { + content.append(name + "\n"); + } + content.append("\n"); + + if ( g.isLexer() ) { + content.append("channel names:\n"); + content.append("DEFAULT_TOKEN_CHANNEL\n"); + content.append("HIDDEN\n"); + for (String channel : g.channelValueToNameList) { + content.append(channel + "\n"); + } + content.append("\n"); + + content.append("mode names:\n"); + for (String mode : ((LexerGrammar)g).modes.keySet()) { + content.append(mode + "\n"); + } + } + content.append("\n"); + + IntegerList serializedATN = ATNSerializer.getSerialized(g.atn); + content.append("atn:\n"); + content.append(serializedATN.toString()); + + try { + Writer fw = getOutputFileWriter(g, g.name + ".interp"); + try { + fw.write(content.toString()); + } + finally { + fw.close(); + } + } + catch (IOException ioe) { + errMgr.toolError(ErrorType.CANNOT_WRITE_FILE, ioe); + } + } + /** This method is used by all code generators to create new output * files. If the outputDir set by -o is not present it will be created. * The final filename is sensitive to the output directory and @@ -758,6 +825,10 @@ public class Tool { * @param fileNameWithPath path to input source */ public File getOutputDirectory(String fileNameWithPath) { + if ( exact_output_dir ) { + return new_getOutputDirectory(fileNameWithPath); + } + File outputDir; String fileDirectory; @@ -783,12 +854,12 @@ public class Tool { // -o . /usr/lib/t.g4 => ./T.java if (fileDirectory != null && (new File(fileDirectory).isAbsolute() || - fileDirectory.startsWith("~"))) { // isAbsolute doesn't count this :( + fileDirectory.startsWith("~"))) { // isAbsolute doesn't count this :( // somebody set the dir, it takes precendence; write new file there outputDir = new File(outputDirectory); } else { - // -o /tmp subdir/t.g4 => /tmp/subdir/t.g4 + // -o /tmp subdir/t.g4 => /tmp/subdir/T.java if (fileDirectory != null) { outputDir = new File(outputDirectory, fileDirectory); } @@ -807,6 +878,37 @@ public class Tool { return outputDir; } + /** @since 4.7.1 in response to -Xexact-output-dir */ + public File new_getOutputDirectory(String fileNameWithPath) { + File outputDir; + String fileDirectory; + + if (fileNameWithPath.lastIndexOf(File.separatorChar) == -1) { + // No path is included in the file name, so make the file + // directory the same as the parent grammar (which might still be just "" + // but when it is not, we will write the file in the correct place. + fileDirectory = "."; + } + else { + fileDirectory = fileNameWithPath.substring(0, fileNameWithPath.lastIndexOf(File.separatorChar)); + } + if ( haveOutputDir ) { + // -o /tmp /var/lib/t.g4 => /tmp/T.java + // -o subdir/output /usr/lib/t.g4 => subdir/output/T.java + // -o . /usr/lib/t.g4 => ./T.java + // -o /tmp subdir/t.g4 => /tmp/T.java + outputDir = new File(outputDirectory); + } + else { + // they didn't specify a -o dir so just write to location + // where grammar is, absolute or relative, this will only happen + // with command line invocation as build tools will always + // supply an output directory. + outputDir = new File(fileDirectory); + } + return outputDir; + } + protected void writeDOTFile(Grammar g, Rule r, String dot) throws IOException { writeDOTFile(g, r.g.name + "." + r.name, dot); } diff --git a/tool/src/org/antlr/v4/analysis/AnalysisPipeline.java b/tool/src/org/antlr/v4/analysis/AnalysisPipeline.java index 6c7b97a..df27ee7 100644 --- a/tool/src/org/antlr/v4/analysis/AnalysisPipeline.java +++ b/tool/src/org/antlr/v4/analysis/AnalysisPipeline.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/tool/src/org/antlr/v4/analysis/LeftRecursionDetector.java b/tool/src/org/antlr/v4/analysis/LeftRecursionDetector.java index c180083..6a4ac00 100644 --- a/tool/src/org/antlr/v4/analysis/LeftRecursionDetector.java +++ b/tool/src/org/antlr/v4/analysis/LeftRecursionDetector.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/tool/src/org/antlr/v4/analysis/LeftRecursiveRuleAltInfo.java b/tool/src/org/antlr/v4/analysis/LeftRecursiveRuleAltInfo.java index 980fcaf..04f2dcf 100644 --- a/tool/src/org/antlr/v4/analysis/LeftRecursiveRuleAltInfo.java +++ b/tool/src/org/antlr/v4/analysis/LeftRecursiveRuleAltInfo.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/tool/src/org/antlr/v4/analysis/LeftRecursiveRuleAnalyzer.java b/tool/src/org/antlr/v4/analysis/LeftRecursiveRuleAnalyzer.java index a2040e7..ae8a441 100644 --- a/tool/src/org/antlr/v4/analysis/LeftRecursiveRuleAnalyzer.java +++ b/tool/src/org/antlr/v4/analysis/LeftRecursiveRuleAnalyzer.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/tool/src/org/antlr/v4/analysis/LeftRecursiveRuleTransformer.java b/tool/src/org/antlr/v4/analysis/LeftRecursiveRuleTransformer.java index 610c550..dd4dce6 100644 --- a/tool/src/org/antlr/v4/analysis/LeftRecursiveRuleTransformer.java +++ b/tool/src/org/antlr/v4/analysis/LeftRecursiveRuleTransformer.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/tool/src/org/antlr/v4/automata/ATNFactory.java b/tool/src/org/antlr/v4/automata/ATNFactory.java index a7de4ff..c0c0642 100644 --- a/tool/src/org/antlr/v4/automata/ATNFactory.java +++ b/tool/src/org/antlr/v4/automata/ATNFactory.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/tool/src/org/antlr/v4/automata/ATNOptimizer.java b/tool/src/org/antlr/v4/automata/ATNOptimizer.java index 36e5009..6c72016 100644 --- a/tool/src/org/antlr/v4/automata/ATNOptimizer.java +++ b/tool/src/org/antlr/v4/automata/ATNOptimizer.java @@ -1,15 +1,17 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ package org.antlr.v4.automata; +import org.antlr.v4.misc.CharSupport; import org.antlr.v4.runtime.atn.ATN; import org.antlr.v4.runtime.atn.ATNState; import org.antlr.v4.runtime.atn.AtomTransition; import org.antlr.v4.runtime.atn.BlockEndState; +import org.antlr.v4.runtime.atn.CodePointTransitions; import org.antlr.v4.runtime.atn.DecisionState; import org.antlr.v4.runtime.atn.EpsilonTransition; import org.antlr.v4.runtime.atn.NotSetTransition; @@ -96,17 +98,24 @@ public class ATNOptimizer { if (matchTransition instanceof NotSetTransition) { throw new UnsupportedOperationException("Not yet implemented."); } - IntervalSet set = matchTransition.label(); - int minElem = set.getMinElement(); - int maxElem = set.getMaxElement(); - for (int k = minElem; k <= maxElem; k++) { - if (matchSet.contains(k)) { - char setMin = (char) set.getMinElement(); - char setMax = (char) set.getMaxElement(); - // TODO: Token is missing (i.e. position in source will not be displayed). - g.tool.errMgr.grammarError(ErrorType.CHARACTERS_COLLISION_IN_SET, g.fileName, - null, (char) minElem + "-" + (char) maxElem, "[" + setMin + "-" + setMax + "]"); - break; + IntervalSet set = matchTransition.label(); + List<Interval> intervals = set.getIntervals(); + int n = intervals.size(); + for (int k = 0; k < n; k++) { + Interval setInterval = intervals.get(k); + int a = setInterval.a; + int b = setInterval.b; + if (a != -1 && b != -1) { + for (int v = a; v <= b; v++) { + if (matchSet.contains(v)) { + // TODO: Token is missing (i.e. position in source will not be displayed). + g.tool.errMgr.grammarError(ErrorType.CHARACTERS_COLLISION_IN_SET, g.fileName, + null, + CharSupport.getANTLRCharLiteralForChar(v), + CharSupport.getIntervalSetEscapedString(matchSet)); + break; + } + } } } matchSet.addAll(set); @@ -115,11 +124,11 @@ public class ATNOptimizer { Transition newTransition; if (matchSet.getIntervals().size() == 1) { if (matchSet.size() == 1) { - newTransition = new AtomTransition(blockEndState, matchSet.getMinElement()); + newTransition = CodePointTransitions.createWithCodePoint(blockEndState, matchSet.getMinElement()); } else { Interval matchInterval = matchSet.getIntervals().get(0); - newTransition = new RangeTransition(blockEndState, matchInterval.a, matchInterval.b); + newTransition = CodePointTransitions.createWithCodePointRange(blockEndState, matchInterval.a, matchInterval.b); } } else { diff --git a/tool/src/org/antlr/v4/automata/ATNPrinter.java b/tool/src/org/antlr/v4/automata/ATNPrinter.java index f3b44a1..acad6e7 100644 --- a/tool/src/org/antlr/v4/automata/ATNPrinter.java +++ b/tool/src/org/antlr/v4/automata/ATNPrinter.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/tool/src/org/antlr/v4/automata/ATNVisitor.java b/tool/src/org/antlr/v4/automata/ATNVisitor.java index 70eb6cb..953ff36 100644 --- a/tool/src/org/antlr/v4/automata/ATNVisitor.java +++ b/tool/src/org/antlr/v4/automata/ATNVisitor.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/tool/src/org/antlr/v4/automata/LexerATNFactory.java b/tool/src/org/antlr/v4/automata/LexerATNFactory.java index b596df5..fb332e5 100644 --- a/tool/src/org/antlr/v4/automata/LexerATNFactory.java +++ b/tool/src/org/antlr/v4/automata/LexerATNFactory.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ @@ -10,6 +10,7 @@ import org.antlr.runtime.CommonToken; import org.antlr.runtime.Token; import org.antlr.v4.codegen.CodeGenerator; import org.antlr.v4.misc.CharSupport; +import org.antlr.v4.misc.EscapeSequenceParsing; import org.antlr.v4.parse.ANTLRParser; import org.antlr.v4.runtime.IntStream; import org.antlr.v4.runtime.Lexer; @@ -17,6 +18,7 @@ import org.antlr.v4.runtime.atn.ATN; import org.antlr.v4.runtime.atn.ATNState; import org.antlr.v4.runtime.atn.ActionTransition; import org.antlr.v4.runtime.atn.AtomTransition; +import org.antlr.v4.runtime.atn.CodePointTransitions; import org.antlr.v4.runtime.atn.LexerAction; import org.antlr.v4.runtime.atn.LexerChannelAction; import org.antlr.v4.runtime.atn.LexerCustomAction; @@ -27,7 +29,6 @@ import org.antlr.v4.runtime.atn.LexerPushModeAction; import org.antlr.v4.runtime.atn.LexerSkipAction; import org.antlr.v4.runtime.atn.LexerTypeAction; import org.antlr.v4.runtime.atn.NotSetTransition; -import org.antlr.v4.runtime.atn.RangeTransition; import org.antlr.v4.runtime.atn.RuleStartState; import org.antlr.v4.runtime.atn.SetTransition; import org.antlr.v4.runtime.atn.TokensStartState; @@ -48,6 +49,7 @@ import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Objects; import java.util.Set; public class LexerATNFactory extends ParserATNFactory { @@ -255,7 +257,7 @@ public class LexerATNFactory extends ParserATNFactory { int t1 = CharSupport.getCharValueFromGrammarCharLiteral(a.getText()); int t2 = CharSupport.getCharValueFromGrammarCharLiteral(b.getText()); checkRange(a, b, t1, t2); - left.addTransition(new RangeTransition(right, t1, t2)); + left.addTransition(CodePointTransitions.createWithCodePointRange(right, t1, t2)); a.atnState = left; b.atnState = left; return new Handle(left, right); @@ -301,8 +303,9 @@ public class LexerATNFactory extends ParserATNFactory { Transition transition; if (set.getIntervals().size() == 1) { Interval interval = set.getIntervals().get(0); - transition = new RangeTransition(right, interval.a, interval.b); - } else { + transition = CodePointTransitions.createWithCodePointRange(right, interval.a, interval.b); + } + else { transition = new SetTransition(right, set); } @@ -343,26 +346,27 @@ public class LexerATNFactory extends ParserATNFactory { String chars = stringLiteralAST.getText(); ATNState left = newState(stringLiteralAST); ATNState right; - chars = CharSupport.getStringFromGrammarStringLiteral(chars); - if (chars == null) { - g.tool.errMgr.grammarError(ErrorType.INVALID_ESCAPE_SEQUENCE, - g.fileName, stringLiteralAST.getToken()); + String s = CharSupport.getStringFromGrammarStringLiteral(chars); + if (s == null) { + // the lexer will already have given an error return new Handle(left, left); } - int n = chars.length(); + int n = s.length(); ATNState prev = left; right = null; - for (int i = 0; i < n; i++) { + for (int i = 0; i < n; ) { right = newState(stringLiteralAST); - prev.addTransition(new AtomTransition(right, chars.charAt(i))); + int codePoint = s.codePointAt(i); + prev.addTransition(CodePointTransitions.createWithCodePoint(right, codePoint)); prev = right; + i += Character.charCount(codePoint); } stringLiteralAST.atnState = left; return new Handle(left, right); } - /** [Aa\t \u1234a-z\]\-] char sets */ + /** [Aa\t \u1234a-z\]\p{Letter}\-] char sets */ @Override public Handle charSetLiteral(GrammarAST charSetAST) { ATNState left = newState(charSetAST); @@ -373,10 +377,68 @@ public class LexerATNFactory extends ParserATNFactory { return new Handle(left, right); } + private static class CharSetParseState { + enum Mode { + NONE, + ERROR, + PREV_CODE_POINT, + PREV_PROPERTY + } + + public static final CharSetParseState NONE = new CharSetParseState(Mode.NONE, false, -1, IntervalSet.EMPTY_SET); + public static final CharSetParseState ERROR = new CharSetParseState(Mode.ERROR, false, -1, IntervalSet.EMPTY_SET); + + public final Mode mode; + public final boolean inRange; + public final int prevCodePoint; + public final IntervalSet prevProperty; + + public CharSetParseState( + Mode mode, + boolean inRange, + int prevCodePoint, + IntervalSet prevProperty) { + this.mode = mode; + this.inRange = inRange; + this.prevCodePoint = prevCodePoint; + this.prevProperty = prevProperty; + } + + @Override + public String toString() { + return String.format( + "%s mode=%s inRange=%s prevCodePoint=%d prevProperty=%s", + super.toString(), + mode, + inRange, + prevCodePoint, + prevProperty); + } + + @Override + public boolean equals(Object other) { + if (!(other instanceof CharSetParseState)) { + return false; + } + CharSetParseState that = (CharSetParseState) other; + if (this == that) { + return true; + } + return Objects.equals(this.mode, that.mode) && + Objects.equals(this.inRange, that.inRange) && + Objects.equals(this.prevCodePoint, that.prevCodePoint) && + Objects.equals(this.prevProperty, that.prevProperty); + } + + @Override + public int hashCode() { + return Objects.hash(mode, inRange, prevCodePoint, prevProperty); + } + } + public IntervalSet getSetFromCharSetLiteral(GrammarAST charSetAST) { String chars = charSetAST.getText(); chars = chars.substring(1, chars.length() - 1); - String cset = '"' + chars + '"'; IntervalSet set = new IntervalSet(); if (chars.length() == 0) { @@ -384,51 +446,127 @@ public class LexerATNFactory extends ParserATNFactory { g.fileName, charSetAST.getToken(), "[]"); return set; } - // unescape all valid escape char like \n, leaving escaped dashes as '\-' - // so we can avoid seeing them as '-' range ops. - chars = CharSupport.getStringFromGrammarStringLiteral(cset); - if (chars == null) { - g.tool.errMgr.grammarError(ErrorType.INVALID_ESCAPE_SEQUENCE, - g.fileName, charSetAST.getToken()); - return set; - } + + CharSetParseState state = CharSetParseState.NONE; + int n = chars.length(); - // now make x-y become set of char - for (int i = 0; i < n; i++) { - int c = chars.charAt(i); - if (c == '\\' && i+1 < n && chars.charAt(i+1) == '-') { // \- - checkSetCollision(charSetAST, set, '-'); - set.add('-'); - i++; + for (int i = 0; i < n; ) { + if (state.mode == CharSetParseState.Mode.ERROR) { + return new IntervalSet(); + } + int c = chars.codePointAt(i); + int offset = Character.charCount(c); + if (c == '\\') { + EscapeSequenceParsing.Result escapeParseResult = + EscapeSequenceParsing.parseEscape(chars, i); + switch (escapeParseResult.type) { + case INVALID: + String invalid = chars.substring(escapeParseResult.startOffset, + escapeParseResult.startOffset+escapeParseResult.parseLength); + g.tool.errMgr.grammarError(ErrorType.INVALID_ESCAPE_SEQUENCE, + g.fileName, charSetAST.getToken(), invalid); + state = CharSetParseState.ERROR; + break; + case CODE_POINT: + state = applyPrevStateAndMoveToCodePoint(charSetAST, set, state, escapeParseResult.codePoint); + break; + case PROPERTY: + state = applyPrevStateAndMoveToProperty(charSetAST, set, state, escapeParseResult.propertyIntervalSet); + break; + } + offset = escapeParseResult.parseLength; } - else if (i+2 < n && chars.charAt(i+1) == '-') { // range x-y - int x = c; - int y = chars.charAt(i+2); - if (x <= y) { - checkSetCollision(charSetAST, set, x, y); - set.add(x,y); + else if (c == '-' && !state.inRange && i != 0 && i != n - 1) { + if (state.mode == CharSetParseState.Mode.PREV_PROPERTY) { + g.tool.errMgr.grammarError(ErrorType.UNICODE_PROPERTY_NOT_ALLOWED_IN_RANGE, + g.fileName, charSetAST.getToken(), charSetAST.getText()); + state = CharSetParseState.ERROR; } else { - g.tool.errMgr.grammarError(ErrorType.EMPTY_STRINGS_AND_SETS_NOT_ALLOWED, - g.fileName, charSetAST.getToken(), "[" + (char) x + "-" + (char) y + "]"); + state = new CharSetParseState(state.mode, true, state.prevCodePoint, state.prevProperty); } - i += 2; } else { - checkSetCollision(charSetAST, set, c); - set.add(c); + state = applyPrevStateAndMoveToCodePoint(charSetAST, set, state, c); } + i += offset; + } + if (state.mode == CharSetParseState.Mode.ERROR) { + return new IntervalSet(); } + // Whether or not we were in a range, we'll add the last code point found to the set. + applyPrevState(charSetAST, set, state); return set; } - protected void checkSetCollision(GrammarAST ast, IntervalSet set, int el) { - if (set.contains(el)) { - g.tool.errMgr.grammarError(ErrorType.CHARACTERS_COLLISION_IN_SET, g.fileName, ast.getToken(), - (char)el, ast.getText()); + private CharSetParseState applyPrevStateAndMoveToCodePoint( + GrammarAST charSetAST, + IntervalSet set, + CharSetParseState state, + int codePoint) { + if (state.inRange) { + if (state.prevCodePoint > codePoint) { + g.tool.errMgr.grammarError( + ErrorType.EMPTY_STRINGS_AND_SETS_NOT_ALLOWED, + g.fileName, + charSetAST.getToken(), + CharSupport.getRangeEscapedString(state.prevCodePoint, codePoint)); + } + checkSetCollision(charSetAST, set, state.prevCodePoint, codePoint); + set.add(state.prevCodePoint, codePoint); + state = CharSetParseState.NONE; + } + else { + applyPrevState(charSetAST, set, state); + state = new CharSetParseState( + CharSetParseState.Mode.PREV_CODE_POINT, + false, + codePoint, + IntervalSet.EMPTY_SET); + } + return state; + } + + private CharSetParseState applyPrevStateAndMoveToProperty( + GrammarAST charSetAST, + IntervalSet set, + CharSetParseState state, + IntervalSet property) { + if (state.inRange) { + g.tool.errMgr.grammarError(ErrorType.UNICODE_PROPERTY_NOT_ALLOWED_IN_RANGE, + g.fileName, charSetAST.getToken(), charSetAST.getText()); + return CharSetParseState.ERROR; + } + else { + applyPrevState(charSetAST, set, state); + state = new CharSetParseState( + CharSetParseState.Mode.PREV_PROPERTY, + false, + -1, + property); + } + return state; + } + + private void applyPrevState(GrammarAST charSetAST, IntervalSet set, CharSetParseState state) { + switch (state.mode) { + case NONE: + case ERROR: + break; + case PREV_CODE_POINT: + checkSetCollision(charSetAST, set, state.prevCodePoint); + set.add(state.prevCodePoint); + break; + case PREV_PROPERTY: + set.addAll(state.prevProperty); + break; } } + protected void checkSetCollision(GrammarAST ast, IntervalSet set, int el) { + checkSetCollision(ast, set, el, el); + } + protected void checkSetCollision(GrammarAST ast, IntervalSet set, int a, int b) { for (int i = a; i <= b; i++) { if (set.contains(i)) { @@ -453,7 +591,7 @@ public class LexerATNFactory extends ParserATNFactory { setText = sb.toString(); } g.tool.errMgr.grammarError(ErrorType.CHARACTERS_COLLISION_IN_SET, g.fileName, ast.getToken(), - (char)a + "-" + (char)b, setText); + CharSupport.getRangeEscapedString(a, b), setText); break; } } @@ -538,23 +676,30 @@ public class LexerATNFactory extends ParserATNFactory { if (command.equals("skip")) { if (ruleCommands.contains("more")) { firstCommand = "more"; - } else if (ruleCommands.contains("type")) { + } + else if (ruleCommands.contains("type")) { firstCommand = "type"; - } else if (ruleCommands.contains("channel")) { + } + else if (ruleCommands.contains("channel")) { firstCommand = "channel"; } - } else if (command.equals("more")) { + } + else if (command.equals("more")) { if (ruleCommands.contains("skip")) { firstCommand = "skip"; - } else if (ruleCommands.contains("type")) { + } + else if (ruleCommands.contains("type")) { firstCommand = "type"; - } else if (ruleCommands.contains("channel")) { + } + else if (ruleCommands.contains("channel")) { firstCommand = "channel"; } - } else if (command.equals("type") || command.equals("channel")) { + } + else if (command.equals("type") || command.equals("channel")) { if (ruleCommands.contains("more")) { firstCommand = "more"; - } else if (ruleCommands.contains("skip")) { + } + else if (ruleCommands.contains("skip")) { firstCommand = "skip"; } } diff --git a/tool/src/org/antlr/v4/automata/ParserATNFactory.java b/tool/src/org/antlr/v4/automata/ParserATNFactory.java index 2cfaccc..830b7fd 100644 --- a/tool/src/org/antlr/v4/automata/ParserATNFactory.java +++ b/tool/src/org/antlr/v4/automata/ParserATNFactory.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ @@ -233,7 +233,12 @@ public class ParserATNFactory implements ATNFactory { @Override public Handle range(GrammarAST a, GrammarAST b) { - throw new UnsupportedOperationException("This construct is not valid in parsers."); + g.tool.errMgr.grammarError(ErrorType.TOKEN_RANGE_IN_PARSER, g.fileName, + a.getToken(), + a.getToken().getText(), + b.getToken().getText()); + // From a..b, yield ATN for just a. + return tokenRef((TerminalAST)a); } protected int getTokenType(GrammarAST atom) { diff --git a/tool/src/org/antlr/v4/automata/TailEpsilonRemover.java b/tool/src/org/antlr/v4/automata/TailEpsilonRemover.java index b534b43..c71e950 100644 --- a/tool/src/org/antlr/v4/automata/TailEpsilonRemover.java +++ b/tool/src/org/antlr/v4/automata/TailEpsilonRemover.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ @@ -45,7 +45,8 @@ public class TailEpsilonRemover extends ATNVisitor { // skip over q if (p.transition(0) instanceof RuleTransition) { ((RuleTransition) p.transition(0)).followState = r; - } else { + } + else { p.transition(0).target = r; } _atn.removeState(q); diff --git a/tool/src/org/antlr/v4/codegen/ActionTranslator.java b/tool/src/org/antlr/v4/codegen/ActionTranslator.java index d9d8208..b214c40 100644 --- a/tool/src/org/antlr/v4/codegen/ActionTranslator.java +++ b/tool/src/org/antlr/v4/codegen/ActionTranslator.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/tool/src/org/antlr/v4/codegen/BlankOutputModelFactory.java b/tool/src/org/antlr/v4/codegen/BlankOutputModelFactory.java index 148d2d5..a040c13 100644 --- a/tool/src/org/antlr/v4/codegen/BlankOutputModelFactory.java +++ b/tool/src/org/antlr/v4/codegen/BlankOutputModelFactory.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/tool/src/org/antlr/v4/codegen/CodeGenPipeline.java b/tool/src/org/antlr/v4/codegen/CodeGenPipeline.java index eb3e998..1f97ddd 100644 --- a/tool/src/org/antlr/v4/codegen/CodeGenPipeline.java +++ b/tool/src/org/antlr/v4/codegen/CodeGenPipeline.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/tool/src/org/antlr/v4/codegen/CodeGenerator.java b/tool/src/org/antlr/v4/codegen/CodeGenerator.java index 68115c1..bc1080f 100644 --- a/tool/src/org/antlr/v4/codegen/CodeGenerator.java +++ b/tool/src/org/antlr/v4/codegen/CodeGenerator.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/tool/src/org/antlr/v4/codegen/CodeGeneratorExtension.java b/tool/src/org/antlr/v4/codegen/CodeGeneratorExtension.java index 53cf2ea..1e21cc9 100644 --- a/tool/src/org/antlr/v4/codegen/CodeGeneratorExtension.java +++ b/tool/src/org/antlr/v4/codegen/CodeGeneratorExtension.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/tool/src/org/antlr/v4/codegen/DefaultOutputModelFactory.java b/tool/src/org/antlr/v4/codegen/DefaultOutputModelFactory.java index 3f84914..78ef4f6 100644 --- a/tool/src/org/antlr/v4/codegen/DefaultOutputModelFactory.java +++ b/tool/src/org/antlr/v4/codegen/DefaultOutputModelFactory.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/tool/src/org/antlr/v4/codegen/LexerFactory.java b/tool/src/org/antlr/v4/codegen/LexerFactory.java index b071861..3a87b8b 100644 --- a/tool/src/org/antlr/v4/codegen/LexerFactory.java +++ b/tool/src/org/antlr/v4/codegen/LexerFactory.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/tool/src/org/antlr/v4/codegen/OutputModelController.java b/tool/src/org/antlr/v4/codegen/OutputModelController.java index 8a38b66..5126670 100644 --- a/tool/src/org/antlr/v4/codegen/OutputModelController.java +++ b/tool/src/org/antlr/v4/codegen/OutputModelController.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ @@ -400,7 +400,7 @@ public class OutputModelController { public List<SrcOp> wildcard(GrammarAST ast, GrammarAST labelAST) { List<SrcOp> ops = delegate.wildcard(ast, labelAST); for (CodeGeneratorExtension ext : extensions) { - ops = ext.set(ops); + ops = ext.wildcard(ops); } return ops; } diff --git a/tool/src/org/antlr/v4/codegen/OutputModelFactory.java b/tool/src/org/antlr/v4/codegen/OutputModelFactory.java index cab452d..8bce642 100644 --- a/tool/src/org/antlr/v4/codegen/OutputModelFactory.java +++ b/tool/src/org/antlr/v4/codegen/OutputModelFactory.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/tool/src/org/antlr/v4/codegen/OutputModelWalker.java b/tool/src/org/antlr/v4/codegen/OutputModelWalker.java index fc3d80b..b740e16 100644 --- a/tool/src/org/antlr/v4/codegen/OutputModelWalker.java +++ b/tool/src/org/antlr/v4/codegen/OutputModelWalker.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/tool/src/org/antlr/v4/codegen/ParserFactory.java b/tool/src/org/antlr/v4/codegen/ParserFactory.java index 4086886..36742fc 100644 --- a/tool/src/org/antlr/v4/codegen/ParserFactory.java +++ b/tool/src/org/antlr/v4/codegen/ParserFactory.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ @@ -33,6 +33,7 @@ import org.antlr.v4.codegen.model.SemPred; import org.antlr.v4.codegen.model.SrcOp; import org.antlr.v4.codegen.model.StarBlock; import org.antlr.v4.codegen.model.TestSetInline; +import org.antlr.v4.codegen.model.Wildcard; import org.antlr.v4.codegen.model.decl.Decl; import org.antlr.v4.codegen.model.decl.RuleContextDecl; import org.antlr.v4.codegen.model.decl.TokenDecl; diff --git a/tool/src/org/antlr/v4/codegen/Target.java b/tool/src/org/antlr/v4/codegen/Target.java index 9a2fc5d..025b42a 100644 --- a/tool/src/org/antlr/v4/codegen/Target.java +++ b/tool/src/org/antlr/v4/codegen/Target.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ @@ -9,6 +9,7 @@ package org.antlr.v4.codegen; import org.antlr.v4.Tool; import org.antlr.v4.codegen.model.RuleFunction; import org.antlr.v4.codegen.model.SerializedATN; +import org.antlr.v4.misc.CharSupport; import org.antlr.v4.misc.Utils; import org.antlr.v4.parse.ANTLRParser; import org.antlr.v4.runtime.RuntimeMetaData; @@ -146,17 +147,22 @@ public abstract class Target { if ( quoted ) { buf.append('"'); } - for (int i=0; i<s.length(); i++) { - int c = s.charAt(i); + for (int i=0; i<s.length(); ) { + int c = s.codePointAt(i); if ( c!='\'' && // don't escape single quotes in strings for java c<targetCharValueEscape.length && targetCharValueEscape[c]!=null ) { buf.append(targetCharValueEscape[c]); } - else { - buf.append((char)c); + else if (shouldUseUnicodeEscapeForCodePointInDoubleQuotedString(c)) { + appendUnicodeEscapedCodePoint(i, buf); + } + else + { + buf.appendCodePoint(c); } + i += Character.charCount(c); } if ( quoted ) { buf.append('"'); @@ -164,6 +170,12 @@ public abstract class Target { return buf.toString(); } + /** + * Escape the Unicode code point appropriately for this language + * and append the escaped value to {@code sb}. + */ + abstract protected void appendUnicodeEscapedCodePoint(int codePoint, StringBuilder sb); + public String getTargetStringLiteralFromString(String s) { return getTargetStringLiteralFromString(s, true); } @@ -194,18 +206,21 @@ public abstract class Target { if ( addQuotes ) sb.append('"'); - for (int i = 1; i < is.length() -1; i++) { - if (is.charAt(i) == '\\') { + for (int i = 1; i < is.length() -1; ) { + int codePoint = is.codePointAt(i); + int toAdvance = Character.charCount(codePoint); + if (codePoint == '\\') { // Anything escaped is what it is! We assume that // people know how to escape characters correctly. However // we catch anything that does not need an escape in Java (which // is what the default implementation is dealing with and remove // the escape. The C target does this for instance. // - switch (is.charAt(i+1)) { + int escapedCodePoint = is.codePointAt(i+toAdvance); + toAdvance++; + switch (escapedCodePoint) { // Pass through any escapes that Java also needs // - case '"': case 'n': case 'r': case 't': @@ -214,32 +229,50 @@ public abstract class Target { case '\\': // Pass the escape through sb.append('\\'); + sb.appendCodePoint(escapedCodePoint); break; - case 'u': // Assume unnnn - // Pass the escape through as double \\ - // so that Java leaves as \u0000 string not char - sb.append('\\'); - sb.append('\\'); + case 'u': // Either unnnn or u{nnnnnn} + if (is.charAt(i+toAdvance) == '{') { + while (is.charAt(i+toAdvance) != '}') { + toAdvance++; + } + toAdvance++; + } + else { + toAdvance += 4; + } + if ( i+toAdvance <= is.length() ) { // we might have an invalid \\uAB or something + String fullEscape = is.substring(i, i+toAdvance); + appendUnicodeEscapedCodePoint( + CharSupport.getCharValueFromCharInGrammarLiteral(fullEscape), + sb); + } break; - default: - // Remove the escape by virtue of not adding it here - // Thus \' becomes ' and so on + if (shouldUseUnicodeEscapeForCodePointInDoubleQuotedString(escapedCodePoint)) { + appendUnicodeEscapedCodePoint(escapedCodePoint, sb); + } + else { + sb.appendCodePoint(escapedCodePoint); + } break; } - - // Go past the \ character - i++; - } else { - // Characters that don't need \ in ANTLR 'strings' but do in Java - if (is.charAt(i) == '"') { - // We need to escape " in Java - sb.append('\\'); + } + else { + if (codePoint == 0x22) { + // ANTLR doesn't escape " in literal strings, + // but every other language needs to do so. + sb.append("\\\""); + } + else if (shouldUseUnicodeEscapeForCodePointInDoubleQuotedString(codePoint)) { + appendUnicodeEscapedCodePoint(codePoint, sb); + } + else { + sb.appendCodePoint(codePoint); } } - // Add in the next character, which may have been escaped - sb.append(is.charAt(i)); + i += toAdvance; } if ( addQuotes ) sb.append('"'); @@ -247,6 +280,19 @@ public abstract class Target { return sb.toString(); } + private static boolean shouldUseUnicodeEscapeForCodePointInDoubleQuotedString(int codePoint) { + // We don't want anyone passing 0x0A (newline) or 0x22 + // (double-quote) here because Java treats \\u000A as + // a literal newline and \\u0022 as a literal + // double-quote, so Unicode escaping doesn't help. + assert codePoint != 0x0A && codePoint != 0x22; + + return + codePoint < 0x20 || // control characters up to but not including space + codePoint == 0x5C || // backslash + codePoint >= 0x7F; // DEL and beyond (keeps source code 7-bit US-ASCII) + } + /** Assume 16-bit char */ public String encodeIntAsCharEscape(int v) { if (v < Character.MIN_VALUE || v > Character.MAX_VALUE) { diff --git a/tool/src/org/antlr/v4/codegen/UnicodeEscapes.java b/tool/src/org/antlr/v4/codegen/UnicodeEscapes.java new file mode 100644 index 0000000..8c88b4a --- /dev/null +++ b/tool/src/org/antlr/v4/codegen/UnicodeEscapes.java @@ -0,0 +1,38 @@ +/* + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. + * Use of this file is governed by the BSD 3-clause license that + * can be found in the LICENSE.txt file in the project root. + */ + +package org.antlr.v4.codegen; + +/** + * Utility class to escape Unicode code points using various + * languages' syntaxes. + */ +public abstract class UnicodeEscapes { + static public void appendJavaStyleEscapedCodePoint(int codePoint, StringBuilder sb) { + if (Character.isSupplementaryCodePoint(codePoint)) { + // char is not an 'integral' type, so we have to explicitly convert + // to int before passing to the %X formatter or else it throws. + sb.append(String.format("\\u%04X", (int)Character.highSurrogate(codePoint))); + sb.append(String.format("\\u%04X", (int)Character.lowSurrogate(codePoint))); + } + else { + sb.append(String.format("\\u%04X", codePoint)); + } + } + + static public void appendPythonStyleEscapedCodePoint(int codePoint, StringBuilder sb) { + if (Character.isSupplementaryCodePoint(codePoint)) { + sb.append(String.format("\\U%08X", codePoint)); + } + else { + sb.append(String.format("\\u%04X", codePoint)); + } + } + + static public void appendSwiftStyleEscapedCodePoint(int codePoint, StringBuilder sb) { + sb.append(String.format("\\u{%04X}", codePoint)); + } +} diff --git a/tool/src/org/antlr/v4/codegen/model/Action.java b/tool/src/org/antlr/v4/codegen/model/Action.java index 5e1e0ef..ada06ae 100644 --- a/tool/src/org/antlr/v4/codegen/model/Action.java +++ b/tool/src/org/antlr/v4/codegen/model/Action.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/tool/src/org/antlr/v4/codegen/model/AddToLabelList.java b/tool/src/org/antlr/v4/codegen/model/AddToLabelList.java index a492ee9..27a169a 100644 --- a/tool/src/org/antlr/v4/codegen/model/AddToLabelList.java +++ b/tool/src/org/antlr/v4/codegen/model/AddToLabelList.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/tool/src/org/antlr/v4/codegen/model/AltBlock.java b/tool/src/org/antlr/v4/codegen/model/AltBlock.java index a632921..db0654f 100644 --- a/tool/src/org/antlr/v4/codegen/model/AltBlock.java +++ b/tool/src/org/antlr/v4/codegen/model/AltBlock.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/tool/src/org/antlr/v4/codegen/model/ArgAction.java b/tool/src/org/antlr/v4/codegen/model/ArgAction.java index c52c72c..29dc530 100644 --- a/tool/src/org/antlr/v4/codegen/model/ArgAction.java +++ b/tool/src/org/antlr/v4/codegen/model/ArgAction.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/tool/src/org/antlr/v4/codegen/model/BaseListenerFile.java b/tool/src/org/antlr/v4/codegen/model/BaseListenerFile.java index 206fcee..6dc6546 100644 --- a/tool/src/org/antlr/v4/codegen/model/BaseListenerFile.java +++ b/tool/src/org/antlr/v4/codegen/model/BaseListenerFile.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/tool/src/org/antlr/v4/codegen/model/BaseVisitorFile.java b/tool/src/org/antlr/v4/codegen/model/BaseVisitorFile.java index 9fff357..6c41307 100644 --- a/tool/src/org/antlr/v4/codegen/model/BaseVisitorFile.java +++ b/tool/src/org/antlr/v4/codegen/model/BaseVisitorFile.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/tool/src/org/antlr/v4/codegen/model/CaptureNextToken.java b/tool/src/org/antlr/v4/codegen/model/CaptureNextToken.java index de49cdc..cbc0f28 100644 --- a/tool/src/org/antlr/v4/codegen/model/CaptureNextToken.java +++ b/tool/src/org/antlr/v4/codegen/model/CaptureNextToken.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/tool/src/org/antlr/v4/codegen/model/CaptureNextTokenType.java b/tool/src/org/antlr/v4/codegen/model/CaptureNextTokenType.java index 094d941..41f08ec 100644 --- a/tool/src/org/antlr/v4/codegen/model/CaptureNextTokenType.java +++ b/tool/src/org/antlr/v4/codegen/model/CaptureNextTokenType.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/tool/src/org/antlr/v4/codegen/model/Choice.java b/tool/src/org/antlr/v4/codegen/model/Choice.java index 9fb8a46..6b0d279 100644 --- a/tool/src/org/antlr/v4/codegen/model/Choice.java +++ b/tool/src/org/antlr/v4/codegen/model/Choice.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/tool/src/org/antlr/v4/codegen/model/CodeBlockForAlt.java b/tool/src/org/antlr/v4/codegen/model/CodeBlockForAlt.java index 12023a7..638f035 100644 --- a/tool/src/org/antlr/v4/codegen/model/CodeBlockForAlt.java +++ b/tool/src/org/antlr/v4/codegen/model/CodeBlockForAlt.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/tool/src/org/antlr/v4/codegen/model/CodeBlockForOuterMostAlt.java b/tool/src/org/antlr/v4/codegen/model/CodeBlockForOuterMostAlt.java index ef3e9d6..236b1fe 100644 --- a/tool/src/org/antlr/v4/codegen/model/CodeBlockForOuterMostAlt.java +++ b/tool/src/org/antlr/v4/codegen/model/CodeBlockForOuterMostAlt.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/tool/src/org/antlr/v4/codegen/model/DispatchMethod.java b/tool/src/org/antlr/v4/codegen/model/DispatchMethod.java index f7b3290..c68b936 100644 --- a/tool/src/org/antlr/v4/codegen/model/DispatchMethod.java +++ b/tool/src/org/antlr/v4/codegen/model/DispatchMethod.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/tool/src/org/antlr/v4/codegen/model/ElementFrequenciesVisitor.java b/tool/src/org/antlr/v4/codegen/model/ElementFrequenciesVisitor.java index ab5b86d..ccd5c80 100644 --- a/tool/src/org/antlr/v4/codegen/model/ElementFrequenciesVisitor.java +++ b/tool/src/org/antlr/v4/codegen/model/ElementFrequenciesVisitor.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ @@ -21,12 +21,30 @@ import java.util.Deque; import java.util.Map; public class ElementFrequenciesVisitor extends GrammarTreeVisitor { + /** + * This special value means "no set", and is used by {@link #minFrequencies} + * to ensure that {@link #combineMin} doesn't merge an empty set (all zeros) + * with the results of the first alternative. + */ + private static final FrequencySet<String> SENTINEL = new FrequencySet<String>(); + final Deque<FrequencySet<String>> frequencies; + private final Deque<FrequencySet<String>> minFrequencies; public ElementFrequenciesVisitor(TreeNodeStream input) { super(input); frequencies = new ArrayDeque<FrequencySet<String>>(); frequencies.push(new FrequencySet<String>()); + minFrequencies = new ArrayDeque<FrequencySet<String>>(); + minFrequencies.push(SENTINEL); + } + + FrequencySet<String> getMinFrequencies() { + assert minFrequencies.size() == 1; + assert minFrequencies.peek() != SENTINEL; + assert SENTINEL.isEmpty(); + + return minFrequencies.peek(); } /** During code gen, we can assume tree is in good shape */ @@ -62,6 +80,31 @@ public class ElementFrequenciesVisitor extends GrammarTreeVisitor { } /** + * Generate a frequency set as the union of two input sets. If an + * element is contained in both sets, the value for the output will be + * the minimum of the two input values. + * + * @param a The first set. + * @param b The second set. If this set is {@link #SENTINEL}, it is treated + * as though no second set were provided. + * @return The union of the two sets, with the minimum value chosen + * whenever both sets contain the same key. + */ + protected static FrequencySet<String> combineMin(FrequencySet<String> a, FrequencySet<String> b) { + if (b == SENTINEL) { + return a; + } + + assert a != SENTINEL; + FrequencySet<String> result = combineAndClip(a, b, Integer.MAX_VALUE); + for (Map.Entry<String, MutableInt> entry : result.entrySet()) { + entry.getValue().v = Math.min(a.count(entry.getKey()), b.count(entry.getKey())); + } + + return result; + } + + /** * Generate a frequency set as the union of two input sets, with the * values clipped to a specified maximum value. If an element is * contained in both sets, the value for the output, prior to clipping, @@ -97,11 +140,13 @@ public class ElementFrequenciesVisitor extends GrammarTreeVisitor { @Override public void tokenRef(TerminalAST ref) { frequencies.peek().add(ref.getText()); + minFrequencies.peek().add(ref.getText()); } @Override public void ruleRef(GrammarAST ref, ActionAST arg) { frequencies.peek().add(ref.getText()); + minFrequencies.peek().add(ref.getText()); } /* @@ -111,21 +156,50 @@ public class ElementFrequenciesVisitor extends GrammarTreeVisitor { @Override protected void enterAlternative(AltAST tree) { frequencies.push(new FrequencySet<String>()); + minFrequencies.push(new FrequencySet<String>()); } @Override protected void exitAlternative(AltAST tree) { frequencies.push(combineMax(frequencies.pop(), frequencies.pop())); + minFrequencies.push(combineMin(minFrequencies.pop(), minFrequencies.pop())); } @Override protected void enterElement(GrammarAST tree) { frequencies.push(new FrequencySet<String>()); + minFrequencies.push(new FrequencySet<String>()); } @Override protected void exitElement(GrammarAST tree) { frequencies.push(combineAndClip(frequencies.pop(), frequencies.pop(), 2)); + minFrequencies.push(combineAndClip(minFrequencies.pop(), minFrequencies.pop(), 2)); + } + + @Override + protected void enterBlockSet(GrammarAST tree) { + frequencies.push(new FrequencySet<String>()); + minFrequencies.push(new FrequencySet<String>()); + } + + @Override + protected void exitBlockSet(GrammarAST tree) { + for (Map.Entry<String, MutableInt> entry : frequencies.peek().entrySet()) { + // This visitor counts a block set as a sequence of elements, not a + // sequence of alternatives of elements. Reset the count back to 1 + // for all items when leaving the set to ensure duplicate entries in + // the set are treated as a maximum of one item. + entry.getValue().v = 1; + } + + if (minFrequencies.peek().size() > 1) { + // Everything is optional + minFrequencies.peek().clear(); + } + + frequencies.push(combineAndClip(frequencies.pop(), frequencies.pop(), 2)); + minFrequencies.push(combineAndClip(minFrequencies.pop(), minFrequencies.pop(), 2)); } @Override @@ -135,6 +209,12 @@ public class ElementFrequenciesVisitor extends GrammarTreeVisitor { entry.getValue().v = 2; } } + + if (tree.getType() == CLOSURE || tree.getType() == OPTIONAL) { + // Everything inside a closure is optional, so the minimum + // number of occurrences for all elements is 0. + minFrequencies.peek().clear(); + } } /* @@ -144,21 +224,25 @@ public class ElementFrequenciesVisitor extends GrammarTreeVisitor { @Override protected void enterLexerAlternative(GrammarAST tree) { frequencies.push(new FrequencySet<String>()); + minFrequencies.push(new FrequencySet<String>()); } @Override protected void exitLexerAlternative(GrammarAST tree) { frequencies.push(combineMax(frequencies.pop(), frequencies.pop())); + minFrequencies.push(combineMin(minFrequencies.pop(), minFrequencies.pop())); } @Override protected void enterLexerElement(GrammarAST tree) { frequencies.push(new FrequencySet<String>()); + minFrequencies.push(new FrequencySet<String>()); } @Override protected void exitLexerElement(GrammarAST tree) { frequencies.push(combineAndClip(frequencies.pop(), frequencies.pop(), 2)); + minFrequencies.push(combineAndClip(minFrequencies.pop(), minFrequencies.pop(), 2)); } @Override @@ -168,5 +252,11 @@ public class ElementFrequenciesVisitor extends GrammarTreeVisitor { entry.getValue().v = 2; } } + + if (tree.getType() == CLOSURE) { + // Everything inside a closure is optional, so the minimum + // number of occurrences for all elements is 0. + minFrequencies.peek().clear(); + } } } diff --git a/tool/src/org/antlr/v4/codegen/model/ExceptionClause.java b/tool/src/org/antlr/v4/codegen/model/ExceptionClause.java index 0eb7ddc..18aac15 100644 --- a/tool/src/org/antlr/v4/codegen/model/ExceptionClause.java +++ b/tool/src/org/antlr/v4/codegen/model/ExceptionClause.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/tool/src/org/antlr/v4/codegen/model/InvokeRule.java b/tool/src/org/antlr/v4/codegen/model/InvokeRule.java index bf83499..fb580cc 100644 --- a/tool/src/org/antlr/v4/codegen/model/InvokeRule.java +++ b/tool/src/org/antlr/v4/codegen/model/InvokeRule.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/tool/src/org/antlr/v4/codegen/model/LL1AltBlock.java b/tool/src/org/antlr/v4/codegen/model/LL1AltBlock.java index 3437388..d1b6d3a 100644 --- a/tool/src/org/antlr/v4/codegen/model/LL1AltBlock.java +++ b/tool/src/org/antlr/v4/codegen/model/LL1AltBlock.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/tool/src/org/antlr/v4/codegen/model/LL1Choice.java b/tool/src/org/antlr/v4/codegen/model/LL1Choice.java index 9ef47d2..745ae4e 100644 --- a/tool/src/org/antlr/v4/codegen/model/LL1Choice.java +++ b/tool/src/org/antlr/v4/codegen/model/LL1Choice.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/tool/src/org/antlr/v4/codegen/model/LL1Loop.java b/tool/src/org/antlr/v4/codegen/model/LL1Loop.java index 839a14d..78cfabd 100644 --- a/tool/src/org/antlr/v4/codegen/model/LL1Loop.java +++ b/tool/src/org/antlr/v4/codegen/model/LL1Loop.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/tool/src/org/antlr/v4/codegen/model/LL1OptionalBlock.java b/tool/src/org/antlr/v4/codegen/model/LL1OptionalBlock.java index 80f8d0b..652967e 100644 --- a/tool/src/org/antlr/v4/codegen/model/LL1OptionalBlock.java +++ b/tool/src/org/antlr/v4/codegen/model/LL1OptionalBlock.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/tool/src/org/antlr/v4/codegen/model/LL1OptionalBlockSingleAlt.java b/tool/src/org/antlr/v4/codegen/model/LL1OptionalBlockSingleAlt.java index 50579ca..6d4fbd2 100644 --- a/tool/src/org/antlr/v4/codegen/model/LL1OptionalBlockSingleAlt.java +++ b/tool/src/org/antlr/v4/codegen/model/LL1OptionalBlockSingleAlt.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/tool/src/org/antlr/v4/codegen/model/LL1PlusBlockSingleAlt.java b/tool/src/org/antlr/v4/codegen/model/LL1PlusBlockSingleAlt.java index 227753f..dcfaf11 100644 --- a/tool/src/org/antlr/v4/codegen/model/LL1PlusBlockSingleAlt.java +++ b/tool/src/org/antlr/v4/codegen/model/LL1PlusBlockSingleAlt.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/tool/src/org/antlr/v4/codegen/model/LL1StarBlockSingleAlt.java b/tool/src/org/antlr/v4/codegen/model/LL1StarBlockSingleAlt.java index fb787c1..27d46ee 100644 --- a/tool/src/org/antlr/v4/codegen/model/LL1StarBlockSingleAlt.java +++ b/tool/src/org/antlr/v4/codegen/model/LL1StarBlockSingleAlt.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/tool/src/org/antlr/v4/codegen/model/LabeledOp.java b/tool/src/org/antlr/v4/codegen/model/LabeledOp.java index b8d5441..73d24d7 100644 --- a/tool/src/org/antlr/v4/codegen/model/LabeledOp.java +++ b/tool/src/org/antlr/v4/codegen/model/LabeledOp.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/tool/src/org/antlr/v4/codegen/model/LeftRecursiveRuleFunction.java b/tool/src/org/antlr/v4/codegen/model/LeftRecursiveRuleFunction.java index abe3f7f..cf6420e 100644 --- a/tool/src/org/antlr/v4/codegen/model/LeftRecursiveRuleFunction.java +++ b/tool/src/org/antlr/v4/codegen/model/LeftRecursiveRuleFunction.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/tool/src/org/antlr/v4/codegen/model/Lexer.java b/tool/src/org/antlr/v4/codegen/model/Lexer.java index 2bc103e..35170e2 100644 --- a/tool/src/org/antlr/v4/codegen/model/Lexer.java +++ b/tool/src/org/antlr/v4/codegen/model/Lexer.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/tool/src/org/antlr/v4/codegen/model/LexerFile.java b/tool/src/org/antlr/v4/codegen/model/LexerFile.java index 3fae78b..9e02c60 100644 --- a/tool/src/org/antlr/v4/codegen/model/LexerFile.java +++ b/tool/src/org/antlr/v4/codegen/model/LexerFile.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/tool/src/org/antlr/v4/codegen/model/ListenerDispatchMethod.java b/tool/src/org/antlr/v4/codegen/model/ListenerDispatchMethod.java index cc861f0..2a8a39f 100644 --- a/tool/src/org/antlr/v4/codegen/model/ListenerDispatchMethod.java +++ b/tool/src/org/antlr/v4/codegen/model/ListenerDispatchMethod.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/tool/src/org/antlr/v4/codegen/model/ListenerFile.java b/tool/src/org/antlr/v4/codegen/model/ListenerFile.java index 24efd74..2066c16 100644 --- a/tool/src/org/antlr/v4/codegen/model/ListenerFile.java +++ b/tool/src/org/antlr/v4/codegen/model/ListenerFile.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ @@ -23,6 +23,7 @@ import java.util.Set; */ public class ListenerFile extends OutputFile { public String genPackage; // from -package cmd-line + public String accessLevel; // from -DaccessLevel cmd-line public String exportMacro; // from -DexportMacro cmd-line public String grammarName; public String parserName; @@ -61,7 +62,8 @@ public class ListenerFile extends OutputFile { } ActionAST ast = g.namedActions.get("header"); if ( ast!=null ) header = new Action(factory, ast); - genPackage = factory.getGrammar().tool.genPackage; - exportMacro = factory.getGrammar().getOptionString("exportMacro"); + genPackage = g.tool.genPackage; + accessLevel = g.getOptionString("accessLevel"); + exportMacro = g.getOptionString("exportMacro"); } } diff --git a/tool/src/org/antlr/v4/codegen/model/Loop.java b/tool/src/org/antlr/v4/codegen/model/Loop.java index 695685e..38d29d6 100644 --- a/tool/src/org/antlr/v4/codegen/model/Loop.java +++ b/tool/src/org/antlr/v4/codegen/model/Loop.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/tool/src/org/antlr/v4/codegen/model/MatchNotSet.java b/tool/src/org/antlr/v4/codegen/model/MatchNotSet.java index 4b86f5e..68687ee 100644 --- a/tool/src/org/antlr/v4/codegen/model/MatchNotSet.java +++ b/tool/src/org/antlr/v4/codegen/model/MatchNotSet.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/tool/src/org/antlr/v4/codegen/model/MatchSet.java b/tool/src/org/antlr/v4/codegen/model/MatchSet.java index ab1f798..48e710c 100644 --- a/tool/src/org/antlr/v4/codegen/model/MatchSet.java +++ b/tool/src/org/antlr/v4/codegen/model/MatchSet.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/tool/src/org/antlr/v4/codegen/model/MatchToken.java b/tool/src/org/antlr/v4/codegen/model/MatchToken.java index 5399113..6756f82 100644 --- a/tool/src/org/antlr/v4/codegen/model/MatchToken.java +++ b/tool/src/org/antlr/v4/codegen/model/MatchToken.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/tool/src/org/antlr/v4/codegen/model/ModelElement.java b/tool/src/org/antlr/v4/codegen/model/ModelElement.java index a0401de..cff1d61 100644 --- a/tool/src/org/antlr/v4/codegen/model/ModelElement.java +++ b/tool/src/org/antlr/v4/codegen/model/ModelElement.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/tool/src/org/antlr/v4/codegen/model/OptionalBlock.java b/tool/src/org/antlr/v4/codegen/model/OptionalBlock.java index 50b8fe3..ade58a1 100644 --- a/tool/src/org/antlr/v4/codegen/model/OptionalBlock.java +++ b/tool/src/org/antlr/v4/codegen/model/OptionalBlock.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/tool/src/org/antlr/v4/codegen/model/OutputFile.java b/tool/src/org/antlr/v4/codegen/model/OutputFile.java index e238dae..cd23324 100644 --- a/tool/src/org/antlr/v4/codegen/model/OutputFile.java +++ b/tool/src/org/antlr/v4/codegen/model/OutputFile.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/tool/src/org/antlr/v4/codegen/model/OutputModelObject.java b/tool/src/org/antlr/v4/codegen/model/OutputModelObject.java index 5cfb1ee..96e44e4 100644 --- a/tool/src/org/antlr/v4/codegen/model/OutputModelObject.java +++ b/tool/src/org/antlr/v4/codegen/model/OutputModelObject.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/tool/src/org/antlr/v4/codegen/model/Parser.java b/tool/src/org/antlr/v4/codegen/model/Parser.java index 298bcff..4e5e135 100644 --- a/tool/src/org/antlr/v4/codegen/model/Parser.java +++ b/tool/src/org/antlr/v4/codegen/model/Parser.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/tool/src/org/antlr/v4/codegen/model/ParserFile.java b/tool/src/org/antlr/v4/codegen/model/ParserFile.java index 49181dd..4cd6b26 100644 --- a/tool/src/org/antlr/v4/codegen/model/ParserFile.java +++ b/tool/src/org/antlr/v4/codegen/model/ParserFile.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/tool/src/org/antlr/v4/codegen/model/PlusBlock.java b/tool/src/org/antlr/v4/codegen/model/PlusBlock.java index 088089c..424ae03 100644 --- a/tool/src/org/antlr/v4/codegen/model/PlusBlock.java +++ b/tool/src/org/antlr/v4/codegen/model/PlusBlock.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/tool/src/org/antlr/v4/codegen/model/Recognizer.java b/tool/src/org/antlr/v4/codegen/model/Recognizer.java index 723dd08..5d79390 100644 --- a/tool/src/org/antlr/v4/codegen/model/Recognizer.java +++ b/tool/src/org/antlr/v4/codegen/model/Recognizer.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ @@ -24,6 +24,7 @@ public abstract class Recognizer extends OutputModelObject { public String name; public String grammarName; public String grammarFileName; + public String accessLevel; public Map<String,Integer> tokens; /** @@ -51,6 +52,7 @@ public abstract class Recognizer extends OutputModelObject { grammarFileName = new File(g.fileName).getName(); grammarName = g.name; name = g.getRecognizerName(); + accessLevel = g.getOptionString("accessLevel"); tokens = new LinkedHashMap<String,Integer>(); for (Map.Entry<String, Integer> entry : g.tokenNameToTypeMap.entrySet()) { Integer ttype = entry.getValue(); diff --git a/tool/src/org/antlr/v4/codegen/model/RuleActionFunction.java b/tool/src/org/antlr/v4/codegen/model/RuleActionFunction.java index 624ffd4..fffd5d2 100644 --- a/tool/src/org/antlr/v4/codegen/model/RuleActionFunction.java +++ b/tool/src/org/antlr/v4/codegen/model/RuleActionFunction.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/tool/src/org/antlr/v4/codegen/model/RuleElement.java b/tool/src/org/antlr/v4/codegen/model/RuleElement.java index ba0c0c0..8643bb9 100644 --- a/tool/src/org/antlr/v4/codegen/model/RuleElement.java +++ b/tool/src/org/antlr/v4/codegen/model/RuleElement.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/tool/src/org/antlr/v4/codegen/model/RuleFunction.java b/tool/src/org/antlr/v4/codegen/model/RuleFunction.java index d6e165e..d8b69b5 100644 --- a/tool/src/org/antlr/v4/codegen/model/RuleFunction.java +++ b/tool/src/org/antlr/v4/codegen/model/RuleFunction.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ @@ -171,49 +171,66 @@ public class RuleFunction extends OutputModelObject { */ public Set<Decl> getDeclsForAllElements(List<AltAST> altASTs) { Set<String> needsList = new HashSet<String>(); + Set<String> nonOptional = new HashSet<String>(); List<GrammarAST> allRefs = new ArrayList<GrammarAST>(); + boolean firstAlt = true; for (AltAST ast : altASTs) { IntervalSet reftypes = new IntervalSet(RULE_REF, TOKEN_REF); List<GrammarAST> refs = ast.getNodesWithType(reftypes); allRefs.addAll(refs); - FrequencySet<String> altFreq = getElementFrequenciesForAlt(ast); + Pair<FrequencySet<String>, FrequencySet<String>> minAndAltFreq = getElementFrequenciesForAlt(ast); + FrequencySet<String> minFreq = minAndAltFreq.a; + FrequencySet<String> altFreq = minAndAltFreq.b; for (GrammarAST t : refs) { String refLabelName = t.getText(); if ( altFreq.count(refLabelName)>1 ) { needsList.add(refLabelName); } + + if (firstAlt && minFreq.count(refLabelName) != 0) { + nonOptional.add(refLabelName); + } + } + + for (String ref : nonOptional.toArray(new String[nonOptional.size()])) { + if (minFreq.count(ref) == 0) { + nonOptional.remove(ref); + } } + + firstAlt = false; } Set<Decl> decls = new LinkedHashSet<Decl>(); for (GrammarAST t : allRefs) { String refLabelName = t.getText(); List<Decl> d = getDeclForAltElement(t, refLabelName, - needsList.contains(refLabelName)); + needsList.contains(refLabelName), + !nonOptional.contains(refLabelName)); decls.addAll(d); } return decls; } /** Given list of X and r refs in alt, compute how many of each there are */ - protected FrequencySet<String> getElementFrequenciesForAlt(AltAST ast) { + protected Pair<FrequencySet<String>, FrequencySet<String>> getElementFrequenciesForAlt(AltAST ast) { try { ElementFrequenciesVisitor visitor = new ElementFrequenciesVisitor(new CommonTreeNodeStream(new GrammarASTAdaptor(), ast)); visitor.outerAlternative(); if (visitor.frequencies.size() != 1) { factory.getGrammar().tool.errMgr.toolError(ErrorType.INTERNAL_ERROR); - return new FrequencySet<String>(); + return new Pair<>(new FrequencySet<String>(), new FrequencySet<String>()); } - return visitor.frequencies.peek(); + return new Pair<>(visitor.getMinFrequencies(), visitor.frequencies.peek()); } catch (RecognitionException ex) { factory.getGrammar().tool.errMgr.toolError(ErrorType.INTERNAL_ERROR, ex); - return new FrequencySet<String>(); + return new Pair<>(new FrequencySet<String>(), new FrequencySet<String>()); } } - public List<Decl> getDeclForAltElement(GrammarAST t, String refLabelName, boolean needList) { + public List<Decl> getDeclForAltElement(GrammarAST t, String refLabelName, boolean needList, boolean optional) { List<Decl> decls = new ArrayList<Decl>(); if ( t.getType()==RULE_REF ) { Rule rref = factory.getGrammar().getRule(t.getText()); @@ -225,7 +242,7 @@ public class RuleFunction extends OutputModelObject { decls.add( new ContextRuleListIndexedGetterDecl(factory, refLabelName, ctxName) ); } else { - decls.add( new ContextRuleGetterDecl(factory, refLabelName, ctxName) ); + decls.add( new ContextRuleGetterDecl(factory, refLabelName, ctxName, optional) ); } } else { @@ -235,7 +252,7 @@ public class RuleFunction extends OutputModelObject { decls.add( new ContextTokenListIndexedGetterDecl(factory, refLabelName) ); } else { - decls.add( new ContextTokenGetterDecl(factory, refLabelName) ); + decls.add( new ContextTokenGetterDecl(factory, refLabelName, optional) ); } } return decls; diff --git a/tool/src/org/antlr/v4/codegen/model/RuleSempredFunction.java b/tool/src/org/antlr/v4/codegen/model/RuleSempredFunction.java index aba9d7e..75d10cb 100644 --- a/tool/src/org/antlr/v4/codegen/model/RuleSempredFunction.java +++ b/tool/src/org/antlr/v4/codegen/model/RuleSempredFunction.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/tool/src/org/antlr/v4/codegen/model/SemPred.java b/tool/src/org/antlr/v4/codegen/model/SemPred.java index e42649b..3496b44 100644 --- a/tool/src/org/antlr/v4/codegen/model/SemPred.java +++ b/tool/src/org/antlr/v4/codegen/model/SemPred.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/tool/src/org/antlr/v4/codegen/model/SerializedATN.java b/tool/src/org/antlr/v4/codegen/model/SerializedATN.java index 6a1094e..1ace766 100644 --- a/tool/src/org/antlr/v4/codegen/model/SerializedATN.java +++ b/tool/src/org/antlr/v4/codegen/model/SerializedATN.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/tool/src/org/antlr/v4/codegen/model/SrcOp.java b/tool/src/org/antlr/v4/codegen/model/SrcOp.java index ff9bd9d..ab51f88 100644 --- a/tool/src/org/antlr/v4/codegen/model/SrcOp.java +++ b/tool/src/org/antlr/v4/codegen/model/SrcOp.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/tool/src/org/antlr/v4/codegen/model/StarBlock.java b/tool/src/org/antlr/v4/codegen/model/StarBlock.java index 41b49f2..6c32293 100644 --- a/tool/src/org/antlr/v4/codegen/model/StarBlock.java +++ b/tool/src/org/antlr/v4/codegen/model/StarBlock.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/tool/src/org/antlr/v4/codegen/model/Sync.java b/tool/src/org/antlr/v4/codegen/model/Sync.java index db5287e..3c4f23a 100644 --- a/tool/src/org/antlr/v4/codegen/model/Sync.java +++ b/tool/src/org/antlr/v4/codegen/model/Sync.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/tool/src/org/antlr/v4/codegen/model/TestSetInline.java b/tool/src/org/antlr/v4/codegen/model/TestSetInline.java index d4c9381..dfbfd03 100644 --- a/tool/src/org/antlr/v4/codegen/model/TestSetInline.java +++ b/tool/src/org/antlr/v4/codegen/model/TestSetInline.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/tool/src/org/antlr/v4/codegen/model/ThrowEarlyExitException.java b/tool/src/org/antlr/v4/codegen/model/ThrowEarlyExitException.java index efadf44..870271f 100644 --- a/tool/src/org/antlr/v4/codegen/model/ThrowEarlyExitException.java +++ b/tool/src/org/antlr/v4/codegen/model/ThrowEarlyExitException.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/tool/src/org/antlr/v4/codegen/model/ThrowNoViableAlt.java b/tool/src/org/antlr/v4/codegen/model/ThrowNoViableAlt.java index 7bb6bbd..bc33dc8 100644 --- a/tool/src/org/antlr/v4/codegen/model/ThrowNoViableAlt.java +++ b/tool/src/org/antlr/v4/codegen/model/ThrowNoViableAlt.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/tool/src/org/antlr/v4/codegen/model/ThrowRecognitionException.java b/tool/src/org/antlr/v4/codegen/model/ThrowRecognitionException.java index 6653038..4f1d630 100644 --- a/tool/src/org/antlr/v4/codegen/model/ThrowRecognitionException.java +++ b/tool/src/org/antlr/v4/codegen/model/ThrowRecognitionException.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/tool/src/org/antlr/v4/codegen/model/VisitorDispatchMethod.java b/tool/src/org/antlr/v4/codegen/model/VisitorDispatchMethod.java index 90c6e82..3c62be8 100644 --- a/tool/src/org/antlr/v4/codegen/model/VisitorDispatchMethod.java +++ b/tool/src/org/antlr/v4/codegen/model/VisitorDispatchMethod.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/tool/src/org/antlr/v4/codegen/model/VisitorFile.java b/tool/src/org/antlr/v4/codegen/model/VisitorFile.java index 4abcdb1..3c746d6 100644 --- a/tool/src/org/antlr/v4/codegen/model/VisitorFile.java +++ b/tool/src/org/antlr/v4/codegen/model/VisitorFile.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ @@ -20,6 +20,7 @@ import java.util.Set; public class VisitorFile extends OutputFile { public String genPackage; // from -package cmd-line + public String accessLevel; // from -DaccessLevel cmd-line public String exportMacro; // from -DexportMacro cmd-line public String grammarName; public String parserName; @@ -58,7 +59,8 @@ public class VisitorFile extends OutputFile { } ActionAST ast = g.namedActions.get("header"); if ( ast!=null ) header = new Action(factory, ast); - genPackage = factory.getGrammar().tool.genPackage; - exportMacro = factory.getGrammar().getOptionString("exportMacro"); + genPackage = g.tool.genPackage; + accessLevel = g.getOptionString("accessLevel"); + exportMacro = g.getOptionString("exportMacro"); } } diff --git a/tool/src/org/antlr/v4/codegen/Wildcard.java b/tool/src/org/antlr/v4/codegen/model/Wildcard.java index 6537cc7..977d22c 100644 --- a/tool/src/org/antlr/v4/codegen/Wildcard.java +++ b/tool/src/org/antlr/v4/codegen/model/Wildcard.java @@ -1,12 +1,12 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ -package org.antlr.v4.codegen; +package org.antlr.v4.codegen.model; -import org.antlr.v4.codegen.model.MatchToken; +import org.antlr.v4.codegen.OutputModelFactory; import org.antlr.v4.tool.ast.GrammarAST; public class Wildcard extends MatchToken { diff --git a/tool/src/org/antlr/v4/codegen/model/chunk/ActionChunk.java b/tool/src/org/antlr/v4/codegen/model/chunk/ActionChunk.java index 36775b5..53d046a 100644 --- a/tool/src/org/antlr/v4/codegen/model/chunk/ActionChunk.java +++ b/tool/src/org/antlr/v4/codegen/model/chunk/ActionChunk.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/tool/src/org/antlr/v4/codegen/model/chunk/ActionTemplate.java b/tool/src/org/antlr/v4/codegen/model/chunk/ActionTemplate.java index 861fd8f..28dd97a 100644 --- a/tool/src/org/antlr/v4/codegen/model/chunk/ActionTemplate.java +++ b/tool/src/org/antlr/v4/codegen/model/chunk/ActionTemplate.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/tool/src/org/antlr/v4/codegen/model/chunk/ActionText.java b/tool/src/org/antlr/v4/codegen/model/chunk/ActionText.java index b1c64c2..4ed87a1 100644 --- a/tool/src/org/antlr/v4/codegen/model/chunk/ActionText.java +++ b/tool/src/org/antlr/v4/codegen/model/chunk/ActionText.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/tool/src/org/antlr/v4/codegen/model/chunk/ArgRef.java b/tool/src/org/antlr/v4/codegen/model/chunk/ArgRef.java index 8544692..4c10fb1 100644 --- a/tool/src/org/antlr/v4/codegen/model/chunk/ArgRef.java +++ b/tool/src/org/antlr/v4/codegen/model/chunk/ArgRef.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/tool/src/org/antlr/v4/codegen/model/chunk/LabelRef.java b/tool/src/org/antlr/v4/codegen/model/chunk/LabelRef.java index 5e5418b..3f703a8 100644 --- a/tool/src/org/antlr/v4/codegen/model/chunk/LabelRef.java +++ b/tool/src/org/antlr/v4/codegen/model/chunk/LabelRef.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/tool/src/org/antlr/v4/codegen/model/chunk/ListLabelRef.java b/tool/src/org/antlr/v4/codegen/model/chunk/ListLabelRef.java index 07dc911..dc941ba 100644 --- a/tool/src/org/antlr/v4/codegen/model/chunk/ListLabelRef.java +++ b/tool/src/org/antlr/v4/codegen/model/chunk/ListLabelRef.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/tool/src/org/antlr/v4/codegen/model/chunk/LocalRef.java b/tool/src/org/antlr/v4/codegen/model/chunk/LocalRef.java index 6cf25ab..6a797cf 100644 --- a/tool/src/org/antlr/v4/codegen/model/chunk/LocalRef.java +++ b/tool/src/org/antlr/v4/codegen/model/chunk/LocalRef.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/tool/src/org/antlr/v4/codegen/model/chunk/NonLocalAttrRef.java b/tool/src/org/antlr/v4/codegen/model/chunk/NonLocalAttrRef.java index db6763f..37698f7 100644 --- a/tool/src/org/antlr/v4/codegen/model/chunk/NonLocalAttrRef.java +++ b/tool/src/org/antlr/v4/codegen/model/chunk/NonLocalAttrRef.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/tool/src/org/antlr/v4/codegen/model/chunk/QRetValueRef.java b/tool/src/org/antlr/v4/codegen/model/chunk/QRetValueRef.java index 245e77a..c31618a 100644 --- a/tool/src/org/antlr/v4/codegen/model/chunk/QRetValueRef.java +++ b/tool/src/org/antlr/v4/codegen/model/chunk/QRetValueRef.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/tool/src/org/antlr/v4/codegen/model/chunk/RetValueRef.java b/tool/src/org/antlr/v4/codegen/model/chunk/RetValueRef.java index 07dfe64..ee67a15 100644 --- a/tool/src/org/antlr/v4/codegen/model/chunk/RetValueRef.java +++ b/tool/src/org/antlr/v4/codegen/model/chunk/RetValueRef.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/tool/src/org/antlr/v4/codegen/model/chunk/RulePropertyRef.java b/tool/src/org/antlr/v4/codegen/model/chunk/RulePropertyRef.java index ded613f..430917a 100644 --- a/tool/src/org/antlr/v4/codegen/model/chunk/RulePropertyRef.java +++ b/tool/src/org/antlr/v4/codegen/model/chunk/RulePropertyRef.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/tool/src/org/antlr/v4/codegen/model/chunk/RulePropertyRef_ctx.java b/tool/src/org/antlr/v4/codegen/model/chunk/RulePropertyRef_ctx.java index 26301a0..aaec94f 100644 --- a/tool/src/org/antlr/v4/codegen/model/chunk/RulePropertyRef_ctx.java +++ b/tool/src/org/antlr/v4/codegen/model/chunk/RulePropertyRef_ctx.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/tool/src/org/antlr/v4/codegen/model/chunk/RulePropertyRef_parser.java b/tool/src/org/antlr/v4/codegen/model/chunk/RulePropertyRef_parser.java index 16646b9..39b6690 100644 --- a/tool/src/org/antlr/v4/codegen/model/chunk/RulePropertyRef_parser.java +++ b/tool/src/org/antlr/v4/codegen/model/chunk/RulePropertyRef_parser.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/tool/src/org/antlr/v4/codegen/model/chunk/RulePropertyRef_start.java b/tool/src/org/antlr/v4/codegen/model/chunk/RulePropertyRef_start.java index ef485e5..8a0977b 100644 --- a/tool/src/org/antlr/v4/codegen/model/chunk/RulePropertyRef_start.java +++ b/tool/src/org/antlr/v4/codegen/model/chunk/RulePropertyRef_start.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/tool/src/org/antlr/v4/codegen/model/chunk/RulePropertyRef_stop.java b/tool/src/org/antlr/v4/codegen/model/chunk/RulePropertyRef_stop.java index d649967..f213ab2 100644 --- a/tool/src/org/antlr/v4/codegen/model/chunk/RulePropertyRef_stop.java +++ b/tool/src/org/antlr/v4/codegen/model/chunk/RulePropertyRef_stop.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/tool/src/org/antlr/v4/codegen/model/chunk/RulePropertyRef_text.java b/tool/src/org/antlr/v4/codegen/model/chunk/RulePropertyRef_text.java index f3d0a08..6ef2313 100644 --- a/tool/src/org/antlr/v4/codegen/model/chunk/RulePropertyRef_text.java +++ b/tool/src/org/antlr/v4/codegen/model/chunk/RulePropertyRef_text.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/tool/src/org/antlr/v4/codegen/model/chunk/SetAttr.java b/tool/src/org/antlr/v4/codegen/model/chunk/SetAttr.java index 6c5755c..dac369b 100644 --- a/tool/src/org/antlr/v4/codegen/model/chunk/SetAttr.java +++ b/tool/src/org/antlr/v4/codegen/model/chunk/SetAttr.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/tool/src/org/antlr/v4/codegen/model/chunk/SetNonLocalAttr.java b/tool/src/org/antlr/v4/codegen/model/chunk/SetNonLocalAttr.java index c2e7924..0345c29 100644 --- a/tool/src/org/antlr/v4/codegen/model/chunk/SetNonLocalAttr.java +++ b/tool/src/org/antlr/v4/codegen/model/chunk/SetNonLocalAttr.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/tool/src/org/antlr/v4/codegen/model/chunk/ThisRulePropertyRef_ctx.java b/tool/src/org/antlr/v4/codegen/model/chunk/ThisRulePropertyRef_ctx.java index 0293664..b14b452 100644 --- a/tool/src/org/antlr/v4/codegen/model/chunk/ThisRulePropertyRef_ctx.java +++ b/tool/src/org/antlr/v4/codegen/model/chunk/ThisRulePropertyRef_ctx.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/tool/src/org/antlr/v4/codegen/model/chunk/ThisRulePropertyRef_parser.java b/tool/src/org/antlr/v4/codegen/model/chunk/ThisRulePropertyRef_parser.java index 22fc377..f1d6541 100644 --- a/tool/src/org/antlr/v4/codegen/model/chunk/ThisRulePropertyRef_parser.java +++ b/tool/src/org/antlr/v4/codegen/model/chunk/ThisRulePropertyRef_parser.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/tool/src/org/antlr/v4/codegen/model/chunk/ThisRulePropertyRef_start.java b/tool/src/org/antlr/v4/codegen/model/chunk/ThisRulePropertyRef_start.java index c4cafec..bfb483c 100644 --- a/tool/src/org/antlr/v4/codegen/model/chunk/ThisRulePropertyRef_start.java +++ b/tool/src/org/antlr/v4/codegen/model/chunk/ThisRulePropertyRef_start.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/tool/src/org/antlr/v4/codegen/model/chunk/ThisRulePropertyRef_stop.java b/tool/src/org/antlr/v4/codegen/model/chunk/ThisRulePropertyRef_stop.java index ea51694..fa1f7c6 100644 --- a/tool/src/org/antlr/v4/codegen/model/chunk/ThisRulePropertyRef_stop.java +++ b/tool/src/org/antlr/v4/codegen/model/chunk/ThisRulePropertyRef_stop.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/tool/src/org/antlr/v4/codegen/model/chunk/ThisRulePropertyRef_text.java b/tool/src/org/antlr/v4/codegen/model/chunk/ThisRulePropertyRef_text.java index 696066a..504b1d4 100644 --- a/tool/src/org/antlr/v4/codegen/model/chunk/ThisRulePropertyRef_text.java +++ b/tool/src/org/antlr/v4/codegen/model/chunk/ThisRulePropertyRef_text.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/tool/src/org/antlr/v4/codegen/model/chunk/TokenPropertyRef.java b/tool/src/org/antlr/v4/codegen/model/chunk/TokenPropertyRef.java index 82c2f69..4a6433d 100644 --- a/tool/src/org/antlr/v4/codegen/model/chunk/TokenPropertyRef.java +++ b/tool/src/org/antlr/v4/codegen/model/chunk/TokenPropertyRef.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/tool/src/org/antlr/v4/codegen/model/chunk/TokenPropertyRef_channel.java b/tool/src/org/antlr/v4/codegen/model/chunk/TokenPropertyRef_channel.java index 1404bf9..b69e096 100644 --- a/tool/src/org/antlr/v4/codegen/model/chunk/TokenPropertyRef_channel.java +++ b/tool/src/org/antlr/v4/codegen/model/chunk/TokenPropertyRef_channel.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/tool/src/org/antlr/v4/codegen/model/chunk/TokenPropertyRef_index.java b/tool/src/org/antlr/v4/codegen/model/chunk/TokenPropertyRef_index.java index bb92a20..b92ea9f 100644 --- a/tool/src/org/antlr/v4/codegen/model/chunk/TokenPropertyRef_index.java +++ b/tool/src/org/antlr/v4/codegen/model/chunk/TokenPropertyRef_index.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/tool/src/org/antlr/v4/codegen/model/chunk/TokenPropertyRef_int.java b/tool/src/org/antlr/v4/codegen/model/chunk/TokenPropertyRef_int.java index b02053f..24b6e22 100644 --- a/tool/src/org/antlr/v4/codegen/model/chunk/TokenPropertyRef_int.java +++ b/tool/src/org/antlr/v4/codegen/model/chunk/TokenPropertyRef_int.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/tool/src/org/antlr/v4/codegen/model/chunk/TokenPropertyRef_line.java b/tool/src/org/antlr/v4/codegen/model/chunk/TokenPropertyRef_line.java index 0229204..f8dd162 100644 --- a/tool/src/org/antlr/v4/codegen/model/chunk/TokenPropertyRef_line.java +++ b/tool/src/org/antlr/v4/codegen/model/chunk/TokenPropertyRef_line.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/tool/src/org/antlr/v4/codegen/model/chunk/TokenPropertyRef_pos.java b/tool/src/org/antlr/v4/codegen/model/chunk/TokenPropertyRef_pos.java index 7badb92..f7e964f 100644 --- a/tool/src/org/antlr/v4/codegen/model/chunk/TokenPropertyRef_pos.java +++ b/tool/src/org/antlr/v4/codegen/model/chunk/TokenPropertyRef_pos.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/tool/src/org/antlr/v4/codegen/model/chunk/TokenPropertyRef_text.java b/tool/src/org/antlr/v4/codegen/model/chunk/TokenPropertyRef_text.java index d049c6e..05bfbab 100644 --- a/tool/src/org/antlr/v4/codegen/model/chunk/TokenPropertyRef_text.java +++ b/tool/src/org/antlr/v4/codegen/model/chunk/TokenPropertyRef_text.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/tool/src/org/antlr/v4/codegen/model/chunk/TokenPropertyRef_type.java b/tool/src/org/antlr/v4/codegen/model/chunk/TokenPropertyRef_type.java index a8b4222..747b676 100644 --- a/tool/src/org/antlr/v4/codegen/model/chunk/TokenPropertyRef_type.java +++ b/tool/src/org/antlr/v4/codegen/model/chunk/TokenPropertyRef_type.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/tool/src/org/antlr/v4/codegen/model/chunk/TokenRef.java b/tool/src/org/antlr/v4/codegen/model/chunk/TokenRef.java index d39115d..b2b6200 100644 --- a/tool/src/org/antlr/v4/codegen/model/chunk/TokenRef.java +++ b/tool/src/org/antlr/v4/codegen/model/chunk/TokenRef.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/tool/src/org/antlr/v4/codegen/model/dbg.java b/tool/src/org/antlr/v4/codegen/model/dbg.java index 0aaeebf..2f6aa07 100644 --- a/tool/src/org/antlr/v4/codegen/model/dbg.java +++ b/tool/src/org/antlr/v4/codegen/model/dbg.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/tool/src/org/antlr/v4/codegen/model/decl/AltLabelStructDecl.java b/tool/src/org/antlr/v4/codegen/model/decl/AltLabelStructDecl.java index b855b28..a0a8cc8 100644 --- a/tool/src/org/antlr/v4/codegen/model/decl/AltLabelStructDecl.java +++ b/tool/src/org/antlr/v4/codegen/model/decl/AltLabelStructDecl.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/tool/src/org/antlr/v4/codegen/model/decl/AttributeDecl.java b/tool/src/org/antlr/v4/codegen/model/decl/AttributeDecl.java index 80af232..0436752 100644 --- a/tool/src/org/antlr/v4/codegen/model/decl/AttributeDecl.java +++ b/tool/src/org/antlr/v4/codegen/model/decl/AttributeDecl.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/tool/src/org/antlr/v4/codegen/model/decl/CodeBlock.java b/tool/src/org/antlr/v4/codegen/model/decl/CodeBlock.java index 89e1f20..2e608a1 100644 --- a/tool/src/org/antlr/v4/codegen/model/decl/CodeBlock.java +++ b/tool/src/org/antlr/v4/codegen/model/decl/CodeBlock.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/tool/src/org/antlr/v4/codegen/model/decl/ContextGetterDecl.java b/tool/src/org/antlr/v4/codegen/model/decl/ContextGetterDecl.java index fc4b262..b708f8a 100644 --- a/tool/src/org/antlr/v4/codegen/model/decl/ContextGetterDecl.java +++ b/tool/src/org/antlr/v4/codegen/model/decl/ContextGetterDecl.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/tool/src/org/antlr/v4/codegen/model/decl/ContextRuleGetterDecl.java b/tool/src/org/antlr/v4/codegen/model/decl/ContextRuleGetterDecl.java index b763965..322054d 100644 --- a/tool/src/org/antlr/v4/codegen/model/decl/ContextRuleGetterDecl.java +++ b/tool/src/org/antlr/v4/codegen/model/decl/ContextRuleGetterDecl.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ @@ -11,8 +11,11 @@ import org.antlr.v4.codegen.OutputModelFactory; /** {@code public XContext X() { }} */ public class ContextRuleGetterDecl extends ContextGetterDecl { public String ctxName; - public ContextRuleGetterDecl(OutputModelFactory factory, String name, String ctxName) { + public boolean optional; + + public ContextRuleGetterDecl(OutputModelFactory factory, String name, String ctxName, boolean optional) { super(factory, name); this.ctxName = ctxName; + this.optional = optional; } } diff --git a/tool/src/org/antlr/v4/codegen/model/decl/ContextRuleListGetterDecl.java b/tool/src/org/antlr/v4/codegen/model/decl/ContextRuleListGetterDecl.java index 4daf182..43e5d3c 100644 --- a/tool/src/org/antlr/v4/codegen/model/decl/ContextRuleListGetterDecl.java +++ b/tool/src/org/antlr/v4/codegen/model/decl/ContextRuleListGetterDecl.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/tool/src/org/antlr/v4/codegen/model/decl/ContextRuleListIndexedGetterDecl.java b/tool/src/org/antlr/v4/codegen/model/decl/ContextRuleListIndexedGetterDecl.java index 175394f..4399c4c 100644 --- a/tool/src/org/antlr/v4/codegen/model/decl/ContextRuleListIndexedGetterDecl.java +++ b/tool/src/org/antlr/v4/codegen/model/decl/ContextRuleListIndexedGetterDecl.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/tool/src/org/antlr/v4/codegen/model/decl/ContextTokenGetterDecl.java b/tool/src/org/antlr/v4/codegen/model/decl/ContextTokenGetterDecl.java index d663181..2ca6478 100644 --- a/tool/src/org/antlr/v4/codegen/model/decl/ContextTokenGetterDecl.java +++ b/tool/src/org/antlr/v4/codegen/model/decl/ContextTokenGetterDecl.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ @@ -10,7 +10,10 @@ import org.antlr.v4.codegen.OutputModelFactory; /** {@code public Token X() { }} */ public class ContextTokenGetterDecl extends ContextGetterDecl { - public ContextTokenGetterDecl(OutputModelFactory factory, String name) { + public boolean optional; + + public ContextTokenGetterDecl(OutputModelFactory factory, String name, boolean optional) { super(factory, name); + this.optional = optional; } } diff --git a/tool/src/org/antlr/v4/codegen/model/decl/ContextTokenListGetterDecl.java b/tool/src/org/antlr/v4/codegen/model/decl/ContextTokenListGetterDecl.java index 7c8e2ff..0005d4b 100644 --- a/tool/src/org/antlr/v4/codegen/model/decl/ContextTokenListGetterDecl.java +++ b/tool/src/org/antlr/v4/codegen/model/decl/ContextTokenListGetterDecl.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/tool/src/org/antlr/v4/codegen/model/decl/ContextTokenListIndexedGetterDecl.java b/tool/src/org/antlr/v4/codegen/model/decl/ContextTokenListIndexedGetterDecl.java index bc722c9..4db542a 100644 --- a/tool/src/org/antlr/v4/codegen/model/decl/ContextTokenListIndexedGetterDecl.java +++ b/tool/src/org/antlr/v4/codegen/model/decl/ContextTokenListIndexedGetterDecl.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/tool/src/org/antlr/v4/codegen/model/decl/Decl.java b/tool/src/org/antlr/v4/codegen/model/decl/Decl.java index 2cbcf9c..b2b9742 100644 --- a/tool/src/org/antlr/v4/codegen/model/decl/Decl.java +++ b/tool/src/org/antlr/v4/codegen/model/decl/Decl.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/tool/src/org/antlr/v4/codegen/model/decl/ElementListDecl.java b/tool/src/org/antlr/v4/codegen/model/decl/ElementListDecl.java index f5fa1bb..e89db11 100644 --- a/tool/src/org/antlr/v4/codegen/model/decl/ElementListDecl.java +++ b/tool/src/org/antlr/v4/codegen/model/decl/ElementListDecl.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/tool/src/org/antlr/v4/codegen/model/decl/RuleContextDecl.java b/tool/src/org/antlr/v4/codegen/model/decl/RuleContextDecl.java index a6c7eff..2169618 100644 --- a/tool/src/org/antlr/v4/codegen/model/decl/RuleContextDecl.java +++ b/tool/src/org/antlr/v4/codegen/model/decl/RuleContextDecl.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/tool/src/org/antlr/v4/codegen/model/decl/RuleContextListDecl.java b/tool/src/org/antlr/v4/codegen/model/decl/RuleContextListDecl.java index 8418dbf..decf0f6 100644 --- a/tool/src/org/antlr/v4/codegen/model/decl/RuleContextListDecl.java +++ b/tool/src/org/antlr/v4/codegen/model/decl/RuleContextListDecl.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/tool/src/org/antlr/v4/codegen/model/decl/StructDecl.java b/tool/src/org/antlr/v4/codegen/model/decl/StructDecl.java index 14a764c..f7a48ee 100644 --- a/tool/src/org/antlr/v4/codegen/model/decl/StructDecl.java +++ b/tool/src/org/antlr/v4/codegen/model/decl/StructDecl.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/tool/src/org/antlr/v4/codegen/model/decl/TokenDecl.java b/tool/src/org/antlr/v4/codegen/model/decl/TokenDecl.java index 6545125..b42e9c7 100644 --- a/tool/src/org/antlr/v4/codegen/model/decl/TokenDecl.java +++ b/tool/src/org/antlr/v4/codegen/model/decl/TokenDecl.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/tool/src/org/antlr/v4/codegen/model/decl/TokenListDecl.java b/tool/src/org/antlr/v4/codegen/model/decl/TokenListDecl.java index ecbfac7..beb334d 100644 --- a/tool/src/org/antlr/v4/codegen/model/decl/TokenListDecl.java +++ b/tool/src/org/antlr/v4/codegen/model/decl/TokenListDecl.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/tool/src/org/antlr/v4/codegen/model/decl/TokenTypeDecl.java b/tool/src/org/antlr/v4/codegen/model/decl/TokenTypeDecl.java index c13ce63..da9e037 100644 --- a/tool/src/org/antlr/v4/codegen/model/decl/TokenTypeDecl.java +++ b/tool/src/org/antlr/v4/codegen/model/decl/TokenTypeDecl.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/tool/src/org/antlr/v4/codegen/target/CSharpTarget.java b/tool/src/org/antlr/v4/codegen/target/CSharpTarget.java index 88d1438..e7a2322 100644 --- a/tool/src/org/antlr/v4/codegen/target/CSharpTarget.java +++ b/tool/src/org/antlr/v4/codegen/target/CSharpTarget.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ @@ -7,6 +7,7 @@ package org.antlr.v4.codegen.target; import org.antlr.v4.codegen.CodeGenerator; import org.antlr.v4.codegen.Target; +import org.antlr.v4.codegen.UnicodeEscapes; import org.antlr.v4.tool.ErrorType; import org.antlr.v4.tool.ast.GrammarAST; import org.stringtemplate.v4.NumberRenderer; @@ -27,7 +28,7 @@ public class CSharpTarget extends Target { @Override public String getVersion() { - return "4.6"; + return "4.7.1"; } @Override @@ -36,78 +37,18 @@ public class CSharpTarget extends Target { throw new IllegalArgumentException(String.format("Cannot encode the specified value: %d", v)); } + String formatted; if (v >= 0 && v < targetCharValueEscape.length && targetCharValueEscape[v] != null) { - return targetCharValueEscape[v]; + formatted = targetCharValueEscape[v]; } - - if (v >= 0x20 && v < 127 && (v < '0' || v > '9') && (v < 'a' || v > 'f') && (v < 'A' || v > 'F')) { - return String.valueOf((char)v); + else if (v >= 0x20 && v < 127 && (v < '0' || v > '9') && (v < 'a' || v > 'f') && (v < 'A' || v > 'F')) { + formatted = Character.toString((char)v); } - - return String.format("\\x%X", v & 0xFFFF); - } - - @Override - public String getTargetStringLiteralFromANTLRStringLiteral( - CodeGenerator generator, - String literal, boolean addQuotes) - { - StringBuilder sb = new StringBuilder(); - String is = literal; - - if ( addQuotes ) sb.append('"'); - - for (int i = 1; i < is.length() -1; i++) { - if (is.charAt(i) == '\\') { - // Anything escaped is what it is! We assume that - // people know how to escape characters correctly. However - // we catch anything that does not need an escape in Java (which - // is what the default implementation is dealing with and remove - // the escape. The C target does this for instance. - // - switch (is.charAt(i+1)) { - // Pass through any escapes that Java also needs - // - case '"': - case 'n': - case 'r': - case 't': - case 'b': - case 'f': - case '\\': - // Pass the escape through - sb.append('\\'); - break; - - case 'u': // Assume unnnn - // Pass the escape through as double \\ - // so that Java leaves as \u0000 string not char - sb.append('\\'); - sb.append('\\'); - break; - - default: - // Remove the escape by virtue of not adding it here - // Thus \' becomes ' and so on - break; - } - - // Go past the \ character - i++; - } else { - // Characters that don't need \ in ANTLR 'strings' but do in Java - if (is.charAt(i) == '"') { - // We need to escape " in Java - sb.append('\\'); - } - } - // Add in the next character, which may have been escaped - sb.append(is.charAt(i)); + else { + formatted = String.format("\\x%X", v & 0xFFFF); } - if ( addQuotes ) sb.append('"'); - - return sb.toString(); + return "'" + formatted + "'"; } @Override @@ -150,4 +91,9 @@ public class CSharpTarget extends Target { return result; } + @Override + protected void appendUnicodeEscapedCodePoint(int codePoint, StringBuilder sb) { + // C# and Python share the same escaping style. + UnicodeEscapes.appendPythonStyleEscapedCodePoint(codePoint, sb); + } } diff --git a/tool/src/org/antlr/v4/codegen/target/CppTarget.java b/tool/src/org/antlr/v4/codegen/target/CppTarget.java index 14ac858..6ef32c2 100644 --- a/tool/src/org/antlr/v4/codegen/target/CppTarget.java +++ b/tool/src/org/antlr/v4/codegen/target/CppTarget.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ @@ -8,6 +8,7 @@ package org.antlr.v4.codegen.target; import org.antlr.v4.codegen.CodeGenerator; import org.antlr.v4.codegen.Target; +import org.antlr.v4.codegen.UnicodeEscapes; import org.antlr.v4.tool.ErrorType; import org.antlr.v4.tool.ast.GrammarAST; import org.stringtemplate.v4.NumberRenderer; @@ -49,7 +50,7 @@ public class CppTarget extends Target { } public String getVersion() { - return "4.6"; + return "4.7.1"; } public boolean needsHeader() { return true; } @@ -68,81 +69,6 @@ public class CppTarget extends Target { badWords.add("parserRule"); } - /** - * {@inheritDoc} - * <p/> - * For C++, this is the translation {@code 'a\n"'} → {@code "a\n\""}. - * Expect single quotes around the incoming literal. Just flip the quotes - * and replace double quotes with {@code \"}. - * <p/> - * Note that we have decided to allow people to use '\"' without penalty, so - * we must build the target string in a loop as {@link String#replace} - * cannot handle both {@code \"} and {@code "} without a lot of messing - * around. - */ - @Override - public String getTargetStringLiteralFromANTLRStringLiteral( - CodeGenerator generator, - String literal, boolean addQuotes) - { - StringBuilder sb = new StringBuilder(); - String is = literal; - - if ( addQuotes ) sb.append('"'); - - for (int i = 1; i < is.length() -1; i++) { - if (is.charAt(i) == '\\') { - // Anything escaped is what it is! We assume that - // people know how to escape characters correctly. However - // we catch anything that does not need an escape in Java (which - // is what the default implementation is dealing with and remove - // the escape. The C target does this for instance. - // - switch (is.charAt(i+1)) { - // Pass through any escapes that Java also needs - // - case '"': - case 'n': - case 'r': - case 't': - case 'b': - case 'f': - case '\\': - // Pass the escape through - sb.append('\\'); - break; - - case 'u': // Assume unnnn - // Pass the escape through as double \\ - // so that Java leaves as \u0000 string not char - sb.append('\\'); - sb.append('\\'); - break; - - default: - // Remove the escape by virtue of not adding it here - // Thus \' becomes ' and so on - break; - } - - // Go past the \ character - i++; - } else { - // Characters that don't need \ in ANTLR 'strings' but do in Java - if (is.charAt(i) == '"') { - // We need to escape " in Java - sb.append('\\'); - } - } - // Add in the next character, which may have been escaped - sb.append(is.charAt(i)); - } - - if ( addQuotes ) sb.append('"'); - - return sb.toString(); - } - @Override public String encodeIntAsCharEscape(int v) { return "0x" + Integer.toHexString(v) + ", "; @@ -232,4 +158,10 @@ public class CppTarget extends Target { return result; } + + @Override + protected void appendUnicodeEscapedCodePoint(int codePoint, StringBuilder sb) { + // C99 and Python share the same escaping style. + UnicodeEscapes.appendPythonStyleEscapedCodePoint(codePoint, sb); + } } diff --git a/tool/src/org/antlr/v4/codegen/target/GoTarget.java b/tool/src/org/antlr/v4/codegen/target/GoTarget.java index 678b532..be4e3cc 100644 --- a/tool/src/org/antlr/v4/codegen/target/GoTarget.java +++ b/tool/src/org/antlr/v4/codegen/target/GoTarget.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ @@ -8,6 +8,7 @@ package org.antlr.v4.codegen.target; import org.antlr.v4.codegen.CodeGenerator; import org.antlr.v4.codegen.Target; +import org.antlr.v4.codegen.UnicodeEscapes; import org.antlr.v4.parse.ANTLRParser; import org.antlr.v4.tool.Grammar; import org.antlr.v4.tool.ast.GrammarAST; @@ -48,8 +49,18 @@ public class GoTarget extends Target { "make", "new", "panic", "print", "println", "real", "recover" }; + // interface definition of RuleContext from runtime/Go/antlr/rule_context.go + private static final String[] goRuleContextInterfaceMethods = { + "Accept", "GetAltNumber", "GetBaseRuleContext", "GetChild", "GetChildCount", + "GetChildren", "GetInvokingState", "GetParent", "GetPayload", "GetRuleContext", + "GetRuleIndex", "GetSourceInterval", "GetText", "IsEmpty", "SetAltNumber", + "SetInvokingState", "SetParent", "String" + }; + /** Avoid grammar symbols in this set to prevent conflicts in gen'd code. */ - private final Set<String> badWords = new HashSet<String>(goKeywords.length + goPredeclaredIdentifiers.length + 2); + private final Set<String> badWords = new HashSet<String>( + goKeywords.length + goPredeclaredIdentifiers.length + goRuleContextInterfaceMethods.length + 3 + ); private static final boolean DO_GOFMT = !Boolean.parseBoolean(System.getenv("ANTLR_GO_DISABLE_GOFMT")) && !Boolean.parseBoolean(System.getProperty("antlr.go.disable-gofmt")); @@ -58,12 +69,12 @@ public class GoTarget extends Target { super(gen, "Go"); } - @Override - public String getVersion() { - return "4.6"; + @Override + public String getVersion() { + return "4.7.1"; } - public Set<String> getBadWords() { + public Set<String> getBadWords() { if (badWords.isEmpty()) { addBadWords(); } @@ -74,8 +85,10 @@ public class GoTarget extends Target { protected void addBadWords() { badWords.addAll(Arrays.asList(goKeywords)); badWords.addAll(Arrays.asList(goPredeclaredIdentifiers)); + badWords.addAll(Arrays.asList(goRuleContextInterfaceMethods)); badWords.add("rule"); badWords.add("parserRule"); + badWords.add("action"); } @Override @@ -214,5 +227,10 @@ public class GoTarget extends Target { } } -} + @Override + protected void appendUnicodeEscapedCodePoint(int codePoint, StringBuilder sb) { + // Go and Python share the same escaping style. + UnicodeEscapes.appendPythonStyleEscapedCodePoint(codePoint, sb); + } +} diff --git a/tool/src/org/antlr/v4/codegen/target/JavaScriptTarget.java b/tool/src/org/antlr/v4/codegen/target/JavaScriptTarget.java index 455c488..adbfc7c 100644 --- a/tool/src/org/antlr/v4/codegen/target/JavaScriptTarget.java +++ b/tool/src/org/antlr/v4/codegen/target/JavaScriptTarget.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ @@ -8,6 +8,7 @@ package org.antlr.v4.codegen.target; import org.antlr.v4.codegen.CodeGenerator; import org.antlr.v4.codegen.Target; +import org.antlr.v4.codegen.UnicodeEscapes; import org.antlr.v4.tool.ast.GrammarAST; import org.stringtemplate.v4.STGroup; import org.stringtemplate.v4.StringRenderer; @@ -50,7 +51,7 @@ public class JavaScriptTarget extends Target { @Override public String getVersion() { - return "4.6"; + return "4.7.1"; } public Set<String> getBadWords() { @@ -67,81 +68,6 @@ public class JavaScriptTarget extends Target { badWords.add("parserRule"); } - /** - * {@inheritDoc} - * <p> - * For Java, this is the translation {@code 'a\n"'} → {@code "a\n\""}. - * Expect single quotes around the incoming literal. Just flip the quotes - * and replace double quotes with {@code \"}. - * <p> - * Note that we have decided to allow people to use '\"' without penalty, so - * we must build the target string in a loop as {@link String#replace} - * cannot handle both {@code \"} and {@code "} without a lot of messing - * around. - */ - @Override - public String getTargetStringLiteralFromANTLRStringLiteral( - CodeGenerator generator, - String literal, boolean addQuotes) - { - StringBuilder sb = new StringBuilder(); - String is = literal; - - if ( addQuotes ) sb.append('"'); - - for (int i = 1; i < is.length() -1; i++) { - if (is.charAt(i) == '\\') { - // Anything escaped is what it is! We assume that - // people know how to escape characters correctly. However - // we catch anything that does not need an escape in Java (which - // is what the default implementation is dealing with and remove - // the escape. The C target does this for instance. - // - switch (is.charAt(i+1)) { - // Pass through any escapes that Java also needs - // - case '"': - case 'n': - case 'r': - case 't': - case 'b': - case 'f': - case '\\': - // Pass the escape through - sb.append('\\'); - break; - - case 'u': // Assume unnnn - // Pass the escape through as double \\ - // so that Java leaves as \u0000 string not char - sb.append('\\'); - sb.append('\\'); - break; - - default: - // Remove the escape by virtue of not adding it here - // Thus \' becomes ' and so on - break; - } - - // Go past the \ character - i++; - } else { - // Characters that don't need \ in ANTLR 'strings' but do in Java - if (is.charAt(i) == '"') { - // We need to escape " in Java - sb.append('\\'); - } - } - // Add in the next character, which may have been escaped - sb.append(is.charAt(i)); - } - - if ( addQuotes ) sb.append('"'); - - return sb.toString(); - } - @Override public String encodeIntAsCharEscape(int v) { if (v < Character.MIN_VALUE || v > Character.MAX_VALUE) { @@ -210,4 +136,10 @@ public class JavaScriptTarget extends Target { public boolean supportsOverloadedMethods() { return false; } + + @Override + protected void appendUnicodeEscapedCodePoint(int codePoint, StringBuilder sb) { + // JavaScript and Java share the same escaping style. + UnicodeEscapes.appendJavaStyleEscapedCodePoint(codePoint, sb); + } } diff --git a/tool/src/org/antlr/v4/codegen/target/JavaTarget.java b/tool/src/org/antlr/v4/codegen/target/JavaTarget.java index a7cb900..2482fef 100644 --- a/tool/src/org/antlr/v4/codegen/target/JavaTarget.java +++ b/tool/src/org/antlr/v4/codegen/target/JavaTarget.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ @@ -9,6 +9,7 @@ package org.antlr.v4.codegen.target; import org.antlr.v4.Tool; import org.antlr.v4.codegen.CodeGenerator; import org.antlr.v4.codegen.Target; +import org.antlr.v4.codegen.UnicodeEscapes; import org.antlr.v4.tool.ast.GrammarAST; import org.stringtemplate.v4.STGroup; import org.stringtemplate.v4.StringRenderer; @@ -99,4 +100,9 @@ public class JavaTarget extends Target { } } + + @Override + protected void appendUnicodeEscapedCodePoint(int codePoint, StringBuilder sb) { + UnicodeEscapes.appendJavaStyleEscapedCodePoint(codePoint, sb); + } } diff --git a/tool/src/org/antlr/v4/codegen/target/Python2Target.java b/tool/src/org/antlr/v4/codegen/target/Python2Target.java index 85cdb9c..3fc6a35 100644 --- a/tool/src/org/antlr/v4/codegen/target/Python2Target.java +++ b/tool/src/org/antlr/v4/codegen/target/Python2Target.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ @@ -8,6 +8,7 @@ package org.antlr.v4.codegen.target; import org.antlr.v4.codegen.CodeGenerator; import org.antlr.v4.codegen.Target; +import org.antlr.v4.codegen.UnicodeEscapes; import org.antlr.v4.tool.ast.GrammarAST; import org.stringtemplate.v4.STGroup; import org.stringtemplate.v4.StringRenderer; @@ -93,7 +94,7 @@ public class Python2Target extends Target { @Override public String getVersion() { - return "4.6"; + return "4.7.1"; } public Set<String> getBadWords() { @@ -109,4 +110,9 @@ public class Python2Target extends Target { badWords.add("rule"); badWords.add("parserRule"); } + + @Override + protected void appendUnicodeEscapedCodePoint(int codePoint, StringBuilder sb) { + UnicodeEscapes.appendPythonStyleEscapedCodePoint(codePoint, sb); + } } diff --git a/tool/src/org/antlr/v4/codegen/target/Python3Target.java b/tool/src/org/antlr/v4/codegen/target/Python3Target.java index e151ea0..388269a 100644 --- a/tool/src/org/antlr/v4/codegen/target/Python3Target.java +++ b/tool/src/org/antlr/v4/codegen/target/Python3Target.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ @@ -8,6 +8,7 @@ package org.antlr.v4.codegen.target; import org.antlr.v4.codegen.CodeGenerator; import org.antlr.v4.codegen.Target; +import org.antlr.v4.codegen.UnicodeEscapes; import org.antlr.v4.tool.ast.GrammarAST; import org.stringtemplate.v4.STGroup; import org.stringtemplate.v4.StringRenderer; @@ -55,7 +56,7 @@ public class Python3Target extends Target { @Override public int getSerializedATNSegmentLimit() { // set to something stupid to avoid segmentation - return 2 ^ 31; + return Integer.MAX_VALUE; } @Override @@ -95,7 +96,7 @@ public class Python3Target extends Target { @Override public String getVersion() { - return "4.6"; + return "4.7.1"; } /** Avoid grammar symbols in this set to prevent conflicts in gen'd code. */ @@ -115,5 +116,8 @@ public class Python3Target extends Target { badWords.add("parserRule"); } - + @Override + protected void appendUnicodeEscapedCodePoint(int codePoint, StringBuilder sb) { + UnicodeEscapes.appendPythonStyleEscapedCodePoint(codePoint, sb); + } } diff --git a/tool/src/org/antlr/v4/codegen/target/SwiftTarget.java b/tool/src/org/antlr/v4/codegen/target/SwiftTarget.java index b4db954..5e28506 100644 --- a/tool/src/org/antlr/v4/codegen/target/SwiftTarget.java +++ b/tool/src/org/antlr/v4/codegen/target/SwiftTarget.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ @@ -8,6 +8,7 @@ package org.antlr.v4.codegen.target; import org.antlr.v4.codegen.CodeGenerator; import org.antlr.v4.codegen.Target; +import org.antlr.v4.codegen.UnicodeEscapes; import org.antlr.v4.runtime.Token; import org.antlr.v4.runtime.atn.ATN; import org.antlr.v4.runtime.atn.ATNDeserializer; @@ -86,7 +87,7 @@ public class SwiftTarget extends Target { @Override public String getVersion() { - return "4.6"; // Java and tool versions move in lock step + return "4.7.1"; // Java and tool versions move in lock step } public Set<String> getBadWords() { @@ -123,7 +124,8 @@ public class SwiftTarget extends Target { if (g.isLexer() && lexerAtnJSON == null) { lexerAtnJSON = getLexerOrParserATNJson(g, fileName); - } else if (!g.isLexer() && parserAtnJSON == null && g.atn != null) { + } + else if (!g.isLexer() && parserAtnJSON == null && g.atn != null) { parserAtnJSON = getLexerOrParserATNJson(g, fileName); } @@ -550,4 +552,9 @@ public class SwiftTarget extends Target { } } + + @Override + protected void appendUnicodeEscapedCodePoint(int codePoint, StringBuilder sb) { + UnicodeEscapes.appendSwiftStyleEscapedCodePoint(codePoint, sb); + } } diff --git a/tool/src/org/antlr/v4/gui/BasicFontMetrics.java b/tool/src/org/antlr/v4/gui/BasicFontMetrics.java index 39ce6bd..f145e85 100644 --- a/tool/src/org/antlr/v4/gui/BasicFontMetrics.java +++ b/tool/src/org/antlr/v4/gui/BasicFontMetrics.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/tool/src/org/antlr/v4/gui/GraphicsSupport.java b/tool/src/org/antlr/v4/gui/GraphicsSupport.java index fbee7f4..74fa215 100644 --- a/tool/src/org/antlr/v4/gui/GraphicsSupport.java +++ b/tool/src/org/antlr/v4/gui/GraphicsSupport.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ @@ -93,7 +93,8 @@ public class GraphicsSupport { job.print(doc, attributes); out.close(); } - } else { + } + else { // parrt: works with [image/jpeg, image/png, image/x-png, image/vnd.wap.wbmp, image/bmp, image/gif] Rectangle rect = comp.getBounds(); BufferedImage image = new BufferedImage(rect.width, rect.height, diff --git a/tool/src/org/antlr/v4/gui/JFileChooserConfirmOverwrite.java b/tool/src/org/antlr/v4/gui/JFileChooserConfirmOverwrite.java index f30b555..e6788e4 100644 --- a/tool/src/org/antlr/v4/gui/JFileChooserConfirmOverwrite.java +++ b/tool/src/org/antlr/v4/gui/JFileChooserConfirmOverwrite.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/tool/src/org/antlr/v4/gui/PostScriptDocument.java b/tool/src/org/antlr/v4/gui/PostScriptDocument.java index 62e2216..e3f9c22 100644 --- a/tool/src/org/antlr/v4/gui/PostScriptDocument.java +++ b/tool/src/org/antlr/v4/gui/PostScriptDocument.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/tool/src/org/antlr/v4/gui/SystemFontMetrics.java b/tool/src/org/antlr/v4/gui/SystemFontMetrics.java index f1b4d91..ff40f1a 100644 --- a/tool/src/org/antlr/v4/gui/SystemFontMetrics.java +++ b/tool/src/org/antlr/v4/gui/SystemFontMetrics.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/tool/src/org/antlr/v4/gui/TestRig.java b/tool/src/org/antlr/v4/gui/TestRig.java index 1fc96f0..a466363 100644 --- a/tool/src/org/antlr/v4/gui/TestRig.java +++ b/tool/src/org/antlr/v4/gui/TestRig.java @@ -1,13 +1,13 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ package org.antlr.v4.gui; -import org.antlr.v4.runtime.ANTLRInputStream; import org.antlr.v4.runtime.CharStream; +import org.antlr.v4.runtime.CharStreams; import org.antlr.v4.runtime.CommonToken; import org.antlr.v4.runtime.CommonTokenStream; import org.antlr.v4.runtime.DiagnosticErrorListener; @@ -19,14 +19,12 @@ import org.antlr.v4.runtime.TokenStream; import org.antlr.v4.runtime.atn.PredictionMode; import javax.print.PrintException; -import java.io.FileInputStream; import java.io.IOException; -import java.io.InputStream; -import java.io.InputStreamReader; -import java.io.Reader; import java.lang.reflect.Constructor; import java.lang.reflect.InvocationTargetException; import java.lang.reflect.Method; +import java.nio.charset.Charset; +import java.nio.file.Paths; import java.util.ArrayList; import java.util.List; @@ -154,42 +152,22 @@ public class TestRig { parser = parserCtor.newInstance((TokenStream)null); } + Charset charset = ( encoding == null ? Charset.defaultCharset () : Charset.forName(encoding) ); if ( inputFiles.size()==0 ) { - InputStream is = System.in; - Reader r; - if ( encoding!=null ) { - r = new InputStreamReader(is, encoding); - } - else { - r = new InputStreamReader(is); - } - - process(lexer, parserClass, parser, is, r); + CharStream charStream = CharStreams.fromStream(System.in, charset); + process(lexer, parserClass, parser, charStream); return; } for (String inputFile : inputFiles) { - InputStream is = System.in; - if ( inputFile!=null ) { - is = new FileInputStream(inputFile); - } - Reader r; - if ( encoding!=null ) { - r = new InputStreamReader(is, encoding); - } - else { - r = new InputStreamReader(is); - } - + CharStream charStream = CharStreams.fromPath(Paths.get(inputFile), charset); if ( inputFiles.size()>1 ) { System.err.println(inputFile); } - process(lexer, parserClass, parser, is, r); + process(lexer, parserClass, parser, charStream); } } - protected void process(Lexer lexer, Class<? extends Parser> parserClass, Parser parser, InputStream is, Reader r) throws IOException, IllegalAccessException, InvocationTargetException, PrintException { - try { - ANTLRInputStream input = new ANTLRInputStream(r); + protected void process(Lexer lexer, Class<? extends Parser> parserClass, Parser parser, CharStream input) throws IOException, IllegalAccessException, InvocationTargetException, PrintException { lexer.setInputStream(input); CommonTokenStream tokens = new CommonTokenStream(lexer); @@ -242,9 +220,4 @@ public class TestRig { System.err.println("No method for rule "+startRuleName+" or it has arguments"); } } - finally { - if ( r!=null ) r.close(); - if ( is!=null ) is.close(); - } - } } diff --git a/tool/src/org/antlr/v4/gui/TreeLayoutAdaptor.java b/tool/src/org/antlr/v4/gui/TreeLayoutAdaptor.java index 295ab99..d821efb 100644 --- a/tool/src/org/antlr/v4/gui/TreeLayoutAdaptor.java +++ b/tool/src/org/antlr/v4/gui/TreeLayoutAdaptor.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/tool/src/org/antlr/v4/gui/TreePostScriptGenerator.java b/tool/src/org/antlr/v4/gui/TreePostScriptGenerator.java index 19f3512..d0411ee 100644 --- a/tool/src/org/antlr/v4/gui/TreePostScriptGenerator.java +++ b/tool/src/org/antlr/v4/gui/TreePostScriptGenerator.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/tool/src/org/antlr/v4/gui/TreeTextProvider.java b/tool/src/org/antlr/v4/gui/TreeTextProvider.java index ff3983e..59c6c38 100644 --- a/tool/src/org/antlr/v4/gui/TreeTextProvider.java +++ b/tool/src/org/antlr/v4/gui/TreeTextProvider.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/tool/src/org/antlr/v4/gui/TreeViewer.java b/tool/src/org/antlr/v4/gui/TreeViewer.java index 21d0f5c..2c000aa 100644 --- a/tool/src/org/antlr/v4/gui/TreeViewer.java +++ b/tool/src/org/antlr/v4/gui/TreeViewer.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ @@ -37,7 +37,10 @@ import java.awt.geom.CubicCurve2D; import java.awt.geom.Rectangle2D; import java.awt.image.BufferedImage; import java.io.File; +import java.io.FileWriter; +import java.io.BufferedWriter; import java.io.IOException; +import java.io.Writer; import java.util.ArrayList; import java.util.Collection; import java.util.List; @@ -159,7 +162,8 @@ public class TreeViewer extends JComponent { double ctrly2 = y1; c.setCurve(x1, y1, ctrlx1, ctrly1, ctrlx2, ctrly2, x2, y2); ((Graphics2D) g).draw(c); - } else { + } + else { g.drawLine((int) x1, (int) y1, (int) x2, (int) y2); } @@ -244,6 +248,69 @@ public class TreeViewer extends JComponent { } } + protected void generateEdges(Writer writer, Tree parent) throws IOException { + if (!getTree().isLeaf(parent)) { + Rectangle2D.Double b1 = getBoundsOfNode(parent); + double x1 = b1.getCenterX(); + double y1 = b1.getCenterY(); + + for (Tree child : getTree().getChildren(parent)) { + Rectangle2D.Double childBounds = getBoundsOfNode(child); + double x2 = childBounds.getCenterX(); + double y2 = childBounds.getMinY(); + writer.write(line(""+x1, ""+y1, ""+x2, ""+y2, + "stroke:black; stroke-width:1px;")); + generateEdges(writer, child); + } + } + } + + protected void generateBox(Writer writer, Tree parent) throws IOException { + + // draw the box in the background + Rectangle2D.Double box = getBoundsOfNode(parent); + writer.write(rect(""+box.x, ""+box.y, ""+box.width, ""+box.height, + "fill:orange; stroke:rgb(0,0,0);", "rx=\"1\"")); + + // draw the text on top of the box (possibly multiple lines) + String line = getText(parent).replace("<","<").replace(">",">"); + int fontSize = 10; + int x = (int) box.x + 2; + int y = (int) box.y + fontSize - 1; + String style = String.format("font-family:sans-serif;font-size:%dpx;", + fontSize); + writer.write(text(""+x, ""+y, style, line)); + } + + private static String line(String x1, String y1, String x2, String y2, + String style) { + return String + .format("<line x1=\"%s\" y1=\"%s\" x2=\"%s\" y2=\"%s\" style=\"%s\" />\n", + x1, y1, x2, y2, style); + } + + private static String rect(String x, String y, String width, String height, + String style, String extraAttributes) { + return String + .format("<rect x=\"%s\" y=\"%s\" width=\"%s\" height=\"%s\" style=\"%s\" %s/>\n", + x, y, width, height, style, extraAttributes); + } + + private static String text(String x, String y, String style, String text) { + return String.format( + "<text x=\"%s\" y=\"%s\" style=\"%s\">\n%s\n</text>\n", x, y, + style, text); + } + + private void paintSVG(Writer writer) throws IOException { + + generateEdges(writer, getTree().getRoot()); + + for (Tree tree : treeLayout.getNodeBounds().keySet()) { + generateBox(writer, tree); + } + } + @Override protected Graphics getComponentGraphics(Graphics g) { Graphics2D g2d=(Graphics2D)g; @@ -306,6 +373,18 @@ public class TreeViewer extends JComponent { ); wrapper.add(png); + // Add an export-to-png button right of the "OK" button + JButton svg = new JButton("Export as SVG"); + svg.addActionListener( + new ActionListener() { + @Override + public void actionPerformed(ActionEvent e) { + generateSVGFile(viewer, dialog); + } + } + ); + wrapper.add(svg); + bottomPanel.add(wrapper, BorderLayout.SOUTH); // Add scale slider @@ -417,29 +496,7 @@ public class TreeViewer extends JComponent { g.dispose(); try { - File suggestedFile = generateNonExistingPngFile(); - JFileChooser fileChooser = new JFileChooserConfirmOverwrite(); - fileChooser.setCurrentDirectory(suggestedFile.getParentFile()); - fileChooser.setSelectedFile(suggestedFile); - FileFilter pngFilter = new FileFilter() { - - @Override - public boolean accept(File pathname) { - if (pathname.isFile()) { - return pathname.getName().toLowerCase().endsWith(".png"); - } - - return true; - } - - @Override - public String getDescription() { - return "PNG Files (*.png)"; - } - }; - - fileChooser.addChoosableFileFilter(pngFilter); - fileChooser.setFileFilter(pngFilter); + JFileChooser fileChooser = getFileChooser(".png", "PNG files"); int returnValue = fileChooser.showSaveDialog(dialog); if (returnValue == JFileChooser.APPROVE_OPTION) { @@ -468,23 +525,85 @@ public class TreeViewer extends JComponent { } } - private static File generateNonExistingPngFile() { + private static JFileChooser getFileChooser(final String fileEnding, + final String description) { + File suggestedFile = generateNonExistingFile(fileEnding); + JFileChooser fileChooser = new JFileChooserConfirmOverwrite(); + fileChooser.setCurrentDirectory(suggestedFile.getParentFile()); + fileChooser.setSelectedFile(suggestedFile); + FileFilter filter = new FileFilter() { + + @Override + public boolean accept(File pathname) { + if (pathname.isFile()) { + return pathname.getName().toLowerCase().endsWith(fileEnding); + } + + return true; + } + + @Override + public String getDescription() { + return description+" (*"+fileEnding+")"; + } + }; + fileChooser.addChoosableFileFilter(filter); + fileChooser.setFileFilter(filter); + return fileChooser; + } + + private static void generateSVGFile(TreeViewer viewer, JFrame dialog) { + + try { + JFileChooser fileChooser = getFileChooser(".svg", "SVG files"); + + int returnValue = fileChooser.showSaveDialog(dialog); + if (returnValue == JFileChooser.APPROVE_OPTION) { + File svgFile = fileChooser.getSelectedFile(); + // save the new svg file here! + BufferedWriter writer = new BufferedWriter(new FileWriter(svgFile)); + // HACK: multiplying with 1.1 should be replaced wit an accurate number + writer.write("<svg width=\"" + viewer.getSize().getWidth() * 1.1 + "\" height=\"" + viewer.getSize().getHeight() * 1.1 + "\" xmlns=\"http://www.w3.org/2000/svg\" xmlns:xlink=\"http://www.w3.org/1999/xlink\">"); + viewer.paintSVG(writer); + writer.write("</svg>"); + writer.flush(); + writer.close(); + try { + // Try to open the parent folder using the OS' native file manager. + Desktop.getDesktop().open(svgFile.getParentFile()); + } catch (Exception ex) { + // We could not launch the file manager: just show a popup that we + // succeeded in saving the PNG file. + JOptionPane.showMessageDialog(dialog, "Saved SVG to: " + + svgFile.getAbsolutePath()); + ex.printStackTrace(); + } + } + } catch (Exception ex) { + JOptionPane.showMessageDialog(dialog, + "Could not export to SVG: " + ex.getMessage(), + "Error", + JOptionPane.ERROR_MESSAGE); + ex.printStackTrace(); + } + } + + private static File generateNonExistingFile(String extension) { final String parent = "."; final String name = "antlr4_parse_tree"; - final String extension = ".png"; - File pngFile = new File(parent, name + extension); + File file = new File(parent, name + extension); int counter = 1; // Keep looping until we create a File that does not yet exist. - while (pngFile.exists()) { - pngFile = new File(parent, name + "_"+ counter + extension); + while (file.exists()) { + file = new File(parent, name + "_" + counter + extension); counter++; } - return pngFile; + return file; } private static void fillTree(TreeNodeWrapper node, Tree tree, TreeViewer viewer) { diff --git a/tool/src/org/antlr/v4/gui/Trees.java b/tool/src/org/antlr/v4/gui/Trees.java index eca200a..5db80ca 100644 --- a/tool/src/org/antlr/v4/gui/Trees.java +++ b/tool/src/org/antlr/v4/gui/Trees.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/tool/src/org/antlr/v4/misc/CharSupport.java b/tool/src/org/antlr/v4/misc/CharSupport.java index 50db0d9..47e0033 100644 --- a/tool/src/org/antlr/v4/misc/CharSupport.java +++ b/tool/src/org/antlr/v4/misc/CharSupport.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ @@ -7,6 +7,10 @@ package org.antlr.v4.misc; import org.antlr.v4.runtime.Lexer; +import org.antlr.v4.runtime.misc.Interval; +import org.antlr.v4.runtime.misc.IntervalSet; + +import java.util.Iterator; /** */ public class CharSupport { @@ -26,47 +30,48 @@ public class CharSupport { ANTLRLiteralEscapedCharValue['b'] = '\b'; ANTLRLiteralEscapedCharValue['f'] = '\f'; ANTLRLiteralEscapedCharValue['\\'] = '\\'; - ANTLRLiteralEscapedCharValue['\''] = '\''; - ANTLRLiteralEscapedCharValue['"'] = '"'; - ANTLRLiteralEscapedCharValue['-'] = '-'; - ANTLRLiteralEscapedCharValue[']'] = ']'; ANTLRLiteralCharValueEscape['\n'] = "\\n"; ANTLRLiteralCharValueEscape['\r'] = "\\r"; ANTLRLiteralCharValueEscape['\t'] = "\\t"; ANTLRLiteralCharValueEscape['\b'] = "\\b"; ANTLRLiteralCharValueEscape['\f'] = "\\f"; ANTLRLiteralCharValueEscape['\\'] = "\\\\"; - ANTLRLiteralCharValueEscape['\''] = "\\'"; } /** Return a string representing the escaped char for code c. E.g., If c - * has value 0x100, you will get "\u0100". ASCII gets the usual - * char (non-hex) representation. Control characters are spit out - * as unicode. While this is specially set up for returning Java strings, - * it can be used by any language target that has the same syntax. :) + * has value 0x100, you will get "\\u0100". ASCII gets the usual + * char (non-hex) representation. Non-ASCII characters are spit out + * as \\uXXXX or \\u{XXXXXX} escapes. */ public static String getANTLRCharLiteralForChar(int c) { - if ( c< Lexer.MIN_CHAR_VALUE ) { - return "'<INVALID>'"; - } - if ( c<ANTLRLiteralCharValueEscape.length && ANTLRLiteralCharValueEscape[c]!=null ) { - return '\''+ANTLRLiteralCharValueEscape[c]+'\''; + String result; + if ( c < Lexer.MIN_CHAR_VALUE ) { + result = "<INVALID>"; } - if ( Character.UnicodeBlock.of((char)c)==Character.UnicodeBlock.BASIC_LATIN && - !Character.isISOControl((char)c) ) { - if ( c=='\\' ) { - return "'\\\\'"; + else { + String charValueEscape = c < ANTLRLiteralCharValueEscape.length ? ANTLRLiteralCharValueEscape[c] : null; + if (charValueEscape != null) { + result = charValueEscape; + } + else if (Character.UnicodeBlock.of((char) c) == Character.UnicodeBlock.BASIC_LATIN && + !Character.isISOControl((char) c)) { + if (c == '\\') { + result = "\\\\"; + } + else if (c == '\'') { + result = "\\'"; + } + else { + result = Character.toString((char) c); + } } - if ( c=='\'') { - return "'\\''"; + else if (c <= 0xFFFF) { + result = String.format("\\u%04X", c); + } else { + result = String.format("\\u{%06X}", c); } - return '\''+Character.toString((char)c)+'\''; } - // turn on the bit above max "\uFFFF" value so that we pad with zeros - // then only take last 4 digits - String hex = Integer.toHexString(c|0x10000).toUpperCase().substring(1,5); - String unicodeStr = "'\\u"+hex+"'"; - return unicodeStr; + return '\'' + result + '\''; } /** Given a literal like (the 3 char sequence with single quotes) 'a', @@ -87,11 +92,26 @@ public class CharSupport { if ( literal.charAt(i) == '\\' ) { end = i+2; if ( i+1 < n && literal.charAt(i+1) == 'u' ) { - for (end = i + 2; end < i + 6; end++) { - if ( end>n ) return null; // invalid escape sequence. - char charAt = literal.charAt(end); - if (!Character.isDigit(charAt) && !(charAt >= 'a' && charAt <= 'f') && !(charAt >= 'A' && charAt <= 'F')) { - return null; // invalid escape sequence. + if ( i+2 < n && literal.charAt(i+2) == '{' ) { // extended escape sequence + end = i + 3; + while (true) { + if ( end + 1 > n ) return null; // invalid escape sequence. + char charAt = literal.charAt(end++); + if (charAt == '}') { + break; + } + if (!Character.isDigit(charAt) && !(charAt >= 'a' && charAt <= 'f') && !(charAt >= 'A' && charAt <= 'F')) { + return null; // invalid escape sequence. + } + } + } + else { + for (end = i + 2; end < i + 6; end++) { + if ( end>n ) return null; // invalid escape sequence. + char charAt = literal.charAt(end); + if (!Character.isDigit(charAt) && !(charAt >= 'a' && charAt <= 'f') && !(charAt >= 'A' && charAt <= 'F')) { + return null; // invalid escape sequence. + } } } } @@ -102,13 +122,13 @@ public class CharSupport { if ( c==-1 ) { return null; // invalid escape sequence. } - else buf.append((char)c); + else buf.appendCodePoint(c); i = end; } return buf.toString(); } - /** Given char x or \t or \u1234 return the char value; + /** Given char x or \\t or \\u1234 return the char value; * Unnecessary escapes like '\{' yield -1. */ public static int getCharValueFromCharInGrammarLiteral(String cstr) { @@ -119,28 +139,67 @@ public class CharSupport { case 2: if ( cstr.charAt(0)!='\\' ) return -1; // '\x' (antlr lexer will catch invalid char) - if ( Character.isDigit(cstr.charAt(1)) ) return -1; - int escChar = cstr.charAt(1); + char escChar = cstr.charAt(1); + if (escChar == '\'') return escChar; // escape quote only in string literals. int charVal = ANTLRLiteralEscapedCharValue[escChar]; - if ( charVal==0 ) return -1; + if (charVal == 0) return -1; return charVal; case 6: - // '\u1234' + // '\\u1234' or '\\u{12}' if ( !cstr.startsWith("\\u") ) return -1; - String unicodeChars = cstr.substring(2, cstr.length()); - int result = -1; - try { - result = Integer.parseInt(unicodeChars, 16); + int startOff; + int endOff; + if ( cstr.charAt(2) == '{' ) { + startOff = 3; + endOff = cstr.indexOf('}'); } - catch (NumberFormatException e) { + else { + startOff = 2; + endOff = cstr.length(); } - return result; + return parseHexValue(cstr, startOff, endOff); default: + if ( cstr.startsWith("\\u{") ) { + return parseHexValue(cstr, 3, cstr.indexOf('}')); + } return -1; } } + public static int parseHexValue(String cstr, int startOff, int endOff) { + if (startOff < 0 || endOff < 0) { + return -1; + } + String unicodeChars = cstr.substring(startOff, endOff); + int result = -1; + try { + result = Integer.parseInt(unicodeChars, 16); + } + catch (NumberFormatException e) { + } + return result; + } + public static String capitalize(String s) { return Character.toUpperCase(s.charAt(0)) + s.substring(1); } + + public static String getIntervalSetEscapedString(IntervalSet intervalSet) { + StringBuilder buf = new StringBuilder(); + Iterator<Interval> iter = intervalSet.getIntervals().iterator(); + while (iter.hasNext()) { + Interval interval = iter.next(); + buf.append(getRangeEscapedString(interval.a, interval.b)); + if (iter.hasNext()) { + buf.append(" | "); + } + } + return buf.toString(); + } + + public static String getRangeEscapedString(int codePointStart, int codePointEnd) { + return codePointStart != codePointEnd + ? getANTLRCharLiteralForChar(codePointStart) + ".." + getANTLRCharLiteralForChar(codePointEnd) + : getANTLRCharLiteralForChar(codePointStart); + } } diff --git a/tool/src/org/antlr/v4/misc/EscapeSequenceParsing.java b/tool/src/org/antlr/v4/misc/EscapeSequenceParsing.java new file mode 100644 index 0000000..d34988d --- /dev/null +++ b/tool/src/org/antlr/v4/misc/EscapeSequenceParsing.java @@ -0,0 +1,186 @@ +/* + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. + * Use of this file is governed by the BSD 3-clause license that + * can be found in the LICENSE.txt file in the project root. + */ + +package org.antlr.v4.misc; + +import org.antlr.v4.runtime.misc.IntervalSet; +import org.antlr.v4.unicode.UnicodeData; + +import java.util.Objects; + +/** + * Utility class to parse escapes like: + * \\n + * \\uABCD + * \\u{10ABCD} + * \\p{Foo} + * \\P{Bar} + * \\p{Baz=Blech} + * \\P{Baz=Blech} + */ +public abstract class EscapeSequenceParsing { + public static class Result { + public enum Type { + INVALID, + CODE_POINT, + PROPERTY + }; + + public final Type type; + public final int codePoint; + public final IntervalSet propertyIntervalSet; + public final int startOffset; + public final int parseLength; + + public Result(Type type, int codePoint, IntervalSet propertyIntervalSet, int startOffset, int parseLength) { + this.type = type; + this.codePoint = codePoint; + this.propertyIntervalSet = propertyIntervalSet; + this.startOffset = startOffset; + this.parseLength = parseLength; + } + + @Override + public String toString() { + return String.format( + "%s type=%s codePoint=%d propertyIntervalSet=%s parseLength=%d", + super.toString(), + type, + codePoint, + propertyIntervalSet, + parseLength); + } + + @Override + public boolean equals(Object other) { + if (!(other instanceof Result)) { + return false; + } + Result that = (Result) other; + if (this == that) { + return true; + } + return Objects.equals(this.type, that.type) && + Objects.equals(this.codePoint, that.codePoint) && + Objects.equals(this.propertyIntervalSet, that.propertyIntervalSet) && + Objects.equals(this.parseLength, that.parseLength); + } + + @Override + public int hashCode() { + return Objects.hash(type, codePoint, propertyIntervalSet, parseLength); + } + } + + /** + * Parses a single escape sequence starting at {@code startOff}. + * + * Returns a type of INVALID if no valid escape sequence was found, a Result otherwise. + */ + public static Result parseEscape(String s, int startOff) { + int offset = startOff; + if (offset + 2 > s.length() || s.codePointAt(offset) != '\\') { + return invalid(startOff, s.length()-1); + } + // Move past backslash + offset++; + int escaped = s.codePointAt(offset); + // Move past escaped code point + offset += Character.charCount(escaped); + if (escaped == 'u') { + // \\u{1} is the shortest we support + if (offset + 3 > s.length()) { + return invalid(startOff, s.length()-1); + } + int hexStartOffset; + int hexEndOffset; // appears to be exclusive + if (s.codePointAt(offset) == '{') { + hexStartOffset = offset + 1; + hexEndOffset = s.indexOf('}', hexStartOffset); + if (hexEndOffset == -1) { + return invalid(startOff, s.length()-1); + } + offset = hexEndOffset + 1; + } + else { + if (offset + 4 > s.length()) { + return invalid(startOff, s.length()-1); + } + hexStartOffset = offset; + hexEndOffset = offset + 4; + offset = hexEndOffset; + } + int codePointValue = CharSupport.parseHexValue(s, hexStartOffset, hexEndOffset); + if (codePointValue == -1 || codePointValue > Character.MAX_CODE_POINT) { + return invalid(startOff, startOff+6-1); + } + return new Result( + Result.Type.CODE_POINT, + codePointValue, + IntervalSet.EMPTY_SET, + startOff, + offset - startOff); + } + else if (escaped == 'p' || escaped == 'P') { + // \p{L} is the shortest we support + if (offset + 3 > s.length()) { + return invalid(startOff, s.length()-1); + } + if (s.codePointAt(offset) != '{') { + return invalid(startOff, offset); + } + int openBraceOffset = offset; + int closeBraceOffset = s.indexOf('}', openBraceOffset); + if (closeBraceOffset == -1) { + return invalid(startOff, s.length()-1); + } + String propertyName = s.substring(openBraceOffset + 1, closeBraceOffset); + IntervalSet propertyIntervalSet = UnicodeData.getPropertyCodePoints(propertyName); + if (propertyIntervalSet == null) { + return invalid(startOff, closeBraceOffset); + } + offset = closeBraceOffset + 1; + if (escaped == 'P') { + propertyIntervalSet = propertyIntervalSet.complement(IntervalSet.COMPLETE_CHAR_SET); + } + return new Result( + Result.Type.PROPERTY, + -1, + propertyIntervalSet, + startOff, + offset - startOff); + } + else if (escaped < CharSupport.ANTLRLiteralEscapedCharValue.length) { + int codePoint = CharSupport.ANTLRLiteralEscapedCharValue[escaped]; + if (codePoint == 0) { + if (escaped != ']' && escaped != '-') { // escape ']' and '-' only in char sets. + return invalid(startOff, startOff+1); + } + else { + codePoint = escaped; + } + } + return new Result( + Result.Type.CODE_POINT, + codePoint, + IntervalSet.EMPTY_SET, + startOff, + offset - startOff); + } + else { + return invalid(startOff,s.length()-1); + } + } + + private static Result invalid(int start, int stop) { // start..stop is inclusive + return new Result( + Result.Type.INVALID, + 0, + IntervalSet.EMPTY_SET, + start, + stop - start + 1); + } +} diff --git a/tool/src/org/antlr/v4/misc/FrequencySet.java b/tool/src/org/antlr/v4/misc/FrequencySet.java index d3d4c73..343f12b 100644 --- a/tool/src/org/antlr/v4/misc/FrequencySet.java +++ b/tool/src/org/antlr/v4/misc/FrequencySet.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/tool/src/org/antlr/v4/misc/Graph.java b/tool/src/org/antlr/v4/misc/Graph.java index ec4c147..ea10cc0 100644 --- a/tool/src/org/antlr/v4/misc/Graph.java +++ b/tool/src/org/antlr/v4/misc/Graph.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/tool/src/org/antlr/v4/misc/MutableInt.java b/tool/src/org/antlr/v4/misc/MutableInt.java index 607b8f6..007f4c2 100644 --- a/tool/src/org/antlr/v4/misc/MutableInt.java +++ b/tool/src/org/antlr/v4/misc/MutableInt.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/tool/src/org/antlr/v4/misc/OrderedHashMap.java b/tool/src/org/antlr/v4/misc/OrderedHashMap.java index 67f9deb..7a988f9 100644 --- a/tool/src/org/antlr/v4/misc/OrderedHashMap.java +++ b/tool/src/org/antlr/v4/misc/OrderedHashMap.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/tool/src/org/antlr/v4/misc/Utils.java b/tool/src/org/antlr/v4/misc/Utils.java index d9a2473..abdedb7 100644 --- a/tool/src/org/antlr/v4/misc/Utils.java +++ b/tool/src/org/antlr/v4/misc/Utils.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ @@ -122,7 +122,8 @@ public class Utils { public static void setSize(List<?> list, int size) { if (size < list.size()) { list.subList(size, list.size()).clear(); - } else { + } + else { while (size > list.size()) { list.add(null); } diff --git a/tool/src/org/antlr/v4/parse/ANTLRLexer.g b/tool/src/org/antlr/v4/parse/ANTLRLexer.g index 7d61c11..4f113a7 100644 --- a/tool/src/org/antlr/v4/parse/ANTLRLexer.g +++ b/tool/src/org/antlr/v4/parse/ANTLRLexer.g @@ -146,6 +146,7 @@ tokens { SEMPRED; TOKEN_REF; RULE_REF; LEXER_CHAR_SET; ARG_ACTION; } */ package org.antlr.v4.parse; import org.antlr.v4.tool.*; +import org.antlr.v4.runtime.misc.Interval; } @@ -615,8 +616,8 @@ SRC : 'src' WSCHARS+ file=ACTION_STRING_LITERAL WSCHARS+ line=INT // // ANTLR makes no disticintion between a single character literal and a // multi-character string. All literals are single quote delimited and -// may contain unicode escape sequences of the form \uxxxx, where x -// is a valid hexadecimal number (as per Java basically). +// may contain unicode escape sequences of the form \uxxxx or \u{xxxxxx}, +// where x is a valid hexadecimal number. STRING_LITERAL : '\'' ( ( ESC_SEQ | ~('\\'|'\''|'\r'|'\n') ) )* ( '\'' @@ -643,30 +644,29 @@ fragment ESC_SEQ : '\\' ( - // The standard escaped character set such as tab, newline, - // etc. - // - 'b'|'t'|'n'|'f'|'r'|'\"'|'\''|'\\' + // The standard escaped character set such as tab, newline, etc... + 'b'|'t'|'n'|'f'|'r'|'\''|'\\' | // A Java style Unicode escape sequence - // UNICODE_ESC + | // A Swift/Hack style Unicode escape sequence + UNICODE_EXTENDED_ESC + | // An illegal escape seqeunce - // + ~('b'|'t'|'n'|'f'|'r'|'\''|'\\'|'u') // \x for any invalid x (make sure to match char here) { - Token t = new CommonToken(input, state.type, state.channel, getCharIndex()-1, getCharIndex()); + Token t = new CommonToken(input, state.type, state.channel, getCharIndex()-2, getCharIndex()-1); t.setText(t.getText()); t.setLine(input.getLine()); - t.setCharPositionInLine(input.getCharPositionInLine()-1); - grammarError(ErrorType.INVALID_ESCAPE_SEQUENCE, t); + t.setCharPositionInLine(input.getCharPositionInLine()-2); + grammarError(ErrorType.INVALID_ESCAPE_SEQUENCE, t, input.substring(getCharIndex()-2,getCharIndex()-1)); } ) ; fragment UNICODE_ESC - @init { // Flag to tell us whether we have a valid number of @@ -710,16 +710,42 @@ UNICODE_ESC // Now check the digit count and issue an error if we need to // { - if (hCount != 4) { - Token t = new CommonToken(input, state.type, state.channel, getCharIndex()-3-hCount, getCharIndex()-1); - t.setText(t.getText()); - t.setLine(input.getLine()); - t.setCharPositionInLine(input.getCharPositionInLine()-hCount-2); - grammarError(ErrorType.INVALID_ESCAPE_SEQUENCE, t); + if (hCount < 4) { + Interval badRange = Interval.of(getCharIndex()-2-hCount, getCharIndex()); + String lastChar = input.substring(badRange.b, badRange.b); + if ( lastChar.codePointAt(0)=='\'' ) { + badRange.b--; + } + String bad = input.substring(badRange.a, badRange.b); + Token t = new CommonToken(input, state.type, state.channel, badRange.a, badRange.b); + t.setLine(input.getLine()); + t.setCharPositionInLine(input.getCharPositionInLine()-hCount-2); + grammarError(ErrorType.INVALID_ESCAPE_SEQUENCE, t, bad); } } ; +fragment +UNICODE_EXTENDED_ESC + : 'u{' // Leadin for unicode extended escape sequence + + HEX_DIGIT+ // One or more hexadecimal digits + + '}' // Leadout for unicode extended escape sequence + + // Now check the digit count and issue an error if we need to + { + int numDigits = getCharIndex()-state.tokenStartCharIndex-6; + if (numDigits > 6) { + Token t = new CommonToken(input, state.type, state.channel, state.tokenStartCharIndex, getCharIndex()-1); + t.setText(t.getText()); + t.setLine(input.getLine()); + t.setCharPositionInLine(input.getCharPositionInLine()-numDigits); + grammarError(ErrorType.INVALID_ESCAPE_SEQUENCE, t, input.substring(state.tokenStartCharIndex,getCharIndex()-1)); + } + } + ; + // ---------- // Whitespace // diff --git a/tool/src/org/antlr/v4/parse/ATNBuilder.g b/tool/src/org/antlr/v4/parse/ATNBuilder.g index 32cdec5..dfd47dc 100644 --- a/tool/src/org/antlr/v4/parse/ATNBuilder.g +++ b/tool/src/org/antlr/v4/parse/ATNBuilder.g @@ -137,7 +137,7 @@ element returns [ATNFactory.Handle p] | ^(ACTION .) {$p = factory.action((ActionAST)$ACTION);} | ^(SEMPRED .) {$p = factory.sempred((PredAST)$SEMPRED);} | ^(NOT b=blockSet[true]) {$p = $b.p;} - | LEXER_CHAR_SET {$p = factory.charSetLiteral($start);} + | LEXER_CHAR_SET {$p = factory.charSetLiteral($start);} ; astOperand returns [ATNFactory.Handle p] diff --git a/tool/src/org/antlr/v4/parse/ActionSplitterListener.java b/tool/src/org/antlr/v4/parse/ActionSplitterListener.java index fe364f9..7ed8b87 100644 --- a/tool/src/org/antlr/v4/parse/ActionSplitterListener.java +++ b/tool/src/org/antlr/v4/parse/ActionSplitterListener.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/tool/src/org/antlr/v4/parse/GrammarASTAdaptor.java b/tool/src/org/antlr/v4/parse/GrammarASTAdaptor.java index 47d6e0a..fd54ec4 100644 --- a/tool/src/org/antlr/v4/parse/GrammarASTAdaptor.java +++ b/tool/src/org/antlr/v4/parse/GrammarASTAdaptor.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/tool/src/org/antlr/v4/parse/GrammarToken.java b/tool/src/org/antlr/v4/parse/GrammarToken.java index edf75b6..1a349ee 100644 --- a/tool/src/org/antlr/v4/parse/GrammarToken.java +++ b/tool/src/org/antlr/v4/parse/GrammarToken.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/tool/src/org/antlr/v4/parse/GrammarTreeVisitor.g b/tool/src/org/antlr/v4/parse/GrammarTreeVisitor.g index c407aa4..9bc3996 100644 --- a/tool/src/org/antlr/v4/parse/GrammarTreeVisitor.g +++ b/tool/src/org/antlr/v4/parse/GrammarTreeVisitor.g @@ -844,7 +844,7 @@ element | SEMPRED {sempredInAlt((PredAST)$SEMPRED);} | ^(ACTION elementOptions) {actionInAlt((ActionAST)$ACTION);} | ^(SEMPRED elementOptions) {sempredInAlt((PredAST)$SEMPRED);} - + | range | ^(NOT blockSet) | ^(NOT block) ; diff --git a/tool/src/org/antlr/v4/parse/ResyncToEndOfRuleBlock.java b/tool/src/org/antlr/v4/parse/ResyncToEndOfRuleBlock.java index 0564d38..802b5ea 100644 --- a/tool/src/org/antlr/v4/parse/ResyncToEndOfRuleBlock.java +++ b/tool/src/org/antlr/v4/parse/ResyncToEndOfRuleBlock.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/tool/src/org/antlr/v4/parse/ScopeParser.java b/tool/src/org/antlr/v4/parse/ScopeParser.java index 25a3f0a..c70ea47 100644 --- a/tool/src/org/antlr/v4/parse/ScopeParser.java +++ b/tool/src/org/antlr/v4/parse/ScopeParser.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ @@ -312,7 +312,8 @@ public class ScopeParser { // do we see a matching '>' ahead? if so, hope it's a generic // and not less followed by expr with greater than p = _splitArgumentList(actionText, p + 1, '>', separatorChar, args); - } else { + } + else { p++; // treat as normal char } break; diff --git a/tool/src/org/antlr/v4/parse/TokenVocabParser.java b/tool/src/org/antlr/v4/parse/TokenVocabParser.java index abfa71c..cf84a92 100644 --- a/tool/src/org/antlr/v4/parse/TokenVocabParser.java +++ b/tool/src/org/antlr/v4/parse/TokenVocabParser.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ @@ -146,6 +146,22 @@ public class TokenVocabParser { // files are generated (in the base, not relative to the input // location.) f = new File(g.tool.outputDirectory, vocabName + CodeGenerator.VOCAB_FILE_EXTENSION); - return f; + if ( f.exists() ) { + return f; + } + + // Still not found? Use the grammar's subfolder then. + String fileDirectory; + + if (g.fileName.lastIndexOf(File.separatorChar) == -1) { + // No path is included in the file name, so make the file + // directory the same as the parent grammar (which might still be just "" + // but when it is not, we will write the file in the correct place. + fileDirectory = "."; + } + else { + fileDirectory = g.fileName.substring(0, g.fileName.lastIndexOf(File.separatorChar)); + } + return new File(fileDirectory, vocabName + CodeGenerator.VOCAB_FILE_EXTENSION); } } diff --git a/tool/src/org/antlr/v4/parse/ToolANTLRLexer.java b/tool/src/org/antlr/v4/parse/ToolANTLRLexer.java index 613c06c..22a1a00 100644 --- a/tool/src/org/antlr/v4/parse/ToolANTLRLexer.java +++ b/tool/src/org/antlr/v4/parse/ToolANTLRLexer.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/tool/src/org/antlr/v4/parse/ToolANTLRParser.java b/tool/src/org/antlr/v4/parse/ToolANTLRParser.java index bbc4964..41d8352 100644 --- a/tool/src/org/antlr/v4/parse/ToolANTLRParser.java +++ b/tool/src/org/antlr/v4/parse/ToolANTLRParser.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/tool/src/org/antlr/v4/parse/v3TreeGrammarException.java b/tool/src/org/antlr/v4/parse/v3TreeGrammarException.java index 8c5c7b5..dad0d6e 100644 --- a/tool/src/org/antlr/v4/parse/v3TreeGrammarException.java +++ b/tool/src/org/antlr/v4/parse/v3TreeGrammarException.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/tool/src/org/antlr/v4/parse/v4ParserException.java b/tool/src/org/antlr/v4/parse/v4ParserException.java index cdcbe30..88fec58 100644 --- a/tool/src/org/antlr/v4/parse/v4ParserException.java +++ b/tool/src/org/antlr/v4/parse/v4ParserException.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/tool/src/org/antlr/v4/semantics/ActionSniffer.java b/tool/src/org/antlr/v4/semantics/ActionSniffer.java index ce25233..506589c 100644 --- a/tool/src/org/antlr/v4/semantics/ActionSniffer.java +++ b/tool/src/org/antlr/v4/semantics/ActionSniffer.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/tool/src/org/antlr/v4/semantics/AttributeChecks.java b/tool/src/org/antlr/v4/semantics/AttributeChecks.java index 5e35e8c..84bbd41 100644 --- a/tool/src/org/antlr/v4/semantics/AttributeChecks.java +++ b/tool/src/org/antlr/v4/semantics/AttributeChecks.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/tool/src/org/antlr/v4/semantics/BasicSemanticChecks.java b/tool/src/org/antlr/v4/semantics/BasicSemanticChecks.java index 4f5e792..4f76b5b 100644 --- a/tool/src/org/antlr/v4/semantics/BasicSemanticChecks.java +++ b/tool/src/org/antlr/v4/semantics/BasicSemanticChecks.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/tool/src/org/antlr/v4/semantics/BlankActionSplitterListener.java b/tool/src/org/antlr/v4/semantics/BlankActionSplitterListener.java index b0c05e7..2e18e67 100644 --- a/tool/src/org/antlr/v4/semantics/BlankActionSplitterListener.java +++ b/tool/src/org/antlr/v4/semantics/BlankActionSplitterListener.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/tool/src/org/antlr/v4/semantics/RuleCollector.java b/tool/src/org/antlr/v4/semantics/RuleCollector.java index 94b68d2..0931986 100644 --- a/tool/src/org/antlr/v4/semantics/RuleCollector.java +++ b/tool/src/org/antlr/v4/semantics/RuleCollector.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/tool/src/org/antlr/v4/semantics/SemanticPipeline.java b/tool/src/org/antlr/v4/semantics/SemanticPipeline.java index 6f95558..dc3cb2f 100644 --- a/tool/src/org/antlr/v4/semantics/SemanticPipeline.java +++ b/tool/src/org/antlr/v4/semantics/SemanticPipeline.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ @@ -108,6 +108,7 @@ public class SemanticPipeline { } symcheck.checkForModeConflicts(g); + symcheck.checkForUnreachableTokens(g); assignChannelTypes(g, collector.channelDefs); diff --git a/tool/src/org/antlr/v4/semantics/SymbolChecks.java b/tool/src/org/antlr/v4/semantics/SymbolChecks.java index 5eccef5..abbf22d 100644 --- a/tool/src/org/antlr/v4/semantics/SymbolChecks.java +++ b/tool/src/org/antlr/v4/semantics/SymbolChecks.java @@ -1,12 +1,15 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ package org.antlr.v4.semantics; +import org.antlr.runtime.tree.CommonTree; +import org.antlr.runtime.tree.Tree; import org.antlr.v4.automata.LexerATNFactory; +import org.antlr.v4.parse.ANTLRLexer; import org.antlr.v4.parse.ANTLRParser; import org.antlr.v4.runtime.Token; import org.antlr.v4.tool.Alternative; @@ -16,11 +19,15 @@ import org.antlr.v4.tool.ErrorManager; import org.antlr.v4.tool.ErrorType; import org.antlr.v4.tool.Grammar; import org.antlr.v4.tool.LabelElementPair; +import org.antlr.v4.tool.LabelType; +import org.antlr.v4.tool.LeftRecursiveRule; import org.antlr.v4.tool.LexerGrammar; import org.antlr.v4.tool.Rule; +import org.antlr.v4.tool.ast.AltAST; import org.antlr.v4.tool.ast.GrammarAST; -import org.antlr.v4.tool.LabelType; +import org.antlr.v4.tool.ast.TerminalAST; +import java.util.ArrayList; import java.util.Collection; import java.util.HashMap; import java.util.HashSet; @@ -35,42 +42,31 @@ import java.util.Set; * Side-effect: strip away redef'd rules. */ public class SymbolChecks { - Grammar g; - SymbolCollector collector; - Map<String, Rule> nameToRuleMap = new HashMap<String, Rule>(); + Grammar g; + SymbolCollector collector; + Map<String, Rule> nameToRuleMap = new HashMap<String, Rule>(); Set<String> tokenIDs = new HashSet<String>(); - Map<String, Set<String>> actionScopeToActionNames = new HashMap<String, Set<String>>(); -// DoubleKeyMap<String, String, GrammarAST> namedActions = -// new DoubleKeyMap<String, String, GrammarAST>(); + Map<String, Set<String>> actionScopeToActionNames = new HashMap<String, Set<String>>(); public ErrorManager errMgr; protected final Set<String> reservedNames = new HashSet<String>(); + { reservedNames.addAll(LexerATNFactory.getCommonConstants()); } - public SymbolChecks(Grammar g, SymbolCollector collector) { - this.g = g; - this.collector = collector; + public SymbolChecks(Grammar g, SymbolCollector collector) { + this.g = g; + this.collector = collector; this.errMgr = g.tool.errMgr; - for (GrammarAST tokenId : collector.tokenIDRefs) { - tokenIDs.add(tokenId.getText()); - } - /* - System.out.println("rules="+collector.rules); - System.out.println("rulerefs="+collector.rulerefs); - System.out.println("tokenIDRefs="+collector.tokenIDRefs); - System.out.println("terminals="+collector.terminals); - System.out.println("strings="+collector.strings); - System.out.println("tokensDef="+collector.tokensDefs); - System.out.println("actions="+collector.actions); - System.out.println("scopes="+collector.scopes); - */ - } - - public void process() { + for (GrammarAST tokenId : collector.tokenIDRefs) { + tokenIDs.add(tokenId.getText()); + } + } + + public void process() { // methods affect fields, but no side-effects outside this object // So, call order sensitive // First collect all rules for later use in checkForLabelConflict() @@ -79,7 +75,6 @@ public class SymbolChecks { } checkReservedNames(g.rules.values()); checkActionRedefinitions(collector.namedActions); - checkForTokenConflicts(collector.tokenIDRefs); // sets tokenIDs checkForLabelConflicts(g.rules.values()); } @@ -92,7 +87,8 @@ public class SymbolChecks { nameNode = (GrammarAST) ampersandAST.getChild(0); if (ampersandAST.getChildCount() == 2) { name = nameNode.getText(); - } else { + } + else { scope = nameNode.getText(); name = ampersandAST.getChild(1).getText(); } @@ -103,72 +99,126 @@ public class SymbolChecks { } if (!scopeActions.contains(name)) { scopeActions.add(name); - } else { + } + else { errMgr.grammarError(ErrorType.ACTION_REDEFINITION, g.fileName, nameNode.token, name); } } } - public void checkForTokenConflicts(List<GrammarAST> tokenIDRefs) { -// for (GrammarAST a : tokenIDRefs) { -// Token t = a.token; -// String ID = t.getText(); -// tokenIDs.add(ID); -// } - } - - /** Make sure a label doesn't conflict with another symbol. - * Labels must not conflict with: rules, tokens, scope names, - * return values, parameters, and rule-scope dynamic attributes - * defined in surrounding rule. Also they must have same type - * for repeated defs. - */ - public void checkForLabelConflicts(Collection<Rule> rules) { + /** + * Make sure a label doesn't conflict with another symbol. + * Labels must not conflict with: rules, tokens, scope names, + * return values, parameters, and rule-scope dynamic attributes + * defined in surrounding rule. Also they must have same type + * for repeated defs. + */ + public void checkForLabelConflicts(Collection<Rule> rules) { for (Rule r : rules) { checkForAttributeConflicts(r); - Map<String, LabelElementPair> labelNameSpace = - new HashMap<String, LabelElementPair>(); - for (int i = 1; i <= r.numberOfAlts; i++) { - if (r.hasAltSpecificContexts()) { - labelNameSpace.clear(); - } + Map<String, LabelElementPair> labelNameSpace = new HashMap<>(); + for (int i = 1; i <= r.numberOfAlts; i++) { Alternative a = r.alt[i]; for (List<LabelElementPair> pairs : a.labelDefs.values()) { - for (LabelElementPair p : pairs) { - checkForLabelConflict(r, p.label); - String name = p.label.getText(); - LabelElementPair prev = labelNameSpace.get(name); - if (prev == null) labelNameSpace.put(name, p); - else checkForTypeMismatch(prev, p); + if (r.hasAltSpecificContexts()) { + // Collect labelName-labeledRules map for rule with alternative labels. + Map<String, List<LabelElementPair>> labelPairs = new HashMap<>(); + for (LabelElementPair p : pairs) { + String labelName = findAltLabelName(p.label); + if (labelName != null) { + List<LabelElementPair> list; + if (labelPairs.containsKey(labelName)) { + list = labelPairs.get(labelName); + } + else { + list = new ArrayList<>(); + labelPairs.put(labelName, list); + } + list.add(p); + } + } + + for (List<LabelElementPair> internalPairs : labelPairs.values()) { + labelNameSpace.clear(); + checkLabelPairs(r, labelNameSpace, internalPairs); + } + } + else { + checkLabelPairs(r, labelNameSpace, pairs); } } } } } - void checkForTypeMismatch(LabelElementPair prevLabelPair, LabelElementPair labelPair) { + private void checkLabelPairs(Rule r, Map<String, LabelElementPair> labelNameSpace, List<LabelElementPair> pairs) { + for (LabelElementPair p : pairs) { + checkForLabelConflict(r, p.label); + String name = p.label.getText(); + LabelElementPair prev = labelNameSpace.get(name); + if (prev == null) { + labelNameSpace.put(name, p); + } + else { + checkForTypeMismatch(r, prev, p); + } + } + } + + private String findAltLabelName(CommonTree label) { + if (label == null) { + return null; + } + else if (label instanceof AltAST) { + AltAST altAST = (AltAST) label; + if (altAST.altLabel != null) { + return altAST.altLabel.toString(); + } + else if (altAST.leftRecursiveAltInfo != null) { + return altAST.leftRecursiveAltInfo.altLabel.toString(); + } + else { + return findAltLabelName(label.parent); + } + } + else { + return findAltLabelName(label.parent); + } + } + + private void checkForTypeMismatch(Rule r, LabelElementPair prevLabelPair, LabelElementPair labelPair) { // label already defined; if same type, no problem if (prevLabelPair.type != labelPair.type) { - String typeMismatchExpr = labelPair.type + "!=" + prevLabelPair.type; + // Current behavior: take a token of rule declaration in case of left-recursive rule + // Desired behavior: take a token of proper label declaration in case of left-recursive rule + // See https://github.com/antlr/antlr4/pull/1585 + // Such behavior is referring to the fact that the warning is typically reported on the actual label redefinition, + // but for left-recursive rules the warning is reported on the enclosing rule. + org.antlr.runtime.Token token = r instanceof LeftRecursiveRule + ? ((GrammarAST) r.ast.getChild(0)).getToken() + : labelPair.label.token; errMgr.grammarError( ErrorType.LABEL_TYPE_CONFLICT, g.fileName, - labelPair.label.token, + token, labelPair.label.getText(), - typeMismatchExpr); + labelPair.type + "!=" + prevLabelPair.type); } if (!prevLabelPair.element.getText().equals(labelPair.element.getText()) && (prevLabelPair.type.equals(LabelType.RULE_LABEL) || prevLabelPair.type.equals(LabelType.RULE_LIST_LABEL)) && (labelPair.type.equals(LabelType.RULE_LABEL) || labelPair.type.equals(LabelType.RULE_LIST_LABEL))) { + org.antlr.runtime.Token token = r instanceof LeftRecursiveRule + ? ((GrammarAST) r.ast.getChild(0)).getToken() + : labelPair.label.token; String prevLabelOp = prevLabelPair.type.equals(LabelType.RULE_LIST_LABEL) ? "+=" : "="; String labelOp = labelPair.type.equals(LabelType.RULE_LIST_LABEL) ? "+=" : "="; errMgr.grammarError( ErrorType.LABEL_TYPE_CONFLICT, g.fileName, - labelPair.label.token, + token, labelPair.label.getText() + labelOp + labelPair.element.getText(), prevLabelPair.label.getText() + prevLabelOp + prevLabelPair.element.getText()); } @@ -225,11 +275,11 @@ public class SymbolChecks { for (Attribute attribute : attributes.attributes.values()) { if (ruleNames.contains(attribute.name)) { errMgr.grammarError( - errorType, - g.fileName, - attribute.token != null ? attribute.token : ((GrammarAST)r.ast.getChild(0)).token, - attribute.name, - r.name); + errorType, + g.fileName, + attribute.token != null ? attribute.token : ((GrammarAST) r.ast.getChild(0)).token, + attribute.name, + r.name); } } } @@ -242,11 +292,11 @@ public class SymbolChecks { Set<String> conflictingKeys = attributes.intersection(referenceAttributes); for (String key : conflictingKeys) { errMgr.grammarError( - errorType, - g.fileName, - attributes.get(key).token != null ? attributes.get(key).token : ((GrammarAST) r.ast.getChild(0)).token, - key, - r.name); + errorType, + g.fileName, + attributes.get(key).token != null ? attributes.get(key).token : ((GrammarAST)r.ast.getChild(0)).token, + key, + r.name); } } @@ -275,8 +325,122 @@ public class SymbolChecks { } } - // CAN ONLY CALL THE TWO NEXT METHODS AFTER GRAMMAR HAS RULE DEFS (see semanticpipeline) + /** + * Algorithm steps: + * 1. Collect all simple string literals (i.e. 'asdf', 'as' 'df', but not [a-z]+, 'a'..'z') + * for all lexer rules in each mode except of autogenerated tokens ({@link #getSingleTokenValues(Rule) getSingleTokenValues}) + * 2. Compare every string literal with each other ({@link #checkForOverlap(Grammar, Rule, Rule, List<String>, List<String>) checkForOverlap}) + * and throw TOKEN_UNREACHABLE warning if the same string found. + * Complexity: O(m * n^2 / 2), approximately equals to O(n^2) + * where m - number of modes, n - average number of lexer rules per mode. + * See also testUnreachableTokens unit test for details. + */ + public void checkForUnreachableTokens(Grammar g) { + if (g.isLexer()) { + LexerGrammar lexerGrammar = (LexerGrammar)g; + for (List<Rule> rules : lexerGrammar.modes.values()) { + // Collect string literal lexer rules for each mode + List<Rule> stringLiteralRules = new ArrayList<>(); + List<List<String>> stringLiteralValues = new ArrayList<>(); + for (int i = 0; i < rules.size(); i++) { + Rule rule = rules.get(i); + + List<String> ruleStringAlts = getSingleTokenValues(rule); + if (ruleStringAlts != null && ruleStringAlts.size() > 0) { + stringLiteralRules.add(rule); + stringLiteralValues.add(ruleStringAlts); + } + } + + // Check string sets intersection + for (int i = 0; i < stringLiteralRules.size(); i++) { + List<String> firstTokenStringValues = stringLiteralValues.get(i); + Rule rule1 = stringLiteralRules.get(i); + checkForOverlap(g, rule1, rule1, firstTokenStringValues, stringLiteralValues.get(i)); + + // Check fragment rules only with themself + if (!rule1.isFragment()) { + for (int j = i + 1; j < stringLiteralRules.size(); j++) { + Rule rule2 = stringLiteralRules.get(j); + if (!rule2.isFragment()) { + checkForOverlap(g, rule1, stringLiteralRules.get(j), firstTokenStringValues, stringLiteralValues.get(j)); + } + } + } + } + } + } + } + + /** + * {@return} list of simple string literals for rule {@param rule} + */ + private List<String> getSingleTokenValues(Rule rule) + { + List<String> values = new ArrayList<>(); + for (Alternative alt : rule.alt) { + if (alt != null) { + // select first alt if token has a command + Tree rootNode = alt.ast.getChildCount() == 2 && + alt.ast.getChild(0) instanceof AltAST && alt.ast.getChild(1) instanceof GrammarAST + ? alt.ast.getChild(0) + : alt.ast; + + if (rootNode.getTokenStartIndex() == -1) { + continue; // ignore autogenerated tokens from combined grammars that start with T__ + } + // Ignore alt if contains not only string literals (repetition, optional) + boolean ignore = false; + StringBuilder currentValue = new StringBuilder(); + for (int i = 0; i < rootNode.getChildCount(); i++) { + Tree child = rootNode.getChild(i); + if (!(child instanceof TerminalAST)) { + ignore = true; + break; + } + + TerminalAST terminalAST = (TerminalAST)child; + if (terminalAST.token.getType() != ANTLRLexer.STRING_LITERAL) { + ignore = true; + break; + } + else { + String text = terminalAST.token.getText(); + currentValue.append(text.substring(1, text.length() - 1)); + } + } + + if (!ignore) { + values.add(currentValue.toString()); + } + } + } + return values; + } + + /** + * For same rule compare values from next index: + * TOKEN_WITH_SAME_VALUES: 'asdf' | 'asdf'; + * For different rules compare from start value: + * TOKEN1: 'asdf'; + * TOKEN2: 'asdf'; + */ + private void checkForOverlap(Grammar g, Rule rule1, Rule rule2, List<String> firstTokenStringValues, List<String> secondTokenStringValues) { + for (int i = 0; i < firstTokenStringValues.size(); i++) { + int secondTokenInd = rule1 == rule2 ? i + 1 : 0; + String str1 = firstTokenStringValues.get(i); + for (int j = secondTokenInd; j < secondTokenStringValues.size(); j++) { + String str2 = secondTokenStringValues.get(j); + if (str1.equals(str2)) { + errMgr.grammarError(ErrorType.TOKEN_UNREACHABLE, g.fileName, + ((GrammarAST) rule2.ast.getChild(0)).token, rule2.name, str2, rule1.name); + } + } + } + } + + // CAN ONLY CALL THE TWO NEXT METHODS AFTER GRAMMAR HAS RULE DEFS (see semanticpipeline) public void checkRuleArgs(Grammar g, List<GrammarAST> rulerefs) { if ( rulerefs==null ) return; for (GrammarAST ref : rulerefs) { @@ -285,12 +449,12 @@ public class SymbolChecks { GrammarAST arg = (GrammarAST)ref.getFirstChildWithType(ANTLRParser.ARG_ACTION); if ( arg!=null && (r==null || r.args==null) ) { errMgr.grammarError(ErrorType.RULE_HAS_NO_ARGS, - g.fileName, ref.token, ruleName); + g.fileName, ref.token, ruleName); } - else if ( arg==null && (r!=null&&r.args!=null) ) { + else if ( arg==null && (r!=null && r.args!=null) ) { errMgr.grammarError(ErrorType.MISSING_RULE_ARGS, - g.fileName, ref.token, ruleName); + g.fileName, ref.token, ruleName); } } } @@ -299,18 +463,18 @@ public class SymbolChecks { for (GrammarAST dot : qualifiedRuleRefs) { GrammarAST grammar = (GrammarAST)dot.getChild(0); GrammarAST rule = (GrammarAST)dot.getChild(1); - g.tool.log("semantics", grammar.getText()+"."+rule.getText()); + g.tool.log("semantics", grammar.getText()+"."+rule.getText()); Grammar delegate = g.getImportedGrammar(grammar.getText()); if ( delegate==null ) { errMgr.grammarError(ErrorType.NO_SUCH_GRAMMAR_SCOPE, - g.fileName, grammar.token, grammar.getText(), - rule.getText()); + g.fileName, grammar.token, grammar.getText(), + rule.getText()); } else { if ( g.getRule(grammar.getText(), rule.getText())==null ) { errMgr.grammarError(ErrorType.NO_SUCH_RULE_IN_SCOPE, - g.fileName, rule.token, grammar.getText(), - rule.getText()); + g.fileName, rule.token, grammar.getText(), + rule.getText()); } } } diff --git a/tool/src/org/antlr/v4/semantics/SymbolCollector.java b/tool/src/org/antlr/v4/semantics/SymbolCollector.java index 2bcf9ca..7c41eef 100644 --- a/tool/src/org/antlr/v4/semantics/SymbolCollector.java +++ b/tool/src/org/antlr/v4/semantics/SymbolCollector.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/tool/src/org/antlr/v4/semantics/UseDefAnalyzer.java b/tool/src/org/antlr/v4/semantics/UseDefAnalyzer.java index 9c6f444..4644198 100644 --- a/tool/src/org/antlr/v4/semantics/UseDefAnalyzer.java +++ b/tool/src/org/antlr/v4/semantics/UseDefAnalyzer.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/tool/src/org/antlr/v4/tool/ANTLRMessage.java b/tool/src/org/antlr/v4/tool/ANTLRMessage.java index 0ed03d6..cf585cb 100644 --- a/tool/src/org/antlr/v4/tool/ANTLRMessage.java +++ b/tool/src/org/antlr/v4/tool/ANTLRMessage.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/tool/src/org/antlr/v4/tool/ANTLRToolListener.java b/tool/src/org/antlr/v4/tool/ANTLRToolListener.java index 5662b34..3aa89e4 100644 --- a/tool/src/org/antlr/v4/tool/ANTLRToolListener.java +++ b/tool/src/org/antlr/v4/tool/ANTLRToolListener.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/tool/src/org/antlr/v4/tool/Alternative.java b/tool/src/org/antlr/v4/tool/Alternative.java index f47543c..2a993c2 100644 --- a/tool/src/org/antlr/v4/tool/Alternative.java +++ b/tool/src/org/antlr/v4/tool/Alternative.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/tool/src/org/antlr/v4/tool/Attribute.java b/tool/src/org/antlr/v4/tool/Attribute.java index 94baaa6..5e15c01 100644 --- a/tool/src/org/antlr/v4/tool/Attribute.java +++ b/tool/src/org/antlr/v4/tool/Attribute.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/tool/src/org/antlr/v4/tool/AttributeDict.java b/tool/src/org/antlr/v4/tool/AttributeDict.java index 6513f9b..2af27f6 100644 --- a/tool/src/org/antlr/v4/tool/AttributeDict.java +++ b/tool/src/org/antlr/v4/tool/AttributeDict.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/tool/src/org/antlr/v4/tool/AttributeResolver.java b/tool/src/org/antlr/v4/tool/AttributeResolver.java index e581f23..18dd9ef 100644 --- a/tool/src/org/antlr/v4/tool/AttributeResolver.java +++ b/tool/src/org/antlr/v4/tool/AttributeResolver.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/tool/src/org/antlr/v4/tool/BuildDependencyGenerator.java b/tool/src/org/antlr/v4/tool/BuildDependencyGenerator.java index be48bfa..c33aa4d 100644 --- a/tool/src/org/antlr/v4/tool/BuildDependencyGenerator.java +++ b/tool/src/org/antlr/v4/tool/BuildDependencyGenerator.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/tool/src/org/antlr/v4/tool/DOTGenerator.java b/tool/src/org/antlr/v4/tool/DOTGenerator.java index 6f7aaea..062f2da 100644 --- a/tool/src/org/antlr/v4/tool/DOTGenerator.java +++ b/tool/src/org/antlr/v4/tool/DOTGenerator.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ @@ -92,7 +92,7 @@ public class DOTGenerator { if ( target.stateNumber == Integer.MAX_VALUE ) continue; int ttype = i-1; // we shift up for EOF as -1 for parser String label = String.valueOf(ttype); - if ( isLexer ) label = "'"+getEdgeLabel(String.valueOf((char) i))+"'"; + if ( isLexer ) label = "'"+getEdgeLabel(new StringBuilder().appendCodePoint(i).toString())+"'"; else if ( grammar!=null ) label = grammar.getTokenDisplayName(ttype); ST st = stlib.getInstanceOf("edge"); st.add("label", label); @@ -259,7 +259,7 @@ public class DOTGenerator { edgeST = stlib.getInstanceOf("edge"); AtomTransition atom = (AtomTransition)edge; String label = String.valueOf(atom.label); - if ( isLexer ) label = "'"+getEdgeLabel(String.valueOf((char)atom.label))+"'"; + if ( isLexer ) label = "'"+getEdgeLabel(new StringBuilder().appendCodePoint(atom.label).toString())+"'"; else if ( grammar!=null ) label = grammar.getTokenDisplayName(atom.label); edgeST.add("label", getEdgeLabel(label)); } @@ -289,7 +289,8 @@ public class DOTGenerator { edgeST.add("arrowhead", arrowhead); if (s.getNumberOfTransitions() > 1) { edgeST.add("transitionIndex", i); - } else { + } + else { edgeST.add("transitionIndex", false); } dot.add("edges", edgeST); diff --git a/tool/src/org/antlr/v4/tool/DefaultToolListener.java b/tool/src/org/antlr/v4/tool/DefaultToolListener.java index 0f64cd0..a2e810b 100644 --- a/tool/src/org/antlr/v4/tool/DefaultToolListener.java +++ b/tool/src/org/antlr/v4/tool/DefaultToolListener.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/tool/src/org/antlr/v4/tool/ErrorManager.java b/tool/src/org/antlr/v4/tool/ErrorManager.java index 34f0d50..0230d2b 100644 --- a/tool/src/org/antlr/v4/tool/ErrorManager.java +++ b/tool/src/org/antlr/v4/tool/ErrorManager.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ @@ -16,7 +16,6 @@ import java.io.File; import java.net.URL; import java.util.Collection; import java.util.EnumSet; -import java.util.Locale; import java.util.Set; public class ErrorManager { @@ -32,8 +31,6 @@ public class ErrorManager { /** The group of templates that represent the current message format. */ STGroup format; - /** Messages should be sensitive to the locale. */ - Locale locale; String formatName; ErrorBuffer initSTListener = new ErrorBuffer(); @@ -63,11 +60,20 @@ public class ErrorManager { locationValid = true; } if (msg.fileName != null) { - File f = new File(msg.fileName); - // Don't show path to file in messages; too long. String displayFileName = msg.fileName; - if ( f.exists() ) { - displayFileName = f.getName(); + if (format.equals("antlr")) { + // Don't show path to file in messages in ANTLR format; + // they're too long. + File f = new File(msg.fileName); + if ( f.exists() ) { + displayFileName = f.getName(); + } + } + else { + // For other message formats, use the full filename in the + // message. This assumes that these formats are intended to + // be parsed by IDEs, and so they need the full path to + // resolve correctly. } locationST.add("file", displayFileName); locationValid = true; diff --git a/tool/src/org/antlr/v4/tool/ErrorSeverity.java b/tool/src/org/antlr/v4/tool/ErrorSeverity.java index 1c527b1..3bc89db 100644 --- a/tool/src/org/antlr/v4/tool/ErrorSeverity.java +++ b/tool/src/org/antlr/v4/tool/ErrorSeverity.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/tool/src/org/antlr/v4/tool/ErrorType.java b/tool/src/org/antlr/v4/tool/ErrorType.java index c4a3861..a034803 100644 --- a/tool/src/org/antlr/v4/tool/ErrorType.java +++ b/tool/src/org/antlr/v4/tool/ErrorType.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ @@ -394,11 +394,11 @@ public enum ErrorType { */ IMPORT_NAME_CLASH(113, "<arg.typeString> grammar <arg.name> and imported <arg2.typeString> grammar <arg2.name> both generate <arg2.recognizerName>", ErrorSeverity.ERROR), /** - * Compiler Error 160. + * Compiler Error 114. * * <p>cannot find tokens file <em>filename</em></p> */ - CANNOT_FIND_TOKENS_FILE_REFD_IN_GRAMMAR(160, "cannot find tokens file <arg>", ErrorSeverity.ERROR), + CANNOT_FIND_TOKENS_FILE_REFD_IN_GRAMMAR(114, "cannot find tokens file <arg>", ErrorSeverity.ERROR), /** * Compiler Warning 118. * @@ -522,7 +522,7 @@ public enum ErrorType { */ USE_OF_BAD_WORD(134, "symbol <arg> conflicts with generated code in target language or runtime", ErrorSeverity.ERROR), /** - * Compiler Error 134. + * Compiler Error 183. * * <p>rule reference <em>rule</em> is not currently supported in a set</p> * @@ -530,7 +530,7 @@ public enum ErrorType { * Note: This error has the same number as the unrelated error * {@link #USE_OF_BAD_WORD}.</p> */ - UNSUPPORTED_REFERENCE_IN_LEXER_SET(134, "rule reference <arg> is not currently supported in a set", ErrorSeverity.ERROR), + UNSUPPORTED_REFERENCE_IN_LEXER_SET(183, "rule reference <arg> is not currently supported in a set", ErrorSeverity.ERROR), /** * Compiler Error 135. * @@ -824,7 +824,7 @@ public enum ErrorType { * * @since 4.2.1 */ - INVALID_ESCAPE_SEQUENCE(156, "invalid escape sequence", ErrorSeverity.ERROR), + INVALID_ESCAPE_SEQUENCE(156, "invalid escape sequence <arg>", ErrorSeverity.WARNING), /** * Compiler Warning 157. * @@ -1043,7 +1043,51 @@ public enum ErrorType { * * TODO: Does not work with fragment rules. */ - CHARACTERS_COLLISION_IN_SET(180, "chars \"<arg>\" used multiple times in set <arg2>", ErrorSeverity.WARNING), + CHARACTERS_COLLISION_IN_SET(180, "chars <arg> used multiple times in set <arg2>", ErrorSeverity.WARNING), + + /** + * Compiler Warning 181 + * + * <p>The token range operator makes no sense in the parser as token types + * are not ordered (except in implementation). + * </p> + * + * <pre> + * grammar T; + * a : 'A'..'Z' ; + * </pre> + * + */ + TOKEN_RANGE_IN_PARSER(181, "token ranges not allowed in parser: <arg>..<arg2>", ErrorSeverity.ERROR), + + /** + * Compiler Error 182. + * + * <p>Unicode properties cannot be part of a lexer charset range</p> + * + * <pre> + * A: [\\p{Letter}-\\p{Number}]; + * </pre> + */ + UNICODE_PROPERTY_NOT_ALLOWED_IN_RANGE( + 182, + "unicode property escapes not allowed in lexer charset range: <arg>", + ErrorSeverity.ERROR), + + /** + * Compiler Warning 184. + * + * <p>The token value overlapped by another token or self</p> + * + * <pre> + * TOKEN1: 'value'; + * TOKEN2: 'value'; // warning + * </pre> + */ + TOKEN_UNREACHABLE( + 184, + "One of the token <arg> values unreachable. <arg2> is always overlapped by token <arg3>", + ErrorSeverity.WARNING), /* * Backward incompatibility errors diff --git a/tool/src/org/antlr/v4/tool/Grammar.java b/tool/src/org/antlr/v4/tool/Grammar.java index ea249f8..b86d622 100644 --- a/tool/src/org/antlr/v4/tool/Grammar.java +++ b/tool/src/org/antlr/v4/tool/Grammar.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ @@ -81,6 +81,7 @@ public class Grammar implements AttributeResolver { parserOptions.add("TokenLabelType"); parserOptions.add("tokenVocab"); parserOptions.add("language"); + parserOptions.add("accessLevel"); parserOptions.add("exportMacro"); } @@ -1301,7 +1302,11 @@ public class Grammar implements AttributeResolver { char[] serializedAtn = ATNSerializer.getSerializedAsChars(atn); ATN deserialized = new ATNDeserializer().deserialize(serializedAtn); - return new LexerInterpreter(fileName, getVocabulary(), Arrays.asList(getRuleNames()), ((LexerGrammar)this).modes.keySet(), deserialized, input); + List<String> allChannels = new ArrayList<String>(); + allChannels.add("DEFAULT_TOKEN_CHANNEL"); + allChannels.add("HIDDEN"); + allChannels.addAll(channelValueToNameList); + return new LexerInterpreter(fileName, getVocabulary(), Arrays.asList(getRuleNames()), allChannels, ((LexerGrammar)this).modes.keySet(), deserialized, input); } /** @since 4.5.1 */ diff --git a/tool/src/org/antlr/v4/tool/GrammarInterpreterRuleContext.java b/tool/src/org/antlr/v4/tool/GrammarInterpreterRuleContext.java index 78d1731..1e7db8b 100644 --- a/tool/src/org/antlr/v4/tool/GrammarInterpreterRuleContext.java +++ b/tool/src/org/antlr/v4/tool/GrammarInterpreterRuleContext.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/tool/src/org/antlr/v4/tool/GrammarParserInterpreter.java b/tool/src/org/antlr/v4/tool/GrammarParserInterpreter.java index b8cbb77..7123b43 100644 --- a/tool/src/org/antlr/v4/tool/GrammarParserInterpreter.java +++ b/tool/src/org/antlr/v4/tool/GrammarParserInterpreter.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/tool/src/org/antlr/v4/tool/GrammarSemanticsMessage.java b/tool/src/org/antlr/v4/tool/GrammarSemanticsMessage.java index 3be7c72..802aff7 100644 --- a/tool/src/org/antlr/v4/tool/GrammarSemanticsMessage.java +++ b/tool/src/org/antlr/v4/tool/GrammarSemanticsMessage.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/tool/src/org/antlr/v4/tool/GrammarSyntaxMessage.java b/tool/src/org/antlr/v4/tool/GrammarSyntaxMessage.java index 6de8861..e3fb5b6 100644 --- a/tool/src/org/antlr/v4/tool/GrammarSyntaxMessage.java +++ b/tool/src/org/antlr/v4/tool/GrammarSyntaxMessage.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/tool/src/org/antlr/v4/tool/GrammarTransformPipeline.java b/tool/src/org/antlr/v4/tool/GrammarTransformPipeline.java index fa27e7f..f6e4cdf 100644 --- a/tool/src/org/antlr/v4/tool/GrammarTransformPipeline.java +++ b/tool/src/org/antlr/v4/tool/GrammarTransformPipeline.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ @@ -161,6 +161,7 @@ public class GrammarTransformPipeline { GrammarAST id = (GrammarAST) root.getChild(0); GrammarASTAdaptor adaptor = new GrammarASTAdaptor(id.token.getInputStream()); + GrammarAST channelsRoot = (GrammarAST)root.getFirstChildWithType(ANTLRParser.CHANNELS); GrammarAST tokensRoot = (GrammarAST)root.getFirstChildWithType(ANTLRParser.TOKENS_SPEC); List<GrammarAST> actionRoots = root.getNodesWithType(ANTLRParser.AT); @@ -172,7 +173,39 @@ public class GrammarTransformPipeline { List<GrammarAST> rootRules = RULES.getNodesWithType(ANTLRParser.RULE); for (GrammarAST r : rootRules) rootRuleNames.add(r.getChild(0).getText()); + // make list of modes we have in root grammar + List<GrammarAST> rootModes = root.getNodesWithType(ANTLRParser.MODE); + Set<String> rootModeNames = new HashSet<String>(); + for (GrammarAST m : rootModes) rootModeNames.add(m.getChild(0).getText()); + List<GrammarAST> addedModes = new ArrayList<GrammarAST>(); + for (Grammar imp : imports) { + // COPY CHANNELS + GrammarAST imp_channelRoot = (GrammarAST)imp.ast.getFirstChildWithType(ANTLRParser.CHANNELS); + if ( imp_channelRoot != null) { + rootGrammar.tool.log("grammar", "imported channels: "+imp_channelRoot.getChildren()); + if (channelsRoot==null) { + channelsRoot = imp_channelRoot.dupTree(); + channelsRoot.g = rootGrammar; + root.insertChild(1, channelsRoot); // ^(GRAMMAR ID TOKENS...) + } else { + for (int c = 0; c < imp_channelRoot.getChildCount(); ++c) { + String channel = imp_channelRoot.getChild(c).getText(); + boolean channelIsInRootGrammar = false; + for (int rc = 0; rc < channelsRoot.getChildCount(); ++rc) { + String rootChannel = channelsRoot.getChild(rc).getText(); + if (rootChannel.equals(channel)) { + channelIsInRootGrammar = true; + break; + } + } + if (!channelIsInRootGrammar) { + channelsRoot.addChild(imp_channelRoot.getChild(c).dupNode()); + } + } + } + } + // COPY TOKENS GrammarAST imp_tokensRoot = (GrammarAST)imp.ast.getFirstChildWithType(ANTLRParser.TOKENS_SPEC); if ( imp_tokensRoot!=null ) { @@ -242,7 +275,54 @@ public class GrammarTransformPipeline { } } + // COPY MODES + // The strategy is to copy all the mode sections rules across to any + // mode section in the new grammar with the same name or a new + // mode section if no matching mode is resolved. Rules which are + // already in the new grammar are ignored for copy. If the mode + // section being added ends up empty it is not added to the merged + // grammar. + List<GrammarAST> modes = imp.ast.getNodesWithType(ANTLRParser.MODE); + if (modes != null) { + for (GrammarAST m : modes) { + rootGrammar.tool.log("grammar", "imported mode: " + m.toStringTree()); + String name = m.getChild(0).getText(); + boolean rootAlreadyHasMode = rootModeNames.contains(name); + GrammarAST destinationAST = null; + if (rootAlreadyHasMode) { + for (GrammarAST m2 : rootModes) { + if (m2.getChild(0).getText().equals(name)) { + destinationAST = m2; + break; + } + } + } else { + destinationAST = m.dupNode(); + destinationAST.addChild(m.getChild(0).dupNode()); + } + + int addedRules = 0; + List<GrammarAST> modeRules = m.getAllChildrenWithType(ANTLRParser.RULE); + for (GrammarAST r : modeRules) { + rootGrammar.tool.log("grammar", "imported rule: "+r.toStringTree()); + String ruleName = r.getChild(0).getText(); + boolean rootAlreadyHasRule = rootRuleNames.contains(ruleName); + if (!rootAlreadyHasRule) { + destinationAST.addChild(r); + addedRules++; + rootRuleNames.add(ruleName); + } + } + if (!rootAlreadyHasMode && addedRules > 0) { + rootGrammar.ast.addChild(destinationAST); + rootModeNames.add(name); + rootModes.add(destinationAST); + } + } + } + // COPY RULES + // Rules copied in the mode copy phase are not copied again. List<GrammarAST> rules = imp.ast.getNodesWithType(ANTLRParser.RULE); if ( rules!=null ) { for (GrammarAST r : rules) { diff --git a/tool/src/org/antlr/v4/tool/LabelElementPair.java b/tool/src/org/antlr/v4/tool/LabelElementPair.java index aa2b292..a415ef2 100644 --- a/tool/src/org/antlr/v4/tool/LabelElementPair.java +++ b/tool/src/org/antlr/v4/tool/LabelElementPair.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/tool/src/org/antlr/v4/tool/LabelType.java b/tool/src/org/antlr/v4/tool/LabelType.java index b28cee2..f28f9c5 100644 --- a/tool/src/org/antlr/v4/tool/LabelType.java +++ b/tool/src/org/antlr/v4/tool/LabelType.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/tool/src/org/antlr/v4/tool/LeftRecursionCyclesMessage.java b/tool/src/org/antlr/v4/tool/LeftRecursionCyclesMessage.java index ca1fbc3..9b17abc 100644 --- a/tool/src/org/antlr/v4/tool/LeftRecursionCyclesMessage.java +++ b/tool/src/org/antlr/v4/tool/LeftRecursionCyclesMessage.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/tool/src/org/antlr/v4/tool/LeftRecursiveRule.java b/tool/src/org/antlr/v4/tool/LeftRecursiveRule.java index fb07c5b..d47ddcb 100644 --- a/tool/src/org/antlr/v4/tool/LeftRecursiveRule.java +++ b/tool/src/org/antlr/v4/tool/LeftRecursiveRule.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/tool/src/org/antlr/v4/tool/LexerGrammar.java b/tool/src/org/antlr/v4/tool/LexerGrammar.java index 5c89ab8..1713a33 100644 --- a/tool/src/org/antlr/v4/tool/LexerGrammar.java +++ b/tool/src/org/antlr/v4/tool/LexerGrammar.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/tool/src/org/antlr/v4/tool/Rule.java b/tool/src/org/antlr/v4/tool/Rule.java index 986ea7f..401dfaa 100644 --- a/tool/src/org/antlr/v4/tool/Rule.java +++ b/tool/src/org/antlr/v4/tool/Rule.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/tool/src/org/antlr/v4/tool/ToolMessage.java b/tool/src/org/antlr/v4/tool/ToolMessage.java index b340466..7af97e2 100644 --- a/tool/src/org/antlr/v4/tool/ToolMessage.java +++ b/tool/src/org/antlr/v4/tool/ToolMessage.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/tool/src/org/antlr/v4/tool/ast/ActionAST.java b/tool/src/org/antlr/v4/tool/ast/ActionAST.java index 9ac97da..3067606 100644 --- a/tool/src/org/antlr/v4/tool/ast/ActionAST.java +++ b/tool/src/org/antlr/v4/tool/ast/ActionAST.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/tool/src/org/antlr/v4/tool/ast/AltAST.java b/tool/src/org/antlr/v4/tool/ast/AltAST.java index 38f5a6b..77e15d4 100644 --- a/tool/src/org/antlr/v4/tool/ast/AltAST.java +++ b/tool/src/org/antlr/v4/tool/ast/AltAST.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/tool/src/org/antlr/v4/tool/ast/BlockAST.java b/tool/src/org/antlr/v4/tool/ast/BlockAST.java index 9ddf349..dba507c 100644 --- a/tool/src/org/antlr/v4/tool/ast/BlockAST.java +++ b/tool/src/org/antlr/v4/tool/ast/BlockAST.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/tool/src/org/antlr/v4/tool/ast/GrammarAST.java b/tool/src/org/antlr/v4/tool/ast/GrammarAST.java index cb59b83..adad395 100644 --- a/tool/src/org/antlr/v4/tool/ast/GrammarAST.java +++ b/tool/src/org/antlr/v4/tool/ast/GrammarAST.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/tool/src/org/antlr/v4/tool/ast/GrammarASTErrorNode.java b/tool/src/org/antlr/v4/tool/ast/GrammarASTErrorNode.java index 472dcf5..dc64413 100644 --- a/tool/src/org/antlr/v4/tool/ast/GrammarASTErrorNode.java +++ b/tool/src/org/antlr/v4/tool/ast/GrammarASTErrorNode.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/tool/src/org/antlr/v4/tool/ast/GrammarASTVisitor.java b/tool/src/org/antlr/v4/tool/ast/GrammarASTVisitor.java index be92f85..c005c8f 100644 --- a/tool/src/org/antlr/v4/tool/ast/GrammarASTVisitor.java +++ b/tool/src/org/antlr/v4/tool/ast/GrammarASTVisitor.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/tool/src/org/antlr/v4/tool/ast/GrammarASTWithOptions.java b/tool/src/org/antlr/v4/tool/ast/GrammarASTWithOptions.java index f6b5bca..19e696f 100644 --- a/tool/src/org/antlr/v4/tool/ast/GrammarASTWithOptions.java +++ b/tool/src/org/antlr/v4/tool/ast/GrammarASTWithOptions.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ @@ -43,7 +43,7 @@ public abstract class GrammarASTWithOptions extends GrammarAST { if ( v.startsWith("'") || v.startsWith("\"") ) { v = CharSupport.getStringFromGrammarStringLiteral(v); if (v == null) { - g.tool.errMgr.grammarError(ErrorType.INVALID_ESCAPE_SEQUENCE, g.fileName, value.getToken()); + g.tool.errMgr.grammarError(ErrorType.INVALID_ESCAPE_SEQUENCE, g.fileName, value.getToken(), value.getText()); v = ""; } } diff --git a/tool/src/org/antlr/v4/tool/ast/GrammarRootAST.java b/tool/src/org/antlr/v4/tool/ast/GrammarRootAST.java index 95f5128..5392145 100644 --- a/tool/src/org/antlr/v4/tool/ast/GrammarRootAST.java +++ b/tool/src/org/antlr/v4/tool/ast/GrammarRootAST.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/tool/src/org/antlr/v4/tool/ast/NotAST.java b/tool/src/org/antlr/v4/tool/ast/NotAST.java index ecd1d5a..620ec6e 100644 --- a/tool/src/org/antlr/v4/tool/ast/NotAST.java +++ b/tool/src/org/antlr/v4/tool/ast/NotAST.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/tool/src/org/antlr/v4/tool/ast/OptionalBlockAST.java b/tool/src/org/antlr/v4/tool/ast/OptionalBlockAST.java index 3a05d89..1b99249 100644 --- a/tool/src/org/antlr/v4/tool/ast/OptionalBlockAST.java +++ b/tool/src/org/antlr/v4/tool/ast/OptionalBlockAST.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/tool/src/org/antlr/v4/tool/ast/PlusBlockAST.java b/tool/src/org/antlr/v4/tool/ast/PlusBlockAST.java index ad0ae26..78881f5 100644 --- a/tool/src/org/antlr/v4/tool/ast/PlusBlockAST.java +++ b/tool/src/org/antlr/v4/tool/ast/PlusBlockAST.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/tool/src/org/antlr/v4/tool/ast/PredAST.java b/tool/src/org/antlr/v4/tool/ast/PredAST.java index 8069916..6cfe9ca 100644 --- a/tool/src/org/antlr/v4/tool/ast/PredAST.java +++ b/tool/src/org/antlr/v4/tool/ast/PredAST.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/tool/src/org/antlr/v4/tool/ast/QuantifierAST.java b/tool/src/org/antlr/v4/tool/ast/QuantifierAST.java index acfa942..3abb879 100644 --- a/tool/src/org/antlr/v4/tool/ast/QuantifierAST.java +++ b/tool/src/org/antlr/v4/tool/ast/QuantifierAST.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/tool/src/org/antlr/v4/tool/ast/RangeAST.java b/tool/src/org/antlr/v4/tool/ast/RangeAST.java index e285ca6..4630e8b 100644 --- a/tool/src/org/antlr/v4/tool/ast/RangeAST.java +++ b/tool/src/org/antlr/v4/tool/ast/RangeAST.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/tool/src/org/antlr/v4/tool/ast/RuleAST.java b/tool/src/org/antlr/v4/tool/ast/RuleAST.java index 3600971..5041ad2 100644 --- a/tool/src/org/antlr/v4/tool/ast/RuleAST.java +++ b/tool/src/org/antlr/v4/tool/ast/RuleAST.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/tool/src/org/antlr/v4/tool/ast/RuleElementAST.java b/tool/src/org/antlr/v4/tool/ast/RuleElementAST.java index e355604..6815d48 100644 --- a/tool/src/org/antlr/v4/tool/ast/RuleElementAST.java +++ b/tool/src/org/antlr/v4/tool/ast/RuleElementAST.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/tool/src/org/antlr/v4/tool/ast/RuleRefAST.java b/tool/src/org/antlr/v4/tool/ast/RuleRefAST.java index be07e58..bdde62e 100644 --- a/tool/src/org/antlr/v4/tool/ast/RuleRefAST.java +++ b/tool/src/org/antlr/v4/tool/ast/RuleRefAST.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/tool/src/org/antlr/v4/tool/ast/SetAST.java b/tool/src/org/antlr/v4/tool/ast/SetAST.java index 53c4c36..6c1fa78 100644 --- a/tool/src/org/antlr/v4/tool/ast/SetAST.java +++ b/tool/src/org/antlr/v4/tool/ast/SetAST.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/tool/src/org/antlr/v4/tool/ast/StarBlockAST.java b/tool/src/org/antlr/v4/tool/ast/StarBlockAST.java index 20b8793..9db14bb 100644 --- a/tool/src/org/antlr/v4/tool/ast/StarBlockAST.java +++ b/tool/src/org/antlr/v4/tool/ast/StarBlockAST.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/tool/src/org/antlr/v4/tool/ast/TerminalAST.java b/tool/src/org/antlr/v4/tool/ast/TerminalAST.java index fcac32d..234dd85 100644 --- a/tool/src/org/antlr/v4/tool/ast/TerminalAST.java +++ b/tool/src/org/antlr/v4/tool/ast/TerminalAST.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/tool/src/org/antlr/v4/unicode/UnicodeDataTemplateController.java b/tool/src/org/antlr/v4/unicode/UnicodeDataTemplateController.java new file mode 100644 index 0000000..da244a3 --- /dev/null +++ b/tool/src/org/antlr/v4/unicode/UnicodeDataTemplateController.java @@ -0,0 +1,418 @@ +/* + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. + * Use of this file is governed by the BSD 3-clause license that + * can be found in the LICENSE.txt file in the project root. + */ + +package org.antlr.v4.unicode; + +import com.ibm.icu.lang.UCharacter; +import com.ibm.icu.lang.UCharacterCategory; +import com.ibm.icu.lang.UProperty; +import com.ibm.icu.lang.UScript; +import com.ibm.icu.text.UnicodeSet; +import com.ibm.icu.util.RangeValueIterator; + +import org.antlr.v4.runtime.misc.IntervalSet; + +import java.util.LinkedHashMap; +import java.util.Locale; +import java.util.Map; + +/** + * StringTemplate controller used to generate parameters to feed + * to {@code unicodedata.st} to code-generate {@code UnicodeData.java}, + * used by the tool for Unicode property escapes like {@code \\p\{Lu\}}. + * + * Uses ICU to iterate over Unicode character categories, properties, + * and script codes, as well as aliases for those codes. + * + * This class exists in its own Maven module to avoid adding a + * dependency from the tool onto the (large) ICU runtime. + */ +public abstract class UnicodeDataTemplateController { + private static void addIntervalForCategory( + Map<String, IntervalSet> categoryMap, + String categoryName, + int start, + int finish) { + IntervalSet intervalSet = categoryMap.get(categoryName); + if (intervalSet == null) { + intervalSet = new IntervalSet(); + categoryMap.put(categoryName, intervalSet); + } + intervalSet.add(start, finish); + } + + private static void addPropertyAliases( + Map<String, String> propertyAliases, + String propertyName, + int property) { + int nameChoice = UProperty.NameChoice.LONG; + while (true) { + String alias; + try { + alias = UCharacter.getPropertyName(property, nameChoice); + } catch (IllegalArgumentException e) { + // No more aliases. + break; + } + assert alias != null; + addPropertyAlias(propertyAliases, alias, propertyName); + nameChoice++; + } + } + + private static void addPropertyAlias( + Map<String, String> propertyAliases, + String alias, + String propertyName) { + propertyAliases.put(alias, propertyName); + } + + public static Map<String, Object> getProperties() { + Map<String, IntervalSet> propertyCodePointRanges = new LinkedHashMap<>(); + addUnicodeCategoryCodesToCodePointRanges(propertyCodePointRanges); + addUnicodeBinaryPropertyCodesToCodePointRanges(propertyCodePointRanges); + addUnicodeIntPropertyCodesToCodePointRanges(propertyCodePointRanges); + addTR35ExtendedPictographicPropertyCodesToCodePointRanges(propertyCodePointRanges); + addEmojiPresentationPropertyCodesToCodePointRanges(propertyCodePointRanges); + + Map<String, String> propertyAliases = new LinkedHashMap<>(); + addUnicodeCategoryCodesToNames(propertyAliases); + addUnicodeBinaryPropertyCodesToNames(propertyAliases); + addUnicodeScriptCodesToNames(propertyAliases); + addUnicodeBlocksToNames(propertyAliases); + addUnicodeIntPropertyCodesToNames(propertyAliases); + propertyAliases.put("EP", "Extended_Pictographic"); + + Map<String, Object> properties = new LinkedHashMap<>(); + properties.put("propertyCodePointRanges", propertyCodePointRanges); + properties.put("propertyAliases", propertyAliases); + return properties; + } + + private static String getShortPropertyName(int property) { + String propertyName = UCharacter.getPropertyName(property, UProperty.NameChoice.SHORT); + // For some reason, a few properties only have long names. + if (propertyName == null) { + propertyName = UCharacter.getPropertyName(property, UProperty.NameChoice.LONG); + } + return propertyName; + } + + private static void addUnicodeCategoryCodesToCodePointRanges(Map<String, IntervalSet> propertyCodePointRanges) { + RangeValueIterator iter = UCharacter.getTypeIterator(); + RangeValueIterator.Element element = new RangeValueIterator.Element(); + while (iter.next(element)) { + String categoryName = UCharacter.getPropertyValueName( + UProperty.GENERAL_CATEGORY_MASK, + 1 << element.value, + UProperty.NameChoice.SHORT); + addIntervalForCategory(propertyCodePointRanges, categoryName, element.start, element.limit - 1); + // Add short category so Ll, Lu, Lo, etc. all show up under L + String shortCategoryName = categoryName.substring(0, 1); + addIntervalForCategory(propertyCodePointRanges, shortCategoryName, element.start, element.limit - 1); + } + } + + private static void addUnicodeCategoryCodesToNames(Map<String, String> propertyAliases) { + RangeValueIterator iter = UCharacter.getTypeIterator(); + RangeValueIterator.Element element = new RangeValueIterator.Element(); + while (iter.next(element)) { + int generalCategoryMask = 1 << element.value; + String categoryName = UCharacter.getPropertyValueName( + UProperty.GENERAL_CATEGORY_MASK, + generalCategoryMask, + UProperty.NameChoice.SHORT); + int nameChoice = UProperty.NameChoice.LONG; + while (true) { + String alias; + try { + alias = UCharacter.getPropertyValueName( + UProperty.GENERAL_CATEGORY_MASK, + generalCategoryMask, + nameChoice); + } catch (IllegalArgumentException e) { + // No more aliases. + break; + } + assert alias != null; + addPropertyAlias(propertyAliases, alias, categoryName); + nameChoice++; + } + } + // Add short categories + addPropertyAlias(propertyAliases, "Control", "C"); + addPropertyAlias(propertyAliases, "Letter", "L"); + addPropertyAlias(propertyAliases, "Number", "N"); + addPropertyAlias(propertyAliases, "Mark", "M"); + addPropertyAlias(propertyAliases, "Punctuation", "P"); + addPropertyAlias(propertyAliases, "Symbol", "S"); + addPropertyAlias(propertyAliases, "Space", "Z"); + } + + private static void addUnicodeBinaryPropertyCodesToCodePointRanges(Map<String, IntervalSet> propertyCodePointRanges) { + for (int property = UProperty.BINARY_START; + property < UProperty.BINARY_LIMIT; + property++) { + String propertyName = getShortPropertyName(property); + IntervalSet intervalSet = new IntervalSet(); + UnicodeSet unicodeSet = new UnicodeSet(); + unicodeSet.applyIntPropertyValue(property, 1); + for (UnicodeSet.EntryRange range : unicodeSet.ranges()) { + intervalSet.add(range.codepoint, range.codepointEnd); + } + propertyCodePointRanges.put(propertyName, intervalSet); + } + } + + private static void addUnicodeBinaryPropertyCodesToNames(Map<String, String> propertyAliases) { + for (int property = UProperty.BINARY_START; + property < UProperty.BINARY_LIMIT; + property++) { + String propertyName = getShortPropertyName(property); + addPropertyAliases(propertyAliases, propertyName, property); + } + } + + private static void addIntPropertyRanges(int property, String namePrefix, Map<String, IntervalSet> propertyCodePointRanges) { + for (int propertyValue = UCharacter.getIntPropertyMinValue(property); + propertyValue <= UCharacter.getIntPropertyMaxValue(property); + propertyValue++) { + UnicodeSet set = new UnicodeSet(); + set.applyIntPropertyValue(property, propertyValue); + String propertyName = namePrefix + UCharacter.getPropertyValueName(property, propertyValue, UProperty.NameChoice.SHORT); + IntervalSet intervalSet = propertyCodePointRanges.get(propertyName); + if (intervalSet == null) { + intervalSet = new IntervalSet(); + propertyCodePointRanges.put(propertyName, intervalSet); + } + addUnicodeSetToIntervalSet(set, intervalSet); + } + } + + private static void addUnicodeSetToIntervalSet(UnicodeSet unicodeSet, IntervalSet intervalSet) { + for (UnicodeSet.EntryRange range : unicodeSet.ranges()) { + intervalSet.add(range.codepoint, range.codepointEnd); + } + } + + private static void addUnicodeIntPropertyCodesToCodePointRanges(Map<String, IntervalSet> propertyCodePointRanges) { + for (int property = UProperty.INT_START; + property < UProperty.INT_LIMIT; + property++) { + String propertyName = getShortPropertyName(property); + addIntPropertyRanges(property, propertyName + "=", propertyCodePointRanges); + } + } + + private static void addTR35ExtendedPictographicPropertyCodesToCodePointRanges(Map<String, IntervalSet> propertyCodePointRanges) { + IntervalSet set = new IntervalSet(); + // Generated using scripts/parse-extended-pictographic/parse.py + set.add(0x1F774, 0x1F77F); + set.add(0x2700, 0x2701); + set.add(0x2703, 0x2704); + set.add(0x270E); + set.add(0x2710, 0x2711); + set.add(0x2765, 0x2767); + set.add(0x1F030, 0x1F093); + set.add(0x1F094, 0x1F09F); + set.add(0x1F10D, 0x1F10F); + set.add(0x1F12F); + set.add(0x1F16C, 0x1F16F); + set.add(0x1F1AD, 0x1F1E5); + set.add(0x1F260, 0x1F265); + set.add(0x1F203, 0x1F20F); + set.add(0x1F23C, 0x1F23F); + set.add(0x1F249, 0x1F24F); + set.add(0x1F252, 0x1F25F); + set.add(0x1F266, 0x1F2FF); + set.add(0x1F7D5, 0x1F7FF); + set.add(0x1F000, 0x1F003); + set.add(0x1F005, 0x1F02B); + set.add(0x1F02C, 0x1F02F); + set.add(0x1F322, 0x1F323); + set.add(0x1F394, 0x1F395); + set.add(0x1F398); + set.add(0x1F39C, 0x1F39D); + set.add(0x1F3F1, 0x1F3F2); + set.add(0x1F3F6); + set.add(0x1F4FE); + set.add(0x1F53E, 0x1F548); + set.add(0x1F54F); + set.add(0x1F568, 0x1F56E); + set.add(0x1F571, 0x1F572); + set.add(0x1F57B, 0x1F586); + set.add(0x1F588, 0x1F589); + set.add(0x1F58E, 0x1F58F); + set.add(0x1F591, 0x1F594); + set.add(0x1F597, 0x1F5A3); + set.add(0x1F5A6, 0x1F5A7); + set.add(0x1F5A9, 0x1F5B0); + set.add(0x1F5B3, 0x1F5BB); + set.add(0x1F5BD, 0x1F5C1); + set.add(0x1F5C5, 0x1F5D0); + set.add(0x1F5D4, 0x1F5DB); + set.add(0x1F5DF, 0x1F5E0); + set.add(0x1F5E2); + set.add(0x1F5E4, 0x1F5E7); + set.add(0x1F5E9, 0x1F5EE); + set.add(0x1F5F0, 0x1F5F2); + set.add(0x1F5F4, 0x1F5F9); + set.add(0x2605); + set.add(0x2607, 0x260D); + set.add(0x260F, 0x2610); + set.add(0x2612); + set.add(0x2616, 0x2617); + set.add(0x2619, 0x261C); + set.add(0x261E, 0x261F); + set.add(0x2621); + set.add(0x2624, 0x2625); + set.add(0x2627, 0x2629); + set.add(0x262B, 0x262D); + set.add(0x2630, 0x2637); + set.add(0x263B, 0x2647); + set.add(0x2654, 0x265F); + set.add(0x2661, 0x2662); + set.add(0x2664); + set.add(0x2667); + set.add(0x2669, 0x267A); + set.add(0x267C, 0x267E); + set.add(0x2680, 0x2691); + set.add(0x2695); + set.add(0x2698); + set.add(0x269A); + set.add(0x269D, 0x269F); + set.add(0x26A2, 0x26A9); + set.add(0x26AC, 0x26AF); + set.add(0x26B2, 0x26BC); + set.add(0x26BF, 0x26C3); + set.add(0x26C6, 0x26C7); + set.add(0x26C9, 0x26CD); + set.add(0x26D0); + set.add(0x26D2); + set.add(0x26D5, 0x26E8); + set.add(0x26EB, 0x26EF); + set.add(0x26F6); + set.add(0x26FB, 0x26FC); + set.add(0x26FE, 0x26FF); + set.add(0x2388); + set.add(0x1FA00, 0x1FFFD); + set.add(0x1F0A0, 0x1F0AE); + set.add(0x1F0B1, 0x1F0BF); + set.add(0x1F0C1, 0x1F0CF); + set.add(0x1F0D1, 0x1F0F5); + set.add(0x1F0AF, 0x1F0B0); + set.add(0x1F0C0); + set.add(0x1F0D0); + set.add(0x1F0F6, 0x1F0FF); + set.add(0x1F80C, 0x1F80F); + set.add(0x1F848, 0x1F84F); + set.add(0x1F85A, 0x1F85F); + set.add(0x1F888, 0x1F88F); + set.add(0x1F8AE, 0x1F8FF); + set.add(0x1F900, 0x1F90B); + set.add(0x1F91F); + set.add(0x1F928, 0x1F92F); + set.add(0x1F931, 0x1F932); + set.add(0x1F94C); + set.add(0x1F95F, 0x1F96B); + set.add(0x1F992, 0x1F997); + set.add(0x1F9D0, 0x1F9E6); + set.add(0x1F90C, 0x1F90F); + set.add(0x1F93F); + set.add(0x1F94D, 0x1F94F); + set.add(0x1F96C, 0x1F97F); + set.add(0x1F998, 0x1F9BF); + set.add(0x1F9C1, 0x1F9CF); + set.add(0x1F9E7, 0x1F9FF); + set.add(0x1F6C6, 0x1F6CA); + set.add(0x1F6D3, 0x1F6D4); + set.add(0x1F6E6, 0x1F6E8); + set.add(0x1F6EA); + set.add(0x1F6F1, 0x1F6F2); + set.add(0x1F6F7, 0x1F6F8); + set.add(0x1F6D5, 0x1F6DF); + set.add(0x1F6ED, 0x1F6EF); + set.add(0x1F6F9, 0x1F6FF); + propertyCodePointRanges.put("Extended_Pictographic", set); + + UnicodeSet emojiRKUnicodeSet = new UnicodeSet("[\\p{GCB=Regional_Indicator}\\*#0-9\\u00a9\\u00ae\\u2122\\u3030\\u303d]"); + IntervalSet emojiRKIntervalSet = new IntervalSet(); + addUnicodeSetToIntervalSet(emojiRKUnicodeSet, emojiRKIntervalSet); + propertyCodePointRanges.put("EmojiRK", emojiRKIntervalSet); + + UnicodeSet emojiNRKUnicodeSet = new UnicodeSet("[\\p{Emoji=Yes}]"); + emojiNRKUnicodeSet.removeAll(emojiRKUnicodeSet); + IntervalSet emojiNRKIntervalSet = new IntervalSet(); + addUnicodeSetToIntervalSet(emojiNRKUnicodeSet, emojiNRKIntervalSet); + propertyCodePointRanges.put("EmojiNRK", emojiNRKIntervalSet); + } + + private static void addEmojiPresentationPropertyCodesToCodePointRanges(Map<String, IntervalSet> propertyCodePointRanges) { + UnicodeSet emojiDefaultUnicodeSet = new UnicodeSet("[[\\p{Emoji=Yes}]&[\\p{Emoji_Presentation=Yes}]]"); + IntervalSet emojiDefaultIntervalSet = new IntervalSet(); + addUnicodeSetToIntervalSet(emojiDefaultUnicodeSet, emojiDefaultIntervalSet); + propertyCodePointRanges.put("EmojiPresentation=EmojiDefault", emojiDefaultIntervalSet); + + UnicodeSet textDefaultUnicodeSet = new UnicodeSet("[[\\p{Emoji=Yes}]&[\\p{Emoji_Presentation=No}]]"); + IntervalSet textDefaultIntervalSet = new IntervalSet(); + addUnicodeSetToIntervalSet(textDefaultUnicodeSet, textDefaultIntervalSet); + propertyCodePointRanges.put("EmojiPresentation=TextDefault", textDefaultIntervalSet); + + UnicodeSet textUnicodeSet = new UnicodeSet("[\\p{Emoji=No}]"); + IntervalSet textIntervalSet = new IntervalSet(); + addUnicodeSetToIntervalSet(textUnicodeSet, textIntervalSet); + propertyCodePointRanges.put("EmojiPresentation=Text", textIntervalSet); + } + + private static void addIntPropertyAliases(int property, String namePrefix, Map<String, String> propertyAliases) { + String propertyName = getShortPropertyName(property); + for (int propertyValue = UCharacter.getIntPropertyMinValue(property); + propertyValue <= UCharacter.getIntPropertyMaxValue(property); + propertyValue++) { + String aliasTarget = propertyName + "=" + UCharacter.getPropertyValueName(property, propertyValue, UProperty.NameChoice.SHORT); + int nameChoice = UProperty.NameChoice.SHORT; + String alias; + while (true) { + try { + alias = namePrefix + UCharacter.getPropertyValueName(property, propertyValue, nameChoice); + } catch (IllegalArgumentException e) { + // No more aliases. + break; + } + assert alias != null; + addPropertyAlias(propertyAliases, alias, aliasTarget); + nameChoice++; + } + } + } + + private static void addUnicodeScriptCodesToNames(Map<String, String> propertyAliases) { + addIntPropertyAliases(UProperty.SCRIPT, "", propertyAliases); + } + + private static void addUnicodeBlocksToNames(Map<String, String> propertyAliases) { + addIntPropertyAliases(UProperty.BLOCK, "In", propertyAliases); + } + + private static void addUnicodeIntPropertyCodesToNames(Map<String, String> propertyAliases) { + for (int property = UProperty.INT_START; + property < UProperty.INT_LIMIT; + property++) { + int nameChoice = UProperty.NameChoice.SHORT + 1; + while (true) { + String propertyNameAlias; + try { + propertyNameAlias = UCharacter.getPropertyName(property, nameChoice); + } catch (IllegalArgumentException e) { + // No more aliases. + break; + } + addIntPropertyAliases(property, propertyNameAlias + "=", propertyAliases); + nameChoice++; + } + } + } +} |
