summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorAndrej Shadura <andrew.shadura@collabora.co.uk>2019-01-29 17:36:41 +0100
committerAndrej Shadura <andrew.shadura@collabora.co.uk>2019-01-29 17:36:41 +0100
commit874048049e4612f67223411775a8d465ecfee617 (patch)
tree68740b579f4ec1898c142462f8447e9259a08e5d
parent99ffe9a3532b02a5623fc19bc5a0771ea5277f71 (diff)
parentec4620c761c5890a4c54eb4dddd9745708b26ff5 (diff)
Update upstream source from tag 'upstream/4.7.2'
Update to upstream version '4.7.2' with Debian dir 35e818e10d63d78e1a7b062fbf5bcd472710a905
-rw-r--r--README.md2
-rw-r--r--antlr4-maven-plugin/pom.xml2
-rw-r--r--contributors.txt32
-rw-r--r--doc/antlr-project-testing.md4
-rw-r--r--doc/building-antlr.md11
-rw-r--r--doc/csharp-target.md5
-rw-r--r--doc/getting-started.md14
-rw-r--r--doc/java-target.md4
-rw-r--r--doc/javascript-target.md43
-rw-r--r--doc/releasing-antlr.md36
-rw-r--r--doc/swift-target.md7
-rw-r--r--pom.xml4
-rw-r--r--runtime-testsuite/annotations/pom.xml2
-rw-r--r--runtime-testsuite/pom.xml17
-rw-r--r--runtime-testsuite/processors/pom.xml10
-rw-r--r--runtime/Java/pom.xml3
-rw-r--r--runtime/Java/src/org/antlr/v4/runtime/CodePointCharStream.java8
-rw-r--r--runtime/Java/src/org/antlr/v4/runtime/RuntimeMetaData.java2
-rw-r--r--runtime/Java/src/org/antlr/v4/runtime/atn/PredictionContext.java4
-rw-r--r--scripts/github_release_notes.py56
-rw-r--r--tool-testsuite/pom.xml4
-rw-r--r--tool-testsuite/test/org/antlr/v4/test/tool/TestCodeGeneration.java19
-rw-r--r--tool/pom.xml6
-rw-r--r--tool/resources/org/antlr/v4/tool/templates/codegen/Cpp/Cpp.stg3
-rw-r--r--tool/resources/org/antlr/v4/tool/templates/codegen/Java/Java.stg36
-rw-r--r--tool/resources/org/antlr/v4/tool/templates/codegen/Python2/Python2.stg4
-rw-r--r--tool/resources/org/antlr/v4/tool/templates/codegen/Python3/Python3.stg11
-rwxr-xr-xtool/resources/org/antlr/v4/tool/templates/codegen/Swift/Swift.stg2
-rw-r--r--tool/src/org/antlr/v4/Tool.java4
-rw-r--r--tool/src/org/antlr/v4/codegen/Target.java2
-rw-r--r--tool/src/org/antlr/v4/codegen/model/ElementFrequenciesVisitor.java10
-rw-r--r--tool/src/org/antlr/v4/codegen/model/RuleFunction.java60
-rw-r--r--tool/src/org/antlr/v4/codegen/target/CSharpTarget.java2
-rw-r--r--tool/src/org/antlr/v4/codegen/target/CppTarget.java2
-rw-r--r--tool/src/org/antlr/v4/codegen/target/GoTarget.java2
-rw-r--r--tool/src/org/antlr/v4/codegen/target/JavaScriptTarget.java2
-rw-r--r--tool/src/org/antlr/v4/codegen/target/Python2Target.java33
-rw-r--r--tool/src/org/antlr/v4/codegen/target/Python3Target.java35
-rw-r--r--tool/src/org/antlr/v4/codegen/target/SwiftTarget.java2
-rw-r--r--tool/src/org/antlr/v4/tool/ErrorManager.java3
-rw-r--r--tool/src/org/antlr/v4/tool/Grammar.java10
41 files changed, 340 insertions, 178 deletions
diff --git a/README.md b/README.md
index efa0181..d63bc4f 100644
--- a/README.md
+++ b/README.md
@@ -1,6 +1,6 @@
# ANTLR v4
-[![Build Travis-CI Status](https://travis-ci.org/antlr/antlr4.png?branch=master)](https://travis-ci.org/antlr/antlr4) [![Build AppVeyor Status](https://ci.appveyor.com/api/projects/status/5acpbx1pg7bhgh8v/branch/master?svg=true)](https://ci.appveyor.com/project/parrt/antlr4) [![Java 7+](https://img.shields.io/badge/java-7+-4c7e9f.svg)](http://java.oracle.com) [![License](https://img.shields.io/badge/license-BSD-blue.svg)](https://raw.githubusercontent.com/antlr/antlr4/master/LICENSE.txt)
+[![Build Travis-CI Status](https://travis-ci.org/antlr/antlr4.svg?branch=master)](https://travis-ci.org/antlr/antlr4) [![Build AppVeyor Status](https://ci.appveyor.com/api/projects/status/5acpbx1pg7bhgh8v/branch/master?svg=true)](https://ci.appveyor.com/project/parrt/antlr4) [![Java 7+](https://img.shields.io/badge/java-7+-4c7e9f.svg)](http://java.oracle.com) [![License](https://img.shields.io/badge/license-BSD-blue.svg)](https://raw.githubusercontent.com/antlr/antlr4/master/LICENSE.txt)
**ANTLR** (ANother Tool for Language Recognition) is a powerful parser generator for reading, processing, executing, or translating structured text or binary files. It's widely used to build languages, tools, and frameworks. From a grammar, ANTLR generates a parser that can build parse trees and also generates a listener interface (or visitor) that makes it easy to respond to the recognition of phrases of interest.
diff --git a/antlr4-maven-plugin/pom.xml b/antlr4-maven-plugin/pom.xml
index c148583..a265ec3 100644
--- a/antlr4-maven-plugin/pom.xml
+++ b/antlr4-maven-plugin/pom.xml
@@ -8,7 +8,7 @@
<parent>
<groupId>org.antlr</groupId>
<artifactId>antlr4-master</artifactId>
- <version>4.7.1</version>
+ <version>4.7.2</version>
</parent>
<artifactId>antlr4-maven-plugin</artifactId>
<packaging>maven-plugin</packaging>
diff --git a/contributors.txt b/contributors.txt
index b85abf0..230e494 100644
--- a/contributors.txt
+++ b/contributors.txt
@@ -178,3 +178,35 @@ YYYY/MM/DD, github id, Full name, email
2017/12/01, DavidMoraisFerreira, David Morais Ferreira, david.moraisferreira@gmail.com
2017/12/01, SebastianLng, Sebastian Lang, sebastian.lang@outlook.com
2017/12/03, oranoran, Oran Epelbaum, oran / epelbaum me
+2017/12/12, janlinde, Jan Lindemann, jan@janware.com
+2017/12/13, enessoylu, Enes Soylu, enessoylutr@gmail.com
+2017/12/20, kbsletten, Kyle Sletten, kbsletten@gmail.com
+2017/12/27, jkmar, Jakub Marciniszyn, marciniszyn.jk@gmail.com
+2018/03/08, dannoc, Daniel Clifford, danno@google.com
+2018/03/10, uvguy, kangjoni76@gmail.com
+2018/01/06, kasbah, Kaspar Emanuel, kaspar@monostable.co.uk
+2018/01/15, xgcssch, Sönke Schau, xgcssch@users.noreply.github.com
+2018/02/08, razfriman, Raz Friman, raz@razfriman.com
+2018/02/11, io7m, Mark Raynsford, code@io7m.com
+2018/04/24, solussd, Joe Smith, joe@uwcreations.com
+2018/15/05, johnvanderholt, jan dillingh johnvanderholte@gmail.com
+2018/06/14, scadgek, Sergey Chupov, scadgek@live.com
+2018/06/16, EternalPhane, Zongyuan Zuo, eternalphane@gmail.com
+2018/06/27, wu-sheng, Wu Sheng, wu.sheng@foxmail.com
+2018/02/25, chaseoxide, Marcus Ong, taccs97[at]gmail[dot]com
+2018/05/15, johnvanderholt, jan dillingh johnvanderholte@gmail.com
+2018/06/16, EternalPhane, Zongyuan Zuo, eternalphane@gmail.com
+2018/05/15, johnvanderholt, jan dillingh johnvanderholte@gmail.com
+2018/05/17, sinopsysHK, Eric Bardes, sinofwd@gmail.com
+2018/05/23, srvance, Stephen Vance, steve@vance.com
+2018/06/14, alecont, Alessandro Contenti, alecontenti@hotmail.com
+2018/06/16, EternalPhane, Zongyuan Zuo, eternalphane@gmail.com
+2018/07/03, jgoppert, James Goppert, james.goppert@gmail.com
+2018/07/27, Maksim Novikov, mnovikov.work@gmail.com
+2018/07/31, Lucas Henrqiue, lucashenrique580@gmail.com
+2018/08/03, ENDOH takanao, djmchl@gmail.com
+2018/10/29, chrisaycock, Christopher Aycock, chris[at]chrisaycock[dot]com
+2018/11/12, vinoski, Steve Vinoski, vinoski@ieee.org
+2018/11/14, nxtstep, Adriaan (Arjan) Duz, codewithadriaan[et]gmail[dot]com
+2018/11/15, amykyta3, Alex Mykyta, amykyta3@users.noreply.github.com
+2018/11/29, hannemann-tamas, Ralf Hannemann-Tamas, ralf.ht@gmail.com \ No newline at end of file
diff --git a/doc/antlr-project-testing.md b/doc/antlr-project-testing.md
index 335f2da..1ff46aa 100644
--- a/doc/antlr-project-testing.md
+++ b/doc/antlr-project-testing.md
@@ -42,9 +42,9 @@ In order to perform the tests on all target languages, you need to have the foll
* `mono` (e.g., `brew install mono`) on non-Windows boxes (on Windows it uses the Microsoft .net stack). Also must [`xbuild` the runtime](https://github.com/antlr/antlr4/blob/master/doc/releasing-antlr.md) before tests will run; see below
* `nodejs`
* Python 2.7
-* Python 3.5
+* Python 3.6
* Go
-* Swift 3 (via XCode 8.x) tested currently only osx
+* Swift 4 (via XCode 10.x) tested currently only osx
* clang (for C++ target)
*
To **install into local repository** `~/.m2/repository/org/antlr`, do this:
diff --git a/doc/building-antlr.md b/doc/building-antlr.md
index 06f1ace..494bea8 100644
--- a/doc/building-antlr.md
+++ b/doc/building-antlr.md
@@ -28,16 +28,7 @@ Checking out files: 100% (1427/1427), done.
# Compile
```bash
-$ cd /tmp
-$ git clone git@github.com:antlr/antlr4.git
-Cloning into 'antlr4'...
-remote: Counting objects: 59858, done.
-remote: Compressing objects: 100% (57/57), done.
-remote: Total 59858 (delta 28), reused 9 (delta 9), pack-reused 59786
-Receiving objects: 100% (59858/59858), 31.10 MiB | 819.00 KiB/s, done.
-Resolving deltas: 100% (31898/31898), done.
-Checking connectivity... done.
-$ cd antlr4
+$ cd /tmp/antlr4
$ export MAVEN_OPTS="-Xmx1G" # don't forget this on linux
$ mvn clean # must be separate, not part of install/compile
$ mvn -DskipTests install
diff --git a/doc/csharp-target.md b/doc/csharp-target.md
index 9158b02..40be518 100644
--- a/doc/csharp-target.md
+++ b/doc/csharp-target.md
@@ -21,7 +21,7 @@ You will find full instructions on the [Git repo page for ANTLR C# runtime](http
Let's suppose that your grammar is named `MyGrammar`. The tool will generate for you the following files:
-* MyGrammarLexer.cs
+* MyGrammarLexer.cs
* MyGrammarParser.cs
* MyGrammarListener.cs (if you have not activated the -no-listener option)
* MyGrammarBaseListener.cs (if you have not activated the -no-listener option)
@@ -32,6 +32,7 @@ Now a fully functioning code might look like the following for start rule `Start
```
using Antlr4.Runtime;
+using Antlr4.Runtime.Tree;
public void MyParseMethod() {
String input = "your text to parse here";
@@ -39,7 +40,7 @@ public void MyParseMethod() {
ITokenSource lexer = new MyGrammarLexer(stream);
ITokenStream tokens = new CommonTokenStream(lexer);
MyGrammarParser parser = new MyGrammarParser(tokens);
- parser.buildParseTrees = true;
+ parser.BuildParseTree = true;
IParseTree tree = parser.StartRule();
}
```
diff --git a/doc/getting-started.md b/doc/getting-started.md
index dabd8c1..ca1fd33 100644
--- a/doc/getting-started.md
+++ b/doc/getting-started.md
@@ -16,10 +16,10 @@ If you are going to integrate ANTLR into your existing build system using mvn, a
1. Download
```
$ cd /usr/local/lib
-$ curl -O http://www.antlr.org/download/antlr-4.7.1-complete.jar
+$ curl -O https://www.antlr.org/download/antlr-4.7.1-complete.jar
```
Or just download in browser from website:
- [http://www.antlr.org/download.html](http://www.antlr.org/download.html)
+ [https://www.antlr.org/download.html](https://www.antlr.org/download.html)
and put it somewhere rational like `/usr/local/lib`.
2. Add `antlr-4.7.1-complete.jar` to your `CLASSPATH`:
@@ -31,7 +31,7 @@ It's also a good idea to put this in your `.bash_profile` or whatever your start
3. Create aliases for the ANTLR Tool, and `TestRig`.
```
$ alias antlr4='java -Xmx500M -cp "/usr/local/lib/antlr-4.7.1-complete.jar:$CLASSPATH" org.antlr.v4.Tool'
-$ alias grun='java org.antlr.v4.gui.TestRig'
+$ alias grun='java -Xmx500M -cp "/usr/local/lib/antlr-4.7.1-complete.jar:$CLASSPATH" org.antlr.v4.gui.TestRig'
```
### WINDOWS
@@ -39,9 +39,9 @@ $ alias grun='java org.antlr.v4.gui.TestRig'
(*Thanks to Graham Wideman*)
0. Install Java (version 1.6 or higher)
-1. Download antlr-4.5.3-complete.jar (or whatever version) from [http://www.antlr.org/download/](http://www.antlr.org/download/)
+1. Download antlr-4.7.1-complete.jar (or whatever version) from [https://www.antlr.org/download/](https://www.antlr.org/download/)
Save to your directory for 3rd party Java libraries, say `C:\Javalib`
-2. Add `antlr-4.5.3-complete.jar` to CLASSPATH, either:
+2. Add `antlr-4.7.1-complete.jar` to CLASSPATH, either:
* Permanently: Using System Properties dialog > Environment variables > Create or append to `CLASSPATH` variable
* Temporarily, at command line:
```
@@ -53,6 +53,10 @@ SET CLASSPATH=.;C:\Javalib\antlr-4.7.1-complete.jar;%CLASSPATH%
java org.antlr.v4.Tool %*
```
```
+@ECHO OFF
+SET TEST_CURRENT_DIR=%CLASSPATH:.;=%
+if "%TEST_CURRENT_DIR%" == "%CLASSPATH%" ( SET CLASSPATH=.;%CLASSPATH% )
+@ECHO ON
java org.antlr.v4.gui.TestRig %*
```
* Or, use doskey commands:
diff --git a/doc/java-target.md b/doc/java-target.md
index f6e6154..60612a9 100644
--- a/doc/java-target.md
+++ b/doc/java-target.md
@@ -16,7 +16,7 @@ Also, I have prepared a [video](https://youtu.be/eW4WFgRtFeY) that will help you
### Eclipse
-Edgar Espina has created an [eclipse plugin for ANTLR v4](https://youtu.be/eW4WFgRtFeY). Features: Advanced Syntax Highlighting, Automatic Code Generation (on save), Manual Code Generation (through External Tools menu), Code Formatter (Ctrl+Shift+F), Syntax Diagrams, Advanced Rule Navigation between files (F3), Quick fixes.
+Edgar Espina has created an [eclipse plugin for ANTLR v4](https://marketplace.eclipse.org/content/antlr-ide). Features: Advanced Syntax Highlighting, Automatic Code Generation (on save), Manual Code Generation (through External Tools menu), Code Formatter (Ctrl+Shift+F), Syntax Diagrams, Advanced Rule Navigation between files (F3), Quick fixes.
### NetBeans
@@ -241,4 +241,4 @@ mvn install
mvn exec:java -Dexec.mainClass=org.abcd.examples.ArrayInit.ArrayInit
{1,2,3}
^D
-``` \ No newline at end of file
+```
diff --git a/doc/javascript-target.md b/doc/javascript-target.md
index 3473e95..2d9973b 100644
--- a/doc/javascript-target.md
+++ b/doc/javascript-target.md
@@ -62,7 +62,7 @@ The steps to create your parsing code are the following:
You are now ready to bundle your parsing code as follows:
- following webpack specs, create a webpack.config file
- in the webpack.config file, exclude node.js only modules using: node: { module: "empty", net: "empty", fs: "empty" }
- - from the cmd line, nag-vigate to the directory containing webpack.config and type: webpack
+ - from the cmd line, navigate to the directory containing webpack.config and type: webpack
This will produce a single js file containing all your parsing code. Easy to include in your web pages!
@@ -95,11 +95,16 @@ Let's suppose that your grammar is named, as above, "MyGrammar". Let's suppose t
Now a fully functioning script might look like the following:
```javascript
+ var antlr4 = require('antlr4');
+ var MyGrammarLexer = require('./MyGrammarLexer').MyGrammarLexer;
+ var MyGrammarParser = require('./MyGrammarParser').MyGrammarParser;
+ var MyGrammarListener = require('./MyGrammarListener').MyGrammarListener;
+
var input = "your text to parse here"
var chars = new antlr4.InputStream(input);
- var lexer = new MyGrammarLexer.MyGrammarLexer(chars);
+ var lexer = new MyGrammarLexer(chars);
var tokens = new antlr4.CommonTokenStream(lexer);
- var parser = new MyGrammarParser.MyGrammarParser(tokens);
+ var parser = new MyGrammarParser(tokens);
parser.buildParseTrees = true;
var tree = parser.MyStartRule();
```
@@ -128,30 +133,30 @@ Let's suppose your MyGrammar grammar comprises 2 rules: "key" and "value". The a
```
In order to provide custom behavior, you might want to create the following class:
-
+
```javascript
- KeyPrinter = function() {
- MyGrammarListener.call(this); // inherit default listener
- return this;
- };
-
-// inherit default listener
+var KeyPrinter = function() {
+ MyGrammarListener.call(this); // inherit default listener
+ return this;
+};
+
+// continue inheriting default listener
KeyPrinter.prototype = Object.create(MyGrammarListener.prototype);
KeyPrinter.prototype.constructor = KeyPrinter;
-
+
// override default listener behavior
- KeyPrinter.prototype.exitKey = function(ctx) {
- console.log("Oh, a key!");
- };
+KeyPrinter.prototype.exitKey = function(ctx) {
+ console.log("Oh, a key!");
+};
```
In order to execute this listener, you would simply add the following lines to the above code:
-
+
```javascript
- ...
- tree = parser.StartRule() - only repeated here for reference
- var printer = new KeyPrinter();
- antlr4.tree.ParseTreeWalker.DEFAULT.walk(printer, tree);
+ ...
+ tree = parser.StartRule() // only repeated here for reference
+var printer = new KeyPrinter();
+antlr4.tree.ParseTreeWalker.DEFAULT.walk(printer, tree);
```
## What about TypeScript?
diff --git a/doc/releasing-antlr.md b/doc/releasing-antlr.md
index 49bcd18..a44e48e 100644
--- a/doc/releasing-antlr.md
+++ b/doc/releasing-antlr.md
@@ -150,13 +150,19 @@ Uploaded: https://oss.sonatype.org/content/repositories/snapshots/org/antlr/antl
The maven deploy lifecycle phased deploys the artifacts and the poms for the ANTLR project to the [sonatype remote staging server](https://oss.sonatype.org/content/repositories/snapshots/).
```bash
-mvn deploy -DskipTests
+export JAVA_HOME=`/usr/libexec/java_home -v 1.7`; mvn deploy -DskipTests
```
With JDK 1.7 (not 6 or 8), do this:
```bash
-mvn release:prepare -Darguments="-DskipTests"
+export JAVA_HOME=`/usr/libexec/java_home -v 1.7`; mvn release:prepare -Darguments="-DskipTests"
+```
+
+Hm...per https://github.com/keybase/keybase-issues/issues/1712 we need this to make gpg work:
+
+```bash
+export GPG_TTY=$(tty)
```
Side note to set jdk 1.7 on os x:
@@ -166,6 +172,13 @@ alias java='/Library/Java/JavaVirtualMachines/jdk1.7.0_21.jdk/Contents/Home/bin/
alias javac='/Library/Java/JavaVirtualMachines/jdk1.7.0_21.jdk/Contents/Home/bin/javac'
alias javadoc='/Library/Java/JavaVirtualMachines/jdk1.7.0_21.jdk/Contents/Home/bin/javadoc'
alias jar='/Library/Java/JavaVirtualMachines/jdk1.7.0_21.jdk/Contents/Home/bin/jar'
+export JAVA_HOME=`/usr/libexec/java_home -v 1.7`
+```
+
+But I think just this on front of mvn works:
+
+```
+export JAVA_HOME=`/usr/libexec/java_home -v 1.7`; mvn ...
```
You should see 0x33 in generated .class files after 0xCAFEBABE; see [Java SE 7 = 51 (0x33 hex)](https://en.wikipedia.org/wiki/Java_class_file):
@@ -260,7 +273,9 @@ popd
### CSharp
-*Publishing to Nuget from Windows*
+Now we have [appveyor create artifact](https://ci.appveyor.com/project/parrt/antlr4/build/artifacts). Go to [nuget](https://www.nuget.org/packages/manage/upload) to upload the `.nupkg`.
+
+### Publishing to Nuget from Windows
**Install the pre-requisites**
@@ -326,13 +341,12 @@ index-servers =
pypitest
[pypi]
-repository: https://pypi.python.org/pypi
username: parrt
-password: XXX
+password: xxx
[pypitest]
-repository: https://testpypi.python.org/pypi
username: parrt
+password: xxx
```
Then run the usual python set up stuff:
@@ -340,8 +354,7 @@ Then run the usual python set up stuff:
```bash
cd ~/antlr/code/antlr4/runtime/Python2
# assume you have ~/.pypirc set up
-python setup.py register -r pypi
-python setup.py sdist bdist_wininst upload -r pypi
+python2 setup.py sdist upload
```
and do again for Python 3 target
@@ -349,8 +362,7 @@ and do again for Python 3 target
```bash
cd ~/antlr/code/antlr4/runtime/Python3
# assume you have ~/.pypirc set up
-python setup.py register -r pypi
-python setup.py sdist bdist_wininst upload -r pypi
+python3 setup.py sdist upload
```
There are links to the artifacts in [download.html](http://www.antlr.org/download.html) already.
@@ -384,12 +396,12 @@ cd runtime/Cpp
cp antlr4-cpp-runtime-source.zip ~/antlr/sites/website-antlr4/download/antlr4-cpp-runtime-4.7-source.zip
```
-On a Windows machine the build scripts checks if VS 2013 and/or VS 2015 are installed and builds binaries for each, if found. This script requires 7z to be installed (http://7-zip.org).
+On a Windows machine the build scripts checks if VS 2013 and/or VS 2015 are installed and builds binaries for each, if found. This script requires 7z to be installed (http://7-zip.org then do `set PATH=%PATH%;C:\Program Files\7-Zip\` from DOS not powershell).
```bash
cd runtime/Cpp
deploy-windows.cmd
-cp antlr4-cpp-runtime-vs2015.zip ~/antlr/sites/website-antlr4/download/antlr4-cpp-runtime-4.7-vs2015.zip
+cp runtime\bin\vs-2015\x64\Release DLL\antlr4-cpp-runtime-vs2015.zip ~/antlr/sites/website-antlr4/download/antlr4-cpp-runtime-4.7-vs2015.zip
```
Move target to website (**_rename to a specific ANTLR version first if needed_**):
diff --git a/doc/swift-target.md b/doc/swift-target.md
index e438908..d023ee4 100644
--- a/doc/swift-target.md
+++ b/doc/swift-target.md
@@ -1,5 +1,12 @@
# ANTLR4 Language Target, Runtime for Swift
+## Requirements
+
+ANTLR 4.7.2 requires Swift 4.2. It works on Swift 4.2.1 also.
+
+ANTLR 4.7.1 requires Swift 4.0, and does not work on Swift 4.2. (The status of
+Swift 4.1 support is unknown.)
+
## Performance Note
To use ANTLR4 Swift target in production environment, make sure to turn on compiler optimizations by following [these instructions](https://github.com/apple/swift-package-manager/blob/master/Documentation/Usage.md#build-configurations) if you use SwiftPM to build your project. If you are using Xcode to build your project, it's unlikely you will not use `release` build for production build.
diff --git a/pom.xml b/pom.xml
index a6759c0..530fadf 100644
--- a/pom.xml
+++ b/pom.xml
@@ -13,7 +13,7 @@
</parent>
<groupId>org.antlr</groupId>
<artifactId>antlr4-master</artifactId>
- <version>4.7.1</version>
+ <version>4.7.2</version>
<packaging>pom</packaging>
<name>ANTLR 4</name>
@@ -138,6 +138,8 @@
<filesets>
<fileset>
<directory>runtime/Swift/.build</directory>
+ </fileset>
+ <fileset>
<directory>runtime/Swift/Tests/Antlr4Tests/gen</directory>
</fileset>
</filesets>
diff --git a/runtime-testsuite/annotations/pom.xml b/runtime-testsuite/annotations/pom.xml
index b476c0a..96f57e3 100644
--- a/runtime-testsuite/annotations/pom.xml
+++ b/runtime-testsuite/annotations/pom.xml
@@ -9,7 +9,7 @@
<parent>
<groupId>org.antlr</groupId>
<artifactId>antlr4-master</artifactId>
- <version>4.7.1</version>
+ <version>4.7.2</version>
<relativePath>../../pom.xml</relativePath>
</parent>
<artifactId>antlr4-runtime-test-annotations</artifactId>
diff --git a/runtime-testsuite/pom.xml b/runtime-testsuite/pom.xml
index 7015303..dc33587 100644
--- a/runtime-testsuite/pom.xml
+++ b/runtime-testsuite/pom.xml
@@ -10,7 +10,7 @@
<parent>
<groupId>org.antlr</groupId>
<artifactId>antlr4-master</artifactId>
- <version>4.7.1</version>
+ <version>4.7.2</version>
</parent>
<artifactId>antlr4-runtime-testsuite</artifactId>
<name>ANTLR 4 Runtime Tests (2nd generation)</name>
@@ -26,7 +26,7 @@
<dependency>
<groupId>org.antlr</groupId>
<artifactId>ST4</artifactId>
- <version>4.0.8</version>
+ <version>4.1</version>
<scope>test</scope>
</dependency>
<dependency>
@@ -66,10 +66,10 @@
<scope>test</scope>
</dependency>
<dependency>
- <groupId>org.eclipse.jetty</groupId>
- <artifactId>jetty-server</artifactId>
- <version>8.1.16.v20140903</version>
- <scope>test</scope>
+ <groupId>org.eclipse.jetty</groupId>
+ <artifactId>jetty-server</artifactId>
+ <version>[9.3.24.v20180605,)</version>
+ <scope>test</scope>
</dependency>
<dependency>
<groupId>org.glassfish</groupId>
@@ -92,6 +92,11 @@
</resource>
<resource>
<directory>../runtime</directory>
+ <excludes>
+ <exclude>**/.build/**</exclude>
+ <exclude>**/target/**</exclude>
+ <exclude>Swift/*.xcodeproj/**</exclude>
+ </excludes>
</resource>
</resources>
<plugins>
diff --git a/runtime-testsuite/processors/pom.xml b/runtime-testsuite/processors/pom.xml
index deccaef..87b351f 100644
--- a/runtime-testsuite/processors/pom.xml
+++ b/runtime-testsuite/processors/pom.xml
@@ -9,7 +9,7 @@
<parent>
<groupId>org.antlr</groupId>
<artifactId>antlr4-master</artifactId>
- <version>4.7.1</version>
+ <version>4.7.2</version>
<relativePath>../../pom.xml</relativePath>
</parent>
<artifactId>antlr4-runtime-test-annotation-processors</artifactId>
@@ -18,11 +18,9 @@
<dependencies>
<dependency>
- <groupId>com.sun</groupId>
- <artifactId>tools</artifactId>
- <version>1.4.2</version>
- <scope>system</scope>
- <systemPath>${java.home}/../lib/tools.jar</systemPath>
+ <groupId>com.github.olivergondza</groupId>
+ <artifactId>maven-jdk-tools-wrapper</artifactId>
+ <version>0.1</version>
</dependency>
<dependency>
<groupId>org.antlr</groupId>
diff --git a/runtime/Java/pom.xml b/runtime/Java/pom.xml
index c700407..eb000a1 100644
--- a/runtime/Java/pom.xml
+++ b/runtime/Java/pom.xml
@@ -9,7 +9,7 @@
<parent>
<groupId>org.antlr</groupId>
<artifactId>antlr4-master</artifactId>
- <version>4.7.1</version>
+ <version>4.7.2</version>
<relativePath>../../pom.xml</relativePath>
</parent>
<artifactId>antlr4-runtime</artifactId>
@@ -96,6 +96,7 @@
<phase>process-classes</phase>
<configuration>
<instructions>
+ <Automatic-Module-Name>org.antlr.antlr4.runtime</Automatic-Module-Name>
<Bundle-SymbolicName>org.antlr.antlr4-runtime</Bundle-SymbolicName>
</instructions>
</configuration>
diff --git a/runtime/Java/src/org/antlr/v4/runtime/CodePointCharStream.java b/runtime/Java/src/org/antlr/v4/runtime/CodePointCharStream.java
index 107faa7..2413545 100644
--- a/runtime/Java/src/org/antlr/v4/runtime/CodePointCharStream.java
+++ b/runtime/Java/src/org/antlr/v4/runtime/CodePointCharStream.java
@@ -203,8 +203,8 @@ public abstract class CodePointCharStream implements CharStream {
/** Return the UTF-16 encoded string for the given interval */
@Override
public String getText(Interval interval) {
- int startIdx = Math.min(interval.a, size - 1);
- int len = Math.min(interval.b - interval.a + 1, size);
+ int startIdx = Math.min(interval.a, size);
+ int len = Math.min(interval.b - interval.a + 1, size - startIdx);
// We know there are no surrogates in this
// array, since otherwise we would be given a
@@ -258,8 +258,8 @@ public abstract class CodePointCharStream implements CharStream {
/** Return the UTF-16 encoded string for the given interval */
@Override
public String getText(Interval interval) {
- int startIdx = Math.min(interval.a, size - 1);
- int len = Math.min(interval.b - interval.a + 1, size);
+ int startIdx = Math.min(interval.a, size);
+ int len = Math.min(interval.b - interval.a + 1, size - startIdx);
// Note that we pass the int[] code points to the String constructor --
// this is supported, and the constructor will convert to UTF-16 internally.
diff --git a/runtime/Java/src/org/antlr/v4/runtime/RuntimeMetaData.java b/runtime/Java/src/org/antlr/v4/runtime/RuntimeMetaData.java
index bc2e478..05e8144 100644
--- a/runtime/Java/src/org/antlr/v4/runtime/RuntimeMetaData.java
+++ b/runtime/Java/src/org/antlr/v4/runtime/RuntimeMetaData.java
@@ -67,7 +67,7 @@ public class RuntimeMetaData {
* omitted.</li>
* </ul>
*/
- public static final String VERSION = "4.7.1";
+ public static final String VERSION = "4.7.2";
/**
* Gets the currently executing version of the ANTLR 4 runtime library.
diff --git a/runtime/Java/src/org/antlr/v4/runtime/atn/PredictionContext.java b/runtime/Java/src/org/antlr/v4/runtime/atn/PredictionContext.java
index c55ba09..25d8bc4 100644
--- a/runtime/Java/src/org/antlr/v4/runtime/atn/PredictionContext.java
+++ b/runtime/Java/src/org/antlr/v4/runtime/atn/PredictionContext.java
@@ -316,14 +316,14 @@ public abstract class PredictionContext {
}
else {
if ( a == EMPTY && b == EMPTY ) return EMPTY; // $ + $ = $
- if ( a == EMPTY ) { // $ + x = [$,x]
+ if ( a == EMPTY ) { // $ + x = [x,$]
int[] payloads = {b.returnState, EMPTY_RETURN_STATE};
PredictionContext[] parents = {b.parent, null};
PredictionContext joined =
new ArrayPredictionContext(parents, payloads);
return joined;
}
- if ( b == EMPTY ) { // x + $ = [$,x] ($ is always first if present)
+ if ( b == EMPTY ) { // x + $ = [x,$] ($ is always last if present)
int[] payloads = {a.returnState, EMPTY_RETURN_STATE};
PredictionContext[] parents = {a.parent, null};
PredictionContext joined =
diff --git a/scripts/github_release_notes.py b/scripts/github_release_notes.py
index a0b12d2..10b8cca 100644
--- a/scripts/github_release_notes.py
+++ b/scripts/github_release_notes.py
@@ -19,36 +19,32 @@ milestone = milestone[0]
issues = repo.get_issues(state="closed", milestone=milestone, sort="created", direction="desc")
-# # dump bugs fixed
-# print()
-# print("## Issues fixed")
-# for x in issues:
-# labels = [l.name for l in x.labels]
-# if x.pull_request is None and not ("type:improvement" in labels or "type:feature" in labels):
-# print("* [%s](%s) (%s)" % (x.title, x.html_url, ", ".join([l.name for l in x.labels])))
-#
-#
-# print()
-# # dump improvements closed for this release (issues or pulls)
-# print("## Improvements, features")
-# for x in issues:
-# labels = [l.name for l in x.labels]
-# if ("type:improvement" in labels or "type:feature" in labels):
-# print("* [%s](%s) (%s)" % (x.title, x.html_url, ", ".join(labels)))
-#
-# print()
-#
-#
-# # dump PRs closed for this release by target
-# print("## Pull requests grouped by target")
-# for target in TARGETS:
-# print()
-# print(f"### {target} target")
-# for x in issues:
-# labels = [l.name for l in x.labels]
-# if x.pull_request is not None and f"target:{target}" in labels:
-# print("* [%s](%s) (%s)" % (x.title, x.html_url, ", ".join(labels)))
-#
+# dump bugs fixed
+print()
+print("## Issues fixed")
+for x in issues:
+ labels = [l.name for l in x.labels]
+ if x.pull_request is None and not ("type:improvement" in labels or "type:feature" in labels):
+ print("* [%s](%s) (%s)" % (x.title, x.html_url, ", ".join([l.name for l in x.labels])))
+
+# dump improvements closed for this release (issues or pulls)
+print()
+print("## Improvements, features")
+for x in issues:
+ labels = [l.name for l in x.labels]
+ if ("type:improvement" in labels or "type:feature" in labels):
+ print("* [%s](%s) (%s)" % (x.title, x.html_url, ", ".join(labels)))
+
+# dump PRs closed for this release by target
+print()
+print("## Pull requests grouped by target")
+for target in TARGETS:
+ print()
+ print(f"### {target} target")
+ for x in issues:
+ labels = [l.name for l in x.labels]
+ if x.pull_request is not None and f"target:{target}" in labels:
+ print("* [%s](%s) (%s)" % (x.title, x.html_url, ", ".join(labels)))
# dump contributors
print()
diff --git a/tool-testsuite/pom.xml b/tool-testsuite/pom.xml
index 5ad476e..fde8434 100644
--- a/tool-testsuite/pom.xml
+++ b/tool-testsuite/pom.xml
@@ -10,7 +10,7 @@
<parent>
<groupId>org.antlr</groupId>
<artifactId>antlr4-master</artifactId>
- <version>4.7.1</version>
+ <version>4.7.2</version>
</parent>
<artifactId>antlr4-tool-testsuite</artifactId>
<name>ANTLR 4 Tool Tests</name>
@@ -26,7 +26,7 @@
<dependency>
<groupId>org.antlr</groupId>
<artifactId>ST4</artifactId>
- <version>4.0.8</version>
+ <version>4.1</version>
<scope>test</scope>
</dependency>
<dependency>
diff --git a/tool-testsuite/test/org/antlr/v4/test/tool/TestCodeGeneration.java b/tool-testsuite/test/org/antlr/v4/test/tool/TestCodeGeneration.java
index 8fb6fb7..6fa9674 100644
--- a/tool-testsuite/test/org/antlr/v4/test/tool/TestCodeGeneration.java
+++ b/tool-testsuite/test/org/antlr/v4/test/tool/TestCodeGeneration.java
@@ -31,6 +31,7 @@ import java.util.ArrayList;
import java.util.List;
import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNotEquals;
public class TestCodeGeneration extends BaseJavaToolTest {
@Before
@@ -52,6 +53,24 @@ public class TestCodeGeneration extends BaseJavaToolTest {
}
}
+ @Test public void AssignTokenNamesToStringLiteralsInGeneratedParserRuleContexts() throws Exception {
+ String g =
+ "grammar T;\n" +
+ "root: 't1';\n" +
+ "Token: 't1';";
+ List<String> evals = getEvalInfoForString(g, "() { return getToken(");
+ assertNotEquals(0, evals.size());
+ }
+
+ @Test public void AssignTokenNamesToStringLiteralArraysInGeneratedParserRuleContexts() throws Exception {
+ String g =
+ "grammar T;\n" +
+ "root: 't1' 't1';\n" +
+ "Token: 't1';";
+ List<String> evals = getEvalInfoForString(g, "() { return getTokens(");
+ assertNotEquals(0, evals.size());
+ }
+
/** Add tags around each attribute/template/value write */
public static class DebugInterpreter extends Interpreter {
List<String> evals = new ArrayList<String>();
diff --git a/tool/pom.xml b/tool/pom.xml
index 72f632d..82c7fb8 100644
--- a/tool/pom.xml
+++ b/tool/pom.xml
@@ -9,7 +9,7 @@
<parent>
<groupId>org.antlr</groupId>
<artifactId>antlr4-master</artifactId>
- <version>4.7.1</version>
+ <version>4.7.2</version>
</parent>
<artifactId>antlr4</artifactId>
<name>ANTLR 4 Tool</name>
@@ -29,7 +29,7 @@
<dependency>
<groupId>org.antlr</groupId>
<artifactId>ST4</artifactId>
- <version>4.0.8</version>
+ <version>4.1</version>
</dependency>
<dependency>
<groupId>org.abego.treelayout</groupId>
@@ -44,7 +44,7 @@
<dependency>
<groupId>com.ibm.icu</groupId>
<artifactId>icu4j</artifactId>
- <version>58.2</version>
+ <version>61.1</version>
</dependency>
</dependencies>
<build>
diff --git a/tool/resources/org/antlr/v4/tool/templates/codegen/Cpp/Cpp.stg b/tool/resources/org/antlr/v4/tool/templates/codegen/Cpp/Cpp.stg
index 6342b66..4c367dd 100644
--- a/tool/resources/org/antlr/v4/tool/templates/codegen/Cpp/Cpp.stg
+++ b/tool/resources/org/antlr/v4/tool/templates/codegen/Cpp/Cpp.stg
@@ -510,6 +510,7 @@ LeftRecursiveRuleFunction(currentRule, args, code, locals, ruleCtx, altLabelCtxs
size_t parentState = getState();
<parser.name>::<currentRule.ctxType> *_localctx = _tracker.createInstance\<<currentRule.ctxType>\>(_ctx, parentState<currentRule.args: {a | , <a.name>}>);
<parser.name>::<currentRule.ctxType> *previousContext = _localctx;
+ (void)previousContext; // Silence compiler, in case the context is not used by generated code.
size_t startState = <currentRule.startState>;
enterRecursionRule(_localctx, <currentRule.startState>, <parser.name>::Rule<currentRule.name; format = "cap">, precedence);
@@ -542,7 +543,7 @@ public:
<if (ctorAttrs)><struct.name>(antlr4::ParserRuleContext *parent, size_t invokingState);<endif>
<struct.name>(antlr4::ParserRuleContext *parent, size_t invokingState<ctorAttrs: {a | , <a>}>);
<if (struct.provideCopyFrom)> <! don't need copy unless we have subclasses !>
- <struct.name>() : antlr4::ParserRuleContext() { }
+ <struct.name>() = default;
void copyFrom(<struct.name> *context);
using antlr4::ParserRuleContext::copyFrom;
<endif>
diff --git a/tool/resources/org/antlr/v4/tool/templates/codegen/Java/Java.stg b/tool/resources/org/antlr/v4/tool/templates/codegen/Java/Java.stg
index 492c566..1683a18 100644
--- a/tool/resources/org/antlr/v4/tool/templates/codegen/Java/Java.stg
+++ b/tool/resources/org/antlr/v4/tool/templates/codegen/Java/Java.stg
@@ -236,9 +236,12 @@ public class <parser.name> extends <superClass; null="Parser"> {
<endif>
public static final int
<parser.rules:{r | RULE_<r.name> = <r.index>}; separator=", ", wrap, anchor>;
- public static final String[] ruleNames = {
- <parser.ruleNames:{r | "<r>"}; separator=", ", wrap, anchor>
- };
+ private static String[] makeRuleNames() {
+ return new String[] {
+ <parser.ruleNames:{r | "<r>"}; separator=", ", wrap, anchor>
+ };
+ }
+ public static final String[] ruleNames = makeRuleNames();
<vocabulary(parser.literalNames, parser.symbolicNames)>
@@ -275,12 +278,18 @@ case <f.ruleIndex>:
>>
vocabulary(literalNames, symbolicNames) ::= <<
-private static final String[] _LITERAL_NAMES = {
- <literalNames:{t | <t>}; null="null", separator=", ", wrap, anchor>
-};
-private static final String[] _SYMBOLIC_NAMES = {
- <symbolicNames:{t | <t>}; null="null", separator=", ", wrap, anchor>
-};
+private static String[] makeLiteralNames() {
+ return new String[] {
+ <literalNames:{t | <t>}; null="null", separator=", ", wrap, anchor>
+ };
+}
+private static final String[] _LITERAL_NAMES = makeLiteralNames();
+private static String[] makeSymbolicNames() {
+ return new String[] {
+ <symbolicNames:{t | <t>}; null="null", separator=", ", wrap, anchor>
+ };
+}
+private static final String[] _SYMBOLIC_NAMES = makeSymbolicNames();
public static final Vocabulary VOCABULARY = new VocabularyImpl(_LITERAL_NAMES, _SYMBOLIC_NAMES);
/**
@@ -914,9 +923,12 @@ public class <lexer.name> extends <superClass; null="Lexer"> {
<lexer.modes:{m| "<m>"}; separator=", ", wrap, anchor>
};
- public static final String[] ruleNames = {
- <lexer.ruleNames:{r | "<r>"}; separator=", ", wrap, anchor>
- };
+ private static String[] makeRuleNames() {
+ return new String[] {
+ <lexer.ruleNames:{r | "<r>"}; separator=", ", wrap, anchor>
+ };
+ }
+ public static final String[] ruleNames = makeRuleNames();
<vocabulary(lexer.literalNames, lexer.symbolicNames)>
diff --git a/tool/resources/org/antlr/v4/tool/templates/codegen/Python2/Python2.stg b/tool/resources/org/antlr/v4/tool/templates/codegen/Python2/Python2.stg
index 570f165..dcaa446 100644
--- a/tool/resources/org/antlr/v4/tool/templates/codegen/Python2/Python2.stg
+++ b/tool/resources/org/antlr/v4/tool/templates/codegen/Python2/Python2.stg
@@ -749,6 +749,10 @@ import sys
>>
Lexer(lexer, atn, actionFuncs, sempredFuncs, superClass) ::= <<
+<if(superClass)>
+from .<superClass> import <superClass>
+
+<endif>
<atn>
diff --git a/tool/resources/org/antlr/v4/tool/templates/codegen/Python3/Python3.stg b/tool/resources/org/antlr/v4/tool/templates/codegen/Python3/Python3.stg
index 34e525b..8e03da7 100644
--- a/tool/resources/org/antlr/v4/tool/templates/codegen/Python3/Python3.stg
+++ b/tool/resources/org/antlr/v4/tool/templates/codegen/Python3/Python3.stg
@@ -119,7 +119,10 @@ Parser(parser, funcs, atn, sempredFuncs, superClass) ::= <<
Parser_(parser, funcs, atn, sempredFuncs, ctor, superClass) ::= <<
<if(superClass)>
-from .<superClass> import <superClass>
+if __name__ is not None and "." in __name__:
+ from .<superClass> import <superClass>
+else:
+ from <superClass> import <superClass>
<endif>
<atn>
@@ -756,7 +759,13 @@ import sys
>>
Lexer(lexer, atn, actionFuncs, sempredFuncs, superClass) ::= <<
+<if(superClass)>
+if __name__ is not None and "." in __name__:
+ from .<superClass> import <superClass>
+else:
+ from <superClass> import <superClass>
+<endif>
<atn>
class <lexer.name>(<if(superClass)><superClass><else>Lexer<endif>):
diff --git a/tool/resources/org/antlr/v4/tool/templates/codegen/Swift/Swift.stg b/tool/resources/org/antlr/v4/tool/templates/codegen/Swift/Swift.stg
index 088cf0c..7d74dc5 100755
--- a/tool/resources/org/antlr/v4/tool/templates/codegen/Swift/Swift.stg
+++ b/tool/resources/org/antlr/v4/tool/templates/codegen/Swift/Swift.stg
@@ -354,7 +354,7 @@ func getVocabulary() -> Vocabulary {
override <accessLevelNotOpen(parser)>
init(_ input:TokenStream) throws {
- RuntimeMetaData.checkVersion("4.7.1", RuntimeMetaData.VERSION)
+ RuntimeMetaData.checkVersion("4.7.2", RuntimeMetaData.VERSION)
try super.init(input)
_interp = ParserATNSimulator(self,<p.name>._ATN,<p.name>._decisionToDFA, <parser.name>._sharedContextCache)
}
diff --git a/tool/src/org/antlr/v4/Tool.java b/tool/src/org/antlr/v4/Tool.java
index 86c49e1..d2c95f5 100644
--- a/tool/src/org/antlr/v4/Tool.java
+++ b/tool/src/org/antlr/v4/Tool.java
@@ -397,7 +397,7 @@ public class Tool {
if ( generate_ATN_dot ) generateATNs(g);
- if ( g.tool.getNumErrors()==0 ) generateInterpreterData(g);
+ if (gencode && g.tool.getNumErrors()==0 ) generateInterpreterData(g);
// PERFORM GRAMMAR ANALYSIS ON ATN: BUILD DECISION DFAs
AnalysisPipeline anal = new AnalysisPipeline(g);
@@ -838,7 +838,7 @@ public class Tool {
// or just or the relative path recorded for the parent grammar. This means
// that when we write the tokens files, or the .java files for imported grammars
// taht we will write them in the correct place.
- if (fileNameWithPath.lastIndexOf(File.separatorChar) == -1) {
+ if ((fileNameWithPath == null) || (fileNameWithPath.lastIndexOf(File.separatorChar) == -1)) {
// No path is included in the file name, so make the file
// directory the same as the parent grammar (which might sitll be just ""
// but when it is not, we will write the file in the correct place.
diff --git a/tool/src/org/antlr/v4/codegen/Target.java b/tool/src/org/antlr/v4/codegen/Target.java
index 025b42a..3634172 100644
--- a/tool/src/org/antlr/v4/codegen/Target.java
+++ b/tool/src/org/antlr/v4/codegen/Target.java
@@ -26,6 +26,8 @@ import org.stringtemplate.v4.STGroupFile;
import org.stringtemplate.v4.StringRenderer;
import org.stringtemplate.v4.misc.STMessage;
+import java.net.URL;
+
/** */
public abstract class Target {
/** For pure strings of Java 16-bit Unicode char, how can we display
diff --git a/tool/src/org/antlr/v4/codegen/model/ElementFrequenciesVisitor.java b/tool/src/org/antlr/v4/codegen/model/ElementFrequenciesVisitor.java
index ccd5c80..ef1aae2 100644
--- a/tool/src/org/antlr/v4/codegen/model/ElementFrequenciesVisitor.java
+++ b/tool/src/org/antlr/v4/codegen/model/ElementFrequenciesVisitor.java
@@ -149,6 +149,16 @@ public class ElementFrequenciesVisitor extends GrammarTreeVisitor {
minFrequencies.peek().add(ref.getText());
}
+ @Override
+ public void stringRef(TerminalAST ref) {
+ String tokenName = ref.g.getTokenName(ref.getText());
+
+ if (tokenName != null && !tokenName.startsWith("T__")) {
+ frequencies.peek().add(tokenName);
+ minFrequencies.peek().add(tokenName);
+ }
+ }
+
/*
* Parser rules
*/
diff --git a/tool/src/org/antlr/v4/codegen/model/RuleFunction.java b/tool/src/org/antlr/v4/codegen/model/RuleFunction.java
index d8b69b5..d6b745f 100644
--- a/tool/src/org/antlr/v4/codegen/model/RuleFunction.java
+++ b/tool/src/org/antlr/v4/codegen/model/RuleFunction.java
@@ -7,6 +7,7 @@
package org.antlr.v4.codegen.model;
import org.antlr.runtime.RecognitionException;
+import org.antlr.runtime.tree.CommonTree;
import org.antlr.runtime.tree.CommonTreeNodeStream;
import org.antlr.v4.codegen.OutputModelFactory;
import org.antlr.v4.codegen.model.decl.AltLabelStructDecl;
@@ -32,6 +33,7 @@ import org.antlr.v4.tool.Rule;
import org.antlr.v4.tool.ast.ActionAST;
import org.antlr.v4.tool.ast.AltAST;
import org.antlr.v4.tool.ast.GrammarAST;
+import org.antlr.v4.tool.ast.PredAST;
import java.util.ArrayList;
import java.util.Collection;
@@ -43,6 +45,7 @@ import java.util.Map;
import java.util.Set;
import static org.antlr.v4.parse.ANTLRParser.RULE_REF;
+import static org.antlr.v4.parse.ANTLRParser.STRING_LITERAL;
import static org.antlr.v4.parse.ANTLRParser.TOKEN_REF;
/** */
@@ -166,7 +169,7 @@ public class RuleFunction extends OutputModelObject {
}
/** for all alts, find which ref X or r needs List
- Must see across alts. If any alt needs X or r as list, then
+ Must see across alts. If any alt needs X or r as list, then
define as list.
*/
public Set<Decl> getDeclsForAllElements(List<AltAST> altASTs) {
@@ -174,21 +177,24 @@ public class RuleFunction extends OutputModelObject {
Set<String> nonOptional = new HashSet<String>();
List<GrammarAST> allRefs = new ArrayList<GrammarAST>();
boolean firstAlt = true;
+ IntervalSet reftypes = new IntervalSet(RULE_REF, TOKEN_REF, STRING_LITERAL);
for (AltAST ast : altASTs) {
- IntervalSet reftypes = new IntervalSet(RULE_REF, TOKEN_REF);
- List<GrammarAST> refs = ast.getNodesWithType(reftypes);
+ List<GrammarAST> refs = getRuleTokens(ast.getNodesWithType(reftypes));
allRefs.addAll(refs);
Pair<FrequencySet<String>, FrequencySet<String>> minAndAltFreq = getElementFrequenciesForAlt(ast);
FrequencySet<String> minFreq = minAndAltFreq.a;
FrequencySet<String> altFreq = minAndAltFreq.b;
for (GrammarAST t : refs) {
- String refLabelName = t.getText();
- if ( altFreq.count(refLabelName)>1 ) {
- needsList.add(refLabelName);
- }
+ String refLabelName = getName(t);
+
+ if (refLabelName != null) {
+ if (altFreq.count(refLabelName) > 1) {
+ needsList.add(refLabelName);
+ }
- if (firstAlt && minFreq.count(refLabelName) != 0) {
- nonOptional.add(refLabelName);
+ if (firstAlt && minFreq.count(refLabelName) != 0) {
+ nonOptional.add(refLabelName);
+ }
}
}
@@ -202,7 +208,12 @@ public class RuleFunction extends OutputModelObject {
}
Set<Decl> decls = new LinkedHashSet<Decl>();
for (GrammarAST t : allRefs) {
- String refLabelName = t.getText();
+ String refLabelName = getName(t);
+
+ if (refLabelName == null) {
+ continue;
+ }
+
List<Decl> d = getDeclForAltElement(t,
refLabelName,
needsList.contains(refLabelName),
@@ -212,6 +223,35 @@ public class RuleFunction extends OutputModelObject {
return decls;
}
+ private List<GrammarAST> getRuleTokens(List<GrammarAST> refs) {
+ List<GrammarAST> result = new ArrayList<>(refs.size());
+ for (GrammarAST ref : refs) {
+ CommonTree r = ref;
+
+ boolean ignore = false;
+ while (r != null) {
+ // Ignore string literals in predicates
+ if (r instanceof PredAST) {
+ ignore = true;
+ break;
+ }
+ r = r.parent;
+ }
+
+ if (!ignore) {
+ result.add(ref);
+ }
+ }
+
+ return result;
+ }
+
+ private String getName(GrammarAST token) {
+ String tokenText = token.getText();
+ String tokenName = token.getType() != STRING_LITERAL ? tokenText : token.g.getTokenName(tokenText);
+ return tokenName == null || tokenName.startsWith("T__") ? null : tokenName; // Do not include tokens with auto generated names
+ }
+
/** Given list of X and r refs in alt, compute how many of each there are */
protected Pair<FrequencySet<String>, FrequencySet<String>> getElementFrequenciesForAlt(AltAST ast) {
try {
diff --git a/tool/src/org/antlr/v4/codegen/target/CSharpTarget.java b/tool/src/org/antlr/v4/codegen/target/CSharpTarget.java
index e7a2322..82f65c6 100644
--- a/tool/src/org/antlr/v4/codegen/target/CSharpTarget.java
+++ b/tool/src/org/antlr/v4/codegen/target/CSharpTarget.java
@@ -28,7 +28,7 @@ public class CSharpTarget extends Target {
@Override
public String getVersion() {
- return "4.7.1";
+ return "4.7.2";
}
@Override
diff --git a/tool/src/org/antlr/v4/codegen/target/CppTarget.java b/tool/src/org/antlr/v4/codegen/target/CppTarget.java
index 6ef32c2..839a362 100644
--- a/tool/src/org/antlr/v4/codegen/target/CppTarget.java
+++ b/tool/src/org/antlr/v4/codegen/target/CppTarget.java
@@ -50,7 +50,7 @@ public class CppTarget extends Target {
}
public String getVersion() {
- return "4.7.1";
+ return "4.7.2";
}
public boolean needsHeader() { return true; }
diff --git a/tool/src/org/antlr/v4/codegen/target/GoTarget.java b/tool/src/org/antlr/v4/codegen/target/GoTarget.java
index be4e3cc..89a40de 100644
--- a/tool/src/org/antlr/v4/codegen/target/GoTarget.java
+++ b/tool/src/org/antlr/v4/codegen/target/GoTarget.java
@@ -71,7 +71,7 @@ public class GoTarget extends Target {
@Override
public String getVersion() {
- return "4.7.1";
+ return "4.7.2";
}
public Set<String> getBadWords() {
diff --git a/tool/src/org/antlr/v4/codegen/target/JavaScriptTarget.java b/tool/src/org/antlr/v4/codegen/target/JavaScriptTarget.java
index adbfc7c..75886fd 100644
--- a/tool/src/org/antlr/v4/codegen/target/JavaScriptTarget.java
+++ b/tool/src/org/antlr/v4/codegen/target/JavaScriptTarget.java
@@ -51,7 +51,7 @@ public class JavaScriptTarget extends Target {
@Override
public String getVersion() {
- return "4.7.1";
+ return "4.7.2";
}
public Set<String> getBadWords() {
diff --git a/tool/src/org/antlr/v4/codegen/target/Python2Target.java b/tool/src/org/antlr/v4/codegen/target/Python2Target.java
index 3fc6a35..1beaf8c 100644
--- a/tool/src/org/antlr/v4/codegen/target/Python2Target.java
+++ b/tool/src/org/antlr/v4/codegen/target/Python2Target.java
@@ -24,27 +24,28 @@ import java.util.Set;
*/
public class Python2Target extends Target {
protected static final String[] python2Keywords = {
- "abs", "all", "any", "apply", "as",
- "bin", "bool", "buffer", "bytearray",
- "callable", "chr", "classmethod", "coerce", "compile", "complex",
- "del", "delattr", "dict", "dir", "divmod",
- "enumerate", "eval", "execfile",
- "file", "filter", "float", "format", "frozenset",
- "getattr", "globals",
+ "abs", "all", "and", "any", "apply", "as", "assert",
+ "bin", "bool", "break", "buffer", "bytearray",
+ "callable", "chr", "class", "classmethod", "coerce", "compile", "complex", "continue",
+ "def", "del", "delattr", "dict", "dir", "divmod",
+ "elif", "else", "enumerate", "eval", "except", "exec", "execfile",
+ "file", "filter", "finally", "float", "for", "format", "from", "frozenset",
+ "getattr", "global", "globals",
"hasattr", "hash", "help", "hex",
- "id", "input", "int", "intern", "isinstance", "issubclass", "iter",
- "len", "list", "locals",
- "map", "max", "min", "next",
+ "id", "if", "import", "in", "input", "int", "intern", "is", "isinstance", "issubclass", "iter",
+ "lambda", "len", "list", "locals",
+ "map", "max", "min", "next", "not",
"memoryview",
- "object", "oct", "open", "ord",
- "pow", "print", "property",
- "range", "raw_input", "reduce", "reload", "repr", "return", "reversed", "round",
+ "object", "oct", "open", "or", "ord",
+ "pass", "pow", "print", "property",
+ "raise", "range", "raw_input", "reduce", "reload", "repr", "return", "reversed", "round",
"set", "setattr", "slice", "sorted", "staticmethod", "str", "sum", "super",
- "tuple", "type",
+ "try", "tuple", "type",
"unichr", "unicode",
"vars",
- "with",
+ "while", "with",
"xrange",
+ "yield",
"zip",
"__import__",
"True", "False", "None"
@@ -94,7 +95,7 @@ public class Python2Target extends Target {
@Override
public String getVersion() {
- return "4.7.1";
+ return "4.7.2";
}
public Set<String> getBadWords() {
diff --git a/tool/src/org/antlr/v4/codegen/target/Python3Target.java b/tool/src/org/antlr/v4/codegen/target/Python3Target.java
index 388269a..753f67c 100644
--- a/tool/src/org/antlr/v4/codegen/target/Python3Target.java
+++ b/tool/src/org/antlr/v4/codegen/target/Python3Target.java
@@ -24,26 +24,27 @@ import java.util.Set;
*/
public class Python3Target extends Target {
protected static final String[] python3Keywords = {
- "abs", "all", "any", "apply", "as",
- "bin", "bool", "buffer", "bytearray",
- "callable", "chr", "classmethod", "coerce", "compile", "complex",
- "del", "delattr", "dict", "dir", "divmod",
- "enumerate", "eval", "execfile",
- "file", "filter", "float", "format", "frozenset",
- "getattr", "globals",
+ "abs", "all", "and", "any", "apply", "as", "assert",
+ "bin", "bool", "break", "buffer", "bytearray",
+ "callable", "chr", "class", "classmethod", "coerce", "compile", "complex", "continue",
+ "def", "del", "delattr", "dict", "dir", "divmod",
+ "elif", "else", "enumerate", "eval", "execfile", "except",
+ "file", "filter", "finally", "float", "for", "format", "from", "frozenset",
+ "getattr", "global", "globals",
"hasattr", "hash", "help", "hex",
- "id", "input", "int", "intern", "isinstance", "issubclass", "iter",
- "len", "list", "locals",
- "map", "max", "min", "next",
- "memoryview",
- "object", "oct", "open", "ord",
- "pow", "print", "property",
- "range", "raw_input", "reduce", "reload", "repr", "return", "reversed", "round",
+ "id", "if", "import", "in", "input", "int", "intern", "is", "isinstance", "issubclass", "iter",
+ "lambda", "len", "list", "locals",
+ "map", "max", "min", "memoryview",
+ "next", "nonlocal", "not",
+ "object", "oct", "open", "or", "ord",
+ "pass", "pow", "print", "property",
+ "raise", "range", "raw_input", "reduce", "reload", "repr", "return", "reversed", "round",
"set", "setattr", "slice", "sorted", "staticmethod", "str", "sum", "super",
- "tuple", "type",
+ "try", "tuple", "type",
"unichr", "unicode",
"vars",
- "with",
+ "with", "while",
+ "yield",
"zip",
"__import__",
"True", "False", "None"
@@ -96,7 +97,7 @@ public class Python3Target extends Target {
@Override
public String getVersion() {
- return "4.7.1";
+ return "4.7.2";
}
/** Avoid grammar symbols in this set to prevent conflicts in gen'd code. */
diff --git a/tool/src/org/antlr/v4/codegen/target/SwiftTarget.java b/tool/src/org/antlr/v4/codegen/target/SwiftTarget.java
index 5e28506..b192344 100644
--- a/tool/src/org/antlr/v4/codegen/target/SwiftTarget.java
+++ b/tool/src/org/antlr/v4/codegen/target/SwiftTarget.java
@@ -87,7 +87,7 @@ public class SwiftTarget extends Target {
@Override
public String getVersion() {
- return "4.7.1"; // Java and tool versions move in lock step
+ return "4.7.2"; // Java and tool versions move in lock step
}
public Set<String> getBadWords() {
diff --git a/tool/src/org/antlr/v4/tool/ErrorManager.java b/tool/src/org/antlr/v4/tool/ErrorManager.java
index 0230d2b..7421266 100644
--- a/tool/src/org/antlr/v4/tool/ErrorManager.java
+++ b/tool/src/org/antlr/v4/tool/ErrorManager.java
@@ -231,8 +231,7 @@ public class ErrorManager {
setFormat("antlr"); // recurse on this rule, trying the default message format
return;
}
-
- format = new STGroupFile(fileName, "UTF-8");
+ format = new STGroupFile(url, "UTF-8", '<', '>');
format.load();
if ( !initSTListener.errors.isEmpty() ) {
diff --git a/tool/src/org/antlr/v4/tool/Grammar.java b/tool/src/org/antlr/v4/tool/Grammar.java
index b86d622..fc98fcf 100644
--- a/tool/src/org/antlr/v4/tool/Grammar.java
+++ b/tool/src/org/antlr/v4/tool/Grammar.java
@@ -631,6 +631,16 @@ public class Grammar implements AttributeResolver {
return i;
}
+ public String getTokenName(String literal) {
+ Grammar grammar = this;
+ while (grammar != null) {
+ if (grammar.stringLiteralToTypeMap.containsKey(literal))
+ return grammar.getTokenName(grammar.stringLiteralToTypeMap.get(literal));
+ grammar = grammar.parent;
+ }
+ return null;
+ }
+
/** Given a token type, get a meaningful name for it such as the ID
* or string literal. If this is a lexer and the ttype is in the
* char vocabulary, compute an ANTLR-valid (possibly escaped) char literal.