diff options
Diffstat (limited to 'runtime/Java/src/org')
175 files changed, 2132 insertions, 453 deletions
diff --git a/runtime/Java/src/org/antlr/v4/runtime/ANTLRErrorListener.java b/runtime/Java/src/org/antlr/v4/runtime/ANTLRErrorListener.java index 013f1cf..d7c8bb6 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/ANTLRErrorListener.java +++ b/runtime/Java/src/org/antlr/v4/runtime/ANTLRErrorListener.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/runtime/Java/src/org/antlr/v4/runtime/ANTLRErrorStrategy.java b/runtime/Java/src/org/antlr/v4/runtime/ANTLRErrorStrategy.java index 90c5528..7604791 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/ANTLRErrorStrategy.java +++ b/runtime/Java/src/org/antlr/v4/runtime/ANTLRErrorStrategy.java @@ -1,11 +1,13 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ package org.antlr.v4.runtime; +import org.antlr.v4.runtime.tree.ErrorNode; + /** * The interface for defining strategies to deal with syntax errors encountered * during a parse by ANTLR-generated parsers. We distinguish between three @@ -89,8 +91,9 @@ public interface ANTLRErrorStrategy { * Tests whether or not {@code recognizer} is in the process of recovering * from an error. In error recovery mode, {@link Parser#consume} adds * symbols to the parse tree by calling - * {@link ParserRuleContext#addErrorNode(Token)} instead of - * {@link ParserRuleContext#addChild(Token)}. + * {@link Parser#createErrorNode(ParserRuleContext, Token)} then + * {@link ParserRuleContext#addErrorNode(ErrorNode)} instead of + * {@link Parser#createTerminalNode(ParserRuleContext, Token)}. * * @param recognizer the parser instance * @return {@code true} if the parser is currently recovering from a parse diff --git a/runtime/Java/src/org/antlr/v4/runtime/ANTLRFileStream.java b/runtime/Java/src/org/antlr/v4/runtime/ANTLRFileStream.java index e31ff32..ede83eb 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/ANTLRFileStream.java +++ b/runtime/Java/src/org/antlr/v4/runtime/ANTLRFileStream.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ @@ -12,6 +12,8 @@ import java.io.IOException; /** * This is an {@link ANTLRInputStream} that is loaded from a file all at once * when you construct the object. + * + * @deprecated as of 4.7 Please use {@link CharStreams} interface. */ public class ANTLRFileStream extends ANTLRInputStream { protected String fileName; diff --git a/runtime/Java/src/org/antlr/v4/runtime/ANTLRInputStream.java b/runtime/Java/src/org/antlr/v4/runtime/ANTLRInputStream.java index 7ee3cca..f71a62d 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/ANTLRInputStream.java +++ b/runtime/Java/src/org/antlr/v4/runtime/ANTLRInputStream.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ @@ -19,6 +19,8 @@ import java.util.Arrays; * {@code char[]} to use. * * <p>If you need encoding, pass in stream/reader with correct encoding.</p> + * + * @deprecated as of 4.7 Please use {@link CharStreams} interface. */ public class ANTLRInputStream implements CharStream { public static final int READ_BUFFER_SIZE = 1024; diff --git a/runtime/Java/src/org/antlr/v4/runtime/BailErrorStrategy.java b/runtime/Java/src/org/antlr/v4/runtime/BailErrorStrategy.java index a7f08c9..d6830bb 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/BailErrorStrategy.java +++ b/runtime/Java/src/org/antlr/v4/runtime/BailErrorStrategy.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/runtime/Java/src/org/antlr/v4/runtime/BaseErrorListener.java b/runtime/Java/src/org/antlr/v4/runtime/BaseErrorListener.java index 7f6ea8e..8453422 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/BaseErrorListener.java +++ b/runtime/Java/src/org/antlr/v4/runtime/BaseErrorListener.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/runtime/Java/src/org/antlr/v4/runtime/BufferedTokenStream.java b/runtime/Java/src/org/antlr/v4/runtime/BufferedTokenStream.java index 8ca3928..e1dc4b1 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/BufferedTokenStream.java +++ b/runtime/Java/src/org/antlr/v4/runtime/BufferedTokenStream.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ @@ -89,6 +89,14 @@ public class BufferedTokenStream implements TokenStream { // no resources to release } + /** + * This method resets the token stream back to the first token in the + * buffer. It is equivalent to calling {@link #seek}{@code (0)}. + * + * @see #setTokenSource(TokenSource) + * @deprecated Use {@code seek(0)} instead. + */ + @Deprecated public void reset() { seek(0); } @@ -252,6 +260,7 @@ public class BufferedTokenStream implements TokenStream { this.tokenSource = tokenSource; tokens.clear(); p = -1; + fetchedEOF = false; } public List<Token> getTokens() { return tokens; } @@ -372,7 +381,7 @@ public class BufferedTokenStream implements TokenStream { /** Collect all hidden tokens (any off-default channel) to the right of * the current token up until we see a token on DEFAULT_TOKEN_CHANNEL - * of EOF. + * or EOF. */ public List<Token> getHiddenTokensToRight(int tokenIndex) { return getHiddenTokensToRight(tokenIndex, -1); diff --git a/runtime/Java/src/org/antlr/v4/runtime/CharStream.java b/runtime/Java/src/org/antlr/v4/runtime/CharStream.java index fcd6bfb..57dbfa9 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/CharStream.java +++ b/runtime/Java/src/org/antlr/v4/runtime/CharStream.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/runtime/Java/src/org/antlr/v4/runtime/CharStreams.java b/runtime/Java/src/org/antlr/v4/runtime/CharStreams.java new file mode 100644 index 0000000..73fb875 --- /dev/null +++ b/runtime/Java/src/org/antlr/v4/runtime/CharStreams.java @@ -0,0 +1,306 @@ +/* + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. + * Use of this file is governed by the BSD 3-clause license that + * can be found in the LICENSE.txt file in the project root. + */ + +package org.antlr.v4.runtime; + +import java.io.IOException; +import java.io.InputStream; +import java.io.Reader; +import java.nio.ByteBuffer; +import java.nio.CharBuffer; +import java.nio.channels.Channels; +import java.nio.channels.ReadableByteChannel; +import java.nio.charset.Charset; +import java.nio.charset.CharsetDecoder; +import java.nio.charset.CoderResult; +import java.nio.charset.CodingErrorAction; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; + +/** This class represents the primary interface for creating {@link CharStream}s + * from a variety of sources as of 4.7. The motivation was to support + * Unicode code points > U+FFFF. {@link ANTLRInputStream} and + * {@link ANTLRFileStream} are now deprecated in favor of the streams created + * by this interface. + * + * DEPRECATED: {@code new ANTLRFileStream("myinputfile")} + * NEW: {@code CharStreams.fromFileName("myinputfile")} + * + * WARNING: If you use both the deprecated and the new streams, you will see + * a nontrivial performance degradation. This speed hit is because the + * {@link Lexer}'s internal code goes from a monomorphic to megamorphic + * dynamic dispatch to get characters from the input stream. Java's + * on-the-fly compiler (JIT) is unable to perform the same optimizations + * so stick with either the old or the new streams, if performance is + * a primary concern. See the extreme debugging and spelunking + * needed to identify this issue in our timing rig: + * + * https://github.com/antlr/antlr4/pull/1781 + * + * The ANTLR character streams still buffer all the input when you create + * the stream, as they have done for ~20 years. If you need unbuffered + * access, please note that it becomes challenging to create + * parse trees. The parse tree has to point to tokens which will either + * point into a stale location in an unbuffered stream or you have to copy + * the characters out of the buffer into the token. That defeats the purpose + * of unbuffered input. Per the ANTLR book, unbuffered streams are primarily + * useful for processing infinite streams *during the parse.* + * + * The new streams also use 8-bit buffers when possible so this new + * interface supports character streams that use half as much memory + * as the old {@link ANTLRFileStream}, which assumed 16-bit characters. + * + * A big shout out to Ben Hamilton (github bhamiltoncx) for his superhuman + * efforts across all targets to get true Unicode 3.1 support for U+10FFFF. + * + * @since 4.7 + */ +public final class CharStreams { + private static final int DEFAULT_BUFFER_SIZE = 4096; + + // Utility class; do not construct. + private CharStreams() { } + + /** + * Creates a {@link CharStream} given a path to a UTF-8 + * encoded file on disk. + * + * Reads the entire contents of the file into the result before returning. + */ + public static CharStream fromPath(Path path) throws IOException { + return fromPath(path, StandardCharsets.UTF_8); + } + + /** + * Creates a {@link CharStream} given a path to a file on disk and the + * charset of the bytes contained in the file. + * + * Reads the entire contents of the file into the result before returning. + */ + public static CharStream fromPath(Path path, Charset charset) throws IOException { + long size = Files.size(path); + try (ReadableByteChannel channel = Files.newByteChannel(path)) { + return fromChannel( + channel, + charset, + DEFAULT_BUFFER_SIZE, + CodingErrorAction.REPLACE, + path.toString(), + size); + } + } + + /** + * Creates a {@link CharStream} given a string containing a + * path to a UTF-8 file on disk. + * + * Reads the entire contents of the file into the result before returning. + */ + public static CharStream fromFileName(String fileName) throws IOException { + return fromPath(Paths.get(fileName), StandardCharsets.UTF_8); + } + + /** + * Creates a {@link CharStream} given a string containing a + * path to a file on disk and the charset of the bytes + * contained in the file. + * + * Reads the entire contents of the file into the result before returning. + */ + public static CharStream fromFileName(String fileName, Charset charset) throws IOException { + return fromPath(Paths.get(fileName), charset); + } + + + /** + * Creates a {@link CharStream} given an opened {@link InputStream} + * containing UTF-8 bytes. + * + * Reads the entire contents of the {@code InputStream} into + * the result before returning, then closes the {@code InputStream}. + */ + public static CharStream fromStream(InputStream is) throws IOException { + return fromStream(is, StandardCharsets.UTF_8); + } + + /** + * Creates a {@link CharStream} given an opened {@link InputStream} and the + * charset of the bytes contained in the stream. + * + * Reads the entire contents of the {@code InputStream} into + * the result before returning, then closes the {@code InputStream}. + */ + public static CharStream fromStream(InputStream is, Charset charset) throws IOException { + return fromStream(is, charset, -1); + } + + public static CharStream fromStream(InputStream is, Charset charset, long inputSize) throws IOException { + try (ReadableByteChannel channel = Channels.newChannel(is)) { + return fromChannel( + channel, + charset, + DEFAULT_BUFFER_SIZE, + CodingErrorAction.REPLACE, + IntStream.UNKNOWN_SOURCE_NAME, + inputSize); + } + } + + /** + * Creates a {@link CharStream} given an opened {@link ReadableByteChannel} + * containing UTF-8 bytes. + * + * Reads the entire contents of the {@code channel} into + * the result before returning, then closes the {@code channel}. + */ + public static CharStream fromChannel(ReadableByteChannel channel) throws IOException { + return fromChannel(channel, StandardCharsets.UTF_8); + } + + /** + * Creates a {@link CharStream} given an opened {@link ReadableByteChannel} and the + * charset of the bytes contained in the channel. + * + * Reads the entire contents of the {@code channel} into + * the result before returning, then closes the {@code channel}. + */ + public static CharStream fromChannel(ReadableByteChannel channel, Charset charset) throws IOException { + return fromChannel( + channel, + DEFAULT_BUFFER_SIZE, + CodingErrorAction.REPLACE, + IntStream.UNKNOWN_SOURCE_NAME); + } + + /** + * Creates a {@link CharStream} given a {@link Reader}. Closes + * the reader before returning. + */ + public static CodePointCharStream fromReader(Reader r) throws IOException { + return fromReader(r, IntStream.UNKNOWN_SOURCE_NAME); + } + + /** + * Creates a {@link CharStream} given a {@link Reader} and its + * source name. Closes the reader before returning. + */ + public static CodePointCharStream fromReader(Reader r, String sourceName) throws IOException { + try { + CodePointBuffer.Builder codePointBufferBuilder = CodePointBuffer.builder(DEFAULT_BUFFER_SIZE); + CharBuffer charBuffer = CharBuffer.allocate(DEFAULT_BUFFER_SIZE); + while ((r.read(charBuffer)) != -1) { + charBuffer.flip(); + codePointBufferBuilder.append(charBuffer); + charBuffer.compact(); + } + return CodePointCharStream.fromBuffer(codePointBufferBuilder.build(), sourceName); + } + finally { + r.close(); + } + } + + /** + * Creates a {@link CharStream} given a {@link String}. + */ + public static CodePointCharStream fromString(String s) { + return fromString(s, IntStream.UNKNOWN_SOURCE_NAME); + } + + /** + * Creates a {@link CharStream} given a {@link String} and the {@code sourceName} + * from which it came. + */ + public static CodePointCharStream fromString(String s, String sourceName) { + // Initial guess assumes no code points > U+FFFF: one code + // point for each code unit in the string + CodePointBuffer.Builder codePointBufferBuilder = CodePointBuffer.builder(s.length()); + // TODO: CharBuffer.wrap(String) rightfully returns a read-only buffer + // which doesn't expose its array, so we make a copy. + CharBuffer cb = CharBuffer.allocate(s.length()); + cb.put(s); + cb.flip(); + codePointBufferBuilder.append(cb); + return CodePointCharStream.fromBuffer(codePointBufferBuilder.build(), sourceName); + } + + /** + * Creates a {@link CharStream} given an opened {@link ReadableByteChannel} + * containing UTF-8 bytes. + * + * Reads the entire contents of the {@code channel} into + * the result before returning, then closes the {@code channel}. + */ + public static CodePointCharStream fromChannel( + ReadableByteChannel channel, + int bufferSize, + CodingErrorAction decodingErrorAction, + String sourceName) + throws IOException + { + return fromChannel(channel, StandardCharsets.UTF_8, bufferSize, decodingErrorAction, sourceName, -1); + } + + public static CodePointCharStream fromChannel( + ReadableByteChannel channel, + Charset charset, + int bufferSize, + CodingErrorAction decodingErrorAction, + String sourceName, + long inputSize) + throws IOException + { + try { + ByteBuffer utf8BytesIn = ByteBuffer.allocate(bufferSize); + CharBuffer utf16CodeUnitsOut = CharBuffer.allocate(bufferSize); + if (inputSize == -1) { + inputSize = bufferSize; + } else if (inputSize > Integer.MAX_VALUE) { + // ByteBuffer et al don't support long sizes + throw new IOException(String.format("inputSize %d larger than max %d", inputSize, Integer.MAX_VALUE)); + } + CodePointBuffer.Builder codePointBufferBuilder = CodePointBuffer.builder((int) inputSize); + CharsetDecoder decoder = charset + .newDecoder() + .onMalformedInput(decodingErrorAction) + .onUnmappableCharacter(decodingErrorAction); + + boolean endOfInput = false; + while (!endOfInput) { + int bytesRead = channel.read(utf8BytesIn); + endOfInput = (bytesRead == -1); + utf8BytesIn.flip(); + CoderResult result = decoder.decode( + utf8BytesIn, + utf16CodeUnitsOut, + endOfInput); + if (result.isError() && decodingErrorAction.equals(CodingErrorAction.REPORT)) { + result.throwException(); + } + utf16CodeUnitsOut.flip(); + codePointBufferBuilder.append(utf16CodeUnitsOut); + utf8BytesIn.compact(); + utf16CodeUnitsOut.compact(); + } + // Handle any bytes at the end of the file which need to + // be represented as errors or substitution characters. + CoderResult flushResult = decoder.flush(utf16CodeUnitsOut); + if (flushResult.isError() && decodingErrorAction.equals(CodingErrorAction.REPORT)) { + flushResult.throwException(); + } + utf16CodeUnitsOut.flip(); + codePointBufferBuilder.append(utf16CodeUnitsOut); + + CodePointBuffer codePointBuffer = codePointBufferBuilder.build(); + return CodePointCharStream.fromBuffer(codePointBuffer, sourceName); + } + finally { + channel.close(); + } + } +} diff --git a/runtime/Java/src/org/antlr/v4/runtime/CodePointBuffer.java b/runtime/Java/src/org/antlr/v4/runtime/CodePointBuffer.java new file mode 100644 index 0000000..58035a0 --- /dev/null +++ b/runtime/Java/src/org/antlr/v4/runtime/CodePointBuffer.java @@ -0,0 +1,389 @@ +/* + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. + * Use of this file is governed by the BSD 3-clause license that + * can be found in the LICENSE.txt file in the project root. + */ + +package org.antlr.v4.runtime; + +import java.nio.ByteBuffer; +import java.nio.CharBuffer; +import java.nio.IntBuffer; + +/** + * Wrapper for {@link ByteBuffer} / {@link CharBuffer} / {@link IntBuffer}. + * + * Because Java lacks generics on primitive types, these three types + * do not share an interface, so we have to write one manually. + */ +public class CodePointBuffer { + public enum Type { + BYTE, + CHAR, + INT + } + private final Type type; + private final ByteBuffer byteBuffer; + private final CharBuffer charBuffer; + private final IntBuffer intBuffer; + + private CodePointBuffer(Type type, ByteBuffer byteBuffer, CharBuffer charBuffer, IntBuffer intBuffer) { + this.type = type; + this.byteBuffer = byteBuffer; + this.charBuffer = charBuffer; + this.intBuffer = intBuffer; + } + + public static CodePointBuffer withBytes(ByteBuffer byteBuffer) { + return new CodePointBuffer(Type.BYTE, byteBuffer, null, null); + } + + public static CodePointBuffer withChars(CharBuffer charBuffer) { + return new CodePointBuffer(Type.CHAR, null, charBuffer, null); + } + + public static CodePointBuffer withInts(IntBuffer intBuffer) { + return new CodePointBuffer(Type.INT, null, null, intBuffer); + } + + public int position() { + switch (type) { + case BYTE: + return byteBuffer.position(); + case CHAR: + return charBuffer.position(); + case INT: + return intBuffer.position(); + } + throw new UnsupportedOperationException("Not reached"); + } + + public void position(int newPosition) { + switch (type) { + case BYTE: + byteBuffer.position(newPosition); + break; + case CHAR: + charBuffer.position(newPosition); + break; + case INT: + intBuffer.position(newPosition); + break; + } + } + + public int remaining() { + switch (type) { + case BYTE: + return byteBuffer.remaining(); + case CHAR: + return charBuffer.remaining(); + case INT: + return intBuffer.remaining(); + } + throw new UnsupportedOperationException("Not reached"); + } + + public int get(int offset) { + switch (type) { + case BYTE: + return byteBuffer.get(offset); + case CHAR: + return charBuffer.get(offset); + case INT: + return intBuffer.get(offset); + } + throw new UnsupportedOperationException("Not reached"); + } + + Type getType() { + return type; + } + + int arrayOffset() { + switch (type) { + case BYTE: + return byteBuffer.arrayOffset(); + case CHAR: + return charBuffer.arrayOffset(); + case INT: + return intBuffer.arrayOffset(); + } + throw new UnsupportedOperationException("Not reached"); + } + + byte[] byteArray() { + assert type == Type.BYTE; + return byteBuffer.array(); + } + + char[] charArray() { + assert type == Type.CHAR; + return charBuffer.array(); + } + + int[] intArray() { + assert type == Type.INT; + return intBuffer.array(); + } + + public static Builder builder(int initialBufferSize) { + return new Builder(initialBufferSize); + } + + public static class Builder { + private Type type; + private ByteBuffer byteBuffer; + private CharBuffer charBuffer; + private IntBuffer intBuffer; + private int prevHighSurrogate; + + private Builder(int initialBufferSize) { + type = Type.BYTE; + byteBuffer = ByteBuffer.allocate(initialBufferSize); + charBuffer = null; + intBuffer = null; + prevHighSurrogate = -1; + } + + Type getType() { + return type; + } + + ByteBuffer getByteBuffer() { + return byteBuffer; + } + + CharBuffer getCharBuffer() { + return charBuffer; + } + + IntBuffer getIntBuffer() { + return intBuffer; + } + + public CodePointBuffer build() { + switch (type) { + case BYTE: + byteBuffer.flip(); + break; + case CHAR: + charBuffer.flip(); + break; + case INT: + intBuffer.flip(); + break; + } + return new CodePointBuffer(type, byteBuffer, charBuffer, intBuffer); + } + + private static int roundUpToNextPowerOfTwo(int i) { + int nextPowerOfTwo = 32 - Integer.numberOfLeadingZeros(i - 1); + return (int) Math.pow(2, nextPowerOfTwo); + } + + public void ensureRemaining(int remainingNeeded) { + switch (type) { + case BYTE: + if (byteBuffer.remaining() < remainingNeeded) { + int newCapacity = roundUpToNextPowerOfTwo(byteBuffer.capacity() + remainingNeeded); + ByteBuffer newBuffer = ByteBuffer.allocate(newCapacity); + byteBuffer.flip(); + newBuffer.put(byteBuffer); + byteBuffer = newBuffer; + } + break; + case CHAR: + if (charBuffer.remaining() < remainingNeeded) { + int newCapacity = roundUpToNextPowerOfTwo(charBuffer.capacity() + remainingNeeded); + CharBuffer newBuffer = CharBuffer.allocate(newCapacity); + charBuffer.flip(); + newBuffer.put(charBuffer); + charBuffer = newBuffer; + } + break; + case INT: + if (intBuffer.remaining() < remainingNeeded) { + int newCapacity = roundUpToNextPowerOfTwo(intBuffer.capacity() + remainingNeeded); + IntBuffer newBuffer = IntBuffer.allocate(newCapacity); + intBuffer.flip(); + newBuffer.put(intBuffer); + intBuffer = newBuffer; + } + break; + } + } + + public void append(CharBuffer utf16In) { + ensureRemaining(utf16In.remaining()); + if (utf16In.hasArray()) { + appendArray(utf16In); + } else { + // TODO + throw new UnsupportedOperationException("TODO"); + } + } + + private void appendArray(CharBuffer utf16In) { + assert utf16In.hasArray(); + + switch (type) { + case BYTE: + appendArrayByte(utf16In); + break; + case CHAR: + appendArrayChar(utf16In); + break; + case INT: + appendArrayInt(utf16In); + break; + } + } + + private void appendArrayByte(CharBuffer utf16In) { + assert prevHighSurrogate == -1; + + char[] in = utf16In.array(); + int inOffset = utf16In.arrayOffset() + utf16In.position(); + int inLimit = utf16In.arrayOffset() + utf16In.limit(); + + byte[] outByte = byteBuffer.array(); + int outOffset = byteBuffer.arrayOffset() + byteBuffer.position(); + + while (inOffset < inLimit) { + char c = in[inOffset]; + if (c <= 0xFF) { + outByte[outOffset] = (byte)(c & 0xFF); + } else { + utf16In.position(inOffset - utf16In.arrayOffset()); + byteBuffer.position(outOffset - byteBuffer.arrayOffset()); + if (!Character.isHighSurrogate(c)) { + byteToCharBuffer(utf16In.remaining()); + appendArrayChar(utf16In); + return; + } else { + byteToIntBuffer(utf16In.remaining()); + appendArrayInt(utf16In); + return; + } + } + inOffset++; + outOffset++; + } + + utf16In.position(inOffset - utf16In.arrayOffset()); + byteBuffer.position(outOffset - byteBuffer.arrayOffset()); + } + + private void appendArrayChar(CharBuffer utf16In) { + assert prevHighSurrogate == -1; + + char[] in = utf16In.array(); + int inOffset = utf16In.arrayOffset() + utf16In.position(); + int inLimit = utf16In.arrayOffset() + utf16In.limit(); + + char[] outChar = charBuffer.array(); + int outOffset = charBuffer.arrayOffset() + charBuffer.position(); + + while (inOffset < inLimit) { + char c = in[inOffset]; + if (!Character.isHighSurrogate(c)) { + outChar[outOffset] = c; + } else { + utf16In.position(inOffset - utf16In.arrayOffset()); + charBuffer.position(outOffset - charBuffer.arrayOffset()); + charToIntBuffer(utf16In.remaining()); + appendArrayInt(utf16In); + return; + } + inOffset++; + outOffset++; + } + + utf16In.position(inOffset - utf16In.arrayOffset()); + charBuffer.position(outOffset - charBuffer.arrayOffset()); + } + + private void appendArrayInt(CharBuffer utf16In) { + char[] in = utf16In.array(); + int inOffset = utf16In.arrayOffset() + utf16In.position(); + int inLimit = utf16In.arrayOffset() + utf16In.limit(); + + int[] outInt = intBuffer.array(); + int outOffset = intBuffer.arrayOffset() + intBuffer.position(); + + while (inOffset < inLimit) { + char c = in[inOffset]; + inOffset++; + if (prevHighSurrogate != -1) { + if (Character.isLowSurrogate(c)) { + outInt[outOffset] = Character.toCodePoint((char) prevHighSurrogate, c); + outOffset++; + prevHighSurrogate = -1; + } else { + // Dangling high surrogate + outInt[outOffset] = prevHighSurrogate; + outOffset++; + if (Character.isHighSurrogate(c)) { + prevHighSurrogate = c & 0xFFFF; + } else { + outInt[outOffset] = c & 0xFFFF; + outOffset++; + prevHighSurrogate = -1; + } + } + } else if (Character.isHighSurrogate(c)) { + prevHighSurrogate = c & 0xFFFF; + } else { + outInt[outOffset] = c & 0xFFFF; + outOffset++; + } + } + + if (prevHighSurrogate != -1) { + // Dangling high surrogate + outInt[outOffset] = prevHighSurrogate & 0xFFFF; + outOffset++; + } + + utf16In.position(inOffset - utf16In.arrayOffset()); + intBuffer.position(outOffset - intBuffer.arrayOffset()); + } + + private void byteToCharBuffer(int toAppend) { + byteBuffer.flip(); + // CharBuffers hold twice as much per unit as ByteBuffers, so start with half the capacity. + CharBuffer newBuffer = CharBuffer.allocate(Math.max(byteBuffer.remaining() + toAppend, byteBuffer.capacity() / 2)); + while (byteBuffer.hasRemaining()) { + newBuffer.put((char) (byteBuffer.get() & 0xFF)); + } + type = Type.CHAR; + byteBuffer = null; + charBuffer = newBuffer; + } + + private void byteToIntBuffer(int toAppend) { + byteBuffer.flip(); + // IntBuffers hold four times as much per unit as ByteBuffers, so start with one quarter the capacity. + IntBuffer newBuffer = IntBuffer.allocate(Math.max(byteBuffer.remaining() + toAppend, byteBuffer.capacity() / 4)); + while (byteBuffer.hasRemaining()) { + newBuffer.put(byteBuffer.get() & 0xFF); + } + type = Type.INT; + byteBuffer = null; + intBuffer = newBuffer; + } + + private void charToIntBuffer(int toAppend) { + charBuffer.flip(); + // IntBuffers hold two times as much per unit as ByteBuffers, so start with one half the capacity. + IntBuffer newBuffer = IntBuffer.allocate(Math.max(charBuffer.remaining() + toAppend, charBuffer.capacity() / 2)); + while (charBuffer.hasRemaining()) { + newBuffer.put(charBuffer.get() & 0xFFFF); + } + type = Type.INT; + charBuffer = null; + intBuffer = newBuffer; + } + } +} diff --git a/runtime/Java/src/org/antlr/v4/runtime/CodePointCharStream.java b/runtime/Java/src/org/antlr/v4/runtime/CodePointCharStream.java new file mode 100644 index 0000000..107faa7 --- /dev/null +++ b/runtime/Java/src/org/antlr/v4/runtime/CodePointCharStream.java @@ -0,0 +1,297 @@ +/* + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. + * Use of this file is governed by the BSD 3-clause license that + * can be found in the LICENSE.txt file in the project root. + */ + +package org.antlr.v4.runtime; + +import org.antlr.v4.runtime.misc.Interval; + +import java.nio.charset.StandardCharsets; + +/** + * Alternative to {@link ANTLRInputStream} which treats the input + * as a series of Unicode code points, instead of a series of UTF-16 + * code units. + * + * Use this if you need to parse input which potentially contains + * Unicode values > U+FFFF. + */ +public abstract class CodePointCharStream implements CharStream { + protected final int size; + protected final String name; + + // To avoid lots of virtual method calls, we directly access + // the state of the underlying code points in the + // CodePointBuffer. + protected int position; + + // Use the factory method {@link #fromBuffer(CodePointBuffer)} to + // construct instances of this type. + private CodePointCharStream(int position, int remaining, String name) { + // TODO + assert position == 0; + this.size = remaining; + this.name = name; + this.position = 0; + } + + // Visible for testing. + abstract Object getInternalStorage(); + + /** + * Constructs a {@link CodePointCharStream} which provides access + * to the Unicode code points stored in {@code codePointBuffer}. + */ + public static CodePointCharStream fromBuffer(CodePointBuffer codePointBuffer) { + return fromBuffer(codePointBuffer, UNKNOWN_SOURCE_NAME); + } + + /** + * Constructs a named {@link CodePointCharStream} which provides access + * to the Unicode code points stored in {@code codePointBuffer}. + */ + public static CodePointCharStream fromBuffer(CodePointBuffer codePointBuffer, String name) { + // Java lacks generics on primitive types. + // + // To avoid lots of calls to virtual methods in the + // very hot codepath of LA() below, we construct one + // of three concrete subclasses. + // + // The concrete subclasses directly access the code + // points stored in the underlying array (byte[], + // char[], or int[]), so we can avoid lots of virtual + // method calls to ByteBuffer.get(offset). + switch (codePointBuffer.getType()) { + case BYTE: + return new CodePoint8BitCharStream( + codePointBuffer.position(), + codePointBuffer.remaining(), + name, + codePointBuffer.byteArray(), + codePointBuffer.arrayOffset()); + case CHAR: + return new CodePoint16BitCharStream( + codePointBuffer.position(), + codePointBuffer.remaining(), + name, + codePointBuffer.charArray(), + codePointBuffer.arrayOffset()); + case INT: + return new CodePoint32BitCharStream( + codePointBuffer.position(), + codePointBuffer.remaining(), + name, + codePointBuffer.intArray(), + codePointBuffer.arrayOffset()); + } + throw new UnsupportedOperationException("Not reached"); + } + + @Override + public final void consume() { + if (size - position == 0) { + assert LA(1) == IntStream.EOF; + throw new IllegalStateException("cannot consume EOF"); + } + position = position + 1; + } + + @Override + public final int index() { + return position; + } + + @Override + public final int size() { + return size; + } + + /** mark/release do nothing; we have entire buffer */ + @Override + public final int mark() { + return -1; + } + + @Override + public final void release(int marker) { + } + + @Override + public final void seek(int index) { + position = index; + } + + @Override + public final String getSourceName() { + if (name == null || name.isEmpty()) { + return UNKNOWN_SOURCE_NAME; + } + + return name; + } + + @Override + public final String toString() { + return getText(Interval.of(0, size - 1)); + } + + // 8-bit storage for code points <= U+00FF. + private static final class CodePoint8BitCharStream extends CodePointCharStream { + private final byte[] byteArray; + + private CodePoint8BitCharStream(int position, int remaining, String name, byte[] byteArray, int arrayOffset) { + super(position, remaining, name); + // TODO + assert arrayOffset == 0; + this.byteArray = byteArray; + } + + /** Return the UTF-16 encoded string for the given interval */ + @Override + public String getText(Interval interval) { + int startIdx = Math.min(interval.a, size); + int len = Math.min(interval.b - interval.a + 1, size - startIdx); + + // We know the maximum code point in byteArray is U+00FF, + // so we can treat this as if it were ISO-8859-1, aka Latin-1, + // which shares the same code points up to 0xFF. + return new String(byteArray, startIdx, len, StandardCharsets.ISO_8859_1); + } + + @Override + public int LA(int i) { + int offset; + switch (Integer.signum(i)) { + case -1: + offset = position + i; + if (offset < 0) { + return IntStream.EOF; + } + return byteArray[offset] & 0xFF; + case 0: + // Undefined + return 0; + case 1: + offset = position + i - 1; + if (offset >= size) { + return IntStream.EOF; + } + return byteArray[offset] & 0xFF; + } + throw new UnsupportedOperationException("Not reached"); + } + + @Override + Object getInternalStorage() { + return byteArray; + } + } + + // 16-bit internal storage for code points between U+0100 and U+FFFF. + private static final class CodePoint16BitCharStream extends CodePointCharStream { + private final char[] charArray; + + private CodePoint16BitCharStream(int position, int remaining, String name, char[] charArray, int arrayOffset) { + super(position, remaining, name); + this.charArray = charArray; + // TODO + assert arrayOffset == 0; + } + + /** Return the UTF-16 encoded string for the given interval */ + @Override + public String getText(Interval interval) { + int startIdx = Math.min(interval.a, size - 1); + int len = Math.min(interval.b - interval.a + 1, size); + + // We know there are no surrogates in this + // array, since otherwise we would be given a + // 32-bit int[] array. + // + // So, it's safe to treat this as if it were + // UTF-16. + return new String(charArray, startIdx, len); + } + + @Override + public int LA(int i) { + int offset; + switch (Integer.signum(i)) { + case -1: + offset = position + i; + if (offset < 0) { + return IntStream.EOF; + } + return charArray[offset] & 0xFFFF; + case 0: + // Undefined + return 0; + case 1: + offset = position + i - 1; + if (offset >= size) { + return IntStream.EOF; + } + return charArray[offset] & 0xFFFF; + } + throw new UnsupportedOperationException("Not reached"); + } + + @Override + Object getInternalStorage() { + return charArray; + } + } + + // 32-bit internal storage for code points between U+10000 and U+10FFFF. + private static final class CodePoint32BitCharStream extends CodePointCharStream { + private final int[] intArray; + + private CodePoint32BitCharStream(int position, int remaining, String name, int[] intArray, int arrayOffset) { + super(position, remaining, name); + this.intArray = intArray; + // TODO + assert arrayOffset == 0; + } + + /** Return the UTF-16 encoded string for the given interval */ + @Override + public String getText(Interval interval) { + int startIdx = Math.min(interval.a, size - 1); + int len = Math.min(interval.b - interval.a + 1, size); + + // Note that we pass the int[] code points to the String constructor -- + // this is supported, and the constructor will convert to UTF-16 internally. + return new String(intArray, startIdx, len); + } + + @Override + public int LA(int i) { + int offset; + switch (Integer.signum(i)) { + case -1: + offset = position + i; + if (offset < 0) { + return IntStream.EOF; + } + return intArray[offset]; + case 0: + // Undefined + return 0; + case 1: + offset = position + i - 1; + if (offset >= size) { + return IntStream.EOF; + } + return intArray[offset]; + } + throw new UnsupportedOperationException("Not reached"); + } + + @Override + Object getInternalStorage() { + return intArray; + } + } +} diff --git a/runtime/Java/src/org/antlr/v4/runtime/CommonToken.java b/runtime/Java/src/org/antlr/v4/runtime/CommonToken.java index 015df7f..a34020f 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/CommonToken.java +++ b/runtime/Java/src/org/antlr/v4/runtime/CommonToken.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/runtime/Java/src/org/antlr/v4/runtime/CommonTokenFactory.java b/runtime/Java/src/org/antlr/v4/runtime/CommonTokenFactory.java index da7bace..d52cbae 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/CommonTokenFactory.java +++ b/runtime/Java/src/org/antlr/v4/runtime/CommonTokenFactory.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/runtime/Java/src/org/antlr/v4/runtime/CommonTokenStream.java b/runtime/Java/src/org/antlr/v4/runtime/CommonTokenStream.java index fd6a8eb..153f421 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/CommonTokenStream.java +++ b/runtime/Java/src/org/antlr/v4/runtime/CommonTokenStream.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/runtime/Java/src/org/antlr/v4/runtime/ConsoleErrorListener.java b/runtime/Java/src/org/antlr/v4/runtime/ConsoleErrorListener.java index f7d36b5..aa2e20d 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/ConsoleErrorListener.java +++ b/runtime/Java/src/org/antlr/v4/runtime/ConsoleErrorListener.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/runtime/Java/src/org/antlr/v4/runtime/DefaultErrorStrategy.java b/runtime/Java/src/org/antlr/v4/runtime/DefaultErrorStrategy.java index db5061c..02b5ee5 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/DefaultErrorStrategy.java +++ b/runtime/Java/src/org/antlr/v4/runtime/DefaultErrorStrategy.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ @@ -37,6 +37,21 @@ public class DefaultErrorStrategy implements ANTLRErrorStrategy { protected IntervalSet lastErrorStates; /** + * This field is used to propagate information about the lookahead following + * the previous match. Since prediction prefers completing the current rule + * to error recovery efforts, error reporting may occur later than the + * original point where it was discoverable. The original context is used to + * compute the true expected sets as though the reporting occurred as early + * as possible. + */ + protected ParserRuleContext nextTokensContext; + + /** + * @see #nextTokensContext + */ + protected int nextTokensState; + + /** * {@inheritDoc} * * <p>The default implementation simply calls {@link #endErrorCondition} to @@ -224,10 +239,21 @@ public class DefaultErrorStrategy implements ANTLRErrorStrategy { int la = tokens.LA(1); // try cheaper subset first; might get lucky. seems to shave a wee bit off - if ( recognizer.getATN().nextTokens(s).contains(la) || la==Token.EOF ) return; + IntervalSet nextTokens = recognizer.getATN().nextTokens(s); + if (nextTokens.contains(la)) { + // We are sure the token matches + nextTokensContext = null; + nextTokensState = ATNState.INVALID_STATE_NUMBER; + return; + } - // Return but don't end recovery. only do that upon valid token match - if (recognizer.isExpectedToken(la)) { + if (nextTokens.contains(Token.EPSILON)) { + if (nextTokensContext == null) { + // It's possible the next token won't match; information tracked + // by sync is restricted for performance. + nextTokensContext = recognizer.getContext(); + nextTokensState = recognizer.getState(); + } return; } @@ -452,7 +478,14 @@ public class DefaultErrorStrategy implements ANTLRErrorStrategy { } // even that didn't work; must throw the exception - throw new InputMismatchException(recognizer); + InputMismatchException e; + if (nextTokensContext == null) { + e = new InputMismatchException(recognizer); + } else { + e = new InputMismatchException(recognizer, nextTokensState, nextTokensContext); + } + + throw e; } /** @@ -550,7 +583,10 @@ public class DefaultErrorStrategy implements ANTLRErrorStrategy { protected Token getMissingSymbol(Parser recognizer) { Token currentSymbol = recognizer.getCurrentToken(); IntervalSet expecting = getExpectedTokens(recognizer); - int expectedTokenType = expecting.getMinElement(); // get any element + int expectedTokenType = Token.INVALID_TYPE; + if ( !expecting.isNil() ) { + expectedTokenType = expecting.getMinElement(); // get any element + } String tokenText; if ( expectedTokenType== Token.EOF ) tokenText = "<missing EOF>"; else tokenText = "<missing "+recognizer.getVocabulary().getDisplayName(expectedTokenType)+">"; diff --git a/runtime/Java/src/org/antlr/v4/runtime/DiagnosticErrorListener.java b/runtime/Java/src/org/antlr/v4/runtime/DiagnosticErrorListener.java index 94a8e9b..03fd0aa 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/DiagnosticErrorListener.java +++ b/runtime/Java/src/org/antlr/v4/runtime/DiagnosticErrorListener.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/runtime/Java/src/org/antlr/v4/runtime/FailedPredicateException.java b/runtime/Java/src/org/antlr/v4/runtime/FailedPredicateException.java index 422a7a9..d9f0fec 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/FailedPredicateException.java +++ b/runtime/Java/src/org/antlr/v4/runtime/FailedPredicateException.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/runtime/Java/src/org/antlr/v4/runtime/InputMismatchException.java b/runtime/Java/src/org/antlr/v4/runtime/InputMismatchException.java index 1f45d7c..08ef67c 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/InputMismatchException.java +++ b/runtime/Java/src/org/antlr/v4/runtime/InputMismatchException.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ @@ -13,4 +13,10 @@ public class InputMismatchException extends RecognitionException { super(recognizer, recognizer.getInputStream(), recognizer._ctx); this.setOffendingToken(recognizer.getCurrentToken()); } + + public InputMismatchException(Parser recognizer, int state, ParserRuleContext ctx) { + super(recognizer, recognizer.getInputStream(), ctx); + this.setOffendingState(state); + this.setOffendingToken(recognizer.getCurrentToken()); + } } diff --git a/runtime/Java/src/org/antlr/v4/runtime/IntStream.java b/runtime/Java/src/org/antlr/v4/runtime/IntStream.java index 4fd59dc..07ccf6d 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/IntStream.java +++ b/runtime/Java/src/org/antlr/v4/runtime/IntStream.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ @@ -53,7 +53,7 @@ public interface IntStream { * filtering streams (e.g. {@link CommonTokenStream} which distinguishes * between "on-channel" and "off-channel" tokens). * - * @throws IllegalStateException if an attempt is made to consume the the + * @throws IllegalStateException if an attempt is made to consume the * end of the stream (i.e. if {@code LA(1)==}{@link #EOF EOF} before calling * {@code consume}). */ diff --git a/runtime/Java/src/org/antlr/v4/runtime/InterpreterRuleContext.java b/runtime/Java/src/org/antlr/v4/runtime/InterpreterRuleContext.java index 893a5a8..862f107 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/InterpreterRuleContext.java +++ b/runtime/Java/src/org/antlr/v4/runtime/InterpreterRuleContext.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/runtime/Java/src/org/antlr/v4/runtime/Lexer.java b/runtime/Java/src/org/antlr/v4/runtime/Lexer.java index bb082aa..bba3668 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/Lexer.java +++ b/runtime/Java/src/org/antlr/v4/runtime/Lexer.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ @@ -28,8 +28,8 @@ public abstract class Lexer extends Recognizer<Integer, LexerATNSimulator> public static final int DEFAULT_TOKEN_CHANNEL = Token.DEFAULT_CHANNEL; public static final int HIDDEN = Token.HIDDEN_CHANNEL; - public static final int MIN_CHAR_VALUE = '\u0000'; - public static final int MAX_CHAR_VALUE = '\uFFFE'; + public static final int MIN_CHAR_VALUE = 0x0000; + public static final int MAX_CHAR_VALUE = 0x10FFFF; public CharStream _input; protected Pair<TokenSource, CharStream> _tokenFactorySourcePair; @@ -320,6 +320,8 @@ public abstract class Lexer extends Recognizer<Integer, LexerATNSimulator> return _channel; } + public String[] getChannelNames() { return null; } + public String[] getModeNames() { return null; } diff --git a/runtime/Java/src/org/antlr/v4/runtime/LexerInterpreter.java b/runtime/Java/src/org/antlr/v4/runtime/LexerInterpreter.java index bb50c53..8a8dfb4 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/LexerInterpreter.java +++ b/runtime/Java/src/org/antlr/v4/runtime/LexerInterpreter.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ @@ -12,6 +12,7 @@ import org.antlr.v4.runtime.atn.LexerATNSimulator; import org.antlr.v4.runtime.atn.PredictionContextCache; import org.antlr.v4.runtime.dfa.DFA; +import java.util.ArrayList; import java.util.Collection; public class LexerInterpreter extends Lexer { @@ -21,6 +22,7 @@ public class LexerInterpreter extends Lexer { @Deprecated protected final String[] tokenNames; protected final String[] ruleNames; + protected final String[] channelNames; protected final String[] modeNames; @@ -32,10 +34,15 @@ public class LexerInterpreter extends Lexer { @Deprecated public LexerInterpreter(String grammarFileName, Collection<String> tokenNames, Collection<String> ruleNames, Collection<String> modeNames, ATN atn, CharStream input) { - this(grammarFileName, VocabularyImpl.fromTokenNames(tokenNames.toArray(new String[tokenNames.size()])), ruleNames, modeNames, atn, input); + this(grammarFileName, VocabularyImpl.fromTokenNames(tokenNames.toArray(new String[tokenNames.size()])), ruleNames, new ArrayList<String>(), modeNames, atn, input); } + @Deprecated public LexerInterpreter(String grammarFileName, Vocabulary vocabulary, Collection<String> ruleNames, Collection<String> modeNames, ATN atn, CharStream input) { + this(grammarFileName, vocabulary, ruleNames, new ArrayList<String>(), modeNames, atn, input); + } + + public LexerInterpreter(String grammarFileName, Vocabulary vocabulary, Collection<String> ruleNames, Collection<String> channelNames, Collection<String> modeNames, ATN atn, CharStream input) { super(input); if (atn.grammarType != ATNType.LEXER) { @@ -50,6 +57,7 @@ public class LexerInterpreter extends Lexer { } this.ruleNames = ruleNames.toArray(new String[ruleNames.size()]); + this.channelNames = channelNames.toArray(new String[channelNames.size()]); this.modeNames = modeNames.toArray(new String[modeNames.size()]); this.vocabulary = vocabulary; @@ -82,6 +90,11 @@ public class LexerInterpreter extends Lexer { } @Override + public String[] getChannelNames() { + return channelNames; + } + + @Override public String[] getModeNames() { return modeNames; } diff --git a/runtime/Java/src/org/antlr/v4/runtime/LexerNoViableAltException.java b/runtime/Java/src/org/antlr/v4/runtime/LexerNoViableAltException.java index 2540d10..8b33196 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/LexerNoViableAltException.java +++ b/runtime/Java/src/org/antlr/v4/runtime/LexerNoViableAltException.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/runtime/Java/src/org/antlr/v4/runtime/ListTokenSource.java b/runtime/Java/src/org/antlr/v4/runtime/ListTokenSource.java index 592c8d9..317027e 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/ListTokenSource.java +++ b/runtime/Java/src/org/antlr/v4/runtime/ListTokenSource.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/runtime/Java/src/org/antlr/v4/runtime/NoViableAltException.java b/runtime/Java/src/org/antlr/v4/runtime/NoViableAltException.java index a805e80..4e20343 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/NoViableAltException.java +++ b/runtime/Java/src/org/antlr/v4/runtime/NoViableAltException.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/runtime/Java/src/org/antlr/v4/runtime/Parser.java b/runtime/Java/src/org/antlr/v4/runtime/Parser.java index 1edc457..566aea0 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/Parser.java +++ b/runtime/Java/src/org/antlr/v4/runtime/Parser.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ @@ -19,9 +19,11 @@ import org.antlr.v4.runtime.dfa.DFA; import org.antlr.v4.runtime.misc.IntegerStack; import org.antlr.v4.runtime.misc.IntervalSet; import org.antlr.v4.runtime.tree.ErrorNode; +import org.antlr.v4.runtime.tree.ErrorNodeImpl; import org.antlr.v4.runtime.tree.ParseTreeListener; import org.antlr.v4.runtime.tree.ParseTreeWalker; import org.antlr.v4.runtime.tree.TerminalNode; +import org.antlr.v4.runtime.tree.TerminalNodeImpl; import org.antlr.v4.runtime.tree.pattern.ParseTreePattern; import org.antlr.v4.runtime.tree.pattern.ParseTreePatternMatcher; @@ -182,7 +184,8 @@ public abstract class Parser extends Recognizer<Token, ParserATNSimulator> { * strategy to attempt recovery. If {@link #getBuildParseTree} is * {@code true} and the token index of the symbol returned by * {@link ANTLRErrorStrategy#recoverInline} is -1, the symbol is added to - * the parse tree by calling {@link ParserRuleContext#addErrorNode}.</p> + * the parse tree by calling {@link #createErrorNode(ParserRuleContext, Token)} then + * {@link ParserRuleContext#addErrorNode(ErrorNode)}.</p> * * @param ttype the token type to match * @return the matched symbol @@ -204,7 +207,7 @@ public abstract class Parser extends Recognizer<Token, ParserATNSimulator> { if ( _buildParseTrees && t.getTokenIndex()==-1 ) { // we must have conjured up a new token during single token insertion // if it's not the current symbol - _ctx.addErrorNode(t); + _ctx.addErrorNode(createErrorNode(_ctx,t)); } } return t; @@ -220,7 +223,8 @@ public abstract class Parser extends Recognizer<Token, ParserATNSimulator> { * strategy to attempt recovery. If {@link #getBuildParseTree} is * {@code true} and the token index of the symbol returned by * {@link ANTLRErrorStrategy#recoverInline} is -1, the symbol is added to - * the parse tree by calling {@link ParserRuleContext#addErrorNode}.</p> + * the parse tree by calling {@link Parser#createErrorNode(ParserRuleContext, Token)}. then + * {@link ParserRuleContext#addErrorNode(ErrorNode)}</p> * * @return the matched symbol * @throws RecognitionException if the current input symbol did not match @@ -238,7 +242,7 @@ public abstract class Parser extends Recognizer<Token, ParserATNSimulator> { if (_buildParseTrees && t.getTokenIndex() == -1) { // we must have conjured up a new token during single token insertion // if it's not the current symbol - _ctx.addErrorNode(t); + _ctx.addErrorNode(createErrorNode(_ctx,t)); } } @@ -553,11 +557,11 @@ public abstract class Parser extends Recognizer<Token, ParserATNSimulator> { * </pre> * * If the parser is not in error recovery mode, the consumed symbol is added - * to the parse tree using {@link ParserRuleContext#addChild(Token)}, and + * to the parse tree using {@link ParserRuleContext#addChild(TerminalNode)}, and * {@link ParseTreeListener#visitTerminal} is called on any parse listeners. * If the parser <em>is</em> in error recovery mode, the consumed symbol is - * added to the parse tree using - * {@link ParserRuleContext#addErrorNode(Token)}, and + * added to the parse tree using {@link #createErrorNode(ParserRuleContext, Token)} then + * {@link ParserRuleContext#addErrorNode(ErrorNode)} and * {@link ParseTreeListener#visitErrorNode} is called on any parse * listeners. */ @@ -569,7 +573,7 @@ public abstract class Parser extends Recognizer<Token, ParserATNSimulator> { boolean hasListener = _parseListeners != null && !_parseListeners.isEmpty(); if (_buildParseTrees || hasListener) { if ( _errHandler.inErrorRecoveryMode(this) ) { - ErrorNode node = _ctx.addErrorNode(o); + ErrorNode node = _ctx.addErrorNode(createErrorNode(_ctx,o)); if (_parseListeners != null) { for (ParseTreeListener listener : _parseListeners) { listener.visitErrorNode(node); @@ -577,7 +581,7 @@ public abstract class Parser extends Recognizer<Token, ParserATNSimulator> { } } else { - TerminalNode node = _ctx.addChild(o); + TerminalNode node = _ctx.addChild(createTerminalNode(_ctx,o)); if (_parseListeners != null) { for (ParseTreeListener listener : _parseListeners) { listener.visitTerminal(node); @@ -588,6 +592,24 @@ public abstract class Parser extends Recognizer<Token, ParserATNSimulator> { return o; } + /** How to create a token leaf node associated with a parent. + * Typically, the terminal node to create is not a function of the parent. + * + * @since 4.7 + */ + public TerminalNode createTerminalNode(ParserRuleContext parent, Token t) { + return new TerminalNodeImpl(t); + } + + /** How to create an error node, given a token, associated with a parent. + * Typically, the error node to create is not a function of the parent. + * + * @since 4.7 + */ + public ErrorNode createErrorNode(ParserRuleContext parent, Token t) { + return new ErrorNodeImpl(t); + } + protected void addContextToParseTree() { ParserRuleContext parent = (ParserRuleContext)_ctx.parent; // add current context to parent if we have a parent @@ -925,3 +947,4 @@ public abstract class Parser extends Recognizer<Token, ParserATNSimulator> { return _tracer != null; } } + diff --git a/runtime/Java/src/org/antlr/v4/runtime/ParserInterpreter.java b/runtime/Java/src/org/antlr/v4/runtime/ParserInterpreter.java index 568495b..6045b27 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/ParserInterpreter.java +++ b/runtime/Java/src/org/antlr/v4/runtime/ParserInterpreter.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ @@ -407,14 +407,17 @@ public class ParserInterpreter extends Parser { if ( e instanceof InputMismatchException ) { InputMismatchException ime = (InputMismatchException)e; Token tok = e.getOffendingToken(); - int expectedTokenType = ime.getExpectedTokens().getMinElement(); // get any element + int expectedTokenType = Token.INVALID_TYPE; + if ( !ime.getExpectedTokens().isNil() ) { + expectedTokenType = ime.getExpectedTokens().getMinElement(); // get any element + } Token errToken = getTokenFactory().create(new Pair<TokenSource, CharStream>(tok.getTokenSource(), tok.getTokenSource().getInputStream()), expectedTokenType, tok.getText(), Token.DEFAULT_CHANNEL, -1, -1, // invalid start/stop tok.getLine(), tok.getCharPositionInLine()); - _ctx.addErrorNode(errToken); + _ctx.addErrorNode(createErrorNode(_ctx,errToken)); } else { // NoViableAlt Token tok = e.getOffendingToken(); @@ -424,7 +427,7 @@ public class ParserInterpreter extends Parser { Token.DEFAULT_CHANNEL, -1, -1, // invalid start/stop tok.getLine(), tok.getCharPositionInLine()); - _ctx.addErrorNode(errToken); + _ctx.addErrorNode(createErrorNode(_ctx,errToken)); } } } @@ -445,3 +448,4 @@ public class ParserInterpreter extends Parser { return rootContext; } } + diff --git a/runtime/Java/src/org/antlr/v4/runtime/ParserRuleContext.java b/runtime/Java/src/org/antlr/v4/runtime/ParserRuleContext.java index 9f3b97f..4513b09 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/ParserRuleContext.java +++ b/runtime/Java/src/org/antlr/v4/runtime/ParserRuleContext.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ @@ -63,8 +63,8 @@ public class ParserRuleContext extends RuleContext { * * The parser setState() method updates field s and adds it to this list * if we are debugging/tracing. - * - * This does not trace states visited during prediction. + * + * This does not trace states visited during prediction. */ // public List<Integer> states; @@ -79,7 +79,8 @@ public class ParserRuleContext extends RuleContext { public ParserRuleContext() { } /** COPY a ctx (I'm deliberately not using copy constructor) to avoid - * confusion with creating node with parent. Does not copy children. + * confusion with creating node with parent. Does not copy children + * (except error leaves). * * This is used in the generated parser code to flip a generic XContext * node for rule X to a YContext for alt label Y. In that sense, it is @@ -101,9 +102,8 @@ public class ParserRuleContext extends RuleContext { this.children = new ArrayList<>(); // reset parent pointer for any error nodes for (ParseTree child : ctx.children) { - if ( child instanceof ErrorNodeImpl ) { - this.children.add(child); - ((ErrorNodeImpl) child).parent = this; + if ( child instanceof ErrorNode ) { + addChild((ErrorNode)child); } } } @@ -118,48 +118,83 @@ public class ParserRuleContext extends RuleContext { public void enterRule(ParseTreeListener listener) { } public void exitRule(ParseTreeListener listener) { } - /** Does not set parent link; other add methods do that */ - public TerminalNode addChild(TerminalNode t) { - if ( children==null ) children = new ArrayList<ParseTree>(); + /** Add a parse tree node to this as a child. Works for + * internal and leaf nodes. Does not set parent link; + * other add methods must do that. Other addChild methods + * call this. + * + * We cannot set the parent pointer of the incoming node + * because the existing interfaces do not have a setParent() + * method and I don't want to break backward compatibility for this. + * + * @since 4.7 + */ + public <T extends ParseTree> T addAnyChild(T t) { + if ( children==null ) children = new ArrayList<>(); children.add(t); return t; } public RuleContext addChild(RuleContext ruleInvocation) { - if ( children==null ) children = new ArrayList<ParseTree>(); - children.add(ruleInvocation); - return ruleInvocation; + return addAnyChild(ruleInvocation); } - /** Used by enterOuterAlt to toss out a RuleContext previously added as - * we entered a rule. If we have # label, we will need to remove - * generic ruleContext object. - */ - public void removeLastChild() { - if ( children!=null ) { - children.remove(children.size()-1); - } + /** Add a token leaf node child and force its parent to be this node. */ + public TerminalNode addChild(TerminalNode t) { + t.setParent(this); + return addAnyChild(t); } -// public void trace(int s) { -// if ( states==null ) states = new ArrayList<Integer>(); -// states.add(s); -// } + /** Add an error node child and force its parent to be this node. + * + * @since 4.7 + */ + public ErrorNode addErrorNode(ErrorNode errorNode) { + errorNode.setParent(this); + return addAnyChild(errorNode); + } + /** Add a child to this node based upon matchedToken. It + * creates a TerminalNodeImpl rather than using + * {@link Parser#createTerminalNode(ParserRuleContext, Token)}. I'm leaving this + * in for compatibility but the parser doesn't use this anymore. + */ + @Deprecated public TerminalNode addChild(Token matchedToken) { TerminalNodeImpl t = new TerminalNodeImpl(matchedToken); - addChild(t); - t.parent = this; + addAnyChild(t); + t.setParent(this); return t; } + /** Add a child to this node based upon badToken. It + * creates a ErrorNodeImpl rather than using + * {@link Parser#createErrorNode(ParserRuleContext, Token)}. I'm leaving this + * in for compatibility but the parser doesn't use this anymore. + */ + @Deprecated public ErrorNode addErrorNode(Token badToken) { ErrorNodeImpl t = new ErrorNodeImpl(badToken); - addChild(t); - t.parent = this; + addAnyChild(t); + t.setParent(this); return t; } +// public void trace(int s) { +// if ( states==null ) states = new ArrayList<Integer>(); +// states.add(s); +// } + + /** Used by enterOuterAlt to toss out a RuleContext previously added as + * we entered a rule. If we have # label, we will need to remove + * generic ruleContext object. + */ + public void removeLastChild() { + if ( children!=null ) { + children.remove(children.size()-1); + } + } + @Override /** Override to make type more specific */ public ParserRuleContext getParent() { @@ -290,13 +325,14 @@ public class ParserRuleContext extends RuleContext { */ public Token getStop() { return stop; } - /** Used for rule context info debugging during parse-time, not so much for ATN debugging */ - public String toInfoString(Parser recognizer) { - List<String> rules = recognizer.getRuleInvocationStack(this); - Collections.reverse(rules); - return "ParserRuleContext"+rules+"{" + - "start=" + start + - ", stop=" + stop + - '}'; - } + /** Used for rule context info debugging during parse-time, not so much for ATN debugging */ + public String toInfoString(Parser recognizer) { + List<String> rules = recognizer.getRuleInvocationStack(this); + Collections.reverse(rules); + return "ParserRuleContext"+rules+"{" + + "start=" + start + + ", stop=" + stop + + '}'; + } } + diff --git a/runtime/Java/src/org/antlr/v4/runtime/ProxyErrorListener.java b/runtime/Java/src/org/antlr/v4/runtime/ProxyErrorListener.java index 35d7546..703fbb6 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/ProxyErrorListener.java +++ b/runtime/Java/src/org/antlr/v4/runtime/ProxyErrorListener.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/runtime/Java/src/org/antlr/v4/runtime/RecognitionException.java b/runtime/Java/src/org/antlr/v4/runtime/RecognitionException.java index 5157421..f5d0399 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/RecognitionException.java +++ b/runtime/Java/src/org/antlr/v4/runtime/RecognitionException.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/runtime/Java/src/org/antlr/v4/runtime/Recognizer.java b/runtime/Java/src/org/antlr/v4/runtime/Recognizer.java index 6b407bc..3e09d15 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/Recognizer.java +++ b/runtime/Java/src/org/antlr/v4/runtime/Recognizer.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/runtime/Java/src/org/antlr/v4/runtime/RuleContext.java b/runtime/Java/src/org/antlr/v4/runtime/RuleContext.java index 91eafd0..8b7f168 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/RuleContext.java +++ b/runtime/Java/src/org/antlr/v4/runtime/RuleContext.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ @@ -163,6 +163,12 @@ public class RuleContext implements RuleNode { */ public void setAltNumber(int altNumber) { } + /** @since 4.7. {@see ParseTree#setParent} comment */ + @Override + public void setParent(RuleContext parent) { + this.parent = parent; + } + @Override public ParseTree getChild(int i) { return null; diff --git a/runtime/Java/src/org/antlr/v4/runtime/RuleContextWithAltNum.java b/runtime/Java/src/org/antlr/v4/runtime/RuleContextWithAltNum.java index 81fe476..38ec190 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/RuleContextWithAltNum.java +++ b/runtime/Java/src/org/antlr/v4/runtime/RuleContextWithAltNum.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/runtime/Java/src/org/antlr/v4/runtime/RuntimeMetaData.java b/runtime/Java/src/org/antlr/v4/runtime/RuntimeMetaData.java index 9355d31..bc2e478 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/RuntimeMetaData.java +++ b/runtime/Java/src/org/antlr/v4/runtime/RuntimeMetaData.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ @@ -67,7 +67,7 @@ public class RuntimeMetaData { * omitted.</li> * </ul> */ - public static final String VERSION = "4.6"; + public static final String VERSION = "4.7.1"; /** * Gets the currently executing version of the ANTLR 4 runtime library. diff --git a/runtime/Java/src/org/antlr/v4/runtime/Token.java b/runtime/Java/src/org/antlr/v4/runtime/Token.java index f25bed5..b6432ca 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/Token.java +++ b/runtime/Java/src/org/antlr/v4/runtime/Token.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/runtime/Java/src/org/antlr/v4/runtime/TokenFactory.java b/runtime/Java/src/org/antlr/v4/runtime/TokenFactory.java index 0053537..ad21f42 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/TokenFactory.java +++ b/runtime/Java/src/org/antlr/v4/runtime/TokenFactory.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ @@ -10,7 +10,7 @@ import org.antlr.v4.runtime.misc.Pair; /** The default mechanism for creating tokens. It's used by default in Lexer and * the error handling strategy (to create missing tokens). Notifying the parser - * of a new factory means that it notifies it's token source and error strategy. + * of a new factory means that it notifies its token source and error strategy. */ public interface TokenFactory<Symbol extends Token> { /** This is the method used to create tokens in the lexer and in the diff --git a/runtime/Java/src/org/antlr/v4/runtime/TokenSource.java b/runtime/Java/src/org/antlr/v4/runtime/TokenSource.java index 1e01bf8..3eb2c99 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/TokenSource.java +++ b/runtime/Java/src/org/antlr/v4/runtime/TokenSource.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/runtime/Java/src/org/antlr/v4/runtime/TokenStream.java b/runtime/Java/src/org/antlr/v4/runtime/TokenStream.java index 18b06f7..4e9887d 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/TokenStream.java +++ b/runtime/Java/src/org/antlr/v4/runtime/TokenStream.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/runtime/Java/src/org/antlr/v4/runtime/TokenStreamRewriter.java b/runtime/Java/src/org/antlr/v4/runtime/TokenStreamRewriter.java index 8b8abca..7195e39 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/TokenStreamRewriter.java +++ b/runtime/Java/src/org/antlr/v4/runtime/TokenStreamRewriter.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/runtime/Java/src/org/antlr/v4/runtime/UnbufferedCharStream.java b/runtime/Java/src/org/antlr/v4/runtime/UnbufferedCharStream.java index 712dc85..3db4970 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/UnbufferedCharStream.java +++ b/runtime/Java/src/org/antlr/v4/runtime/UnbufferedCharStream.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ @@ -12,12 +12,20 @@ import java.io.IOException; import java.io.InputStream; import java.io.InputStreamReader; import java.io.Reader; +import java.nio.charset.Charset; +import java.nio.charset.StandardCharsets; import java.util.Arrays; /** Do not buffer up the entire char stream. It does keep a small buffer * for efficiency and also buffers while a mark exists (set by the * lookahead prediction in parser). "Unbuffered" here refers to fact * that it doesn't buffer all data, not that's it's on demand loading of char. + * + * Before 4.7, this class used the default environment encoding to convert + * bytes to UTF-16, and held the UTF-16 bytes in the buffer as chars. + * + * As of 4.7, the class uses UTF-8 by default, and the buffer holds Unicode + * code points in the buffer as ints. */ public class UnbufferedCharStream implements CharStream { /** @@ -25,7 +33,7 @@ public class UnbufferedCharStream implements CharStream { * we keep adding to buffer. Otherwise, {@link #consume consume()} resets so * we start filling at index 0 again. */ - protected char[] data; + protected int[] data; /** * The number of characters currently in {@link #data data}. @@ -82,7 +90,7 @@ public class UnbufferedCharStream implements CharStream { /** Useful for subclasses that pull char from other than this.input. */ public UnbufferedCharStream(int bufferSize) { n = 0; - data = new char[bufferSize]; + data = new int[bufferSize]; } public UnbufferedCharStream(InputStream input) { @@ -94,8 +102,12 @@ public class UnbufferedCharStream implements CharStream { } public UnbufferedCharStream(InputStream input, int bufferSize) { + this(input, bufferSize, StandardCharsets.UTF_8); + } + + public UnbufferedCharStream(InputStream input, int bufferSize, Charset charset) { this(bufferSize); - this.input = new InputStreamReader(input); + this.input = new InputStreamReader(input, charset); fill(1); // prime } @@ -145,13 +157,42 @@ public class UnbufferedCharStream implements CharStream { */ protected int fill(int n) { for (int i=0; i<n; i++) { - if (this.n > 0 && data[this.n - 1] == (char)IntStream.EOF) { + if (this.n > 0 && data[this.n - 1] == IntStream.EOF) { return i; } try { int c = nextChar(); - add(c); + if (c > Character.MAX_VALUE || c == IntStream.EOF) { + add(c); + } + else { + char ch = (char) c; + if (Character.isLowSurrogate(ch)) { + throw new RuntimeException("Invalid UTF-16 (low surrogate with no preceding high surrogate)"); + } + else if (Character.isHighSurrogate(ch)) { + int lowSurrogate = nextChar(); + if (lowSurrogate > Character.MAX_VALUE) { + throw new RuntimeException("Invalid UTF-16 (high surrogate followed by code point > U+FFFF"); + } + else if (lowSurrogate == IntStream.EOF) { + throw new RuntimeException("Invalid UTF-16 (dangling high surrogate at end of file)"); + } + else { + char lowSurrogateChar = (char) lowSurrogate; + if (Character.isLowSurrogate(lowSurrogateChar)) { + add(Character.toCodePoint(ch, lowSurrogateChar)); + } + else { + throw new RuntimeException("Invalid UTF-16 (dangling high surrogate"); + } + } + } + else { + add(c); + } + } } catch (IOException ioe) { throw new RuntimeException(ioe); @@ -173,7 +214,7 @@ public class UnbufferedCharStream implements CharStream { if ( n>=data.length ) { data = Arrays.copyOf(data, data.length * 2); } - data[n++] = (char)c; + data[n++] = c; } @Override @@ -183,9 +224,7 @@ public class UnbufferedCharStream implements CharStream { int index = p + i - 1; if ( index < 0 ) throw new IndexOutOfBoundsException(); if ( index >= n ) return IntStream.EOF; - char c = data[index]; - if ( c==(char)IntStream.EOF ) return IntStream.EOF; - return c; + return data[index]; } /** diff --git a/runtime/Java/src/org/antlr/v4/runtime/UnbufferedTokenStream.java b/runtime/Java/src/org/antlr/v4/runtime/UnbufferedTokenStream.java index 046d821..f44b1c9 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/UnbufferedTokenStream.java +++ b/runtime/Java/src/org/antlr/v4/runtime/UnbufferedTokenStream.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/runtime/Java/src/org/antlr/v4/runtime/Vocabulary.java b/runtime/Java/src/org/antlr/v4/runtime/Vocabulary.java index c3662ea..83febb2 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/Vocabulary.java +++ b/runtime/Java/src/org/antlr/v4/runtime/Vocabulary.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/runtime/Java/src/org/antlr/v4/runtime/VocabularyImpl.java b/runtime/Java/src/org/antlr/v4/runtime/VocabularyImpl.java index 75b450e..00593e3 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/VocabularyImpl.java +++ b/runtime/Java/src/org/antlr/v4/runtime/VocabularyImpl.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/runtime/Java/src/org/antlr/v4/runtime/WritableToken.java b/runtime/Java/src/org/antlr/v4/runtime/WritableToken.java index bdaaac3..dc3aafd 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/WritableToken.java +++ b/runtime/Java/src/org/antlr/v4/runtime/WritableToken.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/runtime/Java/src/org/antlr/v4/runtime/atn/ATN.java b/runtime/Java/src/org/antlr/v4/runtime/atn/ATN.java index 1e67bea..96f71f8 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/atn/ATN.java +++ b/runtime/Java/src/org/antlr/v4/runtime/atn/ATN.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/runtime/Java/src/org/antlr/v4/runtime/atn/ATNConfig.java b/runtime/Java/src/org/antlr/v4/runtime/atn/ATNConfig.java index 6cededb..a3e4d1d 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/atn/ATNConfig.java +++ b/runtime/Java/src/org/antlr/v4/runtime/atn/ATNConfig.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ @@ -161,7 +161,8 @@ public class ATNConfig { public boolean equals(ATNConfig other) { if (this == other) { return true; - } else if (other == null) { + } + else if (other == null) { return false; } diff --git a/runtime/Java/src/org/antlr/v4/runtime/atn/ATNConfigSet.java b/runtime/Java/src/org/antlr/v4/runtime/atn/ATNConfigSet.java index 423c5d3..d40dfb4 100755 --- a/runtime/Java/src/org/antlr/v4/runtime/atn/ATNConfigSet.java +++ b/runtime/Java/src/org/antlr/v4/runtime/atn/ATNConfigSet.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/runtime/Java/src/org/antlr/v4/runtime/atn/ATNDeserializationOptions.java b/runtime/Java/src/org/antlr/v4/runtime/atn/ATNDeserializationOptions.java index 2191eb1..9aee1f2 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/atn/ATNDeserializationOptions.java +++ b/runtime/Java/src/org/antlr/v4/runtime/atn/ATNDeserializationOptions.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/runtime/Java/src/org/antlr/v4/runtime/atn/ATNDeserializer.java b/runtime/Java/src/org/antlr/v4/runtime/atn/ATNDeserializer.java index 9821b8b..0ebcdeb 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/atn/ATNDeserializer.java +++ b/runtime/Java/src/org/antlr/v4/runtime/atn/ATNDeserializer.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ @@ -45,6 +45,12 @@ public class ATNDeserializer { */ private static final UUID ADDED_LEXER_ACTIONS; /** + * This UUID indicates the serialized ATN contains two sets of + * IntervalSets, where the second set's values are encoded as + * 32-bit integers to support the full Unicode SMP range up to U+10FFFF. + */ + private static final UUID ADDED_UNICODE_SMP; + /** * This list contains all of the currently supported UUIDs, ordered by when * the feature first appeared in this branch. */ @@ -61,15 +67,59 @@ public class ATNDeserializer { BASE_SERIALIZED_UUID = UUID.fromString("33761B2D-78BB-4A43-8B0B-4F5BEE8AACF3"); ADDED_PRECEDENCE_TRANSITIONS = UUID.fromString("1DA0C57D-6C06-438A-9B27-10BCB3CE0F61"); ADDED_LEXER_ACTIONS = UUID.fromString("AADB8D7E-AEEF-4415-AD2B-8204D6CF042E"); + ADDED_UNICODE_SMP = UUID.fromString("59627784-3BE5-417A-B9EB-8131A7286089"); SUPPORTED_UUIDS = new ArrayList<UUID>(); SUPPORTED_UUIDS.add(BASE_SERIALIZED_UUID); SUPPORTED_UUIDS.add(ADDED_PRECEDENCE_TRANSITIONS); SUPPORTED_UUIDS.add(ADDED_LEXER_ACTIONS); + SUPPORTED_UUIDS.add(ADDED_UNICODE_SMP); + + SERIALIZED_UUID = ADDED_UNICODE_SMP; + } + + interface UnicodeDeserializer { + // Wrapper for readInt() or readInt32() + int readUnicode(char[] data, int p); + + // Work around Java not allowing mutation of captured variables + // by returning amount by which to increment p after each read + int size(); + } - SERIALIZED_UUID = ADDED_LEXER_ACTIONS; + enum UnicodeDeserializingMode { + UNICODE_BMP, + UNICODE_SMP } + static UnicodeDeserializer getUnicodeDeserializer(UnicodeDeserializingMode mode) { + if (mode == UnicodeDeserializingMode.UNICODE_BMP) { + return new UnicodeDeserializer() { + @Override + public int readUnicode(char[] data, int p) { + return toInt(data[p]); + } + + @Override + public int size() { + return 1; + } + }; + } + else { + return new UnicodeDeserializer() { + @Override + public int readUnicode(char[] data, int p) { + return toInt32(data, p); + } + + @Override + public int size() { + return 2; + } + }; + } + } private final ATNDeserializationOptions deserializationOptions; @@ -98,7 +148,7 @@ public class ATNDeserializer { * serialized ATN at or after the feature identified by {@code feature} was * introduced; otherwise, {@code false}. */ - protected boolean isFeatureSupported(UUID feature, UUID actualUuid) { + static protected boolean isFeatureSupported(UUID feature, UUID actualUuid) { int featureIndex = SUPPORTED_UUIDS.indexOf(feature); if (featureIndex < 0) { return false; @@ -110,7 +160,22 @@ public class ATNDeserializer { @SuppressWarnings("deprecation") public ATN deserialize(char[] data) { data = data.clone(); - // don't adjust the first value since that's the version number + + // Each char value in data is shifted by +2 at the entry to this method. + // This is an encoding optimization targeting the serialized values 0 + // and -1 (serialized to 0xFFFF), each of which are very common in the + // serialized form of the ATN. In the modified UTF-8 that Java uses for + // compiled string literals, these two character values have multi-byte + // forms. By shifting each value by +2, they become characters 2 and 1 + // prior to writing the string, each of which have single-byte + // representations. Since the shift occurs in the tool during ATN + // serialization, each target is responsible for adjusting the values + // during deserialization. + // + // As a special case, note that the first element of data is not + // adjusted because it contains the major version number of the + // serialized ATN, which was fixed at 3 at the time the value shifting + // was implemented. for (int i = 1; i < data.length; i++) { data[i] = (char)(data[i] - 2); } @@ -243,22 +308,14 @@ public class ATNDeserializer { // SETS // List<IntervalSet> sets = new ArrayList<IntervalSet>(); - int nsets = toInt(data[p++]); - for (int i=0; i<nsets; i++) { - int nintervals = toInt(data[p]); - p++; - IntervalSet set = new IntervalSet(); - sets.add(set); - boolean containsEof = toInt(data[p++]) != 0; - if (containsEof) { - set.add(-1); - } + // First, read all sets with 16-bit Unicode code points <= U+FFFF. + p = deserializeSets(data, p, sets, getUnicodeDeserializer(UnicodeDeserializingMode.UNICODE_BMP)); - for (int j=0; j<nintervals; j++) { - set.add(toInt(data[p]), toInt(data[p + 1])); - p += 2; - } + // Next, if the ATN was serialized with the Unicode SMP feature, + // deserialize sets with 32-bit arguments <= U+10FFFF. + if (isFeatureSupported(ADDED_UNICODE_SMP, uuid)) { + p = deserializeSets(data, p, sets, getUnicodeDeserializer(UnicodeDeserializingMode.UNICODE_SMP)); } // @@ -495,6 +552,30 @@ public class ATNDeserializer { return atn; } + private int deserializeSets(char[] data, int p, List<IntervalSet> sets, UnicodeDeserializer unicodeDeserializer) { + int nsets = toInt(data[p++]); + for (int i=0; i<nsets; i++) { + int nintervals = toInt(data[p]); + p++; + IntervalSet set = new IntervalSet(); + sets.add(set); + + boolean containsEof = toInt(data[p++]) != 0; + if (containsEof) { + set.add(-1); + } + + for (int j=0; j<nintervals; j++) { + int a = unicodeDeserializer.readUnicode(data, p); + p += unicodeDeserializer.size(); + int b = unicodeDeserializer.readUnicode(data, p); + p += unicodeDeserializer.size(); + set.add(a, b); + } + } + return p; + } + /** * Analyze the {@link StarLoopEntryState} states in the specified ATN to set * the {@link StarLoopEntryState#isPrecedenceDecision} field to the diff --git a/runtime/Java/src/org/antlr/v4/runtime/atn/ATNSerializer.java b/runtime/Java/src/org/antlr/v4/runtime/atn/ATNSerializer.java index e5f0344..b1efd14 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/atn/ATNSerializer.java +++ b/runtime/Java/src/org/antlr/v4/runtime/atn/ATNSerializer.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ @@ -14,7 +14,9 @@ import org.antlr.v4.runtime.misc.Utils; import java.io.InvalidClassException; import java.util.ArrayList; +import java.util.Collection; import java.util.HashMap; +import java.util.LinkedHashMap; import java.util.List; import java.util.Locale; import java.util.Map; @@ -24,6 +26,10 @@ public class ATNSerializer { public ATN atn; private List<String> tokenNames; + private interface CodePointSerializer { + void serializeCodePoint(IntegerList data, int cp); + } + public ATNSerializer(ATN atn) { assert atn.grammarType != null; this.atn = atn; @@ -47,9 +53,11 @@ public class ATNSerializer { * (args are token type,actionIndex in lexer else 0,0) * num modes, * mode-0-start-state, mode-1-start-state, ... (parser has 0 modes) - * num sets - * set-0-interval-count intervals, set-1-interval-count intervals, ... - * num total edges, + * num unicode-bmp-sets + * bmp-set-0-interval-count intervals, bmp-set-1-interval-count intervals, ... + * num unicode-smp-sets + * smp-set-0-interval-count intervals, smp-set-1-interval-count intervals, ... + * num total edges, * src, trg, edge-type, edge arg1, optional edge arg2 (present always), ... * num decisions, * decision-0-start-state, decision-1-start-state, ... @@ -66,8 +74,10 @@ public class ATNSerializer { data.add(atn.maxTokenType); int nedges = 0; - Map<IntervalSet, Integer> setIndices = new HashMap<IntervalSet, Integer>(); - List<IntervalSet> sets = new ArrayList<IntervalSet>(); + // Note that we use a LinkedHashMap as a set to + // maintain insertion order while deduplicating + // entries with the same key. + Map<IntervalSet, Boolean> sets = new LinkedHashMap<>(); // dump states, count edges and collect sets while doing so IntegerList nonGreedyStates = new IntegerList(); @@ -114,10 +124,7 @@ public class ATNSerializer { int edgeType = Transition.serializationTypes.get(t.getClass()); if ( edgeType == Transition.SET || edgeType == Transition.NOT_SET ) { SetTransition st = (SetTransition)t; - if (!setIndices.containsKey(st.set)) { - sets.add(st.set); - setIndices.put(st.set, sets.size() - 1); - } + sets.put(st.set, true); } } } @@ -156,34 +163,41 @@ public class ATNSerializer { data.add(modeStartState.stateNumber); } } - - int nsets = sets.size(); - data.add(nsets); - for (IntervalSet set : sets) { - boolean containsEof = set.contains(Token.EOF); - if (containsEof && set.getIntervals().get(0).b == Token.EOF) { - data.add(set.getIntervals().size() - 1); + List<IntervalSet> bmpSets = new ArrayList<>(); + List<IntervalSet> smpSets = new ArrayList<>(); + for (IntervalSet set : sets.keySet()) { + if (set.getMaxElement() <= Character.MAX_VALUE) { + bmpSets.add(set); } else { - data.add(set.getIntervals().size()); + smpSets.add(set); } - - data.add(containsEof ? 1 : 0); - for (Interval I : set.getIntervals()) { - if (I.a == Token.EOF) { - if (I.b == Token.EOF) { - continue; - } - else { - data.add(0); - } + } + serializeSets( + data, + bmpSets, + new CodePointSerializer() { + @Override + public void serializeCodePoint(IntegerList data, int cp) { + data.add(cp); } - else { - data.add(I.a); + }); + serializeSets( + data, + smpSets, + new CodePointSerializer() { + @Override + public void serializeCodePoint(IntegerList data, int cp) { + serializeInt(data, cp); } - - data.add(I.b); - } + }); + Map<IntervalSet, Integer> setIndices = new HashMap<>(); + int setIndex = 0; + for (IntervalSet bmpSet : bmpSets) { + setIndices.put(bmpSet, setIndex++); + } + for (IntervalSet smpSet : smpSets) { + setIndices.put(smpSet, setIndex++); } data.add(nedges); @@ -340,6 +354,7 @@ public class ATNSerializer { } } + // Note: This value shifting loop is documented in ATNDeserializer. // don't adjust the first value since that's the version number for (int i = 1; i < data.size(); i++) { if (data.get(i) < Character.MIN_VALUE || data.get(i) > Character.MAX_VALUE) { @@ -358,6 +373,42 @@ public class ATNSerializer { return data; } + private static void serializeSets( + IntegerList data, + Collection<IntervalSet> sets, + CodePointSerializer codePointSerializer) + { + int nSets = sets.size(); + data.add(nSets); + + for (IntervalSet set : sets) { + boolean containsEof = set.contains(Token.EOF); + if (containsEof && set.getIntervals().get(0).b == Token.EOF) { + data.add(set.getIntervals().size() - 1); + } + else { + data.add(set.getIntervals().size()); + } + + data.add(containsEof ? 1 : 0); + for (Interval I : set.getIntervals()) { + if (I.a == Token.EOF) { + if (I.b == Token.EOF) { + continue; + } + else { + codePointSerializer.serializeCodePoint(data, 0); + } + } + else { + codePointSerializer.serializeCodePoint(data, I.a); + } + + codePointSerializer.serializeCodePoint(data, I.b); + } + } + } + public String decode(char[] data) { data = data.clone(); // don't adjust the first value since that's the version number @@ -436,25 +487,10 @@ public class ATNSerializer { int s = ATNDeserializer.toInt(data[p++]); buf.append("mode ").append(i).append(":").append(s).append('\n'); } - int nsets = ATNDeserializer.toInt(data[p++]); - for (int i=0; i<nsets; i++) { - int nintervals = ATNDeserializer.toInt(data[p++]); - buf.append(i).append(":"); - boolean containsEof = data[p++] != 0; - if (containsEof) { - buf.append(getTokenName(Token.EOF)); - } - - for (int j=0; j<nintervals; j++) { - if ( containsEof || j>0 ) { - buf.append(", "); - } - - buf.append(getTokenName(ATNDeserializer.toInt(data[p]))).append("..").append(getTokenName(ATNDeserializer.toInt(data[p + 1]))); - p += 2; - } - buf.append("\n"); - } + int numBMPSets = ATNDeserializer.toInt(data[p++]); + p = appendSets(buf, data, p, numBMPSets, 0, ATNDeserializer.getUnicodeDeserializer(ATNDeserializer.UnicodeDeserializingMode.UNICODE_BMP)); + int numSMPSets = ATNDeserializer.toInt(data[p++]); + p = appendSets(buf, data, p, numSMPSets, numBMPSets, ATNDeserializer.getUnicodeDeserializer(ATNDeserializer.UnicodeDeserializingMode.UNICODE_SMP)); int nedges = ATNDeserializer.toInt(data[p++]); for (int i=0; i<nedges; i++) { int src = ATNDeserializer.toInt(data[p]); @@ -490,6 +526,31 @@ public class ATNSerializer { return buf.toString(); } + private int appendSets(StringBuilder buf, char[] data, int p, int nsets, int setIndexOffset, ATNDeserializer.UnicodeDeserializer unicodeDeserializer) { + for (int i=0; i<nsets; i++) { + int nintervals = ATNDeserializer.toInt(data[p++]); + buf.append(i+setIndexOffset).append(":"); + boolean containsEof = data[p++] != 0; + if (containsEof) { + buf.append(getTokenName(Token.EOF)); + } + + for (int j=0; j<nintervals; j++) { + if ( containsEof || j>0 ) { + buf.append(", "); + } + + int a = unicodeDeserializer.readUnicode(data, p); + p += unicodeDeserializer.size(); + int b = unicodeDeserializer.readUnicode(data, p); + p += unicodeDeserializer.size(); + buf.append(getTokenName(a)).append("..").append(getTokenName(b)); + } + buf.append("\n"); + } + return p; + } + public String getTokenName(int t) { if ( t==-1 ) return "EOF"; diff --git a/runtime/Java/src/org/antlr/v4/runtime/atn/ATNSimulator.java b/runtime/Java/src/org/antlr/v4/runtime/atn/ATNSimulator.java index 5a1abe0..752a9a7 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/atn/ATNSimulator.java +++ b/runtime/Java/src/org/antlr/v4/runtime/atn/ATNSimulator.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/runtime/Java/src/org/antlr/v4/runtime/atn/ATNState.java b/runtime/Java/src/org/antlr/v4/runtime/atn/ATNState.java index deb845d..955a25d 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/atn/ATNState.java +++ b/runtime/Java/src/org/antlr/v4/runtime/atn/ATNState.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/runtime/Java/src/org/antlr/v4/runtime/atn/ATNType.java b/runtime/Java/src/org/antlr/v4/runtime/atn/ATNType.java index 2ac99b1..613b566 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/atn/ATNType.java +++ b/runtime/Java/src/org/antlr/v4/runtime/atn/ATNType.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/runtime/Java/src/org/antlr/v4/runtime/atn/AbstractPredicateTransition.java b/runtime/Java/src/org/antlr/v4/runtime/atn/AbstractPredicateTransition.java index d00a793..b5aee23 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/atn/AbstractPredicateTransition.java +++ b/runtime/Java/src/org/antlr/v4/runtime/atn/AbstractPredicateTransition.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/runtime/Java/src/org/antlr/v4/runtime/atn/ActionTransition.java b/runtime/Java/src/org/antlr/v4/runtime/atn/ActionTransition.java index e89f5c2..1d35b21 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/atn/ActionTransition.java +++ b/runtime/Java/src/org/antlr/v4/runtime/atn/ActionTransition.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/runtime/Java/src/org/antlr/v4/runtime/atn/AmbiguityInfo.java b/runtime/Java/src/org/antlr/v4/runtime/atn/AmbiguityInfo.java index 647f4b1..8bc2323 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/atn/AmbiguityInfo.java +++ b/runtime/Java/src/org/antlr/v4/runtime/atn/AmbiguityInfo.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/runtime/Java/src/org/antlr/v4/runtime/atn/ArrayPredictionContext.java b/runtime/Java/src/org/antlr/v4/runtime/atn/ArrayPredictionContext.java index 178b085..9e9dcdd 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/atn/ArrayPredictionContext.java +++ b/runtime/Java/src/org/antlr/v4/runtime/atn/ArrayPredictionContext.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/runtime/Java/src/org/antlr/v4/runtime/atn/AtomTransition.java b/runtime/Java/src/org/antlr/v4/runtime/atn/AtomTransition.java index 446f5b3..56cf6e5 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/atn/AtomTransition.java +++ b/runtime/Java/src/org/antlr/v4/runtime/atn/AtomTransition.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/runtime/Java/src/org/antlr/v4/runtime/atn/BasicBlockStartState.java b/runtime/Java/src/org/antlr/v4/runtime/atn/BasicBlockStartState.java index 84216ae..d706162 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/atn/BasicBlockStartState.java +++ b/runtime/Java/src/org/antlr/v4/runtime/atn/BasicBlockStartState.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/runtime/Java/src/org/antlr/v4/runtime/atn/BasicState.java b/runtime/Java/src/org/antlr/v4/runtime/atn/BasicState.java index b139cea..d7b20bc 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/atn/BasicState.java +++ b/runtime/Java/src/org/antlr/v4/runtime/atn/BasicState.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/runtime/Java/src/org/antlr/v4/runtime/atn/BlockEndState.java b/runtime/Java/src/org/antlr/v4/runtime/atn/BlockEndState.java index c8cc871..fae31cc 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/atn/BlockEndState.java +++ b/runtime/Java/src/org/antlr/v4/runtime/atn/BlockEndState.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/runtime/Java/src/org/antlr/v4/runtime/atn/BlockStartState.java b/runtime/Java/src/org/antlr/v4/runtime/atn/BlockStartState.java index 0fb8380..68f03d5 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/atn/BlockStartState.java +++ b/runtime/Java/src/org/antlr/v4/runtime/atn/BlockStartState.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/runtime/Java/src/org/antlr/v4/runtime/atn/CodePointTransitions.java b/runtime/Java/src/org/antlr/v4/runtime/atn/CodePointTransitions.java new file mode 100644 index 0000000..7aedfc4 --- /dev/null +++ b/runtime/Java/src/org/antlr/v4/runtime/atn/CodePointTransitions.java @@ -0,0 +1,52 @@ +/* + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. + * Use of this file is governed by the BSD 3-clause license that + * can be found in the LICENSE.txt file in the project root. + */ + +package org.antlr.v4.runtime.atn; + +import org.antlr.v4.runtime.misc.IntervalSet; + +/** + * Utility class to create {@link AtomTransition}, {@link RangeTransition}, + * and {@link SetTransition} appropriately based on the range of the input. + * + * To keep the serialized ATN size small, we only inline atom and + * range transitions for Unicode code points <= U+FFFF. + * + * Whenever we encounter a Unicode code point > U+FFFF, we represent that + * as a set transition (even if it is logically an atom or a range). + */ +public abstract class CodePointTransitions { + /** + * If {@code codePoint} is <= U+FFFF, returns a new {@link AtomTransition}. + * Otherwise, returns a new {@link SetTransition}. + */ + public static Transition createWithCodePoint(ATNState target, int codePoint) { + if (Character.isSupplementaryCodePoint(codePoint)) { + return new SetTransition(target, IntervalSet.of(codePoint)); + } + else { + return new AtomTransition(target, codePoint); + } + } + + /** + * If {@code codePointFrom} and {@code codePointTo} are both + * <= U+FFFF, returns a new {@link RangeTransition}. + * Otherwise, returns a new {@link SetTransition}. + */ + public static Transition createWithCodePointRange( + ATNState target, + int codePointFrom, + int codePointTo) { + if (Character.isSupplementaryCodePoint(codePointFrom) || + Character.isSupplementaryCodePoint(codePointTo)) { + return new SetTransition(target, IntervalSet.of(codePointFrom, codePointTo)); + } + else { + return new RangeTransition(target, codePointFrom, codePointTo); + } + } +} diff --git a/runtime/Java/src/org/antlr/v4/runtime/atn/ContextSensitivityInfo.java b/runtime/Java/src/org/antlr/v4/runtime/atn/ContextSensitivityInfo.java index e825a74..f6e8413 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/atn/ContextSensitivityInfo.java +++ b/runtime/Java/src/org/antlr/v4/runtime/atn/ContextSensitivityInfo.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/runtime/Java/src/org/antlr/v4/runtime/atn/DecisionEventInfo.java b/runtime/Java/src/org/antlr/v4/runtime/atn/DecisionEventInfo.java index a0a117e..e82ca7a 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/atn/DecisionEventInfo.java +++ b/runtime/Java/src/org/antlr/v4/runtime/atn/DecisionEventInfo.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/runtime/Java/src/org/antlr/v4/runtime/atn/DecisionInfo.java b/runtime/Java/src/org/antlr/v4/runtime/atn/DecisionInfo.java index 98ff1c3..7f0bf66 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/atn/DecisionInfo.java +++ b/runtime/Java/src/org/antlr/v4/runtime/atn/DecisionInfo.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/runtime/Java/src/org/antlr/v4/runtime/atn/DecisionState.java b/runtime/Java/src/org/antlr/v4/runtime/atn/DecisionState.java index e4c710e..3ef93d2 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/atn/DecisionState.java +++ b/runtime/Java/src/org/antlr/v4/runtime/atn/DecisionState.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/runtime/Java/src/org/antlr/v4/runtime/atn/EmptyPredictionContext.java b/runtime/Java/src/org/antlr/v4/runtime/atn/EmptyPredictionContext.java index a7c66e9..a11ff15 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/atn/EmptyPredictionContext.java +++ b/runtime/Java/src/org/antlr/v4/runtime/atn/EmptyPredictionContext.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/runtime/Java/src/org/antlr/v4/runtime/atn/EpsilonTransition.java b/runtime/Java/src/org/antlr/v4/runtime/atn/EpsilonTransition.java index ea1b639..ab0b2db 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/atn/EpsilonTransition.java +++ b/runtime/Java/src/org/antlr/v4/runtime/atn/EpsilonTransition.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/runtime/Java/src/org/antlr/v4/runtime/atn/ErrorInfo.java b/runtime/Java/src/org/antlr/v4/runtime/atn/ErrorInfo.java index 35d9277..7b78e83 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/atn/ErrorInfo.java +++ b/runtime/Java/src/org/antlr/v4/runtime/atn/ErrorInfo.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/runtime/Java/src/org/antlr/v4/runtime/atn/LL1Analyzer.java b/runtime/Java/src/org/antlr/v4/runtime/atn/LL1Analyzer.java index 2df50c3..8d5117b 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/atn/LL1Analyzer.java +++ b/runtime/Java/src/org/antlr/v4/runtime/atn/LL1Analyzer.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/runtime/Java/src/org/antlr/v4/runtime/atn/LexerATNConfig.java b/runtime/Java/src/org/antlr/v4/runtime/atn/LexerATNConfig.java index db7b8e8..ad8766c 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/atn/LexerATNConfig.java +++ b/runtime/Java/src/org/antlr/v4/runtime/atn/LexerATNConfig.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/runtime/Java/src/org/antlr/v4/runtime/atn/LexerATNSimulator.java b/runtime/Java/src/org/antlr/v4/runtime/atn/LexerATNSimulator.java index 81f7f64..ec74ce5 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/atn/LexerATNSimulator.java +++ b/runtime/Java/src/org/antlr/v4/runtime/atn/LexerATNSimulator.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ @@ -372,7 +372,7 @@ public class LexerATNSimulator extends ATNSimulator { protected ATNState getReachableTarget(Transition trans, int t) { - if (trans.matches(t, Character.MIN_VALUE, Character.MAX_VALUE)) { + if (trans.matches(t, Lexer.MIN_CHAR_VALUE, Lexer.MAX_CHAR_VALUE)) { return trans.target; } @@ -544,7 +544,7 @@ public class LexerATNSimulator extends ATNSimulator { case Transition.RANGE: case Transition.SET: if (treatEofAsEpsilon) { - if (t.matches(CharStream.EOF, Character.MIN_VALUE, Character.MAX_VALUE)) { + if (t.matches(CharStream.EOF, Lexer.MIN_CHAR_VALUE, Lexer.MAX_CHAR_VALUE)) { c = new LexerATNConfig(config, t.target); break; } @@ -738,7 +738,8 @@ public class LexerATNSimulator extends ATNSimulator { if ( curChar=='\n' ) { line++; charPositionInLine=0; - } else { + } + else { charPositionInLine++; } input.consume(); diff --git a/runtime/Java/src/org/antlr/v4/runtime/atn/LexerAction.java b/runtime/Java/src/org/antlr/v4/runtime/atn/LexerAction.java index 9657f63..eb10ff4 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/atn/LexerAction.java +++ b/runtime/Java/src/org/antlr/v4/runtime/atn/LexerAction.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/runtime/Java/src/org/antlr/v4/runtime/atn/LexerActionExecutor.java b/runtime/Java/src/org/antlr/v4/runtime/atn/LexerActionExecutor.java index f8c214b..3ec58f7 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/atn/LexerActionExecutor.java +++ b/runtime/Java/src/org/antlr/v4/runtime/atn/LexerActionExecutor.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/runtime/Java/src/org/antlr/v4/runtime/atn/LexerActionType.java b/runtime/Java/src/org/antlr/v4/runtime/atn/LexerActionType.java index e1413d4..4042fd3 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/atn/LexerActionType.java +++ b/runtime/Java/src/org/antlr/v4/runtime/atn/LexerActionType.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/runtime/Java/src/org/antlr/v4/runtime/atn/LexerChannelAction.java b/runtime/Java/src/org/antlr/v4/runtime/atn/LexerChannelAction.java index b37f773..af4419f 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/atn/LexerChannelAction.java +++ b/runtime/Java/src/org/antlr/v4/runtime/atn/LexerChannelAction.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/runtime/Java/src/org/antlr/v4/runtime/atn/LexerCustomAction.java b/runtime/Java/src/org/antlr/v4/runtime/atn/LexerCustomAction.java index 04bec9f..8752b1d 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/atn/LexerCustomAction.java +++ b/runtime/Java/src/org/antlr/v4/runtime/atn/LexerCustomAction.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/runtime/Java/src/org/antlr/v4/runtime/atn/LexerIndexedCustomAction.java b/runtime/Java/src/org/antlr/v4/runtime/atn/LexerIndexedCustomAction.java index 9fcd797..a78c823 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/atn/LexerIndexedCustomAction.java +++ b/runtime/Java/src/org/antlr/v4/runtime/atn/LexerIndexedCustomAction.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/runtime/Java/src/org/antlr/v4/runtime/atn/LexerModeAction.java b/runtime/Java/src/org/antlr/v4/runtime/atn/LexerModeAction.java index a7afcca..1663d6c 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/atn/LexerModeAction.java +++ b/runtime/Java/src/org/antlr/v4/runtime/atn/LexerModeAction.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/runtime/Java/src/org/antlr/v4/runtime/atn/LexerMoreAction.java b/runtime/Java/src/org/antlr/v4/runtime/atn/LexerMoreAction.java index ee1543c..5e0e386 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/atn/LexerMoreAction.java +++ b/runtime/Java/src/org/antlr/v4/runtime/atn/LexerMoreAction.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/runtime/Java/src/org/antlr/v4/runtime/atn/LexerPopModeAction.java b/runtime/Java/src/org/antlr/v4/runtime/atn/LexerPopModeAction.java index 36e7445..afdd2bc 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/atn/LexerPopModeAction.java +++ b/runtime/Java/src/org/antlr/v4/runtime/atn/LexerPopModeAction.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/runtime/Java/src/org/antlr/v4/runtime/atn/LexerPushModeAction.java b/runtime/Java/src/org/antlr/v4/runtime/atn/LexerPushModeAction.java index eb1c138..844c620 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/atn/LexerPushModeAction.java +++ b/runtime/Java/src/org/antlr/v4/runtime/atn/LexerPushModeAction.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/runtime/Java/src/org/antlr/v4/runtime/atn/LexerSkipAction.java b/runtime/Java/src/org/antlr/v4/runtime/atn/LexerSkipAction.java index 762477b..72b7bb6 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/atn/LexerSkipAction.java +++ b/runtime/Java/src/org/antlr/v4/runtime/atn/LexerSkipAction.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/runtime/Java/src/org/antlr/v4/runtime/atn/LexerTypeAction.java b/runtime/Java/src/org/antlr/v4/runtime/atn/LexerTypeAction.java index 9f30ecc..4151d2e 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/atn/LexerTypeAction.java +++ b/runtime/Java/src/org/antlr/v4/runtime/atn/LexerTypeAction.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/runtime/Java/src/org/antlr/v4/runtime/atn/LookaheadEventInfo.java b/runtime/Java/src/org/antlr/v4/runtime/atn/LookaheadEventInfo.java index 4f929fd..90e275e 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/atn/LookaheadEventInfo.java +++ b/runtime/Java/src/org/antlr/v4/runtime/atn/LookaheadEventInfo.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/runtime/Java/src/org/antlr/v4/runtime/atn/LoopEndState.java b/runtime/Java/src/org/antlr/v4/runtime/atn/LoopEndState.java index e06fcbf..d8213a9 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/atn/LoopEndState.java +++ b/runtime/Java/src/org/antlr/v4/runtime/atn/LoopEndState.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/runtime/Java/src/org/antlr/v4/runtime/atn/NotSetTransition.java b/runtime/Java/src/org/antlr/v4/runtime/atn/NotSetTransition.java index 36f0cbb..4599e7b 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/atn/NotSetTransition.java +++ b/runtime/Java/src/org/antlr/v4/runtime/atn/NotSetTransition.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/runtime/Java/src/org/antlr/v4/runtime/atn/OrderedATNConfigSet.java b/runtime/Java/src/org/antlr/v4/runtime/atn/OrderedATNConfigSet.java index 5b3dc3d..dbaa2ca 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/atn/OrderedATNConfigSet.java +++ b/runtime/Java/src/org/antlr/v4/runtime/atn/OrderedATNConfigSet.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/runtime/Java/src/org/antlr/v4/runtime/atn/ParseInfo.java b/runtime/Java/src/org/antlr/v4/runtime/atn/ParseInfo.java index 3f8bd88..1e83ace 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/atn/ParseInfo.java +++ b/runtime/Java/src/org/antlr/v4/runtime/atn/ParseInfo.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/runtime/Java/src/org/antlr/v4/runtime/atn/ParserATNSimulator.java b/runtime/Java/src/org/antlr/v4/runtime/atn/ParserATNSimulator.java index 36965ad..76524eb 100755 --- a/runtime/Java/src/org/antlr/v4/runtime/atn/ParserATNSimulator.java +++ b/runtime/Java/src/org/antlr/v4/runtime/atn/ParserATNSimulator.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ @@ -270,7 +270,7 @@ public class ParserATNSimulator extends ATNSimulator { public static final boolean retry_debug = false; /** Just in case this optimization is bad, add an ENV variable to turn it off */ - public static final boolean TURN_OFF_LR_LOOP_ENTRY_BRANCH_OPT = Boolean.parseBoolean(System.getenv("TURN_OFF_LR_LOOP_ENTRY_BRANCH_OPT")); + public static final boolean TURN_OFF_LR_LOOP_ENTRY_BRANCH_OPT = Boolean.parseBoolean(getSafeEnv("TURN_OFF_LR_LOOP_ENTRY_BRANCH_OPT")); protected final Parser parser; @@ -1541,11 +1541,6 @@ public class ParserATNSimulator extends ATNSimulator { ATNConfig c = getEpsilonTarget(config, t, continueCollecting, depth == 0, fullCtx, treatEofAsEpsilon); if ( c!=null ) { - if (!t.isEpsilon() && !closureBusy.add(c)) { - // avoid infinite recursion for EOF* and EOF+ - continue; - } - int newDepth = depth; if ( config.state instanceof RuleStopState) { assert !fullCtx; @@ -1555,11 +1550,6 @@ public class ParserATNSimulator extends ATNSimulator { // come in handy and we avoid evaluating context dependent // preds if this is > 0. - if (!closureBusy.add(c)) { - // avoid infinite recursion for right-recursive rules - continue; - } - if (_dfa != null && _dfa.isPrecedenceDfa()) { int outermostPrecedenceReturn = ((EpsilonTransition)t).outermostPrecedenceReturn(); if (outermostPrecedenceReturn == _dfa.atnStartState.ruleIndex) { @@ -1568,15 +1558,28 @@ public class ParserATNSimulator extends ATNSimulator { } c.reachesIntoOuterContext++; + + if (!closureBusy.add(c)) { + // avoid infinite recursion for right-recursive rules + continue; + } + configs.dipsIntoOuterContext = true; // TODO: can remove? only care when we add to set per middle of this method assert newDepth > Integer.MIN_VALUE; newDepth--; if ( debug ) System.out.println("dips into outer ctx: "+c); } - else if (t instanceof RuleTransition) { - // latch when newDepth goes negative - once we step out of the entry context we can't return - if (newDepth >= 0) { - newDepth++; + else { + if (!t.isEpsilon() && !closureBusy.add(c)) { + // avoid infinite recursion for EOF* and EOF+ + continue; + } + + if (t instanceof RuleTransition) { + // latch when newDepth goes negative - once we step out of the entry context we can't return + if (newDepth >= 0) { + newDepth++; + } } } @@ -2178,4 +2181,14 @@ public class ParserATNSimulator extends ATNSimulator { public Parser getParser() { return parser; } + + public static String getSafeEnv(String envName) { + try { + return System.getenv(envName); + } + catch(SecurityException e) { + // use the default value + } + return null; + } } diff --git a/runtime/Java/src/org/antlr/v4/runtime/atn/PlusBlockStartState.java b/runtime/Java/src/org/antlr/v4/runtime/atn/PlusBlockStartState.java index 095a48a..025ae46 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/atn/PlusBlockStartState.java +++ b/runtime/Java/src/org/antlr/v4/runtime/atn/PlusBlockStartState.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/runtime/Java/src/org/antlr/v4/runtime/atn/PlusLoopbackState.java b/runtime/Java/src/org/antlr/v4/runtime/atn/PlusLoopbackState.java index 96b5d16..77baf36 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/atn/PlusLoopbackState.java +++ b/runtime/Java/src/org/antlr/v4/runtime/atn/PlusLoopbackState.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/runtime/Java/src/org/antlr/v4/runtime/atn/PrecedencePredicateTransition.java b/runtime/Java/src/org/antlr/v4/runtime/atn/PrecedencePredicateTransition.java index 546bafd..9823c63 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/atn/PrecedencePredicateTransition.java +++ b/runtime/Java/src/org/antlr/v4/runtime/atn/PrecedencePredicateTransition.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/runtime/Java/src/org/antlr/v4/runtime/atn/PredicateEvalInfo.java b/runtime/Java/src/org/antlr/v4/runtime/atn/PredicateEvalInfo.java index d2a73b4..3ee83e1 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/atn/PredicateEvalInfo.java +++ b/runtime/Java/src/org/antlr/v4/runtime/atn/PredicateEvalInfo.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/runtime/Java/src/org/antlr/v4/runtime/atn/PredicateTransition.java b/runtime/Java/src/org/antlr/v4/runtime/atn/PredicateTransition.java index c4381bd..cc4058e 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/atn/PredicateTransition.java +++ b/runtime/Java/src/org/antlr/v4/runtime/atn/PredicateTransition.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/runtime/Java/src/org/antlr/v4/runtime/atn/PredictionContext.java b/runtime/Java/src/org/antlr/v4/runtime/atn/PredictionContext.java index 8a23d34..c55ba09 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/atn/PredictionContext.java +++ b/runtime/Java/src/org/antlr/v4/runtime/atn/PredictionContext.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/runtime/Java/src/org/antlr/v4/runtime/atn/PredictionContextCache.java b/runtime/Java/src/org/antlr/v4/runtime/atn/PredictionContextCache.java index 62c9196..11711ee 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/atn/PredictionContextCache.java +++ b/runtime/Java/src/org/antlr/v4/runtime/atn/PredictionContextCache.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/runtime/Java/src/org/antlr/v4/runtime/atn/PredictionMode.java b/runtime/Java/src/org/antlr/v4/runtime/atn/PredictionMode.java index b0ca601..51a51d2 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/atn/PredictionMode.java +++ b/runtime/Java/src/org/antlr/v4/runtime/atn/PredictionMode.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/runtime/Java/src/org/antlr/v4/runtime/atn/ProfilingATNSimulator.java b/runtime/Java/src/org/antlr/v4/runtime/atn/ProfilingATNSimulator.java index 51ded0a..dd29f02 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/atn/ProfilingATNSimulator.java +++ b/runtime/Java/src/org/antlr/v4/runtime/atn/ProfilingATNSimulator.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/runtime/Java/src/org/antlr/v4/runtime/atn/RangeTransition.java b/runtime/Java/src/org/antlr/v4/runtime/atn/RangeTransition.java index 92001cc..4eee283 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/atn/RangeTransition.java +++ b/runtime/Java/src/org/antlr/v4/runtime/atn/RangeTransition.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ @@ -34,6 +34,11 @@ public final class RangeTransition extends Transition { @Override public String toString() { - return "'"+(char)from+"'..'"+(char)to+"'"; + return new StringBuilder("'") + .appendCodePoint(from) + .append("'..'") + .appendCodePoint(to) + .append("'") + .toString(); } } diff --git a/runtime/Java/src/org/antlr/v4/runtime/atn/RuleStartState.java b/runtime/Java/src/org/antlr/v4/runtime/atn/RuleStartState.java index e7c9629..9265ef5 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/atn/RuleStartState.java +++ b/runtime/Java/src/org/antlr/v4/runtime/atn/RuleStartState.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/runtime/Java/src/org/antlr/v4/runtime/atn/RuleStopState.java b/runtime/Java/src/org/antlr/v4/runtime/atn/RuleStopState.java index b72ee31..fdaf9b3 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/atn/RuleStopState.java +++ b/runtime/Java/src/org/antlr/v4/runtime/atn/RuleStopState.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/runtime/Java/src/org/antlr/v4/runtime/atn/RuleTransition.java b/runtime/Java/src/org/antlr/v4/runtime/atn/RuleTransition.java index ff855d7..c3c0cee 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/atn/RuleTransition.java +++ b/runtime/Java/src/org/antlr/v4/runtime/atn/RuleTransition.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/runtime/Java/src/org/antlr/v4/runtime/atn/SemanticContext.java b/runtime/Java/src/org/antlr/v4/runtime/atn/SemanticContext.java index 922e931..3b888e7 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/atn/SemanticContext.java +++ b/runtime/Java/src/org/antlr/v4/runtime/atn/SemanticContext.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/runtime/Java/src/org/antlr/v4/runtime/atn/SetTransition.java b/runtime/Java/src/org/antlr/v4/runtime/atn/SetTransition.java index 86904bd..a72708a 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/atn/SetTransition.java +++ b/runtime/Java/src/org/antlr/v4/runtime/atn/SetTransition.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/runtime/Java/src/org/antlr/v4/runtime/atn/SingletonPredictionContext.java b/runtime/Java/src/org/antlr/v4/runtime/atn/SingletonPredictionContext.java index 582c6a6..ca5ea66 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/atn/SingletonPredictionContext.java +++ b/runtime/Java/src/org/antlr/v4/runtime/atn/SingletonPredictionContext.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/runtime/Java/src/org/antlr/v4/runtime/atn/StarBlockStartState.java b/runtime/Java/src/org/antlr/v4/runtime/atn/StarBlockStartState.java index 3321e0a..5996806 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/atn/StarBlockStartState.java +++ b/runtime/Java/src/org/antlr/v4/runtime/atn/StarBlockStartState.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/runtime/Java/src/org/antlr/v4/runtime/atn/StarLoopEntryState.java b/runtime/Java/src/org/antlr/v4/runtime/atn/StarLoopEntryState.java index aa1fe02..5951831 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/atn/StarLoopEntryState.java +++ b/runtime/Java/src/org/antlr/v4/runtime/atn/StarLoopEntryState.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/runtime/Java/src/org/antlr/v4/runtime/atn/StarLoopbackState.java b/runtime/Java/src/org/antlr/v4/runtime/atn/StarLoopbackState.java index f7ad420..4e2b449 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/atn/StarLoopbackState.java +++ b/runtime/Java/src/org/antlr/v4/runtime/atn/StarLoopbackState.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/runtime/Java/src/org/antlr/v4/runtime/atn/TokensStartState.java b/runtime/Java/src/org/antlr/v4/runtime/atn/TokensStartState.java index d1be204..846b6ec 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/atn/TokensStartState.java +++ b/runtime/Java/src/org/antlr/v4/runtime/atn/TokensStartState.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/runtime/Java/src/org/antlr/v4/runtime/atn/Transition.java b/runtime/Java/src/org/antlr/v4/runtime/atn/Transition.java index 04b368f..8c0fbc5 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/atn/Transition.java +++ b/runtime/Java/src/org/antlr/v4/runtime/atn/Transition.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/runtime/Java/src/org/antlr/v4/runtime/atn/WildcardTransition.java b/runtime/Java/src/org/antlr/v4/runtime/atn/WildcardTransition.java index 1634196..2707065 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/atn/WildcardTransition.java +++ b/runtime/Java/src/org/antlr/v4/runtime/atn/WildcardTransition.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/runtime/Java/src/org/antlr/v4/runtime/dfa/DFA.java b/runtime/Java/src/org/antlr/v4/runtime/dfa/DFA.java index b508f8b..3c91dab 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/dfa/DFA.java +++ b/runtime/Java/src/org/antlr/v4/runtime/dfa/DFA.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/runtime/Java/src/org/antlr/v4/runtime/dfa/DFASerializer.java b/runtime/Java/src/org/antlr/v4/runtime/dfa/DFASerializer.java index 54b0fab..6504fc6 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/dfa/DFASerializer.java +++ b/runtime/Java/src/org/antlr/v4/runtime/dfa/DFASerializer.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/runtime/Java/src/org/antlr/v4/runtime/dfa/DFAState.java b/runtime/Java/src/org/antlr/v4/runtime/dfa/DFAState.java index 5b74f6c..709094a 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/dfa/DFAState.java +++ b/runtime/Java/src/org/antlr/v4/runtime/dfa/DFAState.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/runtime/Java/src/org/antlr/v4/runtime/dfa/LexerDFASerializer.java b/runtime/Java/src/org/antlr/v4/runtime/dfa/LexerDFASerializer.java index d1d0cca..83d4f8c 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/dfa/LexerDFASerializer.java +++ b/runtime/Java/src/org/antlr/v4/runtime/dfa/LexerDFASerializer.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ @@ -16,6 +16,9 @@ public class LexerDFASerializer extends DFASerializer { @Override protected String getEdgeLabel(int i) { - return "'"+(char)i+"'"; + return new StringBuilder("'") + .appendCodePoint(i) + .append("'") + .toString(); } } diff --git a/runtime/Java/src/org/antlr/v4/runtime/misc/AbstractEqualityComparator.java b/runtime/Java/src/org/antlr/v4/runtime/misc/AbstractEqualityComparator.java index c149d9e..a2bae78 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/misc/AbstractEqualityComparator.java +++ b/runtime/Java/src/org/antlr/v4/runtime/misc/AbstractEqualityComparator.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/runtime/Java/src/org/antlr/v4/runtime/misc/Array2DHashSet.java b/runtime/Java/src/org/antlr/v4/runtime/misc/Array2DHashSet.java index 4595275..9ec8566 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/misc/Array2DHashSet.java +++ b/runtime/Java/src/org/antlr/v4/runtime/misc/Array2DHashSet.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/runtime/Java/src/org/antlr/v4/runtime/misc/DoubleKeyMap.java b/runtime/Java/src/org/antlr/v4/runtime/misc/DoubleKeyMap.java index a612347..a88d17d 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/misc/DoubleKeyMap.java +++ b/runtime/Java/src/org/antlr/v4/runtime/misc/DoubleKeyMap.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/runtime/Java/src/org/antlr/v4/runtime/misc/EqualityComparator.java b/runtime/Java/src/org/antlr/v4/runtime/misc/EqualityComparator.java index 0fee89b..679c6a6 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/misc/EqualityComparator.java +++ b/runtime/Java/src/org/antlr/v4/runtime/misc/EqualityComparator.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/runtime/Java/src/org/antlr/v4/runtime/misc/FlexibleHashMap.java b/runtime/Java/src/org/antlr/v4/runtime/misc/FlexibleHashMap.java index a05008e..3c49eb2 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/misc/FlexibleHashMap.java +++ b/runtime/Java/src/org/antlr/v4/runtime/misc/FlexibleHashMap.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/runtime/Java/src/org/antlr/v4/runtime/misc/IntSet.java b/runtime/Java/src/org/antlr/v4/runtime/misc/IntSet.java index 80d908b..53db074 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/misc/IntSet.java +++ b/runtime/Java/src/org/antlr/v4/runtime/misc/IntSet.java @@ -1,13 +1,11 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ package org.antlr.v4.runtime.misc; -import org.antlr.v4.runtime.Token; - import java.util.List; /** @@ -131,15 +129,6 @@ public interface IntSet { boolean equals(Object obj); /** - * Returns the single value contained in the set, if {@link #size} is 1; - * otherwise, returns {@link Token#INVALID_TYPE}. - * - * @return the single value contained in the set, if {@link #size} is 1; - * otherwise, returns {@link Token#INVALID_TYPE}. - */ - int getSingleElement(); - - /** * Returns {@code true} if the set contains the specified element. * * @param el The element to check for. diff --git a/runtime/Java/src/org/antlr/v4/runtime/misc/IntegerList.java b/runtime/Java/src/org/antlr/v4/runtime/misc/IntegerList.java index 21d5bcd..d6af911 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/misc/IntegerList.java +++ b/runtime/Java/src/org/antlr/v4/runtime/misc/IntegerList.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ @@ -36,7 +36,8 @@ public class IntegerList { if (capacity == 0) { _data = EMPTY_DATA; - } else { + } + else { _data = new int[capacity]; } } @@ -258,7 +259,8 @@ public class IntegerList { int newLength; if (_data.length == 0) { newLength = INITIAL_SIZE; - } else { + } + else { newLength = _data.length; } @@ -272,4 +274,39 @@ public class IntegerList { _data = Arrays.copyOf(_data, newLength); } + /** Convert the list to a UTF-16 encoded char array. If all values are less + * than the 0xFFFF 16-bit code point limit then this is just a char array + * of 16-bit char as usual. For values in the supplementary range, encode + * them as two UTF-16 code units. + */ + public final char[] toCharArray() { + // Optimize for the common case (all data values are + // < 0xFFFF) to avoid an extra scan + char[] resultArray = new char[_size]; + int resultIdx = 0; + boolean calculatedPreciseResultSize = false; + for (int i = 0; i < _size; i++) { + int codePoint = _data[i]; + // Calculate the precise result size if we encounter + // a code point > 0xFFFF + if (!calculatedPreciseResultSize && + Character.isSupplementaryCodePoint(codePoint)) { + resultArray = Arrays.copyOf(resultArray, charArraySize()); + calculatedPreciseResultSize = true; + } + // This will throw IllegalArgumentException if + // the code point is not a valid Unicode code point + int charsWritten = Character.toChars(codePoint, resultArray, resultIdx); + resultIdx += charsWritten; + } + return resultArray; + } + + private int charArraySize() { + int result = 0; + for (int i = 0; i < _size; i++) { + result += Character.charCount(_data[i]); + } + return result; + } } diff --git a/runtime/Java/src/org/antlr/v4/runtime/misc/IntegerStack.java b/runtime/Java/src/org/antlr/v4/runtime/misc/IntegerStack.java index 5250a72..a8f65d9 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/misc/IntegerStack.java +++ b/runtime/Java/src/org/antlr/v4/runtime/misc/IntegerStack.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/runtime/Java/src/org/antlr/v4/runtime/misc/InterpreterDataReader.java b/runtime/Java/src/org/antlr/v4/runtime/misc/InterpreterDataReader.java new file mode 100644 index 0000000..221fa55 --- /dev/null +++ b/runtime/Java/src/org/antlr/v4/runtime/misc/InterpreterDataReader.java @@ -0,0 +1,141 @@ +/* + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. + * Use of this file is governed by the BSD 3-clause license that + * can be found in the LICENSE.txt file in the project root. + */ + +package org.antlr.v4.runtime.misc; + +import org.antlr.v4.runtime.Vocabulary; +import org.antlr.v4.runtime.VocabularyImpl; +import org.antlr.v4.runtime.atn.ATN; +import org.antlr.v4.runtime.atn.ATNDeserializer; +import org.antlr.v4.runtime.dfa.DFA; + +import java.util.Collection; +import java.util.List; +import java.util.ArrayList; +import java.io.BufferedReader; +import java.io.FileReader; + +// A class to read plain text interpreter data produced by ANTLR. +public class InterpreterDataReader { + + public static class InterpreterData { + ATN atn; + Vocabulary vocabulary; + List<String> ruleNames; + List<String> channels; // Only valid for lexer grammars. + List<String> modes; // ditto + }; + + /** + * The structure of the data file is very simple. Everything is line based with empty lines + * separating the different parts. For lexers the layout is: + * token literal names: + * ... + * + * token symbolic names: + * ... + * + * rule names: + * ... + * + * channel names: + * ... + * + * mode names: + * ... + * + * atn: + * <a single line with comma separated int values> enclosed in a pair of squared brackets. + * + * Data for a parser does not contain channel and mode names. + */ + public static InterpreterData parseFile(String fileName) { + InterpreterData result = new InterpreterData(); + result.ruleNames = new ArrayList<String>(); + + try (BufferedReader br = new BufferedReader(new FileReader(fileName))) { + String line; + List<String> literalNames = new ArrayList<String>(); + List<String> symbolicNames = new ArrayList<String>(); + + line = br.readLine(); + if ( !line.equals("token literal names:") ) + throw new RuntimeException("Unexpected data entry"); + while ((line = br.readLine()) != null) { + if ( line.isEmpty() ) + break; + literalNames.add(line.equals("null") ? "" : line); + } + + line = br.readLine(); + if ( !line.equals("token symbolic names:") ) + throw new RuntimeException("Unexpected data entry"); + while ((line = br.readLine()) != null) { + if ( line.isEmpty() ) + break; + symbolicNames.add(line.equals("null") ? "" : line); + } + + result.vocabulary = new VocabularyImpl(literalNames.toArray(new String[0]), symbolicNames.toArray(new String[0])); + + line = br.readLine(); + if ( !line.equals("rule names:") ) + throw new RuntimeException("Unexpected data entry"); + while ((line = br.readLine()) != null) { + if ( line.isEmpty() ) + break; + result.ruleNames.add(line); + } + + if ( line.equals("channel names:") ) { // Additional lexer data. + result.channels = new ArrayList<String>(); + while ((line = br.readLine()) != null) { + if ( line.isEmpty() ) + break; + result.channels.add(line); + } + + line = br.readLine(); + if ( !line.equals("mode names:") ) + throw new RuntimeException("Unexpected data entry"); + result.modes = new ArrayList<String>(); + while ((line = br.readLine()) != null) { + if ( line.isEmpty() ) + break; + result.modes.add(line); + } + } + + line = br.readLine(); + if ( !line.equals("atn:") ) + throw new RuntimeException("Unexpected data entry"); + line = br.readLine(); + String[] elements = line.split(","); + char[] serializedATN = new char[elements.length]; + + for (int i = 0; i < elements.length; ++i) { + int value; + String element = elements[i]; + if ( element.startsWith("[") ) + value = Integer.parseInt(element.substring(1).trim()); + else if ( element.endsWith("]") ) + value = Integer.parseInt(element.substring(0, element.length() - 1).trim()); + else + value = Integer.parseInt(element.trim()); + serializedATN[i] = (char)value; + } + + ATNDeserializer deserializer = new ATNDeserializer(); + result.atn = deserializer.deserialize(serializedATN); + } + catch (java.io.IOException e) { + // We just swallow the error and return empty objects instead. + } + + return result; + } + +} diff --git a/runtime/Java/src/org/antlr/v4/runtime/misc/Interval.java b/runtime/Java/src/org/antlr/v4/runtime/misc/Interval.java index b5d7d1d..ab3f46b 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/misc/Interval.java +++ b/runtime/Java/src/org/antlr/v4/runtime/misc/Interval.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/runtime/Java/src/org/antlr/v4/runtime/misc/IntervalSet.java b/runtime/Java/src/org/antlr/v4/runtime/misc/IntervalSet.java index 27fb574..3dcb0f3 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/misc/IntervalSet.java +++ b/runtime/Java/src/org/antlr/v4/runtime/misc/IntervalSet.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ @@ -384,30 +384,24 @@ public class IntervalSet implements IntSet { @Override public boolean contains(int el) { int n = intervals.size(); - for (int i = 0; i < n; i++) { - Interval I = intervals.get(i); + int l = 0; + int r = n - 1; + // Binary search for the element in the (sorted, + // disjoint) array of intervals. + while (l <= r) { + int m = (l + r) / 2; + Interval I = intervals.get(m); int a = I.a; int b = I.b; - if ( el<a ) { - break; // list is sorted and el is before this interval; not here - } - if ( el>=a && el<=b ) { - return true; // found in this interval + if ( b<el ) { + l = m + 1; + } else if ( a>el ) { + r = m - 1; + } else { // el >= a && el <= b + return true; } } return false; -/* - for (ListIterator iter = intervals.listIterator(); iter.hasNext();) { - Interval I = (Interval) iter.next(); - if ( el<I.a ) { - break; // list is sorted and el is before this interval; not here - } - if ( el>=I.a && el<=I.b ) { - return true; // found in this interval - } - } - return false; - */ } /** {@inheritDoc} */ @@ -416,41 +410,29 @@ public class IntervalSet implements IntSet { return intervals==null || intervals.isEmpty(); } - /** {@inheritDoc} */ - @Override - public int getSingleElement() { - if ( intervals!=null && intervals.size()==1 ) { - Interval I = intervals.get(0); - if ( I.a == I.b ) { - return I.a; - } - } - return Token.INVALID_TYPE; - } - /** - * Returns the maximum value contained in the set. + * Returns the maximum value contained in the set if not isNil(). * - * @return the maximum value contained in the set. If the set is empty, this - * method returns {@link Token#INVALID_TYPE}. + * @return the maximum value contained in the set. + * @throws RuntimeException if set is empty */ public int getMaxElement() { if ( isNil() ) { - return Token.INVALID_TYPE; + throw new RuntimeException("set is empty"); } Interval last = intervals.get(intervals.size()-1); return last.b; } /** - * Returns the minimum value contained in the set. + * Returns the minimum value contained in the set if not isNil(). * - * @return the minimum value contained in the set. If the set is empty, this - * method returns {@link Token#INVALID_TYPE}. + * @return the minimum value contained in the set. + * @throws RuntimeException if set is empty */ public int getMinElement() { if ( isNil() ) { - return Token.INVALID_TYPE; + throw new RuntimeException("set is empty"); } return intervals.get(0).a; @@ -505,11 +487,11 @@ public class IntervalSet implements IntSet { int b = I.b; if ( a==b ) { if ( a==Token.EOF ) buf.append("<EOF>"); - else if ( elemAreChar ) buf.append("'").append((char)a).append("'"); + else if ( elemAreChar ) buf.append("'").appendCodePoint(a).append("'"); else buf.append(a); } else { - if ( elemAreChar ) buf.append("'").append((char)a).append("'..'").append((char)b).append("'"); + if ( elemAreChar ) buf.append("'").appendCodePoint(a).append("'..'").appendCodePoint(b).append("'"); else buf.append(a).append("..").append(b); } if ( iter.hasNext() ) { diff --git a/runtime/Java/src/org/antlr/v4/runtime/misc/LogManager.java b/runtime/Java/src/org/antlr/v4/runtime/misc/LogManager.java index d47b5a2..174a060 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/misc/LogManager.java +++ b/runtime/Java/src/org/antlr/v4/runtime/misc/LogManager.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/runtime/Java/src/org/antlr/v4/runtime/misc/MultiMap.java b/runtime/Java/src/org/antlr/v4/runtime/misc/MultiMap.java index ce7972e..dcf802c 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/misc/MultiMap.java +++ b/runtime/Java/src/org/antlr/v4/runtime/misc/MultiMap.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/runtime/Java/src/org/antlr/v4/runtime/misc/MurmurHash.java b/runtime/Java/src/org/antlr/v4/runtime/misc/MurmurHash.java index c498b06..79e251f 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/misc/MurmurHash.java +++ b/runtime/Java/src/org/antlr/v4/runtime/misc/MurmurHash.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/runtime/Java/src/org/antlr/v4/runtime/misc/NotNull.java b/runtime/Java/src/org/antlr/v4/runtime/misc/NotNull.java index 25c59e4..7805291 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/misc/NotNull.java +++ b/runtime/Java/src/org/antlr/v4/runtime/misc/NotNull.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/runtime/Java/src/org/antlr/v4/runtime/misc/ObjectEqualityComparator.java b/runtime/Java/src/org/antlr/v4/runtime/misc/ObjectEqualityComparator.java index 256a17b..d9c0022 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/misc/ObjectEqualityComparator.java +++ b/runtime/Java/src/org/antlr/v4/runtime/misc/ObjectEqualityComparator.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/runtime/Java/src/org/antlr/v4/runtime/misc/OrderedHashSet.java b/runtime/Java/src/org/antlr/v4/runtime/misc/OrderedHashSet.java index d8f9d1b..7c4c526 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/misc/OrderedHashSet.java +++ b/runtime/Java/src/org/antlr/v4/runtime/misc/OrderedHashSet.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/runtime/Java/src/org/antlr/v4/runtime/misc/Pair.java b/runtime/Java/src/org/antlr/v4/runtime/misc/Pair.java index c01c907..289966f 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/misc/Pair.java +++ b/runtime/Java/src/org/antlr/v4/runtime/misc/Pair.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/runtime/Java/src/org/antlr/v4/runtime/misc/ParseCancellationException.java b/runtime/Java/src/org/antlr/v4/runtime/misc/ParseCancellationException.java index 4e1ebdb..b53d518 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/misc/ParseCancellationException.java +++ b/runtime/Java/src/org/antlr/v4/runtime/misc/ParseCancellationException.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/runtime/Java/src/org/antlr/v4/runtime/misc/Predicate.java b/runtime/Java/src/org/antlr/v4/runtime/misc/Predicate.java index 1c297a5..3107eb7 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/misc/Predicate.java +++ b/runtime/Java/src/org/antlr/v4/runtime/misc/Predicate.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/runtime/Java/src/org/antlr/v4/runtime/misc/TestRig.java b/runtime/Java/src/org/antlr/v4/runtime/misc/TestRig.java index de6f630..2be9784 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/misc/TestRig.java +++ b/runtime/Java/src/org/antlr/v4/runtime/misc/TestRig.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/runtime/Java/src/org/antlr/v4/runtime/misc/Triple.java b/runtime/Java/src/org/antlr/v4/runtime/misc/Triple.java index fdbf1c0..a7960cb 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/misc/Triple.java +++ b/runtime/Java/src/org/antlr/v4/runtime/misc/Triple.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/runtime/Java/src/org/antlr/v4/runtime/misc/Utils.java b/runtime/Java/src/org/antlr/v4/runtime/misc/Utils.java index 450c603..e4d91a1 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/misc/Utils.java +++ b/runtime/Java/src/org/antlr/v4/runtime/misc/Utils.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ @@ -137,11 +137,7 @@ public class Utils { public static char[] toCharArray(IntegerList data) { if ( data==null ) return null; - char[] cdata = new char[data.size()]; - for (int i=0; i<data.size(); i++) { - cdata[i] = (char)data.get(i); - } - return cdata; + return data.toCharArray(); } public static IntervalSet toSet(BitSet bits) { diff --git a/runtime/Java/src/org/antlr/v4/runtime/tree/AbstractParseTreeVisitor.java b/runtime/Java/src/org/antlr/v4/runtime/tree/AbstractParseTreeVisitor.java index f565fbf..93eaa46 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/tree/AbstractParseTreeVisitor.java +++ b/runtime/Java/src/org/antlr/v4/runtime/tree/AbstractParseTreeVisitor.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/runtime/Java/src/org/antlr/v4/runtime/tree/ErrorNode.java b/runtime/Java/src/org/antlr/v4/runtime/tree/ErrorNode.java index 87bf2fa..a54fce5 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/tree/ErrorNode.java +++ b/runtime/Java/src/org/antlr/v4/runtime/tree/ErrorNode.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/runtime/Java/src/org/antlr/v4/runtime/tree/ErrorNodeImpl.java b/runtime/Java/src/org/antlr/v4/runtime/tree/ErrorNodeImpl.java index 9295282..769763a 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/tree/ErrorNodeImpl.java +++ b/runtime/Java/src/org/antlr/v4/runtime/tree/ErrorNodeImpl.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/runtime/Java/src/org/antlr/v4/runtime/tree/IterativeParseTreeWalker.java b/runtime/Java/src/org/antlr/v4/runtime/tree/IterativeParseTreeWalker.java index edb0746..915869f 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/tree/IterativeParseTreeWalker.java +++ b/runtime/Java/src/org/antlr/v4/runtime/tree/IterativeParseTreeWalker.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ @@ -32,9 +32,11 @@ public class IterativeParseTreeWalker extends ParseTreeWalker { // pre-order visit if (currentNode instanceof ErrorNode) { listener.visitErrorNode((ErrorNode) currentNode); - } else if (currentNode instanceof TerminalNode) { + } + else if (currentNode instanceof TerminalNode) { listener.visitTerminal((TerminalNode) currentNode); - } else { + } + else { final RuleNode r = (RuleNode) currentNode; enterRule(listener, r); } diff --git a/runtime/Java/src/org/antlr/v4/runtime/tree/ParseTree.java b/runtime/Java/src/org/antlr/v4/runtime/tree/ParseTree.java index ebb20db..2417068 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/tree/ParseTree.java +++ b/runtime/Java/src/org/antlr/v4/runtime/tree/ParseTree.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ @@ -24,6 +24,24 @@ public interface ParseTree extends SyntaxTree { @Override ParseTree getChild(int i); + + /** Set the parent for this node. + * + * This is not backward compatible as it changes + * the interface but no one was able to create custom + * nodes anyway so I'm adding as it improves internal + * code quality. + * + * One could argue for a restructuring of + * the class/interface hierarchy so that + * setParent, addChild are moved up to Tree + * but that's a major change. So I'll do the + * minimal change, which is to add this method. + * + * @since 4.7 + */ + void setParent(RuleContext parent); + /** The {@link ParseTreeVisitor} needs a double dispatch method. */ <T> T accept(ParseTreeVisitor<? extends T> visitor); diff --git a/runtime/Java/src/org/antlr/v4/runtime/tree/ParseTreeListener.java b/runtime/Java/src/org/antlr/v4/runtime/tree/ParseTreeListener.java index 4fa245e..b9fe5aa 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/tree/ParseTreeListener.java +++ b/runtime/Java/src/org/antlr/v4/runtime/tree/ParseTreeListener.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/runtime/Java/src/org/antlr/v4/runtime/tree/ParseTreeProperty.java b/runtime/Java/src/org/antlr/v4/runtime/tree/ParseTreeProperty.java index ca16b18..22fbddd 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/tree/ParseTreeProperty.java +++ b/runtime/Java/src/org/antlr/v4/runtime/tree/ParseTreeProperty.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/runtime/Java/src/org/antlr/v4/runtime/tree/ParseTreeVisitor.java b/runtime/Java/src/org/antlr/v4/runtime/tree/ParseTreeVisitor.java index f9519da..862ba05 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/tree/ParseTreeVisitor.java +++ b/runtime/Java/src/org/antlr/v4/runtime/tree/ParseTreeVisitor.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/runtime/Java/src/org/antlr/v4/runtime/tree/ParseTreeWalker.java b/runtime/Java/src/org/antlr/v4/runtime/tree/ParseTreeWalker.java index 3d0546d..ede5feb 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/tree/ParseTreeWalker.java +++ b/runtime/Java/src/org/antlr/v4/runtime/tree/ParseTreeWalker.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/runtime/Java/src/org/antlr/v4/runtime/tree/RuleNode.java b/runtime/Java/src/org/antlr/v4/runtime/tree/RuleNode.java index 5e31437..f4a1235 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/tree/RuleNode.java +++ b/runtime/Java/src/org/antlr/v4/runtime/tree/RuleNode.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/runtime/Java/src/org/antlr/v4/runtime/tree/SyntaxTree.java b/runtime/Java/src/org/antlr/v4/runtime/tree/SyntaxTree.java index cedfa19..1502e3d 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/tree/SyntaxTree.java +++ b/runtime/Java/src/org/antlr/v4/runtime/tree/SyntaxTree.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/runtime/Java/src/org/antlr/v4/runtime/tree/TerminalNode.java b/runtime/Java/src/org/antlr/v4/runtime/tree/TerminalNode.java index d8a3e65..c7cfe64 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/tree/TerminalNode.java +++ b/runtime/Java/src/org/antlr/v4/runtime/tree/TerminalNode.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/runtime/Java/src/org/antlr/v4/runtime/tree/TerminalNodeImpl.java b/runtime/Java/src/org/antlr/v4/runtime/tree/TerminalNodeImpl.java index 1882d16..9869cb2 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/tree/TerminalNodeImpl.java +++ b/runtime/Java/src/org/antlr/v4/runtime/tree/TerminalNodeImpl.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ @@ -7,6 +7,7 @@ package org.antlr.v4.runtime.tree; import org.antlr.v4.runtime.Parser; +import org.antlr.v4.runtime.RuleContext; import org.antlr.v4.runtime.Token; import org.antlr.v4.runtime.misc.Interval; @@ -26,6 +27,11 @@ public class TerminalNodeImpl implements TerminalNode { public ParseTree getParent() { return parent; } @Override + public void setParent(RuleContext parent) { + this.parent = parent; + } + + @Override public Token getPayload() { return symbol; } @Override diff --git a/runtime/Java/src/org/antlr/v4/runtime/tree/Tree.java b/runtime/Java/src/org/antlr/v4/runtime/tree/Tree.java index 78f2b75..0a1bed6 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/tree/Tree.java +++ b/runtime/Java/src/org/antlr/v4/runtime/tree/Tree.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/runtime/Java/src/org/antlr/v4/runtime/tree/Trees.java b/runtime/Java/src/org/antlr/v4/runtime/tree/Trees.java index 9c07298..406d2af 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/tree/Trees.java +++ b/runtime/Java/src/org/antlr/v4/runtime/tree/Trees.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ @@ -249,6 +249,8 @@ public class Trees { public static Tree findNodeSuchThat(Tree t, Predicate<Tree> pred) { if ( pred.test(t) ) return t; + if ( t==null ) return null; + int n = t.getChildCount(); for (int i = 0 ; i < n ; i++){ Tree u = findNodeSuchThat(t.getChild(i), pred); diff --git a/runtime/Java/src/org/antlr/v4/runtime/tree/pattern/Chunk.java b/runtime/Java/src/org/antlr/v4/runtime/tree/pattern/Chunk.java index fc33fed..241478f 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/tree/pattern/Chunk.java +++ b/runtime/Java/src/org/antlr/v4/runtime/tree/pattern/Chunk.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/runtime/Java/src/org/antlr/v4/runtime/tree/pattern/ParseTreeMatch.java b/runtime/Java/src/org/antlr/v4/runtime/tree/pattern/ParseTreeMatch.java index 3101916..11204a7 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/tree/pattern/ParseTreeMatch.java +++ b/runtime/Java/src/org/antlr/v4/runtime/tree/pattern/ParseTreeMatch.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/runtime/Java/src/org/antlr/v4/runtime/tree/pattern/ParseTreePattern.java b/runtime/Java/src/org/antlr/v4/runtime/tree/pattern/ParseTreePattern.java index a8f5d14..de7b82a 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/tree/pattern/ParseTreePattern.java +++ b/runtime/Java/src/org/antlr/v4/runtime/tree/pattern/ParseTreePattern.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/runtime/Java/src/org/antlr/v4/runtime/tree/pattern/ParseTreePatternMatcher.java b/runtime/Java/src/org/antlr/v4/runtime/tree/pattern/ParseTreePatternMatcher.java index 7c2c954..0ff53f1 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/tree/pattern/ParseTreePatternMatcher.java +++ b/runtime/Java/src/org/antlr/v4/runtime/tree/pattern/ParseTreePatternMatcher.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/runtime/Java/src/org/antlr/v4/runtime/tree/pattern/RuleTagToken.java b/runtime/Java/src/org/antlr/v4/runtime/tree/pattern/RuleTagToken.java index 1fcb0c6..5531e56 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/tree/pattern/RuleTagToken.java +++ b/runtime/Java/src/org/antlr/v4/runtime/tree/pattern/RuleTagToken.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/runtime/Java/src/org/antlr/v4/runtime/tree/pattern/TagChunk.java b/runtime/Java/src/org/antlr/v4/runtime/tree/pattern/TagChunk.java index 420c892..c55ff0f 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/tree/pattern/TagChunk.java +++ b/runtime/Java/src/org/antlr/v4/runtime/tree/pattern/TagChunk.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/runtime/Java/src/org/antlr/v4/runtime/tree/pattern/TextChunk.java b/runtime/Java/src/org/antlr/v4/runtime/tree/pattern/TextChunk.java index 0ef75a3..da55460 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/tree/pattern/TextChunk.java +++ b/runtime/Java/src/org/antlr/v4/runtime/tree/pattern/TextChunk.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/runtime/Java/src/org/antlr/v4/runtime/tree/pattern/TokenTagToken.java b/runtime/Java/src/org/antlr/v4/runtime/tree/pattern/TokenTagToken.java index 740a898..93251fa 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/tree/pattern/TokenTagToken.java +++ b/runtime/Java/src/org/antlr/v4/runtime/tree/pattern/TokenTagToken.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/runtime/Java/src/org/antlr/v4/runtime/tree/xpath/XPath.java b/runtime/Java/src/org/antlr/v4/runtime/tree/xpath/XPath.java index a4a91bb..1e4416d 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/tree/xpath/XPath.java +++ b/runtime/Java/src/org/antlr/v4/runtime/tree/xpath/XPath.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/runtime/Java/src/org/antlr/v4/runtime/tree/xpath/XPathElement.java b/runtime/Java/src/org/antlr/v4/runtime/tree/xpath/XPathElement.java index 221a729..6db5b9a 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/tree/xpath/XPathElement.java +++ b/runtime/Java/src/org/antlr/v4/runtime/tree/xpath/XPathElement.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/runtime/Java/src/org/antlr/v4/runtime/tree/xpath/XPathLexer.g4 b/runtime/Java/src/org/antlr/v4/runtime/tree/xpath/XPathLexer.g4 deleted file mode 100644 index 1598977..0000000 --- a/runtime/Java/src/org/antlr/v4/runtime/tree/xpath/XPathLexer.g4 +++ /dev/null @@ -1,63 +0,0 @@ -lexer grammar XPathLexer; - -tokens { TOKEN_REF, RULE_REF } - -/* -path : separator? word (separator word)* EOF ; - -separator - : '/' '!' - | '//' '!' - | '/' - | '//' - ; - -word: TOKEN_REF - | RULE_REF - | STRING - | '*' - ; -*/ - -ANYWHERE : '//' ; -ROOT : '/' ; -WILDCARD : '*' ; -BANG : '!' ; - -ID : NameStartChar NameChar* - { - String text = getText(); - if ( Character.isUpperCase(text.charAt(0)) ) setType(TOKEN_REF); - else setType(RULE_REF); - } - ; - -fragment -NameChar : NameStartChar - | '0'..'9' - | '_' - | '\u00B7' - | '\u0300'..'\u036F' - | '\u203F'..'\u2040' - ; - -fragment -NameStartChar - : 'A'..'Z' | 'a'..'z' - | '\u00C0'..'\u00D6' - | '\u00D8'..'\u00F6' - | '\u00F8'..'\u02FF' - | '\u0370'..'\u037D' - | '\u037F'..'\u1FFF' - | '\u200C'..'\u200D' - | '\u2070'..'\u218F' - | '\u2C00'..'\u2FEF' - | '\u3001'..'\uD7FF' - | '\uF900'..'\uFDCF' - | '\uFDF0'..'\uFFFD' - ; // ignores | ['\u10000-'\uEFFFF] ; - -STRING : '\'' .*? '\'' ; - -//WS : [ \t\r\n]+ -> skip ; - diff --git a/runtime/Java/src/org/antlr/v4/runtime/tree/xpath/XPathLexer.java b/runtime/Java/src/org/antlr/v4/runtime/tree/xpath/XPathLexer.java new file mode 100644 index 0000000..4d120a4 --- /dev/null +++ b/runtime/Java/src/org/antlr/v4/runtime/tree/xpath/XPathLexer.java @@ -0,0 +1,179 @@ +/* + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. + * Use of this file is governed by the BSD 3-clause license that + * can be found in the LICENSE.txt file in the project root. + */ +package org.antlr.v4.runtime.tree.xpath; + +import org.antlr.v4.runtime.CharStream; +import org.antlr.v4.runtime.CommonToken; +import org.antlr.v4.runtime.Lexer; +import org.antlr.v4.runtime.LexerNoViableAltException; +import org.antlr.v4.runtime.Token; +import org.antlr.v4.runtime.Vocabulary; +import org.antlr.v4.runtime.VocabularyImpl; +import org.antlr.v4.runtime.atn.ATN; +import org.antlr.v4.runtime.misc.Interval; + +/** Mimic the old XPathLexer from .g4 file */ +public class XPathLexer extends Lexer { + public static final int + TOKEN_REF=1, RULE_REF=2, ANYWHERE=3, ROOT=4, WILDCARD=5, BANG=6, ID=7, + STRING=8; + public static String[] modeNames = { + "DEFAULT_MODE" + }; + + public static final String[] ruleNames = { + "ANYWHERE", "ROOT", "WILDCARD", "BANG", "ID", "NameChar", "NameStartChar", + "STRING" + }; + + private static final String[] _LITERAL_NAMES = { + null, null, null, "'//'", "'/'", "'*'", "'!'" + }; + private static final String[] _SYMBOLIC_NAMES = { + null, "TOKEN_REF", "RULE_REF", "ANYWHERE", "ROOT", "WILDCARD", "BANG", + "ID", "STRING" + }; + public static final Vocabulary VOCABULARY = new VocabularyImpl(_LITERAL_NAMES, _SYMBOLIC_NAMES); + + /** + * @deprecated Use {@link #VOCABULARY} instead. + */ + @Deprecated + public static final String[] tokenNames; + static { + tokenNames = new String[_SYMBOLIC_NAMES.length]; + for (int i = 0; i < tokenNames.length; i++) { + tokenNames[i] = VOCABULARY.getLiteralName(i); + if (tokenNames[i] == null) { + tokenNames[i] = VOCABULARY.getSymbolicName(i); + } + + if (tokenNames[i] == null) { + tokenNames[i] = "<INVALID>"; + } + } + } + + @Override + public String getGrammarFileName() { return "XPathLexer.g4"; } + + @Override + public String[] getRuleNames() { return ruleNames; } + + @Override + public String[] getModeNames() { return modeNames; } + + @Override + @Deprecated + public String[] getTokenNames() { + return tokenNames; + } + + @Override + public Vocabulary getVocabulary() { + return VOCABULARY; + } + + @Override + public ATN getATN() { + return null; + } + + protected int line = 1; + protected int charPositionInLine = 0; + + public XPathLexer(CharStream input) { + super(input); + } + + @Override + public Token nextToken() { + _tokenStartCharIndex = _input.index(); + CommonToken t = null; + while ( t==null ) { + switch ( _input.LA(1) ) { + case '/': + consume(); + if ( _input.LA(1)=='/' ) { + consume(); + t = new CommonToken(ANYWHERE, "//"); + } + else { + t = new CommonToken(ROOT, "/"); + } + break; + case '*': + consume(); + t = new CommonToken(WILDCARD, "*"); + break; + case '!': + consume(); + t = new CommonToken(BANG, "!"); + break; + case '\'': + String s = matchString(); + t = new CommonToken(STRING, s); + break; + case CharStream.EOF : + return new CommonToken(EOF, "<EOF>"); + default: + if ( isNameStartChar(_input.LA(1)) ) { + String id = matchID(); + if ( Character.isUpperCase(id.charAt(0)) ) t = new CommonToken(TOKEN_REF, id); + else t = new CommonToken(RULE_REF, id); + } + else { + throw new LexerNoViableAltException(this, _input, _tokenStartCharIndex, null); + } + break; + } + } + t.setStartIndex(_tokenStartCharIndex); + t.setCharPositionInLine(_tokenStartCharIndex); + t.setLine(line); + return t; + } + + public void consume() { + int curChar = _input.LA(1); + if ( curChar=='\n' ) { + line++; + charPositionInLine=0; + } + else { + charPositionInLine++; + } + _input.consume(); + } + + @Override + public int getCharPositionInLine() { + return charPositionInLine; + } + + public String matchID() { + int start = _input.index(); + consume(); // drop start char + while ( isNameChar(_input.LA(1)) ) { + consume(); + } + return _input.getText(Interval.of(start,_input.index()-1)); + } + + public String matchString() { + int start = _input.index(); + consume(); // drop first quote + while ( _input.LA(1)!='\'' ) { + consume(); + } + consume(); // drop last quote + return _input.getText(Interval.of(start,_input.index()-1)); + } + + public boolean isNameChar(int c) { return Character.isUnicodeIdentifierPart(c); } + + public boolean isNameStartChar(int c) { return Character.isUnicodeIdentifierStart(c); } +} diff --git a/runtime/Java/src/org/antlr/v4/runtime/tree/xpath/XPathLexerErrorListener.java b/runtime/Java/src/org/antlr/v4/runtime/tree/xpath/XPathLexerErrorListener.java index a669850..21bcb54 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/tree/xpath/XPathLexerErrorListener.java +++ b/runtime/Java/src/org/antlr/v4/runtime/tree/xpath/XPathLexerErrorListener.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/runtime/Java/src/org/antlr/v4/runtime/tree/xpath/XPathRuleAnywhereElement.java b/runtime/Java/src/org/antlr/v4/runtime/tree/xpath/XPathRuleAnywhereElement.java index 66684ae..e5a4bfe 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/tree/xpath/XPathRuleAnywhereElement.java +++ b/runtime/Java/src/org/antlr/v4/runtime/tree/xpath/XPathRuleAnywhereElement.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/runtime/Java/src/org/antlr/v4/runtime/tree/xpath/XPathRuleElement.java b/runtime/Java/src/org/antlr/v4/runtime/tree/xpath/XPathRuleElement.java index 2827914..d966666 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/tree/xpath/XPathRuleElement.java +++ b/runtime/Java/src/org/antlr/v4/runtime/tree/xpath/XPathRuleElement.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/runtime/Java/src/org/antlr/v4/runtime/tree/xpath/XPathTokenAnywhereElement.java b/runtime/Java/src/org/antlr/v4/runtime/tree/xpath/XPathTokenAnywhereElement.java index af08ec9..41d34b0 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/tree/xpath/XPathTokenAnywhereElement.java +++ b/runtime/Java/src/org/antlr/v4/runtime/tree/xpath/XPathTokenAnywhereElement.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/runtime/Java/src/org/antlr/v4/runtime/tree/xpath/XPathTokenElement.java b/runtime/Java/src/org/antlr/v4/runtime/tree/xpath/XPathTokenElement.java index 9560af1..4a0e36d 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/tree/xpath/XPathTokenElement.java +++ b/runtime/Java/src/org/antlr/v4/runtime/tree/xpath/XPathTokenElement.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/runtime/Java/src/org/antlr/v4/runtime/tree/xpath/XPathWildcardAnywhereElement.java b/runtime/Java/src/org/antlr/v4/runtime/tree/xpath/XPathWildcardAnywhereElement.java index 43c63ed..4ab3a10 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/tree/xpath/XPathWildcardAnywhereElement.java +++ b/runtime/Java/src/org/antlr/v4/runtime/tree/xpath/XPathWildcardAnywhereElement.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ diff --git a/runtime/Java/src/org/antlr/v4/runtime/tree/xpath/XPathWildcardElement.java b/runtime/Java/src/org/antlr/v4/runtime/tree/xpath/XPathWildcardElement.java index e027299..0de1483 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/tree/xpath/XPathWildcardElement.java +++ b/runtime/Java/src/org/antlr/v4/runtime/tree/xpath/XPathWildcardElement.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2016 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ |