Class Scanner

java.lang.Object
org.aspectj.org.eclipse.jdt.internal.compiler.parser.Scanner
Direct Known Subclasses:
CompletionScanner, JavadocScanner, RecoveryScanner, SelectionScanner
public class Scanner extends Object
IMPORTANT NOTE: Internal Scanner implementation. It is mirrored in org.eclipse.jdt.core.compiler public package where it is API. The mirror implementation is using the backward compatible ITerminalSymbols constant definitions (stable with 2.0), whereas the internal implementation uses TerminalToken which constant values reflect the latest parser generation state.

  • Field Details

    • sourceLevel

      public long sourceLevel

    • complianceLevel

      public long complianceLevel

    • useAssertAsAnIndentifier

      public boolean useAssertAsAnIndentifier

    • containsAssertKeyword

      public boolean containsAssertKeyword

    • previewEnabled

      public boolean previewEnabled

    • useEnumAsAnIndentifier

      public boolean useEnumAsAnIndentifier

    • recordLineSeparator

      public boolean recordLineSeparator

    • currentCharacter

      public char currentCharacter

    • startPosition

      public int startPosition

    • currentPosition

      public int currentPosition

    • initialPosition

      public int initialPosition

    • eofPosition

      public int eofPosition

    • skipComments

      public boolean skipComments

    • tokenizeComments

      public boolean tokenizeComments

    • tokenizeWhiteSpace

      public boolean tokenizeWhiteSpace

    • source

      public char[] source

    • withoutUnicodeBuffer

      public char[] withoutUnicodeBuffer

    • withoutUnicodePtr

      public int withoutUnicodePtr

    • unicodeAsBackSlash

      public boolean unicodeAsBackSlash

    • scanningFloatLiteral

      public boolean scanningFloatLiteral

    • COMMENT_ARRAYS_SIZE

      public static int COMMENT_ARRAYS_SIZE

    • commentStops

      public int[] commentStops

    • commentStarts

      public int[] commentStarts

    • commentIsMarkdown

      public boolean[] commentIsMarkdown

    • commentTagStarts

      public int[] commentTagStarts

    • commentPtr

      public int commentPtr

    • lastCommentLinePosition

      public int lastCommentLinePosition

    • foundTaskTags

      public char[][] foundTaskTags

    • foundTaskMessages

      public char[][] foundTaskMessages

    • foundTaskPriorities

      public char[][] foundTaskPriorities

    • foundTaskPositions

      public int[][] foundTaskPositions

    • foundTaskCount

      public int foundTaskCount

    • taskTags

      public char[][] taskTags

    • taskPriorities

      public char[][] taskPriorities

    • isTaskCaseSensitive

      public boolean isTaskCaseSensitive

    • diet

      public boolean diet

    • lineEnds

      public int[] lineEnds

    • linePtr

      public int linePtr

    • wasAcr

      public boolean wasAcr

    • fakeInModule

      public boolean fakeInModule

    • scanContext

      protected org.aspectj.org.eclipse.jdt.internal.compiler.parser.Scanner.ScanContext scanContext

    • insideModuleInfo

      protected boolean insideModuleInfo

    • END_OF_SOURCE

      public static final String END_OF_SOURCE
      See Also:

    • INVALID_HEXA

      public static final String INVALID_HEXA
      See Also:

    • INVALID_OCTAL

      public static final String INVALID_OCTAL
      See Also:

    • INVALID_CHARACTER_CONSTANT

      public static final String INVALID_CHARACTER_CONSTANT
      See Also:

    • INVALID_ESCAPE

      public static final String INVALID_ESCAPE
      See Also:

    • INVALID_INPUT

      public static final String INVALID_INPUT
      See Also:

    • INVALID_TEXTBLOCK

      public static final String INVALID_TEXTBLOCK
      See Also:

    • INVALID_UNICODE_ESCAPE

      public static final String INVALID_UNICODE_ESCAPE
      See Also:

    • INVALID_FLOAT

      public static final String INVALID_FLOAT
      See Also:

    • INVALID_LOW_SURROGATE

      public static final String INVALID_LOW_SURROGATE
      See Also:

    • INVALID_HIGH_SURROGATE

      public static final String INVALID_HIGH_SURROGATE
      See Also:

    • NULL_SOURCE_STRING

      public static final String NULL_SOURCE_STRING
      See Also:

    • UNTERMINATED_STRING

      public static final String UNTERMINATED_STRING
      See Also:

    • UNTERMINATED_TEXT_BLOCK

      public static final String UNTERMINATED_TEXT_BLOCK
      See Also:

    • UNTERMINATED_COMMENT

      public static final String UNTERMINATED_COMMENT
      See Also:

    • INVALID_CHAR_IN_STRING

      public static final String INVALID_CHAR_IN_STRING
      See Also:

    • INVALID_DIGIT

      public static final String INVALID_DIGIT
      See Also:

    • INVALID_BINARY

      public static final String INVALID_BINARY
      See Also:

    • BINARY_LITERAL_NOT_BELOW_17

      public static final String BINARY_LITERAL_NOT_BELOW_17
      See Also:

    • ILLEGAL_HEXA_LITERAL

      public static final String ILLEGAL_HEXA_LITERAL
      See Also:

    • INVALID_UNDERSCORE

      public static final String INVALID_UNDERSCORE
      See Also:

    • UNDERSCORES_IN_LITERALS_NOT_BELOW_17

      public static final String UNDERSCORES_IN_LITERALS_NOT_BELOW_17
      See Also:

    • TAG_PREFIX

      public static final char[] TAG_PREFIX

    • TAG_PREFIX_LENGTH

      public static final int TAG_PREFIX_LENGTH

    • TAG_POSTFIX

      public static final char TAG_POSTFIX
      See Also:

    • TAG_POSTFIX_LENGTH

      public static final int TAG_POSTFIX_LENGTH
      See Also:

    • IDENTITY_COMPARISON_TAG

      public static final char[] IDENTITY_COMPARISON_TAG

    • validIdentityComparisonLines

      public boolean[] validIdentityComparisonLines

    • checkUninternedIdentityComparison

      public boolean checkUninternedIdentityComparison

    • nlsTagsPtr

      protected int nlsTagsPtr

    • checkNonExternalizedStringLiterals

      public boolean checkNonExternalizedStringLiterals

    • lastPosition

      protected int lastPosition

    • returnOnlyGreater

      public boolean returnOnlyGreater

    • insideRecovery

      public boolean insideRecovery

    • nextToken

      protected TerminalToken nextToken

    • scanningSwitchLabel

      public boolean scanningSwitchLabel

    • RoundBracket

      public static final int RoundBracket
      See Also:

    • SquareBracket

      public static final int SquareBracket
      See Also:

    • CurlyBracket

      public static final int CurlyBracket
      See Also:

    • BracketKinds

      public static final int BracketKinds
      See Also:

    • LOW_SURROGATE_MIN_VALUE

      public static final int LOW_SURROGATE_MIN_VALUE
      See Also:

    • HIGH_SURROGATE_MIN_VALUE

      public static final int HIGH_SURROGATE_MIN_VALUE
      See Also:

    • HIGH_SURROGATE_MAX_VALUE

      public static final int HIGH_SURROGATE_MAX_VALUE
      See Also:

    • LOW_SURROGATE_MAX_VALUE

      public static final int LOW_SURROGATE_MAX_VALUE
      See Also:

    • rawStart

      protected int rawStart

  • Constructor Details

    • Scanner

      public Scanner()

    • Scanner

      public Scanner(boolean tokenizeComments, boolean tokenizeWhiteSpace, boolean checkNonExternalizedStringLiterals, long sourceLevel, long complianceLevel, char[][] taskTags, char[][] taskPriorities, boolean isTaskCaseSensitive, boolean isPreviewEnabled)

    • Scanner

      public Scanner(boolean tokenizeComments, boolean tokenizeWhiteSpace, boolean checkNonExternalizedStringLiterals, long sourceLevel, char[][] taskTags, char[][] taskPriorities, boolean isTaskCaseSensitive, boolean isPreviewEnabled)

    • Scanner

      public Scanner(boolean tokenizeComments, boolean tokenizeWhiteSpace, boolean checkNonExternalizedStringLiterals, long sourceLevel, char[][] taskTags, char[][] taskPriorities, boolean isTaskCaseSensitive)

  • Method Details

    • atEnd

      public final boolean atEnd()

    • checkTaskTag

      public void checkTaskTag(int commentStart, int commentEnd) throws InvalidInputException
      Throws:
      InvalidInputException

    • getCurrentIdentifierSource

      public char[] getCurrentIdentifierSource()

    • getCurrentTokenEndPosition

      public int getCurrentTokenEndPosition()

    • getCurrentTokenSource

      public char[] getCurrentTokenSource()

    • getCurrentTokenString

      public final String getCurrentTokenString()

    • getCurrentTokenSourceString

      public char[] getCurrentTokenSourceString()

    • scanForTextBlockBeginning

      protected final boolean scanForTextBlockBeginning()

    • lineBeginsWithMarkdown

      protected final boolean lineBeginsWithMarkdown() throws InvalidInputException
      Throws:
      InvalidInputException

    • scanForTextBlockClose

      protected final boolean scanForTextBlockClose() throws InvalidInputException
      Throws:
      InvalidInputException

    • getCurrentTextBlock

      public char[] getCurrentTextBlock()

    • getCurrentStringLiteral

      public final String getCurrentStringLiteral()

    • getRawTokenSource

      public final char[] getRawTokenSource()

    • getRawTokenSourceEnd

      public final char[] getRawTokenSourceEnd()

    • getCurrentTokenStartPosition

      public int getCurrentTokenStartPosition()

    • getLineEnd

      public final int getLineEnd(int lineNumber)

    • getLineEnds

      public final int[] getLineEnds()

    • getLineStart

      public final int getLineStart(int lineNumber)
      Search the source position corresponding to the beginning of a given line number Line numbers are 1-based, and relative to the scanner initialPosition. Character positions are 0-based. e.g. getLineStart(1) --> 0 indicates that the first line starts at character 0. In case the given line number is inconsistent, answers -1.
      Parameters:
      lineNumber - int
      Returns:
      int

    • getNextChar

      public final int getNextChar()

    • getNextCharWithBoundChecks

      public final int getNextCharWithBoundChecks()

    • getNextChar

      public final boolean getNextChar(char testedChar)

    • getNextChar

      public final int getNextChar(char testedChar1, char testedChar2)

    • getNextCharAsDigit

      public final boolean getNextCharAsDigit() throws InvalidInputException
      Throws:
      InvalidInputException

    • getNextCharAsDigit

      public final boolean getNextCharAsDigit(int radix)

    • getNextCharAsJavaIdentifierPartWithBoundCheck

      public boolean getNextCharAsJavaIdentifierPartWithBoundCheck()

    • getNextCharAsJavaIdentifierPart

      public boolean getNextCharAsJavaIdentifierPart()

    • scanIdentifier

      public TerminalToken scanIdentifier() throws InvalidInputException
      Throws:
      InvalidInputException

    • ungetToken

      public void ungetToken(TerminalToken unambiguousToken)

    • getNextToken

      public TerminalToken getNextToken() throws InvalidInputException
      Throws:
      InvalidInputException

    • findCommentType

      protected int findCommentType()

    • getNextToken0

      protected TerminalToken getNextToken0() throws InvalidInputException
      Throws:
      InvalidInputException

    • processSingleQuotes

      protected TerminalToken processSingleQuotes(boolean checkIfUnicode) throws InvalidInputException
      Throws:
      InvalidInputException

    • scanForStringLiteral

      protected TerminalToken scanForStringLiteral() throws InvalidInputException
      Throws:
      InvalidInputException

    • scanForTextBlock

      protected TerminalToken scanForTextBlock() throws InvalidInputException
      Throws:
      InvalidInputException

    • getNextUnicodeChar

      public void getNextUnicodeChar() throws InvalidInputException
      Throws:
      InvalidInputException

    • getNLSTags

      public NLSTag[] getNLSTags()

    • getIdentityComparisonLines

      public boolean[] getIdentityComparisonLines()

    • getSource

      public char[] getSource()

    • isFirstTag

      protected boolean isFirstTag()

    • jumpOverMethodBody

      public final void jumpOverMethodBody()

    • jumpOverUnicodeWhiteSpace

      public final boolean jumpOverUnicodeWhiteSpace() throws InvalidInputException
      Throws:
      InvalidInputException

    • isInModuleDeclaration

      public boolean isInModuleDeclaration()

    • areRestrictedModuleKeywordsActive

      protected boolean areRestrictedModuleKeywordsActive()

    • pushLineSeparator

      public final void pushLineSeparator()

    • pushUnicodeLineSeparator

      public final void pushUnicodeLineSeparator()

    • recordComment

      public void recordComment(TerminalToken token)

    • resetTo

      public void resetTo(int begin, int end)
      Reposition the scanner on some portion of the original source. The given endPosition is the last valid position. Beyond this position, the scanner will answer EOF tokens (ITerminalSymbols.TokenNameEOF).
      Parameters:
      begin - the given start position
      end - the given end position

    • resetTo

      public void resetTo(int begin, int end, boolean isModuleInfo)

    • resetTo

      public void resetTo(int begin, int end, boolean isModuleInfo, org.aspectj.org.eclipse.jdt.internal.compiler.parser.Scanner.ScanContext context)
      Reposition the scanner on some portion of the original source. The given endPosition is the last valid position. Beyond this position, the scanner will answer EOF tokens (ITerminalSymbols.TokenNameEOF).
      Parameters:
      begin - the given start position
      end - the given end position
      isModuleInfo - if true apply rules for restricted keywords even without a connection to a properly configured parser
      context - The scan context to use for restricted keyword support, use null to compute

    • scanEscapeCharacter

      protected final void scanEscapeCharacter() throws InvalidInputException
      Throws:
      InvalidInputException

    • scanIdentifierOrKeywordWithBoundCheck

      public TerminalToken scanIdentifierOrKeywordWithBoundCheck()

    • scanIdentifierOrKeyword

      public TerminalToken scanIdentifierOrKeyword()

    • scanIdentifierOrKeywordAj

      public TerminalToken scanIdentifierOrKeywordAj(TerminalToken kind)

    • scanNumber

      public TerminalToken scanNumber(boolean dotPrefix) throws InvalidInputException
      Throws:
      InvalidInputException

    • getLineNumber

      public final int getLineNumber(int position)
      Search the line number corresponding to a specific position
      Parameters:
      position - int
      Returns:
      int

    • setSource

      public final void setSource(char[] sourceString)

    • setSource

      public final void setSource(char[] contents, CompilationResult compilationResult)

    • setSource

      public final void setSource(CompilationResult compilationResult)

    • toString

      public String toString()
      Overrides:
      toString in class Object

    • toStringAction

      public String toStringAction(TerminalToken act)

    • unicodeInitializeBuffer

      public void unicodeInitializeBuffer(int length)

    • unicodeStore

      public void unicodeStore()

    • unicodeStore

      public void unicodeStore(char character)

    • isIdentifier

      public static boolean isIdentifier(TerminalToken token)

    • isLiteral

      public static boolean isLiteral(TerminalToken token)

    • isKeyword

      public static boolean isKeyword(TerminalToken token)

    • maybeAtLambdaOrCast

      protected final boolean maybeAtLambdaOrCast()

    • maybeAtReferenceExpression

      protected final boolean maybeAtReferenceExpression()

    • atTypeAnnotation

      protected final boolean atTypeAnnotation()

    • setActiveParser

      public void setActiveParser(ConflictedParser parser)

    • isRestrictedKeyword

      public static boolean isRestrictedKeyword(TerminalToken token)

    • atMultiCaseComma

      public boolean atMultiCaseComma()

    • mayBeAtCasePattern

      protected final boolean mayBeAtCasePattern(TerminalToken token)

    • isAtAssistIdentifier

      protected boolean isAtAssistIdentifier()

    • fastForward

      public TerminalToken fastForward(Statement unused)

    • getNextNotFakedToken

      protected TerminalToken getNextNotFakedToken() throws InvalidInputException
      Overridable hook, to allow CompletionScanner to hide a faked identifier token.
      Throws:
      InvalidInputException

    • invalidCharacter

      protected static InvalidInputException invalidCharacter()

    • invalidCharInString

      protected static InvalidInputException invalidCharInString()

    • unterminatedString

      protected static InvalidInputException unterminatedString()

    • invalidUnicodeEscape

      protected static InvalidInputException invalidUnicodeEscape()

    • invalidLowSurrogate

      protected static InvalidInputException invalidLowSurrogate()

    • invalidHighSurrogate

      protected static InvalidInputException invalidHighSurrogate()

    • unterminatedComment

      protected static InvalidInputException unterminatedComment()

    • unterminatedTextBlock

      protected static InvalidInputException unterminatedTextBlock()

    • invalidEof

      protected static InvalidInputException invalidEof()

    • invalidUnderscore

      protected static InvalidInputException invalidUnderscore()

    • invalidUnderscoresInLiterals

      protected static InvalidInputException invalidUnderscoresInLiterals()

    • invalidEscape

      protected static InvalidInputException invalidEscape()

    • invalidHexa

      protected static InvalidInputException invalidHexa()

    • illegalHexaLiteral

      protected static InvalidInputException illegalHexaLiteral()

    • invalidFloat

      protected static InvalidInputException invalidFloat()

    • invalidBinaryLiteral

      protected static InvalidInputException invalidBinaryLiteral()

    • invalidBinary

      protected static InvalidInputException invalidBinary()

    • invalidToken

      public static InvalidInputException invalidToken(TerminalToken token)

    • invalidInput

      public static InvalidInputException invalidInput()

    • copyCommentInfo

      public void copyCommentInfo(int to, int from)

    • copyAllCommentInfo

      public void copyAllCommentInfo(int from, int to, int length)

    • growCommentInfoArrays

      protected void growCommentInfoArrays(int length, int newLength)