org.jruby.ext.ripper

Class RipperLexer

java.lang.Object

org.jruby.lexer.LexingCommon

org.jruby.ext.ripper.RipperLexer

public class RipperLexer
extends LexingCommon

Author:

enebo

Nested Class Summary

Nested Classes

Modifier and Type	Class and Description
static class	RipperLexer.Keyword

Field Summary

Fields

Modifier and Type	Field and Description
protected ByteList	delayed

Fields inherited from class org.jruby.lexer.LexingCommon

__end__seen, AMPERSAND, AMPERSAND_AMPERSAND, AMPERSAND_DOT, ASCII8BIT_ENCODING, BACKSLASH, BACKTICK, BANG, BANG_EQ, BANG_TILDE, BEGIN_DOC_MARKER, braceNest, CALL, CARET, cmdArgumentState, CODING, COLON, COLON_COLON, COMMA, commandStart, conditionState, DOLLAR_BANG, DOLLAR_DOT, DOLLAR_UNDERSCORE, DOT, DOT_DOT, DOT_DOT_DOT, END_DOC_MARKER, END_MARKER, EOF, eofp, EQ, EQ_EQ, EQ_EQ_EQ, EQ_GT, EQ_TILDE, EXPR_ARG, EXPR_ARG_ANY, EXPR_BEG, EXPR_BEG_ANY, EXPR_CLASS, EXPR_CMDARG, EXPR_DOT, EXPR_END, EXPR_END_ANY, EXPR_ENDARG, EXPR_ENDFN, EXPR_FITEM, EXPR_FNAME, EXPR_LABEL, EXPR_LABELED, EXPR_MID, EXPR_VALUE, GT, GT_EQ, GT_GT, has_shebang, heredoc_end, heredoc_indent, heredoc_line_indent, inKwarg, last_cr_line, last_state, LBRACKET, LBRACKET_RBRACKET, LBRACKET_RBRACKET_EQ, LCURLY, lex_lastline, lex_nextline, lex_p, lex_pbeg, lex_pend, lex_state, lexb, line_count, line_offset, LT, LT_EQ, LT_EQ_RT, LT_LT, MINUS, MINUS_AT, MINUS_GT, OR, OR_OR, parenNest, PERCENT, PLUS, PLUS_AT, Q, QQ, QUESTION, RBRACKET, RCURLY, RPAREN, ruby_sourceline, SEMICOLON, SLASH, src, STAR, STAR_STAR, str_dquote, str_dsym, str_dword, STR_FUNC_ESCAPE, STR_FUNC_EXPAND, STR_FUNC_LABEL, STR_FUNC_LIST, STR_FUNC_QWORDS, STR_FUNC_REGEXP, STR_FUNC_SYMBOL, STR_FUNC_TERM, str_label, str_regexp, str_squote, str_ssym, str_sword, str_xquote, SUFFIX_ALL, SUFFIX_I, SUFFIX_R, TAB_WIDTH, TILDE, token, tokenSeen, tokline, tokp, USASCII_ENCODING, UTF8_ENCODING, yaccValue

Constructor Summary

Constructors

Constructor and Description
RipperLexer(RipperParserBase parser, LexerSource src)

Method Summary

Modifier and Type	Method and Description
void	addDelayedToken(int tok, int end)
protected void	ambiguousOperator(java.lang.String op, java.lang.String syn)
void	compile_error(java.lang.String message)
IRubyObject	createStr(ByteList buffer, int flags)
void	dispatchDelayedToken(int token)
void	dispatchHeredocEnd()
void	dispatchIgnoredScanEvent(int token)
void	dispatchScanEvent(int token)
protected void	flush_string_content(org.jcodings.Encoding encoding)
java.lang.String	getIdent()
static RipperLexer.Keyword	getKeyword(java.lang.String str)
Ruby	getRuntime()
StrTerm	getStrTerm()
boolean	hasScanEvent()
boolean	hasStarted() Has lexing started yet?
void	heredoc_restore(HeredocTerm here)
boolean	isVerbose()
protected void	mismatchedRegexpEncodingError(org.jcodings.Encoding optionEncoding, org.jcodings.Encoding encoding)
int	nextc()
int	nextToken()
protected boolean	onMagicComment(java.lang.String name, ByteList value)
protected RegexpOptions	parseRegexpFlags()
int	readEscape()
int	readUTFEscape(ByteList buffer, boolean stringLiteral, boolean symbolLiteral)
void	readUTFEscapeRegexpLiteral(ByteList buffer)
protected void	setCompileOptionFlag(java.lang.String name, ByteList value)
protected void	setEncoding(ByteList name)
void	setParser(RipperParserBase parserSupport) Parse must pass its support object for some check at bottom of yylex().
void	setStrTerm(StrTerm strterm)
protected void	setTokenInfo(java.lang.String name, ByteList value)
boolean	tokenAddMBC(int first_byte, ByteList buffer)
int	tokenize_ident(int result)
void	warn(java.lang.String message)
void	warning(java.lang.String fmt)
void	warning(java.lang.String fmt, java.lang.String arg)

Methods inherited from class org.jruby.lexer.LexingCommon

checkRegexpFragment, checkRegexpSyntax, column, comment_at_top, createAsEncodedString, createTokenByteList, createTokenByteList, createTokenString, createTokenString, dedent_string, flush, getBraceNest, getCmdArgumentState, getConditionState, getCurrentArg, getCurrentLine, getEncoding, getFile, getHeredocIndent, getHeredocLineIndent, getLeftParenBegin, getLineOffset, getPosition, getRubySourceline, getState, getTokenCR, incrementParenNest, isAfterOperator, isARG, isASCII, isASCII, isBEG, isEND, isEndSeen, isGlobalCharPunct, isHexChar, isIdentifierChar, isLabelPossible, isLabelSuffix, isLexState, isLexStateAll, isNext_identchar, isOctChar, isSpace, isSpaceArg, lex_goto_eol, lineno, magicCommentEncoding, magicCommentMarker, newtok, numberLiteralSuffix, optionsEncodingChar, p, parser_magic_comment, parser_prepare, parseRegexpFlags, peek, peek, peekVariableName, precise_mbclen, printState, pushback, reset, resetStacks, scanOct, set_file_encoding, setBraceNest, setCurrentArg, setCurrentEncoding, setEncoding, setHeredocIndent, setHeredocLineIndent, setLeftParenBegin, setRegexpEncoding, setRubySourceline, setSource, setState, setValue, strncmp, tokadd_ident, tokadd_mbchar, tokadd_mbchar, tokAdd, tokaddmbc, tokCopy, token, update_heredoc_indent, validateFormalIdentifier, validateFormalIdentifier, value, warn_balanced, was_bol, whole_match_p

Methods inherited from class java.lang.Object

clone, equals, finalize, getClass, hashCode, notify, notifyAll, toString, wait, wait, wait

Field Detail

delayed

protected ByteList delayed

Constructor Detail

RipperLexer

public RipperLexer(RipperParserBase parser,
                   LexerSource src)

Method Detail

ambiguousOperator

protected void ambiguousOperator(java.lang.String op,
                                 java.lang.String syn)

Specified by:

ambiguousOperator in class LexingCommon

onMagicComment

protected boolean onMagicComment(java.lang.String name,
                                 ByteList value)

Overrides:

onMagicComment in class LexingCommon

isVerbose

public boolean isVerbose()

warn

public void warn(java.lang.String message)

warning

public void warning(java.lang.String fmt)

warning

public void warning(java.lang.String fmt,
                    java.lang.String arg)

getKeyword

public static RipperLexer.Keyword getKeyword(java.lang.String str)

hasStarted

public boolean hasStarted()

Has lexing started yet?

flush_string_content

protected void flush_string_content(org.jcodings.Encoding encoding)

addDelayedToken

public void addDelayedToken(int tok,
                            int end)

nextc

public int nextc()

Specified by:

nextc in class LexingCommon

dispatchHeredocEnd

public void dispatchHeredocEnd()

compile_error

public void compile_error(java.lang.String message)

Specified by:

compile_error in class LexingCommon

tokenize_ident

public int tokenize_ident(int result)

Specified by:

tokenize_ident in class LexingCommon

heredoc_restore

public void heredoc_restore(HeredocTerm here)

nextToken

public int nextToken()
              throws java.io.IOException

Throws:

java.io.IOException

getIdent

public java.lang.String getIdent()

getRuntime

public Ruby getRuntime()

setParser

public void setParser(RipperParserBase parserSupport)

Parse must pass its support object for some check at bottom of yylex(). Ruby does it this way as well (i.e. a little parsing logic in the lexer).

Parameters:

parserSupport -

setCompileOptionFlag

protected void setCompileOptionFlag(java.lang.String name,
                                    ByteList value)

Specified by:

setCompileOptionFlag in class LexingCommon

parseRegexpFlags

protected RegexpOptions parseRegexpFlags()
                                  throws java.io.IOException

Specified by:

parseRegexpFlags in class LexingCommon

Throws:

java.io.IOException

mismatchedRegexpEncodingError

protected void mismatchedRegexpEncodingError(org.jcodings.Encoding optionEncoding,
                                             org.jcodings.Encoding encoding)

Specified by:

mismatchedRegexpEncodingError in class LexingCommon

setTokenInfo

protected void setTokenInfo(java.lang.String name,
                            ByteList value)

Specified by:

setTokenInfo in class LexingCommon

setEncoding

protected void setEncoding(ByteList name)

Specified by:

setEncoding in class LexingCommon

getStrTerm

public StrTerm getStrTerm()

setStrTerm

public void setStrTerm(StrTerm strterm)

createStr

public IRubyObject createStr(ByteList buffer,
                             int flags)

hasScanEvent

public boolean hasScanEvent()

dispatchDelayedToken

public void dispatchDelayedToken(int token)

dispatchIgnoredScanEvent

public void dispatchIgnoredScanEvent(int token)

dispatchScanEvent

public void dispatchScanEvent(int token)

readUTFEscapeRegexpLiteral

public void readUTFEscapeRegexpLiteral(ByteList buffer)
                                throws java.io.IOException

Throws:

java.io.IOException

tokenAddMBC

public boolean tokenAddMBC(int first_byte,
                           ByteList buffer)

readUTFEscape

public int readUTFEscape(ByteList buffer,
                         boolean stringLiteral,
                         boolean symbolLiteral)
                  throws java.io.IOException

Throws:

java.io.IOException

readEscape

public int readEscape()
               throws java.io.IOException

Throws:

java.io.IOException