Class: RubyLexer

### Home Classes Methods

### In Files

- lib/ruby_lexer.rb
- lib/ruby_lexer.rex.rb

### Parent

Object

### Namespace

- CLASS RubyLexer::ScanError

### Methods

- ::new
- #action
- #arg_ambiguous
- #arg_state
- #beginning_of_line?
- #bol?
- #check
- #comments
- #d
- #end_of_stream?
- #expr_dot?
- #expr_fname?
- #expr_result
- #heredoc
- #heredoc_identifier
- #in_arg_state?
- #in_fname?
- #in_lex_state?
- #int_with_base
- #is_arg?
- #is_beg?
- #is_end?
- #is_label_possible?
- #is_label_suffix?
- #is_space_arg?
- #lineno=
- #matched
- #matches
- #next_token
- #not_end?
- #parse
- #parse_file
- #parse_quote
- #parse_string
- #process_amper
- #process_backref
- #process_begin
- #process_bracing
- #process_colon1
- #process_colon2
- #process_curly_brace
- #process_float
- #process_gvar
- #process_gvar_oddity
- #process_ivar
- #process_label
- #process_label_or_string
- #process_lchevron
- #process_newline_or_comment
- #process_nthref
- #process_paren
- #process_paren18
- #process_paren19
- #process_percent
- #process_plus_minus
- #process_questionmark
- #process_slash
- #process_square_bracket
- #process_string
- #process_symbol
- #process_token
- #process_token_keyword
- #process_underscore
- #rb_compile_error
- #read_escape
- #regx_options
- #reset
- #ruby18
- #ruby19
- #ruby22?
- #ruby22_label?
- #scan
- #scanner_class
- #space_vs_beginning
- #string
- #tokadd_escape
- #tokadd_string
- #unescape
- #warning
- #was_label?

### Files

- Manifest.txt
- README.txt

### Class/Module Index

Quicksearch

- RubyParserStuff
- RubyParserStuff::Environment
- RubyParserStuff::Keyword
- RubyParserStuff::Keyword::KWtable
- RubyParserStuff::Ruby18Parser
- RubyParserStuff::Ruby19Parser
- RubyParserStuff::Ruby20Parser
- RubyParserStuff::Ruby21Parser
- RubyParserStuff::Ruby22Parser
- RubyParserStuff::RubyParser
- RubyParserStuff::RubyParser::SyntaxError
- RubyParserStuff::Sexp
- RubyParserStuff::StackState
- RubyParserStuff::String
- RubyLexer
- RubyLexer::ScanError
- RPStringScanner
- Ruby18Parser
- Ruby19Parser
- Ruby20Parser
- Ruby21Parser
- Ruby22Parser

No matching classes.

# RubyLexer

lexical scanner definition for ruby

### Constants

ESC
FLOAT
FLOAT_BAD
IDENT
INT_BIN
INT_DEC
INT_DEC2
INT_HEX
INT_OCT
INT_OCT_BAD
NUM_BAD
SIMPLE_STRING
SSTRING

### Attributes

brace_nest[RW]
cmdarg[RW]
command_start[RW]
command_state[RW]
comments[W]
cond[RW]
extra_lineno[RW]
filename[RW]
last_state[RW]
lex_state[RW]

Additional context surrounding tokens that both the lexer and grammar use.

lex_strterm[RW]
lineno[RW]
lpar_beg[RW]
match[RW]
old_lineno=[RW]
paren_nest[RW]
parser[RW]
space_seen[RW]
ss[RW]
state[RW]
string_buffer[RW]
string_nest[RW]
token[RW]

Last token read via next_token.

version[RW]

What version of ruby to parse. 18 and 19 are the only valid values currently supported.

### Public Class Methods

new(v = 18) click to toggle source

    # File lib/ruby_lexer.rb, line 102
    def initialize v = 18
      self.version = v

      reset
    end

### Public Instance Methods

action() click to toggle source

    # File lib/ruby_lexer.rex.rb, line 41
    def action
      yield
    end

arg_ambiguous() click to toggle source

    # File lib/ruby_lexer.rb, line 108
    def arg_ambiguous
      self.warning("Ambiguous first argument. make sure.")
    end

arg_state() click to toggle source

    # File lib/ruby_lexer.rb, line 112
    def arg_state
      in_arg_state? ? :expr_arg : :expr_beg
    end

beginning_of_line?() click to toggle source

    # File lib/ruby_lexer.rb, line 116
    def beginning_of_line?
      ss.bol?
    end

Also aliased as: bol?
bol?() click to toggle source
Alias for: beginning_of_line?
check(re) click to toggle source

    # File lib/ruby_lexer.rb, line 901
    def check re
      ss.check re
    end

comments() click to toggle source

    # File lib/ruby_lexer.rb, line 121
    def comments # TODO: remove this... maybe comment_string + attr_accessor
      c = @comments.join
      @comments.clear
      c
    end

d(o) click to toggle source

    # File lib/ruby_lexer.rb, line 1225
    def d o
      $stderr.puts o.inspect
    end

end_of_stream?() click to toggle source

    # File lib/ruby_lexer.rb, line 127
    def end_of_stream?
      ss.eos?
    end

expr_dot?() click to toggle source

    # File lib/ruby_lexer.rb, line 131
    def expr_dot?
      lex_state == :expr_dot
    end

expr_fname?() click to toggle source

    # File lib/ruby_lexer.rb, line 135
    def expr_fname?
      lex_state == :expr_fname
    end

expr_result(token, text) click to toggle source

    # File lib/ruby_lexer.rb, line 139
    def expr_result token, text
      cond.push false
      cmdarg.push false
      result :expr_beg, token, text
    end

heredoc(here) click to toggle source

    # File lib/ruby_lexer.rb, line 145
    def heredoc here # TODO: rewrite / remove
      _, eos, func, last_line = here

      indent  = (func & STR_FUNC_INDENT) != 0 ? "[ \t]*" : nil
      expand  = (func & STR_FUNC_EXPAND) != 0
      eos_re  = /#{indent}#{Regexp.escape eos}(\r*\n|\z)/
      err_msg = "can't match #{eos_re.inspect} anywhere in "

      rb_compile_error err_msg if end_of_stream?

      if beginning_of_line? && scan(eos_re) then
        self.lineno += 1
        ss.unread_many last_line # TODO: figure out how to remove this
        return :tSTRING_END, eos
      end

      self.string_buffer = []

      if expand then
        case
        when scan(/#[$@]/) then
          ss.pos -= 1 # FIX omg stupid
          return :tSTRING_DVAR, matched
        when scan(/#[{]/) then
          return :tSTRING_DBEG, matched
        when scan(/#/) then
          string_buffer << '#'
        end

        begin
          c = tokadd_string func, "\n", nil

          rb_compile_error err_msg if
            c == RubyLexer::EOF

          if c != "\n" then
            return :tSTRING_CONTENT, string_buffer.join.delete("\r")
          else
            string_buffer << scan(/\n/)
          end

          rb_compile_error err_msg if end_of_stream?
        end until check(eos_re)
      else
        until check(eos_re) do
          string_buffer << scan(/.*(\n|\z)/)
          rb_compile_error err_msg if end_of_stream?
        end
      end

      self.lex_strterm = [:heredoc, eos, func, last_line]

      return :tSTRING_CONTENT, string_buffer.join.delete("\r")
    end

heredoc_identifier() click to toggle source

    # File lib/ruby_lexer.rb, line 200
    def heredoc_identifier # TODO: remove / rewrite
      term, func = nil, STR_FUNC_BORING
      self.string_buffer = []

      case
      when scan(/(-?)([\'\"\`])(.*?)\22//) then
        term = ss[2]
        func |= STR_FUNC_INDENT unless ss[1].empty?
        func |= case term
                when "\'" then
                  STR_SQUOTE
                when '"' then
                  STR_DQUOTE
                else
                  STR_XQUOTE
                end
        string_buffer << ss[3]
      when scan(/-?([\'\"\`])(?!\11**\Z)/) then
        rb_compile_error "unterminated here document identifier"
      when scan(/(-?)(#{IDENT_CHAR}+)/) then
        term = '"'
        func |= STR_DQUOTE
        unless ss[1].empty? then
          func |= STR_FUNC_INDENT
        end
        string_buffer << ss[2]
      else
        return nil
      end

      if scan(/.*\n/) then
        # TODO: think about storing off the char range instead
        line = matched
      else
        line = nil
      end

      self.lex_strterm = [:heredoc, string_buffer.join, func, line]

      if term == '`' then
        result nil, :tXSTRING_BEG, "`"
      else
        result nil, :tSTRING_BEG, "\""
      end
    end

in_arg_state?() click to toggle source

    # File lib/ruby_lexer.rb, line 250
    def in_arg_state? # TODO: rename is_after_operator?
      in_lex_state? :expr_fname, :expr_dot
    end

in_fname?() click to toggle source

    # File lib/ruby_lexer.rb, line 246
    def in_fname?
      in_lex_state? :expr_fname
    end

in_lex_state?(*states) click to toggle source

    # File lib/ruby_lexer.rb, line 254
    def in_lex_state?(*states)
      states.include? lex_state
    end

int_with_base(base) click to toggle source

    # File lib/ruby_lexer.rb, line 258
    def int_with_base base
      rb_compile_error "Invalid numeric format" if matched =~ /__/
      return result(:expr_end, :tINTEGER, matched.to_i(base))
    end

is_arg?() click to toggle source

    # File lib/ruby_lexer.rb, line 263
    def is_arg?
      in_lex_state? :expr_arg, :expr_cmdarg
    end

is_beg?() click to toggle source

    # File lib/ruby_lexer.rb, line 267
    def is_beg?
      in_lex_state? :expr_beg, :expr_value, :expr_mid, :expr_class, :expr_labelarg
    end

is_end?() click to toggle source

    # File lib/ruby_lexer.rb, line 271
    def is_end?
      in_lex_state? :expr_end, :expr_endarg, :expr_endfn
    end

is_label_possible?() click to toggle source

    # File lib/ruby_lexer.rb, line 279
    def is_label_possible?
      (in_lex_state?(:expr_beg, :expr_endfn) && !command_state) || is_arg?
    end

is_label_suffix?() click to toggle source

    # File lib/ruby_lexer.rb, line 283
    def is_label_suffix?
      check(/:(?!:)/)
    end

is_space_arg?(c = "x") click to toggle source

    # File lib/ruby_lexer.rb, line 287
    def is_space_arg? c = "x"
      is_arg? and space_seen and c !~ /\s/
    end

lineno=(n) click to toggle source

    # File lib/ruby_lexer.rb, line 1229
    def lineno= n
      self.old_lineno= n
      where = caller.first.split(/:/).first(2).join(":")
      d :lineno => [n, where, ss && ss.rest[0,40]]
    end

matched() click to toggle source

    # File lib/ruby_lexer.rb, line 291
    def matched
      ss.matched
    end

matches() click to toggle source

    # File lib/ruby_lexer.rex.rb, line 35
    def matches
      m = (1..9).map { |i| ss[i] }
      m.pop until m[-1] or m.empty?
      m
    end

next_token() click to toggle source

    # File lib/ruby_lexer.rex.rb, line 64
    def next_token
      return process_string if lex_strterm
      self.command_state = self.command_start
      self.command_start = false
      self.space_seen    = false
      self.last_state    = lex_state

      token = nil

      until ss.eos? or token do
        token =
          case state
          when nil then
            case
            when text = ss.scan(/[\ \t\r\f\v]/) then
              action { self.space_seen = true; next }
            when text = ss.scan(/\n|\#/) then
              process_newline_or_comment text
            when text = ss.scan(/[\]\)\}]/) then
              process_bracing text
            when ss.check(/\!/) then
              case
              when in_arg_state? && (text = ss.scan(/\!\@/)) then
                action { result :expr_arg, :tUBANG, "!@" }
              when text = ss.scan(/\![=~]?/) then
                action { result :arg_state, TOKENS[text], text }
              end # group /\!/
            when ss.check(/\./) then
              case
              when text = ss.scan(/\.\.\.?/) then
                action { result :expr_beg, TOKENS[text], text }
              when text = ss.scan(/\.\d/) then
                action { rb_compile_error "no .<digit> floating literal anymore put 0 before dot" }
              when text = ss.scan(/\./) then
                action { result :expr_dot, :tDOT, "." }
              end # group /\./
            when text = ss.scan(/\(/) then
              process_paren text
            when text = ss.scan(/\,/) then
              action { result :expr_beg, TOKENS[text], text }
            when ss.check(/=/) then
              case
              when text = ss.scan(/\=\=\=|\=\=|\=~|\=>|\=(?!begin\b)/) then
                action { result arg_state, TOKENS[text], text }
              when bol? && (text = ss.scan(/\=begin(?=\s)/)) then
                process_begin text
              when text = ss.scan(/\=(?=begin\b)/) then
                action { result arg_state, TOKENS[text], text }
              end # group /=/
            when ruby22_label? && (text = ss.scan(/\"(#{SIMPLE_STRING})\":/)) then
              process_label text
            when text = ss.scan(/\"(#{SIMPLE_STRING})\"/) then
              action { result :expr_end, :tSTRING, text[1..-2].gsub(ESC) { unescape $1 } }
            when text = ss.scan(/\"/) then
              action { string STR_DQUOTE; result nil, :tSTRING_BEG, text }
            when text = ss.scan(/\@\@?\d/) then
              action { rb_compile_error "`#{text}` is not allowed as a variable name" }
            when text = ss.scan(/\@\@?#{IDENT_CHAR}+/) then
              process_ivar text
            when ss.check(/:/) then
              case
              when not_end? && (text = ss.scan(/:([a-zA-Z_]#{IDENT_CHAR}*(?:[?]|[!](?!=)|=(?==>)|=(?![=>]))?)/)) then
                process_symbol text
              when not_end? && (text = ss.scan(/\:\"(#{SIMPLE_STRING})\"/)) then
                process_symbol text
              when not_end? && (text = ss.scan(/\:\'(#{SSTRING})\'/)) then
                process_symbol text
              when text = ss.scan(/\:\:/) then
                process_colon2 text
              when text = ss.scan(/\:/) then
                process_colon1 text
              end # group /:/
            when text = ss.scan(/->/) then
              action { result :expr_endfn, :tLAMBDA, nil }
            when text = ss.scan(/[+-]/) then
              process_plus_minus text
            when ss.check(/[+\d]/) then
              case
              when text = ss.scan(/#{NUM_BAD}/) then
                action { rb_compile_error "Invalid numeric format"  }
              when text = ss.scan(/#{INT_DEC}/) then
                action { int_with_base 10                           }
              when text = ss.scan(/#{INT_HEX}/) then
                action { int_with_base 16                           }
              when text = ss.scan(/#{INT_BIN}/) then
                action { int_with_base 2                            }
              when text = ss.scan(/#{INT_OCT_BAD}/) then
                action { rb_compile_error "Illegal octal digit."    }
              when text = ss.scan(/#{INT_OCT}/) then
                action { int_with_base 8                            }
              when text = ss.scan(/#{FLOAT_BAD}/) then
                action { rb_compile_error "Trailing '_' in number." }
              when text = ss.scan(/#{FLOAT}/) then
                process_float text
              when text = ss.scan(/#{INT_DEC2}/) then
                action { int_with_base 10                           }
              when text = ss.scan(/[0-9]/) then
                action { rb_compile_error "Bad number format" }
              end # group /[+\d]/
            when text = ss.scan(/\[/) then
              process_square_bracket text
            when was_label? && (text = ss.scan(/\'#{SSTRING}\':?/)) then
              process_label_or_string text
            when ss.check(/\|/) then
              case
              when text = ss.scan(/\|\|\=/) then
                action { result :expr_beg, :tOP_ASGN, "||" }
              when text = ss.scan(/\|\|/) then
                action { result :expr_beg, :tOROP,    "||" }
              when text = ss.scan(/\|\=/) then
                action { result :expr_beg, :tOP_ASGN, "|" }
              when text = ss.scan(/\|/) then
                action { result :arg_state, :tPIPE,    "|" }
              end # group /\|/
            when text = ss.scan(/\{/) then
              process_curly_brace text
            when ss.check(/\*/) then
              case
              when text = ss.scan(/\*\*=/) then
                action { result :expr_beg, :tOP_ASGN, "**" }
              when text = ss.scan(/\*\*/) then
                action { result(:arg_state, space_vs_beginning(:tDSTAR, :tDSTAR, :tPOW), "**") }
              when text = ss.scan(/\*\=/) then
                action { result(:expr_beg, :tOP_ASGN, "*") }
              when text = ss.scan(/\*/) then
                action { result(:arg_state, space_vs_beginning(:tSTAR, :tSTAR, :tSTAR2), "*") }
              end # group /\*/
            when ss.check(/</) then
              case
              when text = ss.scan(/\<\=\>/) then
                action { result :arg_state, :tCMP, "<=>"    }
              when text = ss.scan(/\<\=/) then
                action { result :arg_state, :tLEQ, "<="     }
              when text = ss.scan(/\<\<\=/) then
                action { result :arg_state, :tOP_ASGN, "<<" }
              when text = ss.scan(/\<\</) then
                process_lchevron text
              when text = ss.scan(/\</) then
                action { result :arg_state, :tLT, "<"       }
              end # group /</
            when ss.check(/>/) then
              case
              when text = ss.scan(/\>\=/) then
                action { result :arg_state, :tGEQ, ">="     }
              when text = ss.scan(/\>\>=/) then
                action { result :arg_state, :tOP_ASGN, ">>" }
              when text = ss.scan(/\>\>/) then
                action { result :arg_state, :tRSHFT, ">>"   }
              when text = ss.scan(/\>/) then
                action { result :arg_state, :tGT, ">"       }
              end # group />/
            when ss.check(/\`/) then
              case
              when expr_fname? && (text = ss.scan(/\`/)) then
                action { result(:expr_end, :tBACK_REF2, "`") }
              when expr_dot? && (text = ss.scan(/\`/)) then
                action { result((command_state ? :expr_cmdarg : :expr_arg), :tBACK_REF2, "`") }
              when text = ss.scan(/\`/) then
                action { string STR_XQUOTE, '`'; result(nil, :tXSTRING_BEG, "`") }
              end # group /\`/
            when text = ss.scan(/\?/) then
              process_questionmark text
            when ss.check(/&/) then
              case
              when text = ss.scan(/\&\&\=/) then
                action { result(:expr_beg, :tOP_ASGN, "&&") }
              when text = ss.scan(/\&\&/) then
                action { result(:expr_beg, :tANDOP,   "&&") }
              when text = ss.scan(/\&\=/) then
                action { result(:expr_beg, :tOP_ASGN, "&" ) }
              when text = ss.scan(/\&/) then
                process_amper text
              end # group /&/
            when text = ss.scan(/\//) then
              process_slash text
            when ss.check(/\^/) then
              case
              when text = ss.scan(/\^=/) then
                action { result(:expr_beg, :tOP_ASGN, "^") }
              when text = ss.scan(/\^/) then
                action { result(:arg_state, :tCARET, "^") }
              end # group /\^/
            when text = ss.scan(/\;/) then
              action { self.command_start = true; result(:expr_beg, :tSEMI, ";") }
            when ss.check(/~/) then
              case
              when in_arg_state? && (text = ss.scan(/\~@/)) then
                action { result(:arg_state, :tTILDE, "~") }
              when text = ss.scan(/\~/) then
                action { result(:arg_state, :tTILDE, "~") }
              end # group /~/
            when ss.check(/\\/) then
              case
              when text = ss.scan(/\\\r?\n/) then
                action { self.lineno += 1; self.space_seen = true; next }
              when text = ss.scan(/\\/) then
                action { rb_compile_error "bare backslash only allowed before newline" }
              end # group /\\/
            when text = ss.scan(/\%/) then
              process_percent text
            when ss.check(/\$/) then
              case
              when text = ss.scan(/\$_\w+/) then
                process_gvar text
              when text = ss.scan(/\$_/) then
                process_gvar text
              when text = ss.scan(/\$[~*$?!@\/\\;,.=:<>\"]|\$-\w?/) then
                process_gvar text
              when in_fname? && (text = ss.scan(/\$([\&\`\'\+])/)) then
                process_gvar text
              when text = ss.scan(/\$([\&\`\'\+])/) then
                process_backref text
              when in_fname? && (text = ss.scan(/\$([1-9]\d*)/)) then
                process_gvar text
              when text = ss.scan(/\$([1-9]\d*)/) then
                process_nthref text
              when text = ss.scan(/\$0/) then
                process_gvar text
              when text = ss.scan(/\$\W|\$\z/) then
                process_gvar_oddity text
              when text = ss.scan(/\$\w+/) then
                process_gvar text
              end # group /\$/
            when text = ss.scan(/\_/) then
              process_underscore text
            when text = ss.scan(/#{IDENT}/) then
              process_token text
            when text = ss.scan(/\0004|\0032|\0000|\Z/) then
              action { [RubyLexer::EOF, RubyLexer::EOF] }
            when text = ss.scan(/./) then
              action { rb_compile_error "Invalid char #{text.inspect} in expression" }
            else
              text = ss.string[ss.pos .. -1]
              raise ScanError, "can not match (#{state.inspect}): '#{text}'"
            end
          else
            raise ScanError, "undefined state: '#{state}'"
          end # token = case state

        next unless token # allow functions to trigger redo w/ nil
      end # while

      raise "bad lexical result: #{token.inspect}" unless
        token.nil? || (Array === token && token.size >= 2)

      # auto-switch state
      self.state = token.last if token && token.first == :state

      token
    end

not_end?() click to toggle source

    # File lib/ruby_lexer.rb, line 295
    def not_end?
      not is_end?
    end

parse(str) click to toggle source

    # File lib/ruby_lexer.rex.rb, line 50
    def parse str
      self.ss     = scanner_class.new str
      self.state  ||= nil

      do_parse
    end

parse_file(path) click to toggle source

    # File lib/ruby_lexer.rex.rb, line 57
    def parse_file path
      self.filename = path
      open path do |f|
        parse f.read
      end
    end

parse_quote() click to toggle source

    # File lib/ruby_lexer.rb, line 1105
    def parse_quote # TODO: remove / rewrite
      beg, nnd, short_hand, c = nil, nil, false, nil

      if scan(/[a-z0-9]{1,2}/) then # Long-hand (e.g. %Q{}).
        rb_compile_error "unknown type of %string" if ss.matched_size == 2
        c, beg, short_hand = matched, ss.getch, false
      else                               # Short-hand (e.g. %{, %., %!, etc)
        c, beg, short_hand = 'Q', ss.getch, true
      end

      if end_of_stream? or c == RubyLexer::EOF or beg == RubyLexer::EOF then
        rb_compile_error "unterminated quoted string meets end of file"
      end

      # Figure nnd-char.  "\0" is special to indicate beg=nnd and that no nesting?
      nnd = { "(" => ")", "[" => "]", "{" => "}", "<" => ">" }[beg]
      nnd, beg = beg, "\00"" if nnd.nil?

      token_type, text = nil, "%#{c}#{beg}"
      token_type, string_type = case c
                                when 'Q' then
                                  ch = short_hand ? nnd : c + beg
                                  text = "%#{ch}"
                                  [:tSTRING_BEG,   STR_DQUOTE]
                                when 'q' then
                                  [:tSTRING_BEG,   STR_SQUOTE]
                                when 'W' then
                                  scan(/\s*/)
                                  [:tWORDS_BEG,    STR_DQUOTE | STR_FUNC_QWORDS]
                                when 'w' then
                                  scan(/\s*/)
                                  [:tQWORDS_BEG,   STR_SQUOTE | STR_FUNC_QWORDS]
                                when 'x' then
                                  [:tXSTRING_BEG,  STR_XQUOTE]
                                when 'r' then
                                  [:tREGEXP_BEG,   STR_REGEXP]
                                when 's' then
                                  self.lex_state  = :expr_fname
                                  [:tSYMBEG,       STR_SSYM]
                                when 'I' then
                                  scan(/\s*/)
                                  [:tSYMBOLS_BEG, STR_DQUOTE | STR_FUNC_QWORDS]
                                when 'i' then
                                  scan(/\s*/)
                                  [:tQSYMBOLS_BEG, STR_SQUOTE | STR_FUNC_QWORDS]
                                end

      rb_compile_error "Bad %string type. Expected [QqWwIixrs], found '#{c}'." if
        token_type.nil?

      raise "huh" unless string_type

      string string_type, nnd, beg

      return token_type, text
    end

parse_string(quote) click to toggle source

    # File lib/ruby_lexer.rb, line 1162
    def parse_string quote # TODO: rewrite / remove
      _, string_type, term, open = quote

      space = false # FIX: remove these
      func = string_type
      paren = open
      term_re = @@regexp_cache[term]

      qwords = (func & STR_FUNC_QWORDS) != 0
      regexp = (func & STR_FUNC_REGEXP) != 0
      expand = (func & STR_FUNC_EXPAND) != 0

      unless func then # nil'ed from qwords below. *sigh*
        return :tSTRING_END, nil
      end

      space = true if qwords and scan(/\s+/)

      if self.string_nest == 0 && scan(/#{term_re}/) then
        if qwords then
          quote[1] = nil
          return :tSPACE, nil
        elsif regexp then
          return :tREGEXP_END, self.regx_options
        else
          return :tSTRING_END, term
        end
      end

      return :tSPACE, nil if space

      self.string_buffer = []

      if expand
        case
        when scan(/#(?=\$(-.|[a-zA-Z_0-9~\*\$\?!@\/\\;,\.=:<>\"\&\`\'+]))/) then
          # TODO: !ISASCII
          # ?! see parser_peek_variable_name
          return :tSTRING_DVAR, nil
        when scan(/#(?=\@\@?[a-zA-Z_])/) then
          # TODO: !ISASCII
          return :tSTRING_DVAR, nil
        when scan(/#[{]/) then
          return :tSTRING_DBEG, nil
        when scan(/#/) then
          string_buffer << '#'
        end
      end

      if tokadd_string(func, term, paren) == RubyLexer::EOF then
        rb_compile_error "unterminated string meets end of file"
      end

      return :tSTRING_CONTENT, string_buffer.join
    end

process_amper(text) click to toggle source

    # File lib/ruby_lexer.rb, line 299
    def process_amper text
      token = if is_arg? && space_seen && !check(/\s/) then
                 warning("`&' interpreted as argument prefix")
                 :tAMPER
               elsif in_lex_state? :expr_beg, :expr_mid then
                 :tAMPER
               else
                 :tAMPER2
               end

      return result(:arg_state, token, "&")
    end

process_backref(text) click to toggle source

    # File lib/ruby_lexer.rb, line 312
    def process_backref text
      token = ss[1].to_sym
      # TODO: can't do lineno hack w/ symbol
      result :expr_end, :tBACK_REF, token
    end

process_begin(text) click to toggle source

    # File lib/ruby_lexer.rb, line 318
    def process_begin text
      @comments << matched

      unless scan(/.*?\n=end( |\t|\f)*[^\n]*(\n|\z)/) then
        @comments.clear
        rb_compile_error("embedded document meets end of file")
      end

      @comments << matched
      self.lineno += matched.count("\n")

      nil # TODO
    end

process_bracing(text) click to toggle source

    # File lib/ruby_lexer.rb, line 332
    def process_bracing text
      cond.lexpop
      cmdarg.lexpop

      case matched
      when "}" then
        self.brace_nest -= 1
        self.lex_state   = :expr_endarg

        # TODO
        # if (c == '}') {
        #     if (!brace_nest--) c = tSTRING_DEND;
        # }

        return :tRCURLY, matched
      when "]" then
        self.paren_nest -= 1
        self.lex_state   = :expr_endarg
        return :tRBRACK, matched
      when ")" then
        self.paren_nest -= 1
        self.lex_state   = :expr_endfn
        return :tRPAREN, matched
      else
        raise "Unknown bracing: #{matched.inspect}"
      end
    end

process_colon1(text) click to toggle source

    # File lib/ruby_lexer.rb, line 360
    def process_colon1 text
      # ?: / then / when
      if is_end? || check(/\s/) then
        return result :expr_beg, :tCOLON, text
      end

      case
      when scan(/\'/) then
        string STR_SSYM
      when scan(/\"/) then
        string STR_DSYM
      end

      result :expr_fname, :tSYMBEG, text
    end

process_colon2(text) click to toggle source

    # File lib/ruby_lexer.rb, line 376
    def process_colon2 text
      if is_beg? || in_lex_state?(:expr_class) || is_space_arg? then
        result :expr_beg, :tCOLON3, text
      else
        result :expr_dot, :tCOLON2, text
      end
    end

process_curly_brace(text) click to toggle source

    # File lib/ruby_lexer.rb, line 384
    def process_curly_brace text
      self.brace_nest += 1
      if lpar_beg && lpar_beg == paren_nest then
        self.lpar_beg = nil
        self.paren_nest -= 1

        return expr_result(:tLAMBEG, "{")
      end

      token = if is_arg? || in_lex_state?(:expr_end, :expr_endfn) then
                 :tLCURLY      #  block (primary)
               elsif in_lex_state?(:expr_endarg) then
                 :tLBRACE_ARG  #  block (expr)
               else
                 :tLBRACE      #  hash
               end

      self.command_start = true unless token == :tLBRACE

      return expr_result(token, "{")
    end

process_float(text) click to toggle source

    # File lib/ruby_lexer.rb, line 406
    def process_float text
      rb_compile_error "Invalid numeric format" if text =~ /__/
      return result(:expr_end, :tFLOAT, text.to_f)
    end

process_gvar(text) click to toggle source

    # File lib/ruby_lexer.rb, line 411
    def process_gvar text
      text.lineno = self.lineno
      result(:expr_end, :tGVAR, text)
    end

process_gvar_oddity(text) click to toggle source

    # File lib/ruby_lexer.rb, line 416
    def process_gvar_oddity text
      return result :expr_end, "$", "$" if text == "$" # TODO: wtf is this?
      rb_compile_error "#{text.inspect} is not allowed as a global variable name"
    end

process_ivar(text) click to toggle source

    # File lib/ruby_lexer.rb, line 421
    def process_ivar text
      tok_id = text =~ /^@@/ ? :tCVAR : :tIVAR
      text.lineno = self.lineno
      return result(:expr_end, tok_id, text)
    end

process_label(text) click to toggle source

    # File lib/ruby_lexer.rb, line 681
    def process_label text
      result = process_symbol text
      result[0] = :tLABEL
      result
    end

process_label_or_string(text) click to toggle source

    # File lib/ruby_lexer.rb, line 669
    def process_label_or_string text
      if @was_label && text =~ /:$/ then
        @was_label = nil
        return process_label text
      elsif text =~ /:$/ then
        ss.pos -= 1 # put back ":"
        text = text[0..-2]
      end

      result :expr_end, :tSTRING, text[1..-2].gsub(/\\\\/, "\\").gsub(/\\'/, "'")
    end

process_lchevron(text) click to toggle source

    # File lib/ruby_lexer.rb, line 427
    def process_lchevron text
      if (!in_lex_state?(:expr_dot, :expr_class) &&
          !is_end? &&
          (!is_arg? || space_seen)) then
        tok = self.heredoc_identifier
        return tok if tok
      end

      return result(:arg_state, :tLSHFT, "\<\<")
    end

process_newline_or_comment(text) click to toggle source

    # File lib/ruby_lexer.rb, line 438
    def process_newline_or_comment text
      c = matched
      hit = false

      if c == '#' then
        ss.pos -= 1

        while scan(/\s*\#.*(\n+|\z)/) do
          hit = true
          self.lineno += matched.lines.to_a.size
          @comments << matched.gsub(/^ +#/, '#').gsub(/^ +$/, '')
        end

        return nil if end_of_stream?
      end

      self.lineno += 1 unless hit

      # Replace a string of newlines with a single one
      self.lineno += matched.lines.to_a.size if scan(/\n+/)

      return if in_lex_state?(:expr_beg, :expr_value, :expr_class,
                              :expr_fname, :expr_dot, :expr_labelarg)

      if scan(/([\ \t\r\f\v]*)\./) then
        self.space_seen = true unless ss[1].empty?

        ss.pos -= 1
        return unless check(/\.\./)
      end

      self.command_start = true

      return result(:expr_beg, :tNL, nil)
    end

process_nthref(text) click to toggle source

    # File lib/ruby_lexer.rb, line 474
    def process_nthref text
      # TODO: can't do lineno hack w/ number
      result :expr_end, :tNTH_REF, ss[1].to_i
    end

process_paren(text) click to toggle source

    # File lib/ruby_lexer.rb, line 479
    def process_paren text
      token = if ruby18 then
                process_paren18
              else
                process_paren19
              end

      self.paren_nest += 1

      return expr_result(token, "(")
    end

process_paren18() click to toggle source

    # File lib/ruby_lexer.rb, line 491
    def process_paren18
      self.command_start = true
      token = :tLPAREN2

      if in_lex_state? :expr_beg, :expr_mid then
        token = :tLPAREN
      elsif space_seen then
        if in_lex_state? :expr_cmdarg then
          token = :tLPAREN_ARG
        elsif in_lex_state? :expr_arg then
          warning "don't put space before argument parentheses"
        end
      else
        # not a ternary -- do nothing?
      end

      token
    end

process_paren19() click to toggle source

    # File lib/ruby_lexer.rb, line 510
    def process_paren19
      if is_beg? then
        :tLPAREN
      elsif is_space_arg? then
        :tLPAREN_ARG
      else
        :tLPAREN2 # plain '(' in parse.y
      end
    end

process_percent(text) click to toggle source

    # File lib/ruby_lexer.rb, line 520
    def process_percent text
      return parse_quote if is_beg?

      return result(:expr_beg, :tOP_ASGN, "%") if scan(/\=/)

      return parse_quote if is_arg? && space_seen && ! check(/\s/)

      return result(:arg_state, :tPERCENT, "%")
    end

process_plus_minus(text) click to toggle source

    # File lib/ruby_lexer.rb, line 530
    def process_plus_minus text
      sign = matched
      utype, type = if sign == "+" then
                      [:tUPLUS, :tPLUS]
                    else
                      [:tUMINUS, :tMINUS]
                    end

      if in_arg_state? then
        if scan(/@/) then
          return result(:expr_arg, utype, "#{sign}@")
        else
          return result(:expr_arg, type, sign)
        end
      end

      return result(:expr_beg, :tOP_ASGN, sign) if scan(/\=/)

      if (is_beg? || (is_arg? && space_seen && !check(/\s/))) then
        arg_ambiguous if is_arg?

        if check(/\d/) then
          return nil if utype == :tUPLUS
          return result(:expr_beg, :tUMINUS_NUM, sign)
        end

        return result(:expr_beg, utype, sign)
      end

      return result(:expr_beg, type, sign)
    end

process_questionmark(text) click to toggle source

    # File lib/ruby_lexer.rb, line 562
    def process_questionmark text
      if is_end? then
        state = ruby18 ? :expr_beg : :expr_value # HACK?
        return result(state, :tEH, "?")
      end

      if end_of_stream? then
        rb_compile_error "incomplete character syntax: parsed #{text.inspect}"
      end

      if check(/\s|\v/) then
        unless is_arg? then
          c2 = { " " => 's',
                "\n" => 'n',
                "\t" => 't',
                "\v" => 'v',
                "\r" => 'r',
                "\f" => 'f' }[matched]

          if c2 then
            warning("invalid character syntax; use ?\\" + c2)
          end
        end

        # ternary
        state = ruby18 ? :expr_beg : :expr_value # HACK?
        return result(state, :tEH, "?")
      elsif check(/\w(?=\w)/) then # ternary, also
        return result(:expr_beg, :tEH, "?")
      end

      c = if scan(/\\/) then
            self.read_escape
          else
            ss.getch
          end

      if version == 18 then
        return result(:expr_end, :tINTEGER, c[0].ord & 0xff)
      else
        return result(:expr_end, :tSTRING, c)
      end
    end

process_slash(text) click to toggle source

    # File lib/ruby_lexer.rb, line 606
    def process_slash text
      if is_beg? then
        string STR_REGEXP

        return result(nil, :tREGEXP_BEG, "/")
      end

      if scan(/\=/) then
        return result(:expr_beg, :tOP_ASGN, "/")
      end

      if is_arg? && space_seen then
        unless scan(/\s/) then
          arg_ambiguous
          string STR_REGEXP, "/"
          return result(nil, :tREGEXP_BEG, "/")
        end
      end

      return result(:arg_state, :tDIVIDE, "/")
    end

process_square_bracket(text) click to toggle source

    # File lib/ruby_lexer.rb, line 628
    def process_square_bracket text
      self.paren_nest += 1

      token = nil

      if in_arg_state? then
        case
        when scan(/\]\=/) then
          self.paren_nest -= 1 # HACK? I dunno, or bug in MRI
          return result(:expr_arg, :tASET, "[]=")
        when scan(/\]/) then
          self.paren_nest -= 1 # HACK? I dunno, or bug in MRI
          return result(:expr_arg, :tAREF, "[]")
        else
          rb_compile_error "unexpected '['"
        end
      elsif is_beg? then
        token = :tLBRACK
      elsif is_arg? && space_seen then
        token = :tLBRACK
      else
        token = :tLBRACK2
      end

      return expr_result(token, "[")
    end

process_string() click to toggle source

    # File lib/ruby_lexer.rb, line 1079
    def process_string # TODO: rewrite / remove
      token = if lex_strterm[0] == :heredoc then
                self.heredoc lex_strterm
              else
                self.parse_string lex_strterm
              end

      token_type, c = token

      if ruby22? && token_type == :tSTRING_END && ["'", '"'].include?(c) then
        if (([:expr_beg, :expr_endfn].include?(lex_state) &&
             !cond.is_in_state) || is_arg?) &&
            is_label_suffix? then
          scan(/:/)
          token_type = token[0] = :tLABEL_END
        end
      end

      if [:tSTRING_END, :tREGEXP_END, :tLABEL_END].include? token_type then
        self.lex_strterm = nil
        self.lex_state   = (token_type == :tLABEL_END) ? :expr_labelarg : :expr_end
      end

      return token
    end

process_symbol(text) click to toggle source

    # File lib/ruby_lexer.rb, line 655
    def process_symbol text
      symbol = match[1].gsub(ESC) { unescape $1 }

      rb_compile_error "symbol cannot contain '\\0'" if
        ruby18 && symbol =~ /\00//

      return result(:expr_end, :tSYMBOL, symbol)
    end

process_token(text) click to toggle source

    # File lib/ruby_lexer.rb, line 687
    def process_token text
      # TODO: make this always return [token, lineno]
      token = self.token = text
      token << matched if scan(/[\!\?](?!=)/)

      tok_id =
        case
        when token =~ /[!?]$/ then
          :tFID
        when in_lex_state?(:expr_fname) && scan(/=(?:(?![~>=])|(?==>))/) then
          # ident=, not =~ => == or followed by =>
          # TODO test lexing of a=>b vs a==>b
          token << matched
          :tIDENTIFIER
        when token =~ /^[A-Z]/ then
          :tCONSTANT
        else
          :tIDENTIFIER
        end

      if !ruby18 and is_label_possible? and is_label_suffix? then
        scan(/:/)
        return result(:expr_labelarg, :tLABEL, [token, self.lineno])
      end

      unless in_lex_state? :expr_dot then
        # See if it is a reserved word.
        keyword = if ruby18 then # REFACTOR need 18/19 lexer subclasses
                    RubyParserStuff::Keyword.keyword18 token
                  else
                    RubyParserStuff::Keyword.keyword19 token
                  end

        return process_token_keyword keyword if keyword
      end # unless in_lex_state? :expr_dot

      # TODO:
      # if (mb == ENC_CODERANGE_7BIT && lex_state != EXPR_DOT) {

      state = if is_beg? or is_arg? or in_lex_state? :expr_dot then
                command_state ? :expr_cmdarg : :expr_arg
              elsif not ruby18 and in_lex_state? :expr_fname then
                :expr_endfn
              else
                :expr_end
              end

      if not [:expr_dot, :expr_fname].include? last_state and
          self.parser.env[token.to_sym] == :lvar then
        state = :expr_end
      end

      token.lineno = self.lineno # yes, on a string. I know... I know...

      return result(state, tok_id, token)
    end

process_token_keyword(keyword) click to toggle source

    # File lib/ruby_lexer.rb, line 744
    def process_token_keyword keyword
      state = keyword.state

      value = [token, self.lineno]

      self.command_start = true if state == :expr_beg and lex_state != :expr_fname

      case
      when lex_state == :expr_fname then
        result(state, keyword.id0, keyword.name)
      when keyword.id0 == :kDO then
        case
        when lpar_beg && lpar_beg == paren_nest then
          self.lpar_beg = nil
          self.paren_nest -= 1
          result(state, :kDO_LAMBDA, value)
        when cond.is_in_state then
          result(state, :kDO_COND, value)
        when cmdarg.is_in_state && lex_state != :expr_cmdarg then
          result(state, :kDO_BLOCK, value)
        when in_lex_state?(:expr_beg, :expr_endarg) then
          result(state, :kDO_BLOCK, value)
        else
          result(state, :kDO, value)
        end
      when in_lex_state?(:expr_beg, :expr_value) then # TODO: :expr_labelarg
        result(state, keyword.id0, value)
      when keyword.id0 != keyword.id1 then
        result(:expr_beg, keyword.id1, value)
      else
        result(state, keyword.id1, value)
      end
    end

process_underscore(text) click to toggle source

    # File lib/ruby_lexer.rb, line 778
    def process_underscore text
      ss.unscan # put back "_"

      if beginning_of_line? && scan(/\__END__(\r?\n|\Z)/) then
        return [RubyLexer::EOF, RubyLexer::EOF]
      elsif scan(/\_\w*/) then
        return process_token matched
      end
    end

rb_compile_error(msg) click to toggle source

    # File lib/ruby_lexer.rb, line 788
    def rb_compile_error msg
      msg += ". near line #{self.lineno}: #{ss.rest[/^.*/].inspect}"
      raise RubyParser::SyntaxError, msg
    end

read_escape() click to toggle source

    # File lib/ruby_lexer.rb, line 793
    def read_escape # TODO: remove / rewrite
      case
      when scan(/\\/) then                  # Backslash
        '\'
      when scan(/n/) then                   # newline
        "\n"
      when scan(/t/) then                   # horizontal tab
        "\t"
      when scan(/r/) then                   # carriage-return
        "\r"
      when scan(/f/) then                   # form-feed
        "\f"
      when scan(/v/) then                   # vertical tab
        "\113""
      when scan(/a/) then                   # alarm(bell)
        "\0007"
      when scan(/e/) then                   # escape
        "\0033"
      when scan(/b/) then                   # backspace
        "\0010"
      when scan(/s/) then                   # space
        " "
      when scan(/[0-7]{1,3}/) then          # octal constant
        (matched.to_i(8) & 0xFF).chr
      when scan(/x([0-9a-fA-F]{1,2})/) then # hex constant
        ss[1].to_i(16).chr
      when check(/M-\\[\\MCc]/) then
        scan(/M-\\/) # eat it
        c = self.read_escape
        c[0] = (c[0].ord | 0x80).chr
        c
      when scan(/M-(.)/) then
        c = ss[1]
        c[0] = (c[0].ord | 0x80).chr
        c
      when check(/(C-|c)\\[\\MCc]/) then
        scan(/(C-|c)\\/) # eat it
        c = self.read_escape
        c[0] = (c[0].ord & 0x9f).chr
        c
      when scan(/C-\?|c\?/) then
        127.chr
      when scan(/(C-|c)(.)/) then
        c = ss[2]
        c[0] = (c[0].ord & 0x9f).chr
        c
      when scan(/^[89]/) then # bad octal or hex... MRI ignores them :(
        matched
      when scan(/u([0-9a-fA-F]{2,4}|\{[0-9a-fA-F]{2,6}\})/) then
        [ss[1].delete("{}").to_i(16)].pack("U")
      when scan(/[McCx0-9]/) || end_of_stream? then
        rb_compile_error("Invalid escape character syntax")
      else
        ss.getch
      end
    end

regx_options() click to toggle source

    # File lib/ruby_lexer.rb, line 850
    def regx_options # TODO: rewrite / remove
      good, bad = [], []

      if scan(/[a-z]+/) then
        good, bad = matched.split(//).partition { |s| s =~ /^[ixmonesu]$/ }
      end

      unless bad.empty? then
        rb_compile_error("unknown regexp option%s - %s" %
                         [(bad.size > 1 ? "s" : ""), bad.join.inspect])
      end

      return good.join
    end

reset() click to toggle source

    # File lib/ruby_lexer.rb, line 865
    def reset
      self.brace_nest    = 0
      self.command_start = true
      self.comments      = []
      self.lex_state     = nil
      self.lex_strterm   = nil
      self.lineno        = 1
      self.lpar_beg      = nil
      self.paren_nest    = 0
      self.space_seen    = false
      self.string_nest   = 0
      self.token         = nil
      self.extra_lineno  = 0

      self.cmdarg = RubyParserStuff::StackState.new(:cmdarg)
      self.cond   = RubyParserStuff::StackState.new(:cond)
    end

ruby18() click to toggle source

    # File lib/ruby_lexer.rb, line 889
    def ruby18
      Ruby18Parser === parser
    end

ruby19() click to toggle source

    # File lib/ruby_lexer.rb, line 893
    def ruby19
      Ruby19Parser === parser
    end

ruby22?() click to toggle source

    # File lib/ruby_lexer.rb, line 1075
    def ruby22?
      Ruby22Parser === parser
    end

ruby22_label?() click to toggle source

    # File lib/ruby_lexer.rb, line 275
    def ruby22_label?
      ruby22? and is_label_possible?
    end

scan(re) click to toggle source

    # File lib/ruby_lexer.rb, line 897
    def scan re
      ss.scan re
    end

scanner_class() click to toggle source

    # File lib/ruby_lexer.rb, line 905
    def scanner_class # TODO: design this out of oedipus_lex. or something.
      RPStringScanner
    end

space_vs_beginning(space_type, beg_type, fallback) click to toggle source

    # File lib/ruby_lexer.rb, line 909
    def space_vs_beginning space_type, beg_type, fallback
      if is_space_arg? check(/./) then
        warning "`**' interpreted as argument prefix"
        space_type
      elsif is_beg? then
        beg_type
      else
        # TODO: warn_balanced("**", "argument prefix");
        fallback
      end
    end

string(type, beg = matched, nnd = "\0") click to toggle source

    # File lib/ruby_lexer.rb, line 921
    def string type, beg = matched, nnd = "\00""
      self.lex_strterm = [:strterm, type, beg, nnd]
    end

tokadd_escape(term) click to toggle source

TODO: consider def src= src

    raise "bad src: #{src.inspect}" unless String === src
    @src = RPStringScanner.new(src)

end

    # File lib/ruby_lexer.rb, line 931
    def tokadd_escape term # TODO: rewrite / remove
      case
      when scan(/\\\n/) then
        # just ignore
      when scan(/\\([0-7]{1,3}|x[0-9a-fA-F]{1,2})/) then
        self.string_buffer << matched
      when scan(/\\([MC]-|c)(?=\\)/) then
        self.string_buffer << matched
        self.tokadd_escape term
      when scan(/\\([MC]-|c)(.)/) then
        self.string_buffer << matched
      when scan(/\\[McCx]/) then
        rb_compile_error "Invalid escape character syntax"
      when scan(/\\(.)/) then
        self.string_buffer << matched
      else
        rb_compile_error "Invalid escape character syntax"
      end
    end

tokadd_string(func, term, paren) click to toggle source

    # File lib/ruby_lexer.rb, line 951
    def tokadd_string(func, term, paren) # TODO: rewrite / remove
      qwords = (func & STR_FUNC_QWORDS) != 0
      escape = (func & STR_FUNC_ESCAPE) != 0
      expand = (func & STR_FUNC_EXPAND) != 0
      regexp = (func & STR_FUNC_REGEXP) != 0
      symbol = (func & STR_FUNC_SYMBOL) != 0

      paren_re = @@regexp_cache[paren]
      term_re  = @@regexp_cache[term]

      until end_of_stream? do
        c = nil
        handled = true

        case
        when paren_re && scan(paren_re) then
          self.string_nest += 1
        when scan(term_re) then
          if self.string_nest == 0 then
            ss.pos -= 1
            break
          else
            self.string_nest -= 1
          end
        when expand && scan(/#(?=[\$\@\{])/) then
          ss.pos -= 1
          break
        when qwords && scan(/\s/) then
          ss.pos -= 1
          break
        when expand && scan(/#(?!\n)/) then
          # do nothing
        when check(/\\/) then
          case
          when qwords && scan(/\\\n/) then
            string_buffer << "\n"
            next
          when qwords && scan(/\\\s/) then
            c = ' '
          when expand && scan(/\\\n/) then
            next
          when regexp && check(/\\/) then
            self.tokadd_escape term
            next
          when expand && scan(/\\/) then
            c = self.read_escape
          when scan(/\\\n/) then
            # do nothing
          when scan(/\\\\/) then
            string_buffer << '\' if escape
            c = '\'
          when scan(/\\/) then
            unless scan(term_re) || paren.nil? || scan(paren_re) then
              string_buffer << "\\"
            end
          else
            handled = false
          end # inner /\\/ case
        else
          handled = false
        end # top case

        unless handled then
          t = Regexp.escape term
          x = Regexp.escape(paren) if paren && paren != "\0000"
          re = if qwords then
                 if RUBY19 then
                   /[^#{t}#{x}\#\00\\\\s]+|./ # |. to pick up whatever
                 else
                   /[^#{t}#{x}\#\00\\\\s\v]+|./ # argh. 1.8's \s doesn't pick up \v
                 end
               else
                 /[^#{t}#{x}\#\00\\\]+|./
               end

          scan re
          c = matched

          rb_compile_error "symbol cannot contain '\\0'" if symbol && c =~ /\00//
        end # unless handled

        c ||= matched
        string_buffer << c
      end # until

      c ||= matched
      c = RubyLexer::EOF if end_of_stream?

      return c
    end

unescape(s) click to toggle source

    # File lib/ruby_lexer.rb, line 1042
    def unescape s
      r = ESCAPES[s]

      self.extra_lineno -= 1 if r && s == "n"

      return r if r

      x = case s
          when /^[0-7]{1,3}/ then
            ($&.to_i(8) & 0xFF).chr
          when /^x([0-9a-fA-F]{1,2})/ then
            $1.to_i(16).chr
          when /^M-(.)/ then
            ($1[0].ord | 0x80).chr
          when /^(C-|c)(.)/ then
            ($2[0].ord & 0x9f).chr
          when /^[89a-f]/ then # bad octal or hex... ignore? that's what MRI does :(
            s
          when /^[McCx0-9]/ then
            rb_compile_error("Invalid escape character syntax")
          when /u([0-9a-fA-F]{2,4}|\{[0-9a-fA-F]{2,6}\})/ then
            [$1.delete("{}").to_i(16)].pack("U")
          else
            s
          end
      x.force_encoding "UTF-8" if RUBY19
      x
    end

warning(s) click to toggle source

    # File lib/ruby_lexer.rb, line 1071
    def warning s
      # do nothing for now
    end

was_label?() click to toggle source

    # File lib/ruby_lexer.rb, line 664
    def was_label?
      @was_label = ruby22_label?
      true
    end

[Validate]

Generated with the Darkfish Rdoc Generator 2.