Class: YARD::Parser::Ruby::Legacy::RubyLex

### Home Classes Methods

### In Files

- lib/yard/parser/ruby/legacy/ruby_lex.rb

### Parent

Object

### Namespace

- CLASS YARD::Parser::Ruby::Legacy::RubyLex::BufferedReader

### Methods

- ::debug?
- ::new
- #char_no
- #get_read
- #getc
- #getc_of_rests
- #gets
- #identify_comment
- #identify_gvar
- #identify_here_document
- #identify_identifier
- #identify_number
- #identify_quotation
- #identify_string
- #lex
- #lex_init
- #lex_int2
- #line_no
- #peek
- #peek_equal?
- #read_escape
- #skip_inner_expression
- #token
- #ungetc

### Included Modules

- YARD::Parser::Ruby::Legacy::RubyToken
- IRB

### Files

- LICENSE.txt
- MIT.txt

### Class/Module Index

Quicksearch

- YARD
- YARD::CLI
- YARD::CLI::Command
- YARD::CLI::CommandParser
- YARD::CLI::Config
- YARD::CLI::Diff
- YARD::CLI::Display
- YARD::CLI::Gems
- YARD::CLI::Graph
- YARD::CLI::GraphOptions
- YARD::CLI::Help
- YARD::CLI::I18n
- YARD::CLI::List
- YARD::CLI::MarkupTypes
- YARD::CLI::Server
- YARD::CLI::Stats
- YARD::CLI::YRI
- YARD::CLI::Yardoc
- YARD::CLI::YardocOptions
- YARD::CLI::YardoptsCommand
- YARD::CodeObjects
- YARD::CodeObjects::Base
- YARD::CodeObjects::ClassObject
- YARD::CodeObjects::ClassVariableObject
- YARD::CodeObjects::CodeObjectList
- YARD::CodeObjects::ConstantObject
- YARD::CodeObjects::ExtendedMethodObject
- YARD::CodeObjects::ExtraFileObject
- YARD::CodeObjects::MacroObject
- YARD::CodeObjects::MethodObject
- YARD::CodeObjects::ModuleObject
- YARD::CodeObjects::NamespaceObject
- YARD::CodeObjects::Proxy
- YARD::CodeObjects::ProxyMethodError
- YARD::CodeObjects::RootObject
- YARD::Config
- YARD::Docstring
- YARD::DocstringParser
- YARD::Handlers
- YARD::Handlers::Base
- YARD::Handlers::C
- YARD::Handlers::C::AliasHandler
- YARD::Handlers::C::AttributeHandler
- YARD::Handlers::C::Base
- YARD::Handlers::C::ClassHandler
- YARD::Handlers::C::ConstantHandler
- YARD::Handlers::C::HandlerMethods
- YARD::Handlers::C::InitHandler
- YARD::Handlers::C::MethodHandler
- YARD::Handlers::C::MixinHandler
- YARD::Handlers::C::ModuleHandler
- YARD::Handlers::C::OverrideCommentHandler
- YARD::Handlers::C::PathHandler
- YARD::Handlers::C::StructHandler
- YARD::Handlers::C::SymbolHandler
- YARD::Handlers::HandlerAborted
- YARD::Handlers::NamespaceMissingError
- YARD::Handlers::Processor
- YARD::Handlers::Ruby
- YARD::Handlers::Ruby::AliasHandler
- YARD::Handlers::Ruby::AttributeHandler
- YARD::Handlers::Ruby::Base
- YARD::Handlers::Ruby::ClassConditionHandler
- YARD::Handlers::Ruby::ClassHandler
- YARD::Handlers::Ruby::ClassVariableHandler
- YARD::Handlers::Ruby::CommentHandler
- YARD::Handlers::Ruby::ConstantHandler
- YARD::Handlers::Ruby::DSLHandler
- YARD::Handlers::Ruby::DSLHandlerMethods
- YARD::Handlers::Ruby::ExceptionHandler
- YARD::Handlers::Ruby::ExtendHandler
- YARD::Handlers::Ruby::HandlesExtension
- YARD::Handlers::Ruby::Legacy
- YARD::Handlers::Ruby::Legacy::AliasHandler
- YARD::Handlers::Ruby::Legacy::AttributeHandler
- YARD::Handlers::Ruby::Legacy::Base
- YARD::Handlers::Ruby::Legacy::ClassConditionHandler
- YARD::Handlers::Ruby::Legacy::ClassHandler
- YARD::Handlers::Ruby::Legacy::ClassVariableHandler
- YARD::Handlers::Ruby::Legacy::CommentHandler
- YARD::Handlers::Ruby::Legacy::ConstantHandler
- YARD::Handlers::Ruby::Legacy::DSLHandler
- YARD::Handlers::Ruby::Legacy::ExceptionHandler
- YARD::Handlers::Ruby::Legacy::ExtendHandler
- YARD::Handlers::Ruby::Legacy::MethodHandler
- YARD::Handlers::Ruby::Legacy::MixinHandler
- YARD::Handlers::Ruby::Legacy::ModuleFunctionHandler
- YARD::Handlers::Ruby::Legacy::ModuleHandler
- YARD::Handlers::Ruby::Legacy::PrivateClassMethodHandler
- YARD::Handlers::Ruby::Legacy::PrivateConstantHandler
- YARD::Handlers::Ruby::Legacy::VisibilityHandler
- YARD::Handlers::Ruby::Legacy::YieldHandler
- YARD::Handlers::Ruby::MethodCallWrapper
- YARD::Handlers::Ruby::MethodConditionHandler
- YARD::Handlers::Ruby::MethodHandler
- YARD::Handlers::Ruby::MixinHandler
- YARD::Handlers::Ruby::ModuleFunctionHandler
- YARD::Handlers::Ruby::ModuleHandler
- YARD::Handlers::Ruby::PrivateClassMethodHandler
- YARD::Handlers::Ruby::PrivateConstantHandler
- YARD::Handlers::Ruby::StructHandlerMethods
- YARD::Handlers::Ruby::TestNodeWrapper
- YARD::Handlers::Ruby::VisibilityHandler
- YARD::Handlers::Ruby::YieldHandler
- YARD::I18n
- YARD::I18n::Locale
- YARD::I18n::Message
- YARD::I18n::Messages
- YARD::I18n::POParser
- YARD::I18n::PotGenerator
- YARD::I18n::Text
- YARD::Logger
- YARD::Options
- YARD::Parser
- YARD::Parser::Base
- YARD::Parser::C
- YARD::Parser::C::BodyStatement
- YARD::Parser::C::CParser
- YARD::Parser::C::Comment
- YARD::Parser::C::CommentParser
- YARD::Parser::C::Statement
- YARD::Parser::C::ToplevelStatement
- YARD::Parser::OrderedParser
- YARD::Parser::ParserSyntaxError
- YARD::Parser::Ruby
- YARD::Parser::Ruby::AstNode
- YARD::Parser::Ruby::ClassNode
- YARD::Parser::Ruby::CommentNode
- YARD::Parser::Ruby::ConditionalNode
- YARD::Parser::Ruby::KeywordNode
- YARD::Parser::Ruby::Legacy
- YARD::Parser::Ruby::Legacy::RubyLex
- YARD::Parser::Ruby::Legacy::RubyLex::BufferedReader
- YARD::Parser::Ruby::Legacy::RubyParser
- YARD::Parser::Ruby::Legacy::RubyToken
- YARD::Parser::Ruby::Legacy::RubyToken::TkBlockContents
- YARD::Parser::Ruby::Legacy::RubyToken::TkError
- YARD::Parser::Ruby::Legacy::RubyToken::TkId
- YARD::Parser::Ruby::Legacy::RubyToken::TkKW
- YARD::Parser::Ruby::Legacy::RubyToken::TkNode
- YARD::Parser::Ruby::Legacy::RubyToken::TkOPASGN
- YARD::Parser::Ruby::Legacy::RubyToken::TkOp
- YARD::Parser::Ruby::Legacy::RubyToken::TkStatementEnd
- YARD::Parser::Ruby::Legacy::RubyToken::TkUnknownChar
- YARD::Parser::Ruby::Legacy::RubyToken::TkVal
- YARD::Parser::Ruby::Legacy::RubyToken::TkWhitespace
- YARD::Parser::Ruby::Legacy::RubyToken::Token
- YARD::Parser::Ruby::Legacy::Statement
- YARD::Parser::Ruby::Legacy::StatementList
- YARD::Parser::Ruby::Legacy::TokenList
- YARD::Parser::Ruby::LiteralNode
- YARD::Parser::Ruby::LoopNode
- YARD::Parser::Ruby::MethodCallNode
- YARD::Parser::Ruby::MethodDefinitionNode
- YARD::Parser::Ruby::ModuleNode
- YARD::Parser::Ruby::ParameterNode
- YARD::Parser::Ruby::ReferenceNode
- YARD::Parser::Ruby::RipperParser
- YARD::Parser::Ruby::RubyParser
- YARD::Parser::SourceParser
- YARD::Parser::UndocumentableError
- YARD::Rake
- YARD::Rake::YardocTask
- YARD::Registry
- YARD::RegistryStore
- YARD::Serializers
- YARD::Serializers::Base
- YARD::Serializers::FileSystemSerializer
- YARD::Serializers::ProcessSerializer
- YARD::Serializers::StdoutSerializer
- YARD::Serializers::YardocSerializer
- YARD::Server
- YARD::Server::Adapter
- YARD::Server::Commands
- YARD::Server::Commands::Base
- YARD::Server::Commands::DisplayFileCommand
- YARD::Server::Commands::DisplayObjectCommand
- YARD::Server::Commands::FramesCommand
- YARD::Server::Commands::LibraryCommand
- YARD::Server::Commands::LibraryIndexCommand
- YARD::Server::Commands::LibraryOptions
- YARD::Server::Commands::ListCommand
- YARD::Server::Commands::SearchCommand
- YARD::Server::Commands::StaticFileCommand
- YARD::Server::DocServerHelper
- YARD::Server::DocServerSerializer
- YARD::Server::FinishRequest
- YARD::Server::LibraryNotPreparedError
- YARD::Server::LibraryVersion
- YARD::Server::NotFoundError
- YARD::Server::RackAdapter
- YARD::Server::RackMiddleware
- YARD::Server::Router
- YARD::Server::StaticCaching
- YARD::Server::WebrickAdapter
- YARD::Server::WebrickServlet
- YARD::StubProxy
- YARD::Tags
- YARD::Tags::AttributeDirective
- YARD::Tags::DefaultFactory
- YARD::Tags::DefaultTag
- YARD::Tags::Directive
- YARD::Tags::EndGroupDirective
- YARD::Tags::GroupDirective
- YARD::Tags::Library
- YARD::Tags::MacroDirective
- YARD::Tags::MethodDirective
- YARD::Tags::OptionTag
- YARD::Tags::OverloadTag
- YARD::Tags::ParseDirective
- YARD::Tags::RefTag
- YARD::Tags::RefTagList
- YARD::Tags::ScopeDirective
- YARD::Tags::Tag
- YARD::Tags::TagFormatError
- YARD::Tags::VisibilityDirective
- YARD::Templates
- YARD::Templates::Engine
- YARD::Templates::ErbCache
- YARD::Templates::Helpers
- YARD::Templates::Helpers::BaseHelper
- YARD::Templates::Helpers::FilterHelper
- YARD::Templates::Helpers::HtmlHelper
- YARD::Templates::Helpers::HtmlSyntaxHighlightHelper
- YARD::Templates::Helpers::Markup
- YARD::Templates::Helpers::Markup::RDocMarkdown
- YARD::Templates::Helpers::Markup::RDocMarkup
- YARD::Templates::Helpers::Markup::RDocMarkup::RDocMarkup
- YARD::Templates::Helpers::Markup::RDocMarkup::RDocMarkup::RDocMarkupToHtml
- YARD::Templates::Helpers::Markup::RDocMarkup::RDocMarkupToHtml
- YARD::Templates::Helpers::MarkupHelper
- YARD::Templates::Helpers::MethodHelper
- YARD::Templates::Helpers::ModuleHelper
- YARD::Templates::Helpers::TextHelper
- YARD::Templates::Helpers::UMLHelper
- YARD::Templates::Section
- YARD::Templates::Template
- YARD::Templates::Template::ClassMethods
- YARD::Templates::TemplateOptions
- YARD::Verifier
- Gem
- Gem::DocManager
- Gem::SourceIndex
- Gem::Specification
- Rack
- Rack::Request
- WEBrick
- WEBrick::HTTPRequest
- Array
- File
- Hash
- Insertion
- Module
- Object
- String
- SymbolHash

No matching classes.

# YARD::Parser::Ruby::Legacy::RubyLex

Lexical analyzer for Ruby source @private

### Constants

ACCEPTS_COLON
DEINDENT_CLAUSE
DLtype2Token
ENINDENT_CLAUSE
Ltype2Token
PERCENT_LTYPE
PERCENT_PAREN

### Attributes

continue[R]
exception_on_syntax_error[RW]
indent[R]
lex_state[R]
read_auto_clean_up[RW]
skip_space[RW]

### Public Class Methods

debug?() click to toggle source

    # File lib/yard/parser/ruby/legacy/ruby_lex.rb, line 460
    def RubyLex.debug?
      false
    end

new(content) click to toggle source

    # File lib/yard/parser/ruby/legacy/ruby_lex.rb, line 464
    def initialize(content)
      lex_init

      @reader = BufferedReader.new(content)

      @exp_line_no = @line_no = 1
      @base_char_no = 0
      @indent = 0

      @ltype = nil
      @quoted = nil
      @lex_state = EXPR_BEG
      @space_seen = false

      @continue = false
      @line = ""

      @skip_space = false
      @read_auto_clean_up = false
      @exception_on_syntax_error = true
    end

### Public Instance Methods

char_no() click to toggle source

    # File lib/yard/parser/ruby/legacy/ruby_lex.rb, line 497
    def char_no
      @reader.column
    end

get_read() click to toggle source

    # File lib/yard/parser/ruby/legacy/ruby_lex.rb, line 501
    def get_read
      @reader.get_read
    end

getc() click to toggle source

    # File lib/yard/parser/ruby/legacy/ruby_lex.rb, line 505
    def getc
      @reader.getc
    end

getc_of_rests() click to toggle source

    # File lib/yard/parser/ruby/legacy/ruby_lex.rb, line 509
    def getc_of_rests
      @reader.getc_already_read
    end

gets() click to toggle source

    # File lib/yard/parser/ruby/legacy/ruby_lex.rb, line 513
    def gets
      c = getc or return
      l = ""
      begin
        l.concat c unless c == "\r"
        break if c == "\n"
      end while c = getc
      l
    end

identify_comment() click to toggle source

    # File lib/yard/parser/ruby/legacy/ruby_lex.rb, line 1289
    def identify_comment
      @ltype = "#"
      comment = "#"
      while ch = getc
        if ch == "\\"
          ch = getc
          if ch == "\n"
            ch = " "
          else
            comment << "\\"
          end
        else
          if ch == "\n"
            @ltype = nil
            ungetc
            break
          end
        end
        comment << ch
      end
      return Token(TkCOMMENT).set_text(comment)
    end

identify_gvar() click to toggle source

    # File lib/yard/parser/ruby/legacy/ruby_lex.rb, line 977
    def identify_gvar
      @lex_state = EXPR_END
      str = "$"

      tk = case ch = getc
           when /[~_*$?!@\/\\;,=:<>".]/   #"
             str << ch
             Token(TkGVAR, str)

           when "-"
             str << "-" << getc
             Token(TkGVAR, str)

           when "&", "`", "'", "+"
             str << ch
             Token(TkBACK_REF, str)

           when /[1-9]/
             str << ch
             while (ch = getc) =~ /[0-9]/
               str << ch
             end
             ungetc
             Token(TkNTH_REF)
           when /\w/
             ungetc
             ungetc
             return identify_identifier
           else
             ungetc
             Token("$")
           end
      tk.set_text(str)
    end

identify_here_document() click to toggle source

    # File lib/yard/parser/ruby/legacy/ruby_lex.rb, line 1094
    def identify_here_document
      ch = getc
      if ch == "-"
        ch = getc
        indent = true
      end
      if /['"`]/ =~ ch            # '
        lt = ch
        quoted = ""
        while (c = getc) && c != lt
          quoted.concat c
        end
      else
        lt = '"'
        quoted = ch.dup
        while (c = getc) && c =~ /\w/
          quoted.concat c
        end
        ungetc
      end

      ltback, @ltype = @ltype, lt
      reserve = ""

      while ch = getc
        reserve << ch
        if ch == "\\"    #"
          ch = getc
          reserve << ch
        elsif ch == "\n"
          break
        end
      end

      str = ""
      while (l = gets)
        l.chomp!
        l.strip! if indent
        break if l == quoted
        str << l.chomp << "\n"
      end

      @reader.divert_read_from(reserve)

      @ltype = ltback
      @lex_state = EXPR_END
      Token(Ltype2Token[lt], str).set_text(str.dump)
    end

identify_identifier() click to toggle source

    # File lib/yard/parser/ruby/legacy/ruby_lex.rb, line 1012
    def identify_identifier
      token = ""
      token.concat getc if peek(0) =~ /[$@]/
      token.concat getc if peek(0) == "@"

      while (ch = getc) =~ /\w|_/
        print ":", ch, ":" if RubyLex.debug?
        token.concat ch
      end
      ungetc

      if ch == "!" or ch == "?"
        token.concat getc
      end
      # fix token

      # $stderr.puts "identifier - #{token}, state = #@lex_state"

      case token
      when /^\$/
        return Token(TkGVAR, token).set_text(token)
      when /^\@/
        @lex_state = EXPR_END
        return Token(TkIVAR, token).set_text(token)
      end

      if @lex_state != EXPR_DOT
        print token, "\n" if RubyLex.debug?

        token_c, *trans = TkReading2Token[token]
        if token_c
          # reserved word?

          if (@lex_state != EXPR_BEG &&
              @lex_state != EXPR_FNAME &&
              trans[1])
            # modifiers
            token_c = TkSymbol2Token[trans[1]]
            @lex_state = trans[0]
          else
            if @lex_state != EXPR_FNAME
              if ENINDENT_CLAUSE.include?(token)
                @indent += 1

                if ACCEPTS_COLON.include?(token)
                  @colonblock_seen = true
                else
                  @colonblock_seen = false
                end
              elsif DEINDENT_CLAUSE.include?(token)
                @indent -= 1
                @colonblock_seen = false
              end
              @lex_state = trans[0]
            else
              @lex_state = EXPR_END
            end
          end
          return Token(token_c, token).set_text(token)
        end
      end

      if @lex_state == EXPR_FNAME
        @lex_state = EXPR_END
        if peek(0) == '='
          token.concat getc
        end
      elsif @lex_state == EXPR_BEG || @lex_state == EXPR_DOT
        @lex_state = EXPR_ARG
      else
        @lex_state = EXPR_END
      end

      if token[0, 1] =~ /[A-Z]/
        return Token(TkCONSTANT, token).set_text(token)
      elsif token[token.size - 1, 1] =~ /[!?]/
        return Token(TkFID, token).set_text(token)
      else
        return Token(TkIDENTIFIER, token).set_text(token)
      end
    end

identify_number(start) click to toggle source

    # File lib/yard/parser/ruby/legacy/ruby_lex.rb, line 1162
    def identify_number(start)
      str = start.dup

      if start == "+" or start == "-" or start == ""
        start = getc
        str << start
      end

      @lex_state = EXPR_END

      if start == "0"
        if peek(0) == "x"
          ch = getc
          str << ch
          match = /[0-9a-f_]/
        else
          match = /[0-7_]/
        end
        while ch = getc
          if ch !~ match
            ungetc
            break
          else
            str << ch
          end
        end
        return Token(TkINTEGER).set_text(str)
      end

      type = TkINTEGER
      allow_point = TRUE
      allow_e = TRUE
      while ch = getc
        case ch
        when /[0-9_]/
          str << ch

        when allow_point && "."
          type = TkFLOAT
          if peek(0) !~ /[0-9]/
            ungetc
            break
          end
          str << ch
          allow_point = false

        when allow_e && "e", allow_e && "E"
          str << ch
          type = TkFLOAT
          if peek(0) =~ /[+-]/
            str << getc
          end
          allow_e = false
          allow_point = false
        else
          ungetc
          break
        end
      end
      Token(type).set_text(str)
    end

identify_quotation(initial_char) click to toggle source

    # File lib/yard/parser/ruby/legacy/ruby_lex.rb, line 1143
    def identify_quotation(initial_char)
      ch = getc
      if lt = PERCENT_LTYPE[ch]
        initial_char += ch
        ch = getc
      elsif ch =~ /\W/
        lt = "\""
      else
        # RubyLex.fail SyntaxError, "unknown type of %string ('#{ch}')"
      end
      # if ch !~ /\W/
      #   ungetc
      #   next
      # end
      # @ltype = lt
      @quoted = ch unless @quoted = PERCENT_PAREN[ch]
      identify_string(lt, @quoted, ch, initial_char) if lt
    end

identify_string(ltype, quoted = ltype, opener=nil, initial_char = nil) click to toggle source

    # File lib/yard/parser/ruby/legacy/ruby_lex.rb, line 1224
    def identify_string(ltype, quoted = ltype, opener=nil, initial_char = nil)
      @ltype = ltype
      @quoted = quoted
      subtype = nil

      str = ""
      str << initial_char if initial_char
      str << (opener||quoted)

      nest = 0
      begin
        while ch = getc
          str << ch
          if @quoted == ch
            if nest == 0
              break
            else
              nest -= 1
            end
          elsif opener == ch
            nest += 1
          elsif @ltype != "'" && @ltype != "]" and ch == "#"
            ch = getc
            if ch == "{"
              subtype = true
              str << ch << skip_inner_expression
            else
              ungetc(ch)
            end
          elsif ch == '\' #'
            str << read_escape
          end
        end
        if @ltype == "/"
          if peek(0) =~ /i|o|n|e|s/
            str << getc
          end
        end
        if subtype
          Token(DLtype2Token[ltype], str)
        else
          Token(Ltype2Token[ltype], str)
        end.set_text(str)
      ensure
        @ltype = nil
        @quoted = nil
        @lex_state = EXPR_END
      end
    end

lex() click to toggle source

    # File lib/yard/parser/ruby/legacy/ruby_lex.rb, line 536
    def lex
      catch(:eof) do
        until (((tk = token).kind_of?(TkNL) || tk.kind_of?(TkEND_OF_SCRIPT)) &&
           !@continue or
           tk.nil?)
        end
        line = get_read

        if line == "" and tk.kind_of?(TkEND_OF_SCRIPT) || tk.nil?
          nil
        else
          line
        end
      end
    end

lex_init() click to toggle source

    # File lib/yard/parser/ruby/legacy/ruby_lex.rb, line 613
    def lex_init()
      @OP = SLex.new
      @OP.def_rules("\00"", "\0004", "\0032") do |chars, io|
        Token(TkEND_OF_SCRIPT).set_text(chars)
      end

      @OP.def_rules(" ", "\t", "\f", "\r", "\113"") do |chars, io|
        @space_seen = TRUE
        while (ch = getc) =~ /[ \t\f\r\113]]/
          chars << ch
        end
        ungetc
        Token(TkSPACE).set_text(chars)
      end

      @OP.def_rule("#") do
        |op, io|
        identify_comment
      end

      @OP.def_rule("=begin", proc{@prev_char_no == 0 && peek(0) =~ /\s/}) do
        |op, io|
        str = op
        @ltype = "="

        begin
          line = ""
          begin
            ch = getc
            line << ch
          end until ch == "\n"
          str << line
        end until line =~ /^=end/

        ungetc

        @ltype = nil

        if str =~ /\A=begin\s+rdoc/
          str.sub!(/\A=begin.*\n/, '')
          str.sub!(/^=end.*/, '')
          Token(TkCOMMENT).set_text(str)
        else
          Token(TkCOMMENT).set_text(str)
        end
      end

      @OP.def_rule("\n") do
        print "\\n\n" if RubyLex.debug?
        @colonblock_seen = false
        case @lex_state
        when EXPR_BEG, EXPR_FNAME, EXPR_DOT
          @continue = TRUE
        else
          @continue = FALSE
          @lex_state = EXPR_BEG
        end
        Token(TkNL).set_text("\n")
      end

      @OP.def_rules("*", "**",
        "!", "!=", "!~",
        "=", "==", "===",
        "=~", "<=>",
        "<", "<=",
        ">", ">=", ">>") do |op, io|
        @lex_state = EXPR_BEG
        Token(op).set_text(op)
      end

      @OP.def_rules("<<") do |op, io|
        tk = nil
        if @lex_state != EXPR_END && @lex_state != EXPR_CLASS &&
           (@lex_state != EXPR_ARG || @space_seen)
          c = peek(0)
          tk = identify_here_document if /[-\w_\"\'\`]/ =~ c
        end
        if !tk
          @lex_state = EXPR_BEG
          tk = Token(op).set_text(op)
        end
        tk
      end

      @OP.def_rules("'", '"') do |op, io|
        identify_string(op)
      end

      @OP.def_rules("`") do |op, io|
        if @lex_state == EXPR_FNAME
          Token(op).set_text(op)
        else
          identify_string(op)
        end
      end

      @OP.def_rules('?') do |op, io|
        if @lex_state == EXPR_END
          @lex_state = EXPR_BEG
          Token(TkQUESTION).set_text(op)
        else
          ch = getc
          if @lex_state == EXPR_ARG && ch !~ /\s/
            ungetc
            @lex_state = EXPR_BEG
            Token(TkQUESTION).set_text(op)
          else
            str = op
            str << ch
            if (ch == '\') #'
              str << read_escape
            end
            @lex_state = EXPR_END
            Token(TkINTEGER).set_text(str)
          end
        end
      end

      @OP.def_rules("&", "&&", "|", "||") do |op, io|
        @lex_state = EXPR_BEG
        Token(op).set_text(op)
      end

      @OP.def_rules("+=", "-=", "*=", "**=",
        "&=", "|=", "^=", "<<=", ">>=", "||=", "&&=") do |op, io|
        @lex_state = EXPR_BEG
        op =~ /^(.*)=$/
        Token(TkOPASGN, $1).set_text(op)
      end

      @OP.def_rule("+@", proc{@lex_state == EXPR_FNAME}) do |op, io|
        Token(TkUPLUS).set_text(op)
      end

      @OP.def_rule("-@", proc{@lex_state == EXPR_FNAME}) do |op, io|
        Token(TkUMINUS).set_text(op)
      end

      @OP.def_rules("+", "-") do |op, io|
        catch(:RET) do
          if @lex_state == EXPR_ARG
            if @space_seen and peek(0) =~ /[0-9]/
              throw :RET, identify_number(op)
            else
              @lex_state = EXPR_BEG
            end
          elsif @lex_state != EXPR_END and peek(0) =~ /[0-9]/
            throw :RET, identify_number(op)
          else
            @lex_state = EXPR_BEG
          end
          Token(op).set_text(op)
        end
      end

      @OP.def_rule(".") do
        @lex_state = EXPR_BEG
        if peek(0) =~ /[0-9]/
          ungetc
          identify_number("")
        else
          # for obj.if
          @lex_state = EXPR_DOT
          Token(TkDOT).set_text(".")
        end
      end

      @OP.def_rules("..", "...") do |op, io|
        @lex_state = EXPR_BEG
        Token(op).set_text(op)
      end

      lex_int2
    end

lex_int2() click to toggle source

    # File lib/yard/parser/ruby/legacy/ruby_lex.rb, line 789
    def lex_int2
      @OP.def_rules("]", "}", ")") do
        |op, io|
        @lex_state = EXPR_END
        @indent -= 1
        Token(op).set_text(op)
      end

      @OP.def_rule(":") do
        if (@colonblock_seen && @lex_state != EXPR_BEG) || peek(0) =~ /\s/
          @lex_state = EXPR_BEG
          tk = Token(TkCOLON)
        else
          @lex_state = EXPR_FNAME
          tk = Token(TkSYMBEG)
        end
        tk.set_text(":")
      end

      @OP.def_rule("::") do
        # p @lex_state.id2name, @space_seen
        if @lex_state == EXPR_BEG or @lex_state == EXPR_ARG && @space_seen
          @lex_state = EXPR_BEG
          tk = Token(TkCOLON3)
        else
          @lex_state = EXPR_DOT
          tk = Token(TkCOLON2)
        end
        tk.set_text("::")
      end

      @OP.def_rule("/") do |op, io|
        if @lex_state == EXPR_BEG || @lex_state == EXPR_MID
          identify_string(op)
        elsif peek(0) == '='
          getc
          @lex_state = EXPR_BEG
          Token(TkOPASGN, :/).set_text("/=") #")
        elsif @lex_state == EXPR_ARG and @space_seen and peek(0) !~ /\s/
          identify_string(op)
        else
          @lex_state = EXPR_BEG
          Token("/").set_text(op)
        end
      end

      @OP.def_rules("^") do
        @lex_state = EXPR_BEG
        Token("^").set_text("^")
      end

      # @OP.def_rules("^=") do
      #   @lex_state = EXPR_BEG
      #   Token(TkOPASGN, :^)
      # end

      @OP.def_rules(",", ";") do |op, io|
        @colonblock_seen = false
        @lex_state = EXPR_BEG
        Token(op).set_text(op)
      end

      @OP.def_rule("~") do
        @lex_state = EXPR_BEG
        Token("~").set_text("~")
      end

      @OP.def_rule("~@", proc{@lex_state = EXPR_FNAME}) do
        @lex_state = EXPR_BEG
        Token("~").set_text("~@")
      end

      @OP.def_rule("(") do
        @indent += 1
        # if @lex_state == EXPR_BEG || @lex_state == EXPR_MID
        #  @lex_state = EXPR_BEG
        #  tk = Token(TkfLPAREN)
        # else
          @lex_state = EXPR_BEG
          tk = Token(TkLPAREN)
        # end
        tk.set_text("(")
      end

      @OP.def_rule("[]", proc{@lex_state == EXPR_FNAME}) do
        Token("[]").set_text("[]")
      end

      @OP.def_rule("[]=", proc{@lex_state == EXPR_FNAME}) do
        Token("[]=").set_text("[]=")
      end

      @OP.def_rule("[") do
        @indent += 1
        # if @lex_state == EXPR_FNAME
        #   t = Token(TkfLBRACK)
        # else
        #   if @lex_state == EXPR_BEG || @lex_state == EXPR_MID
        #     t = Token(TkLBRACK)
        #   elsif @lex_state == EXPR_ARG && @space_seen
        #   else
        #     t = Token(TkfLBRACK)
        #   end
        # end
        t = Token(TkLBRACK)
        @lex_state = EXPR_BEG
        t.set_text("[")
      end

      @OP.def_rule("{") do
        @indent += 1
        # if @lex_state != EXPR_END && @lex_state != EXPR_ARG
        #   t = Token(TkLBRACE)
        # else
        #   t = Token(TkfLBRACE)
        # end
        t = Token(TkLBRACE)
        @lex_state = EXPR_BEG
        t.set_text("{")
      end

      @OP.def_rule('\') do   #'
        if getc == "\n"
          @space_seen = true
          @continue = true
          Token(TkSPACE).set_text("\\\n")
        else
          ungetc
          Token("\\").set_text("\\")  #"
        end
      end

      @OP.def_rule('%') do
        |op, io|
        if @lex_state == EXPR_BEG || @lex_state == EXPR_MID
          identify_quotation('%')
        elsif peek(0) == '='
          getc
          Token(TkOPASGN, "%").set_text("%=")
        elsif @lex_state == EXPR_ARG and @space_seen and peek(0) !~ /\s/
          identify_quotation('%')
        else
          @lex_state = EXPR_BEG
          Token("%").set_text("%")
        end
      end

      @OP.def_rule('$') do  #'
        identify_gvar
      end

      @OP.def_rule('@') do
        if peek(0) =~ /[@\w_]/
          ungetc
          identify_identifier
        else
          Token("@").set_text("@")
        end
      end

      # @OP.def_rule("def", proc{|op, io| /\s/ =~ io.peek(0)}) do
      #   |op, io|
      #   @indent += 1
      #   @lex_state = EXPR_FNAME
      # # @lex_state = EXPR_END
      # # until @rests[0] == "\n" or @rests[0] == ";"
      # #   rests.shift
      # # end
      # end

      @OP.def_rule("__END__", proc{@prev_char_no == 0 && peek(0) =~ /[\r\n]/}) do
        throw :eof
      end

      @OP.def_rule("") do |op, io|
        printf "MATCH: start %s: %s\n", op, io.inspect if RubyLex.debug?
        if peek(0) =~ /[0-9]/
          t = identify_number("")
        elsif peek(0) =~ /[\w_]/
          t = identify_identifier
        end
        printf "MATCH: end %s: %s\n", op, io.inspect if RubyLex.debug?
        t
      end

      p @OP if RubyLex.debug?
    end

line_no() click to toggle source

io functions

    # File lib/yard/parser/ruby/legacy/ruby_lex.rb, line 493
    def line_no
      @reader.line_num
    end

peek(i = 0) click to toggle source

    # File lib/yard/parser/ruby/legacy/ruby_lex.rb, line 532
    def peek(i = 0)
      @reader.peek(i)
    end

peek_equal?(str) click to toggle source

    # File lib/yard/parser/ruby/legacy/ruby_lex.rb, line 528
    def peek_equal?(str)
      @reader.peek_equal(str)
    end

read_escape() click to toggle source

    # File lib/yard/parser/ruby/legacy/ruby_lex.rb, line 1312
    def read_escape
      res = ""
      case ch = getc
      when /[0-7]/
        ungetc ch
        3.times do
        case ch = getc
        when /[0-7]/
        when nil
          break
        else
          ungetc
          break
        end
        res << ch
      end

      when "x"
        res << ch
        2.times do
        case ch = getc
        when /[0-9a-fA-F]/
        when nil
          break
        else
          ungetc
          break
        end
          res << ch
        end

      when "M"
        res << ch
        if (ch = getc) != '-'
          ungetc
        else
          res << ch
          if (ch = getc) == "\\" #"
            res << ch
            res << read_escape
          else
            res << ch
          end
        end

      when "C", "c" #, "^"
        res << ch
        if ch == "C" and (ch = getc) != "-"
          ungetc
        else
          res << ch
          if (ch = getc) == "\\" #"
            res << ch
            res << read_escape
          else
            res << ch
          end
        end
      else
        res << ch
      end
      res
    end

skip_inner_expression() click to toggle source

    # File lib/yard/parser/ruby/legacy/ruby_lex.rb, line 1274
    def skip_inner_expression
      res = ""
      nest = 0
      while (ch = getc)
        res << ch
        if ch == '}'
          break if nest.zero?
          nest -= 1
        elsif ch == '{'
          nest += 1
        end
      end
      res
    end

token() click to toggle source

    # File lib/yard/parser/ruby/legacy/ruby_lex.rb, line 552
    def token
      set_token_position(line_no, char_no)
      catch(:eof) do
        begin
          begin
            tk = @OP.match(self)
            @space_seen = tk.kind_of?(TkSPACE)
          rescue SyntaxError
            abort if @exception_on_syntax_error
            tk = TkError.new(line_no, char_no)
          end
        end while @skip_space and tk.kind_of?(TkSPACE)
        if @read_auto_clean_up
          get_read
        end
        # throw :eof unless tk
        p tk if $DEBUG
        tk.lex_state = lex_state if tk
        tk
      end
    end

ungetc(c = nil) click to toggle source

    # File lib/yard/parser/ruby/legacy/ruby_lex.rb, line 524
    def ungetc(c = nil)
      @reader.ungetc(c)
    end

[Validate]

Generated with the Darkfish Rdoc Generator 2.