Class: CodeRay::Scanners::Python

### Home Classes Methods

### In Files

- lib/coderay/scanners/python.rb

### Parent

CodeRay::Scanners::Scanner

### Methods

- #scan_tokens

### Class/Module Index

Quicksearch

- CodeRay
- CodeRay::Duo
- CodeRay::Encoders
- CodeRay::Encoders::CommentFilter
- CodeRay::Encoders::Count
- CodeRay::Encoders::Debug
- CodeRay::Encoders::DebugLint
- CodeRay::Encoders::Div
- CodeRay::Encoders::Encoder
- CodeRay::Encoders::Filter
- CodeRay::Encoders::HTML
- CodeRay::Encoders::HTML::Output
- CodeRay::Encoders::JSON
- CodeRay::Encoders::LinesOfCode
- CodeRay::Encoders::Lint
- CodeRay::Encoders::Null
- CodeRay::Encoders::Page
- CodeRay::Encoders::Span
- CodeRay::Encoders::Statistic
- CodeRay::Encoders::Terminal
- CodeRay::Encoders::Text
- CodeRay::Encoders::TokenKindFilter
- CodeRay::Encoders::XML
- CodeRay::Encoders::YAML
- CodeRay::FileType
- CodeRay::ForRedCloth
- CodeRay::Plugin
- CodeRay::PluginHost
- CodeRay::Scanners
- CodeRay::Scanners::C
- CodeRay::Scanners::CPlusPlus
- CodeRay::Scanners::CSS
- CodeRay::Scanners::Clojure
- CodeRay::Scanners::Debug
- CodeRay::Scanners::Delphi
- CodeRay::Scanners::Diff
- CodeRay::Scanners::ERB
- CodeRay::Scanners::Go
- CodeRay::Scanners::Groovy
- CodeRay::Scanners::HAML
- CodeRay::Scanners::HTML
- CodeRay::Scanners::JSON
- CodeRay::Scanners::Java
- CodeRay::Scanners::JavaScript
- CodeRay::Scanners::Python
- CodeRay::Scanners::Raydebug
- CodeRay::Scanners::Ruby
- CodeRay::Scanners::SQL
- CodeRay::Scanners::Sass
- CodeRay::Scanners::Scanner
- CodeRay::Scanners::Taskpaper
- CodeRay::Scanners::Text
- CodeRay::Scanners::XML
- CodeRay::Scanners::YAML
- CodeRay::Styles
- CodeRay::Styles::Alpha
- CodeRay::Styles::Style
- CodeRay::Tokens
- CodeRay::TokensProxy
- CodeRay::WordList
- CodeRay::WordList::CaseIgnoring

No matching classes.

# CodeRay::Scanners::Python

Scanner for Python. Supports Python 3.

Based on pygments' PythonLexer, see dev.pocoo.org/projects/pygments/browser/pygments/lexers/agile.py.

### Constants

STRING_CONTENT_REGEXP
STRING_DELIMITER_REGEXP

### Protected Instance Methods

scan_tokens(encoder, options) click to toggle source

    # File lib/coderay/scanners/python.rb, line 103
    def scan_tokens encoder, options

      state = :initial
      string_delimiter = nil
      string_raw = false
      string_type = nil
      docstring_coming = match?(/#{DOCSTRING_COMING}/)
      last_token_dot = false
      unicode = string.respond_to?(:encoding) && string.encoding.name == 'UTF-8'
      from_import_state = []

      until eos?

        if state == :string
          if match = scan(STRING_DELIMITER_REGEXP[string_delimiter])
            encoder.text_token match, :delimiter
            encoder.end_group string_type
            string_type = nil
            state = :initial
            next
          elsif string_delimiter.size == 3 && match = scan(/\n/)
            encoder.text_token match, :content
          elsif match = scan(STRING_CONTENT_REGEXP[string_delimiter])
            encoder.text_token match, :content
          elsif !string_raw && match = scan(/ \\ #{ESCAPE} /x)
            encoder.text_token match, :char
          elsif match = scan(/ \\ #{UNICODE_ESCAPE} /x)
            encoder.text_token match, :char
          elsif match = scan(/ \\ . /)
            encoder.text_token match, :content
          elsif match = scan(/ \\ | $ /)
            encoder.end_group string_type
            string_type = nil
            encoder.text_token match, :error unless match.empty?
            state = :initial
          else
            raise_inspect "else case \" reached; %p not handled." % peek(1), encoder, state
          end

        elsif match = scan(/ [ \t]+ | \\?\n /)
          encoder.text_token match, :space
          if match == "\n"
            state = :initial if state == :include_expected
            docstring_coming = true if match?(/#{DOCSTRING_COMING}/)
          end
          next

        elsif match = scan(/ \# [^\n]* /x)
          encoder.text_token match, :comment
          next

        elsif state == :initial

          if match = scan(/#{OPERATOR}/)
            encoder.text_token match, :operator

          elsif match = scan(/(u?r?|b)?("""|"|'''|')/)
            modifiers = self[1]
            string_delimiter = self[2]
            string_type = docstring_coming ? :docstring : (modifiers == 'b' ? :binary : :string)
            docstring_coming = false if docstring_coming
            encoder.begin_group string_type
            string_raw = false
            unless modifiers.empty?
              string_raw = !!modifiers.index(rr)
              encoder.text_token modifiers, :modifier
              match = string_delimiter
            end
            state = :string
            encoder.text_token match, :delimiter

          # TODO: backticks

          elsif match = scan(unicode ? /#{NAME}/o : /#{NAME}/)
            kind = IDENT_KIND[match]
            # TODO: keyword arguments
            kind = :ident if last_token_dot
            if kind == :old_keyword
              kind = check(/\(/) ? :ident : :keyword
            elsif kind == :predefined && check(/ *=/)
              kind = :ident
            elsif kind == :keyword
              state = DEF_NEW_STATE[match]
              from_import_state << match.to_sym if state == :include_expected
            end
            encoder.text_token match, kind

          elsif match = scan(/@[a-zA-Z0-9_.]+[lL]?/)
            encoder.text_token match, :decorator

          elsif match = scan(/0[xX][0-9A-Fa-f]+[lL]?/)
            encoder.text_token match, :hex

          elsif match = scan(/0[bB][01]+[lL]?/)
            encoder.text_token match, :binary

          elsif match = scan(/(?:\d*\.\d+|\d+\.\d*)(?:[eE][+-]?\d+)?|\d+[eE][+-]?\d+/)
            if scan(/[jJ]/)
              match << matched
              encoder.text_token match, :imaginary
            else
              encoder.text_token match, :float
            end

          elsif match = scan(/0[oO][0-7]+|0[0-7]+(?![89.eE])[lL]?/)
            encoder.text_token match, :octal

          elsif match = scan(/\d+([lL])?/)
            if self[1] == nil && scan(/[jJ]/)
              match << matched
              encoder.text_token match, :imaginary
            else
              encoder.text_token match, :integer
            end

          else
            encoder.text_token getch, :error

          end

        elsif state == :def_expected
          state = :initial
          if match = scan(unicode ? /#{NAME}/o : /#{NAME}/)
            encoder.text_token match, :method
          else
            next
          end

        elsif state == :class_expected
          state = :initial
          if match = scan(unicode ? /#{NAME}/o : /#{NAME}/)
            encoder.text_token match, :class
          else
            next
          end

        elsif state == :include_expected
          if match = scan(unicode ? /#{DESCRIPTOR}/o : /#{DESCRIPTOR}/)
            if match == 'as'
              encoder.text_token match, :keyword
              from_import_state << :as
            elsif from_import_state.first == :from && match == 'import'
              encoder.text_token match, :keyword
              from_import_state << :import
            elsif from_import_state.last == :as
              # encoder.text_token match, match[0,1][unicode ? /[[:upper:]]/u : /[[:upper:]]/] ? :class : :method
              encoder.text_token match, :ident
              from_import_state.pop
            elsif IDENT_KIND[match] == :keyword
              unscan
              match = nil
              state = :initial
              next
            else
              encoder.text_token match, :include
            end
          elsif match = scan(/,/)
            from_import_state.pop if from_import_state.last == :as
            encoder.text_token match, :operator
          else
            from_import_state = []
            state = :initial
            next
          end

        else
          raise_inspect 'Unknown state', encoder, state

        end

        last_token_dot = match == '.'

      end

      if state == :string
        encoder.end_group string_type
      end

      encoder
    end

[Validate]

Generated with the Darkfish Rdoc Generator 2.