Class: CSSPool::CSS::Tokenizer

### Home Classes Methods

### In Files

- lib/csspool/css/tokenizer.rb

### Parent

CSSPool::CSS::Parser

### Namespace

- CLASS CSSPool::CSS::Tokenizer::ScanError

### Methods

- #_next_token
- #action
- #load_file
- #next_token
- #scan
- #scan_file
- #scan_setup
- #scan_str
- #st

### Files

- CHANGELOG.rdoc
- Manifest.txt

### Class/Module Index

Quicksearch

- CSSPool
- CSSPool::CSS
- CSSPool::CSS::Charset
- CSSPool::CSS::Declaration
- CSSPool::CSS::Document
- CSSPool::CSS::DocumentHandler
- CSSPool::CSS::DocumentQuery
- CSSPool::CSS::FontfaceRule
- CSSPool::CSS::ImportRule
- CSSPool::CSS::KeyframesBlock
- CSSPool::CSS::KeyframesRule
- CSSPool::CSS::MediaFeature
- CSSPool::CSS::MediaQuery
- CSSPool::CSS::MediaQueryList
- CSSPool::CSS::MediaType
- CSSPool::CSS::NamespaceRule
- CSSPool::CSS::Parser
- CSSPool::CSS::RuleSet
- CSSPool::CSS::SupportsRule
- CSSPool::CSS::Tokenizer
- CSSPool::CSS::Tokenizer::ScanError
- CSSPool::Collection
- CSSPool::Node
- CSSPool::SAC
- CSSPool::SAC::Document
- CSSPool::SAC::Parser
- CSSPool::Selector
- CSSPool::Selectors
- CSSPool::Selectors::Additional
- CSSPool::Selectors::Attribute
- CSSPool::Selectors::Class
- CSSPool::Selectors::Id
- CSSPool::Selectors::PseudoClass
- CSSPool::Selectors::PseudoElement
- CSSPool::Selectors::Simple
- CSSPool::Selectors::Type
- CSSPool::Selectors::Universal
- CSSPool::Terms
- CSSPool::Terms::Function
- CSSPool::Terms::Hash
- CSSPool::Terms::Ident
- CSSPool::Terms::Math
- CSSPool::Terms::Number
- CSSPool::Terms::Ratio
- CSSPool::Terms::Resolution
- CSSPool::Terms::Rgb
- CSSPool::Terms::String
- CSSPool::Terms::URI
- CSSPool::Visitors
- CSSPool::Visitors::Children
- CSSPool::Visitors::Comparable
- CSSPool::Visitors::Iterator
- CSSPool::Visitors::ToCSS
- CSSPool::Visitors::ToMinifiedCSS
- CSSPool::Visitors::Visitor

No matching classes.

# CSSPool::CSS::Tokenizer

### Attributes

filename[R]
lineno[R]
state[RW]

### Public Instance Methods

_next_token() click to toggle source

    # File lib/csspool/css/tokenizer.rb, line 55
    def _next_token
      text = @ss.peek(1)
      @lineno  +=  1  if text == "\n"
      token = case @state
      when nil
        case
        when (text = @ss.scan(/url\([\s]*("([^\n\r\f\\"]|\\(\n|\r\n|\r|\f)|[^\00--\1177]|\\[0-9A-Fa-f]{1,6}(\r\n|[\s])?|\\[^\n\r\f0-9A-Fa-f])*"|'([^\n\r\f\\']|\\(\n|\r\n|\r|\f)|[^\00--\1177]|\\[0-9A-Fa-f]{1,6}(\r\n|[\s])?|\\[^\n\r\f0-9A-Fa-f])*')[\s]*\)/))
           action { [:URI, st(text)] }

        when (text = @ss.scan(/url\([\s]*([!#\$%&*-~]|[^\00--\1177]|\\[0-9A-Fa-f]{1,6}(\r\n|[\s])?|\\[^\n\r\f0-9A-Fa-f])*[\s]*\)/))
           action { [:URI, st(text)] }

        when (text = @ss.scan(/U\+[0-9a-fA-F?]{1,6}(-[0-9a-fA-F]{1,6})?/))
           action {[:UNICODE_RANGE, st(text)] }

        when (text = @ss.scan(/[\s]*\/\*(.|\s)*?\*\/[\s]*/))
           action { next_token }

        when (text = @ss.scan(/not\([\s]*/))
           action { [:NOT_PSEUDO_CLASS, st(text)] }

        when (text = @ss.scan(/(nth\-child|nth\-last\-child|nth\-of\-type|nth\-last\-of\-type)\([\s]*([\+\-]?[0-9]*n([\s]*[\+\-][\s]*[0-9]+)?|[\+\-]?[0-9]+|odd|even)[\s]*\)/))
           action { [:NTH_PSEUDO_CLASS, st(text)] }

        when (text = @ss.scan(/(\-[A-Za-z]+\-)?(matches|any)\(\s*/))
           action { [:MATCHES_PSEUDO_CLASS, st(text)] }

        when (text = @ss.scan(/(domain|url\-prefix)\([\s]*("([^\n\r\f\\"]|\\(\n|\r\n|\r|\f)|[^\00--\1177]|\\[0-9A-Fa-f]{1,6}(\r\n|[\s])?|\\[^\n\r\f0-9A-Fa-f])*"|'([^\n\r\f\\']|\\(\n|\r\n|\r|\f)|[^\00--\1177]|\\[0-9A-Fa-f]{1,6}(\r\n|[\s])?|\\[^\n\r\f0-9A-Fa-f])*')[\s]*\)/))
           action { [:FUNCTION_NO_QUOTE, st(text)] }

        when (text = @ss.scan(/(domain|url\-prefix)\([\s]*([!#\$%&*-~]|[^\00--\1177]|\\[0-9A-Fa-f]{1,6}(\r\n|[\s])?|\\[^\n\r\f0-9A-Fa-f])*[\s]*\)/))
           action { [:FUNCTION_NO_QUOTE, st(text)] }

        when (text = @ss.scan(/(\-moz\-non\-element|\-moz\-anonymous\-block|\-moz\-anonymous\-positioned\-block|\-moz\-mathml\-anonymous\-block|\-moz\-xul\-anonymous\-block|\-moz\-hframeset\-border|\-moz\-vframeset\-border|\-moz\-line\-frame|\-moz\-button\-content|\-moz\-buttonlabel|\-moz\-cell\-content|\-moz\-dropdown\-list|\-moz\-fieldset\-content|\-moz\-frameset\-blank|\-moz\-display\-comboboxcontrol\-frame|\-moz\-html\-canvas\-content|\-moz\-inline\-table|\-moz\-table|\-moz\-table\-cell|\-moz\-table\-column\-group|\-moz\-table\-column|\-moz\-table\-outer|\-moz\-table\-row\-group|\-moz\-table\-row|\-moz\-canvas|\-moz\-pagebreak|\-moz\-page|\-moz\-pagecontent|\-moz\-page\-sequence|\-moz\-scrolled\-content|\-moz\-scrolled\-canvas|\-moz\-scrolled\-page\-sequence|\-moz\-column\-content|\-moz\-viewport|\-moz\-viewport\-scroll|\-moz\-anonymous\-flex\-item|\-moz\-tree\-column|\-moz\-tree\-row|\-moz\-tree\-separator|\-moz\-tree\-cell|\-moz\-tree\-indentation|\-moz\-tree\-line|\-moz\-tree\-twisty|\-moz\-tree\-image|\-moz\-tree\-cell\-text|\-moz\-tree\-checkbox|\-moz\-tree\-progressmeter|\-moz\-tree\-drop\-feedback|\-moz\-svg\-outer\-svg\-anon\-child|\-moz\-svg\-foreign\-content|\-moz\-svg\-text)\(/))
           action { [:MOZ_PSEUDO_ELEMENT, st(text)] }

        when (text = @ss.scan(/[\s]*@media[\s]*/))
           action { @state = :LOGICQUERY; [:MEDIA_SYM, st(text)] }

        when (text = @ss.scan(/[\s]*@supports[\s]*/))
           action { @state = :LOGICQUERY; [:SUPPORTS_SYM, st(text)] }

        when (text = @ss.scan(/(\-[A-Za-z]+\-)?calc\(\s*/))
           action { [:CALC_SYM, st(text)] }

        when (text = @ss.scan(/\-?([_A-Za-z]|[^\00--\1177]|\\[0-9A-Fa-f]{1,6}(\r\n|[\s])?|\\[^\n\r\f0-9A-Fa-f])([_A-Za-z0-9-]|[^\00--\1177]|\\[0-9A-Fa-f]{1,6}(\r\n|[\s])?|\\[^\n\r\f0-9A-Fa-f])*\(\s*/))
           action { [:FUNCTION, st(text)] }

        when (text = @ss.scan(/[\s]*@import[\s]*/))
           action { [:IMPORT_SYM, st(text)] }

        when (text = @ss.scan(/[\s]*@page[\s]*/))
           action { [:PAGE_SYM, st(text)] }

        when (text = @ss.scan(/[\s]*@charset[\s]*/))
           action { [:CHARSET_SYM, st(text)] }

        when (text = @ss.scan(/[\s]*@(\-[A-Za-z]+\-)?document[\s]*/))
           action { [:DOCUMENT_QUERY_SYM, st(text)] }

        when (text = @ss.scan(/[\s]*@namespace[\s]*/))
           action { [:NAMESPACE_SYM, st(text)] }

        when (text = @ss.scan(/[\s]*@font\-face[\s]*/))
           action { [:FONTFACE_SYM, st(text)] }

        when (text = @ss.scan(/[\s]*@(\-[A-Za-z]+\-)?keyframes[\s]*/))
           action { [:KEYFRAMES_SYM, st(text)] }

        when (text = @ss.scan(/[\s]*!([\s]*|[\s]*\/\*(.|\s)*?\*\/[\s]*)important[\s]*/))
           action { [:IMPORTANT_SYM, st(text)] }

        when (text = @ss.scan(/\-\-([_A-Za-z0-9-]|[^\00--\1177]|\\[0-9A-Fa-f]{1,6}(\r\n|[\s])?|\\[^\n\r\f0-9A-Fa-f])+/))
           action { [:VARIABLE_NAME, st(text)] }

        when (text = @ss.scan(/\-?([_A-Za-z]|[^\00--\1177]|\\[0-9A-Fa-f]{1,6}(\r\n|[\s])?|\\[^\n\r\f0-9A-Fa-f])([_A-Za-z0-9-]|[^\00--\1177]|\\[0-9A-Fa-f]{1,6}(\r\n|[\s])?|\\[^\n\r\f0-9A-Fa-f])*/))
           action { [:IDENT, st(text)] }

        when (text = @ss.scan(/\#([_A-Za-z0-9-]|[^\00--\1177]|\\[0-9A-Fa-f]{1,6}(\r\n|[\s])?|\\[^\n\r\f0-9A-Fa-f])+/))
           action { [:HASH, st(text)] }

        when (text = @ss.scan(/[\s]*~=[\s]*/))
           action { [:INCLUDES, st(text)] }

        when (text = @ss.scan(/[\s]*\|=[\s]*/))
           action { [:DASHMATCH, st(text)] }

        when (text = @ss.scan(/[\s]*\^=[\s]*/))
           action { [:PREFIXMATCH, st(text)] }

        when (text = @ss.scan(/[\s]*\$=[\s]*/))
           action { [:SUFFIXMATCH, st(text)] }

        when (text = @ss.scan(/[\s]*\*=[\s]*/))
           action { [:SUBSTRINGMATCH, st(text)] }

        when (text = @ss.scan(/[\s]*!=[\s]*/))
           action { [:NOT_EQUAL, st(text)] }

        when (text = @ss.scan(/[\s]*=[\s]*/))
           action { [:EQUAL, st(text)] }

        when (text = @ss.scan(/[\s]*\)/))
           action { [:RPAREN, st(text)] }

        when (text = @ss.scan(/[\s]*\(/))
           action { [:LPAREN, st(text)] }

        when (text = @ss.scan(/\[[\s]*/))
           action { [:LSQUARE, st(text)] }

        when (text = @ss.scan(/[\s]*\]/))
           action { [:RSQUARE, st(text)] }

        when (text = @ss.scan(/[\s]*\+[\s]*/))
           action { [:PLUS, st(text)] }

        when (text = @ss.scan(/[\s]*\{[\s]*/))
           action { [:LBRACE, st(text)] }

        when (text = @ss.scan(/[\s]*\}[\s]*/))
           action { [:RBRACE, st(text)] }

        when (text = @ss.scan(/[\s]*>[\s]*/))
           action { [:GREATER, st(text)] }

        when (text = @ss.scan(/[\s]*,[\s]*/))
           action { [:COMMA, st(',')] }

        when (text = @ss.scan(/[\s]*;[\s]*/))
           action { [:SEMI, st(';')] }

        when (text = @ss.scan(/\:/))
           action { [:COLON, st(text)] }

        when (text = @ss.scan(/\*/))
           action { [:STAR, st(text)] }

        when (text = @ss.scan(/[\s]*~[\s]*/))
           action { [:TILDE, st(text)] }

        when (text = @ss.scan(/[\s]*([0-9]*\.[0-9]+|[0-9]+)(px|cm|mm|in|pt|pc|mozmm|em|ex|ch|rem|vh|vw|vmin|vmax)[\s]*/))
           action { [:LENGTH, st(text)] }

        when (text = @ss.scan(/[\s]*([0-9]*\.[0-9]+|[0-9]+)(deg|rad|grad)[\s]*/))
           action { [:ANGLE, st(text)] }

        when (text = @ss.scan(/[\s]*([0-9]*\.[0-9]+|[0-9]+)(ms|s)[\s]*/))
           action { [:TIME, st(text)] }

        when (text = @ss.scan(/[\s]*([0-9]*\.[0-9]+|[0-9]+)[k]?hz[\s]*/))
           action { [:FREQ, st(text)] }

        when (text = @ss.scan(/[\s]*([0-9]*\.[0-9]+|[0-9]+)%[\s]*/))
           action { [:PERCENTAGE, st(text)] }

        when (text = @ss.scan(/[\s]*[1-9][0-9]*[\s]*\/[\s]*[1-9][0-9]*[\s]*/))
           action { [:RATIO, st(text)] }

        when (text = @ss.scan(/[\s]*([0-9]*\.[0-9]+|[0-9]+)[\s]*/))
           action { [:NUMBER, st(text)] }

        when (text = @ss.scan(/[\s]*\/\/[\s]*/))
           action { [:DOUBLESLASH, st(text)] }

        when (text = @ss.scan(/[\s]*\/[\s]*/))
           action { [:SLASH, st('/')] }

        when (text = @ss.scan(/<!--/))
           action { [:CDO, st(text)] }

        when (text = @ss.scan(/-->/))
           action { [:CDC, st(text)] }

        when (text = @ss.scan(/[\s]*\-(?!\-?([_A-Za-z]|[^\00--\1177]|\\[0-9A-Fa-f]{1,6}(\r\n|[\s])?|\\[^\n\r\f0-9A-Fa-f])([_A-Za-z0-9-]|[^\00--\1177]|\\[0-9A-Fa-f]{1,6}(\r\n|[\s])?|\\[^\n\r\f0-9A-Fa-f])*)[\s]*/))
           action { [:MINUS, st(text)] }

        when (text = @ss.scan(/[\s]*\+[\s]*/))
           action { [:PLUS, st(text)] }

        when (text = @ss.scan(/[\s]+/))
           action { [:S, st(text)] }

        when (text = @ss.scan(/("([^\n\r\f\\"]|\\(\n|\r\n|\r|\f)|[^\00--\1177]|\\[0-9A-Fa-f]{1,6}(\r\n|[\s])?|\\[^\n\r\f0-9A-Fa-f])*"|'([^\n\r\f\\']|\\(\n|\r\n|\r|\f)|[^\00--\1177]|\\[0-9A-Fa-f]{1,6}(\r\n|[\s])?|\\[^\n\r\f0-9A-Fa-f])*')/))
           action { [:STRING, st(text)] }

        when (text = @ss.scan(/("([^\n\r\f\\"]|\\(\n|\r\n|\r|\f)|[^\00--\1177]|\\[0-9A-Fa-f]{1,6}(\r\n|[\s])?|\\[^\n\r\f0-9A-Fa-f])*|'([^\n\r\f\\']|\\(\n|\r\n|\r|\f)|[^\00--\1177]|\\[0-9A-Fa-f]{1,6}(\r\n|[\s])?|\\[^\n\r\f0-9A-Fa-f])*)/))
           action { [:INVALID, st(text)] }

        when (text = @ss.scan(/./))
           action { [st(text), st(text)] }

        else
          text = @ss.string[@ss.pos .. -1]
          raise  ScanError, "can not match: '" + text + "'"
        end  # if

      when :LOGICQUERY
        case
        when (text = @ss.scan(/url\([\s]*("([^\n\r\f\\"]|\\(\n|\r\n|\r|\f)|[^\00--\1177]|\\[0-9A-Fa-f]{1,6}(\r\n|[\s])?|\\[^\n\r\f0-9A-Fa-f])*"|'([^\n\r\f\\']|\\(\n|\r\n|\r|\f)|[^\00--\1177]|\\[0-9A-Fa-f]{1,6}(\r\n|[\s])?|\\[^\n\r\f0-9A-Fa-f])*')[\s]*\)/))
           action { [:URI, st(text)] }

        when (text = @ss.scan(/url\([\s]*([!#\$%&*-~]|[^\00--\1177]|\\[0-9A-Fa-f]{1,6}(\r\n|[\s])?|\\[^\n\r\f0-9A-Fa-f])*[\s]*\)/))
           action { [:URI, st(text)] }

        when (text = @ss.scan(/\-?([_A-Za-z]|[^\00--\1177]|\\[0-9A-Fa-f]{1,6}(\r\n|[\s])?|\\[^\n\r\f0-9A-Fa-f])([_A-Za-z0-9-]|[^\00--\1177]|\\[0-9A-Fa-f]{1,6}(\r\n|[\s])?|\\[^\n\r\f0-9A-Fa-f])*\(\s*/))
           action { [:FUNCTION, st(text)] }

        when (text = @ss.scan(/[\s]*(and|only|not|or)[\s]+/))
           action { [text.upcase.strip.intern, st(text)] }

        when (text = @ss.scan(/[\s]*([0-9]*\.[0-9]+|[0-9]+)(dpi|dpcm)/))
           action { [:RESOLUTION, st(text)]}

        when (text = @ss.scan(/[\s]*([0-9]*\.[0-9]+|[0-9]+)(px|cm|mm|in|pt|pc|mozmm|em|ex|ch|rem|vh|vw|vmin|vmax)[\s]*/))
           action { [:LENGTH, st(text)] }

        when (text = @ss.scan(/[\s]*([0-9]*\.[0-9]+|[0-9]+)(deg|rad|grad)[\s]*/))
           action { [:ANGLE, st(text)] }

        when (text = @ss.scan(/[\s]*([0-9]*\.[0-9]+|[0-9]+)(ms|s)[\s]*/))
           action { [:TIME, st(text)] }

        when (text = @ss.scan(/[\s]*([0-9]*\.[0-9]+|[0-9]+)[k]?hz[\s]*/))
           action { [:FREQ, st(text)] }

        when (text = @ss.scan(/[\s]*([0-9]*\.[0-9]+|[0-9]+)%[\s]*/))
           action { [:PERCENTAGE, st(text)] }

        when (text = @ss.scan(/[\s]*[1-9][0-9]*[\s]*\/[\s]*[1-9][0-9]*[\s]*/))
           action { [:RATIO, st(text)] }

        when (text = @ss.scan(/[\s]*([0-9]*\.[0-9]+|[0-9]+)[\s]*/))
           action { [:NUMBER, st(text)] }

        when (text = @ss.scan(/\-?([_A-Za-z]|[^\00--\1177]|\\[0-9A-Fa-f]{1,6}(\r\n|[\s])?|\\[^\n\r\f0-9A-Fa-f])([_A-Za-z0-9-]|[^\00--\1177]|\\[0-9A-Fa-f]{1,6}(\r\n|[\s])?|\\[^\n\r\f0-9A-Fa-f])*/))
           action { [:IDENT, st(text)] }

        when (text = @ss.scan(/[\s]*,[\s]*/))
           action { [:COMMA, st(',')] }

        when (text = @ss.scan(/[\s]*\)/))
           action { [:RPAREN, st(text)] }

        when (text = @ss.scan(/[\s]*\(/))
           action { [:LPAREN, st(text)] }

        when (text = @ss.scan(/\:/))
           action { [:COLON, st(text)] }

        when (text = @ss.scan(/[\s]*!([\s]*|[\s]*\/\*(.|\s)*?\*\/[\s]*)important[\s]*/))
           action { [:IMPORTANT_SYM, st(text)] }

        when (text = @ss.scan(/[\s]*\{[\s]*/))
           action { @state = nil; [:LBRACE, st(text)] }

        when (text = @ss.scan(/[\s]+/))
           action { [:S, st(text)] }

        else
          text = @ss.string[@ss.pos .. -1]
          raise  ScanError, "can not match: '" + text + "'"
        end  # if

      else
        raise  ScanError, "undefined state: '" + state.to_s + "'"
      end  # case state
      token
    end

action() click to toggle source

    # File lib/csspool/css/tokenizer.rb, line 24
    def action
      yield
    end

load_file( filename ) click to toggle source

    # File lib/csspool/css/tokenizer.rb, line 34
    def load_file( filename )
      @filename = filename
      open(filename, "r") do |f|
        scan_setup(f.read)
      end
    end

next_token() click to toggle source

    # File lib/csspool/css/tokenizer.rb, line 47
    def next_token
      return if @ss.eos?

      # skips empty actions
      until token = _next_token or @ss.eos?; end
      token
    end

scan(str) click to toggle source
Alias for: scan_str
scan_file( filename ) click to toggle source

    # File lib/csspool/css/tokenizer.rb, line 41
    def scan_file( filename )
      load_file(filename)
      do_parse
    end

scan_setup(str) click to toggle source

    # File lib/csspool/css/tokenizer.rb, line 18
    def scan_setup(str)
      @ss = StringScanner.new(str)
      @lineno =  1
      @state  = nil
    end

scan_str(str) click to toggle source

    # File lib/csspool/css/tokenizer.rb, line 28
    def scan_str(str)
      scan_setup(str)
      do_parse
    end

Also aliased as: scan
st(o) click to toggle source

    # File lib/csspool/css/tokenizer.rb, line 325
    def st o
      @st ||= Hash.new { |h,k| h[k] = k }
      @st[o]
    end

[Validate]

Generated with the Darkfish Rdoc Generator 2.