Class: WebRobots::RobotsTxt

### Home Classes Methods

### In Files

- lib/webrobots/robotstxt.rb

### Parent

Object

### Namespace

- CLASS WebRobots::RobotsTxt::AccessControlLine
- CLASS WebRobots::RobotsTxt::AgentLine
- CLASS WebRobots::RobotsTxt::AllowLine
- CLASS WebRobots::RobotsTxt::CrawlDelayLine
- CLASS WebRobots::RobotsTxt::DisallowLine
- CLASS WebRobots::RobotsTxt::ExtentionLine
- CLASS WebRobots::RobotsTxt::Line
- CLASS WebRobots::RobotsTxt::Parser
- CLASS WebRobots::RobotsTxt::Record

### Methods

- ::new
- ::unfetchable
- #allow?
- #crawl_delay
- #error!
- #options

### Files

- LICENSE.txt
- README.rdoc

### Class/Module Index

Quicksearch

- WebRobots
- WebRobots::Error
- WebRobots::ParseError
- WebRobots::RobotsTxt
- WebRobots::RobotsTxt::AccessControlLine
- WebRobots::RobotsTxt::AgentLine
- WebRobots::RobotsTxt::AllowLine
- WebRobots::RobotsTxt::CrawlDelayLine
- WebRobots::RobotsTxt::DisallowLine
- WebRobots::RobotsTxt::ExtentionLine
- WebRobots::RobotsTxt::Line
- WebRobots::RobotsTxt::Parser
- WebRobots::RobotsTxt::Record
- Nokogiri
- Nokogiri::HTML
- Nokogiri::HTML::Document
- Webrobots

No matching classes.

# WebRobots::RobotsTxt

### Constants

DISALLOW_ALL

### Attributes

error[RW]
site[R]
sitemaps[R]
timestamp[R]

### Public Class Methods

new(site, records, options = nil) click to toggle source

    # File lib/webrobots/robotstxt.rb, line 533
    def initialize(site, records, options = nil)
      @timestamp = Time.now
      @site = site
      @options = options || {}
      @last_checked_at = nil

      @error = @options[:error]
      @target = @options[:target]
      @sitemaps = @options[:sitemaps] || []
      @crawl_delay_handler = @options[:crawl_delay_handler]

      if records && !records.empty?
        @records, defaults = [], []
        records.each { |record|
          if record.default?
            defaults << record
          elsif !@target || record.match?(@target)
            @records << record
          end
        }
        @records.concat(defaults)
      else
        @records = []
      end
    end

unfetchable(site, reason, target = nil) click to toggle source

    # File lib/webrobots/robotstxt.rb, line 610
    def self.unfetchable(site, reason, target = nil)
      Parser.new(target).parse(DISALLOW_ALL, site).tap { |robots_txt|
        robots_txt.error = reason
      }
    end

### Public Instance Methods

allow?(request_uri, user_agent = nil) click to toggle source

    # File lib/webrobots/robotstxt.rb, line 585
    def allow?(request_uri, user_agent = nil)
      record = find_record(user_agent) or return true
      allow = record.allow?(request_uri)
      if delay = record.delay and @crawl_delay_handler
        @crawl_delay_handler.call(delay, @last_checked_at)
      end
      @last_checked_at = Time.now
      return allow
    end

crawl_delay(user_agent = nil) click to toggle source

    # File lib/webrobots/robotstxt.rb, line 595
    def crawl_delay(user_agent = nil)
      record = find_record(user_agent) or return 0
      record.delay or return 0
    end

error!() click to toggle source

    # File lib/webrobots/robotstxt.rb, line 562
    def error!
      raise @error if @error
    end

options(user_agent = nil) click to toggle source

    # File lib/webrobots/robotstxt.rb, line 600
    def options(user_agent = nil)
      record = find_record(user_agent) or return {}
      record.options
    end

[Validate]

Generated with the Darkfish Rdoc Generator 2.