Module: Scrape

Defined in:
lib/scrape.rb,
lib/scrape/version.rb

Defined Under Namespace

Modules: DSL Classes: Application, CLI, DefaultLoader, FileNotFound, Match, RobotsTxt, RobotsTxtRules, Site

Constant Summary collapse

VERSION = 
'0.2.2'

Class Attribute Summary collapse

Class Method Summary collapse

Class Attribute Details

.user_agent ⇒ Object 



24
25
26
 
# File 'lib/scrape.rb', line 24

def user_agent
  @user_agent || "Scrape/#{Scrape::VERSION}"
end

Class Method Details

.load_scrapefile(path) ⇒ Object 



36
37
38
 
# File 'lib/scrape.rb', line 36

def load_scrapefile path
  Application.new path
end

.logger ⇒ Object 



28
29
30
 
# File 'lib/scrape.rb', line 28

def logger
  @logger ||= Logger.new STDOUT
end

.logger=(log) ⇒ Object 



32
33
34
 
# File 'lib/scrape.rb', line 32

def logger= log
  @logger = log
end

.open(url, headers = {}, &block) ⇒ Object 



40
41
42
43
 
# File 'lib/scrape.rb', line 40

def open url, headers = {}, &block
  headers = {"User-Agent" => user_agent}.merge(headers)
  super(url, headers, &block).read
end