Class: Repub::App::Fetcher::FetcherSupport

Inherits:
Object
  • Object
show all
Includes:
Logger
Defined in:
lib/repub/app/fetcher.rb

Constant Summary collapse

Downloaders = 
{
  :wget     => { :cmd => 'wget', :options => '-nv -E -H -k -p -nH -nd' },
  :httrack  => { :cmd => 'httrack', :options => '-gBqQ -r2 +*.css +*.jpg -*.xml -*.html' }
}

Constants included from Logger

Logger::LOGGER_NORMAL, Logger::LOGGER_QUIET, Logger::LOGGER_VERBOSE

Instance Method Summary collapse

Methods included from Logger

#log

Constructor Details

#initialize(options) ⇒ FetcherSupport

Returns a new instance of FetcherSupport.



37
38
39
40
41
42
43
44
45
46
47
48
49
 
# File 'lib/repub/app/fetcher.rb', line 37

def initialize(options)
  @options = options
  @downloader_path, @downloader_options = ENV['REPUB_DOWNLOADER'], ENV['REPUB_DOWNLOADER_OPTIONS']
  downloader =
    begin
      Downloaders[@options[:helper].to_sym] || Downloaders[:wget]
    rescue
      Downloaders[:wget]
    end
  log.debug "-- Using #{downloader[:cmd]} #{downloader[:options]}"
  @downloader_path ||= which(downloader[:cmd])
  @downloader_options ||= downloader[:options]
end

Instance Method Details

#fetch ⇒ Object

Raises:



51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
 
# File 'lib/repub/app/fetcher.rb', line 51

def fetch
  url = @options[:url]
  raise FetcherException, "empty URL" if !url || url.empty?
  begin
    URI.parse(url)
  rescue
    raise FetcherException, "invalid URL: #{url}"
  end
  Cache.for_url(url) do |cache|
    log.debug "-- Downloading into #{cache.path}"
    cmd = "#{@downloader_path} #{@downloader_options} #{url}"
    unless system(cmd) && !cache.empty?
      raise FetcherException, "Fetch failed."
    end
    unless cache.cached?
      preprocess cache
    end
  end
end