Class: Repub::App::Fetcher::Fetcher

Inherits:
Object
  • Object
show all
Includes:
Logger
Defined in:
lib/repub/app/fetcher.rb

Constant Summary collapse

Downloaders = 
{
  :wget     => { :cmd => 'wget', :options => '-nv -E -H -k -p -nH -nd' },
  :httrack  => { :cmd => 'httrack', :options => '-gB -r2 +*.css +*.jpg -*.xml -*.html' }
}

Constants included from Logger

Logger::LOGGER_NORMAL, Logger::LOGGER_QUIET, Logger::LOGGER_VERBOSE

Instance Method Summary collapse

Methods included from Logger

#log

Constructor Details

#initialize(options) ⇒ Fetcher

Returns a new instance of Fetcher.



36
37
38
39
40
41
42
43
44
45
46
47
 
# File 'lib/repub/app/fetcher.rb', line 36

def initialize(options)
  @options = options
  @downloader_path, @downloader_options = ENV['REPUB_DOWNLOADER'], ENV['REPUB_DOWNLOADER_OPTIONS']
  begin
    downloader = Downloaders[@options[:helper].to_sym] rescue Downloaders[:wget]
    log.debug "-- Using #{downloader[:cmd]} #{downloader[:options]}"
    @downloader_path ||= which(downloader[:cmd])
    @downloader_options ||= downloader[:options]
  rescue RuntimeError
    raise FetcherException, "unknown helper '#{@options[:helper]}'"
  end
end

Instance Method Details

#fetch ⇒ Object

Raises:



49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
 
# File 'lib/repub/app/fetcher.rb', line 49

def fetch
  url = @options[:url]
  raise FetcherException, "empty URL" if !url || url.empty?
  begin
    URI.parse(url)
  rescue
    raise FetcherException, "invalid URL: #{url}"
  end
  Cache.for_url(url) do |cache|
    log.debug "-- Downloading into #{cache.path}"
    cmd = "#{@downloader_path} #{@downloader_options} #{url}"
    unless system(cmd) && !cache.empty?
      raise FetcherException, "Fetch failed."
    end
    unless cache.cached?
      fix_filenames(cache)
      fix_encoding(cache, @options[:encoding])
    end
  end
end