Class: TerrImporter::Application::Downloader

Inherits:
Object
  • Object
show all
Includes:
DownloadHelper
Defined in:
lib/terrimporter/downloader.rb

Instance Method Summary collapse

Methods included from DownloadHelper

#create_dir_path

Constructor Details

#initialize(base_uri) ⇒ Downloader

Returns a new instance of Downloader.



9
10
11
12
 
# File 'lib/terrimporter/downloader.rb', line 9

def initialize(base_uri)
  @base_uri = base_uri
  LOG.debug "Downloader initialized to uri: #{@base_uri}"
end

Instance Method Details

#batch_download(remote_path, local_path, type_filter = "") ⇒ Object 



32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
 
# File 'lib/terrimporter/downloader.rb', line 32

def batch_download(remote_path, local_path, type_filter = "")
  source_path = url(remote_path)
  create_dir_path local_path
  LOG.debug "Download multiple files from #{source_path} to #{local_path} #{"allowed extensions: " + type_filter unless type_filter.empty?}"

  files = html_directory_list(source_path)

  unless type_filter.empty?
    LOG.debug "Apply type filter: #{type_filter}"
    files = files.find_all { |file| file =~ Regexp.new(".*" + type_filter.robust_split.join("|") + "$") }
  end

  LOG.info "Download #{files.size} files..."
  files.each do |file|
    local_file_path = File.join(local_path.to_s, file)
    self.download(File.join(source_path.to_s, file), local_file_path)
  end
end

#download(remote_path, local_path = nil) ⇒ Object 



14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
 
# File 'lib/terrimporter/downloader.rb', line 14

def download(remote_path, local_path = nil)
  remote_url = url(remote_path)
  begin
    if local_path.nil? #download to buffer
      LOG.debug "Download #{remote_url} to buffer"
      data = StringIO.new
      remote_url.open { |io| data = io.read }
      data.to_s
    else
      LOG.info "Download #{remote_url} to local path #{local_path}"
      create_dir_path File.dirname(local_path)
      open(local_path, "wb") { |file| file.write(remote_url.open.read) }
    end
  rescue SocketError => e
    raise DefaultError, "Error opening url #{remote_url}: \n #{e.message}"
  end
end