Class: Apollo::Fetcher::BaseFetcher

Inherits:
Object
  • Object
show all
Defined in:
lib/apollo_crawler/fetcher/base_fetcher.rb

Direct Known Subclasses

SimpleFetcher, SmartFetcher

Class Method Summary collapse

Class Method Details

.fetch(url, options = {}) ⇒ Object 



41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
 
# File 'lib/apollo_crawler/fetcher/base_fetcher.rb', line 41

def self.fetch(url, options = {})
  begin
    uri = URI.parse(url.to_s)
  rescue Exception => e
    puts "EXCEPTION: BaseFetcher::fetch() - Unable to fetch: '#{e.to_s}'"
    return nil
  end

  # See https://github.com/lostisland/faraday
  conn = Faraday.new(:url => url) do |faraday|
    # faraday.request  :url_encoded             # form-encode POST params
    # faraday.response :logger                  # log requests to STDOUT
    faraday.adapter  Faraday.default_adapter  # make requests with Net::HTTP
  end

  # Make request
  begin
    res = conn.get(uri) do |request|
      request.headers = BaseFetcher.get_fake_headers(uri)
    end
  rescue Exception => e
    puts "EXCEPTION: BaseFetcher::fetch() - Unable to fetch: '#{e.to_s}'"
    return nil
  end

  # Return result
  return res
end

.get_fake_headers(url) ⇒ Object 



33
34
35
36
37
38
39
 
# File 'lib/apollo_crawler/fetcher/base_fetcher.rb', line 33

def self.get_fake_headers(url)
  ip = IPAddr.new(rand(2**32), Socket::AF_INET).to_s

  return {
    "X-Forwarded-For" => ip
  }
end