Class: Apollo::Fetcher::SmartFetcher

Inherits:
BaseFetcher show all
Defined in:
lib/apollo_crawler/fetcher/smart_fetcher.rb

Constant Summary collapse

@@DEFAULT_SLEEP = 
0.1
@@LAST_FETCH = 
nil

Class Method Summary collapse

Methods inherited from BaseFetcher

get_fake_headers

Class Method Details

.fetch(url, options = {}) ⇒ Object 



32
33
34
35
36
37
38
39
40
41
42
43
44
45
 
# File 'lib/apollo_crawler/fetcher/smart_fetcher.rb', line 32

def self.fetch(url, options = {})
	# TODO: Throw exception ???
	if(@@LAST_FETCH != nil)
		now = DateTime.now
		last = @@LAST_FETCH
		diff = @@DEFAULT_SLEEP - (last - now)
		sleep(diff)
	end

	res = BaseFetcher::fetch(url)

	@@LAST_FETCH = DateTime.now
	return res
end