Module: TheFox::Sengi

Defined in:

lib/sengi/uri.rb,
lib/sengi/crawler.rb,
lib/sengi/version.rb,
lib/sengi/crawler_worker.rb

Defined Under Namespace

Classes: Crawler, CrawlerWorker, Uri

Constant Summary

VERSION =

'0.1.0'

DATE =

'2016-05-07'

HOMEPAGE =

'https://github.com/TheFox/sengi'

HTTP_USER_AGENT =

HTTP_USER_AGENT = “Sengi SearchENGIne/#VERSION”

'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_4) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/49.0.2623.108 Safari/537.36'

HTTP_REFERER =

'https://www.google.com/'

URL_DELAY =

2

URL_SEPARATE_DELAY =

5

URL_RESCHEDULE =

300