Class: Apollo::PlatformProgram

Inherits:
BaseProgram show all
Defined in:
lib/apollo_crawler/program/platform_program.rb

Constant Summary collapse

DEFAULT_OPTIONS = 
{
  :version => nil
}

Constants inherited from BaseProgram

BaseProgram::CONFIG_DIR

Instance Attribute Summary

Attributes inherited from BaseProgram

#amqp, #config, #mongo, #options, #optparser

Instance Method Summary collapse

Methods inherited from BaseProgram

get_config_path, #init_amqp, #init_mongo, #init_seeds, #init_seeds_crawlers, #load_config, #load_config_file, #load_configs, #parse_options, #request_exit, require_files

Constructor Details

#initialize ⇒ PlatformProgram

Initializer - Constructor



63
64
65
66
67
 
# File 'lib/apollo_crawler/program/platform_program.rb', line 63

def initialize
  super
  
  self.options.merge!(DEFAULT_OPTIONS)
end

Instance Method Details

#enqueue_crawlers_urls(amqp, crawlers = Apollo::Crawler::BaseCrawler.subclasses, opts = {}) ⇒ Object 



100
101
102
103
104
105
 
# File 'lib/apollo_crawler/program/platform_program.rb', line 100

def enqueue_crawlers_urls(amqp, crawlers=Apollo::Crawler::BaseCrawler.subclasses, opts={})
  crawlers.each do |crawler|
    i = crawler.new
    Apollo::Scheduler::BaseScheduler::schedule(i.url, crawler)
  end  
end

#init_agents(amqp, opts = {}) ⇒ Object 



125
126
127
128
129
130
131
 
# File 'lib/apollo_crawler/program/platform_program.rb', line 125

def init_agents(amqp, opts={})
  puts "Initializing agents"

  init_crawlers(amqp, opts)
  init_domainers(amqp, opts)
  init_fetchers(amqp, opts)  
end

#init_crawlers(amqp, opts = {}) ⇒ Object 



107
108
109
110
 
# File 'lib/apollo_crawler/program/platform_program.rb', line 107

def init_crawlers(amqp, opts={})
  crawlers = []
  crawlers << Apollo::Agent::CrawlerAgent.new(amqp, self.options)
end

#init_domainers(amqp, opts = {}) ⇒ Object 



112
113
114
115
 
# File 'lib/apollo_crawler/program/platform_program.rb', line 112

def init_domainers(amqp, opts={})
  domainers = []
  domainers << Apollo::Agent::DomainerAgent.new(amqp, self.options)
end

#init_domains(opts = {}) ⇒ Object 



133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
 
# File 'lib/apollo_crawler/program/platform_program.rb', line 133

def init_domains(opts={})
  path = File.join(File.dirname(__FILE__), "../../../tmp/top-1m.csv")
  puts "#{path}"
  if(File.exists?(path) == false)
    return 0
  end

  Thread::new {
    Apollo::Helper::Mongo::csv_bulk_insert(path, Apollo::Model::Domain, 1000, false) do |row|
      rank = row[0].to_i
      name = row[1]

      res = {
        :rank => rank, 
        :name => name
      }

      res
    end
  }
end

#init_fetchers(amqp, opts = {}) ⇒ Object 



117
118
119
120
121
122
123
 
# File 'lib/apollo_crawler/program/platform_program.rb', line 117

def init_fetchers(amqp, opts={})
  fetchers = []
  fetchers << Apollo::Agent::FetcherAgent.new(amqp, self.options)

  # TODO: This should not be here!
  enqueue_crawlers_urls(amqp, Apollo::Crawler::BaseCrawler.subclasses, opts)
end

#init_options ⇒ Object 



69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
 
# File 'lib/apollo_crawler/program/platform_program.rb', line 69

def init_options()
  self.optparser = OptionParser.new do | opts |
    opts.banner = "Usage: apollo-platform [OPTIONS]"

    opts.separator ""
          opts.separator "Specific options:"

    # This displays the help screen, all programs are
    # assumed to have this option.
    opts.on('-h', '--help', 'Display this screen') do
      self.options[:show_help] = true
    end

    opts.on('-e', '--environment [NAME]', "Environment used, default '#{options[:env]}'") do |name|
      self.options[:env] = name
    end

    opts.on('-d', '--daemon', 'Run Apollo Platform daemon') do
      self.options[:daemon] = true
    end

    opts.on('-v', '--verbose', 'Enable verbose output') do
      self.options[:verbose] = true
    end

    opts.on('-V', '--version', 'Show version info') do
      self.options[:version] = true
    end
  end
end

#init_program(args) ⇒ Object 



155
156
157
158
159
160
161
 
# File 'lib/apollo_crawler/program/platform_program.rb', line 155

def init_program(args)
  res = super(args)
  return res unless res.nil?

  init_agents(self.amqp, self.options)
  return nil
end

#process_options(args) ⇒ Object 



163
164
165
166
167
168
169
170
171
172
173
174
175
176
 
# File 'lib/apollo_crawler/program/platform_program.rb', line 163

def process_options(args)
  if(self.options[:version])
    puts Apollo::VERSION
    return 0
  end

  if(self.options[:show_help])
    puts optparser
    return 0
  end

  # Return nil, it means program can freely continue.
  return nil
end

#requeue_fetching_urls(opts = {}) ⇒ Object 



178
179
180
181
182
183
184
185
186
 
# File 'lib/apollo_crawler/program/platform_program.rb', line 178

def requeue_fetching_urls(opts={})
  urls = Apollo::Model::QueuedUrl.where(:state => :fetching)
  urls.each do |url|
    puts "Requeing '#{url.inspect}'" if opts[:verbose]

    url.state = :queued
    url.save
  end
end

#run(args = ARGV) ⇒ Object

Run Program



189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
 
# File 'lib/apollo_crawler/program/platform_program.rb', line 189

def run(args = ARGV)
  res = super(args)
  return res unless res.nil?

  init_domains()

  requeue_fetching_urls(self.options)

  # Here we start
  # if(ARGV.length < 1)
  #  puts optparser
  #  return 0
  # end

  res_code = 0
  if(self.options[:daemon])
    planner = Apollo::Planner::SmartPlanner.new(self.amqp, self.mongo, self.options)
    res_code = planner.run(self.options)
  end

  return request_exit(res_code)
end