Module: Apollo::Helper::Mongo

Defined in:
lib/apollo_crawler/helper/mongo_helper.rb

Class Method Summary collapse

Class Method Details

.connect(conn, opts = {}) ⇒ Object 


29
30
31
32
33
34
35
36
37
38
39
40
41
 
# File 'lib/apollo_crawler/helper/mongo_helper.rb', line 29

def self.connect(conn, opts={})
  if(opts[:verbose])
    puts "MongoDB connecting - '#{conn.inspect}"
  end

  res = ::Mongo::Connection.new(conn['host'])

  if(opts[:verbose])
    puts "MongoDB connected: #{res.inspect}"
  end

  return res
end

.csv_bulk_insert(path, model, bulk_size, validate = false, &block) ⇒ Object 


43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
 
# File 'lib/apollo_crawler/helper/mongo_helper.rb', line 43

def self.csv_bulk_insert(path, model, bulk_size, validate=false, &block)
  batch = []
  
  CSV.foreach(path) do |row|
    res = nil
    if block_given?
      res = yield row
    end

    if res.nil? == false
      if(!validate || model.where(res).length == 0)
        batch << res
      end
    end

    if((batch.length % bulk_size) == 0)
      # puts "Inserting batch '#{batch.inspect}'"
      
      model.collection.insert(batch)
      batch.clear
    end
  end

  if batch.empty? == false
    model.collection.insert(batch)
    batch.clear
  end
end