Class: DataMapper::YS::Scraper::Composite

Inherits:
Base
  • Object
show all
Defined in:
lib/dm-ys/scraper.rb

Overview

Composite Scraper

Instance Method Summary collapse

Methods inherited from Base

#count, #initialize, #register_properties!, #uri

Methods included from CachedAccessor

included

Constructor Details

This class inherits a constructor from DataMapper::YS::Scraper::Base

Instance Method Details

#labels ⇒ Object 



201
202
203
 
# File 'lib/dm-ys/scraper.rb', line 201

def labels
  pages.first.labels
end

#names ⇒ Object 



197
198
199
 
# File 'lib/dm-ys/scraper.rb', line 197

def names
  pages.first.names
end

#pages ⇒ Object 



193
194
195
 
# File 'lib/dm-ys/scraper.rb', line 193

def pages
  @pages ||= execute
end

#records ⇒ Object 



205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
 
# File 'lib/dm-ys/scraper.rb', line 205

def records
  records = []
  digests = Set.new
  pages.each do |page|
    page.records.each do |entry|
      if config.uniq_entry?
        sha1 = entry.attributes.merge(:id=>nil).inspect
        next if digests.include?(sha1)
        digests << sha1
      end
      records << entry
    end
  end
  return records
end