Class: DataDuck::SEMRush::OrganicResults

Inherits:
IntegrationTable show all
Defined in:
lib/integrations/semrush/organic_results.rb

Instance Attribute Summary

Attributes inherited from Table

#data, #errors

Instance Method Summary collapse

Methods inherited from Table

#actions, #autogenerate_identity?, #batch_size, #building_name, #check_table_valid!, #create_column_names, #create_schema, #distribution_key, #distribution_style, #etl!, #extract_by_clause, #extract_by_column, #extract_query, #include_with_all?, #limit_clause, #load!, #name, output, #output_column_names, #output_schema, #postprocess!, #recreate!, #should_fully_reload?, #show, source, #staging_name, #transform!, transforms, validates

Instance Method Details

#display_limit ⇒ Object 



10
11
12
 
# File 'lib/integrations/semrush/organic_results.rb', line 10

def display_limit
  20
end

#extract!(destination, options = {}) ⇒ Object 



30
31
32
33
34
35
36
37
38
39
40
 
# File 'lib/integrations/semrush/organic_results.rb', line 30

def extract!(destination, options = {})
  self.data = []

  self.phrases.each do |phrase|
    begin
      self.extract_results_for_keyword_and_date!(phrase)
    rescue OrganicResultsAPIError => err
      DataDuck::Logs.error(err)
    end
  end
end

#extract_results_for_keyword_and_date!(phrase) ⇒ Object 



42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
 
# File 'lib/integrations/semrush/organic_results.rb', line 42

def extract_results_for_keyword_and_date!(phrase)
  date = Date.today
  phrase.strip!
  escaped_phrase = URI.escape(phrase)
  semrush_api_url = "http://api.semrush.com/?type=phrase_organic&key=#{ self.key }&display_limit=#{ self.display_limit }&export_columns=Dn,Ur&phrase=#{ escaped_phrase }&database=#{ self.search_database }"

  response = Typhoeus.get(semrush_api_url)
  if response.response_code != 200
    raise OrganicResultsAPIError.new("SEMrush API for phrase #{ phrase } returned error #{ response.response_code } #{ response.body }")
  end

  if response.body.start_with?("ERROR ")
    raise OrganicResultsAPIError.new("SEMrush API for phrase #{ phrase } returned 200 but with a body stating #{ response.body }")
  end

  rank = -1
  response.body.each_line do |line|
    rank += 1
    if rank == 0
      # This is the header line
      next
    end

    domain, url = line.split(';')
    domain.strip!
    url.strip!

    self.data << {
        date: date,
        phrase: phrase,
        rank: rank,
        domain: domain,
        url: url
    }
  end
end

#identify_by_columns ⇒ Object 



79
80
81
 
# File 'lib/integrations/semrush/organic_results.rb', line 79

def identify_by_columns
  ["date", "phrase"]
end

#indexes ⇒ Object 



83
84
85
 
# File 'lib/integrations/semrush/organic_results.rb', line 83

def indexes
  ["date", "phrase", "domain"]
end

#key ⇒ Object 



14
15
16
 
# File 'lib/integrations/semrush/organic_results.rb', line 14

def key
  ENV['semrush_api_key']
end

#phrases ⇒ Object 



18
19
20
 
# File 'lib/integrations/semrush/organic_results.rb', line 18

def phrases
  raise "Must implement phrases method to be an array of the phrases you want."
end

#prefix ⇒ Object 



22
23
24
 
# File 'lib/integrations/semrush/organic_results.rb', line 22

def prefix
  "semrush_"
end

#search_database ⇒ Object 



26
27
28
 
# File 'lib/integrations/semrush/organic_results.rb', line 26

def search_database
  'us'
end