Class: ScrapKit::Recipe
- Inherits:
-
Object
- Object
- ScrapKit::Recipe
- Defined in:
- lib/scrap_kit/recipe.rb
Instance Attribute Summary collapse
-
#browser ⇒ Object
Returns the value of attribute browser.
-
#user_agent ⇒ Object
Returns the value of attribute user_agent.
Class Method Summary collapse
Instance Method Summary collapse
- #elements_from_selector(browser_or_element, selector) ⇒ Object
- #extract_attribute(browser_or_element, selector_or_object) ⇒ Object
- #extract_value_from_element(element) ⇒ Object
- #find_element_by_name_or_selector(browser_or_element, name_or_selector) ⇒ Object
-
#initialize(url: nil, steps: [], attributes: {}) ⇒ Recipe
constructor
A new instance of Recipe.
- #run ⇒ Object
- #run_step(step) ⇒ Object
Constructor Details
permalink #initialize(url: nil, steps: [], attributes: {}) ⇒ Recipe
Returns a new instance of Recipe.
23 24 25 26 27 |
# File 'lib/scrap_kit/recipe.rb', line 23 def initialize(url: nil, steps: [], attributes: {}) @url = url @steps = steps @attributes = attributes end |
Instance Attribute Details
permalink #browser ⇒ Object
Returns the value of attribute browser.
7 8 9 |
# File 'lib/scrap_kit/recipe.rb', line 7 def browser @browser end |
permalink #user_agent ⇒ Object
Returns the value of attribute user_agent.
7 8 9 |
# File 'lib/scrap_kit/recipe.rb', line 7 def user_agent @user_agent end |
Class Method Details
permalink .load(source) ⇒ Object
[View source]
10 11 12 13 14 15 16 17 18 19 20 |
# File 'lib/scrap_kit/recipe.rb', line 10 def load(source) input = if source.is_a?(Hash) source elsif source.is_a?(IO) JSON.parse(source.read) else JSON.parse(File.read(source)) end new(input.deep_symbolize_keys) end |
Instance Method Details
permalink #elements_from_selector(browser_or_element, selector) ⇒ Object
[View source]
67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 |
# File 'lib/scrap_kit/recipe.rb', line 67 def elements_from_selector(browser_or_element, selector) if selector.is_a?(String) browser_or_element.elements(css: selector) elsif selector.is_a?(Hash) browser_or_element.elements(selector) elsif selector.is_a?(Array) *remainder, condition = selector condition_key, condition_value = condition.first elements = browser_or_element if remainder.empty? elements = elements.elements(css: condition_key.to_s) else remainder.each do |item| elements = elements.elements(css: item) end end elements.filter do |element| found_element = element.element(css: condition_key.to_s) extracted_value = extract_value_from_element(found_element) extracted_value.match(condition_value) || extracted_value == condition_value end end end |
permalink #extract_attribute(browser_or_element, selector_or_object) ⇒ Object
[View source]
107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 |
# File 'lib/scrap_kit/recipe.rb', line 107 def extract_attribute(browser_or_element, selector_or_object) if selector_or_object.is_a?(String) extract_value_from_element(browser_or_element.element(css: selector_or_object)) elsif selector_or_object.is_a?(Array) found_elements = elements_from_selector(browser_or_element, selector_or_object) if found_elements.size === 1 extract_value_from_element(found_elements.first) else found_elements.map do |element| extract_value_from_element(element) end end elsif selector_or_object.is_a?(Hash) if selector_or_object[:selector] && selector_or_object[:children_attributes] selector = selector_or_object[:selector] selector_for_children_attributes = selector_or_object[:children_attributes] elements_from_selector(browser_or_element, selector).map do |element| output = {} selector_for_children_attributes.each do |child_attribute_name, child_selector| output[child_attribute_name] = extract_attribute(element, child_selector) end output end elsif selector_or_object[:javascript] @browser.execute_script(selector_or_object[:javascript]) else found_elements = elements_from_selector(browser_or_element, selector_or_object) if found_elements.size === 1 extract_value_from_element(found_elements.first) else found_elements.map do |element| extract_value_from_element(element) end end end end rescue nil end |
permalink #extract_value_from_element(element) ⇒ Object
[View source]
93 94 95 96 97 98 99 100 101 102 103 104 105 |
# File 'lib/scrap_kit/recipe.rb', line 93 def extract_value_from_element(element) return nil unless element.exists? if element&.respond_to?(:tag_name) if element.tag_name.downcase == "input" return element.attribute_value(:value) elsif element.tag_name.downcase == "img" return element.attribute_value(:src) end end element&.text_content end |
permalink #find_element_by_name_or_selector(browser_or_element, name_or_selector) ⇒ Object
[View source]
57 58 59 60 61 62 63 64 65 |
# File 'lib/scrap_kit/recipe.rb', line 57 def find_element_by_name_or_selector(browser_or_element, name_or_selector) element = browser_or_element.element(name: name_or_selector.to_s) return element if element.exists? element = browser_or_element.element(css: name_or_selector.to_s) return element if element.exists? nil end |
permalink #run ⇒ Object
[View source]
29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 |
# File 'lib/scrap_kit/recipe.rb', line 29 def run output = {} @browser = create_browser @browser.goto @url @steps.each do |step| run_step(step) end @attributes.each do |attribute_name, selector| output[attribute_name] = extract_attribute(@browser, selector) end @browser.close @browser = nil output end |
permalink #run_step(step) ⇒ Object
[View source]
49 50 51 52 53 54 55 |
# File 'lib/scrap_kit/recipe.rb', line 49 def run_step(step) return goto(step[:goto]) if step[:goto] return click(step[:click]) if step[:click] return fill_form(step[:fill_form]) if step[:fill_form] nil end |