Class: NtqExcelsior::Importer

Inherits:
Object
  • Object
show all
Defined in:
lib/ntq_excelsior/importer.rb

Instance Attribute Summary collapse

Class Method Summary collapse

Instance Method Summary collapse

Constructor Details

#initialize ⇒ Importer

Returns a new instance of Importer.



57
58
59
 
# File 'lib/ntq_excelsior/importer.rb', line 57

def initialize
  @context = NtqExcelsior::Context.new
end

Instance Attribute Details

#check ⇒ Object

Returns the value of attribute check.



6
7
8
 
# File 'lib/ntq_excelsior/importer.rb', line 6

def check
  @check
end

#context ⇒ Object

Returns the value of attribute context.



7
8
9
 
# File 'lib/ntq_excelsior/importer.rb', line 7

def context
  @context
end

#file ⇒ Object

Returns the value of attribute file.



6
7
8
 
# File 'lib/ntq_excelsior/importer.rb', line 6

def file
  @file
end

#lines ⇒ Object

Returns the value of attribute lines.



6
7
8
 
# File 'lib/ntq_excelsior/importer.rb', line 6

def lines
  @lines
end

#options ⇒ Object

Returns the value of attribute options.



6
7
8
 
# File 'lib/ntq_excelsior/importer.rb', line 6

def options
  @options
end

#status_tracker ⇒ Object

Returns the value of attribute status_tracker.



6
7
8
 
# File 'lib/ntq_excelsior/importer.rb', line 6

def status_tracker
  @status_tracker
end

#success ⇒ Object

Returns the value of attribute success.



6
7
8
 
# File 'lib/ntq_excelsior/importer.rb', line 6

def success
  @success
end

Class Method Details

.after(&block) ⇒ Object 



39
40
41
42
 
# File 'lib/ntq_excelsior/importer.rb', line 39

def after(&block)
  @after = block if block_given?
  @after
end

.autosave(value = nil) ⇒ Object 



10
11
12
 
# File 'lib/ntq_excelsior/importer.rb', line 10

def autosave(value = nil)
  @autosave ||= value
end

.autoset(value = nil) ⇒ Object 



14
15
16
 
# File 'lib/ntq_excelsior/importer.rb', line 14

def autoset(value = nil)
  @autoset ||= value
end

.before(&block) ⇒ Object 



34
35
36
37
 
# File 'lib/ntq_excelsior/importer.rb', line 34

def before(&block)
  @before = block if block_given?
  @before
end

.max_error_count(value = nil) ⇒ Object 



44
45
46
 
# File 'lib/ntq_excelsior/importer.rb', line 44

def max_error_count(value = nil)
  @max_error_count ||= value
end

.model_klass(value = nil) ⇒ Object 



26
27
28
 
# File 'lib/ntq_excelsior/importer.rb', line 26

def model_klass(value = nil)
  @model_klass ||= value
end

.primary_key(value = nil) ⇒ Object 



22
23
24
 
# File 'lib/ntq_excelsior/importer.rb', line 22

def primary_key(value = nil)
  @primary_key ||= value
end

.sample_file(value = nil) ⇒ Object 



52
53
54
 
# File 'lib/ntq_excelsior/importer.rb', line 52

def sample_file(value = nil)
  @sample_file ||= value
end

.schema(value = nil) ⇒ Object 



30
31
32
 
# File 'lib/ntq_excelsior/importer.rb', line 30

def schema(value = nil)
  @schema ||= value
end

.spreadsheet_options(value = nil) ⇒ Object 



18
19
20
 
# File 'lib/ntq_excelsior/importer.rb', line 18

def spreadsheet_options(value = nil)
  @spreadsheet_options ||= value
end

.structure(value = nil) ⇒ Object 



48
49
50
 
# File 'lib/ntq_excelsior/importer.rb', line 48

def structure(value = nil)
  @structure ||= value
end

Instance Method Details

#detect_header_scheme ⇒ Object 



110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
 
# File 'lib/ntq_excelsior/importer.rb', line 110

def detect_header_scheme
  return @header_scheme if @header_scheme

  @header_scheme = {}
  # Read the first line of file (not header)
  l = spreadsheet_data[0].dup || []

  self.class.schema.each do |field, column_config|
    header = column_config.is_a?(Hash) ? column_config[:header] : column_config

    l.each do |parsed_header, _value|
      next unless parsed_header
      next unless (header.is_a?(Regexp) && parsed_header && parsed_header.match?(header)) || header.is_a?(String) && parsed_header == header

      l.delete(parsed_header)
      @header_scheme[parsed_header] = field
    end
  end
  @header_scheme[self.class.primary_key.to_s] = self.class.primary_key.to_s if self.class.primary_key && !self.class.schema[self.class.primary_key.to_sym]

  @header_scheme
end

#find_or_initialize_record(line) ⇒ Object

id for default query in model line in case an override is needed to find correct record



158
159
160
161
162
163
164
165
166
167
168
169
170
 
# File 'lib/ntq_excelsior/importer.rb', line 158

def find_or_initialize_record(line)
  return nil unless self.class.primary_key && self.class.model_klass

  if line[self.class.primary_key.to_sym].present?
    if self.class.primary_key.to_sym == :id
      record = self.class.model_klass.constantize.find_by id: line[self.class.primary_key.to_sym]
    else
      record = self.class.model_klass.constantize.find_or_initialize_by("#{self.class.primary_key}": line[self.class.primary_key.to_sym])
    end
  end
  record = self.class.model_klass.constantize.new unless record
  record
end

#import(save: true, status_tracker: nil) ⇒ Object 



215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
 
# File 'lib/ntq_excelsior/importer.rb', line 215

def import(save: true, status_tracker: nil)
  self.class.before.call(@context, options) if self.class.before.is_a?(Proc)
  at = 0
  errors_lines = []
  success_count = 0
  not_found_count = 0
  lines.each_with_index do |line, index|
    break if errors_lines.size == self.class.max_error_count

    result = import_line(line.with_indifferent_access, save: true)
    case result[:status]
    when :not_found
      not_found_count += 1
    when :success
      success_count += 1
    when :error
      error_line = line.map { |k, v| v }
      error_line << result[:errors]
      errors_lines.push(error_line) 
    end

    if @status_tracker&.is_a?(Proc)
      at = (((index + 1).to_d / lines.size) * 100.to_d) 
      @status_tracker.call(at)
    end
  end

  import_stats = { success_count: success_count, not_found_count: not_found_count, errors: errors_lines }
  @context.success = true if errors_lines.empty?
  self.class.after.call(@context, options) if self.class.after.is_a?(Proc)
  import_stats
end

#import_line(line, save: true) {|record, line| ... } ⇒ Object

Yields:

  • (record, line) 


186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
 
# File 'lib/ntq_excelsior/importer.rb', line 186

def import_line(line, save: true)
  record = find_or_initialize_record(line)
  return { status: :not_found } unless record

  @success = false
  @action = nil
  @errors = []

  if (self.class.autoset)
    record = set_record_fields(record, line)
  end

  yield(record, line) if block_given?

  if (self.class.autosave.nil? || self.class.autosave)
    @action = record.persisted? ? 'update' : 'create'
    if save
      @success = record.save
    else
      @success = record.valid?
    end
    @errors = record.errors.full_messages.concat(@errors) if record.errors.any?
  end

  return { status: :success, action: @action } if @success

  return { status: :error, errors: @errors.join(", ") }
end

#parse_line(line) ⇒ Object 



133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
 
# File 'lib/ntq_excelsior/importer.rb', line 133

def parse_line(line)
  parsed_line = {}
  line.each do |header, value|
    header_scheme = detect_header_scheme
    if header.to_s == self.class.primary_key.to_s
      parsed_line[self.class.primary_key] = value
      next
    end

    header_scheme.each do |header, field|
      parsed_line[field.to_sym] = line[header]
    end
  end

  parsed_line
end

#record_attributes(record) ⇒ Object 



172
173
174
175
176
 
# File 'lib/ntq_excelsior/importer.rb', line 172

def record_attributes(record)
  return @record_attributes if @record_attributes

  @record_attributes = self.class.schema.keys.select { |k| k.to_sym != :id && record.respond_to?(:"#{k}=") }
end

#required_headers ⇒ Object 



69
70
71
72
73
74
75
76
77
78
 
# File 'lib/ntq_excelsior/importer.rb', line 69

def required_headers
  return @required_headers if @required_headers

  @required_columns = self.class.schema.select { |_field, column_config| !column_config.is_a?(Hash) || !column_config.key?(:required) || column_config[:required] }
  @required_headers = @required_columns.values.map { |column| get_column_header(column) }.map { |header| transform_header_to_regexp(header) }
  if self.class.primary_key && !@required_columns.keys.include?(self.class.primary_key)
    @required_headers.unshift(Regexp.new(self.class.primary_key.to_s, "i"))
  end
  @required_headers
end

#set_record_fields(record, line) ⇒ Object 



178
179
180
181
182
183
184
 
# File 'lib/ntq_excelsior/importer.rb', line 178

def set_record_fields(record, line)
  attributes_to_set = record_attributes(record)
  attributes_to_set.each do |attribute|
    record.send(:"#{attribute}=", line[attribute])
  end
  record
end

#spreadsheet ⇒ Object 



61
62
63
64
65
66
67
 
# File 'lib/ntq_excelsior/importer.rb', line 61

def spreadsheet
  return @spreadsheet unless @spreadsheet.nil?

  raise 'File is missing' unless file.present?

  @spreadsheet = Roo::Spreadsheet.open(file, self.class.spreadsheet_options || {})
end

#spreadsheet_data ⇒ Object 



80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
 
# File 'lib/ntq_excelsior/importer.rb', line 80

def spreadsheet_data
  begin
    spreadsheet_data = spreadsheet.sheet(spreadsheet.sheets[0]).parse(header_search: required_headers)
    raise 'File is inconsistent, please check you have data in it or check for invalid characters in headers like , / ; etc...' unless spreadsheet_data.size > 0

    spreadsheet_data
  rescue Roo::HeaderRowNotFoundError => e
    missing_headers = []

    e.message.delete_prefix('[').delete_suffix(']').split(",").map(&:strip).each do |header_missing|
      header_missing_regex = transform_header_to_regexp(header_missing, true)
      header_found = @required_columns.values.find do |column|
        transform_header_to_regexp(get_column_header(column)) == header_missing_regex
      end
      if header_found && header_found.is_a?(Hash)
        if header_found[:header].is_a?(String)
          missing_headers << header_found[:header]
        else
          missing_headers << (header_found[:humanized_header] || header_missing)
        end
      elsif header_found&.is_a?(String)
        missing_headers << header_found
      else
        missing_headers << header_missing
      end
    end
    raise Roo::HeaderRowNotFoundError, missing_headers.join(", ")
  end
end