Class: DataListConverter

Inherits:
Object
  • Object
show all
Defined in:
lib/data_list_converter/base.rb,
lib/data_list_converter/helper.rb,
lib/data_list_converter/version.rb,
lib/data_list_converter/types/basic.rb,
lib/data_list_converter/filters/count.rb,
lib/data_list_converter/filters/limit.rb,
lib/data_list_converter/types/marshal.rb,
lib/data_list_converter/types/records.rb,
lib/data_list_converter/types/csv_file.rb,
lib/data_list_converter/types/xls_file.rb,
lib/data_list_converter/types/xlsx_file.rb,
lib/data_list_converter/types/multi_sheet.rb,
lib/data_list_converter/filters/remove_debug.rb,
lib/data_list_converter/types/fast_xlsx_file.rb

Overview

usage: require ‘data_list_converter/types/fast_xlsx_file’ d = [12.3, b: ‘xx’] DataListConverter.save_to_file(“t.xls”, d, :item_data, file_format: :fast_xlsx_file)

Constant Summary collapse

CONVERTERS = 
{}
FILTERS = 
{}
VERSION = 
"0.6.0".freeze
MULT_SHEET_CONVERTS = 
[[:table_data, :table_iterator],
 [:item_data, :item_iterator],
 [:table_iterator, :item_iterator],
]

Class Attribute Summary collapse

Class Method Summary collapse

Class Attribute Details

.debug ⇒ Object

Returns the value of attribute debug.



10
11
12
 
# File 'lib/data_list_converter/base.rb', line 10

def debug
  @debug
end

Class Method Details

.convert(from_type, to_type, from_value, options = {}) ⇒ Object

Example: convert(:item_iterator, :item_data, iter) convert(:item_iterator, :csv_file, iter, csv_file: ‘result.csv’) convert(:csv_file, :item_data, ‘result.csv’)

can add filter: filter = :limit filter = {size: 2} filter = [{size: 12}, {size: 4}] convert(:item_iterator, :table_data, iter, table_iterator: filter)



45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
 
# File 'lib/data_list_converter/base.rb', line 45

def convert(from_type, to_type, from_value, options={})
  methods = []
  add_filter = lambda { |type|
    filters = (options[type] || {})[:filter]
    return unless filters
    methods += normalize_filters(type, filters)
  }

  route = find_route(from_type, to_type)
  add_filter.call(route[0])

  self.log("route: #{route}")
  (0..(route.length-2)).map do |i|
    from_type, to_type = route[i], route[i+1]
    method = CONVERTERS[[from_type, to_type]]
    raise "cannot find converter #{from_type} -> #{to_type}" unless method
    self.log "#{from_type} -> #{to_type} options: #{options[to_type]}"
    methods.push([method, options[to_type] || {}])
    add_filter.call(to_type)
  end

  self.log("methods: #{methods}")
  methods.inject(from_value) do |v, method|
    method, args = method
    method.call(v, args)
  end
end

.data_to_iterator(data, options = {}) ⇒ Object 



40
41
42
43
44
45
46
 
# File 'lib/data_list_converter/types/basic.rb', line 40

def self.data_to_iterator(data, options={})
  lambda { |&block|
    data.each do |d|
      block.call(d)
    end
  }
end

.file_types ⇒ Object 



30
31
32
33
34
35
 
# File 'lib/data_list_converter/helper.rb', line 30

def self.file_types
  matcher = /(.*)_file$/
  DataListConverter.types.select do |type|
    matcher.match(type)
  end
end

.find_route(from_type, to_type) ⇒ Object

One type of data can be converted into any other types, we have a list of convert methods: CONVERTERS If we want to convert between types, like: convert item_data into csv_file, we need find all the intermidate data type, like: [:item_data, :item_iterator, :table_iterator, :csv_file]

Raises:

  • (Exception) 


95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
 
# File 'lib/data_list_converter/base.rb', line 95

def find_route(from_type, to_type)
  [from_type, to_type].each do |type|
    raise Exception, "cannot find type: #{type}" unless self.types.include?(type)
  end
  raise Exception, "from_type should not equal to to_type: #{from_type}" if from_type == to_type

  # map wide search
  checked = Set.new
  checking = Set.new([from_type])
  directions = {}

  while not checking.empty?
    current_node = checking.first

    next_nodes = route_map[current_node]
    # mark direction from from_type
    next_nodes.each do |node|
      # first marked is the shortest
      directions[node] ||= current_node
    end

    if next_nodes.include?(to_type)
      # get route
      start = to_type
      route = [start]
      while start != from_type
        previous = directions[start]
        raise "cannot find previous for #{start} in #{directions}" if not previous
        route.push(previous)
        start = previous
      end
      return route.reverse
    else
      checking.delete(current_node)
      checked.add(current_node)
      checking += Set.new(next_nodes) - checked
    end
  end

  log = ["Route not found: #{from_type} -> #{to_type}", "Current routes:"]
  log += self.routes.map{|from, to| "#{from} -> #{to}"}
  raise Exception, log.join("\n")
end

.flatten(data, sep = ':', max_level = nil) ⇒ Object

flatten multi level item data into one level, example:

{a: {b: 12}, c: {d: {e: 11}}}
=>
{:"a:b"=>12, :"c:d:e"=>11}



58
59
60
61
62
 
# File 'lib/data_list_converter/helper.rb', line 58

def self.flatten(data, sep=':', max_level=nil)
  out = {}
  recursive_flatten(out, data, nil, sep, 1, max_level)
  out
end

.get_file_format(filename) ⇒ Object 



14
15
16
17
18
19
20
 
# File 'lib/data_list_converter/helper.rb', line 14

def self.get_file_format(filename)
  file_type = (File.extname(filename)[1..-1] + "_file").to_sym
  unless DataListConverter.file_types.include?(file_type)
    raise "unknown file format: #{file_type}"
  end
  file_type
end

.iterator_limit(proc, options) ⇒ Object 



2
3
4
5
6
7
8
9
10
11
12
 
# File 'lib/data_list_converter/filters/limit.rb', line 2

def self.iterator_limit(proc, options)
  limit_size = options[:size] || 10
  lambda { |&block|
    limit = 0
    proc.call do |item|
      block.call(item)
      limit += 1
      break if limit >= limit_size
    end
  }
end

.iterator_to_data(proc, options = {}) ⇒ Object 



32
33
34
35
36
 
# File 'lib/data_list_converter/types/basic.rb', line 32

def self.iterator_to_data(proc, options={})
  out = []
  proc.call { |d| out << d }
  out
end

.load_from_file(filename, data_format = :item_data, options = {}) ⇒ Object 



8
9
10
11
12
 
# File 'lib/data_list_converter/helper.rb', line 8

def self.load_from_file(filename, data_format=:item_data, options={})
  file_format = options.delete(:file_format) || self.get_file_format(filename)
  options[:filename] = filename
  DataListConverter.convert(file_format, data_format, options)
end

.log(msg) ⇒ Object 



19
20
21
22
 
# File 'lib/data_list_converter/base.rb', line 19

def log(msg)
  return unless debug
  puts "#{Time.now.strftime('%Y-%m-%d %H:%M:%S')}\t#{msg}"
end

.marshal_data_to_file(data, options) ⇒ Object 



10
11
12
13
14
15
16
 
# File 'lib/data_list_converter/types/marshal.rb', line 10

def self.marshal_data_to_file(data, options)
  filename = self.parameter(options, :filename, :marshal)
  File.open(filename, 'w+') do |f|
    Marshal.dump(data, f)
  end
  options[:filename]
end

.marshal_file_to_data(input, options = nil) ⇒ Object 



3
4
5
6
7
8
 
# File 'lib/data_list_converter/types/marshal.rb', line 3

def self.marshal_file_to_data(input, options=nil)
  filename = self.parameter(input, :filename, :input)
  File.open(filename) do |f|
    Marshal.load(f)
  end
end

.normalize_filters(type, filters) ⇒ Object 



73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
 
# File 'lib/data_list_converter/base.rb', line 73

def normalize_filters(type, filters)
  # filter list as array
  filters = [filters] unless filters.kind_of?(Array)
  filters.map do |v|
    # fix filter arguments
    case v
    # {:limit, {count: 12}} => [:limit, {count: 12}]
    when Hash; v.first
    # :debug => [:debug, {}]
    when Symbol, String; [v, {}]
    else; v
    end
  end.map do |name, args|
    method = FILTERS[type][name] rescue raise("cannot find method for type #{type} filter #{name}")
    [method, args]
  end
end

.on_debug ⇒ Object 



12
13
14
15
16
17
 
# File 'lib/data_list_converter/base.rb', line 12

def on_debug
  self.debug = true
  yield
ensure
  self.debug = false
end

.parameter(data, key, type) ⇒ Object

Raises:

  • (Exception) 


75
76
77
78
79
 
# File 'lib/data_list_converter/helper.rb', line 75

def self.parameter(data, key, type)
  raise Exception, "`#{type}` should be hash, not `#{data.class}`: #{data}" unless data.kind_of?(Hash)
  raise Exception, "Need `#{key}` for `#{type}`, current: #{data}" if not data.has_key?(key)
  data.fetch(key)
end

.recursive_flatten(out, data, header, sep, level, max_level) ⇒ Object 



64
65
66
67
68
69
70
71
72
73
 
# File 'lib/data_list_converter/helper.rb', line 64

def self.recursive_flatten(out, data, header, sep, level, max_level)
  data.each do |k, v|
    k = header ? :"#{header}#{sep}#{k}" : k
    if v.kind_of?(Hash) and (!max_level or level <= max_level)
      recursive_flatten(out, v, k, sep, level+1, max_level)
    else
      out[k] = v
    end
  end
end

.register_converter(from_type, to_type, &block) ⇒ Object

register_converter(:item_data, :item_iterator){ |data, options| … }



25
26
27
28
 
# File 'lib/data_list_converter/base.rb', line 25

def register_converter(from_type, to_type, &block)
  @route_map = nil # clear cache
  CONVERTERS[[from_type, to_type]] = block
end

.register_filter(type, name, &block) ⇒ Object 



30
31
32
33
 
# File 'lib/data_list_converter/base.rb', line 30

def register_filter(type, name, &block)
  FILTERS[type] ||= {}
  FILTERS[type][name] = block
end

.route_map ⇒ Object

convert adjacency list into quick lookup hash



140
141
142
143
144
145
146
147
148
149
150
 
# File 'lib/data_list_converter/base.rb', line 140

def route_map
  @route_map ||= \
  begin
    CONVERTERS.keys.
      inject({}) do |map, item|
      map[item.first] ||= []
      map[item.first] += [item[1]]
      map
    end
  end
end

.routes ⇒ Object 



26
27
28
 
# File 'lib/data_list_converter/helper.rb', line 26

def self.routes
  CONVERTERS.keys
end

.save_to_file(filename, data, data_format = :item_data, options = {}) ⇒ Object 



2
3
4
5
6
 
# File 'lib/data_list_converter/helper.rb', line 2

def self.save_to_file(filename, data, data_format=:item_data, options={})
  file_format = options.delete(:file_format) || self.get_file_format(filename)
  options[file_format] = {filename: filename}
  DataListConverter.convert(data_format, file_format, data, options)
end

.types ⇒ Object 



22
23
24
 
# File 'lib/data_list_converter/helper.rb', line 22

def self.types
  CONVERTERS.keys.flatten.uniq.sort
end

.unify_item_data_keys(list) ⇒ Object

sometimes item data keys don’t exactly same, like:

[{a: 12}, {b: 11}]

should update to:

[{a: 12, b: nil}, {a: nil, b: 11}]

so it can be convent to table data



42
43
44
45
46
47
48
49
50
51
52
 
# File 'lib/data_list_converter/helper.rb', line 42

def self.unify_item_data_keys(list)
  keys = Set.new
  list.each do |item|
    keys += item.keys
  end
  list.each do |item|
    keys.each do |key|
      item[key] ||= nil
    end
  end
end