Module: Datacraft::Runner

Included in:
Datacraft
Defined in:
lib/datacraft/runner.rb

Overview

The runner for the whole process

Instance Method Summary collapse

Instance Method Details

#build(consumers) ⇒ Object

build and close consumers



80
81
82
83
84
85
 
# File 'lib/datacraft/runner.rb', line 80

def build(consumers)
  consumers.each do |consumer|
    consumer.build if consumer.respond_to? :build
    consumer.close if consumer.respond_to? :close
  end
end

#pprocess_rows ⇒ Object

process rows in parallel



57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
 
# File 'lib/datacraft/runner.rb', line 57

def pprocess_rows
  thread_number = [@context.providers.size,
                   @context.options[:n_threads]].min
  queue = Queue.new
  @context.providers.each { |p| queue << p }
  threads = thread_number.times.map do
    Thread.new do
      begin
        while p = queue.pop(true)
          p.each { |row| process row }
        end
      rescue ThreadError
      end
      # until queue.empty?
      #   p = queue.pop(true)
      #   p.each { |row| process row }
      # end
    end
  end
  threads.each(&:join)
end

#process(row) ⇒ Object

tweak & consume one row



46
47
48
49
50
51
52
53
54
 
# File 'lib/datacraft/runner.rb', line 46

def process(row)
  @context.tweakers.each do |tweaker|
    row = tweaker.tweak row
    return nil unless row
  end
  @context.consumers.each do |consumer|
    consumer << row
  end
end

#process_rows ⇒ Object

process rows sequentially



37
38
39
40
41
42
43
 
# File 'lib/datacraft/runner.rb', line 37

def process_rows
  @context.providers.each do |provider|
    provider.each do |row|
      process row
    end
  end
end

#report(measurements) ⇒ Object

output benchmark results



28
29
30
31
32
33
34
 
# File 'lib/datacraft/runner.rb', line 28

def report(measurements)
  width = measurements.max_by { |m| m.label.size }.label.size + 1
  puts "#{' ' * width}#{Benchmark::CAPTION}"
  measurements.each do |m|
    puts "#{m.label.to_s.ljust width} #{m}"
  end
end

#run(instruction) ⇒ Object

run the instruction



7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
 
# File 'lib/datacraft/runner.rb', line 7

def run(instruction)
  @context = instruction.context
  measurements = []
  measurements << Benchmark.measure('pre build:') do
    @context.pre_hooks.each(&:call)
  end
  measurements << Benchmark.measure('process rows:') do
    @context.options[:parallel] ? pprocess_rows : process_rows
  end

  measurements << Benchmark.measure('build:') do
    build @context.consumers
  end

  measurements << Benchmark.measure('post build:') do
    @context.post_hooks.each(&:call)
  end
  report measurements if @context.options[:benchmark]
end