Class: Mandy::Job

Inherits:
Object
  • Object
show all
Defined in:
lib/job.rb

Instance Attribute Summary collapse

Class Method Summary collapse

Instance Method Summary collapse

Constructor Details

#initialize(name, &blk) ⇒ Job

Returns a new instance of Job.



16
17
18
19
20
21
22
23
24
 
# File 'lib/job.rb', line 16

def initialize(name, &blk)
  @name = name
  @settings = {}
  @modules = []
  @mapper_class = Mandy::Mappers::PassThroughMapper
  @reducer_class = Mandy::Reducers::PassThroughReducer
  set('mapred.job.name', name)
  instance_eval(&blk) if blk
end

Instance Attribute Details

#name ⇒ Object (readonly)

Returns the value of attribute name.



14
15
16
 
# File 'lib/job.rb', line 14

def name
  @name
end

#settings ⇒ Object (readonly)

Returns the value of attribute settings.



13
14
15
 
# File 'lib/job.rb', line 13

def settings
  @settings
end

Class Method Details

.find_by_name(name) ⇒ Object 



8
9
10
 
# File 'lib/job.rb', line 8

def find_by_name(name)
  jobs.find {|job| job.name == name }
end

.jobs ⇒ Object 



4
5
6
 
# File 'lib/job.rb', line 4

def jobs
  @jobs ||= []
end

Instance Method Details

#input_format(format) ⇒ Object 



31
32
33
 
# File 'lib/job.rb', line 31

def input_format(format)
  @input_format = format
end

#map(klass = nil, &blk) ⇒ Object 



60
61
62
63
64
 
# File 'lib/job.rb', line 60

def map(klass=nil, &blk)
  @mapper_class = klass || Mandy::Mappers::Base.compile(&blk)
  @modules.each {|m| @mapper_class.send(:include, m) }
  @mapper_class
end

#map_tasks(count) ⇒ Object 



43
44
45
 
# File 'lib/job.rb', line 43

def map_tasks(count)
  set('mapred.map.tasks', count)
end

#mixin(*modules) ⇒ Object Also known as: serialize 



26
27
28
 
# File 'lib/job.rb', line 26

def mixin(*modules)
  modules.each {|m| @modules << m}
end

#output_format(format) ⇒ Object 



35
36
37
 
# File 'lib/job.rb', line 35

def output_format(format)
  @output_format = format
end

#reduce(klass = nil, &blk) ⇒ Object 



66
67
68
69
70
 
# File 'lib/job.rb', line 66

def reduce(klass=nil, &blk)
  @reducer_class = klass || Mandy::Reducers::Base.compile(&blk)
  @modules.each {|m| @reducer_class.send(:include, m) }
  @reducer_class
end

#reduce_tasks(count) ⇒ Object 



47
48
49
 
# File 'lib/job.rb', line 47

def reduce_tasks(count)
  set('mapred.reduce.tasks', count)
end

#run_map(input = STDIN, output = STDOUT) {|mapper| ... } ⇒ Object

Yields:

  • (mapper) 


72
73
74
75
76
77
 
# File 'lib/job.rb', line 72

def run_map(input=STDIN, output=STDOUT, &blk)
  @mapper_class.send(:include, Mandy::IO::OutputFormatting) unless reducer_defined?
  mapper = @mapper_class.new(input, output, @input_format, @output_format)
  yield(mapper) if blk
  mapper.execute
end

#run_reduce(input = STDIN, output = STDOUT) {|reducer| ... } ⇒ Object

Yields:

  • (reducer) 


79
80
81
82
83
 
# File 'lib/job.rb', line 79

def run_reduce(input=STDIN, output=STDOUT, &blk)
  reducer = @reducer_class.new(input, output, @input_format, @output_format)
  yield(reducer) if blk
  reducer.execute
end

#set(key, value) ⇒ Object 



39
40
41
 
# File 'lib/job.rb', line 39

def set(key, value)
  @settings[key.to_s] = value.to_s
end

#store(type, name, options = {}) ⇒ Object 



51
52
53
54
55
56
57
58
 
# File 'lib/job.rb', line 51

def store(type, name, options={})
  Mandy.stores[name] = case type
  when :hbase
    Stores::HBase.new(options)
  else
    raise "Unknown store type #{type}"
  end
end