Class: Biopsy::Experiment

Inherits:
Object
  • Object
show all
Defined in:
lib/biopsy/experiment.rb

Instance Attribute Summary collapse

Instance Method Summary collapse

Constructor Details

#initialize(target, options: {}, threads: 4, start: nil, algorithm: nil, verbosity: :quiet) ⇒ Experiment

Returns a new Experiment



25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
 
# File 'lib/biopsy/experiment.rb', line 25

def initialize(target, options:{}, threads:4, start:nil, algorithm:nil,
               verbosity: :quiet)
  @threads = threads
  @start = start
  @algorithm = algorithm
  @verbosity = verbosity
  if target.is_a? Target
    @target = target
  else
    self.load_target target
  end
  @options = options
  @objective = ObjectiveHandler.new @target
  self.select_algorithm
  self.select_starting_point
  @scores = {}
  @iteration_count = 0
end

Instance Attribute Details

#algorithm ⇒ Object (readonly)

Returns the value of attribute algorithm.



22
23
24
 
# File 'lib/biopsy/experiment.rb', line 22

def algorithm
  @algorithm
end

#inputs ⇒ Object (readonly)

Returns the value of attribute inputs.



21
22
23
 
# File 'lib/biopsy/experiment.rb', line 21

def inputs
  @inputs
end

#outputs ⇒ Object (readonly)

Returns the value of attribute outputs.



21
22
23
 
# File 'lib/biopsy/experiment.rb', line 21

def outputs
  @outputs
end

#retain_intermediates ⇒ Object (readonly)

Returns the value of attribute retain_intermediates.



21
22
23
 
# File 'lib/biopsy/experiment.rb', line 21

def retain_intermediates
  @retain_intermediates
end

#start ⇒ Object (readonly)

Returns the value of attribute start.



22
23
24
 
# File 'lib/biopsy/experiment.rb', line 22

def start
  @start
end

#target ⇒ Object (readonly)

Returns the value of attribute target.



22
23
24
 
# File 'lib/biopsy/experiment.rb', line 22

def target
  @target
end

Instance Method Details

#cleanup ⇒ Object 



143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
 
# File 'lib/biopsy/experiment.rb', line 143

def cleanup
  # TODO: make this work
  # remove all but essential files
  essential_files = ""
  if Settings.instance.keep_intermediates
    # @objectives isn't mentioned anywhere in the rest of this file
    @objectives.values.each do |objective|
      essential_files += objective.essential_files
    end
  end
  Dir["*"].each do |file|
    next
    # TODO: implement this
    # next if File.directory? file
    # if essential_files && essential_files.include?(file)
    #   `gzip #{file}` if Settings.instance.gzip_intermediates
    #   FileUtils.mv("#{file}.gz", '../output')
    # end
  end
  FileUtils.rm_rf @last_tempdir
end

#create_tempdir ⇒ Object

create a guaranteed random temporary directory for storing outputs return the dirname



167
168
169
170
171
172
173
174
175
176
177
 
# File 'lib/biopsy/experiment.rb', line 167

def create_tempdir
  token = loop do
    # generate random dirnames until we find one that
    # doesn't exist
    test_token = SecureRandom.hex
    break test_token unless File.exist? test_token
  end
  Dir.mkdir(token)
  @last_tempdir = token
  token
end

#load_target(target_name) ⇒ Object

load the target named :target_name



72
73
74
75
 
# File 'lib/biopsy/experiment.rb', line 72

def load_target(target_name)
  @target = Target.new
  @target.load_by_name target_name
end

#print_progress(iteration, params, score, best) ⇒ Object 



134
135
136
137
138
139
140
141
 
# File 'lib/biopsy/experiment.rb', line 134

def print_progress(iteration, params, score, best)
  unless [:silent, :quiet].include? @verbosity
    ptext = params.each_pair.map{ |k, v| "#{k}:#{v}" }.join(", ")
    msg = "run #{iteration}. parameters: #{ptext} | score: #{score}"
    msg += " | best #{best[:score]}" if (best && best.has_key?(:score))
    puts msg
  end
end

#random_start_point ⇒ Object

Return a random set of parameters from the parameter space.



55
56
57
 
# File 'lib/biopsy/experiment.rb', line 55

def random_start_point
  Hash[@target.parameters.map { |p, r| [p, r.sample] }]
end

#run ⇒ Object

Runs the experiment until the completion criteria are met. On completion, returns the best parameter set.



80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
 
# File 'lib/biopsy/experiment.rb', line 80

def run
  in_progress = true
  @algorithm.setup @start
  @current_params = @start
  while in_progress
    run_iteration
    # update the best result
    best = @best
    @best = @algorithm.best
    ptext = @best[:parameters].each_pair.map{ |k, v| "#{k}:#{v}" }.join(", ")
    if @best &&
       @best.key?(:score) &&
       best &&
       best.key?(:score) &&
       @best[:score] > best[:score]
       puts "found a new best score: #{@best[:score]} \
             for parameters #{ptext}"
    end
    # have we finished?
    in_progress = !@algorithm.finished?
  end
  @algorithm.write_data if @algorithm.respond_to? :write_data
  unless @verbosity == :silent
    puts "found optimum score: #{@best[:score]} for parameters \
          #{@best[:parameters]} in #{@iteration_count} iterations."
  end
  return @best
end

#run_iteration ⇒ Object

Runs a single iteration of the optimisation, encompassing the program, objective(s) and optimiser. Returns the output of the optimiser.



112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
 
# File 'lib/biopsy/experiment.rb', line 112

def run_iteration
  # create temp dir
  Dir.chdir(self.create_tempdir) do
    # run the target
    raw_output = @target.run @current_params.merge(@options)
    # evaluate with objectives
    param_key = @current_params.to_s
    result = nil
    if @scores.key? param_key
      result = @scores[param_key]
    else
      result = @objective.run_for_output(raw_output, @threads, nil)
      @iteration_count += 1
      self.print_progress(@iteration_count, @current_params, result, @best)
    end
    @scores[@current_params.to_s] = result
    # get next steps from optimiser
    @current_params = @algorithm.run_one_iteration(@current_params, result)
  end
  self.cleanup
end

#select_algorithm ⇒ Object

select the optimisation algorithm to use



60
61
62
63
64
65
66
67
68
69
 
# File 'lib/biopsy/experiment.rb', line 60

def select_algorithm
  return if algorithm
  max = Settings.instance.sweep_cutoff
  n = @target.count_parameter_permutations
  if n < max
    @algorithm = ParameterSweeper.new(@target.parameters)
  else
    @algorithm = TabuSearch.new(@target.parameters)
  end
end

#select_starting_point ⇒ Object

return the set of parameters to evaluate first



45
46
47
48
49
50
51
52
 
# File 'lib/biopsy/experiment.rb', line 45

def select_starting_point
  return if !@start.nil?
  if @algorithm && @algorithm.knows_starting_point?
    @start = @algorithm.select_starting_point
  else
    @start = self.random_start_point
  end
end