Class: Biopsy::Experiment

Inherits:
Object
  • Object
show all
Defined in:
lib/biopsy/experiment.rb

Instance Attribute Summary collapse

Instance Method Summary collapse

Constructor Details

#initialize(target, options: {}, threads: 4, start: nil, algorithm: nil, timelimit: nil, verbosity: :quiet) ⇒ Experiment

Returns a new Experiment



25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
 
# File 'lib/biopsy/experiment.rb', line 25

def initialize(target, options:{}, threads:4, start:nil, algorithm:nil,
               timelimit:nil, verbosity: :quiet)
  @threads = threads
  @start = start
  @algorithm = algorithm
  @timelimit = timelimit
  @verbosity = verbosity
  if target.is_a? Target
    @target = target
  else
    self.load_target target
  end
  @options = options
  @objective = ObjectiveHandler.new @target
  self.select_algorithm
  self.select_starting_point
  @scores = {}
  @iteration_count = 0
end

Instance Attribute Details

#algorithm ⇒ Object (readonly)

Returns the value of attribute algorithm.



22
23
24
 
# File 'lib/biopsy/experiment.rb', line 22

def algorithm
  @algorithm
end

#inputs ⇒ Object (readonly)

Returns the value of attribute inputs.



21
22
23
 
# File 'lib/biopsy/experiment.rb', line 21

def inputs
  @inputs
end

#outputs ⇒ Object (readonly)

Returns the value of attribute outputs.



21
22
23
 
# File 'lib/biopsy/experiment.rb', line 21

def outputs
  @outputs
end

#retain_intermediates ⇒ Object (readonly)

Returns the value of attribute retain_intermediates.



21
22
23
 
# File 'lib/biopsy/experiment.rb', line 21

def retain_intermediates
  @retain_intermediates
end

#start ⇒ Object (readonly)

Returns the value of attribute start.



22
23
24
 
# File 'lib/biopsy/experiment.rb', line 22

def start
  @start
end

#target ⇒ Object (readonly)

Returns the value of attribute target.



22
23
24
 
# File 'lib/biopsy/experiment.rb', line 22

def target
  @target
end

Instance Method Details

#cleanup ⇒ Object 



150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
 
# File 'lib/biopsy/experiment.rb', line 150

def cleanup
  # TODO: make this work
  # remove all but essential files
  essential_files = ""
  if Settings.instance.keep_intermediates
    # @objectives isn't mentioned anywhere in the rest of this file
    @objectives.values.each do |objective|
      essential_files += objective.essential_files
    end
  end
  Dir["*"].each do |file|
    next
    # TODO: implement this
    # next if File.directory? file
    # if essential_files && essential_files.include?(file)
    #   `gzip #{file}` if Settings.instance.gzip_intermediates
    #   FileUtils.mv("#{file}.gz", '../output')
    # end
  end
  FileUtils.rm_rf @last_tempdir
end

#create_tempdir ⇒ Object

create a guaranteed random temporary directory for storing outputs return the dirname



174
175
176
177
178
179
180
181
182
183
184
 
# File 'lib/biopsy/experiment.rb', line 174

def create_tempdir
  token = loop do
    # generate random dirnames until we find one that
    # doesn't exist
    test_token = SecureRandom.hex
    break test_token unless File.exist? test_token
  end
  Dir.mkdir(token)
  @last_tempdir = token
  token
end

#load_target(target_name) ⇒ Object

load the target named :target_name



73
74
75
76
 
# File 'lib/biopsy/experiment.rb', line 73

def load_target(target_name)
  @target = Target.new
  @target.load_by_name target_name
end

#print_progress(iteration, params, score, best) ⇒ Object 



141
142
143
144
145
146
147
148
 
# File 'lib/biopsy/experiment.rb', line 141

def print_progress(iteration, params, score, best)
  unless [:silent, :quiet].include? @verbosity
    ptext = params.each_pair.map{ |k, v| "#{k}:#{v}" }.join(", ")
    msg = "run #{iteration}. parameters: #{ptext} | score: #{score}"
    msg += " | best #{best[:score]}" if (best && best.has_key?(:score))
    puts msg
  end
end

#random_start_point ⇒ Object

Return a random set of parameters from the parameter space.



56
57
58
 
# File 'lib/biopsy/experiment.rb', line 56

def random_start_point
  Hash[@target.parameters.map { |p, r| [p, r.sample] }]
end

#run ⇒ Object

Runs the experiment until the completion criteria are met. On completion, returns the best parameter set.



81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
 
# File 'lib/biopsy/experiment.rb', line 81

def run
  start_time = Time.now
  in_progress = true
  @algorithm.setup @start
  @current_params = @start
  while in_progress
    run_iteration
    # update the best result
    best = @best
    @best = @algorithm.best
    ptext = @best[:parameters].each_pair.map{ |k, v| "#{k}:#{v}" }.join(", ")
    if @best &&
       @best.key?(:score) &&
       best &&
       best.key?(:score) &&
       @best[:score] > best[:score]
       unless @verbosity == :silent
         puts "found a new best score: #{@best[:score]} "+
              "for parameters #{ptext}"
       end
    end
    # have we finished?
    in_progress = !@algorithm.finished?
    if in_progress && !(@timelimit.nil?)
      in_progress = (Time.now - start_time < @timelimit)
    end
  end
  @algorithm.write_data if @algorithm.respond_to? :write_data
  unless @verbosity == :silent
    puts "found optimum score: #{@best[:score]} for parameters "+
         "#{@best[:parameters]} in #{@iteration_count} iterations."
  end
  return @best
end

#run_iteration ⇒ Object

Runs a single iteration of the optimisation, encompassing the program, objective(s) and optimiser. Returns the output of the optimiser.



119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
 
# File 'lib/biopsy/experiment.rb', line 119

def run_iteration
  # create temp dir
  Dir.chdir(self.create_tempdir) do
    # run the target
    raw_output = @target.run @current_params.merge(@options)
    # evaluate with objectives
    param_key = @current_params.to_s
    result = nil
    if @scores.key? param_key
      result = @scores[param_key]
    else
      result = @objective.run_for_output(raw_output, @threads, nil)
      @iteration_count += 1
      self.print_progress(@iteration_count, @current_params, result, @best)
    end
    @scores[@current_params.to_s] = result
    # get next steps from optimiser
    @current_params = @algorithm.run_one_iteration(@current_params, result)
  end
  self.cleanup
end

#select_algorithm ⇒ Object

select the optimisation algorithm to use



61
62
63
64
65
66
67
68
69
70
 
# File 'lib/biopsy/experiment.rb', line 61

def select_algorithm
  return if algorithm
  max = Settings.instance.sweep_cutoff
  n = @target.count_parameter_permutations
  if n < max
    @algorithm = ParameterSweeper.new(@target.parameters)
  else
    @algorithm = TabuSearch.new(@target.parameters)
  end
end

#select_starting_point ⇒ Object

return the set of parameters to evaluate first



46
47
48
49
50
51
52
53
 
# File 'lib/biopsy/experiment.rb', line 46

def select_starting_point
  return if !@start.nil?
  if @algorithm && @algorithm.knows_starting_point?
    @start = @algorithm.select_starting_point
  else
    @start = self.random_start_point
  end
end