Class: Biopsy::Experiment

Inherits:
Object
  • Object
show all
Defined in:
lib/biopsy/experiment.rb

Instance Attribute Summary collapse

Instance Method Summary collapse

Constructor Details

#initialize(target, options: {}, threads: 4, start: nil, algorithm: nil, timelimit: nil, verbosity: :quiet, id: nil) ⇒ Experiment

Returns a new Experiment



25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
 
# File 'lib/biopsy/experiment.rb', line 25

def initialize(target, options:{}, threads:4, start:nil, algorithm:nil,
               timelimit:nil, verbosity: :quiet, id:nil)
  @threads = threads
  @start = start
  @algorithm = algorithm
  @timelimit = timelimit
  @verbosity = verbosity
  if target.is_a? Target
    @target = target
  else
    self.load_target target
  end
  @options = options
  @objective = ObjectiveHandler.new @target
  self.select_algorithm
  self.select_starting_point
  @scores = {}
  @iteration_count = 0
  set_id id
end

Instance Attribute Details

#algorithm ⇒ Object (readonly)

Returns the value of attribute algorithm.



22
23
24
 
# File 'lib/biopsy/experiment.rb', line 22

def algorithm
  @algorithm
end

#inputs ⇒ Object (readonly)

Returns the value of attribute inputs.



21
22
23
 
# File 'lib/biopsy/experiment.rb', line 21

def inputs
  @inputs
end

#outputs ⇒ Object (readonly)

Returns the value of attribute outputs.



21
22
23
 
# File 'lib/biopsy/experiment.rb', line 21

def outputs
  @outputs
end

#retain_intermediates ⇒ Object (readonly)

Returns the value of attribute retain_intermediates.



21
22
23
 
# File 'lib/biopsy/experiment.rb', line 21

def retain_intermediates
  @retain_intermediates
end

#start ⇒ Object (readonly)

Returns the value of attribute start.



22
23
24
 
# File 'lib/biopsy/experiment.rb', line 22

def start
  @start
end

#target ⇒ Object (readonly)

Returns the value of attribute target.



22
23
24
 
# File 'lib/biopsy/experiment.rb', line 22

def target
  @target
end

Instance Method Details

#cleanup ⇒ Object 



154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
 
# File 'lib/biopsy/experiment.rb', line 154

def cleanup
  return if Settings.instance.no_tempdirs
  # TODO: make this work
  # remove all but essential files
  essential_files = ""
  if Settings.instance.keep_intermediates
    # @objectives isn't mentioned anywhere in the rest of this file
    @objectives.values.each do |objective|
      essential_files += objective.essential_files
    end
  end
  Dir["*"].each do |file|
    next
    # TODO: implement this
    # next if File.directory? file
    # if essential_files && essential_files.include?(file)
    #   `gzip #{file}` if Settings.instance.gzip_intermediates
    #   FileUtils.mv("#{file}.gz", '../output')
    # end
  end
  FileUtils.rm_rf @last_tempdir
end

#create_tempdir ⇒ Object

create a guaranteed random temporary directory for storing outputs return the dirname



179
180
181
182
183
184
185
186
187
188
189
 
# File 'lib/biopsy/experiment.rb', line 179

def create_tempdir
  token = loop do
    # generate random dirnames until we find one that
    # doesn't exist
    test_token = SecureRandom.hex
    break test_token unless File.exist? test_token
  end
  Dir.mkdir(token)
  @last_tempdir = token
  token
end

#load_target(target_name) ⇒ Object

load the target named :target_name



74
75
76
77
 
# File 'lib/biopsy/experiment.rb', line 74

def load_target(target_name)
  @target = Target.new
  @target.load_by_name target_name
end

#print_progress(iteration, params, score, best) ⇒ Object 



145
146
147
148
149
150
151
152
 
# File 'lib/biopsy/experiment.rb', line 145

def print_progress(iteration, params, score, best)
  unless [:silent, :quiet].include? @verbosity
    ptext = params.each_pair.map{ |k, v| "#{k}:#{v}" }.join(", ")
    msg = "run #{iteration}. parameters: #{ptext} | score: #{score}"
    msg += " | best #{best[:score]}" if (best && best.has_key?(:score))
    puts msg
  end
end

#random_start_point ⇒ Object

Return a random set of parameters from the parameter space.



57
58
59
 
# File 'lib/biopsy/experiment.rb', line 57

def random_start_point
  Hash[@target.parameters.map { |p, r| [p, r.sample] }]
end

#run ⇒ Object

Runs the experiment until the completion criteria are met. On completion, returns the best parameter set.



82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
 
# File 'lib/biopsy/experiment.rb', line 82

def run
  start_time = Time.now
  in_progress = true
  @algorithm.setup @start
  @current_params = @start
  max_scores = @target.count_parameter_permutations
  while in_progress
    run_iteration
    # update the best result
    best = @best
    @best = @algorithm.best
    ptext = @best[:parameters].each_pair.map{ |k, v| "#{k}:#{v}" }.join(", ")
    if @best &&
       @best.key?(:score) &&
       best &&
       best.key?(:score) &&
       @best[:score] > best[:score]
       unless @verbosity == :silent
         puts "found a new best score: #{@best[:score]} "+
              "for parameters #{ptext}"
       end
    end
    # have we finished?
    in_progress = !@algorithm.finished? && @scores.size < max_scores
    if in_progress && !(@timelimit.nil?)
      in_progress = (Time.now - start_time < @timelimit)
    end
  end
  @algorithm.write_data if @algorithm.respond_to? :write_data
  unless @verbosity == :silent
    puts "found optimum score: #{@best[:score]} for parameters "+
         "#{@best[:parameters]} in #{@iteration_count} iterations."
  end
  return @best
end

#run_iteration ⇒ Object

Runs a single iteration of the optimisation, encompassing the program, objective(s) and optimiser. Returns the output of the optimiser.



121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
 
# File 'lib/biopsy/experiment.rb', line 121

def run_iteration
  param_key = @current_params.to_s
  result = nil
  # lookup the result if possible
  if @scores.key? param_key
    result = @scores[param_key]
  else
    # create temp dir
    curdir = Dir.pwd
    Dir.chdir(self.create_tempdir) unless Settings.instance.no_tempdirs
    # run the target
    raw_output = @target.run @current_params.merge(@options)
    # evaluate with objectives
    result = @objective.run_for_output(raw_output, @threads, nil)
    @iteration_count += 1
    self.print_progress(@iteration_count, @current_params, result, @best)
    @scores[@current_params.to_s] = result
    self.cleanup
    Dir.chdir(curdir) unless Settings.instance.no_tempdirs
  end
  # get next steps from optimiser
  @current_params = @algorithm.run_one_iteration(@current_params, result)
end

#select_algorithm ⇒ Object

select the optimisation algorithm to use



62
63
64
65
66
67
68
69
70
71
 
# File 'lib/biopsy/experiment.rb', line 62

def select_algorithm
  return if algorithm
  max = Settings.instance.sweep_cutoff
  n = @target.count_parameter_permutations
  if n < max
    @algorithm = ParameterSweeper.new(@target.parameters, @id)
  else
    @algorithm = TabuSearch.new(@target.parameters, @id)
  end
end

#select_starting_point ⇒ Object

return the set of parameters to evaluate first



47
48
49
50
51
52
53
54
 
# File 'lib/biopsy/experiment.rb', line 47

def select_starting_point
  return if !@start.nil?
  if @algorithm && @algorithm.knows_starting_point?
    @start = @algorithm.select_starting_point
  else
    @start = self.random_start_point
  end
end

#set_id(id) ⇒ Object

set experiment ID with either user provided value, or date-time as fallback



193
194
195
196
197
198
199
200
 
# File 'lib/biopsy/experiment.rb', line 193

def set_id id
  @id = id
  if @id.nil?
    t = Time.now
    parts = %w[y m d H M S Z].map{ |p| t.strftime "%#{p}" }
    @id = "experiment_#{parts.join('_')}"
  end
end