Class: Biopsy::Experiment
- Inherits:
-
Object
- Object
- Biopsy::Experiment
- Defined in:
- lib/biopsy/experiment.rb
Instance Attribute Summary collapse
-
#algorithm ⇒ Object
readonly
Returns the value of attribute algorithm.
-
#inputs ⇒ Object
readonly
Returns the value of attribute inputs.
-
#outputs ⇒ Object
readonly
Returns the value of attribute outputs.
-
#retain_intermediates ⇒ Object
readonly
Returns the value of attribute retain_intermediates.
-
#start ⇒ Object
readonly
Returns the value of attribute start.
-
#target ⇒ Object
readonly
Returns the value of attribute target.
Instance Method Summary collapse
- #cleanup ⇒ Object
-
#create_tempdir ⇒ Object
create a guaranteed random temporary directory for storing outputs return the dirname.
-
#initialize(target, options: {}, threads: 4, start: nil, algorithm: nil, timelimit: nil, verbosity: :quiet, id: nil) ⇒ Experiment
constructor
Returns a new Experiment.
-
#load_target(target_name) ⇒ Object
load the target named
:target_name
. - #print_progress(iteration, params, score, best) ⇒ Object
-
#random_start_point ⇒ Object
Return a random set of parameters from the parameter space.
-
#run ⇒ Object
Runs the experiment until the completion criteria are met.
-
#run_iteration ⇒ Object
Runs a single iteration of the optimisation, encompassing the program, objective(s) and optimiser.
-
#select_algorithm ⇒ Object
select the optimisation algorithm to use.
-
#select_starting_point ⇒ Object
return the set of parameters to evaluate first.
-
#set_id(id) ⇒ Object
set experiment ID with either user provided value, or date-time as fallback.
Constructor Details
#initialize(target, options: {}, threads: 4, start: nil, algorithm: nil, timelimit: nil, verbosity: :quiet, id: nil) ⇒ Experiment
Returns a new Experiment
25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 |
# File 'lib/biopsy/experiment.rb', line 25 def initialize(target, options:{}, threads:4, start:nil, algorithm:nil, timelimit:nil, verbosity: :quiet, id:nil) @threads = threads @start = start @algorithm = algorithm @timelimit = timelimit @verbosity = verbosity if target.is_a? Target @target = target else self.load_target target end @options = @objective = ObjectiveHandler.new @target self.select_algorithm self.select_starting_point @scores = {} @iteration_count = 0 set_id id end |
Instance Attribute Details
#algorithm ⇒ Object (readonly)
Returns the value of attribute algorithm.
22 23 24 |
# File 'lib/biopsy/experiment.rb', line 22 def algorithm @algorithm end |
#inputs ⇒ Object (readonly)
Returns the value of attribute inputs.
21 22 23 |
# File 'lib/biopsy/experiment.rb', line 21 def inputs @inputs end |
#outputs ⇒ Object (readonly)
Returns the value of attribute outputs.
21 22 23 |
# File 'lib/biopsy/experiment.rb', line 21 def outputs @outputs end |
#retain_intermediates ⇒ Object (readonly)
Returns the value of attribute retain_intermediates.
21 22 23 |
# File 'lib/biopsy/experiment.rb', line 21 def retain_intermediates @retain_intermediates end |
#start ⇒ Object (readonly)
Returns the value of attribute start.
22 23 24 |
# File 'lib/biopsy/experiment.rb', line 22 def start @start end |
#target ⇒ Object (readonly)
Returns the value of attribute target.
22 23 24 |
# File 'lib/biopsy/experiment.rb', line 22 def target @target end |
Instance Method Details
#cleanup ⇒ Object
154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 |
# File 'lib/biopsy/experiment.rb', line 154 def cleanup return if Settings.instance.no_tempdirs # TODO: make this work # remove all but essential files essential_files = "" if Settings.instance.keep_intermediates # @objectives isn't mentioned anywhere in the rest of this file @objectives.values.each do |objective| essential_files += objective.essential_files end end Dir["*"].each do |file| next # TODO: implement this # next if File.directory? file # if essential_files && essential_files.include?(file) # `gzip #{file}` if Settings.instance.gzip_intermediates # FileUtils.mv("#{file}.gz", '../output') # end end FileUtils.rm_rf @last_tempdir end |
#create_tempdir ⇒ Object
create a guaranteed random temporary directory for storing outputs return the dirname
179 180 181 182 183 184 185 186 187 188 189 |
# File 'lib/biopsy/experiment.rb', line 179 def create_tempdir token = loop do # generate random dirnames until we find one that # doesn't exist test_token = SecureRandom.hex break test_token unless File.exist? test_token end Dir.mkdir(token) @last_tempdir = token token end |
#load_target(target_name) ⇒ Object
load the target named :target_name
74 75 76 77 |
# File 'lib/biopsy/experiment.rb', line 74 def load_target(target_name) @target = Target.new @target.load_by_name target_name end |
#print_progress(iteration, params, score, best) ⇒ Object
145 146 147 148 149 150 151 152 |
# File 'lib/biopsy/experiment.rb', line 145 def print_progress(iteration, params, score, best) unless [:silent, :quiet].include? @verbosity ptext = params.each_pair.map{ |k, v| "#{k}:#{v}" }.join(", ") msg = "run #{iteration}. parameters: #{ptext} | score: #{score}" msg += " | best #{best[:score]}" if (best && best.has_key?(:score)) puts msg end end |
#random_start_point ⇒ Object
Return a random set of parameters from the parameter space.
57 58 59 |
# File 'lib/biopsy/experiment.rb', line 57 def random_start_point Hash[@target.parameters.map { |p, r| [p, r.sample] }] end |
#run ⇒ Object
Runs the experiment until the completion criteria are met. On completion, returns the best parameter set.
82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 |
# File 'lib/biopsy/experiment.rb', line 82 def run start_time = Time.now in_progress = true @algorithm.setup @start @current_params = @start max_scores = @target.count_parameter_permutations while in_progress run_iteration # update the best result best = @best @best = @algorithm.best ptext = @best[:parameters].each_pair.map{ |k, v| "#{k}:#{v}" }.join(", ") if @best && @best.key?(:score) && best && best.key?(:score) && @best[:score] > best[:score] unless @verbosity == :silent puts "found a new best score: #{@best[:score]} "+ "for parameters #{ptext}" end end # have we finished? in_progress = !@algorithm.finished? && @scores.size < max_scores if in_progress && !(@timelimit.nil?) in_progress = (Time.now - start_time < @timelimit) end end @algorithm.write_data if @algorithm.respond_to? :write_data unless @verbosity == :silent puts "found optimum score: #{@best[:score]} for parameters "+ "#{@best[:parameters]} in #{@iteration_count} iterations." end return @best end |
#run_iteration ⇒ Object
Runs a single iteration of the optimisation, encompassing the program, objective(s) and optimiser. Returns the output of the optimiser.
121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 |
# File 'lib/biopsy/experiment.rb', line 121 def run_iteration param_key = @current_params.to_s result = nil # lookup the result if possible if @scores.key? param_key result = @scores[param_key] else # create temp dir curdir = Dir.pwd Dir.chdir(self.create_tempdir) unless Settings.instance.no_tempdirs # run the target raw_output = @target.run @current_params.merge(@options) # evaluate with objectives result = @objective.run_for_output(raw_output, @threads, nil) @iteration_count += 1 self.print_progress(@iteration_count, @current_params, result, @best) @scores[@current_params.to_s] = result self.cleanup Dir.chdir(curdir) unless Settings.instance.no_tempdirs end # get next steps from optimiser @current_params = @algorithm.run_one_iteration(@current_params, result) end |
#select_algorithm ⇒ Object
select the optimisation algorithm to use
62 63 64 65 66 67 68 69 70 71 |
# File 'lib/biopsy/experiment.rb', line 62 def select_algorithm return if algorithm max = Settings.instance.sweep_cutoff n = @target.count_parameter_permutations if n < max @algorithm = ParameterSweeper.new(@target.parameters, @id) else @algorithm = TabuSearch.new(@target.parameters, @id) end end |
#select_starting_point ⇒ Object
return the set of parameters to evaluate first
47 48 49 50 51 52 53 54 |
# File 'lib/biopsy/experiment.rb', line 47 def select_starting_point return if !@start.nil? if @algorithm && @algorithm.knows_starting_point? @start = @algorithm.select_starting_point else @start = self.random_start_point end end |
#set_id(id) ⇒ Object
set experiment ID with either user provided value, or date-time as fallback
193 194 195 196 197 198 199 200 |
# File 'lib/biopsy/experiment.rb', line 193 def set_id id @id = id if @id.nil? t = Time.now parts = %w[y m d H M S Z].map{ |p| t.strftime "%#{p}" } @id = "experiment_#{parts.join('_')}" end end |