Class: LWAC::StorageManager

Inherits:
Object
  • Object
show all
Defined in:
lib/lwac/server/storage_manager.rb

Overview

Handles storage, both file and database based

Instance Attribute Summary collapse

Instance Method Summary collapse

Constructor Details

#initialize(config) ⇒ StorageManager

Returns a new instance of StorageManager.



113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
 
# File 'lib/lwac/server/storage_manager.rb', line 113

def initialize(config)
  @config         = config
  @root           = config[:root]
  @files_per_dir  = config[:files_per_dir]

  # Debug info
  $log.debug "Storage manager starting, serialising using #{config[:serialiser]}"
  @serialiser = Serialiser.new(config[:serialiser])

  # Database storage
  @db = DatabaseStorageManager.new(config[:database])

  # Try to load the current server state
  @state_filename = File.join(@root, config[:state_file])
  if(File.exist?(@state_filename))
    @state = @serialiser.load_file(@state_filename)

    # Version check on the state file that describes the corpus
    if not @state.respond_to?(:version) or not Identity::storage_is_compatible?(@state.version) then
      if @state.respond_to?(:version)
        $log.fatal "The corpus you are trying to load was written by LWAC version #{@state.version}" 
      else 
        $log.fatal "No version info---the corpus was written by a prerelease version of LWAC"
      end
      $log.fatal "This server is only compatible with versions: #{Identity::COMPATIBLE_VERSIONS.sort.join(", ")}"
      raise "Incompatible storage format"
    end

  else
    $log.debug "No state.  Creating a new state file at #{@state_filename}"
    @state = ServerState.new(LWAC::VERSION)
    @serialiser.dump_file(@state, @state_filename)
  end

  # Create the sample subdir
  FileUtils.mkdir_p(get_sample_filepath()) if not File.exist?(get_sample_filepath)
end

Instance Attribute Details

#state ⇒ Object (readonly)

Allow the user to access the server state



111
112
113
 
# File 'lib/lwac/server/storage_manager.rb', line 111

def state
  @state
end

Instance Method Details

#close ⇒ Object

Close the resource and make sure everything is dumped to disk



249
250
251
252
253
 
# File 'lib/lwac/server/storage_manager.rb', line 249

def close
  $log.fatal "Closing storage manager, writing state to #{@state_filename}"
  write_state
  @db.close
end

#count_links(min_id = 0) ⇒ Object

Count links optionally min_id is the lowest id to count from



170
171
172
 
# File 'lib/lwac/server/storage_manager.rb', line 170

def count_links(min_id=0)
  @db.count_links(min_id)
end

#get_dp_filepath(id_or_dp, sample_id = @state.current_sample.id) ⇒ Object

Get a datapoint filepath



267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
 
# File 'lib/lwac/server/storage_manager.rb', line 267

def get_dp_filepath(id_or_dp, sample_id = @state.current_sample.id)
  # Get the numeric link ID from a datapoint, link or raw ID
  id = id_or_dp.to_i        if(id_or_dp.is_a? Integer)
  id = id_or_dp.id          if(id_or_dp.is_a? Link)
  id = id_or_dp.link.id     if(id_or_dp.is_a? DataPoint)

  # Break it up into blocks of @files_per_dir
  dir = (id.to_i/@files_per_dir).floor

  # Ensure dir exists
  filepath = get_sample_filepath( sample_id, dir, true)

  # Join the datapoint ID
  return File.join(filepath, "#{id.to_s}")
end

#get_sample_filepath(sample_id = nil, dir = nil, ensure_exists = false) ⇒ Object

Get a sample filepath, parent of a datapoint filepath



256
257
258
259
260
261
262
263
264
 
# File 'lib/lwac/server/storage_manager.rb', line 256

def get_sample_filepath(sample_id=nil, dir=nil, ensure_exists=false)
  filepath = File.join( @root, @config[:sample_subdir] )
  filepath = File.join( filepath, sample_id.to_s )            if sample_id
  filepath = File.join( filepath, dir.to_s )                  if dir

  FileUtils.mkdir_p(filepath) if not File.exist?(filepath)

  return filepath
end

#read_datapoint(dp_id, sample = @state.current_sample) ⇒ Object

Read a datapoint from disk



183
184
185
186
187
 
# File 'lib/lwac/server/storage_manager.rb', line 183

def read_datapoint(dp_id, sample = @state.current_sample)
  $log.debug "Reading datapoint #{dp_id} (sample #{sample.id}) from disk."
  dp_path = get_dp_filepath(dp_id, sample.id)
  @serialiser.load_file( dp_path )
end

#read_link(id) ⇒ Object

Read a single ID



159
160
161
 
# File 'lib/lwac/server/storage_manager.rb', line 159

def read_link(id)
  @db.read_link(id)
end

#read_link_ids(from = nil, n = nil) ⇒ Object

Read all IDs as a set



164
165
166
 
# File 'lib/lwac/server/storage_manager.rb', line 164

def read_link_ids(from=nil, n=nil)
  @db.read_link_ids(from, n)
end

#read_links(range_low = nil, range_high = nil) ⇒ Object

Read some links from the database using either a range, or an array, depending on the first argument



153
154
155
156
 
# File 'lib/lwac/server/storage_manager.rb', line 153

def read_links(range_low = nil, range_high = nil)
  return @db.read_links_from_array(range_low) if range_high == nil and range_low.is_a?(Array)
  @db.read_links(range_low, range_high)
end

#read_sample(sample_id = @state.last_sample_id) ⇒ Object

Read a finalised sample ID from disk. raises Errno::ENOENT if not there



201
202
203
204
 
# File 'lib/lwac/server/storage_manager.rb', line 201

def read_sample(sample_id = @state.last_sample_id)
  sample_path = File.join( get_sample_filepath(sample_id), @config[:sample_filename])
  @serialiser.load_file( sample_path )
end

#update_state(state) ⇒ Object

Update the server state



243
244
245
246
 
# File 'lib/lwac/server/storage_manager.rb', line 243

def update_state(state)
  @state = state
  write_state
end

#validate_sample(sample_id, verify_datapoints = true) ⇒ Object

Ensure a sample has all of its files on disk, and that they are readable



211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
 
# File 'lib/lwac/server/storage_manager.rb', line 211

def validate_sample(sample_id, verify_datapoints=true)
  $log.debug "Validating sample #{sample_id}..."
  # Check the file exists
  begin
    sample = read_sample(sample_id)
  rescue StandardError => e
    raise "Error loading sample metadata: #{e.to_s}"
  end

  # Load all links and work out which files should
  # actually be in the dir
  all_link_ids = read_link_ids
  sampled = all_link_ids.delete_if{|x| x > sample.last_dp_id} - sample.pending # FIXME

  # Now check they all exist
  if(verify_datapoints) then
    $log.debug "Validating datapoints for #{sample}..."
    sampled.each{ |link_id|
      path = get_dp_filepath( link_id, sample_id )


      raise "Datapoint #{link_id} is missing."          if not File.readable? path
      raise "Cannot read datapoint with ID #{link_id}"  if not File.readable? path
    }
  end

  $log.info "Sample #{sample} passed validation (datapoints checked? #{verify_datapoints})"
  return true
end

#write_datapoint(dp, sample = @state.current_sample) ⇒ Object

Datapoint read/write Write a datapoint to disk



176
177
178
179
180
 
# File 'lib/lwac/server/storage_manager.rb', line 176

def write_datapoint(dp, sample = @state.current_sample)
  $log.debug "Writing datapoint #{dp.link.id} (sample #{sample.id}) to disk."
  dp_path = get_dp_filepath(dp, sample.id)
  @serialiser.dump_file( dp, dp_path)
end

#write_sample(sample = @state.current_sample) ⇒ Object

Sample read/write Write a finalised sample to disk in its proper location.



194
195
196
197
 
# File 'lib/lwac/server/storage_manager.rb', line 194

def write_sample(sample = @state.current_sample)
  sample_path = File.join( get_sample_filepath(sample.id), @config[:sample_filename])
  @serialiser.dump_file( sample, sample_path )
end

#write_state ⇒ Object

Write the server state to disk



284
285
286
 
# File 'lib/lwac/server/storage_manager.rb', line 284

def write_state
  @serialiser.dump_file( @state, @state_filename)
end