Class: PEROBS::EquiBlobsFile

Inherits:
Object
  • Object
show all
Defined in:
lib/perobs/EquiBlobsFile.rb

Overview

This class implements persistent storage space for same size data blobs. The blobs can be stored and retrieved and can be deleted again. The EquiBlobsFile manages the storage of the blobs and free storage spaces. The files grows and shrinks as needed. A blob is referenced by its address. The address is an Integer that must be larger than 0. The value 0 is used to represent an undefined address or nil. The file has a 4 * 8 bytes long header that stores the total entry count, the total space count, the offset of the first entry and the offset of the first space. The header is followed by a custom entry section. Each entry is also 8 bytes long. After the custom entry section the data blobs start. Each data blob starts with a mark byte that indicates if the blob is valid data (2), a free space (0) or reseved space (1). Then it is followed by @entry_bytes number of bytes for the data blob.

Constant Summary collapse

TOTAL_ENTRIES_OFFSET = 
0
TOTAL_SPACES_OFFSET = 
8
FIRST_ENTRY_OFFSET = 
2 * 8
FIRST_SPACE_OFFSET = 
3 * 8
HEADER_SIZE = 
4 * 8

Instance Attribute Summary collapse

Instance Method Summary collapse

Constructor Details

#initialize(dir, name, progressmeter, entry_bytes, first_entry_default = 0) ⇒ EquiBlobsFile

Create a new stack file in the given directory with the given file name.

Parameters:

  • dir (String) —

    Directory

  • name (String) —

    File name

  • progressmeter (ProgressMeter) —

    Reference to a progress meter object

  • entry_bytes (Integer) —

    Number of bytes each entry must have

  • first_entry_default (Integer) (defaults to: 0) —

    Default address of the first blob



64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
 
# File 'lib/perobs/EquiBlobsFile.rb', line 64

def initialize(dir, name, progressmeter, entry_bytes,
               first_entry_default = 0)
  @name = name
  @file_name = File.join(dir, name + '.blobs')
  @progressmeter = progressmeter
  if entry_bytes < 8
    PEROBS.log.fatal "EquiBlobsFile entry size must be at least 8"
  end
  @entry_bytes = entry_bytes
  @first_entry_default = first_entry_default
  clear_custom_data
  reset_counters

  # The File handle.
  @f = nil
end

Instance Attribute Details

#file_name ⇒ Object (readonly)

Returns the value of attribute file_name.



55
56
57
 
# File 'lib/perobs/EquiBlobsFile.rb', line 55

def file_name
  @file_name
end

#first_entry ⇒ Object

Returns the value of attribute first_entry.



55
56
57
 
# File 'lib/perobs/EquiBlobsFile.rb', line 55

def first_entry
  @first_entry
end

#total_entries ⇒ Object (readonly)

Returns the value of attribute total_entries.



55
56
57
 
# File 'lib/perobs/EquiBlobsFile.rb', line 55

def total_entries
  @total_entries
end

#total_spaces ⇒ Object (readonly)

Returns the value of attribute total_spaces.



55
56
57
 
# File 'lib/perobs/EquiBlobsFile.rb', line 55

def total_spaces
  @total_spaces
end

Instance Method Details

#check ⇒ Boolean

Check the file for logical errors.

Returns:

  • (Boolean) —

    true of file has no errors, false otherwise.



364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
 
# File 'lib/perobs/EquiBlobsFile.rb', line 364

def check
  sync

  return false unless check_spaces
  return false unless check_entries

  expected_size = address_to_offset(@total_entries + @total_spaces + 1)
  actual_size = @f.size
  if actual_size != expected_size
    PEROBS.log.error "Size mismatch in EquiBlobsFile #{@file_name}. " +
      "Expected #{expected_size} bytes but found #{actual_size} bytes."
    return false
  end

  true
end

#clear ⇒ Object

Delete all data.



190
191
192
193
194
195
 
# File 'lib/perobs/EquiBlobsFile.rb', line 190

def clear
  @f.truncate(0)
  @f.flush
  reset_counters
  write_header
end

#clear_custom_data ⇒ Object

Reset (delete) all custom data labels that have been registered.



135
136
137
138
139
140
141
142
143
144
 
# File 'lib/perobs/EquiBlobsFile.rb', line 135

def clear_custom_data
  unless @f.nil?
    PEROBS.log.fatal "clear_custom_data should only be called when " +
      "the file is not opened"
  end

  @custom_data_labels = []
  @custom_data_values = []
  @custom_data_defaults = []
end

#close ⇒ Object

Close the blob file. This method must be called before the program is terminated to avoid data loss.



104
105
106
107
108
109
110
111
112
113
114
115
116
 
# File 'lib/perobs/EquiBlobsFile.rb', line 104

def close
  begin
    if @f
      @f.flush
      @f.flock(File::LOCK_UN)
      @f.fsync
      @f.close
      @f = nil
    end
  rescue IOError => e
    PEROBS.log.fatal "Cannot close blob file #{@file_name}: #{e.message}"
  end
end

#delete_blob(address) ⇒ Object

Delete the blob at the given address.

Parameters:

  • address (Integer) —

    Address of blob to delete



328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
 
# File 'lib/perobs/EquiBlobsFile.rb', line 328

def delete_blob(address)
  unless address >= 0
    PEROBS.log.fatal "Blob address must be larger than 0, " +
      "not #{address}"
  end

  offset = address_to_offset(address)
  begin
    @f.seek(offset)
    if (marker = read_char) != 1 && marker != 2
      PEROBS.log.fatal "Cannot delete blob stored at address #{address} " +
        "of EquiBlobsFile #{@file_name}. Blob is " +
        (marker == 0 ? 'empty' : 'corrupted') + '.'
    end
    @f.seek(address_to_offset(address))
    write_char(0)
    write_unsigned_int(@first_space)
  rescue IOError => e
    PEROBS.log.fatal "Cannot delete blob at address #{address}: " +
      e.message
  end

  @first_space = offset
  @total_spaces += 1
  @total_entries -= 1 unless marker == 1
  write_header

  if offset == @f.size - 1 - @entry_bytes
    # We have deleted the last entry in the file. Make sure that all empty
    # entries are removed up to the now new last used entry.
    trim_file
  end
end

#erase ⇒ Object

Erase the backing store. This method should only be called when the file is not currently open.



171
172
173
174
175
 
# File 'lib/perobs/EquiBlobsFile.rb', line 171

def erase
  @f = nil
  File.delete(@file_name) if File.exist?(@file_name)
  reset_counters
end

#file_exist? ⇒ Boolean

Check if the file exists and is larger than 0.

Returns:

  • (Boolean) 


382
383
384
 
# File 'lib/perobs/EquiBlobsFile.rb', line 382

def file_exist?
  File.exist?(@file_name) && File.size(@file_name) > 0
end

#free_address ⇒ Integer

Return the address of a free blob storage space. Addresses start at 0 and increase linearly.

Returns:

  • (Integer) —

    address of a free blob space



207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
 
# File 'lib/perobs/EquiBlobsFile.rb', line 207

def free_address
  if @first_space == 0
    # There is currently no free entry. Create a new reserved entry at the
    # end of the file.
    begin
      offset = @f.size
      @f.seek(offset)
      write_n_bytes([1] + ::Array.new(@entry_bytes, 0))
      write_header
      return offset_to_address(offset)
    rescue IOError => e
      PEROBS.log.fatal "Cannot create reserved space at #{@first_space} " +
        "in EquiBlobsFile #{@file_name}: #{e.message}"
    end
  else
    begin
      free_space_address = offset_to_address(@first_space)
      @f.seek(@first_space)
      marker = read_char
      @first_space = read_unsigned_int
      unless marker == 0
        PEROBS.log.fatal "Free space list of EquiBlobsFile #{@file_name} " +
          "points to non-empty entry at address #{@first_space}"
      end
      # Mark entry as reserved by setting the mark byte to 1.
      @f.seek(-(1 + 8), IO::SEEK_CUR)
      write_char(1)

      # Update the file header
      @total_spaces -= 1
      write_header
      return free_space_address
    rescue IOError => e
      PEROBS.log.fatal "Cannot mark reserved space at " +
        "#{free_space_address} in EquiBlobsFile #{@file_name}: " +
        "#{e.message}"
    end
  end
end

#get_custom_data(name) ⇒ Integer

Get the registered custom data field value.

Parameters:

  • name (String) —

    Label of the offset

Returns:

  • (Integer) —

    Value of the custom data field



161
162
163
164
165
166
167
 
# File 'lib/perobs/EquiBlobsFile.rb', line 161

def get_custom_data(name)
  unless @custom_data_labels.include?(name)
    PEROBS.log.fatal "Unknown custom data field #{name}"
  end

  @custom_data_values[@custom_data_labels.index(name)]
end

#open ⇒ Object

Open the blob file.



82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
 
# File 'lib/perobs/EquiBlobsFile.rb', line 82

def open
  begin
    if File.exist?(@file_name)
      # Open an existing file.
      @f = File.open(@file_name, 'rb+')
      read_header
    else
      # Create a new file by writing a new header.
      @f = File.open(@file_name, 'wb+')
      write_header
    end
  rescue IOError => e
    PEROBS.log.fatal "Cannot open blob file #{@file_name}: #{e.message}"
  end
  unless @f.flock(File::LOCK_NB | File::LOCK_EX)
    PEROBS.log.fatal 'Database blob file is locked by another process'
  end
  @f.sync = true
end

#register_custom_data(name, default_value = 0) ⇒ Object

In addition to the standard offsets for the first entry and the first space any number of additional data fields can be registered. This must be done right after the object is instanciated and before the open() method is called. Each field represents a 64 bit unsigned integer.

Parameters:

  • name (String) —

    The label for this offset

  • default_value (Integer) (defaults to: 0) —

    The default value for the offset



124
125
126
127
128
129
130
131
132
 
# File 'lib/perobs/EquiBlobsFile.rb', line 124

def register_custom_data(name, default_value = 0)
  if @custom_data_labels.include?(name)
    PEROBS.log.fatal "Custom data field #{name} has already been registered"
  end

  @custom_data_labels << name
  @custom_data_values << default_value
  @custom_data_defaults << default_value
end

#retrieve_blob(address) ⇒ String

Retrieve a blob from the given address.

Parameters:

  • address (Integer) —

    Address to store the blob

Returns:

  • (String) —

    blob bytes



298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
 
# File 'lib/perobs/EquiBlobsFile.rb', line 298

def retrieve_blob(address)
  unless address > 0
    PEROBS.log.fatal "Blob retrieval address must be larger than 0, " +
      "not #{address}"
  end

  begin
    if (offset = address_to_offset(address)) >= @f.size
      PEROBS.log.fatal "Cannot retrieve blob at address #{address} " +
        "of EquiBlobsFile #{@file_name}. Address is beyond end of file."
    end

    @f.seek(address_to_offset(address))
    if (marker = read_char) != 2
      PEROBS.log.fatal "Cannot retrieve blob at address #{address} " +
        "of EquiBlobsFile #{@file_name}. Blob is " +
        (marker == 0 ? 'empty' : marker == 1 ? 'reserved' : 'corrupted') +
        '.'
    end
    bytes = @f.read(@entry_bytes)
  rescue IOError => e
    PEROBS.log.fatal "Cannot retrieve blob at adress #{address} " +
      "of EquiBlobsFile #{@file_name}: " + e.message
  end

  bytes
end

#set_custom_data(name, value) ⇒ Object

Set the registered custom data field to the given value.

Parameters:

  • name (String) —

    Label of the offset

  • value (Integer) —

    Value



149
150
151
152
153
154
155
156
 
# File 'lib/perobs/EquiBlobsFile.rb', line 149

def set_custom_data(name, value)
  unless @custom_data_labels.include?(name)
    PEROBS.log.fatal "Unknown custom data field #{name}"
  end

  @custom_data_values[@custom_data_labels.index(name)] = value
  write_header if @f
end

#store_blob(address, bytes) ⇒ Object

Store the given byte blob at the specified address. If the blob space is already in use the content will be overwritten.

Parameters:

  • address (Integer) —

    Address to store the blob

  • bytes (String) —

    bytes to store



251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
 
# File 'lib/perobs/EquiBlobsFile.rb', line 251

def store_blob(address, bytes)
  unless address >= 0
    PEROBS.log.fatal "Blob storage address must be larger than 0, " +
      "not #{address}"
  end
  if bytes.length != @entry_bytes
    PEROBS.log.fatal "All stack entries must be #{@entry_bytes} " +
      "long. This entry is #{bytes.length} bytes long."
  end

  marker = 1
  begin
    offset = address_to_offset(address)
    if offset > (file_size = @f.size)
      PEROBS.log.fatal "Cannot store blob at address #{address} in " +
        "EquiBlobsFile #{@file_name}. Address is larger than file size. " +
        "Offset: #{offset}  File size: #{file_size}"
    end

    @f.seek(offset)
    # The first byte is the marker byte. It's set to 2 for cells that hold
    # a blob. 1 for reserved cells and 0 for empty cells. The cell must be
    # either already be in use or be reserved. It must not be 0.
    if file_size > offset &&
       (marker = read_char) != 1 && marker != 2
      PEROBS.log.fatal "Marker for entry at address #{address} of " +
        "EquiBlobsFile #{@file_name} must be 1 or 2 but is #{marker}"
    end
    @f.seek(offset)
    write_char(2)
    @f.write(bytes)
    @f.flush
  rescue IOError => e
    PEROBS.log.fatal "Cannot store blob at address #{address} in " +
      "EquiBlobsFile #{@file_name}: #{e.message}"
  end

  # Update the entries counter if we inserted a new blob.
  if marker == 1
    @total_entries += 1
    write_header
  end
end

#sync ⇒ Object

Flush out all unwritten data.



178
179
180
181
182
183
184
185
186
187
 
# File 'lib/perobs/EquiBlobsFile.rb', line 178

def sync
  begin
    if @f
      @f.flush
      @f.fsync
    end
  rescue IOError => e
    PEROBS.log.fatal "Cannot sync blob file #{@file_name}: #{e.message}"
  end
end