Class: BigShift::UnloadManifest

Inherits:
Object
  • Object
show all
Defined in:
lib/bigshift/unload_manifest.rb

Instance Attribute Summary collapse

Instance Method Summary collapse

Constructor Details

#initialize(s3_resource, cs_service, bucket_name, prefix) ⇒ UnloadManifest

Returns a new instance of UnloadManifest.



7
8
9
10
11
12
13
 
# File 'lib/bigshift/unload_manifest.rb', line 7

def initialize(s3_resource, cs_service, bucket_name, prefix)
  @s3_resource = s3_resource
  @cs_service = cs_service
  @bucket_name = bucket_name
  @prefix = prefix
  @manifest_key = "#{@prefix}manifest"
end

Instance Attribute Details

#bucket_name ⇒ Object (readonly)

Returns the value of attribute bucket_name.



5
6
7
 
# File 'lib/bigshift/unload_manifest.rb', line 5

def bucket_name
  @bucket_name
end

#manifest_key ⇒ Object (readonly)

Returns the value of attribute manifest_key.



5
6
7
 
# File 'lib/bigshift/unload_manifest.rb', line 5

def manifest_key
  @manifest_key
end

#prefix ⇒ Object (readonly)

Returns the value of attribute prefix.



5
6
7
 
# File 'lib/bigshift/unload_manifest.rb', line 5

def prefix
  @prefix
end

Instance Method Details

#count ⇒ Object 



24
25
26
 
# File 'lib/bigshift/unload_manifest.rb', line 24

def count
  keys.size
end

#keys ⇒ Object 



15
16
17
18
19
20
21
22
 
# File 'lib/bigshift/unload_manifest.rb', line 15

def keys
  @keys ||= begin
    bucket = @s3_resource.bucket(@bucket_name)
    object = bucket.object(@manifest_key)
    manifest = JSON.load(object.get.body)
    manifest['entries'].map { |entry| entry['url'].sub(%r{\As3://[^/]+/}, '') }
  end
end

#total_file_size ⇒ Object 



28
29
30
 
# File 'lib/bigshift/unload_manifest.rb', line 28

def total_file_size
  @total_file_size ||= file_sizes.values.reduce(:+)
end

#validate_transfer(cs_bucket_name) ⇒ Object 



32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
 
# File 'lib/bigshift/unload_manifest.rb', line 32

def validate_transfer(cs_bucket_name)
  objects = @cs_service.list_objects(cs_bucket_name, prefix: @prefix)
  cs_file_sizes = objects.items.each_with_object({}) do |item, acc|
    acc[item.name] = item.size.to_i
  end
  missing_files = (file_sizes.keys - cs_file_sizes.keys)
  extra_files = cs_file_sizes.keys - file_sizes.keys
  common_files = (cs_file_sizes.keys & file_sizes.keys)
  size_mismatches = common_files.select { |name| file_sizes[name] != cs_file_sizes[name] }
  errors = []
  unless missing_files.empty?
    errors << "missing files: #{missing_files.join(', ')}"
  end
  unless extra_files.empty?
    errors << "extra files: #{extra_files.join(', ')}"
  end
  unless size_mismatches.empty?
    messages = size_mismatches.map { |name| sprintf('%s (%d != %d)', name, cs_file_sizes[name], file_sizes[name]) }
    errors << "size mismatches: #{messages.join(', ')}"
  end
  unless errors.empty?
    raise TransferValidationError, "Transferred files don't match unload manifest: #{errors.join('; ')}"
  end
end