Class: SwiftIngest::Ingestor

Inherits:
Object
  • Object
show all
Defined in:
lib/swift_ingest/ingestor.rb

Instance Attribute Summary collapse

Instance Method Summary collapse

Constructor Details

#initialize(connection = {}) ⇒ Ingestor

Returns a new instance of Ingestor.



9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
 
# File 'lib/swift_ingest/ingestor.rb', line 9

def initialize(connection = {})
  extra_opt = { auth_method: 'password',
                service_type: 'object-store' }
  options = SwiftIngest::DEFAULTS.merge(connection).merge(extra_opt)
  options[:api_key] = options.delete :password

  @swift_connection = OpenStack::Connection.create(options)
  @project = connection[:project]

  # connect to the database
  @dbcon = if ENV['DB_HOST'] && ENV['DB_USER'] && ENV['DB_PASSWORD'] && ENV['DB_DATABASE']
             Mysql2::Client.new(host: ENV['DB_HOST'],
                                username: ENV['DB_USER'],
                                password: ENV['DB_PASSWORD'],
                                database: ENV['DB_DATABASE'])
           end
end

Instance Attribute Details

#project ⇒ Object (readonly)

Returns the value of attribute project.



7
8
9
 
# File 'lib/swift_ingest/ingestor.rb', line 7

def project
  @project
end

#swift_connection ⇒ Object (readonly)

Returns the value of attribute swift_connection.



7
8
9
 
# File 'lib/swift_ingest/ingestor.rb', line 7

def swift_connection
  @swift_connection
end

Instance Method Details

#deposit_file(file_name, swift_container, custom_metadata = {}) ⇒ Object 



35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
 
# File 'lib/swift_ingest/ingestor.rb', line 35

def deposit_file(file_name, swift_container, custom_metadata = {})
  file_base_name = File.basename(file_name, '.*')
  checksum = Digest::MD5.file(file_name).hexdigest
  container = swift_connection.container(swift_container)

  # Add swift metadata with in accordance to AIP spec:
  # https://docs.google.com/document/d/154BqhDPAdGW-I9enrqLpBYbhkF9exX9lV3kMaijuwPg/edit#
  metadata = {
    project: @project,
    project_id: file_base_name,
    promise: 'bronze',
    aip_version: '1.0'
  }.merge(custom_metadata)

  # ruby-openstack wants all keys of the metadata to be named like
  # "X-Object-Meta-{{Key}}" so update them
  metadata.transform_keys! { |key| "X-Object-Meta-#{key}" }

  if container.object_exists?(file_base_name)
    # temporary solution until fixed in upstream:
    # for update: construct hash for key/value pairs as strings,
    # and metadata as additional key/value string pairs in the hash
    headers = { 'etag' => checksum,
                'content-type' => 'application/x-tar' }.merge(metadata)
    deposited_file = container.object(file_base_name)
    deposited_file.write(File.open(file_name), headers)
  else
    # for creating new: construct hash with symbols as keys, add metadata as a hash within the header hash
    headers = { etag: checksum,
                content_type:  'application/x-tar',
                metadata: metadata }
    deposited_file = container.create_object(file_base_name, headers, File.open(file_name))
  end

  return deposited_file unless @dbcon

  # update db with deposited file info
  @dbcon.query("INSERT INTO archiveEvent(project, container, ingestTime, \
                objectIdentifier, objectChecksum, objectSize) \
                VALUES('#{@project}', '#{swift_container}', now(), '#{file_base_name}', '#{checksum}', \
                '#{File.size(file_name)}')")
  custom_metadata.each do |key, value|
    @dbcon.query("INSERT INTO customMetadata(eventId, propertyName, propertyValue) \
                  VALUES(LAST_INSERT_ID(), '#{key}', '#{value}' )")
  end

  deposited_file
end

#get_file_from_swit(file_name, swift_container) ⇒ Object 



27
28
29
30
31
32
33
 
# File 'lib/swift_ingest/ingestor.rb', line 27

def get_file_from_swit(file_name, swift_container)
  deposited_file = nil
  file_base_name = File.basename(file_name, '.*')
  container = swift_connection.container(swift_container)
  deposited_file = container.object(file_base_name) if container.object_exists?(file_base_name)
  deposited_file
end