Class: S3TarBackup::Main

Inherits:
Object
  • Object
show all
Defined in:
lib/s3_tar_backup.rb

Constant Summary collapse

UPLOAD_TRIES = 
5

Instance Method Summary collapse

Instance Method Details

#backup(config, backup, verbose = false) ⇒ Object 



190
191
192
193
194
195
196
197
 
# File 'lib/s3_tar_backup.rb', line 190

def backup(config, backup, verbose=false)
  exec(backup.backup_cmd(verbose))
  puts "Uploading #{config[:bucket]}/#{config[:dest_prefix]}/#{File.basename(backup.archive)} (#{bytes_to_human(File.size(backup.archive))})"
  upload(backup.archive, config[:bucket], "#{config[:dest_prefix]}/#{File.basename(backup.archive)}")
  puts "Uploading snar (#{bytes_to_human(File.size(backup.snar_path))})"
  upload(backup.snar_path, config[:bucket], "#{config[:dest_prefix]}/#{File.basename(backup.snar)}")
  File.delete(backup.archive)
end

#backup_full(config, verbose = false) ⇒ Object 



182
183
184
185
186
187
188
 
# File 'lib/s3_tar_backup.rb', line 182

def backup_full(config, verbose=false)
  puts "Starting new full backup"
  backup = Backup.new(config[:backup_dir], config[:name], config[:sources], config[:exclude], config[:compression], config[:gpg_key])
  # Nuke the snar file -- forces a full backup
  File.delete(backup.snar_path) if File.exists?(backup.snar_path)
  backup(config, backup, verbose)
end

#backup_incr(config, verbose = false) ⇒ Object

Config should have the keys backup_dir, name, soruces, exclude, bucket, dest_prefix



158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
 
# File 'lib/s3_tar_backup.rb', line 158

def backup_incr(config, verbose=false)
  puts "Starting new incremental backup"
  backup = Backup.new(config[:backup_dir], config[:name], config[:sources], config[:exclude], config[:compression], config[:gpg_key])

  # Try and get hold of the snar file
  unless backup.snar_exists?
    puts "Failed to find snar file. Attempting to download..."
    s3_snar = "#{config[:dest_prefix]}/#{backup.snar}"
    object = @s3.buckets[config[:bucket]].objects[s3_snar]
    if object.exists?
      puts "Found file on S3. Downloading"
      open(backup.snar_path, 'wb') do |f|
        object.read do |chunk|
          f.write(chunk)
        end
      end
    else
      puts "Failed to download snar file. Defaulting to full backup"
    end
  end

  backup(config, backup, verbose)
end

#bytes_to_human(n) ⇒ Object 



305
306
307
308
309
310
311
312
 
# File 'lib/s3_tar_backup.rb', line 305

def bytes_to_human(n)
  count = 0
  while n >= 1014 && count < 4
    n /= 1024.0
    count += 1
  end
  format("%.2f", n) << %w(B KB MB GB TB)[count]
end

#connect_s3(access_key, secret_key, region) ⇒ Object 



80
81
82
83
 
# File 'lib/s3_tar_backup.rb', line 80

def connect_s3(access_key, secret_key, region)
  warn "No AWS region specified (config key settings.s3_region). Assuming eu-west-1" unless region
  AWS::S3.new(access_key_id: access_key, secret_access_key: secret_key, region: region || 'eu-west-1')
end

#exec(cmd) ⇒ Object 



314
315
316
317
 
# File 'lib/s3_tar_backup.rb', line 314

def exec(cmd)
  puts "Executing: #{cmd}"
  system(cmd)
end

#full_required?(interval_str, objects) ⇒ Boolean

Returns:

  • (Boolean) 


300
301
302
303
 
# File 'lib/s3_tar_backup.rb', line 300

def full_required?(interval_str, objects)
  time = parse_interval(interval_str)
  objects.select{ |o| o[:type] == :full && o[:date] > time }.empty?
end

#gen_backup_config(profile, config) ⇒ Object 



85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
 
# File 'lib/s3_tar_backup.rb', line 85

def gen_backup_config(profile, config)
  bucket, dest_prefix = (config.get("profile.#{profile}.dest", false) || config['settings.dest']).split('/', 2)
  gpg_key = config.get("profile.#{profile}.gpg_key", false) || config['settings.gpg_key']
  backup_config = {
    :backup_dir => config.get("profile.#{profile}.backup_dir", false) || config['settings.backup_dir'],
    :name => profile,
    :gpg_key => gpg_key && !gpg_key.empty? ? gpg_key : nil,
    :sources => [*config.get("profile.#{profile}.source", [])] + [*config.get("settings.source", [])],
    :exclude => [*config.get("profile.#{profile}.exclude", [])] + [*config.get("settings.exclude", [])],
    :bucket => bucket,
    :dest_prefix => dest_prefix.chomp('/'),
    :pre_backup => [*config.get("profile.#{profile}.pre-backup", [])] + [*config.get('settings.pre-backup', [])],
    :post_backup => [*config.get("profile.#{profile}.post-backup", [])] + [*config.get('settings.post-backup', [])],
    :full_if_older_than => config.get("profile.#{profile}.full_if_older_than", false) || config['settings.full_if_older_than'],
    :remove_older_than => config.get("profile.#{profile}.remove_older_than", false) || config.get('settings.remove_older_than', false),
    :remove_all_but_n_full => config.get("profile.#{profile}.remove_all_but_n_full", false) || config.get('settings.remove_all_but_n_full', false),
    :compression => (config.get("profile.#{profile}.compression", false) || config.get('settings.compression', 'bzip2')).to_sym,
    :always_full => config.get('settings.always_full', false) || config.get("profile.#{profile}.always_full", false),
  }
  backup_config
end

#get_objects(bucket, prefix, profile) ⇒ Object 



281
282
283
284
285
286
 
# File 'lib/s3_tar_backup.rb', line 281

def get_objects(bucket, prefix, profile)
  objects = @s3.buckets[bucket].objects.with_prefix(prefix).map do |object|
    Backup.parse_object(object, profile)
  end
  objects.compact.sort_by{ |o| o[:date] }
end

#parse_interval(interval_str) ⇒ Object 



288
289
290
291
292
293
294
295
296
297
298
 
# File 'lib/s3_tar_backup.rb', line 288

def parse_interval(interval_str)
  time = Time.now
  time -= $1.to_i if interval_str =~ /(\d+)s/
  time -= $1.to_i*60 if interval_str =~ /(\d+)m/
  time -= $1.to_i*3600 if interval_str =~ /(\d+)h/
  time -= $1.to_i*86400 if interval_str =~ /(\d+)D/
  time -= $1.to_i*604800 if interval_str =~ /(\d+)W/
  time -= $1.to_i*2592000 if interval_str =~ /(\d+)M/
  time -= $1.to_i*31536000 if interval_str =~ /(\d+)Y/
  time
end

#perform_backup(opts, prev_backups, backup_config) ⇒ Object 



107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
 
# File 'lib/s3_tar_backup.rb', line 107

def perform_backup(opts, prev_backups, backup_config)
  puts "===== Backing up profile #{backup_config[:name]} ====="
  backup_config[:pre_backup].each_with_index do |cmd, i|
    puts "Executing pre-backup hook #{i+1}"
    exec(cmd)
  end
  full_required = full_required?(backup_config[:full_if_older_than], prev_backups)
  puts "Last full backup is too old. Forcing a full backup" if full_required && !opts[:full] && backup_config[:always_full]
  if full_required || opts[:full] || backup_config[:always_full]
    backup_full(backup_config, opts[:verbose])
  else
    backup_incr(backup_config, opts[:verbose])
  end
  backup_config[:post_backup].each_with_index do |cmd, i|
    puts "Executing post-backup hook #{i+1}"
    exec(cmd)
  end
end

#perform_cleanup(prev_backups, backup_config) ⇒ Object 



126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
 
# File 'lib/s3_tar_backup.rb', line 126

def perform_cleanup(prev_backups, backup_config)
  puts "===== Cleaning up profile #{backup_config[:name]} ====="
  remove = []
  if age_str = backup_config[:remove_older_than]
    age = parse_interval(age_str)
    remove = prev_backups.select{ |o| o[:date] < age }
    # Don't want to delete anything before the last full backup
    unless remove.empty?
      kept = remove.slice!(remove.rindex{ |o| o[:type] == :full }..-1).count
      puts "Keeping #{kept} old backups as part of a chain" if kept > 1
    end
  elsif keep_n = backup_config[:remove_all_but_n_full]
    keep_n = keep_n.to_i
    # Get the date of the last full backup to keep
    if last_full_to_keep = prev_backups.select{ |o| o[:type] == :full }[-keep_n]
      # If there is a last full one...
      remove = prev_backups.select{ |o| o[:date] < last_full_to_keep[:date] }
    end
  end

  if remove.empty?
    puts "Nothing to do"
  else
    puts "Removing #{remove.count} old backup files"
  end
  remove.each do |object|
    @s3.buckets[backup_config[:bucket]].objects["#{backup_config[:dest_prefix]}/#{object[:name]}"].delete
  end
end

#perform_list_backups(prev_backups, backup_config) ⇒ Object 



254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
 
# File 'lib/s3_tar_backup.rb', line 254

def perform_list_backups(prev_backups, backup_config)
  # prev_backups alreays contains just the files for the current profile
  puts "===== Backups list for #{backup_config[:name]} ====="
  puts "Type: N:  Date:#{' '*18}Size:       Chain Size:   Format:   Encrypted:\n\n"
  prev_type = ''
  total_size = 0
  chain_length = 0
  chain_cum_size = 0
  prev_backups.each do |object|
    type = object[:type] == prev_type && object[:type] == :incr ? " -- " : object[:type].to_s.capitalize
    prev_type = object[:type]
    chain_length += 1
    chain_length = 0 if object[:type] == :full
    chain_cum_size = 0 if object[:type] == :full
    chain_cum_size += object[:size]

    chain_length_str = (chain_length == 0 ? '' : chain_length.to_s).ljust(3)
    chain_cum_size_str = (object[:type] == :full ? '' : bytes_to_human(chain_cum_size)).ljust(8)
    puts "#{type}  #{chain_length_str} #{object[:date].strftime('%F %T')}    #{bytes_to_human(object[:size]).ljust(8)}    " \
      "#{chain_cum_size_str}      #{object[:compression].to_s.ljust(7)}   #{object[:encryption] ? 'Y' : 'N'}"
    total_size += object[:size]
  end
  puts "\n"
  puts "Total size: #{bytes_to_human(total_size)}"
  puts "\n"
end

#perform_restore(opts, prev_backups, backup_config) ⇒ Object 



216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
 
# File 'lib/s3_tar_backup.rb', line 216

def perform_restore(opts, prev_backups, backup_config)
  puts "===== Restoring profile #{backup_config[:name]} ====="
  # If restore date given, parse
  if opts[:restore_date_given]
    m = opts[:restore_date].match(/(\d\d\d\d)(\d\d)(\d\d)?(\d\d)?(\d\d)?(\d\d)?/)
    raise "Unknown date format in --restore-to" if m.nil?
    restore_to = Time.new(*m[1..-1].map{ |s| s.to_i if s })
  else
    restore_to = Time.now
  end

  # Find the index of the first backup, incremental or full, before that date
  restore_end_index = prev_backups.rindex{ |o| o[:date] < restore_to }
  raise "Failed to find a backup for that date" unless restore_end_index

  # Find the first full backup before that one
  restore_start_index = prev_backups[0..restore_end_index].rindex{ |o| o[:type] == :full }

  restore_dir = opts[:restore].chomp('/') << '/'

  Dir.mkdir(restore_dir) unless Dir.exists?(restore_dir)
  raise "Destination dir is not a directory" unless File.directory?(restore_dir)

  prev_backups[restore_start_index..restore_end_index].each do |object|
    puts "Fetching #{backup_config[:bucket]}/#{backup_config[:dest_prefix]}/#{object[:name]} (#{bytes_to_human(object[:size])})"
    dl_file = "#{backup_config[:backup_dir]}/#{object[:name]}"
    open(dl_file, 'wb') do |f|
      @s3.buckets[backup_config[:bucket]].objects["#{backup_config[:dest_prefix]}/#{object[:name]}"].read do |chunk|
        f.write(chunk)
      end
    end
    puts "Extracting..."
    exec(Backup.restore_cmd(restore_dir, dl_file, opts[:verbose]))

    File.delete(dl_file)
  end
end

#run ⇒ Object 



11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
 
# File 'lib/s3_tar_backup.rb', line 11

def run
  opts = Trollop::options do
    version VERSION
    banner "Backs up files to, and restores files from, Amazon's S3 storage, using tar incremental backups\n\n" \
      "Usage:\ns3-tar-backup -c config.ini [-p profile] --backup [--full] [-v]\n" \
      "s3-tar-backup -c config.ini [-p profile] --cleanup [-v]\n" \
      "s3-tar-backup -c config.ini [-p profile] --restore restore_dir\n\t[--restore_date date] [-v]\n" \
      "s3-tar-backup -c config.ini [-p profile] --backup-config [--verbose]\n" \
      "s3-tar-backup -c config.ini [-p profile] --list-backups\n\n" \
      "Option details:\n"
    opt :config, "Configuration file", :short => 'c', :type => :string, :required => true
    opt :backup, "Make an incremental backup"
    opt :full, "Make the backup a full backup"
    opt :profile, "The backup profile(s) to use (default all)", :short => 'p', :type => :strings
    opt :cleanup, "Clean up old backups"
    opt :restore, "Restore a backup to the specified dir", :type => :string
    opt :restore_date, "Restore a backup from the specified date. Format YYYYMM[DD[hh[mm[ss]]]]", :type => :string
    opt :backup_config, "Backs up the specified configuration file"
    opt :list_backups, "List the stored backup info for one or more profiles"
    opt :verbose, "Show verbose output", :short => 'v'
    conflicts :backup, :cleanup, :restore, :backup_config, :list_backups
  end


  Trollop::die "--full requires --backup" if opts[:full] && !opts[:backup]
  Trollop::die "--restore-date requires --restore" if opts[:restore_date_given] && !opts[:restore_given]
  unless opts[:backup] || opts[:cleanup] || opts[:restore_given] || opts[:backup_config] || opts[:list_backups]
    Trollop::die "Need one of --backup, --cleanup, --restore, --backup-config, --list-backups"
  end

  begin
    raise "Config file #{opts[:config]} not found" unless File.exists?(opts[:config])
    config = IniParser.new(opts[:config]).load
    profiles = opts[:profile] || config.find_sections(/^profile\./).keys.map{ |k| k.to_s.split('.', 2)[1] }
    @s3 = connect_s3(
      ENV['AWS_ACCESS_KEY_ID'] || config['settings.aws_access_key_id'],
      ENV['AWS_SECRET_ACCESS_KEY'] || config['settings.aws_secret_access_key'],
      config.get('settings.aws_region', false)
    )

    # This is a bit of a special case
    if opts[:backup_config]
      dest = config.get('settings.dest', false)
      raise "You must specify a single profile (used to determine the location to back up to) " \
        "if backing up config and dest key is not in [settings]" if !dest && profiles.count != 1
      dest ||= config["profile.#{profiles[0]}.dest"]
      puts "===== Backing up config file #{opts[:config]} ====="
      bucket, prefix = (config.get('settings.dest', false) || config["profile.#{profiles[0]}.dest"]).split('/', 2)
      puts "Uploading #{opts[:config]} to #{bucket}/#{prefix}/#{File.basename(opts[:config])}"
      upload(opts[:config], bucket, "#{prefix}/#{File.basename(opts[:config])}")
      return
    end

    profiles.dup.each do |profile|
      raise "No such profile: #{profile}" unless config.has_section?("profile.#{profile}")
      opts[:profile] = profile
      backup_config = gen_backup_config(opts[:profile], config)
      prev_backups = get_objects(backup_config[:bucket], backup_config[:dest_prefix], opts[:profile])
      perform_backup(opts, prev_backups, backup_config) if opts[:backup]
      perform_cleanup(prev_backups, backup_config) if opts[:backup] || opts[:cleanup]
      perform_restore(opts, prev_backups, backup_config) if opts[:restore_given]
      perform_list_backups(prev_backups, backup_config) if opts[:list_backups]
    end
  rescue Exception => e
    raise e
    Trollop::die e.to_s
  end
end

#upload(source, bucket, dest_name) ⇒ Object 



199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
 
# File 'lib/s3_tar_backup.rb', line 199

def upload(source, bucket, dest_name)
  tries = 0
  begin
    @s3.buckets[bucket].objects.create(dest_name, Pathname.new(source))
  rescue Errno::ECONNRESET => e
    tries += 1
    if tries <= UPLOAD_TRIES
      puts "Upload Exception: #{e}"
      puts "Retrying #{tries}/#{UPLOAD_TRIES}..."
      retry
    else
      raise e
    end
  end
  puts "Succeeded" if tries > 0
end