Class: S3TarBackup::Main

Inherits:
Object
  • Object
show all
Defined in:
lib/s3_tar_backup.rb

Constant Summary collapse

UPLOAD_TRIES = 
5

Instance Method Summary collapse

Instance Method Details

#backup(config, backup, verbose = false) ⇒ Object 



190
191
192
193
194
195
196
197
 
# File 'lib/s3_tar_backup.rb', line 190

def backup(config, backup, verbose=false)
	exec(backup.backup_cmd(verbose))
	puts "Uploading #{config[:bucket]}/#{config[:dest_prefix]}/#{File.basename(backup.archive)} (#{bytes_to_human(File.size(backup.archive))})"
	upload(backup.archive, config[:bucket], "#{config[:dest_prefix]}/#{File.basename(backup.archive)}")
	puts "Uploading snar (#{bytes_to_human(File.size(backup.snar_path))})"
	upload(backup.snar_path, config[:bucket], "#{config[:dest_prefix]}/#{File.basename(backup.snar)}")
	File.delete(backup.archive)
end

#backup_full(config, verbose = false) ⇒ Object 



182
183
184
185
186
187
188
 
# File 'lib/s3_tar_backup.rb', line 182

def backup_full(config, verbose=false)
	puts "Starting new full backup"
	backup = Backup.new(config[:backup_dir], config[:name], config[:sources], config[:exclude], config[:compression], config[:gpg_key])
	# Nuke the snar file -- forces a full backup
	File.delete(backup.snar_path) if File.exists?(backup.snar_path)
	backup(config, backup, verbose)
end

#backup_incr(config, verbose = false) ⇒ Object

Config should have the keys backup_dir, name, soruces, exclude, bucket, dest_prefix



158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
 
# File 'lib/s3_tar_backup.rb', line 158

def backup_incr(config, verbose=false)
	puts "Starting new incremental backup"
	backup = Backup.new(config[:backup_dir], config[:name], config[:sources], config[:exclude], config[:compression], config[:gpg_key])

	# Try and get hold of the snar file
	unless backup.snar_exists?
		puts "Failed to find snar file. Attempting to download..."
		s3_snar = "#{config[:dest_prefix]}/#{backup.snar}"
		object = @s3.buckets[config[:bucket]].objects[s3_snar]
		if object.exists?
			puts "Found file on S3. Downloading"
			open(backup.snar_path, 'wb') do |f|
				object.read do |chunk|
					f.write(chunk)
				end
			end
		else
			puts "Failed to download snar file. Defaulting to full backup"
		end
	end

	backup(config, backup, verbose)
end

#bytes_to_human(n) ⇒ Object 



305
306
307
308
309
310
311
312
 
# File 'lib/s3_tar_backup.rb', line 305

def bytes_to_human(n)
	count = 0
	while n >= 1014 && count < 4
		n /= 1024.0
		count += 1
	end
	format("%.2f", n) << %w(B KB MB GB TB)[count]
end

#connect_s3(access_key, secret_key, region) ⇒ Object 



80
81
82
83
 
# File 'lib/s3_tar_backup.rb', line 80

def connect_s3(access_key, secret_key, region)
	warn "No AWS region specified (config key settings.s3_region). Assuming eu-west-1" unless region
	AWS::S3.new(access_key_id: access_key, secret_access_key: secret_key, region: region || 'eu-west-1')
end

#exec(cmd) ⇒ Object 



314
315
316
317
 
# File 'lib/s3_tar_backup.rb', line 314

def exec(cmd)
	puts "Executing: #{cmd}"
	system(cmd)
end

#full_required?(interval_str, objects) ⇒ Boolean

Returns:

  • (Boolean) 


300
301
302
303
 
# File 'lib/s3_tar_backup.rb', line 300

def full_required?(interval_str, objects)
	time = parse_interval(interval_str)
	objects.select{ |o| o[:type] == :full && o[:date] > time }.empty?
end

#gen_backup_config(profile, config) ⇒ Object 



85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
 
# File 'lib/s3_tar_backup.rb', line 85

def gen_backup_config(profile, config)
	bucket, dest_prefix = (config.get("profile.#{profile}.dest", false) || config['settings.dest']).split('/', 2)
	gpg_key = config.get("profile.#{profile}.gpg_key", false) || config['settings.gpg_key']
	backup_config = {
		:backup_dir => config.get("profile.#{profile}.backup_dir", false) || config['settings.backup_dir'],
		:name => profile,
		:gpg_key => gpg_key && !gpg_key.empty? ? gpg_key : nil,
		:sources => [*config.get("profile.#{profile}.source", [])] + [*config.get("settings.source", [])],
		:exclude => [*config.get("profile.#{profile}.exclude", [])] + [*config.get("settings.exclude", [])],
		:bucket => bucket,
		:dest_prefix => dest_prefix.chomp('/'),
		:pre_backup => [*config.get("profile.#{profile}.pre-backup", [])] + [*config.get('settings.pre-backup', [])],
		:post_backup => [*config.get("profile.#{profile}.post-backup", [])] + [*config.get('settings.post-backup', [])],
		:full_if_older_than => config.get("profile.#{profile}.full_if_older_than", false) || config['settings.full_if_older_than'],
		:remove_older_than => config.get("profile.#{profile}.remove_older_than", false) || config.get('settings.remove_older_than', false),
		:remove_all_but_n_full => config.get("profile.#{profile}.remove_all_but_n_full", false) || config.get('settings.remove_all_but_n_full', false),
		:compression => (config.get("profile.#{profile}.compression", false) || config.get('settings.compression', 'bzip2')).to_sym,
		:always_full => config.get('settings.always_full', false) || config.get("profile.#{profile}.always_full", false),
	}
	backup_config
end

#get_objects(bucket, prefix, profile) ⇒ Object 



281
282
283
284
285
286
 
# File 'lib/s3_tar_backup.rb', line 281

def get_objects(bucket, prefix, profile)
	objects = @s3.buckets[bucket].objects.with_prefix(prefix).map do |object|
		Backup.parse_object(object, profile)
	end
	objects.compact.sort_by{ |o| o[:date] }
end

#parse_interval(interval_str) ⇒ Object 



288
289
290
291
292
293
294
295
296
297
298
 
# File 'lib/s3_tar_backup.rb', line 288

def parse_interval(interval_str)
	time = Time.now
	time -= $1.to_i if interval_str =~ /(\d+)s/
	time -= $1.to_i*60 if interval_str =~ /(\d+)m/
	time -= $1.to_i*3600 if interval_str =~ /(\d+)h/
	time -= $1.to_i*86400 if interval_str =~ /(\d+)D/
	time -= $1.to_i*604800 if interval_str =~ /(\d+)W/
	time -= $1.to_i*2592000 if interval_str =~ /(\d+)M/
	time -= $1.to_i*31536000 if interval_str =~ /(\d+)Y/
	time
end

#perform_backup(opts, prev_backups, backup_config) ⇒ Object 



107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
 
# File 'lib/s3_tar_backup.rb', line 107

def perform_backup(opts, prev_backups, backup_config)
	puts "===== Backing up profile #{backup_config[:name]} ====="
	backup_config[:pre_backup].each_with_index do |cmd, i|
		puts "Executing pre-backup hook #{i+1}"
		exec(cmd)
	end
	full_required = full_required?(backup_config[:full_if_older_than], prev_backups)
	puts "Last full backup is too old. Forcing a full backup" if full_required && !opts[:full] && backup_config[:always_full]
	if full_required || opts[:full] || backup_config[:always_full]
		backup_full(backup_config, opts[:verbose])
	else
		backup_incr(backup_config, opts[:verbose])
	end
	backup_config[:post_backup].each_with_index do |cmd, i|
		puts "Executing post-backup hook #{i+1}"
		exec(cmd)
	end
end

#perform_cleanup(prev_backups, backup_config) ⇒ Object 



126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
 
# File 'lib/s3_tar_backup.rb', line 126

def perform_cleanup(prev_backups, backup_config)
	puts "===== Cleaning up profile #{backup_config[:name]} ====="
	remove = []
	if age_str = backup_config[:remove_older_than]
		age = parse_interval(age_str)
		remove = prev_backups.select{ |o| o[:date] < age }
		# Don't want to delete anything before the last full backup
		unless remove.empty?
			kept = remove.slice!(remove.rindex{ |o| o[:type] == :full }..-1).count
			puts "Keeping #{kept} old backups as part of a chain" if kept > 1
		end
	elsif keep_n = backup_config[:remove_all_but_n_full]
		keep_n = keep_n.to_i
		# Get the date of the last full backup to keep
		if last_full_to_keep = prev_backups.select{ |o| o[:type] == :full }[-keep_n]
			# If there is a last full one...
			remove = prev_backups.select{ |o| o[:date] < last_full_to_keep[:date] }
		end
	end

	if remove.empty?
		puts "Nothing to do"
	else
		puts "Removing #{remove.count} old backup files"
	end
	remove.each do |object|
		@s3.buckets[backup_config[:bucket]].objects["#{backup_config[:dest_prefix]}/#{object[:name]}"].delete
	end
end

#perform_list_backups(prev_backups, backup_config) ⇒ Object 



254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
 
# File 'lib/s3_tar_backup.rb', line 254

def perform_list_backups(prev_backups, backup_config)
	# prev_backups alreays contains just the files for the current profile
	puts "===== Backups list for #{backup_config[:name]} ====="
	puts "Type: N:  Date:#{' '*18}Size:       Chain Size:   Format:   Encrypted:\n\n"
	prev_type = ''
	total_size = 0
	chain_length = 0
	chain_cum_size = 0
	prev_backups.each do |object|
		type = object[:type] == prev_type && object[:type] == :incr ? " -- " : object[:type].to_s.capitalize
		prev_type = object[:type]
		chain_length += 1
		chain_length = 0 if object[:type] == :full
		chain_cum_size = 0 if object[:type] == :full
		chain_cum_size += object[:size]

		chain_length_str = (chain_length == 0 ? '' : chain_length.to_s).ljust(3)
		chain_cum_size_str = (object[:type] == :full ? '' : bytes_to_human(chain_cum_size)).ljust(8)
		puts "#{type}  #{chain_length_str} #{object[:date].strftime('%F %T')}    #{bytes_to_human(object[:size]).ljust(8)}    " \
			"#{chain_cum_size_str}      #{object[:compression].to_s.ljust(7)}   #{object[:encryption] ? 'Y' : 'N'}"
		total_size += object[:size]
	end
	puts "\n"
	puts "Total size: #{bytes_to_human(total_size)}"
	puts "\n"
end

#perform_restore(opts, prev_backups, backup_config) ⇒ Object 



216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
 
# File 'lib/s3_tar_backup.rb', line 216

def perform_restore(opts, prev_backups, backup_config)
	puts "===== Restoring profile #{backup_config[:name]} ====="
	# If restore date given, parse
	if opts[:restore_date_given]
		m = opts[:restore_date].match(/(\d\d\d\d)(\d\d)(\d\d)?(\d\d)?(\d\d)?(\d\d)?/)
		raise "Unknown date format in --restore-to" if m.nil?
		restore_to = Time.new(*m[1..-1].map{ |s| s.to_i if s })
	else
		restore_to = Time.now
	end

	# Find the index of the first backup, incremental or full, before that date
	restore_end_index = prev_backups.rindex{ |o| o[:date] < restore_to }
	raise "Failed to find a backup for that date" unless restore_end_index

	# Find the first full backup before that one
	restore_start_index = prev_backups[0..restore_end_index].rindex{ |o| o[:type] == :full }

	restore_dir = opts[:restore].chomp('/') << '/'

	Dir.mkdir(restore_dir) unless Dir.exists?(restore_dir)
	raise "Destination dir is not a directory" unless File.directory?(restore_dir)

	prev_backups[restore_start_index..restore_end_index].each do |object|
		puts "Fetching #{backup_config[:bucket]}/#{backup_config[:dest_prefix]}/#{object[:name]} (#{bytes_to_human(object[:size])})"
		dl_file = "#{backup_config[:backup_dir]}/#{object[:name]}"
		open(dl_file, 'wb') do |f|
			@s3.buckets[backup_config[:bucket]].objects["#{backup_config[:dest_prefix]}/#{object[:name]}"].read do |chunk|
				f.write(chunk)
			end
		end
		puts "Extracting..."
		exec(Backup.restore_cmd(restore_dir, dl_file, opts[:verbose]))

		File.delete(dl_file)
	end
end

#run ⇒ Object 



11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
 
# File 'lib/s3_tar_backup.rb', line 11

def run
	opts = Trollop::options do
		version VERSION
		banner "Backs up files to, and restores files from, Amazon's S3 storage, using tar incremental backups\n\n" \
			"Usage:\ns3-tar-backup -c config.ini [-p profile] --backup [--full] [-v]\n" \
			"s3-tar-backup -c config.ini [-p profile] --cleanup [-v]\n" \
			"s3-tar-backup -c config.ini [-p profile] --restore restore_dir\n\t[--restore_date date] [-v]\n" \
			"s3-tar-backup -c config.ini [-p profile] --backup-config [--verbose]\n" \
			"s3-tar-backup -c config.ini [-p profile] --list-backups\n\n" \
			"Option details:\n"
		opt :config, "Configuration file", :short => 'c', :type => :string, :required => true
		opt :backup, "Make an incremental backup"
		opt :full, "Make the backup a full backup"
		opt :profile, "The backup profile(s) to use (default all)", :short => 'p', :type => :strings
		opt :cleanup, "Clean up old backups"
		opt :restore, "Restore a backup to the specified dir", :type => :string
		opt :restore_date, "Restore a backup from the specified date. Format YYYYMM[DD[hh[mm[ss]]]]", :type => :string
		opt :backup_config, "Backs up the specified configuration file"
		opt :list_backups, "List the stored backup info for one or more profiles"
		opt :verbose, "Show verbose output", :short => 'v'
		conflicts :backup, :cleanup, :restore, :backup_config, :list_backups
	end


	Trollop::die "--full requires --backup" if opts[:full] && !opts[:backup]
	Trollop::die "--restore-date requires --restore" if opts[:restore_date_given] && !opts[:restore_given]
	unless opts[:backup] || opts[:cleanup] || opts[:restore_given] || opts[:backup_config] || opts[:list_backups]
		Trollop::die "Need one of --backup, --cleanup, --restore, --backup-config, --list-backups"
	end

	begin
		raise "Config file #{opts[:config]} not found" unless File.exists?(opts[:config])
		config = IniParser.new(opts[:config]).load
		profiles = opts[:profile] || config.find_sections(/^profile\./).keys.map{ |k| k.to_s.split('.', 2)[1] }
		@s3 = connect_s3(
			ENV['AWS_ACCESS_KEY_ID'] || config['settings.aws_access_key_id'],
			ENV['AWS_SECRET_ACCESS_KEY'] || config['settings.aws_secret_access_key'],
			config.get('settings.aws_region', false)
		)

		# This is a bit of a special case
		if opts[:backup_config]
			dest = config.get('settings.dest', false)
			raise "You must specify a single profile (used to determine the location to back up to) " \
				"if backing up config and dest key is not in [settings]" if !dest && profiles.count != 1
			dest ||= config["profile.#{profiles[0]}.dest"]
			puts "===== Backing up config file #{opts[:config]} ====="
			bucket, prefix = (config.get('settings.dest', false) || config["profile.#{profiles[0]}.dest"]).split('/', 2)
			puts "Uploading #{opts[:config]} to #{bucket}/#{prefix}/#{File.basename(opts[:config])}"
			upload(opts[:config], bucket, "#{prefix}/#{File.basename(opts[:config])}")
			return
		end

		profiles.dup.each do |profile|
			raise "No such profile: #{profile}" unless config.has_section?("profile.#{profile}")
			opts[:profile] = profile
			backup_config = gen_backup_config(opts[:profile], config)
			prev_backups = get_objects(backup_config[:bucket], backup_config[:dest_prefix], opts[:profile])
			perform_backup(opts, prev_backups, backup_config) if opts[:backup]
			perform_cleanup(prev_backups, backup_config) if opts[:backup] || opts[:cleanup]
			perform_restore(opts, prev_backups, backup_config) if opts[:restore_given]
			perform_list_backups(prev_backups, backup_config) if opts[:list_backups]
		end
	rescue Exception => e
		raise e
		Trollop::die e.to_s
	end
end

#upload(source, bucket, dest_name) ⇒ Object 



199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
 
# File 'lib/s3_tar_backup.rb', line 199

def upload(source, bucket, dest_name)
	tries = 0
	begin
		@s3.buckets[bucket].objects.create(dest_name, Pathname.new(source))
	rescue Errno::ECONNRESET => e
		tries += 1
		if tries <= UPLOAD_TRIES
			puts "Upload Exception: #{e}"
			puts "Retrying #{tries}/#{UPLOAD_TRIES}..."
			retry
		else
			raise e
		end
	end
	puts "Succeeded" if tries > 0
end