Class: RedSnapper

Inherits:
Object
  • Object
show all
Defined in:
lib/redsnapper.rb

Defined Under Namespace

Classes: Group

Constant Summary collapse

TARSNAP = 
'tarsnap'
THREAD_POOL_DEFAULT_SIZE = 
10
EXIT_ERROR = 
"tarsnap: Error exit delayed from previous errors.\n"
NOT_OLDER_ERROR = 
"File on disk is not older; skipping.\n"
@@output_mutex = 
Mutex.new

Instance Method Summary collapse

Constructor Details

#initialize(archive, options = {}) ⇒ RedSnapper

Returns a new instance of RedSnapper.



30
31
32
33
34
35
 
# File 'lib/redsnapper.rb', line 30

def initialize(archive, options = {})
  @archive = archive
  @options = options
  @thread_pool = Thread.pool(options[:thread_pool_size] || THREAD_POOL_DEFAULT_SIZE)
  @error = false
end

Instance Method Details

#empty_dirs(files, dirs) ⇒ Object 



62
63
64
65
66
67
68
69
70
71
72
 
# File 'lib/redsnapper.rb', line 62

def empty_dirs(files, dirs)
  empty_dirs = dirs.clone
  files.each { |f| empty_dirs.delete(File.dirname(f) + '/') }
  dirs.each do |dir|
    components = dir.split('/')[0..-2]
    components.each_with_index do |_, i|
      empty_dirs.delete(components[0, i + 1].join('/') + '/')
    end
  end
  empty_dirs
end

#file_groups ⇒ Object 



82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
 
# File 'lib/redsnapper.rb', line 82

def file_groups
  groups = (1..@thread_pool.max).map { Group.new }
  files_to_extract.sort { |a, b| b.last[:size] <=> a.last[:size] }.each do |name, props|

    # If the previous batch of files had an entry with the same size and date,
    # assume that this is a duplicate and assign it zero weight.  There may be
    # some false positives here since the granularity of the data we have from
    # tarsnap is only "same day".  However, a false positive just affects the
    # queing scheme, not which files get queued.

    size = (@options[:previous] && @options[:previous][name] == props) ? 0 : props[:size]
    groups.sort.last.add(name, size)
  end
  groups.map(&:files).reject(&:empty?)
end

#files ⇒ Object 



37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
 
# File 'lib/redsnapper.rb', line 37

def files
  return @files if @files

  command = [ TARSNAP, '-tvf', @archive, *@options[:tarsnap_options] ]
  command.push(@options[:directory]) if @options[:directory]

  @files = {}

  Open3.popen3(*command) do |_, out, _|
    out.gets(nil).split("\n").each do |entry|
      (_, _, _, _, size, month, day, year_or_time, name) = entry.split(/\s+/, 9)

      date = DateTime.parse("#{month} #{day}, #{year_or_time}")
      date = date.prev_year if date < DateTime.now

      @files[name] = {
        :size => size.to_i,
        :date => date
      }
    end
  end

  @files
end

#files_to_extract ⇒ Object 



74
75
76
77
78
79
80
 
# File 'lib/redsnapper.rb', line 74

def files_to_extract
  files_to_extract, dirs = files.partition { |f| !f.first.end_with?('/') }.map(&:to_h)
  empty_dirs(files_to_extract.keys, dirs.keys).each do |dir|
    files_to_extract[dir] = { :size => 0 }
  end
  files_to_extract
end

#run ⇒ Object 



98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
 
# File 'lib/redsnapper.rb', line 98

def run
  file_groups.each do |chunk|
    @thread_pool.process do
      command = [ TARSNAP, '-xvf', @archive, *(@options[:tarsnap_options] + chunk) ]
      Open3.popen3(*command) do |_, _, err|
        while line = err.gets
          next if line.end_with?(NOT_OLDER_ERROR)
          if line == EXIT_ERROR
            @error = true
            next
          end
          @@output_mutex.synchronize { warn line.chomp }
        end
      end
    end
  end

  @thread_pool.shutdown
  @@output_mutex.synchronize { warn EXIT_ERROR } if @error
end