Module: S3CP

Extended by:
S3CP
Included in:
S3CP
Defined in:
lib/s3cp/version.rb,
lib/s3cp/utils.rb

Overview

Copyright © 2010-2012 Alex Boisvert and Bizo Inc. / All rights reserved.

Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version 2.0 (the “License”); you may not use this file except in compliance with the License. You may obtain a copy of the License at

http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an “AS IS” BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License.

Constant Summary collapse

UNITS =

Valid units for file size formatting

%w{B KB MB GB TB EB ZB YB BB}

%w{
  private
  public-read
  public-read-write
  authenticated-read
  bucket_owner_read
  bucket_owner_full_control
}
VERSION = 
"1.1.42"

Instance Method Summary collapse

Instance Method Details

#bucket_and_key(url) ⇒ Object

Parse URL and return bucket and key.

e.g. s3://bucket/path/to/key => [“bucket”, “path/to/key”]

bucket:path/to/key => ["bucket", "path/to/key"]



70
71
72
73
74
75
76
77
78
79
 
# File 'lib/s3cp/utils.rb', line 70

def bucket_and_key(url)
  if url =~ /s3:\/\/([^\/]+)\/?(.*)/
    bucket = $1
    key = $2
  elsif url =~ /([^:]+):(.*)/
    bucket = $1
    key = $2
  end
  [bucket, key]
end

#connect ⇒ Object

Connect to AWS S3



43
44
45
46
47
48
49
50
51
52
53
54
55
 
# File 'lib/s3cp/utils.rb', line 43

def connect()
  options = {}

  # optional region override
  region = ENV["S3CP_REGION"]
  options[:s3_endpoint] = "s3-#{region}.amazonaws.com" if region && region != "us-east-1"

  # optional endpoint override
  endpoint = ENV["S3CP_ENDPOINT"]
  options[:s3_endpoint] = endpoint if endpoint

  ::AWS::S3.new(options)
end

#format_filesize(num, options = {}) ⇒ Object

Return a formatted string for a file size.

Valid units are “b” (bytes), “kb” (kilobytes), “mb” (megabytes), “gb” (gigabytes), “tb” (terabytes), “eb” (exabytes), “zb” (zettabytes), “yb” (yottabytes), “bb” (brontobytes) and their uppercase equivalents.

If :unit option isn’t specified, the “best” unit is automatically picked. If :precision option isn’t specified, the number is rounded to closest integer.

e.g. format_filesize( 512, :unit => “b”, :precision => 2) => “512B”

format_filesize( 512, :unit => "kb", :precision => 4) =>   "0.5KB"
format_filesize(1512, :unit => "kb", :precision => 3) => "1.477KB"

format_filesize(11789512) => "11MB"  # smart unit selection

format_filesize(11789512, :precision => 2) => "11.24MB"



148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
 
# File 'lib/s3cp/utils.rb', line 148

def format_filesize(num, options = {})
  precision = options[:precision] || 0
  if options[:unit]
    unit = options[:unit].upcase
    fail "Invalid unit" unless UNITS.include?(unit)
    num = num.to_f
    for u in UNITS
      if u == unit
        s = "%0.#{precision}f" % round(num, precision)
        return s + unit
      end
      num = num / 1024
    end
  else
    e = (num == 0) ? 0 : (Math.log(num) / Math.log(1024)).floor
    s = "%0.#{precision}f" % round((num.to_f / 1024**e), precision)
    s + UNITS[e]
  end
end

#headers_array_to_hash(header_array) ⇒ Object 



81
82
83
84
85
86
87
88
89
90
91
92
 
# File 'lib/s3cp/utils.rb', line 81

def headers_array_to_hash(header_array)
  headers = {}
  header_array.each do |header|
    header_parts = header.split(": ", 2)
    if header_parts.size == 2
      headers[header_parts[0]] = header_parts[1]
    else
      fail("Invalid header value; expected single colon delimiter; e.g. Header: Value")
    end
  end
  headers
end

#load_config ⇒ Object

Load user-defined configuration file (e.g. to initialize AWS.config object)



58
59
60
61
62
63
64
 
# File 'lib/s3cp/utils.rb', line 58

def load_config()
  aws_config = File.join(ENV['HOME'], '.s3cp') if ENV['HOME']
  aws_config = ENV['S3CP_CONFIG'] if ENV['S3CP_CONFIG']
  if aws_config && File.exist?(aws_config)
    load aws_config
  end
end

#md5(filename) ⇒ Object

Calculate the MD5 checksum for the given file



118
119
120
121
122
123
124
125
126
127
128
129
 
# File 'lib/s3cp/utils.rb', line 118

def md5(filename)
  digest = Digest::MD5.new()
  file = File.open(filename, 'r')
  begin
    file.each_line do |line|
      digest << line
    end
  ensure
    file.close()
  end
  digest.hexdigest
end

#objects_by_wildcard(bucket, key_regex, &block) ⇒ Object

Yield to block for all objects matching key_regex



230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
 
# File 'lib/s3cp/utils.rb', line 230

def objects_by_wildcard(bucket, key_regex, &block)
  # First, trim multiple wildcards & wildcards on the end
  key = key_regex.gsub(/\*+/, '*')

  if 0 < key.count('*')
    key_split = key.split('*', 2);
    kpfix     = key_split.shift(); # ignore first part as AWS API takes it as a prefix
    regex     = []

    key_split.each do |kpart|
      regex.push Regexp.quote(kpart)
    end

    regex = regex.empty? ? nil : Regexp.new(regex.join('.*') + '$');

    bucket.objects.with_prefix(kpfix).each do |obj|
      yield obj if regex == nil || regex.match(obj.key[kpfix.size..-1])
    end
  else
    # no wildcards, simple:
    yield bucket.objects[key]
  end
end

#parse_days_or_date(d) ⇒ Object 



104
105
106
107
108
109
110
 
# File 'lib/s3cp/utils.rb', line 104

def parse_days_or_date(d)
  if d.to_s =~ /^\d+$/
    d.to_i
  else
    Date.parse(d)
  end
end

#round(n, decimals = 0) ⇒ Object

Round a number at some ‘decimals` level of precision.



113
114
115
 
# File 'lib/s3cp/utils.rb', line 113

def round(n, decimals = 0)
  (n * (10.0 ** decimals)).round * (10.0 ** (-decimals))
end

#set_header_options(options, headers) ⇒ Object 



185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
 
# File 'lib/s3cp/utils.rb', line 185

def set_header_options(options, headers)
  return options unless headers

  # legacy options that were previously passed as headers
  # are now passed explicitly as options/metadata.
  mappings = {
    "Content-Type"  =>  :content_type,
    "x-amz-acl"     =>  :acl,
    "Cache-Control" =>  :cache_control,
    "Content-Disposition" => :content_disposition,
    "x-amz-storage-class" => :reduced_redundancy
  }

  lambdas = {
    "x-amz-storage-class" => lambda { |v| (v =~ /^REDUCED_REDUNDANCY$/i) ? true : false }
  }

  remaining = headers.dup
  headers.each do |hk, hv|
    mappings.each do |mk, mv|
      if hk.to_s =~ /^#{mk}$/i
        lambda = lambdas[mk]
        options[mv] = lambda ? lambda.call(hv) : hv
        remaining.delete(hk)
      end
    end
  end

  metadata = {}
  remaining.each do |k,v|
    metadata[k] = v
  end
  options[:metadata] = metadata if metadata

  options
end

#size_in_bytes(size) ⇒ Object 



168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
 
# File 'lib/s3cp/utils.rb', line 168

def size_in_bytes(size)
  case size
  when /^(\d+)$/ # bytes by default
    $1.to_i
  when /^(\d+)b$/i
    $1.to_i
  when /^(\d+)kb?$/i
    $1.to_i * 1024
  when /^(\d+)mb?$/i
    $1.to_i * 1024 * 1024
  when /^(\d+)gb?$/i
    $1.to_i * 1024 * 1024 * 1024
  else
    fail("Invalid size value: #{size}.")
  end
end

#standard_exception_handling(options) ⇒ Object 



254
255
256
257
258
259
260
261
262
263
264
265
 
# File 'lib/s3cp/utils.rb', line 254

def standard_exception_handling(options)
  begin
    yield
  rescue Exception => ex
    cmd_name ||= File.basename(caller[1].split(/:/)[0], '.*')
    $stderr.print "#{cmd_name} [#{ex.class}] #{ex.message}\n"
    if options[:debug]
      $stderr.print ex.backtrace.join("\n") + "\n"
    end
    exit 1
  end
end

#tableify(rows) ⇒ Object

Convert a series of rows [[“one”, “two”, “three3”], [“foo”, “bar”, “baz”], …] into a table-format.



96
97
98
99
100
101
102
 
# File 'lib/s3cp/utils.rb', line 96

def tableify(rows)
  return "" if rows.nil? || rows.empty?
  cols = rows[0].size
  widths = (1..cols).map { |c| rows.map { |r| r[c-1].length }.max }
  format = widths[0..-2].map { |w| "%-#{w}s" }.join("    ") + "    %s"
  rows.map { |r| format % r }.join("\n")
end

#validate_acl(permission) ⇒ Object 



222
223
224
225
226
227
 
# File 'lib/s3cp/utils.rb', line 222

def validate_acl(permission)
  if !LEGAL_MODS.include?(permission)
    raise "Permissions must be one of the following values: #{LEGAL_MODS}"
  end
  permission
end