Module: S3sync

Includes:

S3Config

Defined in:

lib/s3sync/s3cmd.rb,
lib/s3sync/s3try.rb,
lib/s3sync/s3sync.rb,
lib/s3sync/HTTPStreaming.rb

Overview

module

Defined Under Namespace

Classes: LocalNode, Node, ProgressStream, S3Node

Class Method Summary

• .hashPairs(ar) ⇒ Object

  turn an array into a hash of pairs.

• .main ⇒ Object
• .s3cmdList(bucket, path, max = nil, delim = nil, marker = nil, headers = {}) ⇒ Object

  main.

• .s3cmdMain ⇒ Object
• .s3cmdUsage(message = nil) ⇒ Object
• .s3Prefix?(pre) ⇒ Boolean

  ———- PREFIX PROCESSING ———- #.

• .S3try(command, bucket, *args) ⇒ Object
• .S3tryConnect(bucket, host = '') ⇒ Object
• .s3trySetup ⇒ Object
• .S3url(command, bucket, *args) ⇒ Object
• .s3urlSetup ⇒ Object
• .usage(message = nil) ⇒ Object

Class Method Details

.hashPairs(ar) ⇒ Object

turn an array into a hash of pairs

# File 'lib/s3sync/s3cmd.rb', line 228

def S3sync.hashPairs(ar)
  ret = Hash.new
  ar.each do |item|
    name = (/^(.*?):/.match(item))[1]
    item = (/^.*?:(.*)$/.match(item))[1]
    ret[name] = item
  end if ar
  ret
end

.main ⇒ Object

# File 'lib/s3sync/s3sync.rb', line 52

def S3sync.main   
  # ---------- OPTIONS PROCESSING ---------- # 

  $S3syncOptions = Hash.new
  optionsParser = GetoptLong.new(
      [ '--help',    '-h',  GetoptLong::NO_ARGUMENT ],
      [ '--ssl',     '-s',  GetoptLong::NO_ARGUMENT ],
      [ '--recursive','-r', GetoptLong::NO_ARGUMENT ],
      [ '--public-read','-p', GetoptLong::NO_ARGUMENT ],
      [ '--delete',     GetoptLong::NO_ARGUMENT ],
      [ '--verbose', '-v',  GetoptLong::NO_ARGUMENT ],
      [ '--dryrun',  '-n',  GetoptLong::NO_ARGUMENT ], 
      [ '--debug',   '-d',  GetoptLong::NO_ARGUMENT ],
      [ '--memory',   '-m', GetoptLong::NO_ARGUMENT ],
      [ '--progress', GetoptLong::NO_ARGUMENT ],
             [ '--expires',        GetoptLong::REQUIRED_ARGUMENT ],
             [ '--cache-control',  GetoptLong::REQUIRED_ARGUMENT ],
      [ '--exclude',        GetoptLong::REQUIRED_ARGUMENT ],
      [ '--make-dirs',  GetoptLong::NO_ARGUMENT ]
      )
      
  def S3sync.usage(message = nil)
    $stderr.puts message if message
    name = $0.split('/').last
    $stderr.puts "\#{name} [options] <source> <destination>\\t\\tversion \#{$S3SYNC_VERSION}\n --help    -h          --verbose     -v     --dryrun    -n  \n --ssl     -s          --recursive   -r     --delete\n --public-read -p      --expires=\"<exp>\"    --cache-control=\"<cc>\"\n --exclude=\"<regexp>\"  --progress           --debug   -d\n --make-dirs\nOne of <source> or <destination> must be of S3 format, the other a local path.\nReminders:\n* An S3 formatted item with bucket 'mybucket' and prefix 'mypre' looks like:\n   mybucket:mypre/some/key/name\n* Local paths should always use forward slashes '/' even on Windows\n* Whether you use a trailing slash on the source path makes a difference.\n* For examples see README.\n"
  exit
  end #usage
  
  begin
    optionsParser.each {|opt, arg| $S3syncOptions[opt] = (arg || true)}
  rescue StandardError
    usage # the parser already printed an error message
  end
  usage if $S3syncOptions['--help']
  $S3syncOptions['--verbose'] = true if $S3syncOptions['--dryrun'] or $S3syncOptions['--debug'] or $S3syncOptions['--progress']
  $S3syncOptions['--ssl'] = true if $S3syncOptions['--ssl'] # change from "" to true to appease s3 port chooser

  
  # ---------- CONNECT ---------- #
  S3sync::s3trySetup 

  # ---------- PREFIX PROCESSING ---------- #

  def S3sync.s3Prefix?(pre)
    # allow for dos-like things e.g. C:\ to be treated as local even with colon
    pre.include?(':') and not pre.match('^[A-Za-z]:[\\\\/]')
  end
  sourcePrefix, destinationPrefix = ARGV
  usage("You didn't set up your environment variables; see README.txt") if not($AWS_ACCESS_KEY_ID and $AWS_SECRET_ACCESS_KEY) 
  usage('Need a source and a destination') if sourcePrefix == nil or destinationPrefix == nil
  usage('Both arguments can\'t be on S3') if s3Prefix?(sourcePrefix) and s3Prefix?(destinationPrefix)
  usage('One argument must be on S3') if !s3Prefix?(sourcePrefix) and !s3Prefix?(destinationPrefix)

  # so we can modify them
  sourcePrefix, destinationPrefix = sourcePrefix.dup, destinationPrefix.dup

  # handle trailing slash for source properly
  if(sourcePrefix !~ %r{/$})
    # no slash on end of source means we need to append the last src dir to dst prefix
    # testing for empty isn't good enough here.. needs to be "empty apart from potentially having 'bucket:'"
    slash = (destinationPrefix.empty? or destinationPrefix.match(%r{:$}))? "" : "/"
    # not good enough.. sometimes this coughs up the bucket as a prefix destinationPrefix.replace(destinationPrefix + slash + sourcePrefix.split(/(?:\/|:)/).last)
    # take everything at the end after a slash or colon
    destinationPrefix.replace(destinationPrefix + slash + %r{([^/:]*)$}.match(sourcePrefix)[1])
  end
  # no trailing slash on dest, ever.
  destinationPrefix.sub!(%r{/$}, "")
  
  # don't repeat slashes
  sourcePrefix.squeeze!('/')
  destinationPrefix.squeeze!('/')
  
  # here's where we find out what direction we're going
  sourceIsS3 = s3Prefix?(sourcePrefix)
  # alias these variables to the other strings (in ruby = does not make copies of strings)
  s3Prefix = sourceIsS3 ? sourcePrefix : destinationPrefix
  localPrefix = sourceIsS3 ? destinationPrefix : sourcePrefix
  
  # canonicalize the S3 stuff
  s3Bucket = (/^(.*?):/.match(s3Prefix))[1]
  s3Prefix.replace((/:(.*)$/.match(s3Prefix))[1])
  debug("s3Prefix #{s3Prefix}")
  $S3SyncOriginalS3Prefix = s3Prefix.dup
  
  # canonicalize the local stuff
  # but that can kill a trailing slash, which we need to preserve long enough to know whether we mean "the dir" or "its contents"
  # it will get re-stripped by the local generator after expressing this knowledge
  localTrailingSlash = localPrefix.match(%r{/$}) 
  localPrefix.replace(File.expand_path(localPrefix))
  localPrefix += '/' if localTrailingSlash
  debug("localPrefix #{localPrefix}")
  # used for exclusion parsing
  $S3SyncOriginalLocalPrefix = localPrefix.dup
  
  # exclude preparation
  # we don't want to build then throw away this regexp for each node in the universe; do it once globally
  $S3SyncExclude = Regexp.new($S3syncOptions['--exclude']) if $S3syncOptions['--exclude']
  
  
  # ---------- GENERATORS ---------- #

  
  # a generator that will return the files/dirs of the local tree one by one
  # sorted and decorated for easy comparison with the S3 tree
  localTree = Generator.new do |g|
    def S3sync.localTreeRecurse(g, prefix, path)
      debug("localTreeRecurse #{prefix} #{path}")
      #if $S3syncOptions['--memory']
      #  $stderr.puts "Starting local recurse"
      #  stats = ostats stats 
      #end
      d = nil
      begin
        slash = prefix.empty? ? "" : "/"
        d = Dir.new(prefix + slash + path)
      rescue Errno::ENOENT
        # ok the dir doesn't exist at all (this only really occurs for the root i.e. first dir)
        return nil
      rescue Errno::EACCES
        # vista won't even let us touch some stuff in our own profile
        return nil
      end
      # do some pre-processing
      # the following sleight of hand is to make the recursion match the way s3 sorts
      # take for example the directory 'foo' and the file 'foo.bar'
      # when we encounter the dir we would want to recurse into it
      # but S3 would just say 'period < slash' and sort 'foo.bar' between the dir node 
      # and the contents in that 'dir'
      #
      # so the solution is to not recurse into the directory until the point where
      # it would come up "next" in the S3 list
      # We have to do these hoops on the local side, because we have very little control
      # over how S3 will return its results
      toAdd = Array.new
      d.each do |name|
        slash = path.empty? ? "" : "/"
        partialPath = path + slash + name
        slash = prefix.empty? ? "" : "/"
        fullPath = prefix + slash + partialPath
        if name == "." or name == ".."
          # skip
        else
          # add a dir node if appropriate
          debug("Test #{fullPath}")
          if ((not FileTest.symlink?(fullPath)) and FileTest.directory?(fullPath)) and $S3syncOptions['--recursive']
            debug("Adding it as a dir node")
            toAdd.push(name + '/') # always trail slash here for sorting purposes (removed below with rindex test)
          end
        end
      end
      dItems = d.collect + toAdd
      d.close
      d = toAdd = nil
      dItems.sort! #aws says we will get alpha sorted results but ruby doesn't
      dItems.each do |name|
        isDirNode = false
        if name.rindex('/') == name.length-1
          name = name.slice(0...name.length-1)
          isDirNode = true
          debug("#{name} is a dir node")
        end
        slash = path.empty? ? "" : "/"
        partialPath = path + slash + name
        slash = prefix.empty? ? "" : "/"
        fullPath = prefix + slash + partialPath
        excludePath = fullPath.slice($S3SyncOriginalLocalPrefix.length...fullPath.length)
        if name == "." or name == ".."
          # skip
        elsif $S3SyncExclude and $S3SyncExclude.match(excludePath)
          debug("skipping local item #{excludePath} because of --exclude")
        elsif isDirNode
          localTreeRecurse(g, prefix, partialPath)
        else
          # a normal looking node we should try to process
          debug("local item #{fullPath}")
          g.yield(LocalNode.new(prefix, partialPath))
        end
      end
      #if $S3syncOptions['--memory']
      #  $stderr.puts "Ending local recurse"
      #  stats = ostats stats 
      #end
    end
    # a bit of a special case for local, since "foo/" and "foo" are essentially treated the same by file systems
    # so we need to think harder about what the user really meant in the command line.
    localPrefixTrim = localPrefix
    if localPrefix !~ %r{/$}
      # no trailing slash, so yield the root itself first, then recurse if appropriate
      # gork this is still not quite good enough.. if local is the dest then we don't know whether s3 will have a root dir node yielded a priori, so we can't know whether to do this.  only matters for --erase though
      g.yield(LocalNode.new(localPrefixTrim, "")) # technically we should check this for exclusion, but excluding the root node is kind of senseless.. and that would be a pain to set up here
      localTreeRecurse(g, localPrefixTrim, "") if $S3syncOptions['--recursive']
    else
      # trailing slash, so ignore the root itself, and just go into the first level
      localPrefixTrim.sub!(%r{/$}, "") # strip the slash because of how we do local node slash accounting in the recurse above
      localTreeRecurse(g, localPrefixTrim, "") 
    end
  end
  
  # a generator that will return the nodes in the S3 tree one by one
  # sorted and decorated for easy comparison with the local tree
  s3Tree = Generator.new do |g|
    def S3sync.s3TreeRecurse(g, bucket, prefix, path)
      if $S3syncOptions['--memory']
        $stderr.puts "Starting S3 recurse"
        GC.start
        stats = ostats stats 
      end
      $stderr.puts "s3TreeRecurse #{bucket} #{prefix} #{path}" if $S3syncOptions['--debug']
      nextPage = true
      marker = ''
      while nextPage do
        fullPrefix = prefix + path
        debug("nextPage: #{marker}") if marker != ''
        options = {}
        options['prefix'] = fullPrefix # start at the right depth
        options['delimiter'] = '/' # only one dir at a time please
        options['max-keys'] = '200' # use manageable chunks
        options['marker'] = marker unless marker == ''
        d = S3sync.S3try(:list_bucket, bucket, options)
        $stderr.puts "S3 ERROR: #{d.http_response}" unless d.http_response.is_a? Net::HTTPSuccess
        # the 'directories' and leaf nodes are in two separate collections
        # because a dir will never have the same name as a node, we can just shove them together and sort
        # it's important to evaluate them alphabetically for efficient comparison to the local tree
        tItems = d.entries + d.common_prefix_entries
        tItems.sort! do |a,b|
          aName = a.respond_to?('key') ? a.key : a.prefix
          bName = b.respond_to?('key') ? b.key : b.prefix
          # the full path will be returned, efficient to ignore the part we know will be in common
          aName.slice(fullPrefix.length..aName.length) <=> bName.slice(fullPrefix.length..bName.length)
        end
        # get rid of the big s3 objects asap, just save light-weight nodes and strings
        items = tItems.collect do |item|
          if item.respond_to?('key')
            key = Iconv.iconv($S3SYNC_NATIVE_CHARSET, "UTF-8", item.key).join
            Node.new(key, item.size, item.etag)
          else
            Iconv.iconv($S3SYNC_NATIVE_CHARSET, "UTF-8", item.prefix).join
          end
        end
        nextPage = d.properties.is_truncated
        marker = (d.properties.next_marker)? d.properties.next_marker : ((d.entries.length > 0)? d.entries.last.key : '')
        # get this into native char set (because when we feed it back to s3 that's what it will expect)
        marker = Iconv.iconv($S3SYNC_NATIVE_CHARSET, "UTF-8", marker).join
        tItems = nil
        d = nil # get rid of this before recursing; it's big
        item = nil
        GC.start # not sure but I think yielding before doing this is causing evil closure bloat
        items.each do |item|
          if not (item.kind_of? String)
            # this is an item
            excludePath = item.name.slice($S3SyncOriginalS3Prefix.length...item.name.length)
            if $S3SyncExclude and $S3SyncExclude.match(excludePath)
              debug("skipping S3 item #{excludePath} due to --exclude")
            else
              debug("S3 item #{item.name}")
              g.yield(S3Node.new(bucket, prefix, item))
            end
          else
            # it's a prefix (i.e. there are sub keys)
            partialPath = item.slice(prefix.length..item.length) # will have trailing slash
            excludePath = item.slice($S3SyncOriginalS3Prefix.length...item.length)
            # recurse
            if $S3SyncExclude and $S3SyncExclude.match(excludePath)
              debug("skipping prefix #{excludePath} due to --exclude")
            else
              debug("prefix found: #{partialPath}")
              s3TreeRecurse(g, bucket, prefix, partialPath) if $S3syncOptions['--recursive'] 
            end
          end
        end
        items = nil
      end # of while nextPage
      if $S3syncOptions['--memory']
        $stderr.puts "Ending S3 recurse"
        GC.start
        stats = ostats stats 
      end
    end
    # this will yield the root node first and then recurse
    s3TreeRecurse(g, s3Bucket, s3Prefix, "")
    
  end
  
  # alias the tree objects so we don't care below which direction the transfer is going
  if sourceIsS3
    sourceTree, destinationTree = s3Tree, localTree
  else
    sourceTree, destinationTree = localTree, s3Tree
  end
  
  
  # ---------- COMPARATOR ---------- #
  
  # run the comparison engine and act according to what we find for each check
  nodesToDelete = Array.new # a stack. have to delete in reverse order of normal create/update processing
  
  sourceNode = sourceTree.next? ? sourceTree.next : nil
  destinationNode = destinationTree.next? ? destinationTree.next : nil
  while sourceNode or destinationNode do
    debug("source: #{sourceNode.name}") if sourceNode
    debug("dest: #{destinationNode.name}") if destinationNode
    if (!destinationNode) or (sourceNode and (sourceNode.name < destinationNode.name))
      dNode = 
      if sourceNode.kind_of? LocalNode
        S3Node.new(s3Bucket, s3Prefix, sourceNode.name)
      else
        LocalNode.new(localPrefix, sourceNode.name)
      end
      puts "Create node #{sourceNode.name}" if $S3syncOptions['--verbose']
      dNode.updateFrom(sourceNode) unless $S3syncOptions['--dryrun']
      sourceNode = sourceTree.next? ? sourceTree.next : nil
    elsif (!sourceNode) or (destinationNode and (sourceNode.name > destinationNode.name))
      $stderr.puts "Source does not have #{destinationNode.name}" if $S3syncOptions['--debug']
      if $S3syncOptions['--delete']
        if destinationNode.directory?
          # have to wait
          nodesToDelete.push(destinationNode) 
        else
          puts "Remove node #{destinationNode.name}" if $S3syncOptions['--verbose']
          destinationNode.delete unless $S3syncOptions['--dryrun']
        end
      end
      destinationNode = destinationTree.next? ? destinationTree.next : nil
    elsif sourceNode.name == destinationNode.name
      if (sourceNode.size != destinationNode.size) or (sourceNode.tag != destinationNode.tag)
        puts "Update node #{sourceNode.name}" if $S3syncOptions['--verbose']
        destinationNode.updateFrom(sourceNode) unless $S3syncOptions['--dryrun']
      elsif $S3syncOptions['--debug']
        $stderr.puts "Node #{sourceNode.name} unchanged" 
      end
      sourceNode = sourceTree.next? ? sourceTree.next : nil
      destinationNode = destinationTree.next? ? destinationTree.next : nil
    end         
  end
  
  # get rid of the (now empty, except for other directories) directories
  nodesToDelete.reverse_each do |node|
    puts "Remove node #{node.name}" if $S3syncOptions['--verbose']
    node.delete unless $S3syncOptions['--dryrun']
  end
  
end

.s3cmdList(bucket, path, max = nil, delim = nil, marker = nil, headers = {}) ⇒ Object

main

# File 'lib/s3sync/s3cmd.rb', line 217

def S3sync.s3cmdList(bucket, path, max=nil, delim=nil, marker=nil, headers={})
  debug(max)
  options = Hash.new
  options['prefix'] = path # start at the right depth
  options['max-keys'] = max ? max.to_s : 100
  options['delimiter'] = delim if delim
  options['marker'] = marker if marker
  S3try(:list_bucket, bucket, options, headers)
end

.s3cmdMain ⇒ Object

# File 'lib/s3sync/s3cmd.rb', line 26

def S3sync.s3cmdMain  
  # ---------- OPTIONS PROCESSING ---------- # 

  $S3syncOptions = Hash.new
  optionsParser = GetoptLong.new(
      [ '--help',    '-h',  GetoptLong::NO_ARGUMENT ],
      [ '--ssl',     '-s',  GetoptLong::NO_ARGUMENT ],
      [ '--verbose', '-v',  GetoptLong::NO_ARGUMENT ],
      [ '--dryrun',  '-n',  GetoptLong::NO_ARGUMENT ], 
      [ '--debug',   '-d',  GetoptLong::NO_ARGUMENT ],
      [ '--progress',       GetoptLong::NO_ARGUMENT ],
          [ '--expires-in', GetoptLong::REQUIRED_ARGUMENT ]
      )
      
  def S3sync.s3cmdUsage(message = nil)
    $stderr.puts message if message
    name = $0.split('/').last
    $stderr.puts "\#{name} [options] <command> [arg(s)]\\t\\tversion \#{$S3CMD_VERSION}\n --help    -h        --verbose     -v     --dryrun    -n  \n --ssl     -s        --debug       -d     --progress\n --expires-in=( <# of seconds> | [#d|#h|#m|#s] )\n \nCommands:\n\#{name}  listbuckets  [headers]\n\#{name}  createbucket  <bucket>  [constraint (i.e. EU)]\n\#{name}  deletebucket  <bucket>  [headers]\n\#{name}  list  <bucket>[:prefix]  [max/page]  [delimiter]  [headers]\n\#{name}  location  <bucket> [headers]\n\#{name}  delete  <bucket>:key  [headers]\n\#{name}  deleteall  <bucket>[:prefix]  [headers]\n\#{name}  get|put  <bucket>:key  <file>  [headers]\n"
  exit
  end #usage
  
  begin
    optionsParser.each {|opt, arg| $S3syncOptions[opt] = (arg || true)}
  rescue StandardError
    s3cmdUsage # the parser already printed an error message
  end
  s3cmdUsage if $S3syncOptions['--help']
  $S3syncOptions['--verbose'] = true if $S3syncOptions['--dryrun'] or $S3syncOptions['--debug'] or $S3syncOptions['--progress']
  $S3syncOptions['--ssl'] = true if $S3syncOptions['--ssl'] # change from "" to true to appease s3 port chooser
     
     if $S3syncOptions['--expires-in'] =~ /d|h|m|s/
        e = $S3syncOptions['--expires-in']
        days = (e =~ /(\d+)d/)? (/(\d+)d/.match(e))[1].to_i : 0
        hours = (e =~ /(\d+)h/)? (/(\d+)h/.match(e))[1].to_i : 0
        minutes = (e =~ /(\d+)m/)? (/(\d+)m/.match(e))[1].to_i : 0
        seconds = (e =~ /(\d+)s/)? (/(\d+)s/.match(e))[1].to_i : 0
        $S3syncOptions['--expires-in'] = seconds + 60 * ( minutes + 60 * ( hours + 24 * ( days ) ) )
     end

  # ---------- CONNECT ---------- #
  S3sync::s3trySetup 
  # ---------- COMMAND PROCESSING ---------- #   

  command, path, file = ARGV
  
  s3cmdUsage("You didn't set up your environment variables; see README.txt") if not($AWS_ACCESS_KEY_ID and $AWS_SECRET_ACCESS_KEY) 
  s3cmdUsage("Need a command (etc)") if not command

  path = '' unless path
  path = path.dup # modifiable
  path += ':' unless path.match(':')
  bucket = (/^(.*?):/.match(path))[1]
  path.replace((/:(.*)$/.match(path))[1])

  case command
    when "delete"
      s3cmdUsage("Need a bucket") if bucket == ''
      s3cmdUsage("Need a key") if path == ''
      headers = hashPairs(ARGV[2...ARGV.length])
      $stderr.puts "delete #{bucket}:#{path} #{headers.inspect if headers}" if $S3syncOptions['--verbose']
      S3try(:delete, bucket, path) unless $S3syncOptions['--dryrun']
    when "deleteall"
      s3cmdUsage("Need a bucket") if bucket == ''
      headers = hashPairs(ARGV[2...ARGV.length])
      $stderr.puts "delete ALL entries in #{bucket}:#{path} #{headers.inspect if headers}" if $S3syncOptions['--verbose']
      more = true
      marker = nil
      while more do
        res = s3cmdList(bucket, path, nil, nil, marker)
        res.entries.each do |item|
          # the s3 commands (with my modified UTF-8 conversion) expect native char encoding input
          key = Iconv.iconv($S3SYNC_NATIVE_CHARSET, "UTF-8", item.key).join
          $stderr.puts "delete #{bucket}:#{key} #{headers.inspect if headers}" if $S3syncOptions['--verbose']
          S3try(:delete, bucket, key) unless $S3syncOptions['--dryrun']    
        end
        more = res.properties.is_truncated
        marker = (res.properties.next_marker)? res.properties.next_marker : ((res.entries.length > 0) ? res.entries.last.key : nil)
        # get this into local charset; when we pass it to s3 that is what's expected
        marker = Iconv.iconv($S3SYNC_NATIVE_CHARSET, "UTF-8", marker).join if marker
      end
    when "list"
      s3cmdUsage("Need a bucket") if bucket == ''
      max, delim = ARGV[2..3]
      headers = hashPairs(ARGV[4...ARGV.length])
      $stderr.puts "list #{bucket}:#{path} #{max} #{delim} #{headers.inspect if headers}" if $S3syncOptions['--verbose']
      puts "--------------------"
      
      more = true
      marker = nil
      while more do
        res = s3cmdList(bucket, path, max, delim, marker, headers)
        if delim
          res.common_prefix_entries.each do |item|
            
            puts "dir: " + Iconv.iconv($S3SYNC_NATIVE_CHARSET, "UTF-8", item.prefix).join
          end
          puts "--------------------"
        end
        res.entries.each do |item|
          puts Iconv.iconv($S3SYNC_NATIVE_CHARSET, "UTF-8", item.key).join
        end
        if res.properties.is_truncated
          printf "More? Y/n: "
          more = (STDIN.gets.match('^[Yy]?$'))
          marker = (res.properties.next_marker)? res.properties.next_marker : ((res.entries.length > 0) ? res.entries.last.key : nil)
          # get this into local charset; when we pass it to s3 that is what's expected
          marker = Iconv.iconv($S3SYNC_NATIVE_CHARSET, "UTF-8", marker).join if marker
          
        else
          more = false
        end
      end # more
    when "listbuckets"
      headers = hashPairs(ARGV[1...ARGV.length])
      $stderr.puts "list all buckets #{headers.inspect if headers}" if $S3syncOptions['--verbose']
           if $S3syncOptions['--expires-in']
              $stdout.puts S3url(:list_all_my_buckets, headers)
           else
              res = S3try(:list_all_my_buckets, headers)
              res.entries.each do |item|
           puts item.name
              end
           end
    when "createbucket"
      s3cmdUsage("Need a bucket") if bucket == ''
           lc = ''
           if(ARGV.length > 2)
              lc = '<CreateBucketConfiguration xmlns="http://s3.amazonaws.com/doc/2006-03-01"><LocationConstraint>' + ARGV[2] + '</LocationConstraint></CreateBucketConfiguration>'
           end
      $stderr.puts "create bucket #{bucket} #{lc}" if $S3syncOptions['--verbose']
      S3try(:create_bucket, bucket, lc) unless $S3syncOptions['--dryrun']
    when "deletebucket"
      s3cmdUsage("Need a bucket") if bucket == ''
      headers = hashPairs(ARGV[2...ARGV.length])
      $stderr.puts "delete bucket #{bucket} #{headers.inspect if headers}" if $S3syncOptions['--verbose']
      S3try(:delete_bucket, bucket, headers) unless $S3syncOptions['--dryrun']
    when "location"
      s3cmdUsage("Need a bucket") if bucket == ''
      headers = hashPairs(ARGV[2...ARGV.length])
           query = Hash.new
           query['location'] = 'location'
      $stderr.puts "location request bucket #{bucket} #{query.inspect} #{headers.inspect if headers}" if $S3syncOptions['--verbose']
      S3try(:get_query_stream, bucket, '', query, headers, $stdout) unless $S3syncOptions['--dryrun']
    when "get"
      s3cmdUsage("Need a bucket") if bucket == ''
      s3cmdUsage("Need a key") if path == ''
      s3cmdUsage("Need a file") if file == ''
      headers = hashPairs(ARGV[3...ARGV.length])
      $stderr.puts "get from key #{bucket}:#{path} into #{file} #{headers.inspect if headers}" if $S3syncOptions['--verbose']
      unless $S3syncOptions['--dryrun']
              if $S3syncOptions['--expires-in']
                 $stdout.puts S3url(:get, bucket, path, headers)
              else
                 outStream = File.open(file, 'wb')
                 outStream = ProgressStream.new(outStream) if $S3syncOptions['--progress']
                 S3try(:get_stream, bucket, path, headers, outStream)
                 outStream.close
              end
      end
    when "put"
      s3cmdUsage("Need a bucket") if bucket == ''
      s3cmdUsage("Need a key") if path == ''
      s3cmdUsage("Need a file") if file == ''
      headers = hashPairs(ARGV[3...ARGV.length])
      stream = File.open(file, 'rb')
      stream = ProgressStream.new(stream, File.stat(file).size) if $S3syncOptions['--progress']
      s3o = S3::S3Object.new(stream, {}) # support meta later?
      headers['Content-Length'] = FileTest.size(file).to_s
      $stderr.puts "put to key #{bucket}:#{path} from #{file} #{headers.inspect if headers}" if $S3syncOptions['--verbose']
      S3try(:put, bucket, path, s3o, headers) unless $S3syncOptions['--dryrun']
      stream.close
    else
      s3cmdUsage
  end

end

.s3cmdUsage(message = nil) ⇒ Object

# File 'lib/s3sync/s3cmd.rb', line 40

def S3sync.s3cmdUsage(message = nil)
  $stderr.puts message if message
  name = $0.split('/').last
  $stderr.puts "\#{name} [options] <command> [arg(s)]\\t\\tversion \#{$S3CMD_VERSION}\n--help    -h        --verbose     -v     --dryrun    -n \n--ssl     -s        --debug       -d     --progress\n--expires-in=( <# of seconds> | [#d|#h|#m|#s] )\n\nCommands:\n\#{name}  listbuckets  [headers]\n\#{name}  createbucket  <bucket>  [constraint (i.e. EU)]\n\#{name}  deletebucket  <bucket>  [headers]\n\#{name}  list  <bucket>[:prefix]  [max/page]  [delimiter]  [headers]\n\#{name}  location  <bucket> [headers]\n\#{name}  delete  <bucket>:key  [headers]\n\#{name}  deleteall  <bucket>[:prefix]  [headers]\n\#{name}  get|put  <bucket>:key  <file>  [headers]\n"
exit
end

.s3Prefix?(pre) ⇒ Boolean

———- PREFIX PROCESSING ———- #

Returns:

• (Boolean)

# File 'lib/s3sync/s3sync.rb', line 109

def S3sync.s3Prefix?(pre)
  # allow for dos-like things e.g. C:\ to be treated as local even with colon
  pre.include?(':') and not pre.match('^[A-Za-z]:[\\\\/]')
end

.S3try(command, bucket, *args) ⇒ Object

# File 'lib/s3sync/s3try.rb', line 65

def S3sync.S3try(command, bucket, *args)
     if(not $S3syncHttp or (bucket != $S3syncLastBucket))
        $stderr.puts "Creating new connection" if $S3syncOptions['--debug']
        $S3syncLastBucket = bucket
        S3sync.S3tryConnect(bucket)
     end
     
  result = nil
  delim = $,
  $,=' '
  while $S3syncRetriesLeft > 0 do
        $stderr.puts "Trying command #{command} #{bucket} #{args} with #{$S3syncRetriesLeft} retries left" if $S3syncOptions['--debug']
    forceRetry = false
        now = false
        hush = false
    begin
      result = $S3syncConnection.send(command, bucket, *args)
    rescue Errno::EPIPE => e
      forceRetry = true
      $stderr.puts "Broken pipe: #{e}" 
    rescue Errno::ECONNRESET => e
      forceRetry = true
      $stderr.puts "Connection reset: #{e}" 
    rescue Errno::ECONNABORTED => e
      forceRetry = true
      $stderr.puts "Connection aborted: #{e}" 
    rescue Errno::ETIMEDOUT => e
      forceRetry = true
      $stderr.puts "Connection timed out: #{e}"
    rescue Timeout::Error => e
      forceRetry = true
      $stderr.puts "Connection timed out: #{e}" 
    rescue EOFError => e
      # i THINK this is happening like a connection reset

      forceRetry = true
      $stderr.puts "EOF error: #{e}"
    rescue OpenSSL::SSL::SSLError => e
      forceRetry = true
      $stderr.puts "SSL Error: #{e}"
    rescue NoMethodError
      # we get this when using --progress, and the local item is something unreadable

      $stderr.puts "Null stream error: #{e}"
      break
    end
        if forceRetry
      # kill and reset connection when we receive a non-50x yet retry-able error

      S3sync.S3tryConnect(bucket)
        end
    begin
      debug("Response code: #{result.http_response.code}")
      break if ((200...300).include? result.http_response.code.to_i) and (not forceRetry)
           if result.http_response.code.to_i == 301
              $stderr.puts "Permanent redirect received. Try setting AWS_CALLING_FORMAT to SUBDOMAIN"
           elsif result.http_response.code.to_i == 307
              # move to the other host

              host = %r{https?://([^/]+)}.match(result.http_response['Location'])[1]
              $stderr.puts("Temporary Redirect to: " + host)
              debug("Host: " + host)
              S3sync.S3tryConnect(bucket, host)
              $S3syncRetriesLeft = $S3syncRetriesLeft+1 # don't use one up below

              forceRetry = true
              now = true
              hush = true
           else
              $stderr.puts "S3 command failed:\n#{command} #{args}"
              $stderr.puts "With result #{result.http_response.code} #{result.http_response.message}\n"
              debug(result.http_response.body)
           end
      # only retry 500's, per amazon

      break unless ((500...600).include? result.http_response.code.to_i) or forceRetry
    rescue NoMethodError
      debug("No result available")
    end
    $S3syncRetriesLeft -= 1
    $stderr.puts "#{$S3syncRetriesLeft} retries left" unless hush
    Kernel.sleep $S3SYNC_WAITONERROR unless now
  end
     if $S3syncRetriesLeft <= 0
        $stderr.puts "Ran out of retries; operations did not complete!"
     end
  $, = delim
  result
end

.S3tryConnect(bucket, host = '') ⇒ Object

# File 'lib/s3sync/s3try.rb', line 61

def S3sync.S3tryConnect(bucket, host='')
      $S3syncHttp = $S3syncConnection.make_http(bucket, host, $HTTP_PROXY_HOST, $HTTP_PROXY_PORT, $HTTP_PROXY_USER, $HTTP_PROXY_PASSWORD)
end

.s3trySetup ⇒ Object

# File 'lib/s3sync/s3try.rb', line 39

def S3sync.s3trySetup   
  
  # ---------- CONNECT ---------- #


  $S3syncConnection = S3::AWSAuthConnection.new($AWS_ACCESS_KEY_ID, $AWS_SECRET_ACCESS_KEY, $S3syncOptions['--ssl'], $AWS_S3_HOST)
     $S3syncConnection.calling_format = S3::CallingFormat::string_to_format($AWS_CALLING_FORMAT)
  if $S3syncOptions['--ssl']
    if $SSL_CERT_DIR
      $S3syncConnection.verify_mode = OpenSSL::SSL::VERIFY_PEER
      $S3syncConnection.ca_path = $SSL_CERT_DIR
    elsif $SSL_CERT_FILE
      $S3syncConnection.verify_mode = OpenSSL::SSL::VERIFY_PEER
      $S3syncConnection.ca_file = $SSL_CERT_FILE
    end
  end
end

.S3url(command, bucket, *args) ⇒ Object

# File 'lib/s3sync/s3try.rb', line 149

def S3sync.S3url(command, bucket, *args)
     S3sync.s3urlSetup() unless $S3syncGenerator
  result = nil
  delim = $,
  $,=' '
     $stderr.puts "Calling command #{command} #{bucket} #{args}" if $S3syncOptions['--debug']
     result = $S3syncGenerator.send(command, bucket, *args)
  $, = delim
  result
end

.s3urlSetup ⇒ Object

# File 'lib/s3sync/s3try.rb', line 55

def S3sync.s3urlSetup   
  $S3syncGenerator = S3::QueryStringAuthGenerator.new($AWS_ACCESS_KEY_ID, $AWS_SECRET_ACCESS_KEY, $S3syncOptions['--ssl'], $AWS_S3_HOST)
     $S3syncGenerator.calling_format = S3::CallingFormat::string_to_format($AWS_CALLING_FORMAT)
     $S3syncGenerator.expires_in = $S3syncOptions['--expires-in']
end

.usage(message = nil) ⇒ Object

# File 'lib/s3sync/s3sync.rb', line 73

def S3sync.usage(message = nil)
  $stderr.puts message if message
  name = $0.split('/').last
  $stderr.puts "\#{name} [options] <source> <destination>\\t\\tversion \#{$S3SYNC_VERSION}\n--help    -h          --verbose     -v     --dryrun    -n \n--ssl     -s          --recursive   -r     --delete\n--public-read -p      --expires=\"<exp>\"    --cache-control=\"<cc>\"\n--exclude=\"<regexp>\"  --progress           --debug   -d\n--make-dirs\nOne of <source> or <destination> must be of S3 format, the other a local path.\nReminders:\n* An S3 formatted item with bucket 'mybucket' and prefix 'mypre' looks like:\n  mybucket:mypre/some/key/name\n* Local paths should always use forward slashes '/' even on Windows\n* Whether you use a trailing slash on the source path makes a difference.\n* For examples see README.\n"
exit
end