Class: LogStash::Outputs::S3::TemporaryFile

Inherits:
Object
  • Object
show all
Extended by:
Forwardable
Defined in:
lib/logstash/outputs/s3/temporary_file.rb

Overview

Wrap the actual file descriptor into an utility class Make it more OOP and easier to reason with the paths.

Constant Summary collapse

GZIP_EXTENSION = 
"txt.gz"
TXT_EXTENSION = 
"txt"
RECOVERED_FILE_NAME_TAG = 
"-recovered"

Instance Attribute Summary collapse

Class Method Summary collapse

Instance Method Summary collapse

Constructor Details

#initialize(key, fd, temp_path) ⇒ TemporaryFile

Returns a new instance of TemporaryFile.



26
27
28
29
30
31
 
# File 'lib/logstash/outputs/s3/temporary_file.rb', line 26

def initialize(key, fd, temp_path)
  @fd = fd
  @key = key
  @temp_path = temp_path
  @created_at = Time.now
end

Instance Attribute Details

#fd ⇒ Object (readonly)

Returns the value of attribute fd.



24
25
26
 
# File 'lib/logstash/outputs/s3/temporary_file.rb', line 24

def fd
  @fd
end

Class Method Details

.create_from_existing_file(file_path, temporary_folder) ⇒ Object 



76
77
78
79
80
81
82
83
84
85
86
 
# File 'lib/logstash/outputs/s3/temporary_file.rb', line 76

def self.create_from_existing_file(file_path, temporary_folder)
  key_parts = Pathname.new(file_path).relative_path_from(temporary_folder).to_s.split(::File::SEPARATOR)

  # recover gzip file and compress back before uploading to S3
  if file_path.end_with?("." + GZIP_EXTENSION)
    file_path = self.recover(file_path)
  end
  TemporaryFile.new(key_parts.slice(1, key_parts.size).join("/"),
                 ::File.exist?(file_path) ? ::File.open(file_path, "r") : nil, # for the nil case, file size will be 0 and upload will be ignored.
                 ::File.join(temporary_folder, key_parts.slice(0, 1)))
end

.gzip_extension ⇒ Object 



88
89
90
 
# File 'lib/logstash/outputs/s3/temporary_file.rb', line 88

def self.gzip_extension
  GZIP_EXTENSION
end

.recovery_file_name_tag ⇒ Object 



96
97
98
 
# File 'lib/logstash/outputs/s3/temporary_file.rb', line 96

def self.recovery_file_name_tag
  RECOVERED_FILE_NAME_TAG
end

.text_extension ⇒ Object 



92
93
94
 
# File 'lib/logstash/outputs/s3/temporary_file.rb', line 92

def self.text_extension
  TXT_EXTENSION
end

Instance Method Details

#ctime ⇒ Object 



33
34
35
 
# File 'lib/logstash/outputs/s3/temporary_file.rb', line 33

def ctime
  @created_at
end

#delete! ⇒ Object

Each temporary file is created inside a directory named with an UUID, instead of deleting the file directly and having the risk of deleting other files we delete the root of the UUID, using a UUID also remove the risk of deleting unwanted file, it acts as a sandbox.



62
63
64
65
 
# File 'lib/logstash/outputs/s3/temporary_file.rb', line 62

def delete!
  @fd.close rescue IOError # force close anyway
  FileUtils.rm_r(@temp_path, :secure => true)
end

#empty? ⇒ Boolean

Returns:

  • (Boolean) 


67
68
69
 
# File 'lib/logstash/outputs/s3/temporary_file.rb', line 67

def empty?
  size == 0
end

#key ⇒ Object 



54
55
56
 
# File 'lib/logstash/outputs/s3/temporary_file.rb', line 54

def key
  @key.gsub(/^\//, "")
end

#recoverable? ⇒ Boolean

only to cover the case where LS cannot restore corrupted file, file is not exist

Returns:

  • (Boolean) 


72
73
74
 
# File 'lib/logstash/outputs/s3/temporary_file.rb', line 72

def recoverable?
  !@fd.nil?
end

#size ⇒ Object 



41
42
43
44
45
46
47
48
49
50
51
52
 
# File 'lib/logstash/outputs/s3/temporary_file.rb', line 41

def size
  # Use the fd size to get the accurate result,
  # so we dont have to deal with fsync
  # if the file is close, fd.size raises an IO exception so we use the File::size
  begin
    # fd is nil when LS tries to recover gzip file but fails
    return 0 unless @fd != nil
    @fd.size
  rescue IOError
    ::File.size(path)
  end
end

#temp_path ⇒ Object 



37
38
39
 
# File 'lib/logstash/outputs/s3/temporary_file.rb', line 37

def temp_path
  @temp_path
end