Class: LogStash::Inputs::IMAPRAW

Inherits:
Base
  • Object
show all
Defined in:
lib/logstash/inputs/imap.rb

Overview

Read mails from IMAP server

Periodically scan an IMAP folder (‘INBOX` by default) and move any read messages to the trash.

Instance Method Summary collapse

Instance Method Details

#check_mail(queue) ⇒ Object 



97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
 
# File 'lib/logstash/inputs/imap.rb', line 97

def check_mail(queue)
  # TODO(sissel): handle exceptions happening during runtime:
  # EOFError, OpenSSL::SSL::SSLError
  imap = connect
  imap.select(@folder)
  if @uid_tracking && @uid_last_value
    # If there are no new messages, uid_search returns @uid_last_value
    # because it is the last message, so we need to delete it.
    ids = imap.uid_search(["UID", (@uid_last_value+1..-1)]).delete_if { |uid|
      uid <= @uid_last_value
    }
  else
    ids = imap.uid_search("NOT SEEN")
  end

  ids.each_slice(@fetch_count) do |id_set|
    items = imap.uid_fetch(id_set, ["BODY.PEEK[]", "UID"])
    items.each do |item|
      next unless item.attr.has_key?("BODY[]")
      mail = Mail.read_from_string(item.attr["BODY[]"])
      if @strip_attachments
        queue << parse_mail(mail.without_attachments!, item)
      elsif @save_full_message
        queue << parse_mail(mail, item)
      else
        queue << parse_mail(mail, item)
      end
      # Mark message as processed
      @uid_last_value = item.attr["UID"]
      imap.uid_store(@uid_last_value, '+FLAGS', @delete || @expunge ? :Deleted : :Seen)

      # Stop message processing if it is requested
      break if stop?
    end

    # Expunge deleted messages
    imap.expunge() if @expunge

    # Stop message fetching if it is requested
    break if stop?
  end

rescue => e
  @logger.error("Encountered error #{e.class}", :message => e.message, :backtrace => e.backtrace)
  # Do not raise error, check_mail will be invoked in the next run time

ensure
  # Close the connection (and ignore errors)
  imap.close rescue nil
  imap.disconnect rescue nil

  # Always save @uid_last_value so when tracking is switched from
  # "NOT SEEN" to "UID" we will continue from first unprocessed message
  if @uid_last_value
    @logger.info("Saving \"uid_last_value\": \"#{@uid_last_value}\"")
    File.write(@sincedb_path, @uid_last_value)
  end
end

#connect ⇒ Object

def register



80
81
82
83
84
85
86
87
88
 
# File 'lib/logstash/inputs/imap.rb', line 80

def connect
  sslopt = @secure
  if @secure and not @verify_cert
      sslopt = { :verify_mode => OpenSSL::SSL::VERIFY_NONE }
  end
  imap = Net::IMAP.new(@host, :port => @port, :ssl => sslopt)
  imap.(@user, @password.value)
  return imap
end

#parse_attachments(mail) ⇒ Object 



156
157
158
159
160
161
162
163
164
165
166
 
# File 'lib/logstash/inputs/imap.rb', line 156

def parse_attachments(mail)
  attachments = []
  mail.attachments.each do |attachment|
    if @save_attachments
      attachments << { "filename" => attachment.filename, "data" => attachment.body.encoded }
    else
      attachments << { "filename" => attachment.filename}
    end
  end
  return attachments
end

#parse_mail(mail, item) ⇒ Object 



168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
 
# File 'lib/logstash/inputs/imap.rb', line 168

def parse_mail(mail, item)
  # Add a debug message so we can track what message might cause an error later
  @logger.debug? && @logger.debug("Working with message_id", :message_id => mail.message_id)
  # TODO(sissel): What should a multipart message look like as an event?
  # For now, just take the plain-text part and set it as the message.
  if mail.parts.count == 0
    # No multipart message, just use the body as the event text
    message = mail.body.decoded
  else
    # Multipart message; use the first text/plain part we find
    part = mail.parts.find { |p| p.content_type.match @content_type_re } || mail.parts.first
    message = part.decoded

    # Parse attachments
    attachments = parse_attachments(mail)
  end

  @codec.decode(message) do |event|
    # Use the 'Date' field as the timestamp
    event.timestamp = LogStash::Timestamp.new(mail.date.to_time)

    # Add fields: Add message.header_fields { |h| h.name=> h.value }
    mail.header_fields.each do |header|
      # 'header.name' can sometimes be a Mail::Multibyte::Chars, get it in String form
      name = @lowercase_headers ? header.name.to_s.downcase : header.name.to_s
      # Call .decoded on the header in case it's in encoded-word form.
      # Details at:
      #   https://github.com/mikel/mail/blob/master/README.md#encodings
      #   http://tools.ietf.org/html/rfc2047#section-2
      value = transcode_to_utf8(header.decoded.to_s)

      # Assume we already processed the 'date' above.
      next if name == "Date"

      case (field = event.get(name))
      when String
        # promote string to array if a header appears multiple times
        # (like 'received')
        event.set(name, [field, value])
      when Array
        field << value
        event.set(name, field)
      when nil
        event.set(name, value)
      end
    end

    # Add attachments
    if attachments && attachments.length > 0
      event.set('attachments', attachments)
    end

    #Add Raw Body if configured
    if @save_full_message
      event.set('raw_message_body',item.attr["BODY[]"])
    end

    decorate(event)
    event
  end
end

#register ⇒ Object 



45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
 
# File 'lib/logstash/inputs/imap.rb', line 45

def register
  require "net/imap" # in stdlib
  require "mail" # gem 'mail'

  if @secure and not @verify_cert
    @logger.warn("Running IMAP without verifying the certificate may grant attackers unauthorized access to your mailbox or data")
  end

  if @port.nil?
    if @secure
      @port = 993
    else
      @port = 143
    end
  end

  # Load last processed IMAP uid from file if exists
  if @sincedb_path.nil?
    datapath = File.join(LogStash::SETTINGS.get_value("path.data"), "plugins", "inputs", "imap")
    # Ensure that the filepath exists before writing, since it's deeply nested.
    FileUtils::mkdir_p datapath
    @sincedb_path = File.join(datapath, ".sincedb_" + Digest::MD5.hexdigest("#{@user}_#{@host}_#{@port}_#{@folder}"))
  end
  if File.directory?(@sincedb_path)
    raise ArgumentError.new("The \"sincedb_path\" argument must point to a file, received a directory: \"#{@sincedb_path}\"")
  end
  @logger.info("Using \"sincedb_path\": \"#{@sincedb_path}\"")
  if File.exist?(@sincedb_path)
    @uid_last_value = File.read(@sincedb_path).to_i
    @logger.info("Loading \"uid_last_value\": \"#{@uid_last_value}\"")
  end

  @content_type_re = Regexp.new("^" + @content_type)
end

#run(queue) ⇒ Object 



90
91
92
93
94
95
 
# File 'lib/logstash/inputs/imap.rb', line 90

def run(queue)
  @run_thread = Thread.current
  Stud.interval(@check_interval) do
    check_mail(queue)
  end
end

#stop ⇒ Object 



230
231
232
233
 
# File 'lib/logstash/inputs/imap.rb', line 230

def stop
  Stud.stop!(@run_thread)
  $stdin.close
end