Class: LogStash::Filters::Fingerprint

Inherits:
Base
  • Object
show all
Defined in:
lib/logstash/filters/fingerprint.rb

Overview

Create consistent hashes (fingerprints) of one or more fields and store the result in a new field.

This can e.g. be used to create consistent document ids when inserting events into Elasticsearch, allowing events in Logstash to cause existing documents to be updated rather than new documents to be created.

NOTE: When using any method other than ‘UUID’, ‘PUNCTUATION’ or ‘MURMUR3’ you must set the key, otherwise the plugin will raise an exception

NOTE: When the ‘target` option is set to `UUID` the result won’t be a consistent hash but a random en.wikipedia.org/wiki/Universally_unique_identifier[UUID]. To generate UUIDs, prefer the <<plugins-filters-uuid,uuid filter>>.

Defined Under Namespace

Modules: MinimumSerializationLengthTimestamp

Constant Summary collapse

INTEGER_MAX_32BIT = 
(1 << 31) - 1
INTEGER_MIN_32BIT = 
-(1 << 31)

Instance Method Summary collapse

Constructor Details

#initialize(*params) ⇒ Fingerprint

Returns a new instance of Fingerprint.



107
108
109
110
 
# File 'lib/logstash/filters/fingerprint.rb', line 107

def initialize(*params)
  super
  @target ||= ecs_select[disabled: 'fingerprint', v1: '[event][hash]']
end

Instance Method Details

#filter(event) ⇒ Object 



141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
 
# File 'lib/logstash/filters/fingerprint.rb', line 141

def filter(event)
  case @method
  when :UUID
    event.set(@target, SecureRandom.uuid)
  when :PUNCTUATION
    @source.sort.each do |field|
      next unless event.include?(field)
      # In order to keep some backwards compatibility we should use the unicode version
      # of the regexp because the POSIX one ([[:punct:]]) left some unwanted characters unfiltered (Symbols).
      # gsub(/[^[:punct:]]/,'') should be equivalent to gsub(/[^[\p{P}\p{S}]]/,''), but not 100% in JRuby.
      event.set(@target, event.get(field).gsub(/[^[\p{P}\p{S}]]/,''))
    end
  else
    if @concatenate_sources || @concatenate_all_fields
      to_string = ""
      if @concatenate_all_fields
        deep_sort_hashes(event.to_hash).each do |k,v|
          # Force encoding to UTF-8 to get around https://github.com/jruby/jruby/issues/6748
          to_string << "|#{k}|#{v}".force_encoding("UTF-8")
        end
      else
        @source.sort.each do |k|
          # Force encoding to UTF-8 to get around https://github.com/jruby/jruby/issues/6748
          to_string << "|#{k}|#{deep_sort_hashes(event.get(k))}".force_encoding("UTF-8")
        end
      end
      to_string << "|"
      @logger.debug? && @logger.debug("String built", :to_checksum => to_string)
      event.set(@target, fingerprint(to_string))
    else
      @source.each do |field|
        next unless event.include?(field)
        if event.get(field).is_a?(Array)
          event.set(@target, event.get(field).collect { |v| fingerprint(deep_sort_hashes(v)) })
        else
          event.set(@target, fingerprint(deep_sort_hashes(event.get(field))))
        end
      end
    end
  end
  filter_matched(event)
end

#register ⇒ Object 



112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
 
# File 'lib/logstash/filters/fingerprint.rb', line 112

def register
  # convert to symbol for faster comparisons
  @method = @method.to_sym

  # require any library and set the fingerprint function
  case @method
  when :IPV4_NETWORK
    if @key.nil?
      raise LogStash::ConfigurationError, I18n.t(
        "logstash.runner.configuration.invalid_plugin_register",
        :plugin => "filter",
        :type => "fingerprint",
        :error => "Key value is empty. please fill in a subnet prefix length"
      )
    end
    class << self; alias_method :fingerprint, :fingerprint_ipv4_network; end
  when :MURMUR3
    class << self; alias_method :fingerprint, :fingerprint_murmur3; end
  when :MURMUR3_128
    class << self; alias_method :fingerprint, :fingerprint_murmur3_128; end
  when :UUID
    # nothing
  when :PUNCTUATION
    # nothing
  else
    class << self; alias_method :fingerprint, :fingerprint_openssl; end
  end
end