Class: Fluent::Plugin::FlowCounterOutput

Inherits:
Output
  • Object
show all
Defined in:
lib/fluent/plugin/out_flowcounter.rb

Constant Summary collapse

FOR_MISSING = 
''

Instance Attribute Summary collapse

Instance Method Summary collapse

Instance Attribute Details

#count_all ⇒ Object

Returns the value of attribute count_all.



23
24
25
 
# File 'lib/fluent/plugin/out_flowcounter.rb', line 23

def count_all
  @count_all
end

#counts ⇒ Object

Returns the value of attribute counts.



21
22
23
 
# File 'lib/fluent/plugin/out_flowcounter.rb', line 21

def counts
  @counts
end

#last_checked ⇒ Object

Returns the value of attribute last_checked.



22
23
24
 
# File 'lib/fluent/plugin/out_flowcounter.rb', line 22

def last_checked
  @last_checked
end

#tick ⇒ Object (readonly)

Returns the value of attribute tick.



24
25
26
 
# File 'lib/fluent/plugin/out_flowcounter.rb', line 24

def tick
  @tick
end

Instance Method Details

#configure(conf) ⇒ Object 



26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
 
# File 'lib/fluent/plugin/out_flowcounter.rb', line 26

def configure(conf)
  super

  @tick = case @unit
          when :second then 1
          when :minute then 60
          when :hour then 3600
          when :day then 86400
          else
            raise Fluent::ConfigError, "flowcounter unit allows second/minute/hour/day"
          end
  if @output_style == :tagged and @aggregate != :tag
    raise Fluent::ConfigError, "flowcounter aggregate must be 'tag' when output_style is 'tagged'"
  end
  if @input_tag_remove_prefix
    @removed_prefix_string = @input_tag_remove_prefix + '.'
    @removed_length = @removed_prefix_string.length
  end
  @count_all = false
  if @count_keys && !@count_keys.empty?
    @count_all = (@count_keys == ['*'])
    @count_bytes = true
  else
    @count_bytes = false
  end

  if @timestamp_counting
    @timestamp_timezone_offset = 0
    if @unit == :second
      raise Fluent::ConfigError, "timestamp_counting cannot be enabled with unit: second"
    elsif @unit == :day
      unless @timestamp_timezone
        raise Fluent::ConfigError, "timestamp_counting requires timestamp_timezone to be configured (e.g., '-0700') if unit is day"
      end
      @timestamp_timezone_offset = Time.zone_offset(@timestamp_timezone)
      unless @timestamp_timezone_offset
        raise Fluent::ConfigError, "invalid timestamp_timezone value (specify like '-0700')"
      end
    end
    @last_checked = nil
    @initializer = ->{ now = Fluent::EventTime.now.to_i; @last_checked = now - (now % @tick) - @timestamp_timezone_offset }
    @checker = ->{ Fluent::EventTime.now.to_i - @last_checked >= @tick }
    @updater = ->{ @last_checked += @tick; return Fluent::EventTime.new(@last_checked, 0), @tick }
  else
    @last_checked = nil
    @initializer = ->{ @last_checked = Fluent::Clock.now }
    @checker = ->{ Fluent::Clock.now - @last_checked >= @tick }
    @updater = ->{ prev = @last_checked; @last_checked = Fluent::Clock.now; return Fluent::EventTime.now, @last_checked - prev }
  end

  @counts = count_initialized
  @mutex = Mutex.new
end

#count_initialized(keys = nil) ⇒ Object 



92
93
94
95
96
97
98
99
100
101
102
103
104
105
 
# File 'lib/fluent/plugin/out_flowcounter.rb', line 92

def count_initialized(keys=nil)
  if @aggregate == :all
    if @count_bytes
      {'count' => 0, 'bytes' => 0}
    else
      {'count' => 0}
    end
  elsif keys
    values = Array.new(keys.length){|i| 0 }
    Hash[[keys, values].transpose]
  else
    {}
  end
end

#countup(name, counts, bytes) ⇒ Object 



107
108
109
110
111
112
113
114
115
116
117
118
 
# File 'lib/fluent/plugin/out_flowcounter.rb', line 107

def countup(name, counts, bytes)
  c = 'count'
  b = 'bytes'
  if @aggregate == :tag
    c = name + delimiter + 'count'
    b = name + delimiter + 'bytes' if @count_bytes
  end
  @mutex.synchronize {
    @counts[c] = (@counts[c] || 0) + counts
    @counts[b] = (@counts[b] || 0) + bytes if @count_bytes
  }
end

#flush(step) ⇒ Object 



128
129
130
131
132
 
# File 'lib/fluent/plugin/out_flowcounter.rb', line 128

def flush(step)
  keys = delete_idle ? nil : @counts.keys
  flushed,@counts = @counts,count_initialized(keys)
  generate_output(flushed, step)
end

#flush_emit(now, step) ⇒ Object 



151
152
153
154
155
156
157
158
159
 
# File 'lib/fluent/plugin/out_flowcounter.rb', line 151

def flush_emit(now, step)
  if @output_style == :tagged
    tagged_flush(step).each do |data|
      router.emit(@tag, now, data)
    end
  else
    router.emit(@tag, now, flush(step))
  end
end

#generate_output(counts, step) ⇒ Object 



120
121
122
123
124
125
126
 
# File 'lib/fluent/plugin/out_flowcounter.rb', line 120

def generate_output(counts, step)
  rates = {}
  counts.keys.each {|key|
    rates[key + '_rate'] = ((counts[key] * 100.0) / (1.00 * step)).floor / 100.0
  }
  counts.update(rates)
end

#process(tag, es) ⇒ Object 



163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
 
# File 'lib/fluent/plugin/out_flowcounter.rb', line 163

def process(tag, es)
  name = tag
  if @input_tag_remove_prefix and
      ( (tag.start_with?(@removed_prefix_string) and tag.length > @removed_length) or tag == @input_tag_remove_prefix)
    name = tag[@removed_length..-1]
  end
  c,b = 0,0
  if @count_all
    es.each {|time,record|
      c += 1
      b += record.to_msgpack.bytesize if @count_bytes
    }
  else
    es.each {|time,record|
      c += 1
      b += @count_keys.inject(0){|s,k| s + (record[k] || FOR_MISSING).bytesize} if @count_bytes
    }
  end
  countup(name, c, b)
end

#start ⇒ Object 



80
81
82
83
84
85
86
87
88
89
90
 
# File 'lib/fluent/plugin/out_flowcounter.rb', line 80

def start
  super

  @initializer.call
  timer_execute(:out_flowcounter_watcher, 0.5) do
    if @checker.call
      now, interval = @updater.call
      flush_emit(now, interval)
    end
  end
end

#tagged_flush(step) ⇒ Object 



134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
 
# File 'lib/fluent/plugin/out_flowcounter.rb', line 134

def tagged_flush(step)
  keys = delete_idle ? nil : @counts.keys
  flushed,@counts = @counts,count_initialized(keys)
  names = flushed.keys.select {|x| x.end_with?(delimiter + 'count')}.map {|x| x.chomp(delimiter + 'count')}
  names.map {|name|
    counts = {
      'count' => flushed[name + delimiter + 'count'],
    }
    if @count_bytes
      counts['bytes'] = flushed[name + delimiter + 'bytes']
    end
    data = generate_output(counts, step)
    data['tag'] = name
    data
  }
end