Class: LogStash::Inputs::Kinesis

Inherits:
Base
  • Object
show all
Defined in:
lib/logstash/inputs/kinesis.rb

Overview

Receive events through an AWS Kinesis stream.

This input plugin uses the Java Kinesis Client Library underneath, so the documentation at github.com/awslabs/amazon-kinesis-client will be useful.

AWS credentials can be specified either through environment variables, or an IAM instance role. The library uses a DynamoDB table for worker coordination, so you’ll need to grant access to that as well as to the Kinesis stream. The DynamoDB table has the same name as the ‘application_name` configuration option, which defaults to “logstash”.

The library can optionally also send worker statistics to CloudWatch.

Constant Summary collapse

KCL = 
com.amazonaws.services.kinesis.clientlibrary.lib.worker
KCL_PROCESSOR_FACTORY_CLASS = 
com.amazonaws.services.kinesis.clientlibrary.interfaces.v2.IRecordProcessorFactory

Instance Attribute Summary collapse

Instance Method Summary collapse

Constructor Details

#initialize(params = {}) ⇒ Kinesis

Returns a new instance of Kinesis.



63
64
65
 
# File 'lib/logstash/inputs/kinesis.rb', line 63

def initialize(params = {})
  super(params)
end

Instance Attribute Details

#kcl_config ⇒ Object (readonly)

Returns the value of attribute kcl_config.



31
32
33
 
# File 'lib/logstash/inputs/kinesis.rb', line 31

def kcl_config
  @kcl_config
end

Instance Method Details

#kcl_builder(output_queue) ⇒ Object 



106
107
108
109
110
111
112
113
114
115
 
# File 'lib/logstash/inputs/kinesis.rb', line 106

def kcl_builder(output_queue)
  KCL::Worker::Builder.new.tap do |builder|
    builder.java_send(:recordProcessorFactory, [KCL_PROCESSOR_FACTORY_CLASS.java_class], worker_factory(output_queue))
    builder.config(@kcl_config)

    if metrics_factory
      builder.metricsFactory(metrics_factory)
    end
  end
end

#register ⇒ Object 



67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
 
# File 'lib/logstash/inputs/kinesis.rb', line 67

def register
  # the INFO log level is extremely noisy in KCL
  kinesis_logger = org.apache.commons.logging::LogFactory.getLog("com.amazonaws.services.kinesis").logger
  if kinesis_logger.java_kind_of?(java.util.logging::Logger)
    kinesis_logger.setLevel(java.util.logging::Level::WARNING)
  else
    kinesis_logger.setLevel(org.apache.log4j::Level::WARN)
  end

  worker_id = java.util::UUID.randomUUID.to_s

  # If the AWS profile is set, use the profile credentials provider.
  # Otherwise fall back to the default chain.
  unless @profile.nil?
    creds = com.amazonaws.auth.profile::ProfileCredentialsProvider.new(@profile)
  else
    creds = com.amazonaws.auth::DefaultAWSCredentialsProviderChain.new
  end
  initial_position_in_stream = if @initial_position_in_stream == "TRIM_HORIZON"
    KCL::InitialPositionInStream::TRIM_HORIZON
  else
    KCL::InitialPositionInStream::LATEST
  end

  @kcl_config = KCL::KinesisClientLibConfiguration.new(
    @application_name,
    @kinesis_stream_name,
    creds,
    worker_id).
      withInitialPositionInStream(initial_position_in_stream).
      withRegionName(@region).
      withFailoverTimeMillis(@failover_time_millis)
end

#run(output_queue) ⇒ Object 



101
102
103
104
 
# File 'lib/logstash/inputs/kinesis.rb', line 101

def run(output_queue)
  @kcl_worker = kcl_builder(output_queue).build
  @kcl_worker.run
end

#stop ⇒ Object 



117
118
119
 
# File 'lib/logstash/inputs/kinesis.rb', line 117

def stop
  @kcl_worker.shutdown if @kcl_worker
end

#worker_factory(output_queue) ⇒ Object 



121
122
123
 
# File 'lib/logstash/inputs/kinesis.rb', line 121

def worker_factory(output_queue)
  proc { Worker.new(@codec.clone, output_queue, method(:decorate), @checkpoint_interval_seconds, @logger) }
end