Class: LogStash::Inputs::Kinesis

Inherits:
Base
  • Object
show all
Defined in:
lib/logstash/inputs/kinesis.rb

Overview

Receive events through an AWS Kinesis stream.

This input plugin uses the Java Kinesis Client Library underneath, so the documentation at github.com/awslabs/amazon-kinesis-client will be useful.

AWS credentials can be specified either through environment variables, or an IAM instance role. The library uses a DynamoDB table for worker coordination, so you’ll need to grant access to that as well as to the Kinesis stream. The DynamoDB table has the same name as the ‘application_name` configuration option, which defaults to “logstash”.

The library can optionally also send worker statistics to CloudWatch.

Constant Summary collapse

KCL = 
com.amazonaws.services.kinesis.clientlibrary.lib.worker
KCL_PROCESSOR_FACTORY_CLASS = 
com.amazonaws.services.kinesis.clientlibrary.interfaces.v2.IRecordProcessorFactory

Instance Attribute Summary collapse

Instance Method Summary collapse

Constructor Details

#initialize(params = {}) ⇒ Kinesis

Returns a new instance of Kinesis.



78
79
80
 
# File 'lib/logstash/inputs/kinesis.rb', line 78

def initialize(params = {})
  super(params)
end

Instance Attribute Details

#kcl_config ⇒ Object (readonly)

Returns the value of attribute kcl_config.



33
34
35
 
# File 'lib/logstash/inputs/kinesis.rb', line 33

def kcl_config
  @kcl_config
end

Instance Method Details

#kcl_builder(output_queue) ⇒ Object 



161
162
163
164
165
166
167
168
169
170
 
# File 'lib/logstash/inputs/kinesis.rb', line 161

def kcl_builder(output_queue)
  KCL::Worker::Builder.new.tap do |builder|
    builder.java_send(:recordProcessorFactory, [KCL_PROCESSOR_FACTORY_CLASS.java_class], worker_factory(output_queue))
    builder.config(@kcl_config)

    if metrics_factory
      builder.metricsFactory(metrics_factory)
    end
  end
end

#register ⇒ Object 



82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
 
# File 'lib/logstash/inputs/kinesis.rb', line 82

def register
  # the INFO log level is extremely noisy in KCL
  lg = org.apache.commons.logging::LogFactory.getLog("com.amazonaws.services.kinesis")
  if lg.kind_of?(org.apache.commons.logging.impl::Jdk14Logger)
    kinesis_logger = lg.logger
    if kinesis_logger.kind_of?(java.util.logging::Logger)
      kinesis_logger.setLevel(java.util.logging::Level::WARNING)
    else
      kinesis_logger.setLevel(org.apache.log4j::Level::WARN)
    end
  elsif lg.kind_of?(org.apache.logging.log4jJcl::Log4jLog)
    logContext = org.apache.logging.log4j::LogManager.getContext(false)
    config = logContext.getConfiguration()
    config.getLoggerConfig("com.amazonaws.services.kinesis").setLevel(org.apache.logging.log4j::Level::WARN)
  else
    raise "Can't configure WARN log level for logger wrapper class #{lg.class}"
  end

  @logger.info("Registering logstash-input-kinesis")

  hostname = Socket.gethostname
  uuid = java.util::UUID.randomUUID.to_s
  worker_id = "#{hostname}:#{uuid}"

  # If the AWS profile is set, use the profile credentials provider.
  # Otherwise fall back to the default chain.
  unless @profile.nil?
    creds = com.amazonaws.auth.profile::ProfileCredentialsProvider.new(@profile)
  else
    creds = com.amazonaws.auth::DefaultAWSCredentialsProviderChain.new
  end

  # If a role ARN is set then assume the role as a new layer over the credentials already created
  unless @role_arn.nil?
    kinesis_creds = com.amazonaws.auth::STSAssumeRoleSessionCredentialsProvider.new(creds, @role_arn, @role_session_name)
  else
    kinesis_creds = creds
  end

  initial_position_in_stream = if @initial_position_in_stream == "TRIM_HORIZON"
    KCL::InitialPositionInStream::TRIM_HORIZON
  else
    KCL::InitialPositionInStream::LATEST
  end

  @kcl_config = KCL::KinesisClientLibConfiguration.new(
    @application_name,
    @kinesis_stream_name,
    kinesis_creds, # credential provider for Kinesis, DynamoDB and Cloudwatch access
    worker_id).
      withInitialPositionInStream(initial_position_in_stream).
      withRegionName(@region)

    # Call arbitrary "withX()" functions
    # snake_case => withCamelCase happens automatically
    @additional_settings.each do |key, value|
        fn = "with_#{key}"
        @kcl_config.send(fn, value)
    end

  if @http_proxy && !@http_proxy.value.to_s.strip.empty?
      proxy_uri = URI(@http_proxy.value)
      @logger.info("Using proxy #{proxy_uri.scheme}://#{proxy_uri.user}:*****@#{proxy_uri.host}:#{proxy_uri.port}")
      clnt_cfg = @kcl_config.get_kinesis_client_configuration
      set_client_proxy_settings(clnt_cfg, proxy_uri)
      clnt_cfg = @kcl_config.get_dynamo_db_client_configuration
      set_client_proxy_settings(clnt_cfg, proxy_uri)
      clnt_cfg = @kcl_config.get_cloud_watch_client_configuration
      set_client_proxy_settings(clnt_cfg, proxy_uri)
    end

    @logger.info("Registered logstash-input-kinesis")
end

#run(output_queue) ⇒ Object 



156
157
158
159
 
# File 'lib/logstash/inputs/kinesis.rb', line 156

def run(output_queue)
  @kcl_worker = kcl_builder(output_queue).build
  @kcl_worker.run
end

#stop ⇒ Object 



172
173
174
 
# File 'lib/logstash/inputs/kinesis.rb', line 172

def stop
  @kcl_worker.shutdown if @kcl_worker
end

#worker_factory(output_queue) ⇒ Object 



176
177
178
 
# File 'lib/logstash/inputs/kinesis.rb', line 176

def worker_factory(output_queue)
  proc { Worker.new(@codec.clone, output_queue, method(:decorate), @checkpoint_interval_seconds, @logger) }
end