Module: Spark

Includes:
Helper::System
Defined in:
lib/spark/sampler.rb,
lib/spark.rb,
lib/spark/cli.rb,
lib/spark/rdd.rb,
lib/spark/sort.rb,
lib/spark/sort.rb,
lib/spark/build.rb,
lib/spark/error.rb,
lib/spark/mllib.rb,
lib/spark/config.rb,
lib/spark/ext/io.rb,
lib/spark/helper.rb,
lib/spark/logger.rb,
lib/spark/command.rb,
lib/spark/context.rb,
lib/spark/sampler.rb,
lib/spark/version.rb,
lib/spark/constant.rb,
lib/spark/ext/hash.rb,
lib/spark/broadcast.rb,
lib/spark/ext/module.rb,
lib/spark/ext/object.rb,
lib/spark/ext/string.rb,
lib/spark/serializer.rb,
lib/spark/accumulator.rb,
lib/spark/accumulator.rb,
lib/spark/ext/integer.rb,
lib/spark/java_bridge.rb,
lib/spark/mllib/matrix.rb,
lib/spark/mllib/matrix.rb,
lib/spark/mllib/matrix.rb,
lib/spark/mllib/matrix.rb,
lib/spark/mllib/vector.rb,
lib/spark/mllib/vector.rb,
lib/spark/mllib/vector.rb,
lib/spark/mllib/vector.rb,
lib/spark/stat_counter.rb,
lib/spark/ext/ip_socket.rb,
lib/spark/helper/logger.rb,
lib/spark/helper/parser.rb,
lib/spark/helper/system.rb,
lib/spark/serializer/oj.rb,
lib/spark/storage_level.rb,
lib/spark/command_builder.rb,
lib/spark/java_bridge/rjb.rb,
lib/spark/serializer/base.rb,
lib/spark/serializer/pair.rb,
lib/spark/serializer/text.rb,
lib/spark/helper/serialize.rb,
lib/spark/helper/statistic.rb,
lib/spark/java_bridge/base.rb,
lib/spark/command_validator.rb,
lib/spark/java_bridge/jruby.rb,
lib/spark/serializer/batched.rb,
lib/spark/serializer/marshal.rb,
lib/spark/serializer/cartesian.rb,
lib/spark/serializer/compressed.rb,
lib/spark/mllib/regression/lasso.rb,
lib/spark/mllib/regression/ridge.rb,
lib/spark/mllib/clustering/kmeans.rb,
lib/spark/mllib/clustering/kmeans.rb,
lib/spark/mllib/regression/common.rb,
lib/spark/mllib/regression/common.rb,
lib/spark/mllib/regression/linear.rb,
lib/spark/serializer/auto_batched.rb,
lib/spark/serializer/message_pack.rb,
lib/spark/mllib/classification/svm.rb,
lib/spark/mllib/classification/svm.rb,
lib/spark/mllib/classification/common.rb,
lib/spark/mllib/classification/common.rb,
lib/spark/mllib/regression/labeled_point.rb,
lib/spark/mllib/classification/naive_bayes.rb,
lib/spark/mllib/classification/naive_bayes.rb,
lib/spark/mllib/ruby_matrix/matrix_adapter.rb,
lib/spark/mllib/ruby_matrix/vector_adapter.rb,
lib/spark/mllib/clustering/gaussian_mixture.rb,
lib/spark/mllib/clustering/gaussian_mixture.rb,
lib/spark/mllib/classification/logistic_regression.rb,
lib/spark/mllib/classification/logistic_regression.rb,
lib/spark/mllib/classification/logistic_regression.rb,
ext/ruby_c/ruby-spark.c

Overview

Spark::JavaBridge::Base

Parent for all adapter (ruby - java)

Defined Under Namespace

Modules: Build, CommandValidator, Constant, CoreExtension, Digest, Helper, InternalSorter, JavaBridge, Mllib, RandomGenerator, Sampler, Serializer Classes: Accumulator, AccumulatorError, Broadcast, BroadcastError, BuildError, CLI, Command, CommandBuilder, CommandError, Config, ConfigurationError, Context, ContextError, ExternalSorter, Logger, MllibError, NotImplemented, ParseError, PipelinedRDD, RDD, RDDError, SerializeError, StatCounter, StorageLevel

Constant Summary collapse

VERSION = 
'1.1.0.1'

Class Method Summary collapse

Methods included from Helper::System

included

Class Method Details

.clear_config ⇒ Object

Destroy current configuration. This can be useful for restarting config to set new. It has no effect if context is already started.



72
73
74
 
# File 'lib/spark.rb', line 72

def self.clear_config
  @config = nil
end

.config(&block) ⇒ Object

Returns current configuration. Configurations can be changed until context is initialized. In this case config is locked only for reading.

Configuration can be changed:

Spark.config.set('spark.app.name', 'RubySpark')

Spark.config['spark.app.name'] = 'RubySpark'

Spark.config do
  set 'spark.app.name', 'RubySpark'
end



60
61
62
63
64
65
66
67
68
 
# File 'lib/spark.rb', line 60

def self.config(&block)
  @config ||= Spark::Config.new

  if block_given?
    @config.instance_eval(&block)
  else
    @config
  end
end

.context ⇒ Object Also known as: sc

Return a current active context or nil.

TODO: Run ‘start` if context is nil?



80
81
82
 
# File 'lib/spark.rb', line 80

def self.context
  @context
end

.home ⇒ Object 



119
120
121
 
# File 'lib/spark.rb', line 119

def self.home
  root
end

.java_bridge ⇒ Object Also known as: jb 



163
164
165
 
# File 'lib/spark.rb', line 163

def self.java_bridge
  @java_bridge
end

.load_lib(spark_home = nil) ⇒ Object

Load dependent libraries, can be use once Cannot load before CLI::install

Parameters:

spark_home

path to directory where are located sparks .jar files or single Spark jar



152
153
154
155
156
157
158
159
160
161
 
# File 'lib/spark.rb', line 152

def self.load_lib(spark_home=nil)
  return if @java_bridge

  spark_home ||= Spark.target_dir

  bridge = JavaBridge.get
  @java_bridge = bridge.new(spark_home)
  @java_bridge.load
  nil
end

.logger ⇒ Object 



110
111
112
 
# File 'lib/spark.rb', line 110

def self.logger
  @logger ||= Spark::Logger.new
end

.print_logo(message = nil) ⇒ Object 



32
33
34
35
36
37
38
39
40
41
42
43
44
45
 
# File 'lib/spark.rb', line 32

def self.(message=nil)
  puts <<-STRING

  Welcome to
                __           ____              __
      ______ __/ /  __ __   / __/__  ___ _____/ /__
     / __/ // / _ \\/ // /  _\\ \\/ _ \\/ _ `/ __/  '_/
    /_/  \\_,_/_.__/\\_, /  /___/ .__/\\_,_/_/ /_/\\_\\   version #{Spark::VERSION}
                  /___/      /_/

  #{message}

  STRING
end

.root ⇒ Object

Root of the gem



115
116
117
 
# File 'lib/spark.rb', line 115

def self.root
  @root ||= File.expand_path('..', File.dirname(__FILE__))
end

.ruby_spark_jar ⇒ Object 



133
134
135
 
# File 'lib/spark.rb', line 133

def self.ruby_spark_jar
  @ruby_spark_jar ||= File.join(target_dir, 'ruby-spark.jar')
end

.spark_ext_dir ⇒ Object 



137
138
139
 
# File 'lib/spark.rb', line 137

def self.spark_ext_dir
  @spark_ext_dir ||= File.join(root, 'ext', 'spark')
end

.start ⇒ Object

Initialize spark context if not already. Config will be automatically loaded on constructor. From that point ‘config` will use configuration from running Spark and will be locked only for reading.



87
88
89
90
91
92
93
 
# File 'lib/spark.rb', line 87

def self.start
  if started?
    # Already started
  else
    @context ||= Spark::Context.new
  end
end

.started? ⇒ Boolean

Returns:

  • (Boolean) 


106
107
108
 
# File 'lib/spark.rb', line 106

def self.started?
  !!@context
end

.stop ⇒ Object 



95
96
97
98
99
100
101
102
103
104
 
# File 'lib/spark.rb', line 95

def self.stop
  @context.stop
  RubyWorker.stopServer
  logger.info('Workers were stopped')
rescue
  nil
ensure
  @context = nil
  clear_config
end

.target_dir ⇒ Object

Default directory for java extensions



124
125
126
 
# File 'lib/spark.rb', line 124

def self.target_dir
  @target_dir ||= File.join(root, 'target')
end

.worker_dir ⇒ Object

Directory where is worker.rb



129
130
131
 
# File 'lib/spark.rb', line 129

def self.worker_dir
  @worker_dir ||= File.join(root, 'lib', 'spark', 'worker')
end