Class: Fairy::Import

Inherits:
Object
  • Object
show all
Includes:
Enumerable
Defined in:
lib/fairy/share/port.rb

Defined Under Namespace

Classes: CTLTOKEN, CTLTOKEN_DELAYED_ELEMENT, CTLTOKEN_NULLVALUE, CTLTOKEN_SET_NO_IMPORT

Constant Summary collapse

END_OF_STREAM = 
:END_OF_STREAM
SET_NO_IMPORT = 
CTLTOKEN_SET_NO_IMPORT.new
TOKEN_NULLVALUE = 
CTLTOKEN_NULLVALUE.new

Instance Attribute Summary collapse

Instance Method Summary collapse

Constructor Details

#initialize(policy = nil) ⇒ Import

Returns a new instance of Import.



72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
 
# File 'lib/fairy/share/port.rb', line 72

def initialize(policy = nil)

  @queuing_policy = policy
  @queuing_policy ||= CONF.PREQUEUING_POLICY

  case @queuing_policy
  when Hash
  @queue = eval("#{@queuing_policy[:queuing_class]}").new(@queuing_policy)
  else
  @queue = @queuing_policy
  end

  Log::debug(self, "Using Buffer: #{@queue.class}")
Log::debug(self, "Inspect: #{@queue.inspect}")
  
  @log_import_ntimes_pop = CONF.LOG_IMPORT_NTIMES_POP
  @log_callback_proc = nil

  @njob_id = nil

  @no = nil
  @no_mutex = Mutex.new
  @no_cv = XThread::ConditionVariable.new

  @key = nil

  @no_import = nil

  @no_pop_elements = 0
  @no_eos = 0

  @DEBUG_PORT_WAIT = CONF.DEBUG_PORT_WAIT
end

Instance Attribute Details

#key ⇒ Object

Returns the value of attribute key.



130
131
132
 
# File 'lib/fairy/share/port.rb', line 130

def key
  @key
end

#njob_id ⇒ Object

Returns the value of attribute njob_id.



106
107
108
 
# File 'lib/fairy/share/port.rb', line 106

def njob_id
  @njob_id
end

Instance Method Details

#add_key(key) ⇒ Object 



131
132
133
 
# File 'lib/fairy/share/port.rb', line 131

def add_key(key)
  @key = key
end

#asynchronus_send_with_callback(method, *args, &call_back) ⇒ Object

疑似対応



311
312
313
314
315
316
317
318
319
 
# File 'lib/fairy/share/port.rb', line 311

def asynchronus_send_with_callback(method, *args, &call_back)
  ret = nil
  exp = nil
  begin
  ret = __send__(method, *args)
  rescue => exp
  end
  call_back.call(ret, exp)
end

#context_eval(str) ⇒ Object 



298
299
300
 
# File 'lib/fairy/share/port.rb', line 298

def context_eval(str)
  eval str
end

#each(&block) ⇒ Object 



249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
 
# File 'lib/fairy/share/port.rb', line 249

def each(&block)
  Log::debug(self, "START IMPORT")
  while !@no_import or @no_import > @no_eos
  e = @queue.pop
  case e
  when CTLTOKEN_DELAYED_ELEMENT
    e = e.get_element(self)
  end

  case e
  when CTLTOKEN_SET_NO_IMPORT
  #when SET_NO_IMPORT
    # do nothing
  when END_OF_STREAM
    @no_eos += 1
    if @log_callback_proc
 @log_callback_proc.call "EOS(#{@no_eos}/#{@no_import})", @key
    else
 Log::debug(self, "IMPORT POP key=#{@key}: EOS(#{@no_eos}/#{@no_import})")
    end
  else
    @no_pop_elements += 1
    if @log_import_ntimes_pop && 
   (@no_pop_elements % @log_import_ntimes_pop == 0 || 
    @no_pop_elements == 1)
 if @log_callback_proc
   Log::verbose(self, "IMPORT POP key=#{@key}: #{@no_pop_elements}")
   @log_callback_proc.call @no_pop_elements, @key
 else
   Log::verbose(self, "IMPORT POP key=#{@key}: #{@no_pop_elements}")
 end
    end
    block.call(e)
  end
  end
  if @log_callback_proc
  @log_callback_proc.call "EOS(ALL)", @key
  else
  Log::verbose(self, "IMPORT POP key=#{@key}: EOS(ALL)")
  end
  Log::debug(self, "FINISH IMPORT")
end

#log_id ⇒ Object 



108
109
110
 
# File 'lib/fairy/share/port.rb', line 108

def log_id
  "Import[#{@njob_id}[#{@no}:#{@key}]]"
end

#no ⇒ Object 



112
113
114
115
116
117
118
119
120
121
 
# File 'lib/fairy/share/port.rb', line 112

def no
  @no_mutex.synchronize do
  while !@no
    Log::debug(self, "Wait until set @no") if @DEBUG_PORT_WAIT
    @no_cv.wait(@no_mutex)
    Log::debug(self, "End: Wait until set @no") if @DEBUG_PORT_WAIT
  end
  @no
  end
end

#no=(no) ⇒ Object 



123
124
125
126
127
128
 
# File 'lib/fairy/share/port.rb', line 123

def no=(no)
  @no_mutex.synchronize do
  @no=no
  @no_cv.broadcast
  end
end

#no_import=(n) ⇒ Object 



135
136
137
138
139
 
# File 'lib/fairy/share/port.rb', line 135

def no_import=(n)
  @no_import = n
  @queue.push SET_NO_IMPORT
  nil
end

#pop ⇒ Object 



210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
 
# File 'lib/fairy/share/port.rb', line 210

def pop
  while !@no_import or @no_import > @no_eos
  e = @queue.pop
  case e
  when CTLTOKEN_DELAYED_ELEMENT
    e = e.get_element(self)
  end
  case e
  when CTLTOKEN_DELAYED_ELEMENT
    e = e.get_element(import)
  when CTLTOKEN_SET_NO_IMPORT
  #when SET_NO_IMPORT
    # do nothing
  when END_OF_STREAM
    @no_eos += 1
    Log::debug(self, "IMPORT EOS key=#{@key}: #{@no_eos}/#{@no_import}")
  else
    @no_pop_elements += 1
    if @log_import_ntimes_pop && 
   (@no_pop_elements % @log_import_ntimes_pop == 0 || 
    @no_pop_elements == 1)
 if @log_callback_proc
   @log_callback_proc.call @no_pop_elements, @key
 else
   Log::verbose(self, "IMPORT POP key=#{@key}: #{@no_pop_elements}")
 end
    end

    return e
  end
  end
  if @log_callback_proc
  @log_callback_proc.call "EOS", @key
  else
  Log::verbose(self, "IMPORT POP key=#{@key}: EOS")
  end
  return nil
end

#push(e) ⇒ Object 



141
142
143
144
 
# File 'lib/fairy/share/port.rb', line 141

def push(e)
  @queue.push e
  nil
end

#push_buf(buf) ⇒ Object 



147
148
149
150
151
152
153
154
155
156
157
158
159
160
 
# File 'lib/fairy/share/port.rb', line 147

def push_buf(buf)
  if @queue.respond_to?(:push_all)
  @queue.push_all(buf)
  nil
  else
  begin 
    buf.each{|e| @queue.push e}
    nil
  rescue
    Log::debug_exception(self)
    raise
  end
  end
end

#push_keep_identity(e) ⇒ Object 



182
183
184
185
 
# File 'lib/fairy/share/port.rb', line 182

def push_keep_identity(e)
  @queue.push e
  nil
end

#push_raw(raw) ⇒ Object 



187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
 
# File 'lib/fairy/share/port.rb', line 187

def push_raw(raw)
  if @queue.respond_to?(:push_raw)
  @queue.push_raw(raw)
  nil
  elsif @queue.respond_to?(:push_buf)
  buf = Marshal.load(raw)
  @queue.push_raw(buf)
  nil
  else
  begin 
    buf = Marshal.load(raw)
    @queue.push_raw(buf)
    buf.each{|e| @queue.push e}
    nil
  rescue
    Log::debug_exception(self)
    raise
  end
  end
end

#push_strings(bigstr) ⇒ Object 



163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
 
# File 'lib/fairy/share/port.rb', line 163

def push_strings(bigstr)
  strings = bigstr.split("\t").collect{|e| 
  e.gsub(/(\\t|\\\\)/){|v| v == "\\t" ? "\t" : "\\"}
  }

  if @queue.respond_to?(:push_all)
  @queue.push_all(strings)
  nil
  else
  begin 
    strings.each{|e| @queue.push e}
    nil
  rescue
    Log::debug_exception(self)
    raise
  end
  end
end

#set_log_callback(str = nil, bind = binding, file = __FILE__, line_no = __LINE__, &block) ⇒ Object 



302
303
304
305
306
307
308
 
# File 'lib/fairy/share/port.rb', line 302

def set_log_callback(str = nil, bind = binding, file = __FILE__, line_no = __LINE__, &block)
  if str
    @log_callback_proc = eval(%{proc{#{str}}}, bind, file, line_no)
  else
    @log_callback_proc = block
  end
end

#size ⇒ Object 



292
293
294
295
296
 
# File 'lib/fairy/share/port.rb', line 292

def size
  size = 0
  each{size += 1}
  size
end