Module: MmEsSearch::Models::AbstractSearchModel

Extended by:
ActiveSupport::Concern
Includes:
Api::Facet, Api::Highlight, Api::Query, Api::Sort, MmEsSearch::Models, Utils
Defined in:
lib/mm_es_search/models/abstract_search_model.rb

Defined Under Namespace

Modules: ClassMethods

Instance Method Summary collapse

Instance Method Details

#all_facets_finished? ⇒ Boolean

Returns:

  • (Boolean) 


372
373
374
 
# File 'lib/mm_es_search/models/abstract_search_model.rb', line 372

def all_facets_finished?
  facets.all? { |f| f.current_state == :ready_for_display }
end

#build_filtered_query(query, filters) ⇒ Object 



483
484
485
486
487
488
489
490
491
492
493
494
 
# File 'lib/mm_es_search/models/abstract_search_model.rb', line 483

def build_filtered_query(query, filters)
  if filters.nil? or filters.empty?
    query
  else
    FilteredQuery.new(
    :query => query,
    :filter => AndFilter.new(
      :filters => filters
      )
    )
  end
end

#build_main_query_if_missing ⇒ Object 



287
288
289
 
# File 'lib/mm_es_search/models/abstract_search_model.rb', line 287

def build_main_query_if_missing
  self.query_object ||= build_main_query_object
end

#can_reuse_results? ⇒ Boolean

Returns:

  • (Boolean) 


195
196
197
 
# File 'lib/mm_es_search/models/abstract_search_model.rb', line 195

def can_reuse_results?
  !new_results_requested? && previous_results_fresh? && requested_page_in_top_results_range?
end

#combine_queries(scored, unscored) ⇒ Object 



411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
 
# File 'lib/mm_es_search/models/abstract_search_model.rb', line 411

def combine_queries(scored, unscored)
  query = if scored.empty? and unscored.empty?
    MatchAllQuery.new
  elsif scored.empty?
    ConstantScoreQuery.new(
      :boost => 1,
      :query => BoolQuery.new(
        :musts => unscored
      )
    )
  elsif unscored.empty?
    if scored.length > 1
      BoolQuery.new(
        :musts => scored
      )
    else
      scored.first
    end
  else
    # mod_scored = scored.map {|query| q = query.dup; q.boost = 1e100; q }
    mod_unscored = unscored.map {|query| q = query.dup; q.boost = 0; q }
    BoolQuery.new(
      :musts => scored + mod_unscored
    )
  end
end

#debug_off ⇒ Object 



515
516
517
518
519
 
# File 'lib/mm_es_search/models/abstract_search_model.rb', line 515

def debug_off
  self.debug  = nil
  @search_log = nil
  return self
end

#debug_on ⇒ Object 



502
503
504
505
506
 
# File 'lib/mm_es_search/models/abstract_search_model.rb', line 502

def debug_on
  self.debug = true
  prepare_log unless @search_log
  return self
end

#debug_on? ⇒ Boolean

Returns:

  • (Boolean) 


496
497
498
499
500
 
# File 'lib/mm_es_search/models/abstract_search_model.rb', line 496

def debug_on?
  on = !!debug
  prepare_log if on and @search_log.nil?
  on
end

#default_run_options ⇒ Object 



149
150
151
152
153
154
155
156
157
158
159
160
161
162
 
# File 'lib/mm_es_search/models/abstract_search_model.rb', line 149

def default_run_options
  @default_run_options ||= {
    :target           => :es,
    :force_refresh    => false,
    :page             => 1,
    :per_page         => 10,
    :fields           => [],
    :return           => :results,
    :sorted           => true,
    :highlight        => true,
    :facet_mode       => :auto,
    :autosave         => false
  }
end

#es_request(query_name, options = {}) ⇒ Object 



291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
 
# File 'lib/mm_es_search/models/abstract_search_model.rb', line 291

def es_request(query_name, options = {})
  
  request = {}
  
  if query_name == :main
    
    build_main_query_if_missing
    query = @sorted ? sorted_query : unsorted_query
    request.merge!(:sort => sort_object.to_es_query)            if @sorted and sort_object.is_a?(RootSortModel)
    request.merge!(:highlight => highlight_object.to_es_query)  if @highlight and highlight_object.present?
    
  else
    
    filters = [send("build_#{query_name}_query_object").to_filter]
    query   = build_filtered_query(MatchAllQuery.new, filters)

  end
  
  request.merge!(:query  => query.to_es_query, :query_dsl => false)
  request.merge!(:facets => @facet_es_queries) if @facet_es_queries.present?
  request

end

#execute_query(query_name, for_facets_only = false) ⇒ Object 



247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
 
# File 'lib/mm_es_search/models/abstract_search_model.rb', line 247

def execute_query(query_name, for_facets_only = false)
          
  case @target
  when :es
    
    prepare_facet_queries_for_query query_name unless @facet_mode == :none
    
    if for_facets_only
      page     = 1
      per_page = 0
      request  = es_request query_name, :sorted => false, :highlight => false
    elsif requested_page_in_top_results_range?
      page     = 1
      per_page = NUM_TOP_RESULTS
      request  = es_request query_name
    else
      page     = self.page
      per_page = self.per_page
      request  = es_request query_name
    end
    
    @search_log.info(request.except(:query_dsl).to_json) if debug_on?

    @response = target_collection.search_hits(
      request,
      :page     => page,
      :per_page => per_page,
      :ids_only => true,
      :type     => es_type_for_query(query_name)
    )
    
    @response
    
  when :mongo
    
    
    
  end
end

#extract_page_results_from_top_results ⇒ Object 



199
200
201
 
# File 'lib/mm_es_search/models/abstract_search_model.rb', line 199

def extract_page_results_from_top_results
  self.page_result_ids = top_result_ids[page_range]
end

#facets_as_filters ⇒ Object 



479
480
481
 
# File 'lib/mm_es_search/models/abstract_search_model.rb', line 479

def facets_as_filters
  used_facets.map(&:to_filter).compact.flatten
end

#have_pending_facets? ⇒ Boolean

Returns:

  • (Boolean) 


368
369
370
 
# File 'lib/mm_es_search/models/abstract_search_model.rb', line 368

def have_pending_facets?
  facets.any? { |f| f.current_state != :ready_for_display } || (@auto_explore_needed and type_facet_positively_set?)
end

#have_previous_results? ⇒ Boolean

Returns:

  • (Boolean) 


172
173
174
 
# File 'lib/mm_es_search/models/abstract_search_model.rb', line 172

def have_previous_results?
  top_result_ids.present?
end

#new_results_requested? ⇒ Boolean

Returns:

  • (Boolean) 


191
192
193
 
# File 'lib/mm_es_search/models/abstract_search_model.rb', line 191

def new_results_requested?
  @force_refresh || @raw_es_response
end

#page ⇒ Object 



164
165
166
 
# File 'lib/mm_es_search/models/abstract_search_model.rb', line 164

def page
  @page ||= 1
end

#page_range ⇒ Object 



185
186
187
188
189
 
# File 'lib/mm_es_search/models/abstract_search_model.rb', line 185

def page_range
  lower_index = (page - 1) * per_page
  upper_index = lower_index + per_page
  range       = lower_index...upper_index
end

#page_results ⇒ Object 



203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
 
# File 'lib/mm_es_search/models/abstract_search_model.rb', line 203

def page_results
  
  #fetch records from db in one call and then reorder to match search result ordering
  return paginate_records([]) unless page_result_ids.present?
  return @results if @results.present?
  
  #NOTE: I use #find_with_fields to avoid redefining the standard MM #find method
  # this can be trivially implemented with the plucky #where and #fields methods
  # but is directly implemented in MmUsesUuid
  unordered_records = target_collection.find_with_fields page_result_ids, :fields => @fields
  
  if unordered_records.is_a?(Array)
    records = unordered_records.reorder_by(page_result_ids.map(&:to_s), &Proc.new {|r| r.id.to_s})
  elsif unordered_records.nil?
    records = []
  else
    records = [unordered_records]
  end
  
  paginate_records(records)
  
end

#paginate_records(records) ⇒ Object 



226
227
228
229
230
 
# File 'lib/mm_es_search/models/abstract_search_model.rb', line 226

def paginate_records(records)
  @results = WillPaginate::Collection.new(page, per_page, result_total || 0)
  @results.replace(records)
  @results
end

#per_page ⇒ Object 



168
169
170
 
# File 'lib/mm_es_search/models/abstract_search_model.rb', line 168

def per_page
  @per_page ||= 10
end

#prefix_label(label) ⇒ Object 



376
377
378
 
# File 'lib/mm_es_search/models/abstract_search_model.rb', line 376

def prefix_label(label)
  AbstractFacetModel.prefix_label(self, label)
end

#prepare_facet_queries_for_query(query_name) ⇒ Object 



232
233
234
235
236
237
238
239
 
# File 'lib/mm_es_search/models/abstract_search_model.rb', line 232

def prepare_facet_queries_for_query(query_name)
  @facet_es_queries = {}
  (facets << self).each do |facet| #NOTE we add self, as search object manages exploratory facet queries
    queries = facet.es_facet_queries_for_query(query_name)
    @facet_es_queries.merge!(queries) if queries.present?
  end
  @facet_es_queries
end

#prepare_log ⇒ Object 



508
509
510
511
512
513
 
# File 'lib/mm_es_search/models/abstract_search_model.rb', line 508

def prepare_log
  logfile = File.open(Rails.root.to_s + '/log/search.log', 'a')
  logfile.sync = true
  @search_log = SearchLogger.new(logfile)
  #@search_log.info "#{self.class.name} now logging\n"
end

#previous_results_fresh? ⇒ Boolean

Returns:

  • (Boolean) 


176
177
178
179
 
# File 'lib/mm_es_search/models/abstract_search_model.rb', line 176

def previous_results_fresh?
  return false unless have_previous_results? and last_run_at.present?
  (Time.now - last_run_at) < RESULT_REUSE_PERIOD
end

#process_facet_results(results, target_object = nil) ⇒ Object 



241
242
243
244
245
 
# File 'lib/mm_es_search/models/abstract_search_model.rb', line 241

def process_facet_results(results, target_object = nil)
  results.each do |label, result|
    (target_object || self).send "handle_#{label}", result
  end
end

#process_query_results ⇒ Object 



315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
 
# File 'lib/mm_es_search/models/abstract_search_model.rb', line 315

def process_query_results
  
  case @response.hits.first
  when ElasticSearch::Api::Hit
    ids = @response.hits.map(&:_id)
  else
    ids = @response.hits
  end 

  if requested_page_in_top_results_range?
    self.top_result_ids  = ids
    extract_page_results_from_top_results
  else
    self.top_result_ids  = []
    self.page_result_ids = ids
  end
  
  self.result_total = @response.total_entries
  self.highlights   = @response.response['hits']['hits'].map {|hit| hit['highlight']} if highlight_object.present?
  
  self.last_run_at  = Time.now.utc
  
  if self.class::MIN_FACET_COVERAGE_COUNT and result_total < self.class::MIN_FACET_COVERAGE_COUNT
    @auto_explore_needed = false
  end
  
end

#process_run_options(options = {}) ⇒ Object 



131
132
133
134
135
136
137
138
 
# File 'lib/mm_es_search/models/abstract_search_model.rb', line 131

def process_run_options(options = {})
  #set instance variables for important options e.g. page, per_page
  validate_options(options)
  options = options.symbolize_keys.reverse_merge(default_run_options)
  options.each do |key, value|
    instance_variable_set "@#{key}", value
  end
end

#prune ⇒ Object 



364
365
366
 
# File 'lib/mm_es_search/models/abstract_search_model.rb', line 364

def prune
  puts "PRUNE WAS CALLED IN A #{self.class.name}"
end

#query_and_facets_as_filters ⇒ Object 



467
468
469
470
471
472
473
474
475
476
477
 
# File 'lib/mm_es_search/models/abstract_search_model.rb', line 467

def query_and_facets_as_filters
  filters = facets_as_filters
  unscored_queries = []
  query_as_filter = query_object.present? ? query_object.to_filter : nil
  if query_as_filter
    filters << query_as_filter
  elsif query_object.present?
    unscored_queries << query_object.to_query
  end
  return unscored_queries, filters
end

#remove_optional_facets ⇒ Object 



405
406
407
408
409
 
# File 'lib/mm_es_search/models/abstract_search_model.rb', line 405

def remove_optional_facets
  facets.each do |f|
    remove_facet f unless f.used? or f.required?
  end
end

#requested_page_in_top_results_range? ⇒ Boolean

Returns:

  • (Boolean) 


181
182
183
 
# File 'lib/mm_es_search/models/abstract_search_model.rb', line 181

def requested_page_in_top_results_range?
  page_range.last <= NUM_TOP_RESULTS
end

#required_facets ⇒ Object 



397
398
399
 
# File 'lib/mm_es_search/models/abstract_search_model.rb', line 397

def required_facets
  facets.select(&:required?)
end

#route_facet_query_results ⇒ Object 



343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
 
# File 'lib/mm_es_search/models/abstract_search_model.rb', line 343

def route_facet_query_results
  
  facet_results = @response.facets
  return unless facet_results.present?
  
  grouped_queries = Hash.new { |hash, id| hash[id] = {} }
  facet_results.each_with_object(grouped_queries) do |(label, result), hsh|
    label_parts     = label.split('_')
    id_prefix       = label_parts.shift.to_i
    trimmed_label   = label_parts.join('_')
    hsh[id_prefix].merge!(trimmed_label => result)
  end
  
  grouped_queries.each do |obj_id, results|
    query_owner = ObjectSpace._id2ref(obj_id)
    query_owner.process_facet_results results
  end

  
end

#run(options = {}) ⇒ Object 



38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
 
# File 'lib/mm_es_search/models/abstract_search_model.rb', line 38

def run(options = {})
  
  process_run_options(options)
  
  if can_reuse_results?
    
    puts "INFO: reusing previous results"
    extract_page_results_from_top_results
    page_results
    
  else
    
    @results = nil
    
    if @facet_mode == :auto
      remove_optional_facets
      @auto_explore_needed = true
# #NOTE hack for debugging
# @auto_explore_needed = false
      build_type_facet unless type_facet.present?
    end
    
    facets.each(&:prepare_for_new_data)
    
    if @facet_mode
      self.facet_status = :in_progress
    else
      self.facet_status = :none_requested
    end
    
    execute_query :main
    process_query_results
    route_facet_query_results
    
    if have_pending_facets?
      self.facet_status = :pending
    elsif all_facets_finished?
      self.facet_status = :complete
      puts "MARKING FACETS AS COMPLETE AFTER FIRST RUN"
    end
    
# #NOTE HACK while investigating search
# self.facet_status = :complete

    save if @autosave
    
    case @return
    when :raw_response
      @response
    when :ids
      page_result_ids
    when :results
      page_results
      @results #here as a reminder that this collection is memoized
    end
    
  end
  
end

#run_facets ⇒ Object 



98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
 
# File 'lib/mm_es_search/models/abstract_search_model.rb', line 98

def run_facets
  
  puts "STARTED RUNNING FACETS"
  
  time = Benchmark.measure do
    
    sanity_check = 0
    while have_pending_facets? and sanity_check < 10
      #binding.pry
      facet_parent_queries.each do |parent_query|
        execute_query parent_query, :for_facets_only 
        route_facet_query_results
      end
      sanity_check += 1
    end
    
    self.facet_status = :complete
    
    #NOTE: this can throw a stack overflow if using Fibres to call run_facets async
    #this appears to be due to the limited 4k stack of a Fibre
    #and the fact that saving calls a gazillion methods
    #for this reason I use the "defer" method in Celluloid
    #as this gives async without using fibres... or something...
    #... well it works, whatever it does...
    save if @autosave
    
  end
  
  puts "ENDED RUNNING FACETS #{time.inspect}"
  
end

#sort_query_and_facets_as_filters ⇒ Object 



461
462
463
464
465
 
# File 'lib/mm_es_search/models/abstract_search_model.rb', line 461

def sort_query_and_facets_as_filters
  unscored_queries, filters = query_and_facets_as_filters
  filters << sort_object.to_filter unless (sort_object.nil? or sort_object.is_a?(RootSortModel))
  return unscored_queries, filters
end

#sorted_query ⇒ Object 



445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
 
# File 'lib/mm_es_search/models/abstract_search_model.rb', line 445

def sorted_query
  build_main_query_if_missing
  if (sort_object.nil? and query_object.nil?) or sort_object.is_a?(RootSortModel)
    unsorted_query
  else
    if sort_object.nil?
      query = query_object.to_query
      filters = facets_as_filters
    else
      unscored_queries, filters = query_and_facets_as_filters
      query = combine_queries([sort_object.to_query], unscored_queries)
    end
    build_filtered_query(query, filters)
  end
end

#target_collection ⇒ Object 



521
522
523
524
525
526
 
# File 'lib/mm_es_search/models/abstract_search_model.rb', line 521

def target_collection
  #we assume name is of form klass.name + "Search"
  klass_match = self.class.name.match(/(?<klass>\w*)(?=Search)/)
  raise "expected the class name '#{self.class.name}' to be of form 'SomethingSearch' so that we can extract 'Something' as the target collection" unless klass_match[:klass]
  klass_match[:klass].constantize
end

#type_facet ⇒ Object 



380
381
382
 
# File 'lib/mm_es_search/models/abstract_search_model.rb', line 380

def type_facet
  facets.detect {|facet| facet.virtual_field == type_field}
end

#type_facet_positively_set? ⇒ Boolean

Returns:

  • (Boolean) 


384
385
386
387
 
# File 'lib/mm_es_search/models/abstract_search_model.rb', line 384

def type_facet_positively_set?
  return false unless type_facet.present?
  type_facet.positively_checked_rows.present?
end

#unsorted_query ⇒ Object 



438
439
440
441
442
443
 
# File 'lib/mm_es_search/models/abstract_search_model.rb', line 438

def unsorted_query
  build_main_query_if_missing
  unscored_queries, filters = sort_query_and_facets_as_filters #NOTE: we put non-RootSortModel sorts in as filters as these typically restrict results
  query = combine_queries([], unscored_queries)
  build_filtered_query(query, filters)
end

#unused_facets ⇒ Object 



393
394
395
 
# File 'lib/mm_es_search/models/abstract_search_model.rb', line 393

def unused_facets
  facets.select(&:unused?)
end

#used_facets ⇒ Object 



389
390
391
 
# File 'lib/mm_es_search/models/abstract_search_model.rb', line 389

def used_facets
  facets.select(&:used?)
end

#used_or_required_facets ⇒ Object 



401
402
403
 
# File 'lib/mm_es_search/models/abstract_search_model.rb', line 401

def used_or_required_facets
  facets.select(&:used_or_required?)
end

#validate_options(options) ⇒ Object 



140
141
142
143
144
145
146
147
 
# File 'lib/mm_es_search/models/abstract_search_model.rb', line 140

def validate_options(options)
  valid_options  = default_run_options.keys.to_set
  valid_options += valid_options.map(&:to_s)
  unless valid_options.superset?(options.keys.to_set)
    raise "invalid options passed"
  end
  true
end