Class: SqlLexer::Lexer

Inherits:
Object show all
Defined in:
lib/sql_lexer/lexer.rb

Constant Summary collapse

StartID =

SQL identifiers and key words must begin with a letter (a-z, but also letters with diacritical marks and non-Latin letters) or an underscore (_). Subsequent characters in an identifier or key word can be letters, underscores, digits (0-9), or dollar signs ($). Note that dollar signs are not allowed in identifiers according to the letter of the SQL standard, so their use might render applications less portable. The SQL standard will not define a key word that contains digits or starts or ends with an underscore, so identifiers of this form are safe against possible conflict with future extensions of the standard.

%q<\p{Alpha}_>
PartID = 
%q<\p{Alnum}_$>
OpPart = 
%q<\+|\-(?!-)|\*|/(?!\*)|\<|\>|=|~|!|@|#|%|\^|&|\||\?|\.|,|\(|\)>
WS = 
%q< \t\r\n>
OptWS = 
%Q<[#{WS}]*>
End = 
%Q<;|$>
InOp = 
%Q<IN(?=#{OptWS}\\()>
ArrayOp = 
%q<ARRAY(?=\[)>
ColonColonOp = 
%Q<::(?=[#{StartID}])>
ArrayIndexOp = 
%q<\\[(?:\-?\d+(?::\-?\d+)?|NULL)\\]>
SpecialOps = 
%Q<#{InOp}(?=[#{WS}])|#{ColonColonOp}|#{ArrayOp}|#{ArrayIndexOp}>
StartQuotedID = 
%Q<">
StartTickedID = 
%Q<`>
StartString = 
%Q<[a-zA-Z]?'>
StartDigit = 
%q<[\p{Digit}\.]>
StartSelect = 
%Q<SELECT(?=(?:[#{WS}]|#{OpPart}))>
StartBind =

Binds that are also IDs do not need to be included here, since AfterOp (which uses StartBind) also checks for StartAnyId

%Q<#{StartString}|#{StartDigit}|#{SpecialOps}>
StartNonBind = 
%Q<#{StartQuotedID}|#{StartTickedID}|\\$(?=\\p{Digit})>
TableNext = 
%Q<(#{OptWS}((?=#{StartQuotedID})|(?=#{StartTickedID}))|[#{WS}]+(?=[#{StartID}]))>
StartAnyId = 
%Q<"#{StartID}>
Placeholder = 
%q<\$\p{Digit}+>
AfterID = 
%Q<[#{WS};#{StartNonBind}]|(?:#{OpPart})|(?:#{ColonColonOp})|(?:#{ArrayIndexOp})|$>
ID = 
%Q<[#{StartID}][#{PartID}]*(?=#{AfterID})>
AfterOp = 
%Q<[#{WS}]|[#{StartAnyId}]|[#{StartBind}]|(#{StartNonBind})|;|$>
Op = 
%Q<(?:#{OpPart})+(?=#{AfterOp})>
QuotedID = 
%Q<#{StartQuotedID}(?:[^"]|"")*">
TickedID = 
%Q<#{StartTickedID}(?:[^`]|``)*`>
NonBind = 
%Q<#{ID}|#{Op}|#{QuotedID}|#{TickedID}|#{Placeholder}>
Type = 
%Q<[#{StartID}][#{PartID}]*(?:\\(\d+\\)|\\[\\])?(?=#{AfterID})>
QuotedTable = 
%Q<#{TickedID}|#{QuotedID}>
StringBody = 
%q{(?:''|(?<!\x5C)(?:\x5C\x5C)*\x5C'|[^'])*}
String = 
%Q<#{StartString}#{StringBody}'>
Digits = 
%q<\p{Digit}+>
OptDigits = 
%q<\p{Digit}*>
Exponent = 
%Q<e[+\-]?#{Digits}>
OptExponent = 
%Q<(?:#{Exponent})?>
HeadDecimal = 
%Q<#{Digits}\\.#{OptDigits}#{OptExponent}>
TailDecimal = 
%Q<#{OptDigits}\\.#{Digits}#{OptExponent}>
ExpDecimal = 
%Q<#{Digits}#{Exponent}>
Number = 
%Q<#{HeadDecimal}|#{TailDecimal}|#{ExpDecimal}|#{Digits}>
Literals = 
%Q<(?:NULL|TRUE|FALSE)(?=(?:[#{WS}]|#{OpPart}|#{End}))>
TkWS = 
%r<[#{WS}]+>u
TkOptWS = 
%r<[#{WS}]*>u
TkOp = 
%r<[#{OpPart}]>u
TkComment = 
%r<^#{OptWS}--.*$>u
TkBlockCommentStart = 
%r</\*>u
TkBlockCommentEnd = 
%r<\*/>u
TkPlaceholder = 
%r<#{Placeholder}>u
TkNonBind = 
%r<#{NonBind}>u
TkType = 
%r<#{Type}>u
TkQuotedTable = 
%r<#{QuotedTable}>iu
TkUpdateTable = 
%r<UPDATE#{TableNext}>iu
TkInsertTable = 
%r<INSERT[#{WS}]+INTO#{TableNext}>iu
TkDeleteTable = 
%r<DELETE[#{WS}]+FROM#{TableNext}>iu
TkFromTable = 
%r<FROM#{TableNext}>iu
TkID = 
%r<#{ID}>u
TkEnd = 
%r<;?[#{WS}]*>u
TkBind = 
%r<#{String}|#{Number}|#{Literals}>u
TkIn = 
%r<#{InOp}>iu
TkColonColon = 
%r<#{ColonColonOp}>u
TkArray = 
%r<#{ArrayOp}>iu
TkArrayIndex = 
%r<#{ArrayIndexOp}>iu
TkSpecialOp = 
%r<#{SpecialOps}>iu
TkStartSelect = 
%r<#{StartSelect}>iu
TkStartSubquery = 
%r<\(#{OptWS}#{StartSelect}>iu
TkCloseParen = 
%r<#{OptWS}\)>u
STATE_HANDLERS = 
{
  begin:       :process_begin,
  first_token: :process_first_token,
  tokens:      :process_tokens,
  bind:        :process_bind,
  non_bind:    :process_non_bind,
  placeholder: :process_placeholder,
  table_name:  :process_table_name,
  end:         :process_end,
  special:     :process_special,
  subquery:    :process_subquery,
  in:          :process_in,
  array:       :process_array
}
SCANNER_KEY = 
:__skylight_sql_scanner
LEXER_KEY = 
:__skylight_sql_lexer
SPACE = 
" ".freeze
DEBUG = 

      
        
PLACEHOLDER =
          
        

        
"?".freeze

      
        
UNKNOWN =
          
        

        
"<unknown>".freeze

      
        
OP_SELECT_FROM =
          
        

        
"SELECT FROM".freeze

      
        
OP_UPDATE =
          
        

        
"UPDATE".freeze

      
        
OP_INSERT_INTO =
          
        

        
"INSERT INTO".freeze

      
        
OP_DELETE_FROM =
          
        

        
"DELETE FROM".freeze

      
    

  




  
Instance Attribute Summary collapse

  




  
    

      Class Method Summary
      collapse
    


    
  
    

      Instance Method Summary
      collapse
    


    
  


  

    
Instance Attribute Details

    
      
      
      

  

  
    #binds  ⇒ Object  (readonly)
  

  

  




  

    

Returns the value of attribute binds.



  





  



  

    
      


117
118
119

    		
    
      
# File 'lib/sql_lexer/lexer.rb', line 117

def binds
  @binds
end

    		
  





    
      
      
      

  

  
    #output  ⇒ Object  (readonly)
  

  

  




  

    

Returns the value of attribute output.



  





  



  

    
      


117
118
119

    		
    
      
# File 'lib/sql_lexer/lexer.rb', line 117

def output
  @output
end

    		
  





    
      
      
      

  

  
    #title  ⇒ Object  (readonly)
  

  

  




  

    

Returns the value of attribute title.



  





  



  

    
      


117
118
119

    		
    
      
# File 'lib/sql_lexer/lexer.rb', line 117

def title
  @title
end

    		
  





    
  



  

    
Class Method Details


    
      

  

  
    .bindify(string, binds = nil, strip_comments = false)  ⇒ Object 
  

  

  




  

    
      


111
112
113
114
115

    		
    
      
# File 'lib/sql_lexer/lexer.rb', line 111

def self.bindify(string, binds=nil, strip_comments=false)
  scanner = instance(string)
  scanner.process(binds, strip_comments)
  [scanner.title, scanner.output, scanner.binds]
end

    		
  





    
      

  

  
    .instance(string)  ⇒ Object 
  

  

  




  

    
      


142
143
144
145
146

    		
    
      
# File 'lib/sql_lexer/lexer.rb', line 142

def self.instance(string)
  lexer = Thread.current[LEXER_KEY] ||= new
  lexer.init(string)
  lexer
end

    		
  





    
      

  

  
    .pooled_value(name, default)  ⇒ Object 
  

  

  




  

    
      


119
120
121
122
123
124
125
126
127
128
129
130
131

    		
    
      
# File 'lib/sql_lexer/lexer.rb', line 119

def self.pooled_value(name, default)
  key = :"__skylight_sql_#{name}"

  singleton_class.class_eval do
    define_method(name) do
      value = Thread.current[key] ||= default.dup
      value.clear
      value
    end
  end

  __send__(name)
end

    		
  





    
      

  

  
    .scanner(string = '')  ⇒ Object 
  

  

  




  

    
      


136
137
138
139
140

    		
    
      
# File 'lib/sql_lexer/lexer.rb', line 136

def self.scanner(string='')
  scanner = Thread.current[SCANNER_KEY] ||= StringScanner.new('')
  scanner.string = string
  scanner
end

    		
  





    
  


  

    
Instance Method Details


    
      

  

  
    #init(string)  ⇒ Object 
  

  

  




  

    
      


155
156
157
158
159
160
161
162
163
164

    		
    
      
# File 'lib/sql_lexer/lexer.rb', line 155

def init(string)
  @state   = :begin
  @debug   = DEBUG
  @binds   = self.class.binds
  @table   = self.class.table
  @title   = nil
  @bind    = 0

  self.string = string
end

    		
  





    
      

  

  
    #process(binds, strip_comments)  ⇒ Object 
  

  

  




  

    
      


179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221

    		
    
      
# File 'lib/sql_lexer/lexer.rb', line 179

def process(binds, strip_comments)
  process_comments(strip_comments)

  @operation = nil
  @provided_binds = binds

  while @state
    if @debug
      p @state
      p @scanner
    end

    __send__ STATE_HANDLERS[@state]
  end

  pos = 0
  removed = 0

  # intentionally allocates; the returned binds must
  # be in a newly produced array
  extracted_binds = Array.new(@binds.size / 2)

  if @operation && !@table.empty?
    @title = "" << @operation << SPACE << @table
  end

  while pos < @binds.size
    if @binds[pos] == nil
      extracted_binds[pos/2] = @binds[pos+1]
    else
      slice = @output[@binds[pos] - removed, @binds[pos+1]]
      @output[@binds[pos] - removed, @binds[pos+1]] = PLACEHOLDER

      extracted_binds[pos/2] = slice
      removed += (@binds[pos+1] - 1)
    end

    pos += 2
  end

  @binds = extracted_binds
  nil
end

    		
  





    
      

  

  
    #process_array  ⇒ Object 
  

  

  




  

    
      


478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526

    		
    
      
# File 'lib/sql_lexer/lexer.rb', line 478

def process_array
  nest = 1
  iterations = 0

  @skip_binds = true
  pos = @scanner.charpos - 6

  while nest > 0
    iterations += 1

    if iterations > 10_000
      raise "The SQL '#{@scanner.string}' could not be parsed because of too many iterations in ARRAY"
    end

    if @debug
      p "array loop"
      p @state
      p @scanner
    end

    if @scanner.skip(/\[/u)
      nest += 1
    elsif @scanner.skip(/\]/u)
      nest -= 1

      break if nest.zero?

      # End of final nested array
      next if @scanner.skip(/#{TkOptWS}(?=\])/u)
    end

    # A NULL array
    next if @scanner.skip(/NULL/iu)

    # Another nested array
    next if @scanner.skip(/#{TkOptWS},#{TkOptWS}(?=\[)/u)

    process_tokens

    __send__ STATE_HANDLERS[@state]
  end

  @binds << pos
  @binds << @scanner.charpos - pos

  @skip_binds = false

  @state = :tokens
end

    		
  





    
      

  

  
    #process_begin  ⇒ Object 
  

  

  




  

    
      


301
302
303
304

    		
    
      
# File 'lib/sql_lexer/lexer.rb', line 301

def process_begin
  @scanner.skip(TkOptWS)
  @state = :first_token
end

    		
  





    
      

  

  
    #process_bind  ⇒ Object 
  

  

  




  

    
      


533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548

    		
    
      
# File 'lib/sql_lexer/lexer.rb', line 533

def process_bind
  pos = nil

  unless @skip_binds
    pos = @scanner.charpos
  end

  @scanner.skip(TkBind)

  unless @skip_binds
    @binds << pos
    @binds << @scanner.charpos - pos
  end

  @state = :tokens
end

    		
  





    
      

  

  
    #process_comments(strip_comments)  ⇒ Object 
  

  

  




  

    
      


247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299

    		
    
      
# File 'lib/sql_lexer/lexer.rb', line 247

def process_comments(strip_comments)
  @original_input = @input

  # SQL treats comments as similar to whitespace
  # Here we replace all comments with spaces of the same length so as to not affect binds

  # Remove block comments
  # SQL allows for nested comments so this takes a bit more work
  while @scanner.skip_until(TkBlockCommentStart)
    count = 1
    pos = @scanner.charpos - 2

    while true
      # Determine whether we close the comment or start nesting
      next_open  = @scanner.skip_until(TkBlockCommentStart)
      @scanner.unscan if next_open
      next_close = @scanner.skip_until(TkBlockCommentEnd)
      @scanner.unscan if next_close

      if next_open && next_open < next_close
        # We're nesting
        count += 1
        @scanner.skip_until(TkBlockCommentStart)
      else
        # We're closing
        count -= 1
        @scanner.skip_until(TkBlockCommentEnd)
      end

      if count > 10_000
        raise "The SQL '#{@scanner.string}' could not be parsed because of too many iterations in block comments"
      end

      if count == 0
        # We've closed all comments
        length = @scanner.charpos - pos
        replace_comment(pos, length, strip_comments)
        break
      end
    end
  end

  @scanner.reset

  # Remove single line comments
  while @scanner.skip_until(TkComment)
    pos = @scanner.charpos
    len = @scanner.matched_size
    replace_comment(pos-len, len, strip_comments)
  end

  @scanner.reset
end

    		
  





    
      

  

  
    #process_end  ⇒ Object 
  

  

  




  

    
      


550
551
552
553
554
555
556
557
558
559
560
561
562
563

    		
    
      
# File 'lib/sql_lexer/lexer.rb', line 550

def process_end
  if @scanner.skip(TkEnd)
    if @scanner.eos?
      @state = nil
    else
      process_tokens
    end
  end

  # We didn't hit EOS and we couldn't process any tokens
  if @state == :end
    raise "The SQL '#{@scanner.string}' could not be parsed"
  end
end

    		
  





    
      

  

  
    #process_first_token  ⇒ Object 
  

  

  




  

    
      


311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326

    		
    
      
# File 'lib/sql_lexer/lexer.rb', line 311

def process_first_token
  if @scanner.skip(TkStartSelect)
    @operation = OP_SELECT_FROM
    @state = :tokens
  else
    if @scanner.skip(TkUpdateTable)
      @operation = OP_UPDATE
    elsif @scanner.skip(TkInsertTable)
      @operation = OP_INSERT_INTO
    elsif @scanner.skip(TkDeleteTable)
      @operation = OP_DELETE_FROM
    end

    @state = :table_name
  end
end

    		
  





    
      

  

  
    #process_in  ⇒ Object 
  

  

  




  

    
      


436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476

    		
    
      
# File 'lib/sql_lexer/lexer.rb', line 436

def process_in
  nest = 1
  iterations = 0

  @skip_binds = true
  pos = @scanner.charpos - 1

  while nest > 0
    iterations += 1

    if iterations > 10_000
      raise "The SQL '#{@scanner.string}' could not be parsed because of too many iterations in IN"
    end

    if @debug
      p @state
      p @scanner
      p nest
    end

    if @scanner.skip(/\(/u)
      nest += 1
      process_tokens
    elsif @scanner.skip(TkCloseParen)
      nest -= 1
      break if nest.zero?
      process_tokens
    else
      process_tokens
    end

    __send__ STATE_HANDLERS[@state]
  end

  @binds << pos
  @binds << @scanner.charpos - pos

  @skip_binds = false

  @state = :tokens
end

    		
  





    
      

  

  
    #process_non_bind  ⇒ Object 
  

  

  




  

    
      


528
529
530
531

    		
    
      
# File 'lib/sql_lexer/lexer.rb', line 528

def process_non_bind
  @scanner.skip(TkNonBind)
  @state = :tokens
end

    		
  





    
      

  

  
    #process_placeholder  ⇒ Object 
  

  

  




  

    
      


358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374

    		
    
      
# File 'lib/sql_lexer/lexer.rb', line 358

def process_placeholder
  @scanner.skip(TkPlaceholder)

  binds << nil

  if !@provided_binds
    @binds << UNKNOWN
  elsif !@provided_binds[@bind]
    @binds << UNKNOWN
  else
    @binds << @provided_binds[@bind]
  end

  @bind += 1

  @state = :tokens
end

    		
  





    
      

  

  
    #process_special  ⇒ Object 
  

  

  




  

    
      


376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399

    		
    
      
# File 'lib/sql_lexer/lexer.rb', line 376

def process_special
  if @scanner.skip(TkIn)
    @scanner.skip(TkOptWS)
    if @scanner.skip(TkStartSubquery)
      @state = :subquery
    else
      @scanner.skip(/\(/u)
      @state = :in
    end
  elsif @scanner.skip(TkArray)
    @scanner.skip(/\[/u)
    @state = :array
  elsif @scanner.skip(TkColonColon)
    if @scanner.skip(TkType)
      @state = :tokens
    else
      @state = :end
    end
  elsif @scanner.skip(TkStartSubquery)
    @state = :subquery
  elsif @scanner.skip(TkArrayIndex)
    @state = :tokens
  end
end

    		
  





    
      

  

  
    #process_subquery  ⇒ Object 
  

  

  




  

    
      


401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434

    		
    
      
# File 'lib/sql_lexer/lexer.rb', line 401

def process_subquery
  nest = 1
  iterations = 0

  while nest > 0
    iterations += 1

    if iterations > 10_000
      raise "The SQL '#{@scanner.string}' could not be parsed because of too many iterations in subquery"
    end

    if @debug
      p @state
      p @scanner
      p nest
      p @scanner.peek(1)
    end

    if @scanner.skip(TkStartSubquery)
      nest += 1
      @state = :tokens
    elsif @scanner.skip(TkCloseParen)
      nest -= 1
      break if nest.zero?
      @state = :tokens
    elsif @state == :subquery
      @state = :tokens
    end

    __send__ STATE_HANDLERS[@state]
  end

  @state = :tokens
end

    		
  





    
      

  

  
    #process_table_name  ⇒ Object 
  

  

  




  

    
      


328
329
330
331
332
333
334
335
336
337
338

    		
    
      
# File 'lib/sql_lexer/lexer.rb', line 328

def process_table_name
  pos = @scanner.pos

  if @scanner.skip(TkQuotedTable)
    copy_substr(@input, @table, pos + 1, @scanner.pos - 1)
  elsif @scanner.skip(TkID)
    copy_substr(@input, @table, pos, @scanner.pos)
  end

  @state = :tokens
end

    		
  





    
      

  

  
    #process_tokens  ⇒ Object 
  

  

  




  

    
      


340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356

    		
    
      
# File 'lib/sql_lexer/lexer.rb', line 340

def process_tokens
  @scanner.skip(TkOptWS)

  if @operation == OP_SELECT_FROM && @table.empty? && @scanner.skip(TkFromTable)
    @state = :table_name
  elsif @scanner.match?(TkSpecialOp)
    @state = :special
  elsif @scanner.match?(TkBind)
    @state = :bind
  elsif @scanner.match?(TkPlaceholder)
    @state = :placeholder
  elsif @scanner.match?(TkNonBind)
    @state = :non_bind
  else
    @state = :end
  end
end

    		
  





    
      

  

  
    #replace_comment(pos, length, strip)  ⇒ Object 
  

  

  




  

    
      


223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245

    		
    
      
# File 'lib/sql_lexer/lexer.rb', line 223

def replace_comment(pos, length, strip)
  if strip
    # Dup the string if necessary so we aren't destructive to the original value
    if @input == @original_input
      @input = @input.dup
      @scanner.string = @input
    end

    # Replace the comment with a space to ensure valid SQL
    # Updating the input also updates the scanner
    @input[pos, length] = SPACE
    @output[pos, length] = SPACE

    # Move back to start of removed string
    @scanner.pos = pos
  else
    # Dup the string if necessary so we aren't destructive to the original value
    @scanner.string = @input.dup if @scanner.string == @original_input

    # Replace the comment with spaces
    @scanner.string[pos, length] = SPACE*length
  end
end

    		
  





    
      

  

  
    #string=(value)  ⇒ Object 
  

  

  




  

    
      


166
167
168
169
170
171
172
173
174

    		
    
      
# File 'lib/sql_lexer/lexer.rb', line 166

def string=(value)
  @input   = value

  @scanner = self.class.scanner(value)

  # intentionally allocates; we need to return a new
  # string as part of this API
  @output = value.dup
end