Class: Jrr::Tokenizer

Inherits:
Object
  • Object
show all
Defined in:
lib/jrr/tokenizer.rb

Constant Summary collapse

LPAREN = 
Matcher.new(:grouping, :open)
RPAREN = 
Matcher.new(:grouping, :close)

Instance Attribute Summary collapse

Instance Method Summary collapse

Instance Attribute Details

#case_sensitive ⇒ Object (readonly)

Returns the value of attribute case_sensitive.



8
9
10
 
# File 'lib/jrr/tokenizer.rb', line 8

def case_sensitive
  @case_sensitive
end

Instance Method Details

#last_token ⇒ Object 



35
36
37
 
# File 'lib/jrr/tokenizer.rb', line 35

def last_token
  @tokens.last
end

#scan(string, scanner) ⇒ Object 



39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
 
# File 'lib/jrr/tokenizer.rb', line 39

def scan(string, scanner)
  if tokens = scanner.scan(string, last_token)
    tokens.each do |token|
      if token.empty?
        fail! :unexpected_zero_width_match,
              token_category: token.category, at: string
      end

      @nesting += 1 if LPAREN == token
      @nesting -= 1 if RPAREN == token
      fail! :too_many_closing_parentheses if @nesting < 0

      @tokens << token unless token.is?(:whitespace)
    end

    match_length = tokens.map(&:length).reduce(:+)
    [true, string[match_length..-1]]
  else
    [false, string]
  end
end

#strip_comments(input) ⇒ Object 



61
62
63
 
# File 'lib/jrr/tokenizer.rb', line 61

def strip_comments(input)
  input.gsub(/\/\*[^*]*\*+(?:[^*\/][^*]*\*+)*\//, '')
end

#tokenize(string, options = {}) ⇒ Object 



13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
 
# File 'lib/jrr/tokenizer.rb', line 13

def tokenize(string, options = {})
  @nesting = 0
  @tokens  = []
  input    = strip_comments(string.to_s.dup)
  @case_sensitive = options.fetch(:case_sensitive, false)

  until input.empty?
    scanned = Scanner.scanners(case_sensitive: case_sensitive).any? do |scanner|
      scanned, input = scan(input, scanner)
      scanned
    end

    unless scanned
      fail! :parse_error, at: input
    end
  end

  fail! :too_many_opening_parentheses if @nesting > 0

  @tokens
end