Class: Maxy::Gen::Tokenizer

Inherits:
Object
  • Object
show all
Defined in:
lib/maxy/gen/tokenizer.rb

Constant Summary collapse

TOKEN_TYPES = 
[
  [:arguments, /({[^{}]*})/],
  [:mc_identifier, /(mc\.[^{}()]+~)(?=[-+=*<{}])?/],
  [:escaped_identifier, /(\\[\S][^-+={}<()\\]*)(?=[-+=*<{}])?/],
  [:identifier, /([^-+={}<()\\]+)(?=[-+=*<{}])?/],
  [:oparen, /(\()/],
  [:cparen, /(\))/],
  [:plus, /(\+)/],
  [:less_than, /(<)/],
  [:equals, /(=)/],
  [:dash, /(-)/]
]

Instance Method Summary collapse

Constructor Details

#initialize(pattern) ⇒ Tokenizer

Returns a new instance of Tokenizer.



17
18
19
 
# File 'lib/maxy/gen/tokenizer.rb', line 17

def initialize(pattern)
  @pattern = pattern
end

Instance Method Details

#tokenize ⇒ Object 



21
22
23
24
25
26
27
 
# File 'lib/maxy/gen/tokenizer.rb', line 21

def tokenize
  tokens = []
  until @pattern.empty?
    tokens << tokenize_one_token
  end
  tokens
end

#tokenize_one_token ⇒ Object 



29
30
31
32
33
34
35
36
37
38
39
40
 
# File 'lib/maxy/gen/tokenizer.rb', line 29

def tokenize_one_token
  TOKEN_TYPES.each do |type, re|
    re = /\A#{re}/
    if @pattern =~ re
      value = $1
      @pattern = @pattern[value.length..-1]
      return Token.new(type, value)
    end
  end

  raise "Couldn't match token on #{@pattern.inspect}"
end