Class: Teepee::Tokenizer

Inherits:
Object
  • Object
show all
Defined in:
lib/teepee/tokenizer.rb

Instance Attribute Summary collapse

Instance Method Summary collapse

Constructor Details

#initialize(text) ⇒ Tokenizer

Returns a new instance of Tokenizer.



49
50
51
52
 
# File 'lib/teepee/tokenizer.rb', line 49

def initialize(text)
  @text = text
  tokenize
end

Instance Attribute Details

#text ⇒ Object (readonly)

Returns the value of attribute text.



48
49
50
 
# File 'lib/teepee/tokenizer.rb', line 48

def text
  @text
end

#tokens ⇒ Object (readonly)

Returns the value of attribute tokens.



48
49
50
 
# File 'lib/teepee/tokenizer.rb', line 48

def tokens
  @tokens
end

Instance Method Details

#tokenize ⇒ Object 



54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
 
# File 'lib/teepee/tokenizer.rb', line 54

def tokenize
  @tokens = []
  rest = text.gsub("\r", "")
  while rest.length > 0
    if result =     BackslashToken.matches?(rest) or # Single Character Tokens
       result =     LeftBraceToken.matches?(rest) or
       result =   LeftBracketToken.matches?(rest) or
       result =          PipeToken.matches?(rest) or
       result =    RightBraceToken.matches?(rest) or
       result =  RightBracketToken.matches?(rest) or
       result =     BackquoteToken.matches?(rest) or
       result =      SquiggleToken.matches?(rest) or
       result =        DollarToken.matches?(rest) or
       result = EmptyNewlinesToken.matches?(rest) or # String Tokens
       result =    WhitespaceToken.matches?(rest) or
       result =        NumberToken.matches?(rest) or
       result =          WordToken.matches?(rest)
    then
      @tokens << result[0]
      rest = result[1]
    else
      raise RuntimeError, "Couldn't tokenize the remaining text."
    end
  end
  return @tokens
end