5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
|
# File 'lib/ddql/lexer.rb', line 5
def self.lex(expression, pattern: TokenType.all_types_pattern, available_types: TokenType::ALL)
tokens = LinkedList.new.doubly_linked!
md = pattern.match expression
while md
token_type = available_types.detect { |tt| tt.match?(match_data: md) }
if token_type
tokens << Token.new(
data: token_type.interpreted_data_from(match_data: md),
location: expression.length - md.string.length,
type: token_type,
)
end
md = pattern.match md.post_match
end
tokens
end
|