Class: RsPathTokenizer::Tokenizer

Inherits:
Object
  • Object
show all
Defined in:
lib/rs_path_tokenizer/tokenizer.rb

Constant Summary collapse

PT_DEBUG = 
false

Instance Method Summary collapse

Constructor Details

#initialize(tokens = nil) ⇒ Tokenizer

PT_DEBUG = true


6
7
8
9
10
11
12
13
14
15
16
17
 
# File 'lib/rs_path_tokenizer/tokenizer.rb', line 6

def initialize(tokens = nil)
  return if tokens.nil?
  @single_tokens = {}
  tokens.keys.each do |t|
    parts = url2token(t)
    st = parts[0]
    raise Error.new('Token cant starts with asterisk') if st == '*'
    @single_tokens[st] = [] if @single_tokens[st].nil?
    @single_tokens[st].push parts
  end
  @token_map = tokens
end

Instance Method Details

#marshal_dump ⇒ Object 


19
20
21
 
# File 'lib/rs_path_tokenizer/tokenizer.rb', line 19

def marshal_dump
  [@single_tokens, @token_map]
end

#marshal_load(array) ⇒ Object 


23
24
25
 
# File 'lib/rs_path_tokenizer/tokenizer.rb', line 23

def marshal_load array
  @single_tokens, @token_map = array
end

#tokenize(string) ⇒ Object

best result


28
29
30
31
32
33
 
# File 'lib/rs_path_tokenizer/tokenizer.rb', line 28

def tokenize(string)
  tokens = tokenize_all(string).first
  return if tokens.nil?

  result_to_hash(tokens)
end