Class: Bibtex::Lexer

Inherits:
Object
  • Object
show all
Defined in:
lib/bibtex/lexer.rb

Instance Attribute Summary collapse

Instance Method Summary collapse

Constructor Details

#initialize(ignore_whitespace = false) {|@rules| ... } ⇒ Lexer

Returns a new instance of Lexer.

Yields:

  • (@rules) 


53
54
55
56
57
58
59
60
61
 
# File 'lib/bibtex/lexer.rb', line 53

def initialize(ignore_whitespace = false)
  @scanner = StringScanner.new('')
  @rules = RuleSet.new
  @ignore_whitespace = ignore_whitespace
  @ignore_newlines = ignore_whitespace
  @lineno = 1
  @file_name = '<unknown>'
  yield @rules
end

Instance Attribute Details

#file_name ⇒ Object

Returns the value of attribute file_name.



51
52
53
 
# File 'lib/bibtex/lexer.rb', line 51

def file_name
  @file_name
end

#ignore_newlines ⇒ Object

Returns the value of attribute ignore_newlines.



51
52
53
 
# File 'lib/bibtex/lexer.rb', line 51

def ignore_newlines
  @ignore_newlines
end

#ignore_whitespace ⇒ Object

Returns the value of attribute ignore_whitespace.



50
51
52
 
# File 'lib/bibtex/lexer.rb', line 50

def ignore_whitespace
  @ignore_whitespace
end

#lval ⇒ Object (readonly)

Returns the value of attribute lval.



50
51
52
 
# File 'lib/bibtex/lexer.rb', line 50

def lval
  @lval
end

Instance Method Details

#feed(str) ⇒ Object 



69
70
71
72
 
# File 'lib/bibtex/lexer.rb', line 69

def feed(str)
  @scanner = StringScanner.new(str)
  @cols_prev = 0
end

#more_tokens? ⇒ Boolean

Returns:

  • (Boolean) 


106
107
108
109
 
# File 'lib/bibtex/lexer.rb', line 106

def more_tokens?
  skip_whitespace
  not @scanner.eos?
end

#next_token! ⇒ Object

Raises:



78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
 
# File 'lib/bibtex/lexer.rb', line 78

def next_token!
  if @scanner.check /^\s*\n/ then
    @lineno += 1
    @cols_prev = @scanner.pos + 1
  end
  skip_whitespace
  @rules.each do |regexp, result|
    return result if @lval = @scanner.scan(regexp)
  end
  unexpect = if @scanner.rest.length < 10 then
               @scanner.rest
             else
               "#{@scanner.rest.first 10}..."
             end
  raise LexerError.new("Unexpected input #{unexpect}", src_pos)
end

#peek_lval ⇒ Object 



101
102
103
104
 
# File 'lib/bibtex/lexer.rb', line 101

def peek_lval
  peek_token
  @lval
end

#peek_token ⇒ Object 



95
96
97
98
99
 
# File 'lib/bibtex/lexer.rb', line 95

def peek_token
  tok = self.next_token!
  @scanner.unscan
  return tok
end

#src_pos ⇒ Object 



74
75
76
 
# File 'lib/bibtex/lexer.rb', line 74

def src_pos
  SourcePos.new(@lineno, @scanner.pos - @cols_prev, @file_name)
end