134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
|
# File 'lib/zipf/bleu.rb', line 134
def BLEU::per_sentence_bleu hypothesis, references, n=4, smooth=0.0
h_ng = {}; r_ng = []
num_ref = references.size
num_ref.times { r_ng << {} }
(1).upto(n) { |i| h_ng[i] = []; num_ref.times { |j| r_ng[j][i] = [] } }
ngrams(hypothesis, n) { |i| h_ng[i.size] << i }
references.each_with_index { |reference,j|
ngrams(reference, n) { |i| r_ng[j][i.size] << i }
}
m = [n, references.map { |i| i.split.size }.max].min
add = 0.0
logbleu = 0.0
(1).upto(m) { |i|
counts_clipped = 0
counts_sum = h_ng[i].size
h_ng[i].uniq.each { |j|
max_count = [h_ng[i].count(j), r_ng.map { |r| r[i].count(j) }.max].min
counts_clipped += max_count
}
add = 1.0 if i >= 2
logbleu += Math.log(counts_clipped+add) - Math.log(counts_sum+add);
}
logbleu /= m
hyp_len, best_ref_len = BLEU::best_match_length hypothesis, references
logbleu += brevity_penalty hyp_len, best_ref_len, smooth
return Math.exp logbleu
end
|