Change order to include predicted element

master
Valentin Ochs 2020-02-02 23:34:47 +01:00
parent 7e7980f68b
commit 279f95b855
1 changed files with 7 additions and 8 deletions

View File

@ -133,8 +133,8 @@ protected:
} }
double rate(Mapping const &m, int max_order = -1) const { double rate(Mapping const &m, int max_order = -1) const {
const auto order = const auto order = 1 + ((-1 == max_order) ? corpus.order()
(-1 == max_order) ? corpus.order() : static_cast<size_t>(max_order); : static_cast<size_t>(max_order));
double score = 0.; double score = 0.;
char map[256]; char map[256];
m.create_map(&map[0]); m.create_map(&map[0]);
@ -143,14 +143,13 @@ protected:
for (size_t i = 0, valid = 0; i < len; i++) { for (size_t i = 0, valid = 0; i < len; i++) {
if (plain_text[i]) { if (plain_text[i]) {
valid++; valid++;
if (valid == (order + 1)) { if (valid == order) {
score += log(corpus.probability(&plain_text[i - valid], valid)); score += log(corpus.probability(&plain_text[i - order], order));
} else if (valid > (order + 1)) { } else if (valid > order) {
score += log( score += log(corpus.final_probability(&plain_text[i - order], order));
corpus.final_probability(&plain_text[i - order - 1], order + 1));
} }
} else { } else {
if (valid && (valid <= order)) { if (valid && (valid < order)) {
score += log(corpus.probability(&plain_text[i - valid], valid)); score += log(corpus.probability(&plain_text[i - valid], valid));
} }
valid = 0; valid = 0;