86 lines
4.7 KiB
C++
86 lines
4.7 KiB
C++
#include "autocorrect.h"
|
|
#include <algorithm>
|
|
#include <chrono>
|
|
#include <fstream>
|
|
#include <iostream>
|
|
#include <psapi.h>
|
|
#include <random>
|
|
namespace ks {
|
|
struct AutoCorrectorRegressionAccess {
|
|
static int rank(const AutoCorrector &c,const std::wstring &lang,const std::wstring &word) {
|
|
return c.dictionaries.at(lang)->rank(word);
|
|
}
|
|
static std::vector<int> ranks(const std::filesystem::path &path,const std::vector<std::wstring> &words) {
|
|
AutoCorrector::Dictionary d(path);
|
|
std::vector<int> values;for(const auto &word:words)values.push_back(d.rank(word));return values;
|
|
}
|
|
};
|
|
}
|
|
int main(int argc,char **argv) {
|
|
try {
|
|
auto start=std::chrono::steady_clock::now();
|
|
ks::AutoCorrector c;c.configure(L"dictionaries",{L"en",L"uk",L"ru"});
|
|
PROCESS_MEMORY_COUNTERS_EX m{};
|
|
GetProcessMemoryInfo(GetCurrentProcess(),reinterpret_cast<PROCESS_MEMORY_COUNTERS*>(&m),sizeof(m));
|
|
std::cout<<"dictionary_bytes="<<c.memoryBytes()<<" private_bytes="<<m.PrivateUsage
|
|
<<" load_ms="<<std::chrono::duration<double,std::milli>(std::chrono::steady_clock::now()-start).count()<<'\n';
|
|
struct Query {std::wstring language,word; int rank;};
|
|
std::vector<Query> queries;
|
|
for(const auto *lang:{L"en",L"uk",L"ru"}) {
|
|
std::ifstream input(std::filesystem::path(L"dictionaries")/(std::wstring(lang)+L".txt"));
|
|
std::map<std::wstring,int> expected;
|
|
for(std::string line;std::getline(input,line);) {
|
|
auto tab=line.find('\t');auto word=ks::wide(line.substr(0,tab));auto rank=std::stoi(line.substr(tab+1));
|
|
auto [it,added]=expected.emplace(word,rank);if(!added)it->second=std::min(it->second,rank);
|
|
}
|
|
if(expected.empty())throw std::runtime_error("Missing benchmark dictionary");
|
|
for(auto &[word,rank]:expected) {
|
|
if(ks::AutoCorrectorRegressionAccess::rank(c,lang,word)!=rank)throw std::runtime_error("Dictionary rank differs from source");
|
|
if(ks::AutoCorrectorRegressionAccess::rank(c,lang,word+L"~")!=0)throw std::runtime_error("False dictionary hit");
|
|
queries.push_back({lang,word,rank});queries.push_back({lang,word+L"~",0});
|
|
}
|
|
}
|
|
std::mt19937 random(42);std::shuffle(queries.begin(),queries.end(),random);
|
|
uint64_t checksum=0,expected=0;
|
|
for(auto &q:queries)expected+=q.rank;
|
|
start=std::chrono::steady_clock::now();
|
|
for(int pass=0;pass<5;++pass)for(auto &q:queries)checksum+=ks::AutoCorrectorRegressionAccess::rank(c,q.language,q.word);
|
|
auto ms=std::chrono::duration<double,std::milli>(std::chrono::steady_clock::now()-start).count();
|
|
if(checksum!=expected*5)throw std::runtime_error("Lookup checksum mismatch");
|
|
std::cout<<"verified_words="<<queries.size()/2<<" queries="<<queries.size()*5<<" lookup_ms="<<ms
|
|
<<" ns_per_lookup="<<ms*1000000/(queries.size()*5)<<" checksum="<<checksum<<'\n';
|
|
auto fixture=std::filesystem::temp_directory_path()/(L"KSwitcher-dictionary-"+
|
|
std::to_wstring(GetCurrentProcessId())+L"-"+std::to_wstring(GetTickCount64())+L".txt");
|
|
struct RemoveFixture { std::filesystem::path path; ~RemoveFixture(){std::error_code ec;std::filesystem::remove(path,ec);} } cleanup{fixture};
|
|
auto put=[&](const std::string &text){ks::atomicWrite(fixture,ks::Bytes(text.begin(),text.end()));};
|
|
std::string text="duplicate\t9\r\nduplicate\t2\n";
|
|
std::vector<std::wstring> words;
|
|
std::vector<int> expectedRanks;
|
|
for(int i=129;i>=0;--i) {
|
|
auto word=L"prefix_"+std::to_wstring(i);
|
|
text+=ks::utf8(word)+"\t"+std::to_string(i+1)+"\n";
|
|
words.push_back(word);expectedRanks.push_back(i+1);
|
|
}
|
|
text+=std::string(160,'x')+"\t2147483647"; // Maximum length/rank, no final newline.
|
|
words.insert(words.end(),{std::wstring(160,L'x'),L"DUPLICATE",L"",L"prefix_",L"zzzz"});
|
|
expectedRanks.insert(expectedRanks.end(),{2147483647,2,0,0,0});
|
|
for(auto rank:{65534,65535,65536}) {
|
|
auto word=std::wstring(159,L'x')+wchar_t(L'y'+rank-65534);
|
|
text+="\n"+ks::utf8(word)+"\t"+std::to_string(rank);
|
|
words.push_back(word);expectedRanks.push_back(rank);
|
|
}
|
|
put(text);
|
|
if(ks::AutoCorrectorRegressionAccess::ranks(fixture,words)!=expectedRanks)
|
|
throw std::runtime_error("Block boundaries, duplicate ranks or maximum values failed");
|
|
for(const auto &invalid:std::vector<std::string>{"","missing-tab","x\t0","x\t-1","x\t2147483648",std::string(161,'x')+"\t1","\t1","x\t1bad"}) {
|
|
put(invalid);bool rejected=false;
|
|
try {ks::AutoCorrectorRegressionAccess::ranks(fixture,{L"x"});}catch(const std::runtime_error &){rejected=true;}
|
|
if(!rejected)throw std::runtime_error("Invalid dictionary was accepted");
|
|
}
|
|
const bool baseline=argc==2 && std::string(argv[1])=="--baseline";
|
|
if(!baseline && c.memoryBytes()>=1536*1024)throw std::runtime_error("Bundled dictionaries exceed compact memory budget");
|
|
std::cout<<"Passed complete dictionary equivalence and format regressions\n";
|
|
return 0;
|
|
}catch(const std::exception&e){std::cerr<<e.what()<<'\n';return 1;}
|
|
}
|