hrynco/hrynco-kswitcher
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 46 Wiki Activity

Release v0.6.35: reduce resident memory and compact shared dictionaries

Browse Source
This commit is contained in:
Anatolii Hrynchuk
2026-11-04 14:08:05 +02:00
parent 004fee4976
commit d13e9849a7
21 changed files with 469 additions and 100 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/dictionary.cpp
+85
View File
@@ -0,0 +1,85 @@
#include "autocorrect.h"
#include <algorithm>
#include <chrono>
#include <fstream>
#include <iostream>
#include <psapi.h>
#include <random>
namespace ks {
struct AutoCorrectorRegressionAccess {
static int rank(const AutoCorrector &c,const std::wstring &lang,const std::wstring &word) {
return c.dictionaries.at(lang)->rank(word);
}
static std::vector<int> ranks(const std::filesystem::path &path,const std::vector<std::wstring> &words) {
AutoCorrector::Dictionary d(path);
std::vector<int> values;for(const auto &word:words)values.push_back(d.rank(word));return values;
}
};
}
int main(int argc,char **argv) {
try {
auto start=std::chrono::steady_clock::now();
ks::AutoCorrector c;c.configure(L"dictionaries",{L"en",L"uk",L"ru"});
PROCESS_MEMORY_COUNTERS_EX m{};
GetProcessMemoryInfo(GetCurrentProcess(),reinterpret_cast<PROCESS_MEMORY_COUNTERS*>(&m),sizeof(m));
std::cout<<"dictionary_bytes="<<c.memoryBytes()<<" private_bytes="<<m.PrivateUsage
<<" load_ms="<<std::chrono::duration<double,std::milli>(std::chrono::steady_clock::now()-start).count()<<'\n';
struct Query {std::wstring language,word; int rank;};
std::vector<Query> queries;
for(const auto *lang:{L"en",L"uk",L"ru"}) {
std::ifstream input(std::filesystem::path(L"dictionaries")/(std::wstring(lang)+L".txt"));
std::map<std::wstring,int> expected;
for(std::string line;std::getline(input,line);) {
auto tab=line.find('\t');auto word=ks::wide(line.substr(0,tab));auto rank=std::stoi(line.substr(tab+1));
auto [it,added]=expected.emplace(word,rank);if(!added)it->second=std::min(it->second,rank);
}
if(expected.empty())throw std::runtime_error("Missing benchmark dictionary");
for(auto &[word,rank]:expected) {
if(ks::AutoCorrectorRegressionAccess::rank(c,lang,word)!=rank)throw std::runtime_error("Dictionary rank differs from source");
if(ks::AutoCorrectorRegressionAccess::rank(c,lang,word+L"~")!=0)throw std::runtime_error("False dictionary hit");
queries.push_back({lang,word,rank});queries.push_back({lang,word+L"~",0});
}
}
std::mt19937 random(42);std::shuffle(queries.begin(),queries.end(),random);
uint64_t checksum=0,expected=0;
for(auto &q:queries)expected+=q.rank;
start=std::chrono::steady_clock::now();
for(int pass=0;pass<5;++pass)for(auto &q:queries)checksum+=ks::AutoCorrectorRegressionAccess::rank(c,q.language,q.word);
auto ms=std::chrono::duration<double,std::milli>(std::chrono::steady_clock::now()-start).count();
if(checksum!=expected*5)throw std::runtime_error("Lookup checksum mismatch");
std::cout<<"verified_words="<<queries.size()/2<<" queries="<<queries.size()*5<<" lookup_ms="<<ms
<<" ns_per_lookup="<<ms*1000000/(queries.size()*5)<<" checksum="<<checksum<<'\n';
auto fixture=std::filesystem::temp_directory_path()/(L"KSwitcher-dictionary-"+
std::to_wstring(GetCurrentProcessId())+L"-"+std::to_wstring(GetTickCount64())+L".txt");
struct RemoveFixture { std::filesystem::path path; ~RemoveFixture(){std::error_code ec;std::filesystem::remove(path,ec);} } cleanup{fixture};
auto put=[&](const std::string &text){ks::atomicWrite(fixture,ks::Bytes(text.begin(),text.end()));};
std::string text="duplicate\t9\r\nduplicate\t2\n";
std::vector<std::wstring> words;
std::vector<int> expectedRanks;
for(int i=129;i>=0;--i) {
auto word=L"prefix_"+std::to_wstring(i);
text+=ks::utf8(word)+"\t"+std::to_string(i+1)+"\n";
words.push_back(word);expectedRanks.push_back(i+1);
}
text+=std::string(160,'x')+"\t2147483647"; // Maximum length/rank, no final newline.
words.insert(words.end(),{std::wstring(160,L'x'),L"DUPLICATE",L"",L"prefix_",L"zzzz"});
expectedRanks.insert(expectedRanks.end(),{2147483647,2,0,0,0});
for(auto rank:{65534,65535,65536}) {
auto word=std::wstring(159,L'x')+wchar_t(L'y'+rank-65534);
text+="\n"+ks::utf8(word)+"\t"+std::to_string(rank);
words.push_back(word);expectedRanks.push_back(rank);
}
put(text);
if(ks::AutoCorrectorRegressionAccess::ranks(fixture,words)!=expectedRanks)
throw std::runtime_error("Block boundaries, duplicate ranks or maximum values failed");
for(const auto &invalid:std::vector<std::string>{"","missing-tab","x\t0","x\t-1","x\t2147483648",std::string(161,'x')+"\t1","\t1","x\t1bad"}) {
put(invalid);bool rejected=false;
try {ks::AutoCorrectorRegressionAccess::ranks(fixture,{L"x"});}catch(const std::runtime_error &){rejected=true;}
if(!rejected)throw std::runtime_error("Invalid dictionary was accepted");
}
const bool baseline=argc==2 && std::string(argv[1])=="--baseline";
if(!baseline && c.memoryBytes()>=1536*1024)throw std::runtime_error("Bundled dictionaries exceed compact memory budget");
std::cout<<"Passed complete dictionary equivalence and format regressions\n";
return 0;
}catch(const std::exception&e){std::cerr<<e.what()<<'\n';return 1;}
}
tests/input.cpp
+56
View File
@@ -25,6 +25,38 @@ LRESULT CALLBACK countControlMoves(HWND w, UINT message, WPARAM wp, LPARAM lp, U
}
namespace ks {
struct InputRegressionAccess {
static bool idleUndo(App &a) {
a.last=LastTransform{}; a.last->at=100;
a.releaseIdleMemory(120100);
if (!a.last) return false;
a.busy=true;
a.releaseIdleMemory(120101);
if (!a.last) { a.busy=false; return false; }
a.busy=false;
a.releaseIdleMemory(120101);
return !a.last;
}
static bool releasedSync(const App &a) {
return a.syncSnapshot.capacity()==0 && a.syncResult.settings.capacity()==0 &&
a.syncResult.base.capacity()==0 && a.syncResult.conflicts.capacity()==0;
}
static bool completedSync(App &a, int mode) {
sampleConflicts(a);
auto conflicts=encodeSyncConflicts(a.pendingSyncConflicts);
a.syncConnection=loadSyncConfig(a.data);
a.syncSnapshot=encodeSettings(a.settings);
a.syncResult={mode!=1,sharedSyncSettings(a.settings),2,L"test"};
a.syncResult.base=a.syncResult.settings;
a.syncResult.conflicts=a.pendingSyncConflicts;
if(mode==2) a.syncResult.settings={1,2,3}; // Decode exception after joining.
a.syncWorker=std::thread([]{});
a.syncRunning=true; a.syncDone=true; a.busy=true;
a.pollSync();
bool retained=!a.syncSnapshot.empty() && a.syncRunning;
a.busy=false; a.pollSync();
return retained && releasedSync(a) && !a.syncRunning &&
encodeSyncConflicts(a.pendingSyncConflicts)==conflicts;
}
static inline bool delivery = true;
static inline int notifications = 0;
static bool notify(const std::wstring &) { ++notifications; return delivery; }
@@ -44,6 +76,7 @@ struct InputRegressionAccess {
a.syncRunning=true; a.syncDone=true;
a.pollSync();
return encodeSettings(a.settings)==before && !a.syncRunning &&
releasedSync(a) &&
!std::filesystem::exists(a.data/L"sync-base.bin");
}
static bool syncBackoff(App &a) {
@@ -791,6 +824,10 @@ int main(int argc, char **argv) {
require(IsWindowEnabled(app.settingsWindow),"closing sync restores parent");
require(ks::InputRegressionAccess::staleSync(app,false),"in-flight sync cannot overwrite newer local edits");
require(ks::InputRegressionAccess::staleSync(app,true),"disconnect discards in-flight downloaded profile");
require(ks::InputRegressionAccess::idleUndo(app),"idle cleanup retains available/busy Undo and frees expired Undo");
for(int mode=0;mode<3;++mode)
require(ks::InputRegressionAccess::completedSync(app,mode),
"completed, failed and invalid sync release buffers but preserve conflict choices");
require(ks::InputRegressionAccess::syncBackoff(app),"saving settings preserves authentication error backoff");
SendMessageW(app.settingsWindow,WM_COMMAND,255,0);
auto autoOptions=FindWindowW(L"KSwitcher.AutoOptions",nullptr);
@@ -902,6 +939,25 @@ int main(int argc, char **argv) {
SetDlgItemTextW(history,101,L"");
require(pumpUntil(2),"clearing search restores history");
require(GetFocus()==originalFocus,"history search preserves focus");
SendDlgItemMessageW(history,102,LB_SETCURSEL,1,0);
ks::refreshHistory(app); // A result may still be in flight when closing.
SendMessageW(history,WM_CLOSE,0,0);
SendMessageW(history,WM_TIMER,1,0); // KillTimer cannot retract a posted timer message.
require(pumpUntil(0),"closing history releases list strings");
auto until=GetTickCount64()+250;
while(GetTickCount64()<until) {
MSG message{};
while(PeekMessageW(&message,nullptr,0,0,PM_REMOVE)) {
TranslateMessage(&message); DispatchMessageW(&message);
}
Sleep(5);
}
require(SendDlgItemMessageW(history,102,LB_GETCOUNT,0,0)==0,
"late search results do not refill closed history");
ks::refreshHistory(app);
require(pumpUntil(2),"history reloads after releasing hidden rows");
require(SendDlgItemMessageW(history,102,LB_GETCURSEL,0,0)==1,
"history retains selected record across close and reload");
}
auto childRect=[&](int id) { RECT r{}; GetWindowRect(GetDlgItem(history,id),&r); MapWindowPoints(nullptr,history,(POINT*)&r,2); return r; };
RECT historyFrame{}; GetWindowRect(history,&historyFrame);
tests/tests.cpp
+66 -14
View File
@@ -23,20 +23,6 @@ void rejects(const std::function<void()> &f, const char *name)
bool thrown = false;
try
{
{
std::map<std::wstring,int> previews;
std::set<std::wstring> visible;
for(int i=0;i<40;++i) previews[std::to_wstring(i)]=i;
for(int i=0;i<8;++i) visible.insert(std::to_wstring(i));
ks::trimHistoryPreviews(previews,visible);
require(previews.size()==32,"preview cache stays bounded");
for(const auto &id:visible) require(previews.contains(id),"visible preview survives eviction");
for(int i=0;i<40;++i) { auto id=std::to_wstring(i); previews[id]=i; visible.insert(id); }
ks::trimHistoryPreviews(previews,visible);
require(previews.size()==40,"tall viewport does not thrash visible previews");
visible.clear(); ks::trimHistoryPreviews(previews,visible);
require(previews.size()==32,"cache shrinks after leaving tall viewport");
}
f();
}
catch (...)
@@ -60,6 +46,34 @@ int main(int argc, char **argv)
}
try
{
// Byte budgets must retain visible entries even when they alone exceed the budget.
{
std::map<std::wstring, size_t> cache{{L"a", 800}, {L"b", 800}, {L"c", 100}};
std::set<std::wstring> visible{L"b"};
auto weight=[](size_t bytes) { return bytes; };
ks::trimHistoryPreviews(cache, visible, 32, 1000, weight);
require(cache.size()==2 && cache.contains(L"b") && cache.contains(L"c"),
"byte budget evicts offscreen previews below count limit");
ks::trimHistoryPreviews(cache, visible, 32, 100, weight);
require(cache.size()==1 && cache.contains(L"b"), "oversized visible preview never thrashes");
visible.clear();
ks::trimHistoryPreviews(cache, visible, 32, 100, weight);
require(cache.empty(), "oversized preview is freed after leaving viewport");
}
{
std::map<std::wstring,int> previews;
std::set<std::wstring> visible;
for(int i=0;i<40;++i) previews[std::to_wstring(i)]=i;
for(int i=0;i<8;++i) visible.insert(std::to_wstring(i));
ks::trimHistoryPreviews(previews,visible);
require(previews.size()==32,"preview cache stays bounded");
for(const auto &id:visible) require(previews.contains(id),"visible preview survives eviction");
for(int i=0;i<40;++i) { auto id=std::to_wstring(i); previews[id]=i; visible.insert(id); }
ks::trimHistoryPreviews(previews,visible);
require(previews.size()==40,"tall viewport does not thrash visible previews");
visible.clear(); ks::trimHistoryPreviews(previews,visible);
require(previews.size()==32,"cache shrinks after leaving tall viewport");
}
{
ks::DiagnosticJson record{{"attempt", "42"}, {"process", "editor.exe"}, {"command", "Convert / cycle"},
{"source", "English"}, {"target", "Ukrainian"}, {"outcome", "Original text remained"},
@@ -99,6 +113,10 @@ int main(int argc, char **argv)
"large diagnostic selections are explicitly truncated");
STARTUPINFOW startup{sizeof(startup)};
ks::DiagnosticBuffer diagnostics;
diagnostics.record(L"idle-expiry", false, 0);
require(diagnostics.expire(30000)==0,"diagnostics remain available at expiry boundary");
require(diagnostics.expire(30001)==1,"idle expiry releases context without another log event");
require(diagnostics.expire(60000)==0,"repeated idle expiry is safe");
require(diagnostics.record(L"successful-attempt", false, 100).empty(),
"successful diagnostics stay in memory");
auto incident = diagnostics.record(L"failed-attempt", true, 200);
@@ -217,6 +235,26 @@ int main(int argc, char **argv)
require(!autoSettings.autoCorrect, "auto-correction defaults off");
autoSettings.cycle = {L"en", L"uk"};
corrector.configure(L"dictionaries", autoSettings.autoLanguages);
{
ks::AutoCorrector consumer;
consumer.shareDictionaries(corrector);
require(consumer.suggest(L"ghbdsn ", 0, layouts, autoSettings).has_value(),
"shared dictionaries produce the same correction");
consumer.reject(L"ghbdsn");
consumer.shareDictionaries(corrector);
require(!consumer.suggest(L"ghbdsn ", 0, layouts, autoSettings),
"refreshing shared dictionaries preserves consumer undo rejections");
require(corrector.suggest(L"ghbdsn ", 0, layouts, autoSettings).has_value(),
"consumer rejections do not affect the predictor");
ks::AutoCorrector retained;
retained.shareDictionaries(corrector);
corrector.configure({}, {});
require(retained.suggest(L"ghbdsn ", 0, layouts, autoSettings).has_value(),
"in-flight dictionary snapshot survives source reset");
consumer.shareDictionaries(corrector);
require(consumer.memoryBytes() == 0, "disabled correction releases consumer dictionaries");
corrector.shareDictionaries(retained);
}
{
auto options = autoSettings;
options.autoTypos = true;
@@ -545,6 +583,20 @@ int main(int argc, char **argv)
require(!ambiguous.suggest(L"hell ", 0, layouts, options), "frequency filter does not conceal ambiguous typo targets");
}
ks::Pixels pixels{2, 1, {10, 20, 30, 255, 40, 50, 60, 255}};
{
auto png = ks::encodePng(pixels);
ks::Clip encoded{{{0, L"PNG", png}}};
for (int i=0; i<3; ++i) {
auto decoded=ks::decodeImage(encoded);
require(decoded.bgra==pixels.bgra && decoded.width==2 && decoded.height==1,
"borrowed PNG stream preserves decoded pixels on repeated reads");
require(encoded.formats[0].data==png,"PNG decoder never modifies the borrowed input");
}
ks::Clip fallback{{{0,L"PNG",{}},ks::imageDib(pixels)}};
require(ks::decodeImage(fallback).bgra==pixels.bgra,"empty PNG falls back to DIB");
ks::Clip invalid{{{0,L"PNG",{1,2,3}}}};
rejects([&] { ks::decodeImage(invalid); },"invalid borrowed PNG fails safely");
}
ks::Clip firstImage{{ks::imageDib(pixels), {0, L"Ole Private Data", {1, 2, 3}}}};
auto thumbnail = ks::decodeImage(firstImage, 1, 1);
require(thumbnail.width == 1 && thumbnail.height == 1 && thumbnail.bgra.size() == 4,