hrynco/hrynco-kswitcher
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 55 Wiki Activity

Release v0.6.35: reduce resident memory and compact shared dictionaries

Browse Source
This commit is contained in:
Anatolii Hrynchuk
2026-11-04 14:08:05 +02:00
parent 004fee4976
commit d13e9849a7
21 changed files with 469 additions and 100 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/autocorrect.cpp
+122 -44
View File
@@ -42,82 +42,160 @@ bool safeAutoWord(const std::wstring &word, int minimumLength)
auto small = lower(word);
return word == small || (word.substr(1) == small.substr(1) && word != upper(word));
}
std::string_view AutoCorrector::Dictionary::word(size_t index) const
namespace {
constexpr size_t DictionaryBlockWords = 32;
}
std::string_view AutoCorrector::Dictionary::word(size_t block) const
{
auto start = offsets[index];
auto end = bytes.find('\t', start);
return std::string_view(bytes).substr(start, end - start);
size_t start = offsets[block];
// Every block starts with a complete word (zero shared prefix).
size_t length = static_cast<unsigned char>(bytes[start++]) & 15;
if (length == 15) length = static_cast<unsigned char>(bytes[start++]);
return std::string_view(bytes).substr(start, length);
}
AutoCorrector::Dictionary::Dictionary(const std::filesystem::path &path)
{
auto raw = readFile(path, 2 * 1024 * 1024);
bytes.assign(raw.begin(), raw.end());
for (size_t start = 0; start < bytes.size();)
const auto raw = readFile(path, 2 * 1024 * 1024);
const std::string_view source(reinterpret_cast<const char *>(raw.data()), raw.size());
std::vector<uint32_t> lines;
lines.reserve(std::count(source.begin(), source.end(), '\n') + 1);
for (size_t start = 0; start < source.size();)
{
auto end = bytes.find('\n', start);
if (end == bytes.npos)
end = bytes.size();
auto tab = bytes.find('\t', start);
if (tab == bytes.npos || tab >= end || tab == start || tab - start > 160)
auto end = source.find('\n', start);
if (end == source.npos) end = source.size();
auto tab = source.find('\t', start);
if (tab == source.npos || tab >= end || tab == start || tab - start > 160)
throw std::runtime_error("Invalid auto-correction dictionary");
int rank{};
auto rankEnd = end > tab + 1 && bytes[end - 1] == '\r' ? end - 1 : end;
auto parse = std::from_chars(bytes.data() + tab + 1, bytes.data() + rankEnd, rank);
if (parse.ec != std::errc() || parse.ptr != bytes.data() + rankEnd || rank <= 0)
auto rankEnd = end > tab + 1 && source[end - 1] == '\r' ? end - 1 : end;
auto parse = std::from_chars(source.data() + tab + 1, source.data() + rankEnd, rank);
if (parse.ec != std::errc() || parse.ptr != source.data() + rankEnd || rank <= 0)
throw std::runtime_error("Invalid dictionary rank");
offsets.push_back((uint32_t)start);
lines.push_back(static_cast<uint32_t>(start));
start = end + 1;
}
if (offsets.empty())
throw std::runtime_error("Empty auto-correction dictionary");
auto entry = [&](uint32_t offset) { return std::string_view(bytes).substr(offset, bytes.find('\t', offset) - offset); };
auto frequency = [&](uint32_t offset)
{
if (lines.empty()) throw std::runtime_error("Empty auto-correction dictionary");
auto entry = [&](uint32_t offset) { return source.substr(offset, source.find('\t', offset) - offset); };
auto frequency = [&](uint32_t offset) {
int rank{};
auto start = offset + entry(offset).size() + 1;
std::from_chars(bytes.data() + start, bytes.data() + bytes.size(), rank);
std::from_chars(source.data() + start, source.data() + source.size(), rank);
return rank;
};
std::sort(offsets.begin(), offsets.end(), [&](uint32_t a, uint32_t b)
{ return entry(a) == entry(b) ? frequency(a) < frequency(b) : entry(a) < entry(b); });
offsets.erase(std::unique(offsets.begin(), offsets.end(), [&](uint32_t a, uint32_t b)
{ return entry(a) == entry(b); }), offsets.end());
offsets.shrink_to_fit();
bytes.shrink_to_fit();
std::sort(lines.begin(), lines.end(), [&](uint32_t a, uint32_t b) {
return entry(a) == entry(b) ? frequency(a) < frequency(b) : entry(a) < entry(b);
});
lines.erase(std::unique(lines.begin(), lines.end(), [&](uint32_t a, uint32_t b) {
return entry(a) == entry(b);
}), lines.end());
offsets.reserve((lines.size() + DictionaryBlockWords - 1) / DictionaryBlockWords);
std::string_view previous;
// Allocate the packed payload exactly once. Geometric growth and a final
// shrink otherwise leave discarded buffers in the resident process heap.
size_t packedSize = 0;
for (size_t i = 0; i < lines.size(); ++i)
{
auto current = entry(lines[i]);
size_t prefix = 0;
if (i % DictionaryBlockWords != 0)
while (prefix < std::min(previous.size(), current.size()) && previous[prefix] == current[prefix]) ++prefix;
auto suffix = current.size() - prefix;
packedSize += 1 + (prefix >= 15) + (suffix >= 15) + suffix + 2;
auto rank = static_cast<uint32_t>(frequency(lines[i]));
if (rank >= 65535)
do { ++packedSize; rank >>= 7; } while (rank);
previous = current;
}
bytes.reserve(packedSize);
previous = {};
for (size_t i = 0; i < lines.size(); ++i)
{
auto current = entry(lines[i]);
size_t prefix = 0;
if (i % DictionaryBlockWords == 0)
offsets.push_back(static_cast<uint32_t>(bytes.size()));
else
while (prefix < std::min(previous.size(), current.size()) && previous[prefix] == current[prefix]) ++prefix;
size_t suffix = current.size() - prefix;
bytes.push_back(static_cast<char>((std::min(prefix, size_t(15)) << 4) | std::min(suffix, size_t(15))));
if (prefix >= 15) bytes.push_back(static_cast<char>(prefix));
if (suffix >= 15) bytes.push_back(static_cast<char>(suffix));
bytes.append(current.substr(prefix));
auto rank = static_cast<uint32_t>(frequency(lines[i]));
auto shortRank = std::min(rank, uint32_t(65535));
bytes.push_back(static_cast<char>(shortRank & 255));
bytes.push_back(static_cast<char>(shortRank >> 8));
if (rank >= 65535)
do {
auto next = rank & 127;
rank >>= 7;
bytes.push_back(static_cast<char>(next | (rank ? 128 : 0)));
} while (rank);
previous = current;
}
}
int AutoCorrector::Dictionary::rank(const std::wstring &text) const
{
auto key = utf8(lower(text));
size_t begin = 0, end = offsets.size();
// Find the last block whose first word is <= the search key.
while (begin < end)
{
auto mid = begin + (end - begin) / 2;
if (word(mid) < key)
begin = mid + 1;
else
end = mid;
if (word(mid) <= key) begin = mid + 1;
else end = mid;
}
if (begin == offsets.size() || word(begin) != key)
return 0;
auto start = offsets[begin] + word(begin).size() + 1;
int rank{};
std::from_chars(bytes.data() + start, bytes.data() + bytes.size(), rank);
return rank;
if (!begin) return 0;
size_t block = begin - 1;
size_t at = offsets[block];
const size_t limit = begin < offsets.size() ? offsets[begin] : bytes.size();
char decoded[160]; // The loader validates lengths; lookup never allocates a decoded word.
while (at < limit)
{
auto lengths = static_cast<unsigned char>(bytes[at++]);
size_t prefix = lengths >> 4, suffix = lengths & 15;
if (prefix == 15) prefix = static_cast<unsigned char>(bytes[at++]);
if (suffix == 15) suffix = static_cast<unsigned char>(bytes[at++]);
memcpy(decoded + prefix, bytes.data() + at, suffix);
at += suffix;
uint32_t rank = static_cast<unsigned char>(bytes[at]) |
(uint32_t(static_cast<unsigned char>(bytes[at + 1])) << 8);
at += 2;
if (rank == 65535) {
rank = 0;
uint32_t shift = 0;
unsigned char part;
do {
part = static_cast<unsigned char>(bytes[at++]);
rank |= uint32_t(part & 127) << shift;
shift += 7;
} while (part & 128);
}
auto compared = std::string_view(decoded, size_t(prefix) + suffix).compare(key);
if (compared == 0) return static_cast<int>(rank);
if (compared > 0) return 0;
}
return 0;
}
void AutoCorrector::configure(const std::filesystem::path &folder, const std::vector<std::wstring> &wanted)
{
if (languages == wanted)
return;
std::map<std::wstring, Dictionary> next;
std::map<std::wstring, std::shared_ptr<const Dictionary>> next;
for (auto &lang : wanted)
{
if (lang != L"en" && lang != L"uk" && lang != L"ru")
throw std::runtime_error("Unsupported dictionary language");
next.emplace(lang, Dictionary(folder / (lang + L".txt")));
next.emplace(lang, std::make_shared<const Dictionary>(folder / (lang + L".txt")));
}
dictionaries = std::move(next);
languages = wanted;
}
void AutoCorrector::shareDictionaries(const AutoCorrector &source)
{
dictionaries = source.dictionaries;
languages = source.languages;
}
void AutoCorrector::reject(std::wstring word, const Settings &settings)
{
while (!word.empty() && openingDelimiter(word.front(), settings))
@@ -132,7 +210,7 @@ size_t AutoCorrector::memoryBytes() const
{
size_t bytes = 0;
for (auto &[language, d] : dictionaries)
bytes += d.bytes.capacity() + d.offsets.capacity() * sizeof(uint32_t);
bytes += d->bytes.capacity() + d->offsets.capacity() * sizeof(uint32_t);
return bytes;
}
std::optional<AutoSuggestion> AutoCorrector::suggest(const std::wstring &typed, int source,
@@ -191,7 +269,7 @@ std::optional<AutoSuggestion> AutoCorrector::suggest(const std::wstring &typed,
};
// Valid words from either selected language are never changed.
for (auto &[language, d] : dictionaries)
if (d.rank(word) || personal(language, word))
if (d->rank(word) || personal(language, word))
return {};
std::optional<AutoSuggestion> candidate;
std::optional<std::wstring> matchedText;
@@ -211,7 +289,7 @@ std::optional<AutoSuggestion> AutoCorrector::suggest(const std::wstring &typed,
auto converted = convert(word, layouts[source], layouts[i], true);
if (converted == word || !safeAutoWord(converted, settings.autoMinLength))
continue;
int rank = personal(dictionary->first, converted) ? 1 : dictionary->second.rank(converted);
int rank = personal(dictionary->first, converted) ? 1 : dictionary->second->rank(converted);
if (!rank)
continue;
if (matchedText && *matchedText != converted)
@@ -233,7 +311,7 @@ std::optional<AutoSuggestion> AutoCorrector::suggest(const std::wstring &typed,
auto consider = [&](const std::wstring &text)
{
if (text == small || text == found || text.size() < 4 || text.size() > 32) return true;
int rank = personalTargets.contains(text) ? 1 : dictionaries.at(sourceLanguage).rank(text);
int rank = personalTargets.contains(text) ? 1 : dictionaries.at(sourceLanguage)->rank(text);
if (!rank) return true;
if (!found.empty()) return false;
found = text;