Release v0.6.10: preserve typing during correction and add personal dictionaries
This commit is contained in:
@@ -3,6 +3,8 @@ from pathlib import Path
|
||||
import urllib.request, hashlib, json
|
||||
ROOT=Path(__file__).resolve().parents[1]/'dictionaries'
|
||||
REV='525f9b560de45753a5ea01069454e72e9aa541c6'
|
||||
# Supplemental vocabulary has no corpus frequency: rank after the 50k source list.
|
||||
SUPPLEMENTAL={'uk': {'реліз': 50001}}
|
||||
ROOT.mkdir(exist_ok=True)
|
||||
manifest={}
|
||||
for lang in ('en','uk','ru'):
|
||||
@@ -15,8 +17,12 @@ for lang in ('en','uk','ru'):
|
||||
if not (2<=len(word)<=40 and all(c in alphabet for c in word)):continue
|
||||
if int(count)<5:continue
|
||||
rows.setdefault(word,rank)
|
||||
for word,rank in SUPPLEMENTAL.get(lang,{}).items():
|
||||
rows.setdefault(word,rank)
|
||||
out=''.join(f'{word}\t{rows[word]}\n' for word in sorted(rows))
|
||||
(ROOT/f'{lang}.txt').write_bytes(out.encode('utf-8'))
|
||||
manifest[lang]={'source':url,'sourceSha256':hashlib.sha256(data).hexdigest(),'words':len(rows),'bytes':len(out.encode()),'sha256':hashlib.sha256(out.encode()).hexdigest()}
|
||||
if lang in SUPPLEMENTAL:
|
||||
manifest[lang]['supplementalWords']=SUPPLEMENTAL[lang]
|
||||
(ROOT/'sources.json').write_text(json.dumps(manifest,indent=2)+'\n',encoding='utf-8')
|
||||
print(json.dumps(manifest,indent=2))
|
||||
|
||||
Reference in New Issue
Block a user