"""Build compact offline dictionaries from pinned FrequencyWords data (CC BY-SA 4.0)."""
from pathlib import Path
import urllib.request, hashlib, json
ROOT=Path(__file__).resolve().parents[1]/'dictionaries'
REV='525f9b560de45753a5ea01069454e72e9aa541c6'
ROOT.mkdir(exist_ok=True)
manifest={}
for lang in ('en','uk','ru'):
    url=f'https://raw.githubusercontent.com/hermitdave/FrequencyWords/{REV}/content/2016/{lang}/{lang}_50k.txt'
    data=urllib.request.urlopen(url).read()
    rows={}
    for rank,line in enumerate(data.decode('utf-8').splitlines(),1):
        word,count=line.rsplit(' ',1);word=word.lower()
        alphabet='abcdefghijklmnopqrstuvwxyz' if lang=='en' else ('абвгґдеєжзиіїйклмнопрстуфхцчшщьюя' if lang=='uk' else 'абвгдеёжзийклмнопрстуфхцчшщъыьэюя')
        if not (2<=len(word)<=40 and all(c in alphabet for c in word)):continue
        if int(count)<5:continue
        rows.setdefault(word,rank)
    out=''.join(f'{word}\t{rows[word]}\n' for word in sorted(rows))
    (ROOT/f'{lang}.txt').write_bytes(out.encode('utf-8'))
    manifest[lang]={'source':url,'sourceSha256':hashlib.sha256(data).hexdigest(),'words':len(rows),'bytes':len(out.encode()),'sha256':hashlib.sha256(out.encode()).hexdigest()}
(ROOT/'sources.json').write_text(json.dumps(manifest,indent=2)+'\n',encoding='utf-8')
print(json.dumps(manifest,indent=2))