23 lines
1.4 KiB
Python
23 lines
1.4 KiB
Python
"""Build compact offline dictionaries from pinned FrequencyWords data (CC BY-SA 4.0)."""
|
|
from pathlib import Path
|
|
import urllib.request, hashlib, json
|
|
ROOT=Path(__file__).resolve().parents[1]/'dictionaries'
|
|
REV='525f9b560de45753a5ea01069454e72e9aa541c6'
|
|
ROOT.mkdir(exist_ok=True)
|
|
manifest={}
|
|
for lang in ('en','uk','ru'):
|
|
url=f'https://raw.githubusercontent.com/hermitdave/FrequencyWords/{REV}/content/2016/{lang}/{lang}_50k.txt'
|
|
data=urllib.request.urlopen(url).read()
|
|
rows={}
|
|
for rank,line in enumerate(data.decode('utf-8').splitlines(),1):
|
|
word,count=line.rsplit(' ',1);word=word.lower()
|
|
alphabet='abcdefghijklmnopqrstuvwxyz' if lang=='en' else ('абвгґдеєжзиіїйклмнопрстуфхцчшщьюя' if lang=='uk' else 'абвгдеёжзийклмнопрстуфхцчшщъыьэюя')
|
|
if not (2<=len(word)<=40 and all(c in alphabet for c in word)):continue
|
|
if int(count)<5:continue
|
|
rows.setdefault(word,rank)
|
|
out=''.join(f'{word}\t{rows[word]}\n' for word in sorted(rows))
|
|
(ROOT/f'{lang}.txt').write_bytes(out.encode('utf-8'))
|
|
manifest[lang]={'source':url,'sourceSha256':hashlib.sha256(data).hexdigest(),'words':len(rows),'bytes':len(out.encode()),'sha256':hashlib.sha256(out.encode()).hexdigest()}
|
|
(ROOT/'sources.json').write_text(json.dumps(manifest,indent=2)+'\n',encoding='utf-8')
|
|
print(json.dumps(manifest,indent=2))
|