29 lines
1.7 KiB
Python
29 lines
1.7 KiB
Python
"""Build compact offline dictionaries from pinned FrequencyWords data (CC BY-SA 4.0)."""
|
|
from pathlib import Path
|
|
import urllib.request, hashlib, json
|
|
ROOT=Path(__file__).resolve().parents[1]/'dictionaries'
|
|
REV='525f9b560de45753a5ea01069454e72e9aa541c6'
|
|
# Supplemental vocabulary has no corpus frequency: rank after the 50k source list.
|
|
SUPPLEMENTAL={'uk': {'реліз': 50001}}
|
|
ROOT.mkdir(exist_ok=True)
|
|
manifest={}
|
|
for lang in ('en','uk','ru'):
|
|
url=f'https://raw.githubusercontent.com/hermitdave/FrequencyWords/{REV}/content/2016/{lang}/{lang}_50k.txt'
|
|
data=urllib.request.urlopen(url).read()
|
|
rows={}
|
|
for rank,line in enumerate(data.decode('utf-8').splitlines(),1):
|
|
word,count=line.rsplit(' ',1);word=word.lower()
|
|
alphabet='abcdefghijklmnopqrstuvwxyz' if lang=='en' else ('абвгґдеєжзиіїйклмнопрстуфхцчшщьюя' if lang=='uk' else 'абвгдеёжзийклмнопрстуфхцчшщъыьэюя')
|
|
if not (2<=len(word)<=40 and all(c in alphabet for c in word)):continue
|
|
if int(count)<5:continue
|
|
rows.setdefault(word,rank)
|
|
for word,rank in SUPPLEMENTAL.get(lang,{}).items():
|
|
rows.setdefault(word,rank)
|
|
out=''.join(f'{word}\t{rows[word]}\n' for word in sorted(rows))
|
|
(ROOT/f'{lang}.txt').write_bytes(out.encode('utf-8'))
|
|
manifest[lang]={'source':url,'sourceSha256':hashlib.sha256(data).hexdigest(),'words':len(rows),'bytes':len(out.encode()),'sha256':hashlib.sha256(out.encode()).hexdigest()}
|
|
if lang in SUPPLEMENTAL:
|
|
manifest[lang]['supplementalWords']=SUPPLEMENTAL[lang]
|
|
(ROOT/'sources.json').write_text(json.dumps(manifest,indent=2)+'\n',encoding='utf-8')
|
|
print(json.dumps(manifest,indent=2))
|