"""Rebuild licensed character shards from downloaded upstream snapshots.
Usage: python3 scripts/characters/import-data.py /absolute/source-directory
No network calls, generated readings, or invented stroke geometry.
"""
import sys,json,zipfile,re,unicodedata,hashlib
from pathlib import Path
from collections import defaultdict
src=Path(sys.argv[1]);root=Path(__file__).resolve().parents[2];out=root/'public/characters';out.mkdir(exist_ok=True)
(out/'shards').mkdir(exist_ok=True);(out/'licenses').mkdir(exist_ok=True)
fields={'kDefinition':'definition','kMandarin':'mandarin','kCantonese':'cantonese','kJapaneseOn':'on','kJapaneseKun':'kun','kJapanese':'japanese','kRSUnicode':'rs','kTotalStrokes':'totalStrokes','kSimplifiedVariant':'simplified','kTraditionalVariant':'traditional','kJoyoKanji':'joyo','kJinmeiyoKanji':'jinmeiyo','kGradeLevel':'grade'}
entries=defaultdict(dict)
z=zipfile.ZipFile(src/'Unihan.zip')
for name in z.namelist():
 for line in z.read(name).decode().splitlines():
  if not line.startswith('U+'):continue
  cp,key,value=line.split('\t',2);c=chr(int(cp[2:],16));entries[c]['char']=c
  if key in fields:entries[c][fields[key]]=value
kanji=zipfile.ZipFile(src/'kanjivg.zip');jp=0
for name in kanji.namelist():
 m=re.search(r'/kanji/([0-9a-f]{5,6})\.svg$',name)
 if not m:continue
 c=chr(int(m[1],16))
 if c not in entries:continue
 xml=kanji.read(name).decode();paths=re.findall(r'<path\b[^>]*\bd="([^"]+)"',xml)
 if not paths:continue
 components=list(dict.fromkeys(re.findall(r'kvg:element="([^"]+)"',xml)))[1:]
 entries[c]['jp']={'strokes':paths,'components':components};jp+=1
(out/'licenses/KanjiVG-COPYING.txt').write_bytes(kanji.read('kanjivg-master/COPYING'))
# Dictionary and geometry retain their separate upstream licenses.
for line in (src/'hanzi-dictionary.txt').read_text().splitlines():
 row=json.loads(line);c=row['character'];entries[c]['char']=c
 entries[c]['hanzi']={k:row[k] for k in ['definition','pinyin','decomposition','radical'] if k in row}
cn=0
for line in (src/'hanzi-graphics.txt').read_text().splitlines():
 row=json.loads(line);c=row['character'];entries[c]['char']=c
 assert row['strokes'] and len(row['strokes'])==len(row['medians'])
 entries[c]['cn']={'strokes':row['strokes'],'medians':row['medians']};cn+=1
radicals=[]
for n in range(1,215):
 symbol=chr(0x2f00+n-1);char=unicodedata.normalize('NFKC',symbol)
 radicals.append({'number':n,'symbol':char,'name':unicodedata.name(symbol).replace('KANGXI RADICAL ','').lower(),'forms':[]})
for line in (src/'CJKRadicals.txt').read_text().splitlines():
 if not line or line.startswith('#'):continue
 num,rad,unified=[p.strip() for p in line.split('#')[0].split(';')]
 n=int(num.replace("'",''))
 form=chr(int(unified,16))
 if form!=radicals[n-1]['symbol']:radicals[n-1]['forms'].append(form)
for row in entries.values():
 if row.get('rs'):row['radical']=int(row['rs'].split('.')[0].replace("'",''))
 row['isJoyo']=row.get('joyo')=='2010'
# Keep rare-character lookups available without sending the complete corpus to a phone.
shards=defaultdict(dict)
for c,row in entries.items():shards[f'{ord(c)>>8:x}'][c]=row
def write(path,data):path.write_text(json.dumps(data,ensure_ascii=False,separators=(',',':'))+'\n')
for key,rows in shards.items():write(out/'shards'/f'{key}.json',rows)
for track,field in [('japanese','jp'),('chinese','cn')]:
 index=[]
 for c,row in entries.items():
  if field not in row:continue
  index.append({k:row[k] for k in ['char','definition','on','kun','mandarin','cantonese','radical','isJoyo'] if k in row}|{'strokes':len(row[field]['strokes']),'pinyin':row.get('hanzi',{}).get('pinyin',[]),'gloss':row.get('hanzi',{}).get('definition','')})
 index.sort(key=lambda r:ord(r['char']));write(out/f'{track}.json',index)
write(out/'radicals.json',radicals)
manifest={'checked':'2026-09-15','unicodeVersion':'17.0.0','referenceEntries':len(entries),'japaneseStrokeEntries':jp,'chineseStrokeEntries':cn,'joyoEntries':sum(r.get('isJoyo',False) for r in entries.values()),'joyoWithStrokes':sum(r.get('isJoyo',False) and 'jp' in r for r in entries.values()),'radicals':214,'shards':len(shards),'inputs':{p.name:hashlib.sha256(p.read_bytes()).hexdigest() for p in src.iterdir() if p.name in ['kanjivg.zip','Unihan.zip','hanzi-graphics.txt','hanzi-dictionary.txt','CJKRadicals.txt']}}
write(out/'manifest.json',manifest)
for name in ['unicode-license.txt','hanzi-COPYING','hanzi-LGPL','hanzi-APL','hanzi-GPL']:
 if (src/name).exists():(out/'licenses'/f'{name}.txt').write_bytes((src/name).read_bytes())
print(json.dumps(manifest,indent=2))
assert manifest['joyoEntries']==2136 and manifest['joyoWithStrokes']==2136