"""Recompute the Sydney trades index from preserved answer JSON, without network access.

Counts are descriptive of this frozen convenience sample, never estimates for
Sydney businesses or consumers. A citation is a captured direct answer link;
source-panel-only pages are separate. One answer can use several source types.
"""
import csv
import hashlib
import itertools
import json
import re
from collections import Counter, defaultdict
from datetime import datetime
from pathlib import Path
from urllib.parse import urlsplit, urlunsplit, parse_qsl, urlencode
from zoneinfo import ZoneInfo

RUN = Path(__file__).resolve().parent
ENGINES = ['chatgpt', 'google-ai-mode', 'google-ai-overview', 'perplexity']
LABELS = {'chatgpt':'ChatGPT','google-ai-mode':'Google AI Mode','google-ai-overview':'Google AI Overviews','perplexity':'Perplexity'}
DIRECTORIES = {'hipages.com.au','oneflare.com.au','yelp.com','yelp.com.au','localsearch.com.au','whereis.com','yellowpages.com.au','whitepages.com.au','wordofmouth.com.au','serviceseeking.com.au','trueblue.com.au','truelocal.com.au','service.com.au','houzz.com.au','houzz.com'}
REVIEWS = {'productreview.com.au','getbirdeye.com.au','birdeye.com','trustpilot.com','trustindex.io'}
FORUMS = {'reddit.com','forums.whirlpool.net.au','whirlpool.net.au'}
LISTICLES = {'bestinhood.com','starworks.com.au','homedone.com.au','eehotwater.com.au','onyamagazine.com','sydneyuncovered.com','theurbanlist.com','expertise.com'}

def domain(url):
    return urlsplit(url).netloc.lower().removeprefix('www.')

def norm(name):
    # Punctuation and legal suffixes only. No fuzzy or location-based merges.
    name = re.sub(r'\bpty\.?\s+ltd\.?\b', '', name, flags=re.I)
    return ' '.join(re.findall(r'[a-z0-9]+', name.casefold().replace('’', "'")))

def category(url):
    d=domain(url)
    if any(d==root or d.endswith('.'+root) for root in DIRECTORIES): return 'directory'
    if d=='masterlocksmiths.com.au' and '/sys/publicprofile/' in url.lower(): return 'directory'
    if any(d==root or d.endswith('.'+root) for root in REVIEWS): return 'review platform'
    if d in FORUMS or d.endswith('.reddit.com'): return 'community forum'
    if d in LISTICLES: return 'editorial list or guide'
    if d in {'google.com','maps.google.com','maps.app.goo.gl','google.com.au'}: return 'Google business or map record'
    if d.endswith('.gov.au') or d.endswith('.gov'): return 'government'
    return 'other or business website'

def clean_url(url):
    u=urlsplit(url)
    return urlunsplit((u.scheme,u.netloc,u.path,urlencode([(k,v) for k,v in parse_qsl(u.query) if not k.startswith('utm_')]),''))

def citation_counts(rows):
    """Recount manual source matches; never infer ownership from domain spelling."""
    path=RUN/'citation-review.json'
    if not path.exists(): return {'complete':False,'by_engine':{}}
    review=json.loads(path.read_text())
    lookup={a['engine']+'-'+a['question_id']:a for a in rows if a['status']=='answered'}
    counts={e:Counter() for e in ENGINES}; reviewed=set()
    for a in review['answers']:
        key=a['answer_key']
        assert key in lookup,key
        source=lookup[key]
        if (RUN/'answers').is_dir():
            assert hashlib.sha256((RUN/a['answer_path']).read_bytes()).hexdigest()==a['answer_sha256'],key
            assert hashlib.sha256((RUN/a['raw_path']).read_bytes()).hexdigest()==a['raw_sha256'],key
        else:
            payload={k:v for k,v in source.items() if k!='file'}
            assert hashlib.sha256(json.dumps(payload,ensure_ascii=False,sort_keys=True).encode()).hexdigest()==a['public_answer_sha256'],key
        assert [r['business_name'] for r in a['records']]==source['businesses_named'],key
        captured={clean_url(u) for u in source.get('cited_urls',[])}
        c=counts[source['engine']];c['reviewed_answers']+=1;reviewed.add(key)
        for r in a['records']:
            c['named_business_occurrences']+=1
            for field in ['own_site_urls','directory_urls','review_platform_urls','community_urls','google_business_urls','other_urls']:
                urls=[u for u in r.get(field,[]) if clean_url(u) in captured]
                if field=='own_site_urls' and r.get('identity_uncertain'): urls=[]
                c[field.removesuffix('_urls')+'_confirmed']+=bool(urls)
            c['unresolved_own_site']+=bool(r.get('unresolved_own_site'))
            c['unresolved_group']+=bool(r.get('unresolved_group'))
            c['uncertain_identity']+=bool(r.get('identity_uncertain'))
    totals=Counter()
    for c in counts.values():totals.update(c)
    return {'complete':set(lookup)==reviewed and review.get('collection_settled',False),'reviewed_answer_count':len(reviewed),'answered_denominator':len(lookup),'by_engine':{e:dict(c) for e,c in counts.items()},'totals':dict(totals),'method':review['method']}

def main():
    questions=json.loads((RUN/'question-set.json').read_text())
    frozen=json.loads((RUN/'freeze.json').read_text())
    assert hashlib.sha256((RUN/'question-set.json').read_bytes()).hexdigest()==frozen['question_set_sha256'],'Frozen question file changed'
    qs={q['id']:q for q in questions['questions']}
    rows=[]
    internal=(RUN/'answers').is_dir()
    inputs=[(path,json.loads(path.read_text())) for path in sorted((RUN/'answers').glob('*.json'))] if internal else [(Path('answers.json'),a) for a in json.loads((RUN/'answers.json').read_text())]
    for path,a in inputs:
        assert a['question_id'] in qs, path
        assert a['engine'] in ENGINES, path
        assert a['question']==qs[a['question_id']]['question'], path
        assert a['question_set_sha256']==frozen['question_set_sha256'], path
        assert a['status'] in {'answered','blocked','failed','no_overview'}, path
        if a['status']=='answered':
            assert a.get('answer_text','').strip(), path
            assert isinstance(a.get('businesses_named'),list), path
            assert a.get('completion_evidence'), path
            for n in a['businesses_named']:
                assert n in a['answer_text'], (path,n)
        if internal: assert (RUN/a['raw_evidence']).is_file(), path
        a['file']=str(path.relative_to(RUN)) if internal else 'answers.json#'+a['engine']+'-'+a['question_id']
        rows.append(a)
    keys=[(a['engine'],a['question_id']) for a in rows]
    assert len(keys)==len(set(keys)), 'Duplicate question/engine'
    answered=[a for a in rows if a['status']=='answered']
    businesses={}; sources={}; by_trade={}; domain_rows={}; named_rows=[]
    for engine in ENGINES:
        allrows=[a for a in rows if a['engine']==engine]
        ok=[a for a in allrows if a['status']=='answered']
        freq=Counter(); spellings={}; sourcefreq=Counter(); domains=Counter()
        for a in ok:
            for n in dict.fromkeys(norm(n) for n in a['businesses_named']): freq[n]+=1
            for n in a['businesses_named']: spellings.setdefault(norm(n),n)
            for cat in {category(u) for u in a.get('cited_urls',[])}: sourcefreq[cat]+=1
            for d in {domain(u) for u in a.get('cited_urls',[])}: domains[d]+=1
            for position,n in enumerate(a['businesses_named'],1):
                named_rows.append({'engine':engine,'question_id':a['question_id'],'trade':a['trade'],'position':position,'business_name':n,'normalised_name':norm(n),'answer_file':a['file']})
        businesses[engine]=[{'name':spellings[n],'normalised_name':n,'answer_count':count,'answered_denominator':len(ok)} for n,count in freq.most_common()]
        sources[engine]=[{'source_type':cat,'answers_with_source_type':count,'answered_denominator':len(ok)} for cat,count in sorted(sourcefreq.items(),key=lambda x:(-x[1],x[0]))]
        domain_rows[engine]=[{'domain':d,'answer_count':n,'answered_denominator':len(ok)} for d,n in sorted(domains.items(),key=lambda x:(-x[1],x[0]))]
        by_trade[engine]={}
        for trade in dict.fromkeys(q['trade'] for q in qs.values()):
            tradeok=[a for a in ok if a['trade']==trade]
            by_trade[engine][trade]={'answered':len(tradeok),'named_business_occurrences':sum(len(set(norm(n) for n in a['businesses_named'])) for a in tradeok),'unique_businesses':len({norm(n) for a in tradeok for n in a['businesses_named']}),'no_business_answers':sum(not a['businesses_named'] for a in tradeok)}
            tf=Counter(norm(n) for a in tradeok for n in dict.fromkeys(a['businesses_named']))
            by_trade[engine][trade]['businesses']=[{'name':spellings[n],'answer_count':count,'answered_denominator':len(tradeok)} for n,count in tf.most_common()]
    counts={}
    for e in ENGINES:
        erows=[a for a in rows if a['engine']==e]; ok=[a for a in erows if a['status']=='answered']
        c=Counter(a['status'] for a in erows)
        counts[e]={**{s:c[s] for s in ['answered','blocked','failed','no_overview']},'missing':len(qs)-len(erows),'attempted':sum(a.get('attempted',True) for a in erows),'named_business_answers':sum(bool(a['businesses_named']) for a in ok),'no_business_answers':sum(not a['businesses_named'] for a in ok),'unique_businesses':len(businesses[e]),'named_business_occurrences':sum(len(set(norm(n) for n in a['businesses_named'])) for a in ok),'answers_with_direct_citations':sum(bool(a.get('cited_urls')) for a in ok),'unique_cited_domains':len(domain_rows[e])}
    pairs=[]; same_question=[]
    lookup={(a['engine'],a['question_id']):a for a in answered}
    for a,b in itertools.combinations(ENGINES,2):
        na={x['normalised_name'] for x in businesses[a]}; nb={x['normalised_name'] for x in businesses[b]}
        common=sorted(na&nb)
        pairs.append({'engine_a':a,'engine_b':b,'shared_businesses':len(common),'combined_unique_businesses':len(na|nb),'jaccard_percent':round(100*len(common)/len(na|nb),1) if na|nb else None,'names':common})
        matched=[]
        for qid in qs:
            if (a,qid) not in lookup or (b,qid) not in lookup: continue
            aa=lookup[a,qid]; bb=lookup[b,qid]
            an={norm(n) for n in aa['businesses_named']}; bn={norm(n) for n in bb['businesses_named']}
            matched.append({'question_id':qid,'shared':sorted(an&bn),'union_count':len(an|bn),'both_name_business':bool(an and bn)})
        same_question.append({'engine_a':a,'engine_b':b,'paired_answered_denominator':len(matched),'pairs_with_shared_business':sum(bool(m['shared']) for m in matched),'paired_both_name_denominator':sum(m['both_name_business'] for m in matched),'details':matched})
    complete=[qid for qid in qs if all((e,qid) in lookup for e in ENGINES)]
    fourway=[]
    for qid in complete:
        sets=[{norm(n) for n in lookup[e,qid]['businesses_named']} for e in ENGINES]
        fourway.append({'question_id':qid,'shared_all_four':sorted(set.intersection(*sets)),'union_count':len(set.union(*sets))})
    dates=sorted({datetime.fromisoformat(a['captured_at'].replace('Z','+00:00')).astimezone(ZoneInfo('Australia/Sydney')).strftime('%-d %B %Y') for a in rows})
    result={'run_id':rows[0]['run_id'] if rows else RUN.name,'question_count':len(qs),'trade_count':len({q['trade'] for q in qs.values()}),'question_source_count':len(questions.get('sources',[])),'planned_observations':len(qs)*len(ENGINES),'saved_records':len(rows),'answered_total':len(answered),'date_labels':dates,'counts':counts,'businesses':businesses,'by_trade':by_trade,'source_types':sources,'domains':domain_rows,'cross_engine_unique_overlap':pairs,'same_question_overlap':same_question,'four_engine_paired_denominator':len(complete),'four_engine_pairs_with_shared_business':sum(bool(x['shared_all_four']) for x in fourway),'four_engine_details':fourway,'method_notes':['Answers only are denominators for no-business and direct-citation rates.','Absent Overviews, blocks, failures and unattempted lane-block rows are not no-business answers.','Business comparison removes punctuation, case and Pty Ltd suffix only.','Source types can co-occur in an answer; totals do not form a market-share measure.','Other or business website is not equivalent to a named business own site.','This run observes existing signed-in account state, not a representative or controlled population.']}
    result['citation_matches']=citation_counts(rows)
    result['market_mismatches']=[{'engine':a['engine'],'question_id':a['question_id'],'reason':a.get('market_mismatch_reason')} for a in answered if a.get('market_mismatch')]
    (RUN/'analysis.json').write_text(json.dumps(result,ensure_ascii=False,indent=2)+'\n')
    with (RUN/'business-mentions.csv').open('w') as f:
        w=csv.DictWriter(f,fieldnames=list(named_rows[0]) if named_rows else ['engine','question_id','trade','position','business_name','normalised_name','answer_file']);w.writeheader();w.writerows(named_rows)
    table=['| Engine | Answered | Blocked | Failed | No Overview | Missing | No business / answered |','| --- | ---: | ---: | ---: | ---: | ---: | --- |']
    for e,c in counts.items(): table.append(f"| {LABELS[e]} | {c['answered']} | {c['blocked']} | {c['failed']} | {c['no_overview']} | {c['missing']} | {c['no_business_answers']}/{c['answered']} |")
    output='\n'.join([f"Frozen questions: {len(qs)}; trades: {result['trade_count']}; planned observations: {result['planned_observations']}",f"Saved records: {len(rows)}; answered: {len(answered)}",*table,f"Four-engine paired answers: {len(complete)}; questions sharing a business across all four: {result['four_engine_pairs_with_shared_business']}/{len(complete)}",'Record validation: PASS (frozen prompt, hash, status, answer, name substrings, raw evidence, duplicate keys)',f"Collection completeness: {'PASS' if len(rows)==len(qs)*len(ENGINES) else 'INCOMPLETE'}"])
    (RUN/'recompute-output.txt').write_text(output+'\n'); print(output)

if __name__=='__main__': main()