#!/usr/bin/env python3
"""Arkansas small-business website report: the numbers.

Reads one prospect scan (prospects.json, made by xiobo-site tools/prospects/prospects.php) and writes:
  out/frame.json      every study cell: the place_ids it could pick from and the 9 it picked (the seed makes it repeatable)
  out/numbers.json    the page's numbers (same shape as xiobo-site src/data/report-arkansas-2026.json)
  out/data.csv        one row per business, NO names, addresses or place ids (the public download)
  out/lines.json      ready sentences for outreach, per trade statewide and per area (no link; groups with base >= 30)
  out/qc-sample.csv   30 random websites for the hand check (private: has the website address)
  out/handcheck.csv   every "no" behind a headline number, to confirm by hand (private)
If out/handcheck-done.csv exists (same columns + `confirmed` = yes/no), a "no" a person did not confirm is dropped.
If out/qc-done.csv exists (qc-revisar.csv with `wrong_answers` = how many of its 5 answers were wrong), qc.agree is computed.

Run:  python3 study.py <scan>/prospects.json --out <dir> [--seed xiobo-ar-2026] [--per-cell 9]
Python 3.9+, standard library only. Same input + same seed = same output.
"""
import argparse, csv, json, os, random, sys
from collections import defaultdict

TRADES = [('restaurants', 'Restaurants'), ('salons_barbers', 'Salons and barbers'), ('home_services', 'Home services'),
          ('auto_repair', 'Auto repair'), ('small_health', 'Health practices'), ('local_retail', 'Local shops')]
AREAS = [('lr', 'Little Rock area'), ('nwa', 'Northwest Arkansas'), ('conway', 'Conway'), ('jonesboro', 'Jonesboro'),
         ('fortsmith', 'Fort Smith'), ('hotsprings', 'Hot Springs')]
# People age 5+ who speak Spanish at home: Census table C16001 (C16001003 / C16001001), ACS 2020-2024 5-year
# (acs2024_5yr), read 20 Oct 2026 through api.censusreporter.org; margin = 90% MOE of the share (Census formula).
CENSUS = [('Arkansas', 6.1, 0.1), ('Springdale', 31.8, 2.1), ('Rogers', 22.7, 2.6), ('Fort Smith', 15.6, 0.9), ('Little Rock', 9.0, 0.7), ('Conway', 5.6, 0.9)]
BROKEN = {'dns', 'timeout', 'fetch', '4xx', '5xx', 'empty', 'redirects', 'private'}   # each one re-opened in a real browser before we count it (browser-check.mjs)
OLD_YEAR = 2024        # "shows a copyright year of 2024 or older"
# Speed = Google PageSpeed (Lighthouse) mobile LAB test, which simulates a mid-range phone on a slow 4G connection.
# We use Google's own score bands for that test (90-100 good, 50-89 needs improvement, 0-49 poor), not the 2.5 s LCP
# line: that line is for real-visit (field) data, and comparing a lab number with it would overstate the problem.
GOOD_SCORE, POOR_SCORE = 90, 50
HEADLINE = ['noTel', 'noHours', 'pdfMenu', 'linkBroken']   # every "no" behind these is re-checked (browser-check.mjs) and spot-checked by a person


def load(path):
    with open(path) as f:
        return json.load(f)


def pick(rows, seed, per_cell):
    """Each business belongs to the study cell of its first study hit. Frame = open, not a chain. Pick per_cell at random."""
    frame = defaultdict(list)
    for r in rows:
        cells = [h['study_cell'] for h in r.get('hits', []) if h.get('study_cell')]
        if not cells or r.get('chain'):
            continue
        g = r.get('google') or {}
        if g.get('business_status') not in (None, 'OPERATIONAL'):
            continue
        frame[cells[0]].append(r['place_id'])
    picked, out = {}, {}
    for t, _ in TRADES:
        for a, _ in AREAS:
            cell = f'{t}|{a}'
            ids = sorted(set(frame.get(cell, [])))
            rnd = random.Random(f'{seed}|{cell}')
            chosen = sorted(rnd.sample(ids, min(per_cell, len(ids))))
            out[cell] = {'frame': len(ids), 'picked': chosen, 'frame_ids': ids}
            for pid in chosen:
                picked[pid] = (t, a)
    return picked, out


# Google listings that link to a directory or lead page, not to the business's own website (found in the 30-site
# reviews, 20 Oct). They count as "only a Facebook or directory page", like Yelp or Linktree.
DIRECTORY_HOSTS = ('flowershopnetwork.com', 'yourhelper.org')


def facts(r):
    """The checks for one business, from the raw measures. None = not measured (left out of that base)."""
    kind = r.get('site_kind') or 'none'
    if kind == 'own' and any((r.get('site_host') or '').endswith(h) for h in DIRECTORY_HOSTS):
        kind = 'directory'
    c = r.get('check') or {}
    raw = c.get('raw') or {}
    st = c.get('fetch_status')
    f = {'site': 'own' if kind == 'own' else ('none' if kind == 'none' else 'social'), 'kind': kind,
         'opened': kind == 'own' and st == 'ok', 'broken': kind == 'own' and st in BROKEN, 'robots': st == 'robots_disallow'}
    if f['opened']:
        lcp = raw.get('psi_mobile_lcp_s') if raw.get('psi_status') == 'ok' else None
        score = raw.get('psi_performance_score') if raw.get('psi_status') == 'ok' else None
        f.update({
            'https': bool(raw.get('https_ok')), 'viewport': bool(raw.get('viewport_meta')),
            'lcp': lcp, 'score': score, 'speed': None if score is None else ('good' if score >= GOOD_SCORE else 'slow' if score >= POOR_SCORE else 'poor'),
            'tel': bool(raw.get('tel_link')), 'hours': bool(raw.get('hours_on_page')), 'address': bool(raw.get('address_on_page')),
            'contact': bool(raw.get('contact_form') or raw.get('mailto') or raw.get('contact_page_link')),
            'phone_text': bool(raw.get('phone_in_text')),
            'pdf_menu': raw.get('pdf_menu_or_price_list') == 'menu', 'booking': bool(raw.get('online_booking_or_ordering')),
            'year': raw.get('copyright_year'), 'spanish': bool(raw.get('spanish')), 'platform': raw.get('platform') or 'Unknown',
            'url': raw.get('final_url') or r.get('site_host'),
        })
    return f


# metric id -> (in base?, fails?)
METRICS = {
    'noOwn':      (lambda f, t: True, lambda f: f['site'] != 'own'),
    'linkBroken': (lambda f, t: f['site'] == 'own' and not f['robots'], lambda f: f['broken']),
    'noHttps':    (lambda f, t: f['opened'], lambda f: not f['https']),
    'noViewport': (lambda f, t: f['opened'], lambda f: not f['viewport']),
    'slow':       (lambda f, t: f['opened'] and f['speed'] is not None, lambda f: f['speed'] != 'good'),
    'noTel':      (lambda f, t: f['opened'], lambda f: not f['tel']),
    'noHours':    (lambda f, t: f['opened'], lambda f: not f['hours']),
    'noAddress':  (lambda f, t: f['opened'], lambda f: not f['address']),
    'noContact':  (lambda f, t: f['opened'], lambda f: not f['contact']),
    'oldYear':    (lambda f, t: f['opened'], lambda f: f['year'] is not None and f['year'] <= OLD_YEAR),
    'noSpanish':  (lambda f, t: f['opened'], lambda f: not f['spanish']),
    'pdfMenu':    (lambda f, t: f['opened'] and t == 'restaurants', lambda f: f['pdf_menu']),
}


def main():
    ap = argparse.ArgumentParser()
    ap.add_argument('scan')
    ap.add_argument('--out', required=True)
    ap.add_argument('--seed', default='xiobo-ar-2026')
    ap.add_argument('--per-cell', type=int, default=9)
    A = ap.parse_args()
    os.makedirs(A.out, exist_ok=True)
    S = load(A.scan)
    rows = {r['place_id']: r for r in S['rows']}
    picked, frame = pick(S['rows'], A.seed, A.per_cell)
    with open(f'{A.out}/frame.json', 'w') as fh:
        json.dump({'scan_date': S.get('scan_date'), 'code_commit': S.get('code_commit'), 'seed': A.seed, 'per_cell': A.per_cell,
                   'cells': {k: {'frame': v['frame'], 'picked': v['picked'], 'frame_ids': v['frame_ids']} for k, v in frame.items()}}, fh, indent=1)

    F = {pid: (t, a, facts(rows[pid])) for pid, (t, a) in picked.items()}

    # second look in a real browser at every website that opened (browser-all.mjs): it answers tap-to-call and hours
    # again, both ways, and catches pages that open on an error ("Not found"). Its answer replaces the first check's.
    B = {}
    bp = f'{A.out}/browser-all.json'
    if os.path.exists(bp):
        B = json.load(open(bp))
    changed = {'tel': 0, 'hours': 0, 'notFound': 0, 'sites': 0}
    for pid, (t, a, f) in F.items():
        b = B.get(pid)
        if not f['opened'] or not b or not b.get('ok'):
            continue
        changed['sites'] += 1
        if b.get('notFound'):
            f.update(opened=False, broken=True); changed['notFound'] += 1
            continue
        for k, bk in (('tel', 'tel'), ('hours', 'hours'), ('address', 'address'), ('contact', 'reach'), ('spanish', 'spanish')):
            if bk not in b:
                continue
            if bool(b.get(bk)) != f[k]:
                changed[k] = changed.get(k, 0) + 1
            f[k] = bool(b.get(bk))

    # hand checks: drop a "no" a person did not confirm (the machine was wrong)
    wrong = set()
    done = f'{A.out}/handcheck-done.csv'
    if os.path.exists(done):
        for row in csv.DictReader(open(done)):
            if row.get('confirmed', '').strip().lower() == 'no' and (row['metric'] == 'linkBroken' or not B):
                wrong.add((row['place_id'], row['metric']))   # with browser-all.json, tel/hours come from the browser itself

    def fails(pid, mid, f):
        return METRICS[mid][1](f) and (pid, mid) not in wrong

    N = {'sample': False, 'edition': 2026, 'n': len(F), 'seed': A.seed,
         'checked': max((rows[p].get('check') or {}).get('checked_at', '') or '' for p in picked)[:10] or S.get('scan_date'),
         'checkedFrom': min(((rows[p].get('check') or {}).get('checked_at') or '9') for p in picked)[:10],
         'trades': [], 'areas': [], 'metrics': {}, 'byTrade': {}, 'byArea': {}, 'web': {'own': 0, 'social': 0, 'none': 0}}
    for pid, (t, a, f) in F.items():
        N['web'][f['site']] += 1
    for mid, (inbase, _) in METRICS.items():
        tot = [0, 0]; bt = {k: [0, 0] for k, _ in TRADES}; ba = {k: [0, 0] for k, _ in AREAS}
        for pid, (t, a, f) in F.items():
            if not inbase(f, t):
                continue
            x = 1 if fails(pid, mid, f) else 0
            for acc in (tot, bt[t], ba[a]):
                acc[0] += x; acc[1] += 1
        N['metrics'][mid] = {'fail': tot[0], 'base': tot[1]}
        N['byTrade'][mid] = bt
        N['byArea'][mid] = ba
    N['trades'] = [{'id': k, 'name': n, 'n': sum(1 for t, _, _ in F.values() if t == k)} for k, n in TRADES]
    N['areas'] = [{'id': k, 'name': n, 'n': sum(1 for _, a, _ in F.values() if a == k)} for k, n in AREAS]
    sp = [f['speed'] for _, _, f in F.values() if f['opened'] and f['speed']]
    N['speed'] = {'base': len(sp), 'good': sp.count('good'), 'slow': sp.count('slow'), 'poor': sp.count('poor')}
    lc = sorted(f['lcp'] for _, _, f in F.values() if f['opened'] and f['lcp'] is not None)
    N['speed']['lcpMedian'] = round(lc[len(lc) // 2], 1) if lc else None
    # of the websites with no tap-to-call link, how many still show the number as text (the customer must copy it)
    nt = [f for pid, (_, _, f) in F.items() if f['opened'] and fails(pid, 'noTel', f)]
    N['telText'] = {'fail': sum(1 for f in nt if f['phone_text']), 'base': len(nt)}
    sb = N['metrics']['noSpanish']
    N['spanish'] = {'yes': sb['base'] - sb['fail'], 'base': sb['base'], 'census': [{'place': p, 'pct': v, 'moe': m} for p, v, m in CENSUS]}
    plat = defaultdict(int)
    for _, _, f in F.values():
        if f['opened']:
            plat[f['platform']] += 1
    N['platforms'] = dict(sorted(plat.items(), key=lambda kv: -kv[1]))
    N['browser'] = changed
    hc = list(csv.DictReader(open(done))) if os.path.exists(done) else []
    N['handcheck'] = {'checked': len(hc), 'dropped': len(wrong), 'byMetric': {m: sum(1 for r in hc if r['metric'] == m and r.get('confirmed', '').strip().lower() == 'no') for m in HEADLINE}}
    N['qc'] = {'checked': 0, 'agree': None}
    # the reported check = the second, fresh sample of 30 (qc2-done.csv), read before any rule was tuned on it;
    # the first 30 (qc-done.csv) were used to improve the rules, so their match rate would flatter us
    qcd = f'{A.out}/qc2-done.csv' if os.path.exists(f'{A.out}/qc2-done.csv') else f'{A.out}/qc-done.csv'
    if os.path.exists(qcd):
        q = list(csv.DictReader(open(qcd)))
        q = [r for r in q if (r.get('wrong_answers') or '').strip() != '']   # qc-revisar.csv filled in: 5 answers per website
        n = 5 * len(q); ok = n - sum(int(r['wrong_answers']) for r in q)
        N['qc'] = {'checked': len(q), 'answers': n, 'match': ok, 'agree': round(ok / n, 3) if n else None}
    N['frame'] = {k: v['frame'] for k, v in frame.items()}
    # the pool: every business the study searches found, before chains and closed places were taken out (frame = after)
    N['found'] = sum(1 for r in S['rows'] if any(h.get('study_cell') for h in r.get('hits', [])))
    N['searches'] = sum(1 for q in S.get('queries', []) if any(c.get('id') == q.get('category') and c.get('study') for c in S.get('config', {}).get('categories', [])))
    with open(f'{A.out}/numbers.json', 'w') as fh:
        json.dump(N, fh, indent=1)

    # public CSV: no names, no addresses, no place ids, no website addresses
    cols = ['trade', 'area', 'website', 'opened', 'https', 'phone_ready', 'lcp_seconds_lab', 'speed_score_lab', 'speed', 'tap_to_call', 'hours', 'address',
            'contact', 'pdf_menu', 'booking_or_ordering', 'copyright_year', 'spanish', 'platform']
    with open(f'{A.out}/data.csv', 'w', newline='') as fh:
        w = csv.writer(fh); w.writerow(cols)
        for pid in sorted(F, key=lambda p: (F[p][0], F[p][1], random.Random(p).random())):
            t, a, f = F[pid]
            o = f['opened']
            w.writerow([t, a, f['kind'], o] + ([f['https'], f['viewport'], f['lcp'], f['score'], f['speed'], f['tel'], f['hours'], f['address'],
                        f['contact'], f['pdf_menu'] if t == 'restaurants' else '', f['booking'], f['year'] or '', f['spanish'], f['platform']]
                        if o else [''] * 14))

    # hand-check list and QC sample (private)
    with open(f'{A.out}/handcheck.csv', 'w', newline='') as fh:
        w = csv.writer(fh); w.writerow(['place_id', 'metric', 'trade', 'area', 'url', 'confirmed'])
        for pid, (t, a, f) in sorted(F.items()):
            for mid in HEADLINE:
                if METRICS[mid][0](f, t) and METRICS[mid][1](f):   # the machine's "no" (before any hand result)
                    w.writerow([pid, mid, t, a, f.get('url') or ('https://' + (rows[pid].get('site_host') or '') + '/'), ''])
    own = sorted(p for p, (_, _, f) in F.items() if f['opened'])
    with open(f'{A.out}/qc-sample.csv', 'w', newline='') as fh:
        w = csv.writer(fh); w.writerow(['place_id', 'url', 'machine', 'answers', 'agree'])
        for pid in sorted(random.Random(f'{A.seed}|qc').sample(own, min(30, len(own)))):
            f = F[pid][2]
            m = {k: f[k] for k in ('https', 'viewport', 'tel', 'hours', 'address', 'contact', 'pdf_menu', 'booking', 'year', 'spanish')}
            w.writerow([pid, f['url'], json.dumps(m), '', ''])

    # outreach lines (no link): the trade statewide, and the area across all trades. A trade x area cell has only ~9
    # businesses, too few to quote. Only groups with base >= 30.
    WHAT = {'noTel': 'have no tap-to-call phone number on their website', 'noHours': 'do not show their hours on the home page',
            'pdfMenu': 'put the menu in a PDF', 'slow': 'have a website that is slow on a phone', 'noOwn': 'have no website of their own'}
    ADJ = {'restaurants': 'restaurant', 'salons_barbers': 'salon and barber shop', 'home_services': 'home service',
           'auto_repair': 'auto repair', 'small_health': 'health practice', 'local_retail': 'local shop'}

    def say(mid, f_, b_, who):
        pct = round(100 * f_ / b_)
        return pct, f'We checked {b_} {who}: {pct}% {WHAT[mid]}.'
    lines = []
    for mid in ['noTel', 'noHours', 'pdfMenu', 'slow', 'noOwn']:
        unit = 'businesses' if mid == 'noOwn' else 'websites'
        for t, _ in TRADES:
            f_, b_ = N['byTrade'][mid][t]
            if b_ >= 30:
                pct, s_ = say(mid, f_, b_, f'{ADJ[t]} {unit} across Arkansas')
                lines.append({'metric': mid, 'by': 'trade', 'trade': t, 'fail': f_, 'base': b_, 'pct': pct, 'sentence': s_})
        if mid == 'pdfMenu':
            continue
        for a, an in AREAS:
            f_, b_ = N['byArea'][mid][a]
            if b_ >= 30:
                pct, s_ = say(mid, f_, b_, f'small businesses in {an}' if mid == 'noOwn' else f'small-business websites in {an}')
                lines.append({'metric': mid, 'by': 'area', 'area': a, 'fail': f_, 'base': b_, 'pct': pct, 'sentence': s_})
    with open(f'{A.out}/lines.json', 'w') as fh:
        json.dump({'note': 'No link in the sentence (step 0). Link the study page in the day-3 follow-up.', 'lines': lines}, fh, indent=1)

    small = {k: v['frame'] for k, v in frame.items() if v['frame'] < A.per_cell}
    print(f"picked {len(F)} of {len(TRADES) * len(AREAS) * A.per_cell}; web {N['web']}; cells short: {small or 'none'}", file=sys.stderr)


if __name__ == '__main__':
    main()