"""Round 12: exact synthetic statistical diagnostics, not an LLM benchmark.

Run: python3 diagnostics.py
Requires scipy. All fixtures are constructed; no model output is represented.
"""
from __future__ import annotations

from copy import deepcopy
from hashlib import sha256
import json
import math
from pathlib import Path

import scipy
from scipy.stats import beta, binom


def upper_bound(errors: int, accepted: int, delta: float) -> float:
    """One-sided exact binomial bound; zero accepted is NOT risk zero."""
    if not 0 <= errors <= accepted or not 0 < delta < 1:
        raise ValueError("Invalid counts or delta")
    if accepted == 0 or errors == accepted:
        return 1.0
    return float(beta.ppf(1 - delta, errors + 1, accepted - errors))


def certify(records: list[dict], thresholds: list[float], alpha: float,
            delta: float) -> list[dict]:
    """Bonferroni over the ENTIRE predeclared threshold family."""
    if len(set(thresholds)) != len(thresholds) or not thresholds:
        raise ValueError("Unique nonempty predeclared threshold family required")
    if len({r['family_id'] for r in records}) != len(records):
        raise ValueError("Fixture contract: one episode per independent family")
    if any(r['error'] not in (0, 1) for r in records):
        raise ValueError("Exact-binomial rule requires binary episode loss")
    rows = []
    for threshold in thresholds:
        selected = [r for r in records if r['score'] >= threshold]
        m, k = len(selected), sum(r['error'] for r in selected)
        p = float(binom.cdf(k, m, alpha)) if m else 1.0
        upper = upper_bound(k, m, delta / len(thresholds))
        rows.append(dict(threshold=threshold, accepted=m, errors=k,
                         coverage=m / len(records),
                         empirical_risk=k / m if m else None,
                         p_value=p, upper_bound=upper,
                         certified=bool(m and p <= delta / len(thresholds))))
    return rows


def digest(text: str) -> str:
    return sha256(text.encode()).hexdigest()


def release_valid(record: dict, certificate: dict) -> tuple[bool, str]:
    """Structural guard only. Cannot establish source truth or iid sampling."""
    for field in ('pipeline_digest', 'score_digest', 'evidence_policy_digest',
                  'label_contract_digest', 'population_id'):
        if record.get(field) != certificate.get(field):
            return False, field + " mismatch"
    if record.get('scored_answer_digest') != digest(record['final_answer']):
        return False, "answer changed after scoring"
    if record.get('schema_version') != 1:
        return False, "schema version"
    required_usage = ('input_tokens', 'output_tokens', 'reasoning_tokens',
                      'tool_cost_units', 'retries', 'wall_ms')
    if any(k not in record.get('usage', {}) for k in required_usage):
        return False, "incomplete usage"
    if any(not isinstance(v, (int, float)) or v < 0
           for v in record['usage'].values()):
        return False, "invalid usage"
    if not record.get('evidence_complete') or not record.get('requirements_met'):
        return False, "incomplete answer or evidence"
    if not certificate.get('certified'):
        return False, "no certified rule"
    if record['score'] < certificate['threshold']:
        return False, "below threshold"
    return True, "structural checks passed"


def main() -> None:
    alpha, delta = .02, .05
    result = {'status': 'Executed exact synthetic diagnostics; no LLM benchmark',
              'scipy_version': scipy.__version__, 'alpha': alpha, 'delta': delta}
    result['zero_error_bounds'] = [
        {'accepted': m, 'candidates': candidates,
         'upper': upper_bound(0, m, delta / candidates)}
        for m, candidates in [(50, 1), (149, 1), (300, 1), (300, 20)]]
    result['minimum_zero_error_acceptances'] = {
        str(c): math.ceil(math.log(delta / c) / math.log1p(-alpha))
        for c in (1, 3, 20, 160)}

    records = []
    for score, count, errors in [(.99, 300, 0), (.95, 300, 4), (.8, 400, 26)]:
        for j in range(count):
            records.append({'family_id': f'synthetic-{len(records)}',
                            'score': score, 'error': int(j < errors)})
    result['worked_calibration'] = certify(records, [.8, .95, .99], alpha, delta)
    chosen = [r for r in result['worked_calibration'] if r['certified']]
    result['selected_threshold'] = max(chosen, key=lambda r: r['coverage'])['threshold']

    # Exact familywise false certification in a constructed all-bad family.
    # Each candidate has independent Bernoulli errors across episodes AND candidates.
    n, candidates, bad_risk = 300, 20, .0201
    all_k = list(range(n + 1))
    family_rows = []
    for name, cutoff in [('uncorrected_search', delta),
                         ('bonferroni', delta / candidates)]:
        eligible = [k for k in all_k if binom.cdf(k, n, alpha) <= cutoff]
        max_errors = max(eligible) if eligible else -1
        one = float(binom.cdf(max_errors, n, bad_risk))
        family_rows.append({'method': name, 'n': n, 'candidates': candidates,
                            'true_risk': bad_risk, 'max_errors': max_errors,
                            'single_false_certificate_probability': one,
                            'family_false_certificate_probability': 1 - (1 - one)**candidates})
    result['multiplicity_counterexample'] = family_rows

    result['risk_denominators'] = {
        'tasks': 1000, 'released': 100, 'wrong_released': 20,
        'unconditional_wrong_release_rate': 20/1000,
        'selective_risk': 20/100,
        'coverage': 100/1000,
        'mean_FDP_if_every_answer_has_1_error_in_100_claims': .01,
        'any_error_answer_rate_in_same_fixture': 1.0}
    result['threshold_nonmonotonicity'] = {
        'two_claim_FDP_before': 1/2, 'after_rejecting_correct_claim': 1.0,
        'answer_risk_at_score_threshold_0_8': 20/1000,
        'answer_risk_at_score_threshold_0_99': 20/100}
    result['dependent_duplicates'] = {
        'independent_families': 10, 'copies_per_family': 100, 'true_risk': .1,
        'actual_zero_error_probability': .9**10,
        'incorrect_iid_zero_error_probability': .9**1000,
        'naive_upper_from_1000_zero_errors': upper_bound(0, 1000, .05),
        'family_upper_from_10_zero_errors': upper_bound(0, 10, .05)}
    result['pooled_vs_revised'] = {
        'unchanged': {'n':900, 'errors':9}, 'revised': {'n':100, 'errors':20},
        'pooled_error':29/1000, 'revised_error':20/100}

    cert = {'pipeline_digest':'pipeline-v1', 'score_digest':'score-v1',
            'evidence_policy_digest':'frozen-sources-v1',
            'label_contract_digest':'material-error-v1',
            'population_id':'synthetic-contract-v1',
            'certified':True,'threshold':.95}
    record = {k:v for k,v in cert.items() if k.endswith('_digest') or k=='population_id'}
    record.update({'schema_version':1,'final_answer':'The setting is disabled.',
                   'scored_answer_digest':digest('The setting is disabled.'),
                   'score':.97,'evidence_complete':True,'requirements_met':True,
                   'usage':{'input_tokens':1200,'output_tokens':200,'reasoning_tokens':0,
                            'tool_cost_units':2,'retries':0,'wall_ms':2500}})
    mutations = {
        'valid': lambda r: None,
        'changed_answer': lambda r: r.update(final_answer='The setting is enabled.'),
        'changed_pipeline': lambda r: r.update(pipeline_digest='pipeline-v2'),
        'changed_score': lambda r: r.update(score_digest='score-v2'),
        'changed_sources': lambda r: r.update(evidence_policy_digest='live-sources-v2'),
        'changed_population': lambda r: r.update(population_id='shifted-domain'),
        'changed_label_contract': lambda r: r.update(label_contract_digest='lenient-labels'),
        'missing_usage': lambda r: r['usage'].pop('reasoning_tokens'),
        'incomplete_answer': lambda r: r.update(requirements_met=False),
        'below_threshold': lambda r: r.update(score=.90)}
    guards=[]
    for name, mutate in mutations.items():
        r=deepcopy(record); mutate(r)
        valid, reason=release_valid(r,cert)
        expected=name=='valid'
        assert valid == expected, (name, valid, expected)
        guards.append({'fixture':name,'released':valid,'reason':reason,'passed':True})
    result['structural_guard_checks']=guards
    result['fixture_notes'] = [
        'All scores, labels, costs, and answers are invented synthetic fixtures.',
        'Digest matching cannot detect an unrecorded provider update or false provenance.',
        'Counting families is not proof that distinct families are independent.',
        'An unavailable reasoning-token count must be recorded as unknown in real studies; this guard requires a complete accounting adapter and otherwise cannot claim exact compute matching.']
    assert result['selected_threshold'] == .95
    assert upper_bound(0,0,.05)==1.0
    assert math.isclose(upper_bound(0,50,.05),1-.05**(1/50),rel_tol=1e-12)
    Path(__file__).with_name('diagnostic_results.json').write_text(json.dumps(result,indent=2)+'\n')
    print(json.dumps(result,indent=2))


if __name__ == '__main__':
    main()