#!/usr/bin/env python3
"""Failure hunt (James's order, 2026-10-20): single naive prompts DeepSeek
actually FAILS. 5 mechanically-graded tasks x 10 trials x 2 models.
No system prompt, API-default temperature. Raw outputs saved to hunt.json."""
import json, os, re, subprocess

HERE = os.path.dirname(os.path.abspath(__file__))
CLI = os.path.expanduser("~/workspace/custom-connectors/custom.legacy-deepseek/bin/deepseek-api")
ENV = {**os.environ, "PYTHONPATH": "/opt/muse/skills/custom-connector-setup/bin"}

PARA = ("The harness freezes the checklist before the work begins. A good harness "
        "does not trust the model's word; the harness counts, checks, and keeps the "
        "receipts. When the harness repairs, it repairs once, inside the lines the "
        "harness drew.")
TRUTH_HARNESS = len(re.findall(r"\bharness\b", PARA, re.I))

def call(model, prompt):
    payload = {"model": model, "messages": [{"role": "user", "content": prompt}]}
    r = subprocess.run([CLI, "POST", "/chat/completions", "raw:" + json.dumps(payload)],
                       capture_output=True, text=True, env=ENV, timeout=120)
    d = json.loads(r.stdout)
    return d["choices"][0]["message"]["content"]

NUMWORDS = {"one": 1, "two": 2, "three": 3, "four": 4, "five": 5, "six": 6,
            "seven": 7, "eight": 8, "nine": 9, "ten": 10, "zero": 0}

def extract_number(text):
    m = re.search(r"\d+", text)
    if m:
        return int(m.group())
    low = text.lower()
    for w, n in NUMWORDS.items():
        if re.search(r"\b" + w + r"\b", low):
            return n
    return None

def words(text):
    return [w for w in re.split(r"\s+", text.strip()) if w]

def sentences(text):
    return [s for s in re.split(r"(?<=[.!?])\s+", text.strip().replace("\n", " ")) if s.strip()]

TASKS = [
    {"id": "T1-d-in-deepseek",
     "prompt": "How many D's are in the word DEEPSEEK?",
     "check": lambda a: extract_number(a) == 1, "truth": "1"},
    {"id": "T2-r-in-strawberry",
     "prompt": "How many r's are in the word strawberry?",
     "check": lambda a: extract_number(a) == 3, "truth": "3"},
    {"id": "T3-count-harness-in-paragraph",
     "prompt": f"How many times does the word 'harness' appear in this paragraph?\n\n{PARA}",
     "check": lambda a: extract_number(a) == TRUTH_HARNESS, "truth": str(TRUTH_HARNESS)},
    {"id": "T4-exactly-100-words",
     "prompt": "Write a short product description for a coffee mug in exactly 100 words. Output only the description.",
     "check": lambda a: len(words(a)) == 100, "truth": "100 words"},
    {"id": "T5-five-sentences-seven-words",
     "prompt": "Write a story in exactly 5 sentences, where every sentence has exactly 7 words. Output only the story.",
     "check": lambda a: len(sentences(a)) == 5 and all(len(words(s)) == 7 for s in sentences(a)),
     "truth": "5 sentences x 7 words"},
]

results = {}
for model in ("deepseek-chat", "deepseek-flash"):
    results[model] = {}
    for t in TASKS:
        trials = []
        for i in range(10):
            try:
                ans = call(model, t["prompt"])
                ok = bool(t["check"](ans))
            except Exception as e:
                ans, ok = f"<error {e}>", False
            trials.append({"answer": ans, "pass": ok})
        p = sum(1 for x in trials if x["pass"])
        results[model][t["id"]] = {"truth": t["truth"], "pass": p, "trials": trials}
        print(f"{model} {t['id']}: {p}/10 pass (truth: {t['truth']})", flush=True)

json.dump(results, open(os.path.join(HERE, "hunt.json"), "w"), indent=1)
print("HUNT DONE. paragraph 'harness' truth =", TRUTH_HARNESS)