scripts/bias_pattern_detector.py
7.3 KB| 1 | #!/usr/bin/env python3 |
| 2 | """bias_pattern_detector.py — Scan conversation text for 5-bias signal patterns. |
| 3 | |
| 4 | Stdlib-only. Scans a conversation transcript and flags patterns indicative |
| 5 | of each of the 5 cognitive biases (confirmation, sunk_cost, anchoring, |
| 6 | complexity, recency). |
| 7 | |
| 8 | The detector is HEURISTIC. It surfaces candidate patterns; the reflect |
| 9 | skill's reasoning applies judgment on top. |
| 10 | |
| 11 | NO LLM CALLS. Pure regex + counting. |
| 12 | |
| 13 | Usage: |
| 14 | python bias_pattern_detector.py --conversation /tmp/transcript.txt |
| 15 | python bias_pattern_detector.py --conversation /tmp/transcript.txt --output json |
| 16 | python bias_pattern_detector.py --sample |
| 17 | """ |
| 18 | |
| 19 | import argparse |
| 20 | import json |
| 21 | import re |
| 22 | import sys |
| 23 | from pathlib import Path |
| 24 | from typing import Any, Dict, List |
| 25 | |
| 26 | |
| 27 | BIAS_PATTERNS = { |
| 28 | "confirmation": { |
| 29 | "supporting": [ |
| 30 | r"\bconfirms?\b", |
| 31 | r"\bsupports?\b", |
| 32 | r"\bas expected\b", |
| 33 | r"\bproves?\b", |
| 34 | r"\bverifies?\b", |
| 35 | ], |
| 36 | "counter_dismissal": [ |
| 37 | r"\bbut that doesn'?t apply\b", |
| 38 | r"\bedge case\b", |
| 39 | r"\bnot relevant here\b", |
| 40 | r"\boutlier\b", |
| 41 | r"\bexception\b", |
| 42 | ], |
| 43 | }, |
| 44 | "sunk_cost": [ |
| 45 | r"\bwe'?ve\s+(already\s+)?(invested|spent|put in)\b", |
| 46 | r"\btoo far along\b", |
| 47 | r"\btoo much work\b", |
| 48 | r"\bafter all (that|this) work\b", |
| 49 | r"\bwe'?re committed\b", |
| 50 | r"\bcan'?t back out\b", |
| 51 | r"\bdon'?t want to lose\b", |
| 52 | ], |
| 53 | "anchoring": [ |
| 54 | r"\bcompared to (the )?(first|original|initial)\b", |
| 55 | r"\bvs (the )?first option\b", |
| 56 | r"\bvariation of\b", |
| 57 | r"\bmodification of\b", |
| 58 | r"\bbuilding on the (first|original)\b", |
| 59 | r"\bsticking with\b", |
| 60 | ], |
| 61 | "complexity": [ |
| 62 | r"\bwhat about\s+\w+", |
| 63 | r"\bwe should also\b", |
| 64 | r"\bwe need to handle\b", |
| 65 | r"\badd (a|an)\s+\w+\s+(layer|wrapper|check|safeguard|fallback)\b", |
| 66 | r"\bjust in case\b", |
| 67 | r"\bfor robustness\b", |
| 68 | ], |
| 69 | "recency": [ |
| 70 | r"\bbased on what we'?ve been discussing\b", |
| 71 | r"\brecently\s+\w+\b", |
| 72 | r"\bjust now\b", |
| 73 | r"\bthe last few\b", |
| 74 | ], |
| 75 | } |
| 76 | |
| 77 | |
| 78 | SAMPLE_CONVERSATION = """User: I want to build a notification system for my SaaS app. Should support email + push + in-app. |
| 79 | Assistant: Great. Let's start with the data model — what events trigger notifications? |
| 80 | User: Mainly account changes, billing alerts, and team-mention notifications. |
| 81 | Assistant: OK, I'll propose a queue-based architecture with Redis + workers. |
| 82 | User: Sounds good. Can we also add a feature flag system for rollout? |
| 83 | Assistant: Yes, we can layer in feature flags. We should also add a rate limiter for safety. |
| 84 | User: What about retry logic for failed deliveries? |
| 85 | Assistant: Good point. Adding exponential backoff with jitter. We should also handle dead-letter queues. |
| 86 | User: What about a webhook system for third-party integrations? |
| 87 | Assistant: We can extend to webhooks. We should add HMAC signature verification just in case. |
| 88 | User: What about analytics tracking? |
| 89 | Assistant: Adding event analytics. We should also handle GDPR consent tracking for robustness. |
| 90 | User: We've invested a lot in this architecture already. What about adding a template system? |
| 91 | Assistant: We're far enough along that adding templates makes sense. Just sticking with the queue-based foundation. |
| 92 | User: Hmm, are we missing something? This feels complex. |
| 93 | """ |
| 94 | |
| 95 | |
| 96 | def detect_biases(conversation: str) -> Dict[str, Any]: |
| 97 | results: Dict[str, Any] = {} |
| 98 | |
| 99 | # Confirmation: supporting cites count vs counter dismissal |
| 100 | confirmation_data = BIAS_PATTERNS["confirmation"] |
| 101 | supporting_count = sum( |
| 102 | len(re.findall(p, conversation, re.IGNORECASE)) |
| 103 | for p in confirmation_data["supporting"] |
| 104 | ) |
| 105 | dismissal_count = sum( |
| 106 | len(re.findall(p, conversation, re.IGNORECASE)) |
| 107 | for p in confirmation_data["counter_dismissal"] |
| 108 | ) |
| 109 | confirmation_signal = supporting_count >= 2 or dismissal_count >= 1 |
| 110 | results["confirmation"] = { |
| 111 | "detected": confirmation_signal, |
| 112 | "supporting_hits": supporting_count, |
| 113 | "counter_dismissal_hits": dismissal_count, |
| 114 | "rationale": ( |
| 115 | "Multiple confirming phrases + dismissed counter-evidence" |
| 116 | if confirmation_signal else "No strong confirmation-bias signal" |
| 117 | ), |
| 118 | } |
| 119 | |
| 120 | for bias in ["sunk_cost", "anchoring", "complexity", "recency"]: |
| 121 | patterns = BIAS_PATTERNS[bias] |
| 122 | hits = [] |
| 123 | for p in patterns: |
| 124 | matches = re.findall(p, conversation, re.IGNORECASE) |
| 125 | if matches: |
| 126 | hits.extend(matches) |
| 127 | threshold = 2 if bias == "complexity" else 1 |
| 128 | detected = len(hits) >= threshold |
| 129 | results[bias] = { |
| 130 | "detected": detected, |
| 131 | "hits": len(hits), |
| 132 | "match_examples": hits[:3], |
| 133 | "rationale": ( |
| 134 | f"Found {len(hits)} signal(s) (threshold: {threshold})" |
| 135 | if detected else f"Found {len(hits)} signal(s), below threshold {threshold}" |
| 136 | ), |
| 137 | } |
| 138 | |
| 139 | detected_biases = [b for b, d in results.items() if d["detected"]] |
| 140 | return { |
| 141 | "biases_detected": detected_biases, |
| 142 | "biases_clear": [b for b in results if b not in detected_biases], |
| 143 | "details": results, |
| 144 | } |
| 145 | |
| 146 | |
| 147 | def render_human(result: Dict[str, Any]) -> str: |
| 148 | out: List[str] = [] |
| 149 | if result["biases_detected"]: |
| 150 | out.append(f"⚠️ Potential biases detected ({len(result['biases_detected'])}):") |
| 151 | for bias in result["biases_detected"]: |
| 152 | d = result["details"][bias] |
| 153 | out.append(f"") |
| 154 | out.append(f" [!] {bias.upper()}") |
| 155 | out.append(f" Rationale: {d['rationale']}") |
| 156 | if "match_examples" in d and d["match_examples"]: |
| 157 | out.append(f" Example matches: {d['match_examples']}") |
| 158 | else: |
| 159 | out.append("[ok] No strong bias signals detected.") |
| 160 | |
| 161 | if result["biases_clear"]: |
| 162 | out.append("") |
| 163 | out.append("Biases checked but not detected:") |
| 164 | for bias in result["biases_clear"]: |
| 165 | out.append(f" - {bias}") |
| 166 | |
| 167 | out.append("") |
| 168 | out.append("Note: detector is HEURISTIC. Reflect skill's reasoning applies judgment on top.") |
| 169 | return "\n".join(out) |
| 170 | |
| 171 | |
| 172 | def main(argv: List[str]) -> int: |
| 173 | parser = argparse.ArgumentParser(description=__doc__.split("\n")[0]) |
| 174 | parser.add_argument("--conversation", help="Path to conversation transcript text file") |
| 175 | parser.add_argument("--sample", action="store_true", help="Run on embedded sample (multi-bias scenario)") |
| 176 | parser.add_argument("--output", choices=["human", "json"], default="human") |
| 177 | args = parser.parse_args(argv) |
| 178 | |
| 179 | if args.sample: |
| 180 | text = SAMPLE_CONVERSATION |
| 181 | elif args.conversation: |
| 182 | p = Path(args.conversation) |
| 183 | if not p.exists(): |
| 184 | print(f"error: {args.conversation} not found", file=sys.stderr) |
| 185 | return 2 |
| 186 | text = p.read_text(encoding="utf-8") |
| 187 | else: |
| 188 | parser.print_help() |
| 189 | return 0 |
| 190 | |
| 191 | result = detect_biases(text) |
| 192 | if args.output == "json": |
| 193 | print(json.dumps(result, indent=2)) |
| 194 | else: |
| 195 | print(render_human(result)) |
| 196 | return 0 |
| 197 | |
| 198 | |
| 199 | if __name__ == "__main__": |
| 200 | sys.exit(main(sys.argv[1:])) |
文件内容是平台检查时读出的文本，只供查看。平台不会运行包里的任何脚本或程序。
