scripts/ai_content_detector.py
10.8 KB| 1 | #!/usr/bin/env python3 |
| 2 | """ |
| 3 | ai_content_detector.py — Detect AI-generated content patterns. |
| 4 | |
| 5 | Three detection methods: |
| 6 | 1. Burstiness analysis — human writing has high variance in sentence length; |
| 7 | AI writing has consistently medium-length sentences |
| 8 | 2. Vocabulary diversity (Type-Token Ratio) — AI reuses words more than humans |
| 9 | 3. Known AI phrases — specific phrases that appear disproportionately in |
| 10 | AI-generated text |
| 11 | |
| 12 | This is a heuristic tool, NOT a proof engine. False positives are expected. |
| 13 | The goal is to flag passages that FEEL AI-generated so a human editor can |
| 14 | inject voice, variance, and specificity. |
| 15 | |
| 16 | Usage: |
| 17 | python ai_content_detector.py article.md |
| 18 | python ai_content_detector.py article.md --json |
| 19 | python ai_content_detector.py --demo |
| 20 | |
| 21 | Scoring: |
| 22 | 0-20 = likely human (high burstiness, diverse vocab, no AI phrases) |
| 23 | 21-50 = mixed signals (review flagged passages) |
| 24 | 51-100 = likely AI (flat burstiness, repetitive vocab, AI phrase density) |
| 25 | """ |
| 26 | from __future__ import annotations |
| 27 | import argparse |
| 28 | import json |
| 29 | import math |
| 30 | import re |
| 31 | import sys |
| 32 | from collections import Counter |
| 33 | from pathlib import Path |
| 34 | |
| 35 | # --- Known AI phrases (commonly overrepresented in LLM output) --- |
| 36 | |
| 37 | AI_PHRASES = [ |
| 38 | "in today's digital landscape", |
| 39 | "in today's fast-paced", |
| 40 | "it's worth noting that", |
| 41 | "it is important to note", |
| 42 | "delve into", |
| 43 | "dive deep into", |
| 44 | "leverage", |
| 45 | "game-changer", |
| 46 | "game changer", |
| 47 | "unlock the potential", |
| 48 | "unlock the power", |
| 49 | "harness the power", |
| 50 | "at the end of the day", |
| 51 | "in conclusion", |
| 52 | "in summary", |
| 53 | "navigating the complexities", |
| 54 | "a comprehensive guide", |
| 55 | "seamlessly integrate", |
| 56 | "robust solution", |
| 57 | "cutting-edge", |
| 58 | "state-of-the-art", |
| 59 | "empower you to", |
| 60 | "take your .* to the next level", |
| 61 | "in the realm of", |
| 62 | "tapestry of", |
| 63 | "multifaceted", |
| 64 | "it's crucial to", |
| 65 | "paramount", |
| 66 | "foster a .* environment", |
| 67 | "elevate your", |
| 68 | ] |
| 69 | |
| 70 | AI_PHRASE_RES = [re.compile(p, re.IGNORECASE) for p in AI_PHRASES] |
| 71 | |
| 72 | # --- Sentence splitting --- |
| 73 | |
| 74 | SENTENCE_RE = re.compile(r"[^.!?]+[.!?]+", re.DOTALL) |
| 75 | WORD_RE = re.compile(r"[a-zA-Z]+") |
| 76 | |
| 77 | DEMO_CONTENT = """In today's digital landscape, leveraging AI tools has become a game-changer for content creators. It's worth noting that the ability to harness the power of large language models can unlock the potential of your marketing efforts. This comprehensive guide will delve into the multifaceted world of AI-assisted content creation. |
| 78 | |
| 79 | The integration of AI into content workflows is a robust solution that seamlessly integrates with existing processes. By navigating the complexities of modern content production, you can elevate your brand's voice and foster a creative environment that empowers your team to take their content to the next level. |
| 80 | |
| 81 | At the end of the day, it's crucial to understand that AI is a tool, not a replacement. The cutting-edge capabilities of state-of-the-art models are paramount for staying competitive in the realm of digital marketing. In conclusion, the tapestry of modern content creation requires both human creativity and artificial intelligence working in harmony.""" |
| 82 | |
| 83 | |
| 84 | def extract_sentences(text): |
| 85 | body = re.sub(r"^---.*?---\s*", "", text, count=1, flags=re.DOTALL) |
| 86 | body = re.sub(r"^#+\s+.*$", "", body, flags=re.MULTILINE) |
| 87 | body = re.sub(r"```.*?```", "", body, flags=re.DOTALL) |
| 88 | body = re.sub(r"`[^`]+`", "", body) |
| 89 | body = re.sub(r"\[([^\]]+)\]\([^)]+\)", r"\1", body) |
| 90 | sentences = SENTENCE_RE.findall(body) |
| 91 | return [s.strip() for s in sentences if len(s.strip().split()) >= 3] |
| 92 | |
| 93 | |
| 94 | def burstiness_score(sentences): |
| 95 | """Compute burstiness (sentence length variance). High = human, low = AI.""" |
| 96 | if len(sentences) < 5: |
| 97 | return {"score": 50, "mean": 0, "std": 0, "cv": 0, "note": "too few sentences"} |
| 98 | lengths = [len(s.split()) for s in sentences] |
| 99 | mean = sum(lengths) / len(lengths) |
| 100 | variance = sum((l - mean) ** 2 for l in lengths) / len(lengths) |
| 101 | std = math.sqrt(variance) |
| 102 | cv = std / mean if mean > 0 else 0 # coefficient of variation |
| 103 | |
| 104 | # Human writing: CV typically 0.4-0.8+ (high variance) |
| 105 | # AI writing: CV typically 0.15-0.35 (consistently medium) |
| 106 | if cv >= 0.5: |
| 107 | ai_prob = max(0, 30 - (cv - 0.5) * 60) |
| 108 | elif cv >= 0.35: |
| 109 | ai_prob = 30 + (0.5 - cv) * 130 |
| 110 | else: |
| 111 | ai_prob = 50 + (0.35 - cv) * 250 |
| 112 | |
| 113 | ai_prob = max(0, min(100, ai_prob)) |
| 114 | return { |
| 115 | "score": round(ai_prob, 1), |
| 116 | "mean_sentence_length": round(mean, 1), |
| 117 | "std_sentence_length": round(std, 1), |
| 118 | "coefficient_of_variation": round(cv, 3), |
| 119 | "note": "low CV = flat sentence lengths (AI-like)" if cv < 0.35 else "healthy variance", |
| 120 | } |
| 121 | |
| 122 | |
| 123 | def vocabulary_diversity(text): |
| 124 | """Type-Token Ratio. Low TTR = repetitive vocabulary (AI-like).""" |
| 125 | body = re.sub(r"^---.*?---\s*", "", text, count=1, flags=re.DOTALL) |
| 126 | words = [w.lower() for w in WORD_RE.findall(body) if len(w) > 2] |
| 127 | if len(words) < 50: |
| 128 | return {"score": 50, "ttr": 0, "unique": 0, "total": len(words), "note": "too few words"} |
| 129 | # Use a sliding window TTR for length-independence |
| 130 | window = min(200, len(words)) |
| 131 | ttrs = [] |
| 132 | for i in range(0, len(words) - window + 1, window // 2): |
| 133 | chunk = words[i : i + window] |
| 134 | ttrs.append(len(set(chunk)) / len(chunk)) |
| 135 | avg_ttr = sum(ttrs) / len(ttrs) |
| 136 | |
| 137 | # Human: TTR 0.55-0.75+ (varied vocabulary) |
| 138 | # AI: TTR 0.35-0.50 (repetitive patterns) |
| 139 | if avg_ttr >= 0.60: |
| 140 | ai_prob = max(0, 25 - (avg_ttr - 0.60) * 150) |
| 141 | elif avg_ttr >= 0.45: |
| 142 | ai_prob = 25 + (0.60 - avg_ttr) * 330 |
| 143 | else: |
| 144 | ai_prob = 75 + (0.45 - avg_ttr) * 250 |
| 145 | |
| 146 | ai_prob = max(0, min(100, ai_prob)) |
| 147 | |
| 148 | # Top repeated words (excluding stop words) |
| 149 | stop = {"the", "and", "for", "that", "this", "with", "from", "are", "was", "were", "been", |
| 150 | "have", "has", "had", "not", "but", "can", "will", "your", "you", "they", "their", |
| 151 | "more", "than", "also", "into", "when", "how", "what", "which", "about", "each"} |
| 152 | content_words = [w for w in words if w not in stop] |
| 153 | top_repeated = Counter(content_words).most_common(5) |
| 154 | |
| 155 | return { |
| 156 | "score": round(ai_prob, 1), |
| 157 | "avg_ttr": round(avg_ttr, 3), |
| 158 | "unique_words": len(set(words)), |
| 159 | "total_words": len(words), |
| 160 | "top_repeated": [{"word": w, "count": c} for w, c in top_repeated], |
| 161 | "note": "low TTR = repetitive vocabulary (AI-like)" if avg_ttr < 0.45 else "healthy diversity", |
| 162 | } |
| 163 | |
| 164 | |
| 165 | def phrase_detection(text): |
| 166 | """Count known AI phrases.""" |
| 167 | found = [] |
| 168 | text_lower = text.lower() |
| 169 | for i, pattern in enumerate(AI_PHRASE_RES): |
| 170 | matches = pattern.findall(text_lower) |
| 171 | if matches: |
| 172 | found.append({"phrase": AI_PHRASES[i], "count": len(matches)}) |
| 173 | |
| 174 | total_matches = sum(f["count"] for f in found) |
| 175 | word_count = len(text.split()) |
| 176 | density = (total_matches / (word_count / 1000)) if word_count > 0 else 0 |
| 177 | |
| 178 | # 0-2 per 1K words = normal; 3-5 = suspect; 6+ = likely AI |
| 179 | if density <= 2: |
| 180 | ai_prob = density * 15 |
| 181 | elif density <= 5: |
| 182 | ai_prob = 30 + (density - 2) * 20 |
| 183 | else: |
| 184 | ai_prob = min(100, 90 + (density - 5) * 5) |
| 185 | |
| 186 | return { |
| 187 | "score": round(ai_prob, 1), |
| 188 | "phrases_found": len(found), |
| 189 | "total_matches": total_matches, |
| 190 | "density_per_1k_words": round(density, 1), |
| 191 | "matches": found[:10], |
| 192 | "note": f"{density:.1f} AI phrases per 1K words" if found else "no known AI phrases", |
| 193 | } |
| 194 | |
| 195 | |
| 196 | def analyze(text): |
| 197 | sentences = extract_sentences(text) |
| 198 | burst = burstiness_score(sentences) |
| 199 | vocab = vocabulary_diversity(text) |
| 200 | phrases = phrase_detection(text) |
| 201 | |
| 202 | # Weighted composite: burstiness 35%, vocab 30%, phrases 35% |
| 203 | composite = burst["score"] * 0.35 + vocab["score"] * 0.30 + phrases["score"] * 0.35 |
| 204 | composite = round(min(100, max(0, composite)), 1) |
| 205 | |
| 206 | if composite <= 20: |
| 207 | verdict = "LIKELY_HUMAN" |
| 208 | elif composite <= 50: |
| 209 | verdict = "MIXED" |
| 210 | else: |
| 211 | verdict = "LIKELY_AI" |
| 212 | |
| 213 | return { |
| 214 | "status": "ok", |
| 215 | "composite_score": composite, |
| 216 | "verdict": verdict, |
| 217 | "burstiness": burst, |
| 218 | "vocabulary": vocab, |
| 219 | "phrases": phrases, |
| 220 | "sentences_analyzed": len(sentences), |
| 221 | "recommendations": _recommendations(burst, vocab, phrases), |
| 222 | } |
| 223 | |
| 224 | |
| 225 | def _recommendations(burst, vocab, phrases): |
| 226 | recs = [] |
| 227 | if burst["score"] > 40: |
| 228 | recs.append("Vary sentence lengths: mix short punchy sentences (5-8 words) with longer explanatory ones (20-30 words).") |
| 229 | if vocab["score"] > 40: |
| 230 | recs.append("Diversify vocabulary: replace repeated words with synonyms. Use domain-specific jargon where appropriate.") |
| 231 | if phrases["total_matches"] > 0: |
| 232 | top = phrases["matches"][:3] |
| 233 | recs.append(f"Remove/replace AI phrases: {', '.join(p['phrase'] for p in top)}") |
| 234 | if not recs: |
| 235 | recs.append("Content reads naturally. No humanization needed.") |
| 236 | return recs |
| 237 | |
| 238 | |
| 239 | def main(): |
| 240 | p = argparse.ArgumentParser( |
| 241 | description="Detect AI-generated content via burstiness, vocabulary diversity, and phrase analysis.", |
| 242 | epilog="Score 0-20 = likely human, 21-50 = mixed, 51-100 = likely AI. Run with --demo.", |
| 243 | ) |
| 244 | p.add_argument("file", nargs="?", help="Markdown/text file to analyze") |
| 245 | p.add_argument("--json", action="store_true", help="JSON output") |
| 246 | p.add_argument("--demo", action="store_true", help="Run with AI-heavy demo text") |
| 247 | args = p.parse_args() |
| 248 | |
| 249 | if args.demo: |
| 250 | text = DEMO_CONTENT |
| 251 | elif args.file: |
| 252 | path = Path(args.file) |
| 253 | if not path.exists(): |
| 254 | print(f"[error] {path} not found", file=sys.stderr) |
| 255 | sys.exit(1) |
| 256 | text = path.read_text(encoding="utf-8", errors="replace") |
| 257 | else: |
| 258 | p.print_help() |
| 259 | sys.exit(0) |
| 260 | |
| 261 | result = analyze(text) |
| 262 | |
| 263 | if args.json: |
| 264 | print(json.dumps(result, indent=2)) |
| 265 | return |
| 266 | |
| 267 | print(f"AI Content Detection — Composite: {result['composite_score']}/100 ({result['verdict']})") |
| 268 | print() |
| 269 | b = result["burstiness"] |
| 270 | print(f" Burstiness: {b['score']}/100 — CV={b['coefficient_of_variation']} ({b['note']})") |
| 271 | v = result["vocabulary"] |
| 272 | print(f" Vocabulary: {v['score']}/100 — TTR={v['avg_ttr']} ({v['note']})") |
| 273 | ph = result["phrases"] |
| 274 | print(f" AI Phrases: {ph['score']}/100 — {ph['total_matches']} matches, {ph['density_per_1k_words']}/1K words") |
| 275 | if ph["matches"]: |
| 276 | for m in ph["matches"][:5]: |
| 277 | print(f" → \"{m['phrase']}\" (×{m['count']})") |
| 278 | print() |
| 279 | print("Recommendations:") |
| 280 | for r in result["recommendations"]: |
| 281 | print(f" → {r}") |
| 282 | |
| 283 | |
| 284 | if __name__ == "__main__": |
| 285 | main() |
文件内容是平台检查时读出的文本，只供查看。平台不会运行包里的任何脚本或程序。
