scripts/extract_citations.py
9.8 KB| 1 | #!/usr/bin/env python3 |
| 2 | """ |
| 3 | research-summarizer: Citation Extractor |
| 4 | |
| 5 | Extract and format citations from text documents. Detects DOIs, URLs, |
| 6 | author-year patterns, and numbered references. Outputs in APA, IEEE, |
| 7 | Chicago, Harvard, or MLA format. |
| 8 | |
| 9 | Usage: |
| 10 | python scripts/extract_citations.py document.txt |
| 11 | python scripts/extract_citations.py document.txt --format ieee |
| 12 | python scripts/extract_citations.py document.txt --format apa --output json |
| 13 | python scripts/extract_citations.py --stdin < document.txt |
| 14 | """ |
| 15 | |
| 16 | import argparse |
| 17 | import json |
| 18 | import re |
| 19 | import sys |
| 20 | from collections import OrderedDict |
| 21 | |
| 22 | |
| 23 | # --- Citation Detection Patterns --- |
| 24 | |
| 25 | PATTERNS = { |
| 26 | "doi": re.compile( |
| 27 | r"(?:https?://doi\.org/|doi:\s*)(10\.\d{4,}/[^\s,;}\]]+)", re.IGNORECASE |
| 28 | ), |
| 29 | "url": re.compile( |
| 30 | r"https?://[^\s,;}\])\"'>]+", re.IGNORECASE |
| 31 | ), |
| 32 | "author_year": re.compile( |
| 33 | r"(?:^|\(|\s)([A-Z][a-z]+(?:\s(?:&|and)\s[A-Z][a-z]+)?(?:\set\sal\.?)?)\s*\((\d{4})\)", |
| 34 | ), |
| 35 | "numbered_ref": re.compile( |
| 36 | r"^\[(\d+)\]\s+(.+)$", re.MULTILINE |
| 37 | ), |
| 38 | "footnote": re.compile( |
| 39 | r"^\d+\.\s+([A-Z].+?(?:\d{4}).+)$", re.MULTILINE |
| 40 | ), |
| 41 | } |
| 42 | |
| 43 | |
| 44 | def extract_dois(text): |
| 45 | """Extract DOI references.""" |
| 46 | citations = [] |
| 47 | for match in PATTERNS["doi"].finditer(text): |
| 48 | doi = match.group(1).rstrip(".") |
| 49 | citations.append({ |
| 50 | "type": "doi", |
| 51 | "doi": doi, |
| 52 | "raw": match.group(0).strip(), |
| 53 | "url": f"https://doi.org/{doi}", |
| 54 | }) |
| 55 | return citations |
| 56 | |
| 57 | |
| 58 | def extract_urls(text): |
| 59 | """Extract URL references (excluding DOI URLs already captured).""" |
| 60 | citations = [] |
| 61 | for match in PATTERNS["url"].finditer(text): |
| 62 | url = match.group(0).rstrip(".,;)") |
| 63 | if "doi.org" in url: |
| 64 | continue |
| 65 | citations.append({ |
| 66 | "type": "url", |
| 67 | "url": url, |
| 68 | "raw": url, |
| 69 | }) |
| 70 | return citations |
| 71 | |
| 72 | |
| 73 | def extract_author_year(text): |
| 74 | """Extract author-year citations like (Smith, 2023) or Smith & Jones (2021).""" |
| 75 | citations = [] |
| 76 | for match in PATTERNS["author_year"].finditer(text): |
| 77 | author = match.group(1).strip() |
| 78 | year = match.group(2) |
| 79 | citations.append({ |
| 80 | "type": "author_year", |
| 81 | "author": author, |
| 82 | "year": year, |
| 83 | "raw": f"{author} ({year})", |
| 84 | }) |
| 85 | return citations |
| 86 | |
| 87 | |
| 88 | def extract_numbered_refs(text): |
| 89 | """Extract numbered reference list entries like [1] Author. Title...""" |
| 90 | citations = [] |
| 91 | for match in PATTERNS["numbered_ref"].finditer(text): |
| 92 | num = match.group(1) |
| 93 | content = match.group(2).strip() |
| 94 | citations.append({ |
| 95 | "type": "numbered", |
| 96 | "number": int(num), |
| 97 | "content": content, |
| 98 | "raw": f"[{num}] {content}", |
| 99 | }) |
| 100 | return citations |
| 101 | |
| 102 | |
| 103 | def deduplicate(citations): |
| 104 | """Remove duplicate citations based on raw text.""" |
| 105 | seen = OrderedDict() |
| 106 | for c in citations: |
| 107 | key = c.get("doi") or c.get("url") or c.get("raw", "") |
| 108 | key = key.lower().strip() |
| 109 | if key and key not in seen: |
| 110 | seen[key] = c |
| 111 | return list(seen.values()) |
| 112 | |
| 113 | |
| 114 | def classify_source(citation): |
| 115 | """Classify citation as primary, secondary, or tertiary.""" |
| 116 | raw = citation.get("content", citation.get("raw", "")).lower() |
| 117 | if any(kw in raw for kw in ["meta-analysis", "systematic review", "literature review", "survey of"]): |
| 118 | return "secondary" |
| 119 | if any(kw in raw for kw in ["textbook", "encyclopedia", "handbook", "dictionary"]): |
| 120 | return "tertiary" |
| 121 | return "primary" |
| 122 | |
| 123 | |
| 124 | # --- Formatting --- |
| 125 | |
| 126 | def format_apa(citation): |
| 127 | """Format citation in APA 7 style.""" |
| 128 | if citation["type"] == "doi": |
| 129 | return f"https://doi.org/{citation['doi']}" |
| 130 | if citation["type"] == "url": |
| 131 | return f"Retrieved from {citation['url']}" |
| 132 | if citation["type"] == "author_year": |
| 133 | return f"{citation['author']} ({citation['year']})." |
| 134 | if citation["type"] == "numbered": |
| 135 | return citation["content"] |
| 136 | return citation.get("raw", "") |
| 137 | |
| 138 | |
| 139 | def format_ieee(citation): |
| 140 | """Format citation in IEEE style.""" |
| 141 | if citation["type"] == "doi": |
| 142 | return f"doi: {citation['doi']}" |
| 143 | if citation["type"] == "url": |
| 144 | return f"[Online]. Available: {citation['url']}" |
| 145 | if citation["type"] == "author_year": |
| 146 | return f"{citation['author']}, {citation['year']}." |
| 147 | if citation["type"] == "numbered": |
| 148 | return f"[{citation['number']}] {citation['content']}" |
| 149 | return citation.get("raw", "") |
| 150 | |
| 151 | |
| 152 | def format_chicago(citation): |
| 153 | """Format citation in Chicago style.""" |
| 154 | if citation["type"] == "doi": |
| 155 | return f"https://doi.org/{citation['doi']}." |
| 156 | if citation["type"] == "url": |
| 157 | return f"{citation['url']}." |
| 158 | if citation["type"] == "author_year": |
| 159 | return f"{citation['author']}. {citation['year']}." |
| 160 | if citation["type"] == "numbered": |
| 161 | return citation["content"] |
| 162 | return citation.get("raw", "") |
| 163 | |
| 164 | |
| 165 | def format_harvard(citation): |
| 166 | """Format citation in Harvard style.""" |
| 167 | if citation["type"] == "doi": |
| 168 | return f"doi:{citation['doi']}" |
| 169 | if citation["type"] == "url": |
| 170 | return f"Available at: {citation['url']}" |
| 171 | if citation["type"] == "author_year": |
| 172 | return f"{citation['author']} ({citation['year']})" |
| 173 | if citation["type"] == "numbered": |
| 174 | return citation["content"] |
| 175 | return citation.get("raw", "") |
| 176 | |
| 177 | |
| 178 | def format_mla(citation): |
| 179 | """Format citation in MLA 9 style.""" |
| 180 | if citation["type"] == "doi": |
| 181 | return f"doi:{citation['doi']}." |
| 182 | if citation["type"] == "url": |
| 183 | return f"{citation['url']}." |
| 184 | if citation["type"] == "author_year": |
| 185 | return f"{citation['author']}. {citation['year']}." |
| 186 | if citation["type"] == "numbered": |
| 187 | return citation["content"] |
| 188 | return citation.get("raw", "") |
| 189 | |
| 190 | |
| 191 | FORMATTERS = { |
| 192 | "apa": format_apa, |
| 193 | "ieee": format_ieee, |
| 194 | "chicago": format_chicago, |
| 195 | "harvard": format_harvard, |
| 196 | "mla": format_mla, |
| 197 | } |
| 198 | |
| 199 | |
| 200 | # --- Demo Data --- |
| 201 | |
| 202 | DEMO_TEXT = """ |
| 203 | Recent studies in product management have shown significant shifts in methodology. |
| 204 | According to Smith & Jones (2023), agile adoption has increased by 47% since 2020. |
| 205 | Patel et al. (2022) found that cross-functional teams deliver 2.3x faster. |
| 206 | |
| 207 | Several frameworks have been proposed: |
| 208 | [1] Cagan, M. Inspired: How to Create Tech Products Customers Love. Wiley, 2018. |
| 209 | [2] Torres, T. Continuous Discovery Habits. Product Talk LLC, 2021. |
| 210 | [3] Gothelf, J. & Seiden, J. Lean UX. O'Reilly Media, 2021. doi: 10.1234/leanux.2021 |
| 211 | |
| 212 | For further reading, see https://www.svpg.com/articles/ and the meta-analysis |
| 213 | by Chen (2024) on product discovery effectiveness. |
| 214 | |
| 215 | Related work: doi: 10.1145/3544548.3581388 |
| 216 | """ |
| 217 | |
| 218 | |
| 219 | def run_extraction(text, fmt, output_mode): |
| 220 | """Run full extraction pipeline.""" |
| 221 | all_citations = [] |
| 222 | all_citations.extend(extract_dois(text)) |
| 223 | all_citations.extend(extract_author_year(text)) |
| 224 | all_citations.extend(extract_numbered_refs(text)) |
| 225 | all_citations.extend(extract_urls(text)) |
| 226 | |
| 227 | citations = deduplicate(all_citations) |
| 228 | |
| 229 | for c in citations: |
| 230 | c["classification"] = classify_source(c) |
| 231 | |
| 232 | formatter = FORMATTERS.get(fmt, format_apa) |
| 233 | |
| 234 | if output_mode == "json": |
| 235 | result = { |
| 236 | "format": fmt, |
| 237 | "total": len(citations), |
| 238 | "citations": [], |
| 239 | } |
| 240 | for i, c in enumerate(citations, 1): |
| 241 | result["citations"].append({ |
| 242 | "index": i, |
| 243 | "type": c["type"], |
| 244 | "classification": c["classification"], |
| 245 | "formatted": formatter(c), |
| 246 | "raw": c.get("raw", ""), |
| 247 | }) |
| 248 | print(json.dumps(result, indent=2)) |
| 249 | else: |
| 250 | print(f"Citations ({fmt.upper()}) — {len(citations)} found\n") |
| 251 | primary = [c for c in citations if c["classification"] == "primary"] |
| 252 | secondary = [c for c in citations if c["classification"] == "secondary"] |
| 253 | tertiary = [c for c in citations if c["classification"] == "tertiary"] |
| 254 | |
| 255 | for label, group in [("Primary Sources", primary), ("Secondary Sources", secondary), ("Tertiary Sources", tertiary)]: |
| 256 | if group: |
| 257 | print(f"### {label}") |
| 258 | for i, c in enumerate(group, 1): |
| 259 | print(f" {i}. {formatter(c)}") |
| 260 | print() |
| 261 | |
| 262 | return citations |
| 263 | |
| 264 | |
| 265 | def main(): |
| 266 | parser = argparse.ArgumentParser( |
| 267 | description="research-summarizer: Extract and format citations from text" |
| 268 | ) |
| 269 | parser.add_argument("file", nargs="?", help="Input text file (omit for demo)") |
| 270 | parser.add_argument( |
| 271 | "--format", "-f", |
| 272 | choices=["apa", "ieee", "chicago", "harvard", "mla"], |
| 273 | default="apa", |
| 274 | help="Citation format (default: apa)", |
| 275 | ) |
| 276 | parser.add_argument( |
| 277 | "--output", "-o", |
| 278 | choices=["text", "json"], |
| 279 | default="text", |
| 280 | help="Output mode (default: text)", |
| 281 | ) |
| 282 | parser.add_argument( |
| 283 | "--stdin", |
| 284 | action="store_true", |
| 285 | help="Read from stdin instead of file", |
| 286 | ) |
| 287 | args = parser.parse_args() |
| 288 | |
| 289 | if args.stdin: |
| 290 | text = sys.stdin.read() |
| 291 | elif args.file: |
| 292 | try: |
| 293 | with open(args.file, "r", encoding="utf-8") as f: |
| 294 | text = f.read() |
| 295 | except FileNotFoundError: |
| 296 | print(f"Error: File not found: {args.file}", file=sys.stderr) |
| 297 | sys.exit(1) |
| 298 | except IOError as e: |
| 299 | print(f"Error reading file: {e}", file=sys.stderr) |
| 300 | sys.exit(1) |
| 301 | else: |
| 302 | print("No input file provided. Running demo...\n") |
| 303 | text = DEMO_TEXT |
| 304 | |
| 305 | run_extraction(text, args.format, args.output) |
| 306 | |
| 307 | |
| 308 | if __name__ == "__main__": |
| 309 | main() |
文件内容是平台检查时读出的文本，只供查看。平台不会运行包里的任何脚本或程序。
