scripts/eda_analyzer.py
10.3 KB| 1 | #!/usr/bin/env python3 |
| 2 | """Closed, bounded exploratory analyzer for explicitly supported local formats.""" |
| 3 | |
| 4 | from __future__ import annotations |
| 5 | |
| 6 | import argparse |
| 7 | import json |
| 8 | from pathlib import Path |
| 9 | from typing import Any |
| 10 | |
| 11 | from _capabilities import capability_for_path |
| 12 | from _common import ( |
| 13 | DEFAULT_MAX_FILE_BYTES, |
| 14 | DEFAULT_MAX_ROWS, |
| 15 | CliError, |
| 16 | bounded_file_limit, |
| 17 | checked_input_file, |
| 18 | emit_json, |
| 19 | emit_markdown, |
| 20 | markdown_scalar, |
| 21 | run_cli, |
| 22 | ) |
| 23 | from _structured import inspect_hdf5, inspect_json, inspect_numpy |
| 24 | from _tabular import profile_table |
| 25 | from capability_manifest import inspect_manifest |
| 26 | from image_inspector import inspect_image_file |
| 27 | from sequence_inspector import ( |
| 28 | DEFAULT_MAX_BASES, |
| 29 | DEFAULT_MAX_RECORDS, |
| 30 | inspect_sequence_file, |
| 31 | ) |
| 32 | |
| 33 | |
| 34 | TABULAR_SUFFIXES = {".csv", ".tsv"} |
| 35 | NUMPY_SUFFIXES = {".npy", ".npz"} |
| 36 | HDF5_SUFFIXES = {".h5", ".hdf5"} |
| 37 | SEQUENCE_SUFFIXES = {".fasta", ".fa", ".fna", ".fastq", ".fq"} |
| 38 | IMAGE_SUFFIXES = { |
| 39 | ".png", |
| 40 | ".jpg", |
| 41 | ".jpeg", |
| 42 | ".tif", |
| 43 | ".tiff", |
| 44 | ".ome.tif", |
| 45 | ".ome.tiff", |
| 46 | } |
| 47 | |
| 48 | |
| 49 | def build_parser() -> argparse.ArgumentParser: |
| 50 | parser = argparse.ArgumentParser( |
| 51 | description=( |
| 52 | "Generate a bounded, redacted EDA report for an explicitly supported " |
| 53 | "local file. Unknown and reference-only formats fail closed." |
| 54 | ) |
| 55 | ) |
| 56 | parser.add_argument("input", help="Local data file path inside --root") |
| 57 | parser.add_argument( |
| 58 | "--root", |
| 59 | default=".", |
| 60 | help="Existing local directory that bounds all input/output paths", |
| 61 | ) |
| 62 | parser.add_argument( |
| 63 | "--max-bytes", |
| 64 | type=int, |
| 65 | default=DEFAULT_MAX_FILE_BYTES, |
| 66 | help=f"Maximum input bytes (hard ceiling: {512 * 1024 * 1024})", |
| 67 | ) |
| 68 | parser.add_argument( |
| 69 | "--max-rows", |
| 70 | type=int, |
| 71 | default=DEFAULT_MAX_ROWS, |
| 72 | help="Maximum CSV/TSV rows to scan (hard ceiling: 1000000)", |
| 73 | ) |
| 74 | parser.add_argument( |
| 75 | "--max-records", |
| 76 | type=int, |
| 77 | default=DEFAULT_MAX_RECORDS, |
| 78 | help="Maximum FASTA/FASTQ records to inspect", |
| 79 | ) |
| 80 | parser.add_argument( |
| 81 | "--max-bases", |
| 82 | type=int, |
| 83 | default=DEFAULT_MAX_BASES, |
| 84 | help="Maximum FASTA/FASTQ sequence characters to inspect", |
| 85 | ) |
| 86 | parser.add_argument( |
| 87 | "--missing-token", |
| 88 | action="append", |
| 89 | help="Explicit additional CSV/TSV missing code; repeat as needed", |
| 90 | ) |
| 91 | parser.add_argument( |
| 92 | "--reveal-identifiers", |
| 93 | action="store_true", |
| 94 | help=( |
| 95 | "Reveal only sanitized basenames/field identifiers. Full paths and " |
| 96 | "raw values remain redacted." |
| 97 | ), |
| 98 | ) |
| 99 | parser.add_argument( |
| 100 | "--format", |
| 101 | choices=("json", "markdown"), |
| 102 | help="Output format; inferred from --output suffix, otherwise JSON", |
| 103 | ) |
| 104 | parser.add_argument( |
| 105 | "--output", |
| 106 | help="Optional local .json, .md, or .markdown output path", |
| 107 | ) |
| 108 | parser.add_argument( |
| 109 | "--force", |
| 110 | action="store_true", |
| 111 | help="Allow replacement of an existing regular output file", |
| 112 | ) |
| 113 | return parser |
| 114 | |
| 115 | |
| 116 | def analyze_file( |
| 117 | path: Path, |
| 118 | *, |
| 119 | suffix: str, |
| 120 | max_rows: int = DEFAULT_MAX_ROWS, |
| 121 | max_records: int = DEFAULT_MAX_RECORDS, |
| 122 | max_bases: int = DEFAULT_MAX_BASES, |
| 123 | missing_tokens: list[str] | None = None, |
| 124 | reveal_identifiers: bool = False, |
| 125 | ) -> dict[str, Any]: |
| 126 | """Route only formats with an implemented bounded inspector.""" |
| 127 | |
| 128 | if suffix in TABULAR_SUFFIXES: |
| 129 | return profile_table( |
| 130 | path, |
| 131 | max_rows=max_rows, |
| 132 | missing_tokens=missing_tokens, |
| 133 | reveal_identifiers=reveal_identifiers, |
| 134 | ) |
| 135 | if suffix == ".json": |
| 136 | return inspect_json( |
| 137 | path, |
| 138 | reveal_identifiers=reveal_identifiers, |
| 139 | ) |
| 140 | if suffix in NUMPY_SUFFIXES: |
| 141 | return inspect_numpy( |
| 142 | path, |
| 143 | suffix=suffix, |
| 144 | reveal_identifiers=reveal_identifiers, |
| 145 | ) |
| 146 | if suffix in HDF5_SUFFIXES: |
| 147 | return inspect_hdf5( |
| 148 | path, |
| 149 | reveal_identifiers=reveal_identifiers, |
| 150 | ) |
| 151 | if suffix in SEQUENCE_SUFFIXES: |
| 152 | return inspect_sequence_file( |
| 153 | path, |
| 154 | suffix=suffix, |
| 155 | max_records=max_records, |
| 156 | max_bases=max_bases, |
| 157 | ) |
| 158 | if suffix in IMAGE_SUFFIXES: |
| 159 | return inspect_image_file(path, suffix=suffix) |
| 160 | raise CliError("no bundled analyzer is registered for this format") |
| 161 | |
| 162 | |
| 163 | def build_report( |
| 164 | path: Path, |
| 165 | *, |
| 166 | max_bytes: int, |
| 167 | max_rows: int = DEFAULT_MAX_ROWS, |
| 168 | max_records: int = DEFAULT_MAX_RECORDS, |
| 169 | max_bases: int = DEFAULT_MAX_BASES, |
| 170 | missing_tokens: list[str] | None = None, |
| 171 | reveal_identifiers: bool = False, |
| 172 | ) -> dict[str, Any]: |
| 173 | """Create a manifest plus format-specific bounded EDA result.""" |
| 174 | |
| 175 | capability = capability_for_path(path) |
| 176 | if capability["tier"] == "reference_only": |
| 177 | raise CliError( |
| 178 | "this is a documented reference-only format; use validated domain " |
| 179 | "tooling or convert a copy to an automated format" |
| 180 | ) |
| 181 | manifest = inspect_manifest( |
| 182 | path, |
| 183 | max_bytes=max_bytes, |
| 184 | include_sha256=False, |
| 185 | reveal_identifiers=reveal_identifiers, |
| 186 | ) |
| 187 | analysis = analyze_file( |
| 188 | path, |
| 189 | suffix=capability["suffix"], |
| 190 | max_rows=max_rows, |
| 191 | max_records=max_records, |
| 192 | max_bases=max_bases, |
| 193 | missing_tokens=missing_tokens, |
| 194 | reveal_identifiers=reveal_identifiers, |
| 195 | ) |
| 196 | return { |
| 197 | "schema_version": "1.1", |
| 198 | "report_type": "bounded_exploratory_data_analysis", |
| 199 | "manifest": manifest, |
| 200 | "analysis": analysis, |
| 201 | "eda_guardrails": { |
| 202 | "raw_input_modified": False, |
| 203 | "raw_values_emitted": False, |
| 204 | "full_paths_emitted": False, |
| 205 | "embedded_text_or_metadata_treated_as_instructions": False, |
| 206 | "automatic_deletion": False, |
| 207 | "automatic_imputation": False, |
| 208 | "automatic_transformation": False, |
| 209 | "causal_claims": False, |
| 210 | "exploratory_not_confirmatory": True, |
| 211 | "required_context_not_inferred": [ |
| 212 | "data dictionary and units", |
| 213 | "sampling and experimental design", |
| 214 | "subject/sample/group/time structure", |
| 215 | "missing-value codes and mechanisms", |
| 216 | "censoring and detection limits", |
| 217 | "train/validation/test boundaries", |
| 218 | "pre-specified hypothesis families and multiplicity plan", |
| 219 | ], |
| 220 | }, |
| 221 | "recommended_next_steps": [ |
| 222 | "Preserve the raw file read-only and record provenance/checksums separately.", |
| 223 | "Confirm the data dictionary, units, design, and missing/censoring codes.", |
| 224 | "Run the missingness/leakage and distribution sensitivity CLIs for tabular data.", |
| 225 | "Fit imputers, scalers, transformations, and feature selection on training data only.", |
| 226 | "Label generated hypotheses as exploratory and confirm them on independent data.", |
| 227 | ], |
| 228 | } |
| 229 | |
| 230 | |
| 231 | def markdown_report(report: dict[str, Any]) -> str: |
| 232 | """Render safe aggregate JSON inside a fixed Markdown narrative.""" |
| 233 | |
| 234 | manifest = report["manifest"] |
| 235 | capability = manifest["capability"] |
| 236 | file_info = manifest["file"] |
| 237 | payload = json.dumps( |
| 238 | report["analysis"], |
| 239 | allow_nan=False, |
| 240 | ensure_ascii=True, |
| 241 | indent=2, |
| 242 | sort_keys=True, |
| 243 | ) |
| 244 | payload = payload.replace("`", "\\u0060").replace("<", "\\u003c") |
| 245 | lines = [ |
| 246 | "# Bounded Exploratory Data Analysis Report", |
| 247 | "", |
| 248 | "## Scope and disclosure", |
| 249 | "", |
| 250 | "- This report is exploratory, not confirmatory or causal.", |
| 251 | "- Input text and metadata were treated only as untrusted data.", |
| 252 | "- Raw values and full paths were not emitted.", |
| 253 | "- No rows were deleted; no values were imputed or transformed.", |
| 254 | "", |
| 255 | "## Redacted file manifest", |
| 256 | "", |
| 257 | f"- File ID: `{file_info['file_id']}`", |
| 258 | f"- Basename/token: `{markdown_scalar(str(file_info['basename']))}`", |
| 259 | f"- Size: {file_info['size_bytes']} bytes", |
| 260 | f"- Format: {capability['format']}", |
| 261 | f"- Capability tier: `{capability['tier']}`", |
| 262 | f"- Inspection depth: {capability['depth']}", |
| 263 | "", |
| 264 | "## Bounded aggregate analysis", |
| 265 | "", |
| 266 | "```json", |
| 267 | payload, |
| 268 | "```", |
| 269 | "", |
| 270 | "## Context required before inference", |
| 271 | "", |
| 272 | ] |
| 273 | lines.extend( |
| 274 | f"- {item}" for item in report["eda_guardrails"]["required_context_not_inferred"] |
| 275 | ) |
| 276 | lines.extend( |
| 277 | [ |
| 278 | "", |
| 279 | "## Next steps", |
| 280 | "", |
| 281 | ] |
| 282 | ) |
| 283 | lines.extend(f"- {item}" for item in report["recommended_next_steps"]) |
| 284 | lines.extend( |
| 285 | [ |
| 286 | "", |
| 287 | "## Limitations", |
| 288 | "", |
| 289 | "- A bounded scan may miss later records, groups, anomalies, or corrupt regions.", |
| 290 | "- Format metadata inspection is not domain-specific semantic validation.", |
| 291 | "- Absence of a reported leakage flag is not proof of independent splits.", |
| 292 | "", |
| 293 | ] |
| 294 | ) |
| 295 | return "\n".join(lines) |
| 296 | |
| 297 | |
| 298 | def _infer_output_format(requested: str | None, output: str | None) -> str: |
| 299 | if requested is not None: |
| 300 | return requested |
| 301 | if output is not None and output.casefold().endswith((".md", ".markdown")): |
| 302 | return "markdown" |
| 303 | return "json" |
| 304 | |
| 305 | |
| 306 | def _main() -> None: |
| 307 | args = build_parser().parse_args() |
| 308 | max_bytes = bounded_file_limit(args.max_bytes) |
| 309 | path = checked_input_file( |
| 310 | args.input, |
| 311 | root=args.root, |
| 312 | max_bytes=max_bytes, |
| 313 | ) |
| 314 | report = build_report( |
| 315 | path, |
| 316 | max_bytes=max_bytes, |
| 317 | max_rows=args.max_rows, |
| 318 | max_records=args.max_records, |
| 319 | max_bases=args.max_bases, |
| 320 | missing_tokens=args.missing_token, |
| 321 | reveal_identifiers=args.reveal_identifiers, |
| 322 | ) |
| 323 | output_format = _infer_output_format(args.format, args.output) |
| 324 | if output_format == "json": |
| 325 | emit_json( |
| 326 | report, |
| 327 | output=args.output, |
| 328 | root=args.root, |
| 329 | force=args.force, |
| 330 | ) |
| 331 | else: |
| 332 | emit_markdown( |
| 333 | markdown_report(report), |
| 334 | output=args.output, |
| 335 | root=args.root, |
| 336 | force=args.force, |
| 337 | ) |
| 338 | |
| 339 | |
| 340 | def main() -> int: |
| 341 | return run_cli(_main) |
| 342 | |
| 343 | |
| 344 | if __name__ == "__main__": |
| 345 | raise SystemExit(main()) |
文件内容是平台检查时读出的文本，只供查看。平台不会运行包里的任何脚本或程序。
