scripts/_structured.py
14.3 KB| 1 | #!/usr/bin/env python3 |
| 2 | """Bounded inspectors for JSON, NumPy containers, and HDF5 metadata.""" |
| 3 | |
| 4 | from __future__ import annotations |
| 5 | |
| 6 | import itertools |
| 7 | import math |
| 8 | from pathlib import Path |
| 9 | from typing import Any |
| 10 | |
| 11 | from _capabilities import preflight_npz |
| 12 | from _common import ( |
| 13 | MAX_JSON_BYTES, |
| 14 | CliError, |
| 15 | display_identifier, |
| 16 | load_strict_json, |
| 17 | ) |
| 18 | from _tabular import profile_json_structure |
| 19 | |
| 20 | |
| 21 | MAX_ARRAY_SAMPLE = 4096 |
| 22 | MAX_HDF5_OBJECTS = 1000 |
| 23 | MAX_HDF5_DEPTH = 16 |
| 24 | |
| 25 | |
| 26 | def inspect_json( |
| 27 | path: Path, |
| 28 | *, |
| 29 | reveal_identifiers: bool = False, |
| 30 | ) -> dict[str, Any]: |
| 31 | """Strictly parse bounded JSON and emit structure, never scalar values.""" |
| 32 | |
| 33 | document = load_strict_json(path, max_bytes=MAX_JSON_BYTES) |
| 34 | return profile_json_structure( |
| 35 | document, |
| 36 | reveal_identifiers=reveal_identifiers, |
| 37 | ) |
| 38 | |
| 39 | |
| 40 | def _dtype_report( |
| 41 | dtype: Any, |
| 42 | *, |
| 43 | reveal_identifiers: bool, |
| 44 | ) -> dict[str, Any]: |
| 45 | fields = getattr(dtype, "fields", None) |
| 46 | report: dict[str, Any] = { |
| 47 | "kind": str(dtype.kind), |
| 48 | "itemsize": int(dtype.itemsize), |
| 49 | "byteorder": str(dtype.byteorder), |
| 50 | "contains_python_objects": bool(dtype.hasobject), |
| 51 | "structured": fields is not None, |
| 52 | } |
| 53 | if fields is not None: |
| 54 | names = list(dtype.names or ()) |
| 55 | report["field_count"] = len(names) |
| 56 | report["field_ids"] = [ |
| 57 | display_identifier( |
| 58 | name, |
| 59 | kind="field", |
| 60 | reveal_identifiers=reveal_identifiers, |
| 61 | ) |
| 62 | for name in names[:128] |
| 63 | ] |
| 64 | report["field_ids_truncated"] = len(names) > 128 |
| 65 | else: |
| 66 | report["dtype"] = str(dtype) |
| 67 | return report |
| 68 | |
| 69 | |
| 70 | def _sample_array(array: Any, np: Any) -> Any: |
| 71 | size = int(array.size) |
| 72 | if size <= MAX_ARRAY_SAMPLE: |
| 73 | return np.asarray(array).reshape(-1) |
| 74 | positions = np.linspace( |
| 75 | 0, |
| 76 | size - 1, |
| 77 | num=MAX_ARRAY_SAMPLE, |
| 78 | dtype=np.int64, |
| 79 | ) |
| 80 | # ndarray.flat indexes in C order without materializing a full C-order |
| 81 | # copy of a Fortran-contiguous memory map. |
| 82 | return np.asarray(array.flat[positions]) |
| 83 | |
| 84 | |
| 85 | def _array_report( |
| 86 | array: Any, |
| 87 | np: Any, |
| 88 | *, |
| 89 | array_id: str | None, |
| 90 | reveal_identifiers: bool, |
| 91 | ) -> dict[str, Any]: |
| 92 | if bool(array.dtype.hasobject): |
| 93 | raise CliError("NumPy object arrays are not accepted because they require pickle") |
| 94 | report: dict[str, Any] = { |
| 95 | "shape": [int(value) for value in array.shape], |
| 96 | "dimension_count": int(array.ndim), |
| 97 | "element_count": int(array.size), |
| 98 | "dtype": _dtype_report( |
| 99 | array.dtype, |
| 100 | reveal_identifiers=reveal_identifiers, |
| 101 | ), |
| 102 | } |
| 103 | if array_id is not None: |
| 104 | report["array_id"] = array_id |
| 105 | if int(array.size) == 0: |
| 106 | report["numeric_sample"] = {"sample_count": 0} |
| 107 | return report |
| 108 | if np.issubdtype(array.dtype, np.bool_): |
| 109 | sample = _sample_array(array, np) |
| 110 | report["boolean_sample"] = { |
| 111 | "sample_count": int(sample.size), |
| 112 | "true_count": int(np.count_nonzero(sample)), |
| 113 | "sample_is_bounded": int(sample.size) < int(array.size), |
| 114 | } |
| 115 | return report |
| 116 | if np.issubdtype(array.dtype, np.number): |
| 117 | sample = _sample_array(array, np) |
| 118 | if np.iscomplexobj(sample): |
| 119 | finite = np.isfinite(sample.real) & np.isfinite(sample.imag) |
| 120 | magnitudes = np.abs(sample[finite]) |
| 121 | report["numeric_sample"] = { |
| 122 | "sample_count": int(sample.size), |
| 123 | "finite_count": int(np.count_nonzero(finite)), |
| 124 | "complex_values_summarized_by_magnitude": True, |
| 125 | "magnitude_mean": ( |
| 126 | float(np.mean(magnitudes)) if magnitudes.size else None |
| 127 | ), |
| 128 | "magnitude_minimum": ( |
| 129 | float(np.min(magnitudes)) if magnitudes.size else None |
| 130 | ), |
| 131 | "magnitude_maximum": ( |
| 132 | float(np.max(magnitudes)) if magnitudes.size else None |
| 133 | ), |
| 134 | "sample_is_bounded": int(sample.size) < int(array.size), |
| 135 | } |
| 136 | else: |
| 137 | finite = np.isfinite(sample) |
| 138 | finite_values = sample[finite] |
| 139 | report["numeric_sample"] = { |
| 140 | "sample_count": int(sample.size), |
| 141 | "finite_count": int(np.count_nonzero(finite)), |
| 142 | "nan_count": int(np.count_nonzero(np.isnan(sample))) |
| 143 | if np.issubdtype(sample.dtype, np.inexact) |
| 144 | else 0, |
| 145 | "infinite_count": int(np.count_nonzero(np.isinf(sample))) |
| 146 | if np.issubdtype(sample.dtype, np.inexact) |
| 147 | else 0, |
| 148 | "mean": ( |
| 149 | float(np.mean(finite_values)) if finite_values.size else None |
| 150 | ), |
| 151 | "minimum": ( |
| 152 | float(np.min(finite_values)) if finite_values.size else None |
| 153 | ), |
| 154 | "maximum": ( |
| 155 | float(np.max(finite_values)) if finite_values.size else None |
| 156 | ), |
| 157 | "sample_is_bounded": int(sample.size) < int(array.size), |
| 158 | } |
| 159 | return report |
| 160 | |
| 161 | |
| 162 | def inspect_numpy( |
| 163 | path: Path, |
| 164 | *, |
| 165 | suffix: str, |
| 166 | reveal_identifiers: bool = False, |
| 167 | ) -> dict[str, Any]: |
| 168 | """Inspect NPY/NPZ with pickle disabled and bounded decompression.""" |
| 169 | |
| 170 | try: |
| 171 | import numpy as np |
| 172 | except ImportError as exc: |
| 173 | raise CliError( |
| 174 | 'optional dependency missing; install with: uv pip install "numpy==2.5.3"' |
| 175 | ) from exc |
| 176 | if suffix == ".npy": |
| 177 | try: |
| 178 | array = np.load( |
| 179 | path, |
| 180 | mmap_mode="r", |
| 181 | allow_pickle=False, |
| 182 | max_header_size=10_000, |
| 183 | ) |
| 184 | except (OSError, ValueError, TypeError, MemoryError) as exc: |
| 185 | raise CliError( |
| 186 | "the NPY file could not be inspected safely; object arrays are rejected" |
| 187 | ) from exc |
| 188 | return { |
| 189 | "profile_type": "numpy_npy_bounded_profile", |
| 190 | "pickle_allowed": False, |
| 191 | "array": _array_report( |
| 192 | array, |
| 193 | np, |
| 194 | array_id=None, |
| 195 | reveal_identifiers=reveal_identifiers, |
| 196 | ), |
| 197 | "raw_values_emitted": False, |
| 198 | } |
| 199 | if suffix != ".npz": |
| 200 | raise CliError("the NumPy inspector received an unsupported suffix") |
| 201 | preflight = preflight_npz(path) |
| 202 | arrays: list[dict[str, Any]] = [] |
| 203 | try: |
| 204 | with np.load( |
| 205 | path, |
| 206 | allow_pickle=False, |
| 207 | max_header_size=10_000, |
| 208 | ) as archive: |
| 209 | for name in archive.files: |
| 210 | array = archive[name] |
| 211 | arrays.append( |
| 212 | _array_report( |
| 213 | array, |
| 214 | np, |
| 215 | array_id=display_identifier( |
| 216 | name, |
| 217 | kind="array", |
| 218 | reveal_identifiers=reveal_identifiers, |
| 219 | ), |
| 220 | reveal_identifiers=reveal_identifiers, |
| 221 | ) |
| 222 | ) |
| 223 | del array |
| 224 | except (OSError, ValueError, TypeError, MemoryError) as exc: |
| 225 | raise CliError( |
| 226 | "the NPZ arrays could not be inspected safely; object arrays are rejected" |
| 227 | ) from exc |
| 228 | return { |
| 229 | "profile_type": "numpy_npz_bounded_profile", |
| 230 | "pickle_allowed": False, |
| 231 | "archive_preflight": preflight, |
| 232 | "arrays": arrays, |
| 233 | "raw_values_emitted": False, |
| 234 | } |
| 235 | |
| 236 | |
| 237 | def _shape_elements(shape: Any) -> int | None: |
| 238 | if shape is None: |
| 239 | return None |
| 240 | total = 1 |
| 241 | for value in shape: |
| 242 | total *= int(value) |
| 243 | if total > 2**63 - 1: |
| 244 | return None |
| 245 | return total |
| 246 | |
| 247 | |
| 248 | def inspect_hdf5( |
| 249 | path: Path, |
| 250 | *, |
| 251 | reveal_identifiers: bool = False, |
| 252 | ) -> dict[str, Any]: |
| 253 | """Inspect HDF5 links and metadata without reading dataset values.""" |
| 254 | |
| 255 | try: |
| 256 | import h5py |
| 257 | except ImportError as exc: |
| 258 | raise CliError( |
| 259 | 'optional dependency missing; install with: uv pip install "h5py==3.16.0"' |
| 260 | ) from exc |
| 261 | objects: list[dict[str, Any]] = [] |
| 262 | link_counts = {"hard": 0, "soft_not_followed": 0, "external_not_followed": 0} |
| 263 | object_limit_reached = False |
| 264 | depth_limit_reached = False |
| 265 | seen_addresses: set[int] = set() |
| 266 | try: |
| 267 | with h5py.File(path, "r") as handle: |
| 268 | stack: list[tuple[Any, str, int]] = [(handle, "/", 0)] |
| 269 | while stack: |
| 270 | group, logical_path, depth = stack.pop() |
| 271 | if len(objects) >= MAX_HDF5_OBJECTS: |
| 272 | object_limit_reached = True |
| 273 | break |
| 274 | for name in itertools.islice( |
| 275 | group.keys(), |
| 276 | MAX_HDF5_OBJECTS - len(objects) + 1, |
| 277 | ): |
| 278 | if len(objects) >= MAX_HDF5_OBJECTS: |
| 279 | object_limit_reached = True |
| 280 | break |
| 281 | link = group.get(name, getlink=True) |
| 282 | child_path = ( |
| 283 | f"/{name}" if logical_path == "/" else f"{logical_path}/{name}" |
| 284 | ) |
| 285 | object_id = display_identifier( |
| 286 | child_path, |
| 287 | kind="hdf_object", |
| 288 | reveal_identifiers=reveal_identifiers, |
| 289 | ) |
| 290 | if isinstance(link, h5py.ExternalLink): |
| 291 | link_counts["external_not_followed"] += 1 |
| 292 | objects.append( |
| 293 | { |
| 294 | "object_id": object_id, |
| 295 | "type": "external_link", |
| 296 | "followed": False, |
| 297 | } |
| 298 | ) |
| 299 | continue |
| 300 | if isinstance(link, h5py.SoftLink): |
| 301 | link_counts["soft_not_followed"] += 1 |
| 302 | objects.append( |
| 303 | { |
| 304 | "object_id": object_id, |
| 305 | "type": "soft_link", |
| 306 | "followed": False, |
| 307 | } |
| 308 | ) |
| 309 | continue |
| 310 | if not isinstance(link, h5py.HardLink): |
| 311 | raise CliError("the HDF5 file contains an unsupported link type") |
| 312 | link_counts["hard"] += 1 |
| 313 | child = group.get(name) |
| 314 | if child is None: |
| 315 | raise CliError("the HDF5 file contains an unresolved hard link") |
| 316 | address = int(h5py.h5o.get_info(child.id).addr) |
| 317 | already_seen = address in seen_addresses |
| 318 | seen_addresses.add(address) |
| 319 | if isinstance(child, h5py.Dataset): |
| 320 | compression = child.compression |
| 321 | external_count = len(child.external or ()) |
| 322 | objects.append( |
| 323 | { |
| 324 | "object_id": object_id, |
| 325 | "type": "dataset", |
| 326 | "hard_link_alias": already_seen, |
| 327 | "shape": [int(value) for value in child.shape] |
| 328 | if child.shape is not None |
| 329 | else None, |
| 330 | "element_count": _shape_elements(child.shape), |
| 331 | "dtype": _dtype_report( |
| 332 | child.dtype, |
| 333 | reveal_identifiers=reveal_identifiers, |
| 334 | ), |
| 335 | "attribute_count": len(child.attrs), |
| 336 | "chunked": child.chunks is not None, |
| 337 | "chunk_shape": [int(value) for value in child.chunks] |
| 338 | if child.chunks is not None |
| 339 | else None, |
| 340 | "compressed": compression is not None, |
| 341 | "compression_id": display_identifier( |
| 342 | str(compression), |
| 343 | kind="compression", |
| 344 | reveal_identifiers=reveal_identifiers, |
| 345 | ) |
| 346 | if compression is not None |
| 347 | else None, |
| 348 | "external_storage_file_count": external_count, |
| 349 | "virtual_dataset": bool(child.is_virtual), |
| 350 | "values_read": False, |
| 351 | } |
| 352 | ) |
| 353 | elif isinstance(child, h5py.Group): |
| 354 | objects.append( |
| 355 | { |
| 356 | "object_id": object_id, |
| 357 | "type": "group", |
| 358 | "hard_link_alias": already_seen, |
| 359 | "attribute_count": len(child.attrs), |
| 360 | } |
| 361 | ) |
| 362 | if already_seen: |
| 363 | continue |
| 364 | if depth >= MAX_HDF5_DEPTH: |
| 365 | depth_limit_reached = True |
| 366 | else: |
| 367 | stack.append((child, child_path, depth + 1)) |
| 368 | else: |
| 369 | raise CliError("the HDF5 file contains an unsupported object type") |
| 370 | except CliError: |
| 371 | raise |
| 372 | except (OSError, RuntimeError, ValueError, TypeError, MemoryError) as exc: |
| 373 | raise CliError("the HDF5 metadata could not be inspected safely") from exc |
| 374 | return { |
| 375 | "profile_type": "hdf5_metadata_only", |
| 376 | "object_count_reported": len(objects), |
| 377 | "object_limit_reached": object_limit_reached, |
| 378 | "depth_limit_reached": depth_limit_reached, |
| 379 | "links": link_counts, |
| 380 | "objects": objects, |
| 381 | "dataset_values_read": False, |
| 382 | "attributes_values_read": False, |
| 383 | "soft_links_followed": False, |
| 384 | "external_links_followed": False, |
| 385 | "external_dataset_storage_read": False, |
| 386 | "filter_plugins_invoked_for_data": False, |
| 387 | "raw_values_emitted": False, |
| 388 | "limitations": [ |
| 389 | "Generic hierarchy inspection is not semantic validation of H5AD, Loom, or other HDF5 conventions.", |
| 390 | "Dataset payloads are intentionally not read, so value-level statistics are unavailable.", |
| 391 | ], |
| 392 | } |
文件内容是平台检查时读出的文本，只供查看。平台不会运行包里的任何脚本或程序。
