feat(conformance): in-repo, reproducible conformance sweep
conformance/run.sh fetches eight public HDF5 corpora pinned by commit (conformance/corpus.txt) into a gitignored cache, reads every file with clawhdf5 (conformance/probe, a crate outside the workspace) and with h5py/libhdf5 (ref.py), and the CVE files with h5dump, each under a timeout and an address-space limit; compare.py classifies the files, report.py writes CONFORMANCE.md and check.py gates on panics/hangs/crashes/OOM and on regressions against conformance/baseline.json. ~25 s once cached. Changes from the ad-hoc audit harness: - the probe compares non-IEEE-layout floats (N-Bit) and integers with a bit offset or reduced precision as the values libhdf5 converts them to, not raw file bytes: 8 files that showed as mismatches now read identically; - ref.py exits without tearing down h5py objects: libhdf5 2.0 aborts while freeing them for two files about half the time, which flipped them between ok and h5py-cannot-read from run to run; - the file list is defined (list_files.py): netCDF classic files are left out, 11 HDF5 files the ad-hoc sweep missed are in. Co-Authored-By: Claude Opus 5.5 (1M context) <[email protected]>
This commit is contained in:
osobh
Generated
+458
@@ -0,0 +1,458 @@
|
||||
# This file is automatically @generated by Cargo.
|
||||
# It is not intended for manual editing.
|
||||
version = 4
|
||||
|
||||
[[package]]
|
||||
name = "adler2"
|
||||
version = "2.0.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa"
|
||||
|
||||
[[package]]
|
||||
name = "better_io"
|
||||
version = "0.2.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ef0a3155e943e341e557863e69a708999c94ede624e37865c8e2a91b94efa78f"
|
||||
|
||||
[[package]]
|
||||
name = "block-buffer"
|
||||
version = "0.10.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71"
|
||||
dependencies = [
|
||||
"generic-array",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "byteorder"
|
||||
version = "1.5.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b"
|
||||
|
||||
[[package]]
|
||||
name = "cc"
|
||||
version = "1.5.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f360145194ee8e21db5ee7f3fcd4fe52210864c75c985dae33218202c8bbe040"
|
||||
dependencies = [
|
||||
"find-msvc-tools",
|
||||
"jobserver",
|
||||
"libc",
|
||||
"shlex",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "cfg-if"
|
||||
version = "1.0.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "4e7648175b45a9a48536d676f68d918270699102aa8dab5496df06904c914600"
|
||||
|
||||
[[package]]
|
||||
name = "clawhdf5-format"
|
||||
version = "2.7.0"
|
||||
dependencies = [
|
||||
"byteorder",
|
||||
"flate2",
|
||||
"libaec-sys",
|
||||
"lz4_flex",
|
||||
"pco",
|
||||
"portable-atomic",
|
||||
"sha2",
|
||||
"zstd",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "conformance-probe"
|
||||
version = "0.1.0"
|
||||
dependencies = [
|
||||
"clawhdf5-format",
|
||||
"serde_json",
|
||||
"sha2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "cpufeatures"
|
||||
version = "0.2.17"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "59ed5838eebb26a2bb2e58f6d5b5316989ae9d08bab10e0e6d103e656d1b0280"
|
||||
dependencies = [
|
||||
"libc",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "crc32fast"
|
||||
version = "1.5.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "01a7799fd6b852db0e61728dde9a204c423b44d689dbd432522543614b490e78"
|
||||
dependencies = [
|
||||
"cfg-if",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "crunchy"
|
||||
version = "0.2.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "460fbee9c2c2f33933d720630a6a0bac33ba7053db5344fac858d4b8952d77d5"
|
||||
|
||||
[[package]]
|
||||
name = "crypto-common"
|
||||
version = "0.1.7"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "78c8292055d1c1df0cce5d180393dc8cce0abec0a7102adb6c7b1eef6016d60a"
|
||||
dependencies = [
|
||||
"generic-array",
|
||||
"typenum",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "digest"
|
||||
version = "0.10.7"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292"
|
||||
dependencies = [
|
||||
"block-buffer",
|
||||
"crypto-common",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "dtype_dispatch"
|
||||
version = "0.2.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ab23e69df104e2fd85ee63a533a22d2132ef5975dc6b36f9f3e5a7305e4a8ed7"
|
||||
|
||||
[[package]]
|
||||
name = "find-msvc-tools"
|
||||
version = "0.1.14"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "aedcfb3409746eddb02b9e19ebda1c3394f759a152e48ee875a0844d1b955484"
|
||||
|
||||
[[package]]
|
||||
name = "flate2"
|
||||
version = "1.1.10"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "6e634e2e0ebac1ee034020da1ca582e17ffe4e0f5e985823721e168928136dcb"
|
||||
dependencies = [
|
||||
"crc32fast",
|
||||
"miniz_oxide",
|
||||
"zlib-rs",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "generic-array"
|
||||
version = "0.14.7"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a"
|
||||
dependencies = [
|
||||
"typenum",
|
||||
"version_check",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "getrandom"
|
||||
version = "0.4.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "300e883d756b2e4ec94e02791f39b04b522276138852cfc41d9fb7e904106099"
|
||||
dependencies = [
|
||||
"cfg-if",
|
||||
"libc",
|
||||
"r-efi",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "half"
|
||||
version = "2.7.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "6ea2d84b969582b4b1864a92dc5d27cd2b77b622a8d79306834f1be5ba20d84b"
|
||||
dependencies = [
|
||||
"cfg-if",
|
||||
"crunchy",
|
||||
"zerocopy",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "itoa"
|
||||
version = "1.0.18"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682"
|
||||
|
||||
[[package]]
|
||||
name = "jobserver"
|
||||
version = "0.1.35"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "1c00acbd29eabad4a2392fa0e921c874934dbbf4194312ad20f04a0ed67a3cb3"
|
||||
dependencies = [
|
||||
"getrandom",
|
||||
"libc",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "libaec-sys"
|
||||
version = "0.1.0"
|
||||
dependencies = [
|
||||
"pkg-config",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "libc"
|
||||
version = "0.2.189"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "3eaf3ede3fee6db1a4c2ee091bf8a8b4dccdc6d17f656fb07896ee72867612f2"
|
||||
|
||||
[[package]]
|
||||
name = "lz4_flex"
|
||||
version = "0.11.6"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "373f5eceeeab7925e0c1098212f2fbc4d416adec9d35051a6ab251e824c1854a"
|
||||
dependencies = [
|
||||
"twox-hash",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "memchr"
|
||||
version = "2.8.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98"
|
||||
|
||||
[[package]]
|
||||
name = "miniz_oxide"
|
||||
version = "0.9.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b63fbc4a50860e98e7b2aa7804ded1db5cbc3aff9193adaff57a6931bf7c4b4c"
|
||||
dependencies = [
|
||||
"adler2",
|
||||
"simd-adler32",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "pco"
|
||||
version = "1.0.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "386342cad4c6e97f081568e5d910ea7d871314c843aa8fc564f2a6b64cab9456"
|
||||
dependencies = [
|
||||
"better_io",
|
||||
"dtype_dispatch",
|
||||
"half",
|
||||
"rand_xoshiro",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "pkg-config"
|
||||
version = "0.3.34"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f6b464fbc74e149a392436b17d523f769e057cb6877f6a5c4618bc6f11800548"
|
||||
|
||||
[[package]]
|
||||
name = "portable-atomic"
|
||||
version = "1.15.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "05c8b63e8d9609db387f0324918f81d68fe27748f084ef092fb35954d0539a85"
|
||||
|
||||
[[package]]
|
||||
name = "proc-macro2"
|
||||
version = "1.0.107"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "985e7ec9bb745e6ce6535b544d84d6cd6f7ad8bd711c398938ae983b91a766d9"
|
||||
dependencies = [
|
||||
"unicode-ident",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "quote"
|
||||
version = "1.0.47"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "1fbf4db142a473a8d80c26bbf18454ed458bf8d26c8219c331daecfdbd079001"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "r-efi"
|
||||
version = "6.0.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf"
|
||||
|
||||
[[package]]
|
||||
name = "rand_core"
|
||||
version = "0.6.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c"
|
||||
|
||||
[[package]]
|
||||
name = "rand_xoshiro"
|
||||
version = "0.6.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "6f97cdb2a36ed4183de61b2f824cc45c9f1037f28afe0a322e9fff4c108b5aaa"
|
||||
dependencies = [
|
||||
"rand_core",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "serde"
|
||||
version = "1.0.229"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "4148590afebada386688f18773da617792bf2ef03ffc1e4cbd2b1d45b023e0ba"
|
||||
dependencies = [
|
||||
"serde_core",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "serde_core"
|
||||
version = "1.0.229"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "67dca2c9c51e58a4791a4b1ed58308b39c64224d349a935ab5039aa360942a48"
|
||||
dependencies = [
|
||||
"serde_derive",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "serde_derive"
|
||||
version = "1.0.229"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "e7a5d71263a5a7d47b41f6b3f06ba276f10cc18b0931f1799f710578e2309348"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn 3.0.6",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "serde_json"
|
||||
version = "1.0.151"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "c841b55ecdae098c80dcae9cf767f6f8a0c2cdb3416bbef72181df4d0fe73f14"
|
||||
dependencies = [
|
||||
"itoa",
|
||||
"memchr",
|
||||
"serde",
|
||||
"serde_core",
|
||||
"zmij",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "sha2"
|
||||
version = "0.10.9"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283"
|
||||
dependencies = [
|
||||
"cfg-if",
|
||||
"cpufeatures",
|
||||
"digest",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "shlex"
|
||||
version = "2.0.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f8fadd59c855ef2080decdef8ff161eb6661b86933c9d82e5ba29dc602a55aba"
|
||||
|
||||
[[package]]
|
||||
name = "simd-adler32"
|
||||
version = "0.3.10"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "3a219298ac11a56ea9a6d2120044824d6f01aeb034955e7af7bc16858527deea"
|
||||
|
||||
[[package]]
|
||||
name = "syn"
|
||||
version = "2.0.119"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "872831b642d1a07999a962a351ed35b955ea2cfc8f3862091e2a240a84f17297"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"unicode-ident",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "syn"
|
||||
version = "3.0.6"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "8593e8e72159ed2257d083c7a454a85cbf854f37a0966d8d483aff8c8a3ebcee"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"unicode-ident",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "twox-hash"
|
||||
version = "2.1.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "5283634e518fe9e82c7b20520bb4bc209009fd16c82077c802f8111ecbb0117a"
|
||||
|
||||
[[package]]
|
||||
name = "typenum"
|
||||
version = "1.20.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b6f5e870be6c3b371b77fe0ee0bafb859fa4964b4404c27de1d380043c4dda20"
|
||||
|
||||
[[package]]
|
||||
name = "unicode-ident"
|
||||
version = "1.0.26"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d245f478577f809a851594d02313b640fb437e0bb33866753cff937863096954"
|
||||
|
||||
[[package]]
|
||||
name = "version_check"
|
||||
version = "0.9.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a"
|
||||
|
||||
[[package]]
|
||||
name = "zerocopy"
|
||||
version = "0.8.59"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "6df92bf3d9227be3d53173901ddbffac2babc27ae50f397776ffd6dc33f800cb"
|
||||
dependencies = [
|
||||
"zerocopy-derive",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "zerocopy-derive"
|
||||
version = "0.8.59"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ac4f328cf2f05d084e496c3e9c3f33ed0a183656a16e1fcec4d464d8373aec82"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn 2.0.119",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "zlib-rs"
|
||||
version = "0.6.8"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b268e58e7c693d7c271f93ffc4ba3b380412554231c85bf61ca7af91042a4112"
|
||||
|
||||
[[package]]
|
||||
name = "zmij"
|
||||
version = "1.0.23"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "29666d0abbfad1e3dc4dcf6144730dd3a3ab225bbbdac83319345b1b44ccfc1b"
|
||||
|
||||
[[package]]
|
||||
name = "zstd"
|
||||
version = "0.13.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "e91ee311a569c327171651566e07972200e76fcfe2242a4fa446149a3881c08a"
|
||||
dependencies = [
|
||||
"zstd-safe",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "zstd-safe"
|
||||
version = "7.3.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "64d80649ab6db9d9f6f9c80a40becd948eda4714a0a5ac8c4d157a32231c7882"
|
||||
dependencies = [
|
||||
"zstd-sys",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "zstd-sys"
|
||||
version = "2.1.0+zstd.1.5.7"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "0ef0a8027ec3ee71300ab3bcbcd0393f434aa72b91ca6d635a39941deae8eea0"
|
||||
dependencies = [
|
||||
"cc",
|
||||
"pkg-config",
|
||||
]
|
||||
@@ -0,0 +1,25 @@
|
||||
[package]
|
||||
name = "conformance-probe"
|
||||
version = "0.1.0"
|
||||
edition = "2024"
|
||||
rust-version = "1.92"
|
||||
publish = false
|
||||
description = "Walks an HDF5 file with clawhdf5-format and prints a canonical JSON description (see conformance/README.md)"
|
||||
|
||||
# Deliberately outside the main workspace: `cargo test --workspace` never
|
||||
# builds it, and it links the optional C codecs (zstd, libaec) that the core
|
||||
# crates' default build must not.
|
||||
[workspace]
|
||||
|
||||
[dependencies]
|
||||
clawhdf5-format = { path = "../../crates/clawhdf5-format", features = ["lz4", "zstd", "szip", "pcodec"] }
|
||||
serde_json = "1"
|
||||
sha2 = "0.10"
|
||||
|
||||
[profile.release]
|
||||
# Keep panics catchable (the probe records them per object) and turn integer
|
||||
# overflow into a reported panic instead of silent wraparound.
|
||||
debug = 1
|
||||
overflow-checks = true
|
||||
debug-assertions = true
|
||||
panic = "unwind"
|
||||
@@ -0,0 +1,854 @@
|
||||
//! Conformance probe: walks an HDF5 file with clawhdf5-format (the same calls
|
||||
//! the `clawhdf5` facade makes) and prints a canonical JSON description:
|
||||
//! every hard-linked object (sorted-name DFS, deduplicated by header address),
|
||||
//! and for each dataset / attribute its shape plus the SHA-256 of its values
|
||||
//! in a canonical encoding shared with `ref.py`.
|
||||
//!
|
||||
//! Canonical value encoding (per element, concatenated, row-major):
|
||||
//! int / float / bitfield / enum / time : element bytes, little-endian
|
||||
//! non-IEEE-layout float (e.g. N-Bit) : the IEEE float of the same size it converts to
|
||||
//! int with bit offset / short precision: the full-width integer it converts to
|
||||
//! opaque : raw bytes
|
||||
//! compound : members in declaration order (padding dropped)
|
||||
//! array : base elements row-major
|
||||
//! string (fixed or VL) : b'S' + u32le len + bytes (cut at first NUL, trailing spaces stripped)
|
||||
//! VL sequence : b'V' + u32le count + base elements
|
||||
//! reference : b'R' (payload not compared)
|
||||
//!
|
||||
//! Every object is processed inside catch_unwind; a caught panic is recorded
|
||||
//! with its message, location and the clawhdf5 frames of its backtrace.
|
||||
|
||||
use std::cell::RefCell;
|
||||
use std::collections::{HashMap, HashSet};
|
||||
use std::panic::{self, AssertUnwindSafe};
|
||||
use std::rc::Rc;
|
||||
|
||||
use clawhdf5_format::attribute::extract_attributes_full;
|
||||
use clawhdf5_format::data_layout::DataLayout;
|
||||
use clawhdf5_format::data_read;
|
||||
use clawhdf5_format::dataspace::{Dataspace, DataspaceType};
|
||||
use clawhdf5_format::datatype::{Datatype, DatatypeByteOrder};
|
||||
use clawhdf5_format::filter_pipeline::FilterPipeline;
|
||||
use clawhdf5_format::global_heap::GlobalHeapCollection;
|
||||
use clawhdf5_format::group_v1::{self, GroupEntry};
|
||||
use clawhdf5_format::group_v2;
|
||||
use clawhdf5_format::message_type::MessageType;
|
||||
use clawhdf5_format::object_header::ObjectHeader;
|
||||
use clawhdf5_format::signature;
|
||||
use clawhdf5_format::superblock::Superblock;
|
||||
use clawhdf5_format::symbol_table::SymbolTableMessage;
|
||||
use serde_json::{Map, Value, json};
|
||||
use sha2::{Digest, Sha256};
|
||||
|
||||
const MAX_BYTES: u64 = 200 * 1024 * 1024;
|
||||
const MAX_OBJECTS: usize = 200_000;
|
||||
|
||||
thread_local! {
|
||||
static LAST_PANIC: RefCell<Option<String>> = const { RefCell::new(None) };
|
||||
}
|
||||
|
||||
fn install_hook() {
|
||||
panic::set_hook(Box::new(|info| {
|
||||
let msg = if let Some(s) = info.payload().downcast_ref::<&str>() {
|
||||
s.to_string()
|
||||
} else if let Some(s) = info.payload().downcast_ref::<String>() {
|
||||
s.clone()
|
||||
} else {
|
||||
"<non-string panic>".into()
|
||||
};
|
||||
let loc = info
|
||||
.location()
|
||||
.map(|l| format!("{}:{}", l.file(), l.line()))
|
||||
.unwrap_or_default();
|
||||
let bt = std::backtrace::Backtrace::force_capture().to_string();
|
||||
// keep only frames from clawhdf5 code
|
||||
let mut frames = Vec::new();
|
||||
let lines: Vec<&str> = bt.lines().collect();
|
||||
for (i, l) in lines.iter().enumerate() {
|
||||
let t = l.trim();
|
||||
if t.contains("clawhdf5_format::") || t.contains("conformance_probe::") {
|
||||
let at = lines
|
||||
.get(i + 1)
|
||||
.map(|n| n.trim())
|
||||
.filter(|n| n.starts_with("at "))
|
||||
.map(|n| {
|
||||
let n = n.trim_start_matches("at ");
|
||||
match n.find("/crates/") {
|
||||
Some(p) => n[p + 1..].to_string(),
|
||||
None => n.to_string(),
|
||||
}
|
||||
})
|
||||
.unwrap_or_default();
|
||||
let name = t.split_once(": ").map(|x| x.1).unwrap_or(t);
|
||||
frames.push(format!("{name} ({at})"));
|
||||
if frames.len() >= 12 {
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
let full = format!("PANIC: {msg} @ {loc}\n {}", frames.join("\n "));
|
||||
eprintln!("{full}");
|
||||
LAST_PANIC.with(|p| *p.borrow_mut() = Some(full));
|
||||
}));
|
||||
}
|
||||
|
||||
/// Run `f`, turning a panic into Err("PANIC: ...").
|
||||
fn guarded<T>(f: impl FnOnce() -> Result<T, String>) -> Result<T, String> {
|
||||
match panic::catch_unwind(AssertUnwindSafe(f)) {
|
||||
Ok(r) => r,
|
||||
Err(_) => Err(LAST_PANIC
|
||||
.with(|p| p.borrow_mut().take())
|
||||
.unwrap_or_else(|| "PANIC: <unknown>".into())),
|
||||
}
|
||||
}
|
||||
|
||||
fn e<E: std::fmt::Debug>(x: E) -> String {
|
||||
format!("{x:?}")
|
||||
}
|
||||
|
||||
struct Ctx<'a> {
|
||||
data: &'a [u8],
|
||||
os: u8,
|
||||
ls: u8,
|
||||
base_dir: std::path::PathBuf,
|
||||
heaps: RefCell<HashMap<u64, Result<Rc<GlobalHeapCollection>, String>>>,
|
||||
}
|
||||
|
||||
impl<'a> Ctx<'a> {
|
||||
fn header(&self, addr: u64) -> Result<ObjectHeader, String> {
|
||||
ObjectHeader::parse(self.data, addr as usize, self.os, self.ls).map_err(e)
|
||||
}
|
||||
|
||||
fn payload(&self, h: &ObjectHeader, t: MessageType) -> Result<Option<Vec<u8>>, String> {
|
||||
match h.messages.iter().find(|m| m.msg_type == t) {
|
||||
None => Ok(None),
|
||||
Some(m) => {
|
||||
clawhdf5_format::shared_message::message_data(self.data, m, self.os, self.ls)
|
||||
.map(|c| Some(c.into_owned()))
|
||||
.map_err(e)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn heap_obj(&self, addr: u64, idx: u32) -> Result<Vec<u8>, String> {
|
||||
let coll = {
|
||||
let mut cache = self.heaps.borrow_mut();
|
||||
cache
|
||||
.entry(addr)
|
||||
.or_insert_with(|| {
|
||||
GlobalHeapCollection::parse(self.data, addr as usize, self.ls)
|
||||
.map(Rc::new)
|
||||
.map_err(e)
|
||||
})
|
||||
.clone()?
|
||||
};
|
||||
coll.get_object(idx as u16)
|
||||
.map(|o| o.data.clone())
|
||||
.ok_or_else(|| {
|
||||
format!("GlobalHeapObjectNotFound {{ collection_address: {addr}, index: {idx} }}")
|
||||
})
|
||||
}
|
||||
|
||||
fn read_offset(&self, b: &[u8]) -> u64 {
|
||||
let mut v = 0u64;
|
||||
for (i, x) in b.iter().take(self.os as usize).enumerate() {
|
||||
v |= (*x as u64) << (8 * i);
|
||||
}
|
||||
v
|
||||
}
|
||||
|
||||
fn canon(&self, dt: &Datatype, b: &[u8], out: &mut Vec<u8>) -> Result<(), String> {
|
||||
let size = dt.type_size() as usize;
|
||||
if b.len() < size {
|
||||
return Err(format!(
|
||||
"canon: element slice {} < type size {size}",
|
||||
b.len()
|
||||
));
|
||||
}
|
||||
match dt {
|
||||
Datatype::FloatingPoint { .. } if !ieee_layout(dt) => {
|
||||
canon_custom_float(dt, &b[..size], out)?
|
||||
}
|
||||
Datatype::FixedPoint { .. } if partial_int(dt) => {
|
||||
canon_partial_int(dt, &b[..size], out)?
|
||||
}
|
||||
Datatype::FixedPoint { byte_order, .. }
|
||||
| Datatype::BitField { byte_order, .. }
|
||||
| Datatype::FloatingPoint { byte_order, .. } => match byte_order {
|
||||
DatatypeByteOrder::LittleEndian => out.extend_from_slice(&b[..size]),
|
||||
DatatypeByteOrder::BigEndian => out.extend(b[..size].iter().rev()),
|
||||
DatatypeByteOrder::Vax => return Err("canon: VAX byte order".into()),
|
||||
},
|
||||
Datatype::Time { .. } | Datatype::Opaque { .. } => out.extend_from_slice(&b[..size]),
|
||||
Datatype::String { .. } => canon_str(&b[..size], out),
|
||||
Datatype::Compound { members, .. } => {
|
||||
for m in members {
|
||||
let off = m.byte_offset as usize;
|
||||
let ms = m.datatype.type_size() as usize;
|
||||
if off.checked_add(ms).is_none_or(|end| end > size) {
|
||||
return Err(format!("canon: member {} out of bounds", m.name));
|
||||
}
|
||||
self.canon(&m.datatype, &b[off..off + ms], out)?;
|
||||
}
|
||||
}
|
||||
Datatype::Reference { .. } => out.push(b'R'),
|
||||
Datatype::Enumeration { base_type, .. } => self.canon(base_type, b, out)?,
|
||||
Datatype::Array {
|
||||
base_type,
|
||||
dimensions,
|
||||
} => {
|
||||
let n: usize = dimensions.iter().map(|d| *d as usize).product();
|
||||
let bs = base_type.type_size() as usize;
|
||||
for i in 0..n {
|
||||
self.canon(base_type, &b[i * bs..], out)?;
|
||||
}
|
||||
}
|
||||
Datatype::VariableLength {
|
||||
is_string,
|
||||
base_type,
|
||||
..
|
||||
} => {
|
||||
let len = u32::from_le_bytes([b[0], b[1], b[2], b[3]]) as usize;
|
||||
let addr = self.read_offset(&b[4..]);
|
||||
let idx_off = 4 + self.os as usize;
|
||||
let idx = u32::from_le_bytes([
|
||||
b[idx_off],
|
||||
b[idx_off + 1],
|
||||
b[idx_off + 2],
|
||||
b[idx_off + 3],
|
||||
]);
|
||||
let obj = if len == 0 || addr == 0 || addr == u64::MAX >> (64 - 8 * self.os as u32)
|
||||
{
|
||||
Vec::new()
|
||||
} else {
|
||||
self.heap_obj(addr, idx)?
|
||||
};
|
||||
if *is_string {
|
||||
let l = len.min(obj.len());
|
||||
canon_str(&obj[..l], out);
|
||||
} else {
|
||||
let bs = base_type.type_size() as usize;
|
||||
if bs == 0 {
|
||||
return Err("canon: VL base size 0".into());
|
||||
}
|
||||
let need = len.checked_mul(bs).ok_or("canon: VL overflow")?;
|
||||
if len > 0 && obj.len() < need {
|
||||
return Err(format!("canon: VL object {} < {need}", obj.len()));
|
||||
}
|
||||
out.push(b'V');
|
||||
out.extend_from_slice(&(len as u32).to_le_bytes());
|
||||
for i in 0..len {
|
||||
self.canon(base_type, &obj[i * bs..], out)?;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Returns (shape json, n_elements)
|
||||
fn shape(ds: &Dataspace) -> (Value, u64) {
|
||||
match ds.space_type {
|
||||
DataspaceType::Null => (Value::String("null".into()), 0),
|
||||
DataspaceType::Scalar => (json!([]), 1),
|
||||
DataspaceType::Simple => {
|
||||
let n = ds.dimensions.iter().fold(1u64, |a, d| a.saturating_mul(*d));
|
||||
(json!(ds.dimensions), n)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn hash_values(
|
||||
&self,
|
||||
dt: &Datatype,
|
||||
raw: &[u8],
|
||||
n: u64,
|
||||
rec: &mut Map<String, Value>,
|
||||
) -> Result<(), String> {
|
||||
let size = dt.type_size() as usize;
|
||||
let need = (n as usize).checked_mul(size).ok_or("n*size overflow")?;
|
||||
if raw.len() != need {
|
||||
return Err(format!(
|
||||
"raw length {} != n_elements {n} * type_size {size}",
|
||||
raw.len()
|
||||
));
|
||||
}
|
||||
let mut canon = Vec::with_capacity(need);
|
||||
for i in 0..n as usize {
|
||||
self.canon(dt, &raw[i * size..(i + 1) * size], &mut canon)?;
|
||||
}
|
||||
let h = Sha256::digest(&canon);
|
||||
rec.insert("hash".into(), Value::String(hex(&h)));
|
||||
rec.insert(
|
||||
"head".into(),
|
||||
Value::String(hex(&canon[..canon.len().min(48)])),
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn read_dataset(&self, h: &ObjectHeader, rec: &mut Map<String, Value>) -> Result<(), String> {
|
||||
let dtb = self
|
||||
.payload(h, MessageType::Datatype)?
|
||||
.ok_or("MissingMessage(Datatype)")?;
|
||||
let (dt, _) = Datatype::parse(&dtb).map_err(e)?;
|
||||
rec.insert("dtype".into(), Value::String(dtype_str(&dt)));
|
||||
let dsb = self
|
||||
.payload(h, MessageType::Dataspace)?
|
||||
.ok_or("MissingMessage(Dataspace)")?;
|
||||
let ds = Dataspace::parse(&dsb, self.ls).map_err(e)?;
|
||||
let (shape, n) = Self::shape(&ds);
|
||||
rec.insert("shape".into(), shape);
|
||||
if n.saturating_mul(dt.type_size() as u64) > MAX_BYTES {
|
||||
rec.insert("skipped".into(), Value::String("too large".into()));
|
||||
return Ok(());
|
||||
}
|
||||
let lm = h
|
||||
.messages
|
||||
.iter()
|
||||
.find(|m| m.msg_type == MessageType::DataLayout)
|
||||
.ok_or("MissingMessage(DataLayout)")?;
|
||||
let dl = DataLayout::parse(&lm.data, self.os, self.ls).map_err(e)?;
|
||||
rec.insert(
|
||||
"layout".into(),
|
||||
Value::String(
|
||||
match &dl {
|
||||
DataLayout::Compact { .. } => "compact",
|
||||
DataLayout::Contiguous { .. } => "contiguous",
|
||||
DataLayout::Chunked { .. } => "chunked",
|
||||
DataLayout::Virtual { .. } => "virtual",
|
||||
}
|
||||
.into(),
|
||||
),
|
||||
);
|
||||
let pipeline = match self.payload(h, MessageType::FilterPipeline)? {
|
||||
Some(p) => Some(FilterPipeline::parse(&p).map_err(e)?),
|
||||
None => None,
|
||||
};
|
||||
if let Some(p) = &pipeline {
|
||||
rec.insert(
|
||||
"filters".into(),
|
||||
json!(p.filters.iter().map(|f| f.filter_id).collect::<Vec<_>>()),
|
||||
);
|
||||
}
|
||||
let raw = if matches!(dl, DataLayout::Virtual { .. }) {
|
||||
let base = self.base_dir.clone();
|
||||
let resolver = move |name: &str| -> Option<Vec<u8>> {
|
||||
let p = std::path::Path::new(name);
|
||||
if p.is_absolute()
|
||||
|| p.components()
|
||||
.any(|c| matches!(c, std::path::Component::ParentDir))
|
||||
{
|
||||
return None;
|
||||
}
|
||||
std::fs::read(base.join(p)).ok()
|
||||
};
|
||||
data_read::read_raw_data_full_with_resolver(
|
||||
self.data,
|
||||
&dl,
|
||||
&ds,
|
||||
&dt,
|
||||
pipeline.as_ref(),
|
||||
self.os,
|
||||
self.ls,
|
||||
Some(&resolver),
|
||||
)
|
||||
.map_err(e)?
|
||||
} else {
|
||||
let cache = clawhdf5_format::chunk_cache::ChunkCache::new();
|
||||
clawhdf5_format::fill_value::read_full_with_fill::<clawhdf5_format::error::FormatError>(
|
||||
&h.messages,
|
||||
self.data,
|
||||
&dl,
|
||||
&ds,
|
||||
dt.type_size() as usize,
|
||||
self.os,
|
||||
self.ls,
|
||||
|| {
|
||||
data_read::read_raw_data_cached(
|
||||
self.data,
|
||||
&dl,
|
||||
&ds,
|
||||
&dt,
|
||||
pipeline.as_ref(),
|
||||
self.os,
|
||||
self.ls,
|
||||
&cache,
|
||||
)
|
||||
},
|
||||
)
|
||||
.map_err(e)?
|
||||
};
|
||||
self.hash_values(&dt, &raw, n, rec)
|
||||
}
|
||||
|
||||
fn attrs(&self, h: &ObjectHeader) -> Result<Map<String, Value>, String> {
|
||||
let msgs = extract_attributes_full(self.data, h, self.os, self.ls).map_err(e)?;
|
||||
let mut out = Map::new();
|
||||
for a in &msgs {
|
||||
let r = guarded(|| {
|
||||
let mut rec = Map::new();
|
||||
rec.insert("dtype".into(), Value::String(dtype_str(&a.datatype)));
|
||||
let (shape, n) = Self::shape(&a.dataspace);
|
||||
rec.insert("shape".into(), shape);
|
||||
self.hash_values(&a.datatype, &a.raw_data, n, &mut rec)?;
|
||||
Ok(rec)
|
||||
});
|
||||
let v = match r {
|
||||
Ok(rec) => Value::Object(rec),
|
||||
Err(msg) => json!({ "error": msg }),
|
||||
};
|
||||
out.insert(a.name.clone(), v);
|
||||
}
|
||||
Ok(out)
|
||||
}
|
||||
|
||||
fn entries(&self, h: &ObjectHeader) -> Result<Vec<GroupEntry>, String> {
|
||||
let v1 = h
|
||||
.messages
|
||||
.iter()
|
||||
.find(|m| m.msg_type == MessageType::SymbolTable);
|
||||
if let Some(m) = v1 {
|
||||
let stm = SymbolTableMessage::parse(&m.data, self.os).map_err(e)?;
|
||||
group_v1::resolve_v1_group_entries(self.data, &stm, self.os, self.ls).map_err(e)
|
||||
} else if h
|
||||
.messages
|
||||
.iter()
|
||||
.any(|m| m.msg_type == MessageType::LinkInfo || m.msg_type == MessageType::Link)
|
||||
{
|
||||
group_v2::resolve_v2_group_entries(self.data, h, self.os, self.ls).map_err(e)
|
||||
} else {
|
||||
Ok(Vec::new())
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Element bytes as an unsigned integer (at most 16 bytes), honouring byte order.
|
||||
fn element_bits(b: &[u8], byte_order: &DatatypeByteOrder) -> Result<u128, String> {
|
||||
if b.len() > 16 {
|
||||
return Err(format!("canon: {}-byte numeric element", b.len()));
|
||||
}
|
||||
let mut v = 0u128;
|
||||
match byte_order {
|
||||
DatatypeByteOrder::LittleEndian => {
|
||||
for (i, x) in b.iter().enumerate() {
|
||||
v |= u128::from(*x) << (8 * i);
|
||||
}
|
||||
}
|
||||
DatatypeByteOrder::BigEndian => {
|
||||
for x in b {
|
||||
v = (v << 8) | u128::from(*x);
|
||||
}
|
||||
}
|
||||
DatatypeByteOrder::Vax => return Err("canon: VAX byte order".into()),
|
||||
}
|
||||
Ok(v)
|
||||
}
|
||||
|
||||
fn field(v: u128, pos: u32, len: u32) -> u128 {
|
||||
if len == 0 || pos >= 128 {
|
||||
return 0;
|
||||
}
|
||||
let v = v >> pos;
|
||||
if len >= 128 {
|
||||
v
|
||||
} else {
|
||||
v & ((1u128 << len) - 1)
|
||||
}
|
||||
}
|
||||
|
||||
/// True when a float's bit fields are exactly IEEE 754 binary16/32/64 for its
|
||||
/// size. h5py hands back such a type's bytes untouched; any other layout (an
|
||||
/// N-Bit `H5Tset_precision` float, say) is *converted* by libhdf5 into the
|
||||
/// numpy float of the same size, so comparing raw bytes would be meaningless.
|
||||
fn ieee_layout(dt: &Datatype) -> bool {
|
||||
let Datatype::FloatingPoint {
|
||||
size,
|
||||
bit_offset,
|
||||
bit_precision,
|
||||
exponent_location,
|
||||
exponent_size,
|
||||
mantissa_location,
|
||||
mantissa_size,
|
||||
exponent_bias,
|
||||
..
|
||||
} = dt
|
||||
else {
|
||||
return true;
|
||||
};
|
||||
let std = match size {
|
||||
2 => (16, 10, 5, 10, 15),
|
||||
4 => (32, 23, 8, 23, 127),
|
||||
8 => (64, 52, 11, 52, 1023),
|
||||
_ => return true, // no same-size numpy float to convert to: compare raw
|
||||
};
|
||||
*bit_offset == 0
|
||||
&& (
|
||||
*bit_precision,
|
||||
*exponent_location,
|
||||
*exponent_size,
|
||||
*mantissa_size,
|
||||
*exponent_bias,
|
||||
) == (std.0, std.1, std.2, std.3, std.4)
|
||||
&& *mantissa_location == 0
|
||||
}
|
||||
|
||||
/// Canonicalise a non-IEEE-layout float the way libhdf5's float->float
|
||||
/// conversion presents it to h5py: as the IEEE float of the same size.
|
||||
/// Assumes the implied-leading-one normalisation and the sign bit at the top
|
||||
/// of the precision (what `H5Tset_precision` produces; the parser does not
|
||||
/// keep either field).
|
||||
fn canon_custom_float(dt: &Datatype, b: &[u8], out: &mut Vec<u8>) -> Result<(), String> {
|
||||
let Datatype::FloatingPoint {
|
||||
size,
|
||||
byte_order,
|
||||
bit_offset,
|
||||
bit_precision,
|
||||
exponent_location,
|
||||
exponent_size,
|
||||
mantissa_location,
|
||||
mantissa_size,
|
||||
exponent_bias,
|
||||
} = dt
|
||||
else {
|
||||
unreachable!()
|
||||
};
|
||||
let (esize, msize) = (u32::from(*exponent_size), u32::from(*mantissa_size));
|
||||
if esize == 0 || esize > 30 || msize > 64 {
|
||||
return Err(format!("canon: unsupported float layout e{esize} m{msize}"));
|
||||
}
|
||||
let v = element_bits(b, byte_order)?;
|
||||
let sign_pos = (u32::from(*bit_offset) + u32::from(*bit_precision)).saturating_sub(1);
|
||||
let neg = field(v, sign_pos, 1) == 1;
|
||||
let e = field(v, u32::from(*exponent_location), esize) as i64;
|
||||
let m = field(v, u32::from(*mantissa_location), msize);
|
||||
let emax = (1i64 << esize) - 1;
|
||||
let bias = i64::from(*exponent_bias);
|
||||
let mag = if e == emax {
|
||||
if m == 0 { f64::INFINITY } else { f64::NAN }
|
||||
} else if e == 0 {
|
||||
(m as f64) * 2f64.powi((1 - bias - msize as i64) as i32)
|
||||
} else {
|
||||
((1u128 << msize) as f64 + m as f64) * 2f64.powi((e - bias - msize as i64) as i32)
|
||||
};
|
||||
let x = if neg { -mag } else { mag };
|
||||
match size {
|
||||
2 => out
|
||||
.extend_from_slice(&clawhdf5_format::float16::f32_to_f16_bits(x as f32).to_le_bytes()),
|
||||
4 => out.extend_from_slice(&(x as f32).to_le_bytes()),
|
||||
8 => out.extend_from_slice(&x.to_le_bytes()),
|
||||
_ => unreachable!("ieee_layout keeps other sizes raw"),
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Integers stored with a bit offset or reduced precision (N-Bit): libhdf5
|
||||
/// converts them to the full-width integer of the same size, shifting the
|
||||
/// value down and sign-extending from the top precision bit.
|
||||
fn canon_partial_int(dt: &Datatype, b: &[u8], out: &mut Vec<u8>) -> Result<(), String> {
|
||||
let Datatype::FixedPoint {
|
||||
size,
|
||||
byte_order,
|
||||
signed,
|
||||
bit_offset,
|
||||
bit_precision,
|
||||
} = dt
|
||||
else {
|
||||
unreachable!()
|
||||
};
|
||||
let prec = u32::from(*bit_precision);
|
||||
let v = element_bits(b, byte_order)?;
|
||||
let mut x = field(v, u32::from(*bit_offset), prec);
|
||||
if *signed && prec > 0 && prec < 128 && field(x, prec - 1, 1) == 1 {
|
||||
x |= !0u128 << prec;
|
||||
}
|
||||
out.extend_from_slice(&x.to_le_bytes()[..*size as usize]);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn partial_int(dt: &Datatype) -> bool {
|
||||
matches!(dt, Datatype::FixedPoint { size, bit_offset, bit_precision, .. }
|
||||
if *bit_offset != 0 || u32::from(*bit_precision) != size * 8)
|
||||
}
|
||||
|
||||
fn canon_str(b: &[u8], out: &mut Vec<u8>) {
|
||||
let cut = b.iter().position(|&c| c == 0).unwrap_or(b.len());
|
||||
let mut s = &b[..cut];
|
||||
while let [rest @ .., b' '] = s {
|
||||
s = rest;
|
||||
}
|
||||
out.push(b'S');
|
||||
out.extend_from_slice(&(s.len() as u32).to_le_bytes());
|
||||
out.extend_from_slice(s);
|
||||
}
|
||||
|
||||
fn hex(b: &[u8]) -> String {
|
||||
b.iter().map(|x| format!("{x:02x}")).collect()
|
||||
}
|
||||
|
||||
fn dtype_str(dt: &Datatype) -> String {
|
||||
match dt {
|
||||
Datatype::FixedPoint {
|
||||
size,
|
||||
signed,
|
||||
byte_order,
|
||||
..
|
||||
} => {
|
||||
format!(
|
||||
"{}{}{}",
|
||||
bo(byte_order),
|
||||
if *signed { "i" } else { "u" },
|
||||
size
|
||||
)
|
||||
}
|
||||
Datatype::FloatingPoint {
|
||||
size, byte_order, ..
|
||||
} => format!("{}f{}", bo(byte_order), size),
|
||||
Datatype::BitField {
|
||||
size, byte_order, ..
|
||||
} => format!("{}b{}", bo(byte_order), size),
|
||||
Datatype::Time { size, .. } => format!("time{size}"),
|
||||
Datatype::String { size, .. } => format!("S{size}"),
|
||||
Datatype::Opaque { size, .. } => format!("V{size}"),
|
||||
Datatype::Compound { size, members } => format!(
|
||||
"{{{}}}{size}",
|
||||
members
|
||||
.iter()
|
||||
.map(|m| format!("{}:{}", m.name, dtype_str(&m.datatype)))
|
||||
.collect::<Vec<_>>()
|
||||
.join(",")
|
||||
),
|
||||
Datatype::Reference { ref_type, .. } => format!("ref({ref_type:?})"),
|
||||
Datatype::Enumeration { base_type, .. } => format!("enum({})", dtype_str(base_type)),
|
||||
Datatype::VariableLength {
|
||||
is_string: true, ..
|
||||
} => "vlstr".into(),
|
||||
Datatype::VariableLength { base_type, .. } => format!("vlen({})", dtype_str(base_type)),
|
||||
Datatype::Array {
|
||||
base_type,
|
||||
dimensions,
|
||||
} => format!("({}){dimensions:?}", dtype_str(base_type)),
|
||||
}
|
||||
}
|
||||
|
||||
fn bo(b: &DatatypeByteOrder) -> &'static str {
|
||||
match b {
|
||||
DatatypeByteOrder::LittleEndian => "<",
|
||||
DatatypeByteOrder::BigEndian => ">",
|
||||
DatatypeByteOrder::Vax => "vax",
|
||||
}
|
||||
}
|
||||
|
||||
fn is_group(h: &ObjectHeader) -> bool {
|
||||
h.messages.iter().any(|m| {
|
||||
matches!(
|
||||
m.msg_type,
|
||||
MessageType::LinkInfo | MessageType::Link | MessageType::SymbolTable
|
||||
)
|
||||
})
|
||||
}
|
||||
|
||||
fn main() {
|
||||
install_hook();
|
||||
let path = std::env::args().nth(1).expect("usage: probe <file>");
|
||||
let mut top = Map::new();
|
||||
top.insert("file".into(), Value::String(path.clone()));
|
||||
let data = match std::fs::read(&path) {
|
||||
Ok(d) => d,
|
||||
Err(err) => {
|
||||
top.insert("open_error".into(), Value::String(format!("Io({err})")));
|
||||
println!("{}", Value::Object(top));
|
||||
return;
|
||||
}
|
||||
};
|
||||
let sb = guarded(|| {
|
||||
let off = signature::find_signature(&data).map_err(e)?;
|
||||
Superblock::parse(&data, off).map_err(e)
|
||||
});
|
||||
let sb = match sb {
|
||||
Ok(sb) => sb,
|
||||
Err(msg) => {
|
||||
top.insert("open_error".into(), Value::String(msg));
|
||||
println!("{}", Value::Object(top));
|
||||
return;
|
||||
}
|
||||
};
|
||||
top.insert("superblock_version".into(), json!(sb.version));
|
||||
let ctx = Ctx {
|
||||
data: &data,
|
||||
os: sb.offset_size,
|
||||
ls: sb.length_size,
|
||||
base_dir: std::path::Path::new(&path)
|
||||
.parent()
|
||||
.map(|p| p.to_path_buf())
|
||||
.unwrap_or_default(),
|
||||
heaps: RefCell::new(HashMap::new()),
|
||||
};
|
||||
let mut objects: Vec<Value> = Vec::new();
|
||||
let mut visited = HashSet::new();
|
||||
let mut soft_v1 = 0u64;
|
||||
// explicit DFS stack: (address, path)
|
||||
let mut stack: Vec<(u64, String)> = vec![(sb.root_group_address, "/".to_string())];
|
||||
while let Some((addr, p)) = stack.pop() {
|
||||
if objects.len() >= MAX_OBJECTS {
|
||||
top.insert("truncated".into(), json!(true));
|
||||
break;
|
||||
}
|
||||
if !visited.insert(addr) {
|
||||
continue;
|
||||
}
|
||||
let mut rec = Map::new();
|
||||
rec.insert("path".into(), Value::String(p.clone()));
|
||||
let r = guarded(|| {
|
||||
let h = ctx.header(addr)?;
|
||||
Ok(h)
|
||||
});
|
||||
let h = match r {
|
||||
Ok(h) => h,
|
||||
Err(msg) => {
|
||||
rec.insert("kind".into(), Value::String("unknown".into()));
|
||||
rec.insert("error".into(), Value::String(msg));
|
||||
objects.push(Value::Object(rec));
|
||||
continue;
|
||||
}
|
||||
};
|
||||
let is_ds = h
|
||||
.messages
|
||||
.iter()
|
||||
.any(|m| m.msg_type == MessageType::DataLayout);
|
||||
let kind = if is_ds {
|
||||
"dataset"
|
||||
} else if is_group(&h) || addr == sb.root_group_address {
|
||||
"group"
|
||||
} else if h
|
||||
.messages
|
||||
.iter()
|
||||
.any(|m| m.msg_type == MessageType::Datatype)
|
||||
{
|
||||
"datatype"
|
||||
} else {
|
||||
"unknown"
|
||||
};
|
||||
rec.insert("kind".into(), Value::String(kind.into()));
|
||||
if kind == "dataset"
|
||||
&& let Err(msg) = guarded(|| ctx.read_dataset(&h, &mut rec))
|
||||
{
|
||||
rec.insert("error".into(), Value::String(msg));
|
||||
}
|
||||
if kind != "datatype" {
|
||||
match guarded(|| ctx.attrs(&h)) {
|
||||
Ok(m) => {
|
||||
rec.insert("attrs".into(), Value::Object(m));
|
||||
}
|
||||
Err(msg) => {
|
||||
rec.insert("attrs_error".into(), Value::String(msg));
|
||||
}
|
||||
}
|
||||
}
|
||||
if kind == "group" {
|
||||
match guarded(|| ctx.entries(&h)) {
|
||||
Ok(mut ents) => {
|
||||
ents.retain(|en| {
|
||||
if en.cache_type == 2 {
|
||||
soft_v1 += 1;
|
||||
false
|
||||
} else {
|
||||
true
|
||||
}
|
||||
});
|
||||
ents.sort_by(|a, b| a.name.cmp(&b.name));
|
||||
let base = if p == "/" { String::new() } else { p.clone() };
|
||||
for en in ents.into_iter().rev() {
|
||||
stack.push((en.object_header_address, format!("{base}/{}", en.name)));
|
||||
}
|
||||
}
|
||||
Err(msg) => {
|
||||
rec.insert("list_error".into(), Value::String(msg));
|
||||
}
|
||||
}
|
||||
}
|
||||
objects.push(Value::Object(rec));
|
||||
}
|
||||
if soft_v1 > 0 {
|
||||
top.insert("v1_soft_link_entries".into(), json!(soft_v1));
|
||||
}
|
||||
top.insert("objects".into(), Value::Array(objects));
|
||||
println!("{}", Value::Object(top));
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
/// The N-Bit float of libhdf5's `test/testfiles/le_data.h5`
|
||||
/// (`Nbit_float_data_le`): offset 7, precision 20, sign bit 26, exponent
|
||||
/// 20+6 (bias 31), mantissa 7+13.
|
||||
fn nbit_f32(byte_order: DatatypeByteOrder) -> Datatype {
|
||||
Datatype::FloatingPoint {
|
||||
size: 4,
|
||||
byte_order,
|
||||
bit_offset: 7,
|
||||
bit_precision: 20,
|
||||
exponent_location: 20,
|
||||
exponent_size: 6,
|
||||
mantissa_location: 7,
|
||||
mantissa_size: 13,
|
||||
exponent_bias: 31,
|
||||
}
|
||||
}
|
||||
|
||||
fn canon_one(dt: &Datatype, bytes: &[u8]) -> Vec<u8> {
|
||||
let mut out = Vec::new();
|
||||
canon_custom_float(dt, bytes, &mut out).unwrap();
|
||||
out
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn nbit_float_canonicalises_to_the_value_libhdf5_returns() {
|
||||
let le = nbit_f32(DatatypeByteOrder::LittleEndian);
|
||||
let be = nbit_f32(DatatypeByteOrder::BigEndian);
|
||||
assert!(!ieee_layout(&le));
|
||||
// 1.0: exponent = bias, mantissa 0
|
||||
let one: u32 = 31 << 20;
|
||||
assert_eq!(canon_one(&le, &one.to_le_bytes()), 1.0f32.to_le_bytes());
|
||||
assert_eq!(canon_one(&be, &one.to_be_bytes()), 1.0f32.to_le_bytes());
|
||||
// -2.1999512 (h5py's reading of the file's -2.2): sign, e = 32, m = 819
|
||||
let v: u32 = (1 << 26) | (32 << 20) | (819 << 7);
|
||||
assert_eq!(
|
||||
canon_one(&le, &v.to_le_bytes()),
|
||||
(-2.199_951_2f32).to_le_bytes()
|
||||
);
|
||||
assert_eq!(canon_one(&le, &[0; 4]), 0.0f32.to_le_bytes());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn ieee_floats_keep_their_raw_bytes() {
|
||||
let f32le = Datatype::FloatingPoint {
|
||||
size: 4,
|
||||
byte_order: DatatypeByteOrder::LittleEndian,
|
||||
bit_offset: 0,
|
||||
bit_precision: 32,
|
||||
exponent_location: 23,
|
||||
exponent_size: 8,
|
||||
mantissa_location: 0,
|
||||
mantissa_size: 23,
|
||||
exponent_bias: 127,
|
||||
};
|
||||
assert!(ieee_layout(&f32le));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn partial_precision_int_is_shifted_and_sign_extended() {
|
||||
let dt = Datatype::FixedPoint {
|
||||
size: 4,
|
||||
byte_order: DatatypeByteOrder::BigEndian,
|
||||
signed: true,
|
||||
bit_offset: 4,
|
||||
bit_precision: 17,
|
||||
};
|
||||
assert!(partial_int(&dt));
|
||||
let stored = (((-5i32) as u32) & 0x1_FFFF) << 4;
|
||||
let mut out = Vec::new();
|
||||
canon_partial_int(&dt, &stored.to_be_bytes(), &mut out).unwrap();
|
||||
assert_eq!(out, (-5i32).to_le_bytes());
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user