Merge branch 'feat/p1-plugin-filters' into feat/p1-proof
# Conflicts: # crates/clawhdf5/tests/h5py_chunked_read_tests.rs # docs/known-issues.md
This commit is contained in:
osobh
@@ -31,7 +31,7 @@ name = "parallel_bench"
|
||||
harness = false
|
||||
|
||||
[features]
|
||||
default = ["mmap", "provenance"]
|
||||
default = ["mmap", "provenance", "lzf"]
|
||||
mmap = ["clawhdf5-io/mmap"]
|
||||
parallel = ["clawhdf5-format/parallel", "rayon"]
|
||||
# zlib-ng (C, needs cmake) instead of the default pure-Rust zlib-rs.
|
||||
@@ -41,6 +41,14 @@ zstd = ["clawhdf5-format/zstd"]
|
||||
blake3_hash = ["clawhdf5-format/blake3_hash"]
|
||||
lz4 = ["clawhdf5-format/lz4"]
|
||||
pcodec = ["clawhdf5-format/pcodec"]
|
||||
# Plugin filters, pure Rust (no C). LZF (32000) is h5py's built-in
|
||||
# compression; it has no dependencies, so it is on by default.
|
||||
lzf = ["clawhdf5-format/lzf"]
|
||||
bitshuffle = ["clawhdf5-format/bitshuffle"]
|
||||
bzip2 = ["clawhdf5-format/bzip2"]
|
||||
blosc = ["clawhdf5-format/blosc"]
|
||||
# Every plugin filter.
|
||||
plugin-filters = ["lzf", "bitshuffle", "bzip2", "blosc"]
|
||||
# Dataset::verify_provenance() — recompute a dataset's SHA-256 and compare
|
||||
# against its stored _provenance_sha256 attribute. On by default, matching
|
||||
# clawhdf5-format's own default-on `provenance` feature.
|
||||
|
||||
@@ -381,3 +381,77 @@ with h5py.File("{p}", "r") as f:
|
||||
let fa = file.dataset("fa").unwrap().read_i32().unwrap();
|
||||
assert!(fa.iter().copied().eq(0..3 * 70000), "fa");
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Chunks that decode short
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/// HDF5 stores every chunk at the full chunk size, so a chunk whose filters
|
||||
/// decode to fewer bytes is corrupt. libhdf5 returns the rest of such a
|
||||
/// chunk uninitialised (or, for filters that check, fails); clawhdf5 padded
|
||||
/// it with zeros and returned it as data. Every read path must fail,
|
||||
/// naming the chunk, and chunks that decode fully must still read.
|
||||
#[test]
|
||||
fn h5py_short_decoded_chunk_is_an_error() {
|
||||
skip_if_no_python!();
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let path = dir.path().join("short.h5");
|
||||
let p = path.display().to_string();
|
||||
run_python(&format!(
|
||||
r#"
|
||||
import h5py, numpy as np, zlib
|
||||
with h5py.File("{p}", "w") as f:
|
||||
# 1-D, 4 gzip chunks of 8 i4; chunk 2 (offset 16) inflates to 16 bytes.
|
||||
ds = f.create_dataset("line", shape=(32,), chunks=(8,), dtype="<i4", compression="gzip")
|
||||
ds[...] = np.arange(32, dtype="<i4") + 1
|
||||
ds.id.write_direct_chunk((16,), zlib.compress(np.arange(4, dtype="<i4").tobytes()))
|
||||
# 2-D with a partial edge chunk; the short chunk is the edge one (8, 4).
|
||||
g = f.create_dataset("grid", shape=(10, 7), chunks=(4, 4), dtype="<f8", compression="gzip")
|
||||
g[...] = np.arange(70, dtype="<f8").reshape(10, 7) + 1
|
||||
g.id.write_direct_chunk((8, 4), zlib.compress(np.ones(3, dtype="<f8").tobytes()))
|
||||
# 40 chunks (enough for the parallel decoder), one short, shuffle + gzip.
|
||||
m = f.create_dataset("many", shape=(320,), chunks=(8,), dtype="<i4",
|
||||
compression="gzip", shuffle=True)
|
||||
m[...] = np.arange(320, dtype="<i4") + 1
|
||||
m.id.write_direct_chunk((200,), zlib.compress(bytes(31)))
|
||||
"#
|
||||
));
|
||||
|
||||
let hyperslab = |start: u64, count: u64| Selection::Hyperslab {
|
||||
start: vec![start],
|
||||
stride: vec![1],
|
||||
count: vec![1],
|
||||
block: vec![count],
|
||||
};
|
||||
let file = File::open(&path).unwrap();
|
||||
for (name, coords) in [("line", "[16"), ("grid", "[8, 4"), ("many", "[200")] {
|
||||
let ds = file.dataset(name).unwrap();
|
||||
let full = || {
|
||||
if name == "grid" {
|
||||
ds.read_f64().map(|_| ())
|
||||
} else {
|
||||
ds.read_i32().map(|_| ())
|
||||
}
|
||||
};
|
||||
let err = full().expect_err(name).to_string();
|
||||
assert!(err.contains(coords), "{name}: {err}");
|
||||
// Cached reads decode the same way; a second read fails too.
|
||||
assert!(full().is_err(), "{name}");
|
||||
assert!(ds.read_selection(&Selection::All).is_err(), "{name}");
|
||||
}
|
||||
let line = file.dataset("line").unwrap();
|
||||
assert!(line.read_selection(&hyperslab(14, 4)).is_err());
|
||||
// A selection that avoids the short chunk still reads.
|
||||
let want: Vec<u8> = (1..=16i32).flat_map(i32::to_le_bytes).collect();
|
||||
assert_eq!(line.read_selection(&hyperslab(0, 16)).unwrap(), want);
|
||||
let many = file.dataset("many").unwrap();
|
||||
assert!(many.read_selection(&hyperslab(190, 20)).is_err());
|
||||
|
||||
// The memory-mapped and lazy readers.
|
||||
let mm = clawhdf5::MmapFile::open(&path).unwrap();
|
||||
assert!(mm.dataset("line").unwrap().read_i32().is_err());
|
||||
assert!(mm.dataset("many").unwrap().read_i32().is_err());
|
||||
let lazy = clawhdf5::LazyFile::open_mmap(&path).unwrap();
|
||||
assert!(lazy.dataset("line").unwrap().read_i32().is_err());
|
||||
assert!(lazy.dataset("grid").unwrap().read_f64().is_err());
|
||||
}
|
||||
|
||||
@@ -0,0 +1,469 @@
|
||||
//! Plugin filters (LZF, bitshuffle, bzip2, blosc) against libhdf5.
|
||||
//!
|
||||
//! Read direction: h5py (with hdf5plugin for everything but LZF, which h5py
|
||||
//! ships) writes each filter over a matrix of dtypes (1-8 bytes, both byte
|
||||
//! orders), 1-3 dimensional shapes whose chunks do not divide them (so the
|
||||
//! edge chunks are partial), compressible and incompressible data, and the
|
||||
//! filter's own options; every dataset has an unfiltered twin, and clawhdf5
|
||||
//! must read the filtered one byte for byte equal to it.
|
||||
//!
|
||||
//! Write direction: clawhdf5 writes with its encoder, and h5py must read the
|
||||
//! values back.
|
||||
//!
|
||||
//! Skipped when python3 with h5py (and hdf5plugin) is unavailable, unless
|
||||
//! `CLAWHDF5_REQUIRE_INTEROP=1`.
|
||||
#![allow(dead_code)]
|
||||
|
||||
use std::process::Command;
|
||||
|
||||
use clawhdf5::File;
|
||||
use clawhdf5_format::selection::Selection;
|
||||
|
||||
fn python() -> String {
|
||||
std::env::var("CLAWHDF5_PYTHON").unwrap_or_else(|_| "python3".to_string())
|
||||
}
|
||||
|
||||
fn interop_required() -> bool {
|
||||
std::env::var("CLAWHDF5_REQUIRE_INTEROP").is_ok_and(|v| v == "1")
|
||||
}
|
||||
|
||||
fn python_has(modules: &str) -> bool {
|
||||
Command::new(python())
|
||||
.args(["-c", &format!("import {modules}")])
|
||||
.output()
|
||||
.map(|o| o.status.success())
|
||||
.unwrap_or(false)
|
||||
}
|
||||
|
||||
/// Whether the interop test can run; panics instead of skipping when
|
||||
/// `CLAWHDF5_REQUIRE_INTEROP=1`.
|
||||
fn have_python(modules: &str) -> bool {
|
||||
if python_has(modules) {
|
||||
return true;
|
||||
}
|
||||
assert!(
|
||||
!interop_required(),
|
||||
"CLAWHDF5_REQUIRE_INTEROP=1 but python3 with {modules} is not available"
|
||||
);
|
||||
eprintln!("SKIP: python3 with {modules} not available");
|
||||
false
|
||||
}
|
||||
|
||||
fn run_python(script: &str, args: &[&str]) -> String {
|
||||
let output = Command::new(python())
|
||||
.arg("-c")
|
||||
.arg(script)
|
||||
.args(args)
|
||||
.output()
|
||||
.expect("failed to run python");
|
||||
if !output.status.success() {
|
||||
panic!(
|
||||
"Python script failed:\nSTDOUT: {}\nSTDERR: {}",
|
||||
String::from_utf8_lossy(&output.stdout),
|
||||
String::from_utf8_lossy(&output.stderr)
|
||||
);
|
||||
}
|
||||
String::from_utf8_lossy(&output.stdout).trim().to_string()
|
||||
}
|
||||
|
||||
/// Writes `f{i}` (filtered) and `r{i}` (unfiltered twin) for every filter
|
||||
/// setting in `FILTERS` × every case; prints the number of pairs.
|
||||
const GENERATE: &str = r#"
|
||||
import sys
|
||||
import numpy as np, h5py
|
||||
try:
|
||||
import hdf5plugin
|
||||
except ImportError:
|
||||
hdf5plugin = None
|
||||
path = sys.argv[1]
|
||||
FILTERS = eval('(' + sys.argv[2] + ')')
|
||||
cases = [
|
||||
('<u1', (1000,), (128,), 'ramp'),
|
||||
('<i2', (37, 53), (10, 16), 'ramp'),
|
||||
('<i4', (2000,), (300,), 'ramp'),
|
||||
('>i4', (37, 53), (8, 8), 'ramp'),
|
||||
('<f4', (5, 6, 7), (2, 3, 4), 'ramp'),
|
||||
('<f8', (1500,), (512,), 'ramp'),
|
||||
('<i8', (33, 17), (33, 17), 'ramp'),
|
||||
('<u2', (4097,), (4097,), 'ramp'),
|
||||
('<f8', (3000,), (1000,), 'noise'),
|
||||
('<u1', (5000,), (5000,), 'noise'),
|
||||
('<i4', (1, 1), (1, 1), 'ramp'),
|
||||
('<f8', (100000,), (40000,), 'ramp'),
|
||||
]
|
||||
rng = np.random.default_rng(7)
|
||||
i = 0
|
||||
with h5py.File(path, 'w') as f:
|
||||
for label, kw in FILTERS:
|
||||
for dt, shape, chunks, kind in cases:
|
||||
n = int(np.prod(shape))
|
||||
if kind == 'noise':
|
||||
raw = rng.integers(0, 256, n * np.dtype(dt).itemsize, dtype=np.uint8)
|
||||
data = raw.view(dt)
|
||||
if np.dtype(dt).kind == 'f':
|
||||
data = np.nan_to_num(data)
|
||||
else:
|
||||
base = (np.arange(n) * 3) % 251 + rng.integers(0, 4, n)
|
||||
data = (base / 7).astype(dt) if np.dtype(dt).kind == 'f' else base.astype(dt)
|
||||
data = data.reshape(shape)
|
||||
f.create_dataset(f'f{i}', data=data, chunks=chunks, **kw)
|
||||
f.create_dataset(f'r{i}', data=data)
|
||||
f[f'f{i}'].attrs['case'] = f'{label} {dt} {shape} chunks={chunks} {kind}'
|
||||
i += 1
|
||||
print(i)
|
||||
"#;
|
||||
|
||||
/// Have h5py write every filter setting in `filters` (a Python list of
|
||||
/// `(label, create_dataset kwargs)`), then check that clawhdf5 reads each
|
||||
/// filtered dataset exactly as its unfiltered twin.
|
||||
fn check_h5py_written(tag: &str, filters: &str) {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let path = dir.path().join(format!("{tag}.h5"));
|
||||
let n: usize = run_python(GENERATE, &[path.to_str().unwrap(), filters])
|
||||
.parse()
|
||||
.unwrap();
|
||||
assert!(n > 0);
|
||||
let file = File::open(&path).unwrap();
|
||||
for i in 0..n {
|
||||
let filtered = file.dataset(&format!("f{i}")).unwrap();
|
||||
let case = format!("{:?}", filtered.attrs().unwrap().get("case"));
|
||||
let got = filtered
|
||||
.read_selection(&Selection::All)
|
||||
.unwrap_or_else(|e| panic!("{tag} f{i} {case}: {e}"));
|
||||
let want = file
|
||||
.dataset(&format!("r{i}"))
|
||||
.unwrap()
|
||||
.read_selection(&Selection::All)
|
||||
.unwrap();
|
||||
assert!(got == want, "{tag} f{i} {case}: data differs");
|
||||
}
|
||||
}
|
||||
|
||||
/// Values the write-direction tests store: row-major, compressible with some
|
||||
/// variation.
|
||||
fn ramp_i32(n: usize) -> Vec<i32> {
|
||||
(0..n).map(|i| ((i * 3) % 251) as i32 - 60).collect()
|
||||
}
|
||||
fn ramp_f64(n: usize) -> Vec<f64> {
|
||||
(0..n).map(|i| ((i * 3) % 251) as f64 / 7.0).collect()
|
||||
}
|
||||
fn ramp_u8(n: usize) -> Vec<u8> {
|
||||
(0..n).map(|i| ((i * 7) % 256) as u8).collect()
|
||||
}
|
||||
|
||||
/// h5py checks the datasets `write_ours` wrote against the same ramps.
|
||||
const VERIFY: &str = r#"
|
||||
import sys
|
||||
import numpy as np, h5py
|
||||
try:
|
||||
import hdf5plugin
|
||||
except ImportError:
|
||||
pass
|
||||
bad = []
|
||||
with h5py.File(sys.argv[1], 'r') as f:
|
||||
for name in f:
|
||||
ds = f[name]
|
||||
n = ds.size
|
||||
k = np.arange(n)
|
||||
if ds.dtype == np.int32:
|
||||
want = ((k * 3) % 251 - 60).astype(np.int32)
|
||||
elif ds.dtype == np.float64:
|
||||
want = ((k * 3) % 251) / 7.0
|
||||
else:
|
||||
want = ((k * 7) % 256).astype(np.uint8)
|
||||
got = ds[()].reshape(-1)
|
||||
if not np.array_equal(got, want):
|
||||
bad.append(name)
|
||||
# The dataset really is filtered by the plugin under test.
|
||||
if int(sys.argv[2]) not in [int(x) for x in ds._filters.keys() if x.isdigit()] \
|
||||
and sys.argv[3] not in ds._filters:
|
||||
bad.append(name + ':filter-missing:' + repr(ds._filters))
|
||||
print('OK' if not bad else 'BAD ' + ' '.join(bad))
|
||||
"#;
|
||||
|
||||
/// Write i32/f64/u8 datasets (1-D and 2-D, partial edge chunks) with
|
||||
/// `configure` applying the filter, then have h5py read them back.
|
||||
fn check_ours_read_by_h5py(
|
||||
tag: &str,
|
||||
filter_id: u16,
|
||||
filter_name: &str,
|
||||
configure: impl Fn(&mut clawhdf5_format::type_builders::DatasetBuilder),
|
||||
) {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let path = dir.path().join(format!("{tag}_ours.h5"));
|
||||
let mut fb = clawhdf5::FileBuilder::new();
|
||||
{
|
||||
let ds = fb.create_dataset("i32_1d");
|
||||
ds.with_i32_data(&ramp_i32(10_000)).with_chunks(&[3000]);
|
||||
configure(ds);
|
||||
}
|
||||
{
|
||||
let ds = fb.create_dataset("f64_2d");
|
||||
ds.with_f64_data(&ramp_f64(37 * 53))
|
||||
.with_shape(&[37, 53])
|
||||
.with_chunks(&[10, 16]);
|
||||
configure(ds);
|
||||
}
|
||||
{
|
||||
let ds = fb.create_dataset("u8_1d");
|
||||
ds.with_u8_data(&ramp_u8(5000)).with_chunks(&[777]);
|
||||
configure(ds);
|
||||
}
|
||||
{
|
||||
let ds = fb.create_dataset("f64_big");
|
||||
ds.with_f64_data(&ramp_f64(100_000)).with_chunks(&[40_000]);
|
||||
configure(ds);
|
||||
}
|
||||
fb.write(&path).unwrap();
|
||||
|
||||
// clawhdf5 reads its own output.
|
||||
let file = File::open(&path).unwrap();
|
||||
assert_eq!(
|
||||
file.dataset("i32_1d").unwrap().read_i32().unwrap(),
|
||||
ramp_i32(10_000)
|
||||
);
|
||||
assert_eq!(
|
||||
file.dataset("f64_2d").unwrap().read_f64().unwrap(),
|
||||
ramp_f64(37 * 53)
|
||||
);
|
||||
|
||||
let out = run_python(
|
||||
VERIFY,
|
||||
&[path.to_str().unwrap(), &filter_id.to_string(), filter_name],
|
||||
);
|
||||
assert_eq!(out, "OK", "{tag}: h5py could not read our output");
|
||||
}
|
||||
|
||||
#[cfg(feature = "bitshuffle")]
|
||||
#[test]
|
||||
fn bitshuffle_written_by_hdf5plugin_reads_exactly() {
|
||||
if !have_python("h5py, hdf5plugin") {
|
||||
return;
|
||||
}
|
||||
check_h5py_written(
|
||||
"bitshuffle",
|
||||
r#"[('none', hdf5plugin.Bitshuffle(cname='none')),
|
||||
('lz4', hdf5plugin.Bitshuffle(cname='lz4')),
|
||||
('lz4 nelems=16', hdf5plugin.Bitshuffle(nelems=16, cname='lz4')),
|
||||
('none nelems=64', hdf5plugin.Bitshuffle(nelems=64, cname='none')),
|
||||
('zstd', hdf5plugin.Bitshuffle(cname='zstd')),
|
||||
('zstd clevel=19 nelems=2048', hdf5plugin.Bitshuffle(nelems=2048, cname='zstd', clevel=19))]"#,
|
||||
);
|
||||
}
|
||||
|
||||
#[cfg(feature = "bitshuffle")]
|
||||
#[test]
|
||||
fn bitshuffle_written_by_clawhdf5_reads_in_hdf5plugin() {
|
||||
use clawhdf5_format::chunked_write::{BitshuffleCompression, PluginFilter};
|
||||
if !have_python("h5py, hdf5plugin") {
|
||||
return;
|
||||
}
|
||||
for (tag, compression) in [
|
||||
("bshuf_none", BitshuffleCompression::None),
|
||||
("bshuf_lz4", BitshuffleCompression::Lz4),
|
||||
("bshuf_zstd", BitshuffleCompression::Zstd { level: 3 }),
|
||||
] {
|
||||
check_ours_read_by_h5py(tag, 32008, "bitshuffle", |ds| {
|
||||
ds.with_bitshuffle(compression);
|
||||
});
|
||||
check_ours_read_by_h5py(tag, 32008, "bitshuffle", |ds| {
|
||||
ds.with_plugin_filter(PluginFilter::Bitshuffle {
|
||||
block_size: 40,
|
||||
compression,
|
||||
});
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(feature = "bzip2")]
|
||||
#[test]
|
||||
fn bzip2_written_by_hdf5plugin_reads_exactly() {
|
||||
if !have_python("h5py, hdf5plugin") {
|
||||
return;
|
||||
}
|
||||
check_h5py_written(
|
||||
"bzip2",
|
||||
r#"[('bzip2 9', hdf5plugin.BZip2()),
|
||||
('bzip2 1', hdf5plugin.BZip2(blocksize=1)),
|
||||
('bzip2 5 + shuffle', dict(**hdf5plugin.BZip2(blocksize=5), shuffle=True))]"#,
|
||||
);
|
||||
}
|
||||
|
||||
#[cfg(feature = "bzip2")]
|
||||
#[test]
|
||||
fn bzip2_written_by_clawhdf5_reads_in_hdf5plugin() {
|
||||
if !have_python("h5py, hdf5plugin") {
|
||||
return;
|
||||
}
|
||||
check_ours_read_by_h5py("bzip2", 307, "bzip2", |ds| {
|
||||
ds.with_bzip2(9);
|
||||
});
|
||||
check_ours_read_by_h5py("bzip2_1", 307, "bzip2", |ds| {
|
||||
ds.with_bzip2(1).without_shuffle();
|
||||
});
|
||||
}
|
||||
|
||||
#[cfg(feature = "blosc")]
|
||||
#[test]
|
||||
fn blosc_written_by_hdf5plugin_reads_exactly() {
|
||||
if !have_python("h5py, hdf5plugin") {
|
||||
return;
|
||||
}
|
||||
// Every codec hdf5plugin's Blosc offers, each shuffle mode, and levels
|
||||
// from "store" to maximum.
|
||||
check_h5py_written(
|
||||
"blosc",
|
||||
r#"[(f'{c} {s} {l}', hdf5plugin.Blosc(cname=c, clevel=l, shuffle=s))
|
||||
for c in ['blosclz', 'lz4', 'lz4hc', 'snappy', 'zlib', 'zstd']
|
||||
for s, l in [(hdf5plugin.Blosc.NOSHUFFLE, 5),
|
||||
(hdf5plugin.Blosc.SHUFFLE, 9),
|
||||
(hdf5plugin.Blosc.BITSHUFFLE, 1)]]
|
||||
+ [('blosclz level 0', hdf5plugin.Blosc(cname='blosclz', clevel=0))]"#,
|
||||
);
|
||||
}
|
||||
|
||||
#[cfg(feature = "blosc")]
|
||||
#[test]
|
||||
fn blosc_written_by_clawhdf5_reads_in_hdf5plugin() {
|
||||
use clawhdf5_format::chunked_write::{BloscCodec, BloscShuffle};
|
||||
if !have_python("h5py, hdf5plugin") {
|
||||
return;
|
||||
}
|
||||
for codec in [
|
||||
BloscCodec::Lz4,
|
||||
BloscCodec::Snappy,
|
||||
BloscCodec::Zlib,
|
||||
BloscCodec::Zstd,
|
||||
] {
|
||||
for (shuffle, level) in [
|
||||
(BloscShuffle::None, 5),
|
||||
(BloscShuffle::Byte, 9),
|
||||
(BloscShuffle::Bit, 1),
|
||||
(BloscShuffle::Byte, 0),
|
||||
] {
|
||||
check_ours_read_by_h5py(
|
||||
&format!("blosc_{codec:?}_{shuffle:?}_{level}"),
|
||||
32001,
|
||||
"blosc",
|
||||
|ds| {
|
||||
ds.with_blosc(codec, level, shuffle);
|
||||
},
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(feature = "lzf")]
|
||||
#[test]
|
||||
fn lzf_written_by_h5py_reads_exactly() {
|
||||
if !have_python("h5py") {
|
||||
return;
|
||||
}
|
||||
check_h5py_written(
|
||||
"lzf",
|
||||
r#"[('lzf', dict(compression='lzf')),
|
||||
('lzf+shuffle', dict(compression='lzf', shuffle=True)),
|
||||
('lzf+shuffle+fletcher32', dict(compression='lzf', shuffle=True, fletcher32=True))]"#,
|
||||
);
|
||||
}
|
||||
|
||||
#[cfg(feature = "lzf")]
|
||||
#[test]
|
||||
fn lzf_written_by_clawhdf5_reads_in_h5py() {
|
||||
if !have_python("h5py") {
|
||||
return;
|
||||
}
|
||||
check_ours_read_by_h5py("lzf", 32000, "lzf", |ds| {
|
||||
ds.with_lzf();
|
||||
});
|
||||
check_ours_read_by_h5py("lzf_noshuffle", 32000, "lzf", |ds| {
|
||||
ds.with_lzf().without_shuffle();
|
||||
});
|
||||
}
|
||||
|
||||
/// Blosc2 and ZFP are not implemented: reading them must be a clear error
|
||||
/// naming the filter, never data.
|
||||
#[test]
|
||||
fn unimplemented_filters_are_a_clear_error() {
|
||||
if !have_python("h5py, hdf5plugin") {
|
||||
return;
|
||||
}
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let path = dir.path().join("unimplemented.h5");
|
||||
run_python(
|
||||
r#"
|
||||
import sys
|
||||
import numpy as np, h5py, hdf5plugin
|
||||
with h5py.File(sys.argv[1], 'w') as f:
|
||||
d = np.arange(4096, dtype='<f4').reshape(64, 64)
|
||||
f.create_dataset('blosc2', data=d, chunks=(16, 16), **hdf5plugin.Blosc2())
|
||||
f.create_dataset('zfp', data=d, chunks=(16, 16), **hdf5plugin.Zfp(reversible=True))
|
||||
"#,
|
||||
&[path.to_str().unwrap()],
|
||||
);
|
||||
let file = File::open(&path).unwrap();
|
||||
for (name, id, label) in [("blosc2", 32026u16, "Blosc2"), ("zfp", 32013, "ZFP")] {
|
||||
let err = file
|
||||
.dataset(name)
|
||||
.unwrap()
|
||||
.read_selection(&Selection::All)
|
||||
.expect_err("an unimplemented filter must not read");
|
||||
let msg = err.to_string();
|
||||
assert!(
|
||||
msg.contains(&id.to_string()) && msg.contains(label),
|
||||
"{name}: {msg}"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
/// A corrupt chunk must never read as zeros: a Blosc frame that declares no
|
||||
/// data (libhdf5's filter fails it), and Blosc, LZF and bzip2 chunks that
|
||||
/// decode short (libhdf5 returns the rest of the chunk uninitialised).
|
||||
#[test]
|
||||
fn short_decoding_chunks_are_errors() {
|
||||
if !have_python("h5py, hdf5plugin") {
|
||||
return;
|
||||
}
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let path = dir.path().join("short.h5");
|
||||
run_python(
|
||||
r#"
|
||||
import sys, struct, bz2
|
||||
import numpy as np, h5py, hdf5plugin
|
||||
def lzf(b):
|
||||
# One literal run per 32 bytes: a valid LZF stream.
|
||||
return b"".join(bytes([len(b[i:i+32]) - 1]) + b[i:i+32] for i in range(0, len(b), 32))
|
||||
with h5py.File(sys.argv[1], 'w') as f:
|
||||
kw = dict(shape=(16,), dtype='<i4', chunks=(16,))
|
||||
empty = f.create_dataset('blosc_empty', **kw, **hdf5plugin.Blosc(cname='lz4'))
|
||||
empty.id.write_direct_chunk((0,), bytes([2, 1, 0x20, 4]) + struct.pack('<III', 0, 64, 16))
|
||||
short = f.create_dataset('blosc_short', **kw, **hdf5plugin.Blosc(cname='lz4'))
|
||||
short.id.write_direct_chunk((0,), bytes([2, 1, 0x22, 4]) + struct.pack('<III', 32, 32, 48) + bytes(32))
|
||||
try:
|
||||
f['blosc_empty'][...]
|
||||
except OSError:
|
||||
pass
|
||||
else:
|
||||
raise SystemExit('blosc_empty: libhdf5 read it')
|
||||
lz = f.create_dataset('lzf_short', **kw, compression='lzf')
|
||||
lz.id.write_direct_chunk((0,), lzf(np.arange(8, dtype='<i4').tobytes()))
|
||||
bz = f.create_dataset('bzip2_short', **kw, **hdf5plugin.BZip2())
|
||||
bz.id.write_direct_chunk((0,), bz2.compress(np.arange(8, dtype='<i4').tobytes()))
|
||||
ok = f.create_dataset('lzf_ok', **kw, compression='lzf')
|
||||
ok.id.write_direct_chunk((0,), lzf(np.arange(16, dtype='<i4').tobytes()))
|
||||
"#,
|
||||
&[path.to_str().unwrap()],
|
||||
);
|
||||
let file = File::open(&path).unwrap();
|
||||
for name in ["blosc_empty", "blosc_short", "lzf_short", "bzip2_short"] {
|
||||
let ds = file.dataset(name).unwrap();
|
||||
assert!(
|
||||
ds.read_i32().is_err(),
|
||||
"{name}: {:?}",
|
||||
ds.read_i32().unwrap()
|
||||
);
|
||||
assert!(ds.read_selection(&Selection::All).is_err(), "{name}");
|
||||
}
|
||||
let want: Vec<i32> = (0..16).collect();
|
||||
assert_eq!(file.dataset("lzf_ok").unwrap().read_i32().unwrap(), want);
|
||||
}
|
||||
Reference in New Issue
Block a user