Merge branch 'feat/p2-python-bindings' into feat/p2-perf-coverage
# Conflicts: # CHANGELOG.md # README.md
This commit is contained in:
osobh
@@ -176,6 +176,22 @@ impl File {
|
||||
})
|
||||
}
|
||||
|
||||
/// A `Dataset` handle for the object header at `address` (an address
|
||||
/// from a group listing, or one kept from an earlier lookup), without
|
||||
/// resolving a path. Resolving a path walks every group on it, which in
|
||||
/// a large group costs a scan of its links; keep the address instead to
|
||||
/// open the same dataset repeatedly.
|
||||
pub fn dataset_at(&self, address: u64) -> Result<Dataset<'_>, Error> {
|
||||
let hdr = self.parse_header(address)?;
|
||||
if !has_message(&hdr, MessageType::DataLayout) {
|
||||
return Err(Error::NotADataset(format!("object at address {address}")));
|
||||
}
|
||||
Ok(Dataset {
|
||||
file: self,
|
||||
header: hdr,
|
||||
})
|
||||
}
|
||||
|
||||
/// Resolve a path and return a `Group` handle.
|
||||
///
|
||||
/// The path uses `/` separators (e.g., `"sensors"`).
|
||||
@@ -606,8 +622,16 @@ impl<'f> Dataset<'f> {
|
||||
|
||||
/// Read selected elements as raw bytes.
|
||||
///
|
||||
/// Only the elements matching the [`clawhdf5_format::selection::Selection`] are returned. For chunked
|
||||
/// datasets, only intersecting chunks are decompressed.
|
||||
/// Only the elements matching the [`clawhdf5_format::selection::Selection`] are returned.
|
||||
///
|
||||
/// What is read to get them: when the selection's bounding box covers at
|
||||
/// most half the dataset, only that box — the chunks overlapping it, or
|
||||
/// the rows of a contiguous dataset. The whole dataset is decoded instead
|
||||
/// when the box covers more than half (a strided selection spanning the
|
||||
/// dataset does), for compact and virtual layouts, for a dataset with no
|
||||
/// storage, and for a chunked dataset with a non-default fill value.
|
||||
/// [`Selection::All`](clawhdf5_format::selection::Selection::All) goes
|
||||
/// through the file's chunk cache; other selections do not.
|
||||
pub fn read_selection(
|
||||
&self,
|
||||
selection: &clawhdf5_format::selection::Selection,
|
||||
|
||||
@@ -986,3 +986,35 @@ fn u64_data_roundtrip() {
|
||||
values
|
||||
);
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Opening a dataset by address
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
#[test]
|
||||
fn dataset_at_opens_the_same_dataset_as_its_path() {
|
||||
let mut b = FileBuilder::new();
|
||||
let mut g = b.create_group("grp");
|
||||
g.create_dataset("vals").with_f64_data(&[1.0, 2.5, -3.0]);
|
||||
b.add_group(g.finish());
|
||||
let file = File::from_bytes(b.finish().unwrap()).unwrap();
|
||||
|
||||
let addr =
|
||||
clawhdf5_format::group_v2::resolve_path_any(file.as_bytes(), file.superblock(), "grp/vals")
|
||||
.unwrap();
|
||||
let by_addr = file.dataset_at(addr).unwrap();
|
||||
assert_eq!(by_addr.read_f64().unwrap(), vec![1.0, 2.5, -3.0]);
|
||||
assert_eq!(
|
||||
by_addr.shape().unwrap(),
|
||||
file.dataset("grp/vals").unwrap().shape().unwrap()
|
||||
);
|
||||
|
||||
// The group's own header is not a dataset.
|
||||
let group_addr =
|
||||
clawhdf5_format::group_v2::resolve_path_any(file.as_bytes(), file.superblock(), "grp")
|
||||
.unwrap();
|
||||
assert!(matches!(
|
||||
file.dataset_at(group_addr),
|
||||
Err(clawhdf5::Error::NotADataset(_))
|
||||
));
|
||||
}
|
||||
|
||||
@@ -0,0 +1,241 @@
|
||||
//! Partial hyperslab reads of every v4 (`libver='latest'`) chunk index type,
|
||||
//! compared element for element with h5py.
|
||||
//!
|
||||
//! `Dataset::read_selection` takes two routes: `partial_read` materialises
|
||||
//! the selection's bounding box when it covers at most half the dataset, and
|
||||
//! `data_read::read_raw_data_selection` handles the rest. The second route
|
||||
//! once passed the layout's full chunk dimensions (which carry the element
|
||||
//! size as an extra, last dimension) to the implicit-index chunk generator,
|
||||
//! which then indexed past the dataset's rank and panicked. So every case
|
||||
//! below reads both a small window and one covering most of the dataset.
|
||||
//!
|
||||
//! Each case asserts which chunk index the file actually uses (parsed from
|
||||
//! the layout message), so a change in how h5py lays the file out can't turn
|
||||
//! this into a test of the wrong index.
|
||||
//!
|
||||
//! Skipped when python3 with h5py is unavailable, unless
|
||||
//! `CLAWHDF5_REQUIRE_INTEROP=1`.
|
||||
|
||||
use std::process::Command;
|
||||
|
||||
use clawhdf5::File;
|
||||
use clawhdf5_format::data_layout::DataLayout;
|
||||
use clawhdf5_format::message_type::MessageType;
|
||||
use clawhdf5_format::object_header::ObjectHeader;
|
||||
use clawhdf5_format::selection::Selection;
|
||||
use clawhdf5_format::superblock::Superblock;
|
||||
|
||||
fn python() -> String {
|
||||
std::env::var("CLAWHDF5_PYTHON").unwrap_or_else(|_| "python3".to_string())
|
||||
}
|
||||
|
||||
fn interop_required() -> bool {
|
||||
std::env::var("CLAWHDF5_REQUIRE_INTEROP").is_ok_and(|v| v == "1")
|
||||
}
|
||||
|
||||
fn python_available() -> bool {
|
||||
Command::new(python())
|
||||
.args(["-c", "import h5py"])
|
||||
.output()
|
||||
.map(|o| o.status.success())
|
||||
.unwrap_or(false)
|
||||
}
|
||||
|
||||
fn run_python(script: &str) -> String {
|
||||
let output = Command::new(python())
|
||||
.args(["-c", script])
|
||||
.output()
|
||||
.expect("failed to run python");
|
||||
assert!(
|
||||
output.status.success(),
|
||||
"Python script failed:\nSTDOUT: {}\nSTDERR: {}",
|
||||
String::from_utf8_lossy(&output.stdout),
|
||||
String::from_utf8_lossy(&output.stderr)
|
||||
);
|
||||
String::from_utf8_lossy(&output.stdout).trim().to_string()
|
||||
}
|
||||
|
||||
/// The v4 chunk index type recorded in `name`'s layout message
|
||||
/// (1 single chunk, 2 implicit, 3 fixed array, 4 extensible array, 5 B-tree v2).
|
||||
fn chunk_index_type(path: &std::path::Path, name: &str) -> u8 {
|
||||
let data = std::fs::read(path).unwrap();
|
||||
let sb = Superblock::parse(&data, 0).unwrap();
|
||||
let addr = clawhdf5_format::group_v2::resolve_path_any(&data, &sb, name).unwrap();
|
||||
let hdr = ObjectHeader::parse(&data, addr as usize, sb.offset_size, sb.length_size).unwrap();
|
||||
let msg = hdr
|
||||
.messages
|
||||
.iter()
|
||||
.find(|m| m.msg_type == MessageType::DataLayout)
|
||||
.expect("layout message");
|
||||
match DataLayout::parse(&msg.data, sb.offset_size, sb.length_size).unwrap() {
|
||||
DataLayout::Chunked {
|
||||
version: 4,
|
||||
chunk_index_type: Some(t),
|
||||
..
|
||||
} => t,
|
||||
other => panic!("{name}: expected a v4 chunked layout, got {other:?}"),
|
||||
}
|
||||
}
|
||||
|
||||
struct Case {
|
||||
name: &'static str,
|
||||
/// Python keyword arguments to `create_dataset` besides `data`.
|
||||
kwargs: &'static str,
|
||||
index_type: u8,
|
||||
}
|
||||
|
||||
const SHAPE: [u64; 2] = [37, 23];
|
||||
|
||||
const CASES: &[Case] = &[
|
||||
Case {
|
||||
name: "implicit",
|
||||
// Early allocation, no filters, fixed maximum: the implicit index.
|
||||
kwargs: "chunks=(5, 4), dcpl=early()",
|
||||
index_type: 2,
|
||||
},
|
||||
Case {
|
||||
name: "fixed_array",
|
||||
kwargs: "chunks=(5, 4), compression='gzip'",
|
||||
index_type: 3,
|
||||
},
|
||||
Case {
|
||||
name: "extensible_array",
|
||||
kwargs: "chunks=(5, 4), maxshape=(None, 23), compression='gzip'",
|
||||
index_type: 4,
|
||||
},
|
||||
Case {
|
||||
name: "btree2",
|
||||
kwargs: "chunks=(5, 4), maxshape=(None, None), compression='gzip'",
|
||||
index_type: 5,
|
||||
},
|
||||
Case {
|
||||
name: "single_chunk",
|
||||
kwargs: "chunks=(37, 23), compression='gzip'",
|
||||
index_type: 1,
|
||||
},
|
||||
Case {
|
||||
name: "single_chunk_unfiltered",
|
||||
kwargs: "chunks=(37, 23)",
|
||||
index_type: 1,
|
||||
},
|
||||
];
|
||||
|
||||
/// `(start, stride, count, block)` per dimension; the first few stay below
|
||||
/// half the dataset (bounding-box path), the rest exceed it (full path).
|
||||
/// `(start, stride, count, block)` of a 2-D hyperslab.
|
||||
type Hyperslab2 = ([u64; 2], [u64; 2], [u64; 2], [u64; 2]);
|
||||
|
||||
fn selections() -> Vec<Hyperslab2> {
|
||||
vec![
|
||||
([0, 0], [1, 1], [3, 23], [1, 1]), // ds[0:3]
|
||||
([7, 3], [1, 1], [9, 6], [1, 1]), // interior window across chunks
|
||||
([36, 22], [1, 1], [1, 1], [1, 1]), // last element (edge chunk)
|
||||
([2, 1], [3, 4], [4, 3], [1, 1]), // strided, small
|
||||
([0, 0], [1, 1], [30, 23], [1, 1]), // most rows
|
||||
([1, 0], [2, 1], [18, 23], [1, 1]), // every other row, spanning all
|
||||
([0, 2], [1, 1], [37, 20], [1, 1]), // columns 2..22 of every row
|
||||
([3, 1], [5, 3], [7, 7], [2, 2]), // strided blocks over everything
|
||||
]
|
||||
}
|
||||
|
||||
fn py_slice(start: u64, stride: u64, count: u64, block: u64) -> String {
|
||||
// Each case is expressible as a numpy index when block == 1; with a block
|
||||
// the selected indices are listed explicitly.
|
||||
let idx: Vec<String> = (0..count)
|
||||
.flat_map(|c| (0..block).map(move |b| start + c * stride + b))
|
||||
.map(|i| i.to_string())
|
||||
.collect();
|
||||
format!("[{}]", idx.join(","))
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn partial_hyperslabs_of_every_v4_chunk_index_match_h5py() {
|
||||
if !python_available() {
|
||||
assert!(
|
||||
!interop_required(),
|
||||
"CLAWHDF5_REQUIRE_INTEROP=1 but python3 with h5py is not available"
|
||||
);
|
||||
eprintln!("SKIP: python3 with h5py not available");
|
||||
return;
|
||||
}
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let path = dir.path().join("v4_index_selection.h5");
|
||||
let path_str = path.display().to_string();
|
||||
|
||||
// h5py writes the file, then reads every selection back and prints the
|
||||
// values, one line per (case, selection).
|
||||
let mut script = format!(
|
||||
"import h5py, numpy as np\n\
|
||||
def early():\n\
|
||||
\x20 p = h5py.h5p.create(h5py.h5p.DATASET_CREATE)\n\
|
||||
\x20 p.set_alloc_time(h5py.h5d.ALLOC_TIME_EARLY)\n\
|
||||
\x20 return p\n\
|
||||
data = (np.arange({n}, dtype='<i4') * 7 - 1000).reshape({r}, {c})\n\
|
||||
with h5py.File(r'{path_str}', 'w', libver='latest') as f:\n",
|
||||
n = SHAPE[0] * SHAPE[1],
|
||||
r = SHAPE[0],
|
||||
c = SHAPE[1],
|
||||
);
|
||||
for case in CASES {
|
||||
script += &format!(
|
||||
" f.create_dataset('{}', data=data, {})\n",
|
||||
case.name, case.kwargs
|
||||
);
|
||||
}
|
||||
script += &format!("with h5py.File(r'{path_str}', 'r') as f:\n");
|
||||
for case in CASES {
|
||||
for (start, stride, count, block) in selections() {
|
||||
let rows = py_slice(start[0], stride[0], count[0], block[0]);
|
||||
let cols = py_slice(start[1], stride[1], count[1], block[1]);
|
||||
script += &format!(
|
||||
" print(' '.join(map(str, f['{}'][{rows}][:, {cols}].ravel())))\n",
|
||||
case.name
|
||||
);
|
||||
}
|
||||
}
|
||||
let out = run_python(&script);
|
||||
let mut expected = out.lines();
|
||||
|
||||
let file = File::open(&path).unwrap();
|
||||
for case in CASES {
|
||||
assert_eq!(
|
||||
chunk_index_type(&path, case.name),
|
||||
case.index_type,
|
||||
"{}: h5py did not produce the intended chunk index",
|
||||
case.name
|
||||
);
|
||||
let ds = file.dataset(case.name).unwrap();
|
||||
assert_eq!(ds.shape().unwrap(), SHAPE);
|
||||
for (start, stride, count, block) in selections() {
|
||||
let sel = Selection::Hyperslab {
|
||||
start: start.to_vec(),
|
||||
stride: stride.to_vec(),
|
||||
count: count.to_vec(),
|
||||
block: block.to_vec(),
|
||||
};
|
||||
let want: Vec<i32> = expected
|
||||
.next()
|
||||
.expect("h5py printed too few lines")
|
||||
.split_whitespace()
|
||||
.map(|v| v.parse().unwrap())
|
||||
.collect();
|
||||
let raw = ds
|
||||
.read_selection(&sel)
|
||||
.unwrap_or_else(|e| panic!("{}: read_selection {sel:?} failed: {e}", case.name));
|
||||
let got: Vec<i32> = raw
|
||||
.as_chunks::<4>()
|
||||
.0
|
||||
.iter()
|
||||
.map(|b| i32::from_le_bytes(*b))
|
||||
.collect();
|
||||
assert_eq!(got, want, "{}: selection {sel:?}", case.name);
|
||||
assert_eq!(
|
||||
ds.read_i32_selection(&sel).unwrap(),
|
||||
want,
|
||||
"{}: read_i32_selection {sel:?}",
|
||||
case.name
|
||||
);
|
||||
}
|
||||
}
|
||||
assert!(expected.next().is_none(), "h5py printed extra lines");
|
||||
}
|
||||
Reference in New Issue
Block a user