quantumclaw/clawhdf5
3
1
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases 1 Wiki Activity

perf(format): back large read buffers with transparent huge pages

Browse Source 
A full read of a contiguous dataset is one memcpy from the mapped file,
yet ran at a quarter of h5py's speed on one thread: the fresh output Vec
took a page fault and a kernel page clear for every 4 KiB page written,
16384 per 64 MiB, costing several times the copy (the benchmark spent
6.2 s of 8 s in the kernel, 4.3M minor faults). numpy, so h5py, madvises
MADV_HUGEPAGE on allocations of 4 MiB or more; the typed readers' output,
the raw contiguous read and the chunk assembly buffer now do the same
(Linux only, libc as a Linux-only dependency; no-op otherwise).

New h5py comparison tests cover full and selection reads of contiguous
data for every 1-8-byte integer and float type, both byte orders, ranks
1-4, empty selections, and datasets past the 4 MiB threshold.

Co-Authored-By: Claude Opus 5.5 (1M context) <[email protected]>
This commit is contained in:
osobh
2026-09-26 08:12:55 -05:00
co-authored by Claude Opus 5.5
parent 63648c7000
commit 78c769f179
7 changed files with 521 additions and 11 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/clawhdf5-format/src/data_read.rs
+16 -11
View File
@@ -180,7 +180,9 @@ fn read_raw_data_full_impl(
});
}
ensure_len(file_data, addr, sz)?;
Ok(file_data[addr..addr + sz].to_vec())
let mut out = crate::bulk_alloc::vec_for_bulk(sz);
out.extend_from_slice(&file_data[addr..addr + sz]);
Ok(out)
}
DataLayout::Chunked { .. } => read_chunked_data(
file_data,
@@ -765,7 +767,7 @@ fn get_size(dt: &Datatype) -> usize {
fn native_le_to_vec<T: Copy>(raw: &[u8], count: usize) -> Vec<T> {
let bytes = count * core::mem::size_of::<T>();
debug_assert!(bytes <= raw.len());
let mut result: Vec<T> = Vec::with_capacity(count);
let mut result: Vec<T> = crate::bulk_alloc::vec_for_bulk(count);
// SAFETY: `result` has capacity for `count` values of `T`, i.e. `bytes`
// bytes; `raw` holds at least `bytes` bytes (callers derive `count` from
// `raw.len() / size_of::<T>()`); the regions cannot overlap because
@@ -803,7 +805,7 @@ pub fn read_as_f64(raw: &[u8], datatype: &Datatype) -> Result<Vec<f64>, FormatEr
}
let order = get_byte_order(datatype);
let mut result = Vec::with_capacity(count);
let mut result = crate::bulk_alloc::vec_for_bulk(count);
if let Datatype::FloatingPoint { .. } = datatype {
let format = FloatFormat::of(datatype)?;
for chunk in raw.chunks_exact(elem_size) {
@@ -955,7 +957,7 @@ pub fn read_as_i64(raw: &[u8], datatype: &Datatype) -> Result<Vec<i64>, FormatEr
}
let order = get_byte_order(datatype);
let mut result = Vec::with_capacity(count);
let mut result = crate::bulk_alloc::vec_for_bulk(count);
for i in 0..count {
let chunk = &raw[i * elem_size..(i + 1) * elem_size];
result.push(decode_scalar(chunk, datatype, &order)?.to_i64());
@@ -985,7 +987,7 @@ pub fn read_as_u64(raw: &[u8], datatype: &Datatype) -> Result<Vec<u64>, FormatEr
}
let count = raw.len() / elem_size;
let order = get_byte_order(datatype);
let mut result = Vec::with_capacity(count);
let mut result = crate::bulk_alloc::vec_for_bulk(count);
for i in 0..count {
let chunk = &raw[i * elem_size..(i + 1) * elem_size];
result.push(decode_scalar(chunk, datatype, &order)?.to_u64());
@@ -1018,14 +1020,17 @@ pub fn read_as_f32(raw: &[u8], datatype: &Datatype) -> Result<Vec<f32>, FormatEr
// Little-endian IEEE half precision (numpy float16): widen directly.
if is_native_le_float(datatype, FloatFormat::Half) {
let (halves, _) = raw[..count * 2].as_chunks::<2>();
return Ok(halves
.iter()
.map(|&b| f16_bits_to_f32(u16::from_le_bytes(b)))
.collect());
let mut result = crate::bulk_alloc::vec_for_bulk(count);
result.extend(
halves
.iter()
.map(|&b| f16_bits_to_f32(u16::from_le_bytes(b))),
);
return Ok(result);
}
let order = get_byte_order(datatype);
let mut result = Vec::with_capacity(count);
let mut result = crate::bulk_alloc::vec_for_bulk(count);
if let Datatype::FloatingPoint { .. } = datatype {
let format = FloatFormat::of(datatype)?;
for chunk in raw.chunks_exact(elem_size) {
@@ -1114,7 +1119,7 @@ pub fn read_as_i32(raw: &[u8], datatype: &Datatype) -> Result<Vec<i32>, FormatEr
}
let order = get_byte_order(datatype);
let mut result = Vec::with_capacity(count);
let mut result = crate::bulk_alloc::vec_for_bulk(count);
for i in 0..count {
let chunk = &raw[i * elem_size..(i + 1) * elem_size];
result.push(decode_scalar(chunk, datatype, &order)?.to_i32());