quantumclaw/clawhdf5
3
1
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 1 Wiki Activity

ci: lint all targets, run interop suites for real, compile benches

Browse Source 
- clippy --all-targets plus a clawhdf5-format feature matrix (parallel, lz4,
  zstd, pcodec, fast-checksum); fix the accumulated lint backlog in test,
  bench and feature-gated code (no behaviour changes).
- Install python3 + h5py/numpy/netCDF4/xarray in the CI container and set
  CLAWHDF5_REQUIRE_INTEROP=1, which makes a missing interop dependency a test
  failure. Every h5py/netCDF4 interop test used to skip silently in CI. Run
  the #[ignore]d writer_h5py_tests suite explicitly.
- cargo bench --no-run so benches can't rot; fix bench.rs and memory_bench.rs,
  which no longer compiled against the current strategy/consolidation APIs.
- Optional fuzz smoke run via CLAWHDF5_FUZZ_SECONDS.
- CHANGELOG and docs/known-issues.md updated.

Co-Authored-By: Claude Fable 5.1 <[email protected]>
This commit is contained in:
osobh
2026-09-19 05:36:22 -07:00
co-authored by Claude Fable 5.1
parent 706189c3ef
commit bbe1baa208
30 changed files with 588 additions and 405 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/clawhdf5-format/src/attribute.rs
+3 -4
View File
@@ -472,14 +472,13 @@ mod tests {
// Name padded to 8 bytes
data.extend_from_slice(name);
while data.len() % 8 != 0 || data.len() == 8 {
if data.len() % 8 != 0 || data.len() == 8 {
// Pad name to 8-byte boundary from start of name
let name_start = 8;
let name_padded = pad8(name_size);
while data.len() < name_start + name_padded {
data.push(0);
}
break;
}
// Datatype padded to 8 bytes
@@ -749,11 +748,11 @@ mod tests {
data.extend_from_slice(name);
data.extend_from_slice(&dt_bytes);
data.extend_from_slice(&ds_bytes);
data.extend_from_slice(&3.14f64.to_le_bytes());
data.extend_from_slice(&3.25f64.to_le_bytes());
let attr = AttributeMessage::parse(&data, 8).unwrap();
let vals = attr.read_as_f64().unwrap();
assert_eq!(vals, vec![3.14]);
assert_eq!(vals, vec![3.25]);
}
#[test]
crates/clawhdf5-format/src/btree_v2.rs
+1
View File
@@ -416,6 +416,7 @@ fn header_max_total_records(max_leaf_nrec: u64, depth: u16) -> u64 {
mod tests {
use super::*;
#[allow(clippy::too_many_arguments)]
fn build_btree_v2_header(
tree_type: u8,
node_size: u32,
crates/clawhdf5-format/src/chunked_read.rs
+4 -4
View File
@@ -1657,9 +1657,9 @@ mod tests {
let chunk_bytes = chunk_size_elems * elem_size; // full chunk allocation
// Write chunk data (full chunk size, padding with zeros)
for i in start..end {
for (i, value) in values.iter().enumerate().take(end).skip(start) {
let byte_offset = data_offset + (i - start) * elem_size;
file_data[byte_offset..byte_offset + 8].copy_from_slice(&values[i].to_le_bytes());
file_data[byte_offset..byte_offset + 8].copy_from_slice(&value.to_le_bytes());
}
chunk_infos.push(ChunkInfo {
@@ -1837,8 +1837,8 @@ mod tests {
for chunk_idx in 0..2 {
let start = chunk_idx * chunk_elems;
let mut chunk_bytes = Vec::new();
for i in start..start + chunk_elems {
chunk_bytes.extend_from_slice(&values[i].to_le_bytes());
for value in values.iter().skip(start).take(chunk_elems) {
chunk_bytes.extend_from_slice(&value.to_le_bytes());
}
let compressed = compress_chunk(&chunk_bytes, &pipeline, elem_size as u32).unwrap();
crates/clawhdf5-format/src/data_read.rs
+6 -6
View File
@@ -1763,11 +1763,11 @@ mod tests {
fn f16_bits(v: f32) -> u16 {
// Encode a few exact values used by the test.
match v {
x if x == 0.0 => 0x0000,
x if x == 1.0 => 0x3c00,
x if x == -2.0 => 0xc000,
x if x == 0.5 => 0x3800,
x if x == 65504.0 => 0x7bff, // f16 max
0.0 => 0x0000,
1.0 => 0x3c00,
-2.0 => 0xc000,
0.5 => 0x3800,
65504.0 => 0x7bff, // f16 max
_ => panic!("unsupported test value {v}"),
}
}
@@ -2186,7 +2186,7 @@ mod tests {
],
};
let mut raw = Vec::new();
raw.extend_from_slice(&3.14f64.to_le_bytes());
raw.extend_from_slice(&3.25f64.to_le_bytes());
raw.extend_from_slice(&42i32.to_le_bytes());
let field = read_compound_field(&raw, &dt, "id").unwrap();
crates/clawhdf5-format/src/dataspace.rs
+3 -15
View File
@@ -189,11 +189,7 @@ mod tests {
fn build_v1_dataspace(rank: u8, flags: u8, dims: &[u64], max_dims: Option<&[u64]>) -> Vec<u8> {
let length_size = 8u8;
let mut buf = Vec::new();
buf.push(1); // version
buf.push(rank);
buf.push(flags);
buf.push(0); // reserved
let mut buf = vec![1, rank, flags, 0]; // version, rank, flags, reserved
buf.extend_from_slice(&[0u8; 4]); // reserved(4)
for &d in dims {
buf.extend_from_slice(&d.to_le_bytes());
@@ -214,11 +210,7 @@ mod tests {
dims: &[u64],
max_dims: Option<&[u64]>,
) -> Vec<u8> {
let mut buf = Vec::new();
buf.push(2); // version
buf.push(rank);
buf.push(flags);
buf.push(type_byte);
let mut buf = vec![2, rank, flags, type_byte]; // version, rank, flags, type
for &d in dims {
buf.extend_from_slice(&d.to_le_bytes());
}
@@ -298,11 +290,7 @@ mod tests {
#[test]
fn v1_with_4byte_length() {
let mut buf = Vec::new();
buf.push(1); // version
buf.push(1); // rank
buf.push(0); // flags
buf.push(0); // reserved
let mut buf = vec![1, 1, 0, 0]; // version, rank, flags, reserved
buf.extend_from_slice(&[0u8; 4]); // reserved(4)
buf.extend_from_slice(&10u32.to_le_bytes()); // dim with length_size=4
let ds = Dataspace::parse(&buf, 4).unwrap();
crates/clawhdf5-format/src/filters.rs
+21 -15
View File
@@ -1045,24 +1045,30 @@ fn pcodec_compress(data: &[u8], element_size: usize) -> Result<Vec<u8>, FormatEr
match element_size {
4 => {
let nums: Vec<f32> = data
.chunks_exact(4)
.map(|b| f32::from_le_bytes(b.try_into().unwrap()))
.as_chunks::<4>()
.0
.iter()
.map(|b| f32::from_le_bytes(*b))
.collect();
simple_compress(&nums, &config)
.map_err(|e| FormatError::CompressionError(format!("pco: {e}")))
}
8 => {
let nums: Vec<f64> = data
.chunks_exact(8)
.map(|b| f64::from_le_bytes(b.try_into().unwrap()))
.as_chunks::<8>()
.0
.iter()
.map(|b| f64::from_le_bytes(*b))
.collect();
simple_compress(&nums, &config)
.map_err(|e| FormatError::CompressionError(format!("pco: {e}")))
}
_ => {
let nums: Vec<u32> = data
.chunks_exact(4)
.map(|b| u32::from_le_bytes(b.try_into().unwrap()))
.as_chunks::<4>()
.0
.iter()
.map(|b| u32::from_le_bytes(*b))
.collect();
simple_compress(&nums, &config)
.map_err(|e| FormatError::CompressionError(format!("pco: {e}")))
@@ -1092,11 +1098,7 @@ fn pcodec_decompress(
} else {
MAX_DECOMPRESS_SIZE
};
let n = if element_size != 0 {
limit_bytes / element_size
} else {
0
};
let n = limit_bytes.checked_div(element_size).unwrap_or(0);
match element_size {
4 => {
let mut buf = vec![0f32; n];
@@ -1543,8 +1545,10 @@ mod tests {
fn as_f32(bytes: &[u8]) -> Vec<f32> {
bytes
.chunks_exact(4)
.map(|c| f32::from_le_bytes(c.try_into().unwrap()))
.as_chunks::<4>()
.0
.iter()
.map(|c| f32::from_le_bytes(*c))
.collect()
}
@@ -1578,8 +1582,10 @@ mod tests {
fn as_f64(bytes: &[u8]) -> Vec<f64> {
bytes
.chunks_exact(8)
.map(|c| f64::from_le_bytes(c.try_into().unwrap()))
.as_chunks::<8>()
.0
.iter()
.map(|c| f64::from_le_bytes(*c))
.collect()
}
crates/clawhdf5-format/src/global_heap.rs
+1 -3
View File
@@ -184,9 +184,7 @@ mod tests {
buf.extend_from_slice(data);
// Pad to 8 bytes
let padded = pad8(data.len());
for _ in data.len()..padded {
buf.push(0);
}
buf.resize(buf.len() + (padded - data.len()), 0);
}
// Free space marker
crates/clawhdf5-format/src/link_message.rs
+4 -11
View File
@@ -413,11 +413,8 @@ mod tests {
#[test]
fn soft_link() {
let target = "/group1/dataset";
let mut data = Vec::new();
data.push(1); // version
data.push(0x08); // flags: bit 3 = link type present, name size = 1 byte (bits 0-1 = 0)
data.push(1); // link type = soft
data.push(4); // name length = 4
// version, flags (bit 3 = link type present, name size = 1 byte), link type = soft, name length = 4
let mut data = vec![1, 0x08, 1, 4];
data.extend_from_slice(b"link");
data.extend_from_slice(&(target.len() as u16).to_le_bytes());
data.extend_from_slice(target.as_bytes());
@@ -455,12 +452,8 @@ mod tests {
#[test]
fn invalid_link_type() {
let mut data = Vec::new();
data.push(1); // version
data.push(0x08); // flags: bit 3 = link type present
data.push(99); // invalid link type
data.push(1); // name length = 1
data.push(b'x');
// version, flags (bit 3 = link type present), invalid link type = 99, name length = 1, name = 'x'
let data = vec![1, 0x08, 99, 1, b'x'];
let err = LinkMessage::parse(&data, 8).unwrap_err();
assert_eq!(err, FormatError::InvalidLinkType(99));
}
crates/clawhdf5-format/src/selection.rs
+1 -1
View File
@@ -509,7 +509,7 @@ mod tests {
#[test]
fn selection_slice_1d() {
let sel = Selection::slice(&[5..15]);
let sel = Selection::slice(std::slice::from_ref(&(5..15)));
assert_eq!(sel.num_elements(&[100]), 10);
assert_eq!(sel.output_shape(&[100]), vec![10]);
}
crates/clawhdf5-format/tests/integration_test.rs
+27 -25
View File
@@ -343,7 +343,11 @@ fn attrs_h5_dataset_scale() {
let scale_attr = find_attribute(&attrs, "scale").expect("scale attr not found");
let vals = scale_attr.read_as_f64().unwrap();
assert_eq!(vals.len(), 1);
assert!((vals[0] - 3.14).abs() < 1e-10);
// 3.14 here is the literal value baked into the binary fixture (fixtures/attrs.h5),
// not an arbitrary sample value, so it cannot be swapped for another constant.
#[allow(clippy::approx_constant)]
let expected = 3.14;
assert!((vals[0] - expected).abs() < 1e-10);
}
#[test]
@@ -556,8 +560,8 @@ fn chunked_deflate_read_values() {
let (raw, datatype, _) = read_chunked_dataset(file_data, "data");
let values = read_as_f64(&raw, &datatype).unwrap();
assert_eq!(values.len(), 100);
for i in 0..100 {
assert_eq!(values[i], i as f64, "mismatch at index {i}");
for (i, &v) in values.iter().enumerate() {
assert_eq!(v, i as f64, "mismatch at index {i}");
}
}
@@ -567,8 +571,8 @@ fn chunked_shuffle_deflate_read_values() {
let (raw, datatype, _) = read_chunked_dataset(file_data, "data");
let values = read_as_f64(&raw, &datatype).unwrap();
assert_eq!(values.len(), 100);
for i in 0..100 {
assert_eq!(values[i], i as f64, "mismatch at index {i}");
for (i, &v) in values.iter().enumerate() {
assert_eq!(v, i as f64, "mismatch at index {i}");
}
}
@@ -578,8 +582,8 @@ fn chunked_fletcher32_read_values() {
let (raw, datatype, _) = read_chunked_dataset(file_data, "data");
let values = read_as_f64(&raw, &datatype).unwrap();
assert_eq!(values.len(), 100);
for i in 0..100 {
assert_eq!(values[i], i as f64, "mismatch at index {i}");
for (i, &v) in values.iter().enumerate() {
assert_eq!(v, i as f64, "mismatch at index {i}");
}
}
@@ -589,11 +593,10 @@ fn chunked_2d_read_values() {
let (raw, datatype, _) = read_chunked_dataset(file_data, "matrix");
let values = read_as_f32(&raw, &datatype).unwrap();
assert_eq!(values.len(), 60);
for i in 0..60 {
for (i, &v) in values.iter().enumerate() {
assert!(
(values[i] - i as f32).abs() < 1e-6,
"mismatch at index {i}: got {}",
values[i]
(v - i as f32).abs() < 1e-6,
"mismatch at index {i}: got {v}"
);
}
}
@@ -604,8 +607,8 @@ fn chunked_large_read_values() {
let (raw, datatype, _) = read_chunked_dataset(file_data, "big");
let values = read_as_i32(&raw, &datatype).unwrap();
assert_eq!(values.len(), 1000);
for i in 0..1000 {
assert_eq!(values[i], i as i32, "mismatch at index {i}");
for (i, &v) in values.iter().enumerate() {
assert_eq!(v, i as i32, "mismatch at index {i}");
}
}
@@ -615,8 +618,8 @@ fn chunked_nofilter_read_values() {
let (raw, datatype, _) = read_chunked_dataset(file_data, "raw");
let values = read_as_f64(&raw, &datatype).unwrap();
assert_eq!(values.len(), 50);
for i in 0..50 {
assert_eq!(values[i], i as f64, "mismatch at index {i}");
for (i, &v) in values.iter().enumerate() {
assert_eq!(v, i as f64, "mismatch at index {i}");
}
}
@@ -646,8 +649,8 @@ fn v4_implicit_read() {
let (raw, datatype, _) = read_chunked_dataset(file_data, "data");
let values = read_as_f64(&raw, &datatype).unwrap();
assert_eq!(values.len(), 100);
for i in 0..100 {
assert_eq!(values[i], i as f64, "mismatch at index {i}");
for (i, &v) in values.iter().enumerate() {
assert_eq!(v, i as f64, "mismatch at index {i}");
}
}
@@ -657,8 +660,8 @@ fn v4_fixed_array_read() {
let (raw, datatype, _) = read_chunked_dataset(file_data, "data");
let values = read_as_f64(&raw, &datatype).unwrap();
assert_eq!(values.len(), 100);
for i in 0..100 {
assert_eq!(values[i], i as f64, "mismatch at index {i}");
for (i, &v) in values.iter().enumerate() {
assert_eq!(v, i as f64, "mismatch at index {i}");
}
}
@@ -871,11 +874,10 @@ fn v4_2d_fixed_array_read() {
let (raw, datatype, _) = read_chunked_dataset(file_data, "matrix");
let values = read_as_f32(&raw, &datatype).unwrap();
assert_eq!(values.len(), 60);
for i in 0..60 {
for (i, &v) in values.iter().enumerate() {
assert!(
(values[i] - i as f32).abs() < 1e-6,
"mismatch at index {i}: got {}",
values[i]
(v - i as f32).abs() < 1e-6,
"mismatch at index {i}: got {v}"
);
}
}
@@ -1272,7 +1274,7 @@ fn write_roundtrip_scalar_f64_attr() {
let mut fw = FileWriter::new();
fw.create_dataset("data")
.with_f64_data(&[1.0])
.set_attr("scale", AttrValue::F64(3.14));
.set_attr("scale", AttrValue::F64(3.25));
let bytes = fw.finish().unwrap();
let sig = find_signature(&bytes).unwrap();
@@ -1283,7 +1285,7 @@ fn write_roundtrip_scalar_f64_attr() {
let scale = find_attribute(&attrs, "scale").expect("scale attr not found");
let vals = scale.read_as_f64().unwrap();
assert_eq!(vals.len(), 1);
assert!((vals[0] - 3.14).abs() < 1e-10);
assert!((vals[0] - 3.25).abs() < 1e-10);
}
#[test]
crates/clawhdf5-format/tests/reference_tests.rs
+9 -4
View File
@@ -180,15 +180,20 @@ print('ok')
let output = match output {
Ok(o) if o.status.success() => o,
_ => {
// CI sets CLAWHDF5_REQUIRE_INTEROP=1 so this can't silently skip.
assert!(
!std::env::var("CLAWHDF5_REQUIRE_INTEROP").is_ok_and(|v| v == "1"),
"CLAWHDF5_REQUIRE_INTEROP=1 but python3 with h5py is not available"
);
eprintln!("skipping h5py_object_reference_roundtrip: python3+h5py not available");
return;
}
};
let stdout = String::from_utf8(output.stdout).unwrap();
if !stdout.trim().contains("ok") {
eprintln!("skipping h5py_object_reference_roundtrip: h5py script failed");
return;
}
assert!(
stdout.trim().contains("ok"),
"h5py reference-file generator did not report ok: {stdout}"
);
// Read the file and parse object references
let file_data = std::fs::read(&path).unwrap();