//! Regression tests for writer metadata bugs that produced files libhdf5 //! refuses (or reads differently from us), plus the reader-side counterparts. //! //! The plain tests check the bytes we write with our own parser. The //! `#[ignore]`d ones are the interop half: they open what we write in h5py //! (`CLAWHDF5_PYTHON`, as in `writer_h5py_tests.rs`) and run `h5dump` over it. use clawhdf5_format::data_layout::DataLayout; use clawhdf5_format::file_writer::{AttrValue, FileWriter}; use clawhdf5_format::group_v2::resolve_path_any; use clawhdf5_format::message_type::MessageType; use clawhdf5_format::object_header::ObjectHeader; use clawhdf5_format::signature; use clawhdf5_format::superblock::Superblock; use clawhdf5_format::type_builders::make_u8_type; // ---- helpers ---- fn header_at(bytes: &[u8], path: &str) -> (Superblock, ObjectHeader) { let sig = signature::find_signature(bytes).unwrap(); let sb = Superblock::parse(bytes, sig).unwrap(); let addr = if path == "/" { sb.root_group_address } else { resolve_path_any(bytes, &sb, path).unwrap() }; let oh = ObjectHeader::parse(bytes, addr as usize, sb.offset_size, sb.length_size).unwrap(); (sb, oh) } fn layout_of(bytes: &[u8], path: &str) -> DataLayout { let (sb, oh) = header_at(bytes, path); let msg = oh .messages .iter() .find(|m| m.msg_type == MessageType::DataLayout) .unwrap(); DataLayout::parse(&msg.data, sb.offset_size, sb.length_size).unwrap() } fn python() -> String { std::env::var("CLAWHDF5_PYTHON").unwrap_or_else(|_| "python3".to_string()) } fn write_tmp(name: &str, bytes: &[u8]) -> std::path::PathBuf { let path = std::env::temp_dir().join(format!("clawhdf5_writer_meta_{name}.h5")); std::fs::write(&path, bytes).unwrap(); path } /// Run `script` (with `path` bound to the file) under h5py; return stdout. fn h5py(path: &std::path::Path, script: &str) -> String { let full = format!( "import h5py, numpy as np, json\npath = {:?}\n{script}", path.display().to_string() ); let o = std::process::Command::new(python()) .args(["-c", &full]) .output() .expect("python interpreter"); assert!( o.status.success(), "h5py failed: {}", String::from_utf8_lossy(&o.stderr) ); String::from_utf8(o.stdout).unwrap().trim().to_string() } /// `h5dump` must read the whole file without error. fn h5dump_ok(path: &std::path::Path) { let o = std::process::Command::new("h5dump") .arg(path) .output() .expect("h5dump"); assert!( o.status.success(), "h5dump failed: {}{}", String::from_utf8_lossy(&o.stdout), String::from_utf8_lossy(&o.stderr) ); } fn u8_ramp(n: usize) -> Vec { (0..n).map(|i| (i % 251) as u8).collect() } // ---- 1. object header message size limit ---- #[test] fn attribute_too_big_for_a_header_message_is_an_error() { // Measured: a 70000-byte attribute was written with its message size // wrapped to 16 bits, and libhdf5 refused the whole root group. let mut fw = FileWriter::new(); fw.set_root_attr( "a", AttrValue::Raw { datatype: make_u8_type(), shape: vec![70_000], data: u8_ramp(70_000), }, ); assert!(fw.finish().is_err()); // 65500 bytes still fits and still works. let mut fw = FileWriter::new(); fw.set_root_attr( "a", AttrValue::Raw { datatype: make_u8_type(), shape: vec![65_500], data: u8_ramp(65_500), }, ); let bytes = fw.finish().unwrap(); let (sb, oh) = header_at(&bytes, "/"); let attrs = clawhdf5_format::attribute::extract_attributes(&oh, sb.length_size).unwrap(); assert_eq!(attrs[0].raw_data, u8_ramp(65_500)); } #[test] fn compact_layout_falls_back_to_contiguous_past_the_message_limit() { // Layout message = 4 bytes + data; data may be at most 65531 bytes. for (n, compact) in [(65_531, true), (65_532, false), (65_534, false)] { let mut fw = FileWriter::new(); fw.create_dataset("d").with_u8_data(&u8_ramp(n)).compact(); let bytes = fw.finish().unwrap(); match layout_of(&bytes, "d") { DataLayout::Compact { data } => { assert!(compact, "{n} bytes must not be compact"); assert_eq!(data, u8_ramp(n)); } DataLayout::Contiguous { .. } => assert!(!compact, "{n} bytes should be compact"), other => panic!("unexpected layout {other:?}"), } } } #[test] #[ignore = "requires Python h5py module and h5dump"] fn h5py_reads_compact_datasets_at_the_limit() { for n in [65_531usize, 65_534] { let mut fw = FileWriter::new(); fw.create_dataset("d").with_u8_data(&u8_ramp(n)).compact(); let path = write_tmp(&format!("compact_{n}"), &fw.finish().unwrap()); let out = h5py( &path, "f = h5py.File(path, 'r'); v = f['d'][()]\n\ print(bool((v == (np.arange(v.size) % 251).astype(np.uint8)).all()), v.size)", ); assert_eq!(out, format!("True {n}")); h5dump_ok(&path); } }