feat(tools): h5rs, pure-Rust HDF5 tools (ls, dump, stat, diff, check)
New workspace crate clawhdf5-tools with one binary, h5rs, built only on the clawhdf5 facade and clawhdf5-format (no libhdf5, no C): - ls [-r] [-v] FILE[/path]: h5ls's listing (same text in its first two columns) plus the datatype; -v adds address, link count, layout and chunk index, chunk size, storage, filters, datatype and attributes. - dump [--json] [-A] [-p] [-d PATH] FILE: h5dump DDL (byte-identical to h5dump 1.14.6 on the test files) or hdf5-json. - stat FILE: h5stat's object/link/rank/layout/filter/attribute counts, raw data and total size. - diff [-r] [-q] [-d D] [-p R] A B [OBJ1 [OBJ2]]: structural and value differences, exit 0/1/2 like h5diff. - check [--data] FILE: walks every object, parses every message, verifies the checksums of every v2+ structure (including the fractal heap blocks the library never checks), checks chunk indexes against their datasets and raw data for out-of-file or overlapping extents; every problem with its address. Values over --max-bytes are reported, not read; dense-storage heaps are verified before objects are read from them; panics are caught (exit 3). Tests compare with h5ls, h5stat, h5dump and h5diff and with h5py's values, and flip the checksum of every checksummed structure in a v1.14-format file. Co-Authored-By: Claude Opus 5.5 (1M context) <[email protected]>
This commit is contained in:
osobh
@@ -0,0 +1,385 @@
|
||||
//! `h5rs ls`: list a file's objects like h5ls.
|
||||
|
||||
use std::collections::HashMap;
|
||||
|
||||
use clawhdf5_format::attribute::AttributeMessage;
|
||||
use clawhdf5_format::dataspace::DataspaceType;
|
||||
use clawhdf5_format::object_header::ObjectHeader;
|
||||
|
||||
use crate::cli::{Args, Out};
|
||||
use crate::h5::{H5, Kind, Link, LinkKind, split_file_arg};
|
||||
use crate::info::{self, DsInfo};
|
||||
use crate::value::{self, Decoder};
|
||||
|
||||
pub const USAGE: &str = "\
|
||||
usage: h5rs ls [-r] [-v] [--max-bytes N] FILE[/OBJECT]
|
||||
|
||||
List the objects in FILE (or in the group OBJECT), one per line, like h5ls:
|
||||
name, kind, shape and, for datasets, the datatype.
|
||||
|
||||
-r, --recursive list every object below, with full paths
|
||||
-v, --verbose also print address, link count, layout, chunking,
|
||||
storage, filters, datatype and attributes
|
||||
--max-bytes N largest attribute value decoded for -v (default 1 GiB)
|
||||
|
||||
Exit status: 0 listed, 1 some object could not be described, 2 error.";
|
||||
|
||||
struct Ls<'a> {
|
||||
h5: &'a H5,
|
||||
verbose: bool,
|
||||
kinds: HashMap<u64, Kind>,
|
||||
problems: usize,
|
||||
}
|
||||
|
||||
pub fn run(args: &mut Args, out: &mut Out) -> std::io::Result<i32> {
|
||||
let mut recursive = false;
|
||||
let mut verbose = false;
|
||||
let mut max_bytes = None;
|
||||
let mut target = None;
|
||||
while let Some(a) = args.next() {
|
||||
match a.as_str() {
|
||||
"-r" | "--recursive" => recursive = true,
|
||||
"-v" | "--verbose" => verbose = true,
|
||||
"-rv" | "-vr" => {
|
||||
recursive = true;
|
||||
verbose = true;
|
||||
}
|
||||
"--max-bytes" => match args.number() {
|
||||
Some(n) => max_bytes = Some(n),
|
||||
None => return args.usage_error(out, "--max-bytes needs a number", USAGE),
|
||||
},
|
||||
"-h" | "--help" => {
|
||||
writeln!(out.o, "{USAGE}")?;
|
||||
return Ok(0);
|
||||
}
|
||||
s if s.starts_with('-') && s.len() > 1 => {
|
||||
return args.usage_error(out, &format!("unknown option {a}"), USAGE);
|
||||
}
|
||||
_ if target.is_none() => target = Some(a),
|
||||
_ => return args.usage_error(out, &format!("unexpected argument {a}"), USAGE),
|
||||
}
|
||||
}
|
||||
let Some(target) = target else {
|
||||
return args.usage_error(out, "missing FILE", USAGE);
|
||||
};
|
||||
let (file, obj) = split_file_arg(&target);
|
||||
let mut h5 = match H5::open(std::path::Path::new(&file)) {
|
||||
Ok(h) => h,
|
||||
Err(e) => {
|
||||
writeln!(out.e, "h5rs ls: {e}")?;
|
||||
return Ok(2);
|
||||
}
|
||||
};
|
||||
if let Some(m) = max_bytes {
|
||||
h5.max_bytes = m;
|
||||
}
|
||||
let mut ls = Ls {
|
||||
h5: &h5,
|
||||
verbose,
|
||||
kinds: HashMap::new(),
|
||||
problems: 0,
|
||||
};
|
||||
let obj_path = obj.unwrap_or_else(|| "/".into());
|
||||
let addr = match h5.resolve(&obj_path) {
|
||||
Ok(a) => a,
|
||||
Err(e) => {
|
||||
writeln!(out.e, "h5rs ls: {obj_path}: not found: {e}")?;
|
||||
return Ok(2);
|
||||
}
|
||||
};
|
||||
let header = match h5.header(addr) {
|
||||
Ok(h) => h,
|
||||
Err(e) => {
|
||||
writeln!(out.e, "h5rs ls: {obj_path}: {e}")?;
|
||||
return Ok(2);
|
||||
}
|
||||
};
|
||||
let is_root = obj_path.trim_matches('/').is_empty();
|
||||
if Kind::of(&header) != Kind::Group && !is_root {
|
||||
// h5ls names a single object by its base name, or with -r by its
|
||||
// path as given (without the leading slash).
|
||||
let name = if recursive {
|
||||
obj_path.trim_start_matches('/').to_string()
|
||||
} else {
|
||||
obj_path.rsplit('/').next().unwrap_or(&obj_path).to_string()
|
||||
};
|
||||
ls.object_line(out, &name, &obj_path, addr, &Ok(header))?;
|
||||
} else if recursive {
|
||||
let start = if is_root { "/" } else { "" };
|
||||
let mut io_err = None;
|
||||
let res = h5.walk_from(addr, start, &mut |item| {
|
||||
if io_err.is_some() || (item.link.is_none() && !is_root) {
|
||||
return; // listing a group's contents, not the group
|
||||
}
|
||||
let path = if item.path.is_empty() { "/" } else { item.path };
|
||||
let full = if is_root {
|
||||
path.to_string()
|
||||
} else {
|
||||
format!("{}{path}", obj_path.trim_end_matches('/'))
|
||||
};
|
||||
if let Err(e) = ls.item(
|
||||
out,
|
||||
path,
|
||||
&full,
|
||||
item.link,
|
||||
item.addr,
|
||||
item.first_path,
|
||||
item.header,
|
||||
) {
|
||||
io_err = Some(e);
|
||||
}
|
||||
});
|
||||
if let Some(e) = io_err {
|
||||
return Err(e);
|
||||
}
|
||||
if let Err(e) = res {
|
||||
writeln!(out.e, "h5rs ls: {e}")?;
|
||||
ls.problems += 1;
|
||||
}
|
||||
} else {
|
||||
match h5.links(&header) {
|
||||
Ok(links) => {
|
||||
for l in &links {
|
||||
let (a, hdr) = match l.kind {
|
||||
LinkKind::Hard(a) => (Some(a), Some(h5.header(a))),
|
||||
_ => (None, None),
|
||||
};
|
||||
let full = format!("{}/{}", obj_path.trim_end_matches('/'), l.name);
|
||||
ls.item(out, &l.name, &full, Some(l), a, None, hdr.as_ref())?;
|
||||
}
|
||||
}
|
||||
Err(e) => {
|
||||
writeln!(out.e, "h5rs ls: {obj_path}: cannot list group: {e}")?;
|
||||
ls.problems += 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
Ok(if ls.problems > 0 { 1 } else { 0 })
|
||||
}
|
||||
|
||||
impl Ls<'_> {
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
fn item(
|
||||
&mut self,
|
||||
out: &mut Out,
|
||||
name: &str,
|
||||
full: &str,
|
||||
link: Option<&Link>,
|
||||
addr: Option<u64>,
|
||||
first: Option<&str>,
|
||||
header: Option<&crate::h5::Result<ObjectHeader>>,
|
||||
) -> std::io::Result<()> {
|
||||
if let Some(l) = link {
|
||||
match &l.kind {
|
||||
LinkKind::Soft(t) => return writeln!(out.o, "{name:<24} Soft Link {{{t}}}"),
|
||||
LinkKind::External { file, path } => {
|
||||
return writeln!(out.o, "{name:<24} External Link {{{file}/{path}}}");
|
||||
}
|
||||
LinkKind::UserDefined(t) => {
|
||||
return writeln!(out.o, "{name:<24} User-defined link (type {t})");
|
||||
}
|
||||
LinkKind::Hard(_) => {}
|
||||
}
|
||||
}
|
||||
let Some(addr) = addr else { return Ok(()) };
|
||||
if let Some(first) = first {
|
||||
let k = self.kinds.get(&addr).copied().unwrap_or(Kind::Unknown);
|
||||
return writeln!(out.o, "{name:<24} {}, same as {first}", k.name());
|
||||
}
|
||||
match header {
|
||||
Some(h) => self.object_line(out, name, full, addr, h),
|
||||
None => Ok(()),
|
||||
}
|
||||
}
|
||||
|
||||
fn object_line(
|
||||
&mut self,
|
||||
out: &mut Out,
|
||||
name: &str,
|
||||
full: &str,
|
||||
addr: u64,
|
||||
header: &crate::h5::Result<ObjectHeader>,
|
||||
) -> std::io::Result<()> {
|
||||
let h = match header {
|
||||
Ok(h) => h,
|
||||
Err(e) => {
|
||||
self.problems += 1;
|
||||
writeln!(out.o, "{name:<24} ** error **")?;
|
||||
return writeln!(out.e, "h5rs ls: {name}: {e}");
|
||||
}
|
||||
};
|
||||
let kind = Kind::of(h);
|
||||
self.kinds.insert(addr, kind);
|
||||
match kind {
|
||||
Kind::Dataset => {
|
||||
let info = DsInfo::read(self.h5, full, h);
|
||||
let shape = match &info.ds {
|
||||
Ok(ds) => info::shape_text(ds, self.verbose),
|
||||
Err(_) => "{?}".into(),
|
||||
};
|
||||
if self.verbose {
|
||||
writeln!(out.o, "{name:<24} Dataset {shape}")?;
|
||||
} else {
|
||||
let t = match &info.dt {
|
||||
Ok(dt) => crate::dtype::short(dt),
|
||||
Err(_) => "?".into(),
|
||||
};
|
||||
writeln!(out.o, "{name:<24} Dataset {shape} {t}")?;
|
||||
}
|
||||
for e in [
|
||||
info.dt.as_ref().err(),
|
||||
info.ds.as_ref().err(),
|
||||
info.layout.as_ref().err(),
|
||||
info.filters.as_ref().err(),
|
||||
]
|
||||
.into_iter()
|
||||
.flatten()
|
||||
{
|
||||
self.problems += 1;
|
||||
writeln!(out.e, "h5rs ls: {name}: {e}")?;
|
||||
}
|
||||
if self.verbose {
|
||||
self.dataset_details(out, addr, h, &info)?;
|
||||
}
|
||||
}
|
||||
k => {
|
||||
writeln!(out.o, "{name:<24} {}", k.name())?;
|
||||
if self.verbose {
|
||||
writeln!(out.o, " Address: {addr}")?;
|
||||
writeln!(out.o, " Links: {}", info::link_count(h))?;
|
||||
if k == Kind::Datatype
|
||||
&& let Ok(dt) = self.h5.datatype(h)
|
||||
{
|
||||
writeln!(out.o, " Type: {}", crate::dtype::long(&dt))?;
|
||||
}
|
||||
self.attributes(out, name, h)?;
|
||||
}
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn dataset_details(
|
||||
&mut self,
|
||||
out: &mut Out,
|
||||
addr: u64,
|
||||
h: &ObjectHeader,
|
||||
info: &DsInfo,
|
||||
) -> std::io::Result<()> {
|
||||
writeln!(out.o, " Address: {addr}")?;
|
||||
writeln!(out.o, " Links: {}", info::link_count(h))?;
|
||||
if let Ok(l) = &info.layout {
|
||||
let idx = info::chunk_index_name(l);
|
||||
if idx.is_empty() {
|
||||
writeln!(out.o, " Layout: {}", info::layout_name(l))?;
|
||||
} else {
|
||||
writeln!(
|
||||
out.o,
|
||||
" Layout: {} ({idx} index)",
|
||||
info::layout_name(l)
|
||||
)?;
|
||||
}
|
||||
if let clawhdf5_format::data_layout::DataLayout::Chunked {
|
||||
chunk_dimensions, ..
|
||||
} = l
|
||||
{
|
||||
let rank = chunk_dimensions.len().saturating_sub(1);
|
||||
let dims = &chunk_dimensions[..rank];
|
||||
let esize = info
|
||||
.dt
|
||||
.as_ref()
|
||||
.map(|d| u64::from(d.type_size()))
|
||||
.unwrap_or(0);
|
||||
let bytes = dims
|
||||
.iter()
|
||||
.try_fold(esize, |a, &d| a.checked_mul(u64::from(d)))
|
||||
.map(|b| b.to_string())
|
||||
.unwrap_or_else(|| "?".into());
|
||||
writeln!(
|
||||
out.o,
|
||||
" Chunks: {{{}}} {bytes} bytes",
|
||||
dims.iter()
|
||||
.map(|d| d.to_string())
|
||||
.collect::<Vec<_>>()
|
||||
.join(", ")
|
||||
)?;
|
||||
}
|
||||
}
|
||||
if info.external {
|
||||
writeln!(out.o, " External: raw data in external files")?;
|
||||
}
|
||||
let logical = info.logical_bytes();
|
||||
match (logical, info::allocated_bytes(self.h5, info)) {
|
||||
(Some(l), Ok(a)) => {
|
||||
if a > 0 {
|
||||
writeln!(
|
||||
out.o,
|
||||
" Storage: {l} logical bytes, {a} allocated bytes, {:.2}% utilization",
|
||||
l as f64 * 100.0 / a as f64
|
||||
)?;
|
||||
} else {
|
||||
writeln!(out.o, " Storage: {l} logical bytes, 0 allocated bytes")?;
|
||||
}
|
||||
}
|
||||
(_, Err(e)) => {
|
||||
self.problems += 1;
|
||||
writeln!(out.e, "h5rs ls: {e}")?;
|
||||
}
|
||||
_ => {}
|
||||
}
|
||||
if let Ok(Some(p)) = &info.filters {
|
||||
for (i, f) in p.filters.iter().enumerate() {
|
||||
writeln!(out.o, " Filter-{i}: {}", info::filter_text(f))?;
|
||||
}
|
||||
}
|
||||
if let Ok(dt) = &info.dt {
|
||||
writeln!(out.o, " Type: {}", crate::dtype::long(dt))?;
|
||||
}
|
||||
self.attributes(out, "", h)
|
||||
}
|
||||
|
||||
fn attributes(&mut self, out: &mut Out, name: &str, h: &ObjectHeader) -> std::io::Result<()> {
|
||||
let (attrs, errs) = match self.h5.attributes(h) {
|
||||
Ok(x) => x,
|
||||
Err(e) => {
|
||||
self.problems += 1;
|
||||
return writeln!(out.e, "h5rs ls: {name}: {e}");
|
||||
}
|
||||
};
|
||||
for e in errs {
|
||||
self.problems += 1;
|
||||
writeln!(out.e, "h5rs ls: {name}: attribute: {e}")?;
|
||||
}
|
||||
for a in &attrs {
|
||||
self.attribute(out, a)?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn attribute(&mut self, out: &mut Out, a: &AttributeMessage) -> std::io::Result<()> {
|
||||
let shape = match a.dataspace.space_type {
|
||||
DataspaceType::Scalar => "scalar".to_string(),
|
||||
DataspaceType::Null => "null".to_string(),
|
||||
DataspaceType::Simple => info::shape_text(&a.dataspace, false),
|
||||
};
|
||||
writeln!(out.o, " Attribute: {} {shape}", a.name)?;
|
||||
writeln!(
|
||||
out.o,
|
||||
" Type: {}",
|
||||
crate::dtype::long(&a.datatype)
|
||||
)?;
|
||||
let n = crate::h5::num_elements(&a.dataspace).unwrap_or(0);
|
||||
if n == 0 {
|
||||
return Ok(());
|
||||
}
|
||||
let dec = Decoder::new(self.h5);
|
||||
let shown = n.min(8) as usize;
|
||||
let mut vals = Vec::with_capacity(shown);
|
||||
for i in 0..shown {
|
||||
let v = dec.element(&a.datatype, &a.raw_data, i);
|
||||
vals.push(value::text(&v, &|_| None));
|
||||
}
|
||||
let more = if n > 8 { ", ..." } else { "" };
|
||||
writeln!(out.o, " Data: {}{more}", vals.join(", "))
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user