feat(tools): h5rs, pure-Rust HDF5 tools (ls, dump, stat, diff, check)
New workspace crate clawhdf5-tools with one binary, h5rs, built only on the clawhdf5 facade and clawhdf5-format (no libhdf5, no C): - ls [-r] [-v] FILE[/path]: h5ls's listing (same text in its first two columns) plus the datatype; -v adds address, link count, layout and chunk index, chunk size, storage, filters, datatype and attributes. - dump [--json] [-A] [-p] [-d PATH] FILE: h5dump DDL (byte-identical to h5dump 1.14.6 on the test files) or hdf5-json. - stat FILE: h5stat's object/link/rank/layout/filter/attribute counts, raw data and total size. - diff [-r] [-q] [-d D] [-p R] A B [OBJ1 [OBJ2]]: structural and value differences, exit 0/1/2 like h5diff. - check [--data] FILE: walks every object, parses every message, verifies the checksums of every v2+ structure (including the fractal heap blocks the library never checks), checks chunk indexes against their datasets and raw data for out-of-file or overlapping extents; every problem with its address. Values over --max-bytes are reported, not read; dense-storage heaps are verified before objects are read from them; panics are caught (exit 3). Tests compare with h5ls, h5stat, h5dump and h5diff and with h5py's values, and flip the checksum of every checksummed structure in a v1.14-format file. Co-Authored-By: Claude Opus 5.5 (1M context) <[email protected]>
This commit is contained in:
osobh
@@ -0,0 +1,944 @@
|
||||
//! `h5rs dump`: the whole file (or one dataset) as h5dump-style DDL text or
|
||||
//! as hdf5-json.
|
||||
|
||||
use std::cell::OnceCell;
|
||||
use std::collections::HashMap;
|
||||
|
||||
use clawhdf5_format::attribute::AttributeMessage;
|
||||
use clawhdf5_format::data_layout::DataLayout;
|
||||
use clawhdf5_format::dataspace::{Dataspace, DataspaceType};
|
||||
use clawhdf5_format::datatype::{Datatype, StringPadding};
|
||||
use clawhdf5_format::object_header::ObjectHeader;
|
||||
use serde_json::{Map, Value as J, json};
|
||||
|
||||
use crate::cli::{Args, Out};
|
||||
use crate::h5::{Error, H5, Kind, Link, LinkKind};
|
||||
use crate::info::{self, DsInfo};
|
||||
use crate::value::{self, Decoder, Value};
|
||||
|
||||
pub const USAGE: &str = "\
|
||||
usage: h5rs dump [--json] [-A] [-p] [-d PATH] [--max-bytes N] FILE
|
||||
|
||||
Print FILE's groups, datasets, named datatypes, links and attributes, with
|
||||
their values, as h5dump-style DDL text (default) or as JSON.
|
||||
|
||||
--json hdf5-json layout (see the crate README for the schema)
|
||||
-A, --header no dataset values (attribute values are still printed,
|
||||
as with h5dump -A)
|
||||
-p, --properties also print each dataset's storage layout and filters
|
||||
-d, --dataset P dump only the dataset at path P
|
||||
--max-bytes N largest dataset or attribute decoded (default 1 GiB);
|
||||
a larger one is reported instead of read
|
||||
|
||||
Exit status: 0 dumped, 1 something could not be read (reported on stderr
|
||||
and marked in the output), 2 error.";
|
||||
|
||||
struct Opts {
|
||||
json: bool,
|
||||
header_only: bool,
|
||||
props: bool,
|
||||
}
|
||||
|
||||
struct Dump<'a> {
|
||||
h5: &'a H5,
|
||||
opts: Opts,
|
||||
problems: usize,
|
||||
paths: OnceCell<HashMap<u64, String>>,
|
||||
}
|
||||
|
||||
pub fn run(args: &mut Args, out: &mut Out) -> std::io::Result<i32> {
|
||||
let mut opts = Opts {
|
||||
json: false,
|
||||
header_only: false,
|
||||
props: false,
|
||||
};
|
||||
let mut dataset = None;
|
||||
let mut max_bytes = None;
|
||||
let mut file = None;
|
||||
while let Some(a) = args.next() {
|
||||
match a.as_str() {
|
||||
"--json" | "-j" => opts.json = true,
|
||||
"-A" | "--header" => opts.header_only = true,
|
||||
"-p" | "--properties" => opts.props = true,
|
||||
"-d" | "--dataset" => match args.value() {
|
||||
Some(p) => dataset = Some(p),
|
||||
None => return args.usage_error(out, "-d needs a path", USAGE),
|
||||
},
|
||||
"--max-bytes" => match args.number() {
|
||||
Some(n) => max_bytes = Some(n),
|
||||
None => return args.usage_error(out, "--max-bytes needs a number", USAGE),
|
||||
},
|
||||
"-h" | "--help" => {
|
||||
writeln!(out.o, "{USAGE}")?;
|
||||
return Ok(0);
|
||||
}
|
||||
s if s.starts_with('-') && s.len() > 1 => {
|
||||
return args.usage_error(out, &format!("unknown option {a}"), USAGE);
|
||||
}
|
||||
_ if file.is_none() => file = Some(a),
|
||||
_ => return args.usage_error(out, &format!("unexpected argument {a}"), USAGE),
|
||||
}
|
||||
}
|
||||
let Some(file) = file else {
|
||||
return args.usage_error(out, "missing FILE", USAGE);
|
||||
};
|
||||
let mut h5 = match H5::open(std::path::Path::new(&file)) {
|
||||
Ok(h) => h,
|
||||
Err(e) => {
|
||||
writeln!(out.e, "h5rs dump: {e}")?;
|
||||
return Ok(2);
|
||||
}
|
||||
};
|
||||
if let Some(m) = max_bytes {
|
||||
h5.max_bytes = m;
|
||||
}
|
||||
let mut d = Dump {
|
||||
h5: &h5,
|
||||
opts,
|
||||
problems: 0,
|
||||
paths: OnceCell::new(),
|
||||
};
|
||||
let fname = std::path::Path::new(&file)
|
||||
.file_name()
|
||||
.map(|s| s.to_string_lossy().into_owned())
|
||||
.unwrap_or(file.clone());
|
||||
let code = if d.opts.json {
|
||||
d.json(out, dataset.as_deref())?
|
||||
} else {
|
||||
d.ddl(out, &fname, dataset.as_deref())?
|
||||
};
|
||||
if code != 0 {
|
||||
return Ok(code);
|
||||
}
|
||||
Ok(if d.problems > 0 { 1 } else { 0 })
|
||||
}
|
||||
|
||||
/// A full path for `name` inside the group at `base`.
|
||||
fn join(base: &str, name: &str) -> String {
|
||||
if base == "/" {
|
||||
format!("/{name}")
|
||||
} else {
|
||||
format!("{base}/{name}")
|
||||
}
|
||||
}
|
||||
|
||||
fn quote(s: &str) -> String {
|
||||
s.replace('\\', "\\\\").replace('"', "\\\"")
|
||||
}
|
||||
|
||||
impl Dump<'_> {
|
||||
/// Object address -> first path, for printing references.
|
||||
fn paths(&self) -> &HashMap<u64, String> {
|
||||
self.paths.get_or_init(|| {
|
||||
let mut m = HashMap::new();
|
||||
let _ = self.h5.walk(|it| {
|
||||
if let (Some(a), None) = (it.addr, it.first_path) {
|
||||
m.entry(a).or_insert_with(|| it.path.to_string());
|
||||
}
|
||||
});
|
||||
m
|
||||
})
|
||||
}
|
||||
|
||||
fn problem(&mut self, out: &mut Out, what: &str, e: &Error) -> std::io::Result<()> {
|
||||
self.problems += 1;
|
||||
writeln!(out.e, "h5rs dump: {what}: {e}")
|
||||
}
|
||||
|
||||
// ------------------------------------------------------------------
|
||||
// DDL
|
||||
// ------------------------------------------------------------------
|
||||
|
||||
fn ddl(&mut self, out: &mut Out, fname: &str, only: Option<&str>) -> std::io::Result<i32> {
|
||||
writeln!(out.o, "HDF5 \"{}\" {{", quote(fname))?;
|
||||
if let Some(p) = only {
|
||||
let h = match self.h5.resolve(p).and_then(|a| self.h5.header(a)) {
|
||||
Ok(h) => h,
|
||||
Err(e) => {
|
||||
writeln!(out.o, "}}")?;
|
||||
writeln!(out.e, "h5rs dump: {p}: {e}")?;
|
||||
return Ok(2);
|
||||
}
|
||||
};
|
||||
if Kind::of(&h) != Kind::Dataset {
|
||||
writeln!(out.o, "}}")?;
|
||||
writeln!(out.e, "h5rs dump: {p}: not a dataset")?;
|
||||
return Ok(2);
|
||||
}
|
||||
let full = if p.starts_with('/') {
|
||||
p.to_string()
|
||||
} else {
|
||||
format!("/{p}")
|
||||
};
|
||||
self.ddl_dataset(out, &full, &full, &h, 0)?;
|
||||
} else {
|
||||
let root = self.h5.root();
|
||||
let mut seen = HashMap::new();
|
||||
match self.h5.header(root) {
|
||||
Ok(h) => self.ddl_group(out, "/", "/", root, &h, 0, &mut seen)?,
|
||||
Err(e) => {
|
||||
writeln!(out.o, "GROUP \"/\" {{\n}}")?;
|
||||
self.problem(out, "/", &e)?;
|
||||
}
|
||||
}
|
||||
}
|
||||
writeln!(out.o, "}}")?;
|
||||
Ok(0)
|
||||
}
|
||||
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
fn ddl_group(
|
||||
&mut self,
|
||||
out: &mut Out,
|
||||
name: &str,
|
||||
path: &str,
|
||||
addr: u64,
|
||||
h: &ObjectHeader,
|
||||
ind: usize,
|
||||
seen: &mut HashMap<u64, String>,
|
||||
) -> std::io::Result<()> {
|
||||
let pad = " ".repeat(ind);
|
||||
seen.insert(addr, path.to_string());
|
||||
writeln!(out.o, "{pad}GROUP \"{}\" {{", quote(name))?;
|
||||
self.ddl_attributes(out, path, h, ind + 3)?;
|
||||
let links = match self.h5.links(h) {
|
||||
Ok(l) => l,
|
||||
Err(e) => {
|
||||
self.problem(out, path, &e)?;
|
||||
Vec::new()
|
||||
}
|
||||
};
|
||||
// The DDL recursion follows the group nesting; bound it (the walk used
|
||||
// by the other commands is iterative).
|
||||
if seen.len() > crate::h5::MAX_OBJECTS || ind > 3 * MAX_DDL_DEPTH {
|
||||
let e = Error::new("too many objects, or groups nested too deeply; stopped");
|
||||
self.problem(out, path, &e)?;
|
||||
return writeln!(out.o, "{pad}}}");
|
||||
}
|
||||
for l in &links {
|
||||
self.ddl_link(out, path, l, ind + 3, seen)?;
|
||||
}
|
||||
writeln!(out.o, "{pad}}}")
|
||||
}
|
||||
|
||||
fn ddl_link(
|
||||
&mut self,
|
||||
out: &mut Out,
|
||||
base: &str,
|
||||
l: &Link,
|
||||
ind: usize,
|
||||
seen: &mut HashMap<u64, String>,
|
||||
) -> std::io::Result<()> {
|
||||
let pad = " ".repeat(ind);
|
||||
let inner = " ".repeat(ind + 3);
|
||||
let name = quote(&l.name);
|
||||
let path = join(base, &l.name);
|
||||
match &l.kind {
|
||||
LinkKind::Soft(t) => writeln!(
|
||||
out.o,
|
||||
"{pad}SOFTLINK \"{name}\" {{\n{inner}LINKTARGET \"{}\"\n{pad}}}",
|
||||
quote(t)
|
||||
),
|
||||
LinkKind::External { file, path: p } => writeln!(
|
||||
out.o,
|
||||
"{pad}EXTERNAL_LINK \"{name}\" {{\n{inner}TARGETFILE \"{}\"\n{inner}TARGETPATH \"{}\"\n{pad}}}",
|
||||
quote(file),
|
||||
quote(p)
|
||||
),
|
||||
LinkKind::UserDefined(t) => writeln!(
|
||||
out.o,
|
||||
"{pad}USERDEFINED_LINK \"{name}\" {{\n{inner}LINKCLASS {t}\n{pad}}}"
|
||||
),
|
||||
LinkKind::Hard(a) => {
|
||||
let h = match self.h5.header(*a) {
|
||||
Ok(h) => h,
|
||||
Err(e) => {
|
||||
self.problem(out, &path, &e)?;
|
||||
return writeln!(out.o, "{pad}UNKNOWN_OBJECT \"{name}\" {{\n{pad}}}");
|
||||
}
|
||||
};
|
||||
let kind = Kind::of(&h);
|
||||
if let Some(first) = seen.get(a) {
|
||||
let word = match kind {
|
||||
Kind::Group => "GROUP",
|
||||
Kind::Dataset => "DATASET",
|
||||
Kind::Datatype => "DATATYPE",
|
||||
Kind::Unknown => "OBJECT",
|
||||
};
|
||||
return writeln!(
|
||||
out.o,
|
||||
"{pad}{word} \"{name}\" {{\n{inner}HARDLINK \"{}\"\n{pad}}}",
|
||||
quote(first)
|
||||
);
|
||||
}
|
||||
match kind {
|
||||
Kind::Group => self.ddl_group(out, &l.name, &path, *a, &h, ind, seen),
|
||||
Kind::Dataset => {
|
||||
seen.insert(*a, path.clone());
|
||||
self.ddl_dataset(out, &l.name, &path, &h, ind)
|
||||
}
|
||||
Kind::Datatype => {
|
||||
seen.insert(*a, path.clone());
|
||||
match self.h5.datatype(&h) {
|
||||
Ok(dt) => writeln!(
|
||||
out.o,
|
||||
"{pad}DATATYPE \"{name}\" {};",
|
||||
crate::dtype::ddl(&dt, ind)
|
||||
),
|
||||
Err(e) => {
|
||||
self.problem(out, &path, &e)?;
|
||||
writeln!(out.o, "{pad}DATATYPE \"{name}\" ?;")
|
||||
}
|
||||
}
|
||||
}
|
||||
Kind::Unknown => {
|
||||
seen.insert(*a, path.clone());
|
||||
writeln!(out.o, "{pad}UNKNOWN_OBJECT \"{name}\" {{\n{pad}}}")
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn ddl_dataset(
|
||||
&mut self,
|
||||
out: &mut Out,
|
||||
name: &str,
|
||||
path: &str,
|
||||
h: &ObjectHeader,
|
||||
ind: usize,
|
||||
) -> std::io::Result<()> {
|
||||
let pad = " ".repeat(ind);
|
||||
let inner = " ".repeat(ind + 3);
|
||||
writeln!(out.o, "{pad}DATASET \"{}\" {{", quote(name))?;
|
||||
let info = DsInfo::read(self.h5, path, h);
|
||||
match &info.dt {
|
||||
Ok(dt) => writeln!(out.o, "{inner}DATATYPE {}", crate::dtype::ddl(dt, ind + 3))?,
|
||||
Err(e) => self.problem(out, path, e)?,
|
||||
}
|
||||
match &info.ds {
|
||||
Ok(ds) => writeln!(out.o, "{inner}DATASPACE {}", info::dataspace_ddl(ds))?,
|
||||
Err(e) => self.problem(out, path, e)?,
|
||||
}
|
||||
if self.opts.props {
|
||||
self.ddl_properties(out, path, &info, ind + 3)?;
|
||||
}
|
||||
if !self.opts.header_only
|
||||
&& let (Ok(dt), Ok(ds)) = (&info.dt, &info.ds)
|
||||
{
|
||||
match self.h5.read_dataset(path, dt, ds) {
|
||||
Ok(raw) => self.ddl_data(out, dt, ds, &raw, ind + 3)?,
|
||||
Err(e) => {
|
||||
writeln!(out.o, "{inner}DATA {{\n{inner}<not read: {e}>\n{inner}}}")?;
|
||||
self.problem(out, path, &e)?;
|
||||
}
|
||||
}
|
||||
}
|
||||
self.ddl_attributes(out, path, h, ind + 3)?;
|
||||
writeln!(out.o, "{pad}}}")
|
||||
}
|
||||
|
||||
fn ddl_properties(
|
||||
&mut self,
|
||||
out: &mut Out,
|
||||
path: &str,
|
||||
info: &DsInfo,
|
||||
ind: usize,
|
||||
) -> std::io::Result<()> {
|
||||
let pad = " ".repeat(ind);
|
||||
let inner = " ".repeat(ind + 3);
|
||||
let Ok(layout) = &info.layout else {
|
||||
if let Err(e) = &info.layout {
|
||||
self.problem(out, path, e)?;
|
||||
}
|
||||
return Ok(());
|
||||
};
|
||||
writeln!(out.o, "{pad}STORAGE_LAYOUT {{")?;
|
||||
match layout {
|
||||
DataLayout::Chunked {
|
||||
chunk_dimensions, ..
|
||||
} => {
|
||||
let rank = chunk_dimensions.len().saturating_sub(1);
|
||||
writeln!(
|
||||
out.o,
|
||||
"{inner}CHUNKED ( {} )",
|
||||
chunk_dimensions[..rank]
|
||||
.iter()
|
||||
.map(|d| d.to_string())
|
||||
.collect::<Vec<_>>()
|
||||
.join(", ")
|
||||
)?;
|
||||
writeln!(
|
||||
out.o,
|
||||
"{inner}INDEX {}",
|
||||
info::chunk_index_name(layout).to_uppercase()
|
||||
)?;
|
||||
}
|
||||
DataLayout::Contiguous { address, size } => {
|
||||
writeln!(out.o, "{inner}CONTIGUOUS")?;
|
||||
match address {
|
||||
Some(a) => writeln!(out.o, "{inner}OFFSET {a}")?,
|
||||
None => writeln!(out.o, "{inner}NOT ALLOCATED")?,
|
||||
}
|
||||
writeln!(out.o, "{inner}SIZE {size}")?;
|
||||
}
|
||||
DataLayout::Compact { data } => {
|
||||
writeln!(out.o, "{inner}COMPACT")?;
|
||||
writeln!(out.o, "{inner}SIZE {}", data.len())?;
|
||||
}
|
||||
DataLayout::Virtual { .. } => writeln!(out.o, "{inner}VIRTUAL")?,
|
||||
}
|
||||
if let Ok(a) = info::allocated_bytes(self.h5, info)
|
||||
&& matches!(layout, DataLayout::Chunked { .. })
|
||||
{
|
||||
writeln!(out.o, "{inner}SIZE {a}")?;
|
||||
}
|
||||
writeln!(out.o, "{pad}}}")?;
|
||||
writeln!(out.o, "{pad}FILTERS {{")?;
|
||||
match &info.filters {
|
||||
Ok(Some(p)) if !p.filters.is_empty() => {
|
||||
for f in &p.filters {
|
||||
writeln!(
|
||||
out.o,
|
||||
"{inner}{} {{ ID {}; PARAMS {{ {} }} }}",
|
||||
info::filter_name(f).to_uppercase(),
|
||||
f.filter_id,
|
||||
f.client_data
|
||||
.iter()
|
||||
.map(|c| c.to_string())
|
||||
.collect::<Vec<_>>()
|
||||
.join(" ")
|
||||
)?;
|
||||
}
|
||||
}
|
||||
Ok(_) => writeln!(out.o, "{inner}NONE")?,
|
||||
Err(e) => self.problem(out, path, e)?,
|
||||
}
|
||||
writeln!(out.o, "{pad}}}")
|
||||
}
|
||||
|
||||
fn ddl_attributes(
|
||||
&mut self,
|
||||
out: &mut Out,
|
||||
path: &str,
|
||||
h: &ObjectHeader,
|
||||
ind: usize,
|
||||
) -> std::io::Result<()> {
|
||||
let (attrs, errs) = match self.h5.attributes(h) {
|
||||
Ok(x) => x,
|
||||
Err(e) => return self.problem(out, path, &e),
|
||||
};
|
||||
for e in errs {
|
||||
self.problem(out, path, &Error::new(format!("attribute: {e}")))?;
|
||||
}
|
||||
for a in &attrs {
|
||||
self.ddl_attribute(out, path, a, ind)?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn ddl_attribute(
|
||||
&mut self,
|
||||
out: &mut Out,
|
||||
path: &str,
|
||||
a: &AttributeMessage,
|
||||
ind: usize,
|
||||
) -> std::io::Result<()> {
|
||||
let pad = " ".repeat(ind);
|
||||
let inner = " ".repeat(ind + 3);
|
||||
writeln!(out.o, "{pad}ATTRIBUTE \"{}\" {{", quote(&a.name))?;
|
||||
writeln!(
|
||||
out.o,
|
||||
"{inner}DATATYPE {}",
|
||||
crate::dtype::ddl(&a.datatype, ind + 3)
|
||||
)?;
|
||||
writeln!(
|
||||
out.o,
|
||||
"{inner}DATASPACE {}",
|
||||
info::dataspace_ddl(&a.dataspace)
|
||||
)?;
|
||||
{
|
||||
match self.checked_attr(a) {
|
||||
Ok(()) => self.ddl_data(out, &a.datatype, &a.dataspace, &a.raw_data, ind + 3)?,
|
||||
Err(e) => {
|
||||
writeln!(out.o, "{inner}DATA {{\n{inner}<not read: {e}>\n{inner}}}")?;
|
||||
self.problem(out, &format!("{path} attribute {}", a.name), &e)?;
|
||||
}
|
||||
}
|
||||
}
|
||||
writeln!(out.o, "{pad}}}")
|
||||
}
|
||||
|
||||
/// An attribute's value holds as many bytes as its dataspace says.
|
||||
fn checked_attr(&self, a: &AttributeMessage) -> crate::h5::Result<()> {
|
||||
let need = crate::h5::byte_len(&a.dataspace, &a.datatype)?;
|
||||
if need > self.h5.max_bytes {
|
||||
return Err(Error::new(format!(
|
||||
"attribute is {need} bytes, over the {} byte limit (--max-bytes)",
|
||||
self.h5.max_bytes
|
||||
)));
|
||||
}
|
||||
if (a.raw_data.len() as u64) < need {
|
||||
return Err(Error::new(format!(
|
||||
"attribute holds {} bytes, its dataspace needs {need}",
|
||||
a.raw_data.len()
|
||||
)));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// An h5dump DATA block: elements row by row, each line starting with
|
||||
/// the index of its first element, wrapped where h5dump wraps (78 columns).
|
||||
fn ddl_data(
|
||||
&mut self,
|
||||
out: &mut Out,
|
||||
dt: &Datatype,
|
||||
ds: &Dataspace,
|
||||
raw: &[u8],
|
||||
ind: usize,
|
||||
) -> std::io::Result<()> {
|
||||
let pad = " ".repeat(ind);
|
||||
writeln!(out.o, "{pad}DATA {{")?;
|
||||
let dims: Vec<u64> = match ds.space_type {
|
||||
DataspaceType::Null => {
|
||||
return writeln!(out.o, "{pad}}}");
|
||||
}
|
||||
DataspaceType::Scalar => vec![1],
|
||||
DataspaceType::Simple => ds.dimensions.clone(),
|
||||
};
|
||||
let n = crate::h5::num_elements(ds).unwrap_or(0) as usize;
|
||||
let dec = Decoder::new(self.h5);
|
||||
let last = dims.last().copied().unwrap_or(1).max(1) as usize;
|
||||
let paths = |a: u64| self.paths().get(&a).cloned();
|
||||
let mut line = String::new();
|
||||
let mut errors = 0usize;
|
||||
for i in 0..n {
|
||||
let v = dec.element(dt, raw, i);
|
||||
if matches!(v, Value::Error(_)) {
|
||||
errors += 1;
|
||||
}
|
||||
let comma = if i + 1 < n { "," } else { "" };
|
||||
if let Value::Compound(members) = &v {
|
||||
// h5dump prints each compound element as a block, one
|
||||
// member per line.
|
||||
if !line.is_empty() {
|
||||
writeln!(out.o, "{line}")?;
|
||||
line.clear();
|
||||
}
|
||||
writeln!(out.o, "{pad}({}): {{", index_text(i as u64, &dims))?;
|
||||
for (k, (_, m)) in members.iter().enumerate() {
|
||||
let sep = if k + 1 < members.len() { "," } else { "" };
|
||||
writeln!(out.o, "{pad} {}{sep}", value::text(m, &paths))?;
|
||||
}
|
||||
writeln!(out.o, "{pad} }}{comma}")?;
|
||||
continue;
|
||||
}
|
||||
let t = match dt {
|
||||
// h5dump shows a null-padded string's padding.
|
||||
Datatype::String {
|
||||
padding: StringPadding::NullPad,
|
||||
size,
|
||||
..
|
||||
} if !matches!(v, Value::Error(_)) => {
|
||||
let sz = *size as usize;
|
||||
let b = raw.get(i * sz..(i + 1) * sz).unwrap_or_default();
|
||||
format!("{}{comma}", value::quote_bytes(b))
|
||||
}
|
||||
_ => format!("{}{comma}", value::text(&v, &paths)),
|
||||
};
|
||||
let row_start = i % last == 0;
|
||||
if row_start || line.len() + 1 + t.len() > 77 {
|
||||
if !line.is_empty() {
|
||||
writeln!(out.o, "{line}")?;
|
||||
}
|
||||
line = format!("{pad}({}): {t}", index_text(i as u64, &dims));
|
||||
} else {
|
||||
line.push(' ');
|
||||
line.push_str(&t);
|
||||
}
|
||||
}
|
||||
if !line.is_empty() {
|
||||
writeln!(out.o, "{line}")?;
|
||||
}
|
||||
if errors > 0 {
|
||||
self.problems += 1;
|
||||
writeln!(out.e, "h5rs dump: {errors} element(s) could not be decoded")?;
|
||||
}
|
||||
writeln!(out.o, "{pad}}}")
|
||||
}
|
||||
|
||||
// ------------------------------------------------------------------
|
||||
// JSON
|
||||
// ------------------------------------------------------------------
|
||||
|
||||
fn json(&mut self, out: &mut Out, only: Option<&str>) -> std::io::Result<i32> {
|
||||
let mut groups = Map::new();
|
||||
let mut datasets = Map::new();
|
||||
let mut datatypes = Map::new();
|
||||
let mut doc = Map::new();
|
||||
doc.insert("apiVersion".into(), json!("1.1.1"));
|
||||
if let Some(p) = only {
|
||||
let full = if p.starts_with('/') {
|
||||
p.to_string()
|
||||
} else {
|
||||
format!("/{p}")
|
||||
};
|
||||
let r = self
|
||||
.h5
|
||||
.resolve(&full)
|
||||
.and_then(|a| Ok((a, self.h5.header(a)?)));
|
||||
match r {
|
||||
Ok((a, h)) if Kind::of(&h) == Kind::Dataset => {
|
||||
let obj = self.json_dataset(out, &full, &h)?;
|
||||
datasets.insert(obj_id(Kind::Dataset, a), obj);
|
||||
}
|
||||
Ok(_) => {
|
||||
writeln!(out.e, "h5rs dump: {p}: not a dataset")?;
|
||||
return Ok(2);
|
||||
}
|
||||
Err(e) => {
|
||||
writeln!(out.e, "h5rs dump: {p}: {e}")?;
|
||||
return Ok(2);
|
||||
}
|
||||
}
|
||||
} else {
|
||||
doc.insert("root".into(), json!(obj_id(Kind::Group, self.h5.root())));
|
||||
let (items, walk) = self.h5.walk_collect();
|
||||
if let Err(e) = walk {
|
||||
self.problem(out, "/", &e)?;
|
||||
}
|
||||
// Aliases: every path an object is reachable by.
|
||||
let mut aliases: HashMap<u64, Vec<String>> = HashMap::new();
|
||||
for it in &items {
|
||||
if let Some(a) = it.addr {
|
||||
aliases.entry(a).or_default().push(it.path.clone());
|
||||
}
|
||||
}
|
||||
for it in items {
|
||||
let path = it.path;
|
||||
let (Some(addr), None, Some(header)) = (it.addr, it.first_path, it.header) else {
|
||||
continue;
|
||||
};
|
||||
let h = match header {
|
||||
Ok(h) => h,
|
||||
Err(e) => {
|
||||
self.problem(out, &path, &e)?;
|
||||
continue;
|
||||
}
|
||||
};
|
||||
let kind = Kind::of(&h);
|
||||
let alias = aliases.remove(&addr).unwrap_or_default();
|
||||
let mut obj = match kind {
|
||||
Kind::Dataset => self.json_dataset(out, &path, &h)?,
|
||||
Kind::Datatype => {
|
||||
let mut m = Map::new();
|
||||
match self.h5.datatype(&h) {
|
||||
Ok(dt) => {
|
||||
m.insert("type".into(), crate::dtype::json(&dt));
|
||||
}
|
||||
Err(e) => self.problem(out, &path, &e)?,
|
||||
}
|
||||
m.insert("attributes".into(), self.json_attributes(out, &path, &h)?);
|
||||
J::Object(m)
|
||||
}
|
||||
_ => {
|
||||
let mut m = Map::new();
|
||||
m.insert("attributes".into(), self.json_attributes(out, &path, &h)?);
|
||||
let links = match self.h5.links(&h) {
|
||||
Ok(l) => l,
|
||||
Err(e) => {
|
||||
if kind == Kind::Group {
|
||||
self.problem(out, &path, &e)?;
|
||||
}
|
||||
Vec::new()
|
||||
}
|
||||
};
|
||||
let mut jl = Vec::new();
|
||||
for l in &links {
|
||||
jl.push(self.json_link(l));
|
||||
}
|
||||
m.insert("links".into(), J::Array(jl));
|
||||
J::Object(m)
|
||||
}
|
||||
};
|
||||
if let J::Object(m) = &mut obj {
|
||||
m.insert("alias".into(), json!(alias));
|
||||
}
|
||||
let id = obj_id(kind, addr);
|
||||
match kind {
|
||||
Kind::Dataset => datasets.insert(id, obj),
|
||||
Kind::Datatype => datatypes.insert(id, obj),
|
||||
_ => groups.insert(id, obj),
|
||||
};
|
||||
}
|
||||
doc.insert("groups".into(), J::Object(groups));
|
||||
}
|
||||
doc.insert("datasets".into(), J::Object(datasets));
|
||||
if only.is_none() {
|
||||
doc.insert("datatypes".into(), J::Object(datatypes));
|
||||
}
|
||||
let text = serde_json::to_string_pretty(&J::Object(doc)).map_err(std::io::Error::other)?;
|
||||
writeln!(out.o, "{text}")?;
|
||||
Ok(0)
|
||||
}
|
||||
|
||||
fn json_link(&self, l: &Link) -> J {
|
||||
match &l.kind {
|
||||
LinkKind::Hard(a) => {
|
||||
let kind = self
|
||||
.h5
|
||||
.header(*a)
|
||||
.map(|h| Kind::of(&h))
|
||||
.unwrap_or(Kind::Unknown);
|
||||
let coll = match kind {
|
||||
Kind::Dataset => "datasets",
|
||||
Kind::Datatype => "datatypes",
|
||||
_ => "groups",
|
||||
};
|
||||
json!({"class": "H5L_TYPE_HARD", "title": l.name, "collection": coll,
|
||||
"id": obj_id(kind, *a)})
|
||||
}
|
||||
LinkKind::Soft(t) => json!({"class": "H5L_TYPE_SOFT", "title": l.name, "h5path": t}),
|
||||
LinkKind::External { file, path } => json!({"class": "H5L_TYPE_EXTERNAL",
|
||||
"title": l.name, "file": file, "h5path": path}),
|
||||
LinkKind::UserDefined(t) => json!({"class": "H5L_TYPE_USER_DEFINED",
|
||||
"title": l.name, "linkClass": t}),
|
||||
}
|
||||
}
|
||||
|
||||
fn json_dataset(&mut self, out: &mut Out, path: &str, h: &ObjectHeader) -> std::io::Result<J> {
|
||||
let info = DsInfo::read(self.h5, path, h);
|
||||
let mut m = Map::new();
|
||||
match &info.dt {
|
||||
Ok(dt) => {
|
||||
m.insert("type".into(), crate::dtype::json(dt));
|
||||
}
|
||||
Err(e) => self.problem(out, path, e)?,
|
||||
}
|
||||
match &info.ds {
|
||||
Ok(ds) => {
|
||||
m.insert("shape".into(), shape_json(ds));
|
||||
}
|
||||
Err(e) => self.problem(out, path, e)?,
|
||||
}
|
||||
let mut cp = Map::new();
|
||||
if let Ok(l) = &info.layout {
|
||||
let mut lj = Map::new();
|
||||
lj.insert(
|
||||
"class".into(),
|
||||
json!(match l {
|
||||
DataLayout::Compact { .. } => "H5D_COMPACT",
|
||||
DataLayout::Contiguous { .. } => "H5D_CONTIGUOUS",
|
||||
DataLayout::Chunked { .. } => "H5D_CHUNKED",
|
||||
DataLayout::Virtual { .. } => "H5D_VIRTUAL",
|
||||
}),
|
||||
);
|
||||
if let DataLayout::Chunked {
|
||||
chunk_dimensions, ..
|
||||
} = l
|
||||
{
|
||||
let rank = chunk_dimensions.len().saturating_sub(1);
|
||||
lj.insert("dims".into(), json!(chunk_dimensions[..rank]));
|
||||
}
|
||||
cp.insert("layout".into(), J::Object(lj));
|
||||
} else if let Err(e) = &info.layout {
|
||||
self.problem(out, path, e)?;
|
||||
}
|
||||
if let Ok(Some(p)) = &info.filters {
|
||||
let fl: Vec<J> = p
|
||||
.filters
|
||||
.iter()
|
||||
.map(|f| {
|
||||
json!({"id": f.filter_id, "name": info::filter_name(f),
|
||||
"class": filter_class(f.filter_id), "parameters": f.client_data})
|
||||
})
|
||||
.collect();
|
||||
cp.insert("filters".into(), J::Array(fl));
|
||||
}
|
||||
m.insert("creationProperties".into(), J::Object(cp));
|
||||
if !self.opts.header_only
|
||||
&& let (Ok(dt), Ok(ds)) = (&info.dt, &info.ds)
|
||||
{
|
||||
// JSON values are built in memory, some 32+ bytes per element:
|
||||
// hold them to the same budget as the raw data.
|
||||
let n = crate::h5::num_elements(ds).unwrap_or(u64::MAX);
|
||||
let read = if n.saturating_mul(JSON_BYTES_PER_ELEMENT) > self.h5.max_bytes {
|
||||
Err(Error::new(format!(
|
||||
"{n} elements are too many to hold as JSON within --max-bytes {}",
|
||||
self.h5.max_bytes
|
||||
))
|
||||
.with_kind(crate::h5::ErrorKind::Limit))
|
||||
} else {
|
||||
self.h5.read_dataset(path, dt, ds)
|
||||
};
|
||||
match read {
|
||||
Ok(raw) => {
|
||||
let v = self.json_values(out, dt, ds, &raw)?;
|
||||
m.insert("value".into(), v);
|
||||
}
|
||||
Err(e) => {
|
||||
m.insert("value_error".into(), json!(e.to_string()));
|
||||
self.problem(out, path, &e)?;
|
||||
}
|
||||
}
|
||||
}
|
||||
m.insert("attributes".into(), self.json_attributes(out, path, h)?);
|
||||
Ok(J::Object(m))
|
||||
}
|
||||
|
||||
fn json_attributes(
|
||||
&mut self,
|
||||
out: &mut Out,
|
||||
path: &str,
|
||||
h: &ObjectHeader,
|
||||
) -> std::io::Result<J> {
|
||||
let (attrs, errs) = match self.h5.attributes(h) {
|
||||
Ok(x) => x,
|
||||
Err(e) => {
|
||||
self.problem(out, path, &e)?;
|
||||
return Ok(J::Array(Vec::new()));
|
||||
}
|
||||
};
|
||||
for e in errs {
|
||||
self.problem(out, path, &Error::new(format!("attribute: {e}")))?;
|
||||
}
|
||||
let mut v = Vec::new();
|
||||
for a in &attrs {
|
||||
let mut m = Map::new();
|
||||
m.insert("name".into(), json!(a.name));
|
||||
m.insert("shape".into(), shape_json(&a.dataspace));
|
||||
m.insert("type".into(), crate::dtype::json(&a.datatype));
|
||||
{
|
||||
match self.checked_attr(a) {
|
||||
Ok(()) => {
|
||||
let val = self.json_values(out, &a.datatype, &a.dataspace, &a.raw_data)?;
|
||||
m.insert("value".into(), val);
|
||||
}
|
||||
Err(e) => {
|
||||
m.insert("value_error".into(), json!(e.to_string()));
|
||||
self.problem(out, &format!("{path} attribute {}", a.name), &e)?;
|
||||
}
|
||||
}
|
||||
}
|
||||
v.push(J::Object(m));
|
||||
}
|
||||
Ok(J::Array(v))
|
||||
}
|
||||
|
||||
fn json_values(
|
||||
&mut self,
|
||||
out: &mut Out,
|
||||
dt: &Datatype,
|
||||
ds: &Dataspace,
|
||||
raw: &[u8],
|
||||
) -> std::io::Result<J> {
|
||||
let n = crate::h5::num_elements(ds).unwrap_or(0) as usize;
|
||||
let dec = Decoder::new(self.h5);
|
||||
let paths = |a: u64| self.paths().get(&a).cloned();
|
||||
let mut flat = Vec::with_capacity(n);
|
||||
let mut errors = 0usize;
|
||||
for i in 0..n {
|
||||
let v = dec.element(dt, raw, i);
|
||||
if matches!(v, Value::Error(_)) {
|
||||
errors += 1;
|
||||
}
|
||||
flat.push(value::to_json(&v, &paths));
|
||||
}
|
||||
if errors > 0 {
|
||||
self.problems += 1;
|
||||
writeln!(out.e, "h5rs dump: {errors} element(s) could not be decoded")?;
|
||||
}
|
||||
Ok(match ds.space_type {
|
||||
DataspaceType::Null => J::Null,
|
||||
DataspaceType::Scalar => flat.into_iter().next().unwrap_or(J::Null),
|
||||
DataspaceType::Simple => nest(flat, &ds.dimensions),
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
/// Deepest group nesting the DDL output follows.
|
||||
const MAX_DDL_DEPTH: usize = 256;
|
||||
|
||||
/// Memory budgeted per element for a dataset's value held as JSON.
|
||||
const JSON_BYTES_PER_ELEMENT: u64 = 64;
|
||||
|
||||
/// `(i,j,k)` index of flat element `i` of an array with `dims`.
|
||||
fn index_text(mut i: u64, dims: &[u64]) -> String {
|
||||
let mut idx = vec![0u64; dims.len()];
|
||||
for (k, &d) in dims.iter().enumerate().rev() {
|
||||
let d = d.max(1);
|
||||
idx[k] = i % d;
|
||||
i /= d;
|
||||
}
|
||||
idx.iter()
|
||||
.map(|x| x.to_string())
|
||||
.collect::<Vec<_>>()
|
||||
.join(",")
|
||||
}
|
||||
|
||||
/// Turn a flat row-major list into nested lists of shape `dims`.
|
||||
fn nest(flat: Vec<J>, dims: &[u64]) -> J {
|
||||
if dims.len() <= 1 {
|
||||
return J::Array(flat);
|
||||
}
|
||||
let inner: usize = dims[1..].iter().map(|&d| d as usize).product();
|
||||
if inner == 0 {
|
||||
return J::Array((0..dims[0]).map(|_| nest(Vec::new(), &dims[1..])).collect());
|
||||
}
|
||||
let mut it = flat.into_iter();
|
||||
let mut outer = Vec::with_capacity(dims[0] as usize);
|
||||
for _ in 0..dims[0] {
|
||||
let part: Vec<J> = it.by_ref().take(inner).collect();
|
||||
outer.push(nest(part, &dims[1..]));
|
||||
}
|
||||
J::Array(outer)
|
||||
}
|
||||
|
||||
fn shape_json(ds: &Dataspace) -> J {
|
||||
match ds.space_type {
|
||||
DataspaceType::Null => json!({"class": "H5S_NULL"}),
|
||||
DataspaceType::Scalar => json!({"class": "H5S_SCALAR"}),
|
||||
DataspaceType::Simple => {
|
||||
let mut m = Map::new();
|
||||
m.insert("class".into(), json!("H5S_SIMPLE"));
|
||||
m.insert("dims".into(), json!(ds.dimensions));
|
||||
if let Some(max) = &ds.max_dimensions {
|
||||
let mx: Vec<J> = max
|
||||
.iter()
|
||||
.map(|&d| {
|
||||
if d == u64::MAX {
|
||||
json!("H5S_UNLIMITED")
|
||||
} else {
|
||||
json!(d)
|
||||
}
|
||||
})
|
||||
.collect();
|
||||
m.insert("maxdims".into(), J::Array(mx));
|
||||
}
|
||||
J::Object(m)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn filter_class(id: u16) -> &'static str {
|
||||
match id {
|
||||
1 => "H5Z_FILTER_DEFLATE",
|
||||
2 => "H5Z_FILTER_SHUFFLE",
|
||||
3 => "H5Z_FILTER_FLETCHER32",
|
||||
4 => "H5Z_FILTER_SZIP",
|
||||
5 => "H5Z_FILTER_NBIT",
|
||||
6 => "H5Z_FILTER_SCALEOFFSET",
|
||||
_ => "H5Z_FILTER_USER",
|
||||
}
|
||||
}
|
||||
|
||||
/// Deterministic object id: kind prefix + header address (hdf5-json uses
|
||||
/// UUIDs; these are stable for a given file instead).
|
||||
pub fn obj_id(kind: Kind, addr: u64) -> String {
|
||||
let p = match kind {
|
||||
Kind::Dataset => 'd',
|
||||
Kind::Datatype => 't',
|
||||
_ => 'g',
|
||||
};
|
||||
format!("{p}-{addr:016x}")
|
||||
}
|
||||
Reference in New Issue
Block a user