osobhandClaude Opus 5.5
c4d96c1390
A null-padded fixed string inside a compound or an array member printed
trimmed ("" for three NULs, "a" for "a\0b"), where h5dump prints every
byte ("\000\000\000", "a\000b"); only top-level strings were shown in
full. DATA blocks now render elements through one function that keeps
the padding at any depth.
The README now lists the remaining known differences from h5dump:
nested compounds print inline, and long double values are printed as
errors (exit 1) with the datatype as an H5T_FLOAT block.
Co-Authored-By: Claude Opus 5.5 (1M context) <[email protected]>
1003 lines
35 KiB
Rust
1003 lines
35 KiB
Rust
//! `h5rs dump`: the whole file (or one dataset) as h5dump-style DDL text or
|
|
//! as hdf5-json.
|
|
|
|
use std::cell::OnceCell;
|
|
use std::collections::HashMap;
|
|
|
|
use clawhdf5_format::attribute::AttributeMessage;
|
|
use clawhdf5_format::data_layout::DataLayout;
|
|
use clawhdf5_format::dataspace::{Dataspace, DataspaceType};
|
|
use clawhdf5_format::datatype::{Datatype, StringPadding};
|
|
use clawhdf5_format::object_header::ObjectHeader;
|
|
use serde_json::{Map, Value as J, json};
|
|
|
|
use crate::cli::{Args, Out};
|
|
use crate::h5::{Error, H5, Kind, Link, LinkKind};
|
|
use crate::info::{self, DsInfo};
|
|
use crate::value::{self, Decoder, Value};
|
|
|
|
pub const USAGE: &str = "\
|
|
usage: h5rs dump [--json] [-A] [-p] [-d PATH] [--max-bytes N] FILE
|
|
|
|
Print FILE's groups, datasets, named datatypes, links and attributes, with
|
|
their values, as h5dump-style DDL text (default) or as JSON.
|
|
|
|
--json hdf5-json layout (see the crate README for the schema)
|
|
-A, --header no dataset values (attribute values are still printed,
|
|
as with h5dump -A)
|
|
-p, --properties also print each dataset's storage layout and filters
|
|
-d, --dataset P dump only the dataset at path P
|
|
--max-bytes N largest dataset or attribute decoded (default 1 GiB);
|
|
a larger one is reported instead of read
|
|
|
|
Exit status: 0 dumped, 1 something could not be read (reported on stderr
|
|
and marked in the output), 2 error.";
|
|
|
|
struct Opts {
|
|
json: bool,
|
|
header_only: bool,
|
|
props: bool,
|
|
}
|
|
|
|
struct Dump<'a> {
|
|
h5: &'a H5,
|
|
opts: Opts,
|
|
problems: usize,
|
|
paths: OnceCell<HashMap<u64, String>>,
|
|
}
|
|
|
|
pub fn run(args: &mut Args, out: &mut Out) -> std::io::Result<i32> {
|
|
let mut opts = Opts {
|
|
json: false,
|
|
header_only: false,
|
|
props: false,
|
|
};
|
|
let mut dataset = None;
|
|
let mut max_bytes = None;
|
|
let mut file = None;
|
|
while let Some(a) = args.next() {
|
|
match a.as_str() {
|
|
"--json" | "-j" => opts.json = true,
|
|
"-A" | "--header" => opts.header_only = true,
|
|
"-p" | "--properties" => opts.props = true,
|
|
"-d" | "--dataset" => match args.value() {
|
|
Some(p) => dataset = Some(p),
|
|
None => return args.usage_error(out, "-d needs a path", USAGE),
|
|
},
|
|
"--max-bytes" => match args.number() {
|
|
Some(n) => max_bytes = Some(n),
|
|
None => return args.usage_error(out, "--max-bytes needs a number", USAGE),
|
|
},
|
|
"-h" | "--help" => {
|
|
writeln!(out.o, "{USAGE}")?;
|
|
return Ok(0);
|
|
}
|
|
s if s.starts_with('-') && s.len() > 1 => {
|
|
return args.usage_error(out, &format!("unknown option {a}"), USAGE);
|
|
}
|
|
_ if file.is_none() => file = Some(a),
|
|
_ => return args.usage_error(out, &format!("unexpected argument {a}"), USAGE),
|
|
}
|
|
}
|
|
let Some(file) = file else {
|
|
return args.usage_error(out, "missing FILE", USAGE);
|
|
};
|
|
let mut h5 = match H5::open(std::path::Path::new(&file)) {
|
|
Ok(h) => h,
|
|
Err(e) => {
|
|
writeln!(out.e, "h5rs dump: {e}")?;
|
|
return Ok(2);
|
|
}
|
|
};
|
|
if let Some(m) = max_bytes {
|
|
h5.max_bytes = m;
|
|
}
|
|
let mut d = Dump {
|
|
h5: &h5,
|
|
opts,
|
|
problems: 0,
|
|
paths: OnceCell::new(),
|
|
};
|
|
let fname = std::path::Path::new(&file)
|
|
.file_name()
|
|
.map(|s| s.to_string_lossy().into_owned())
|
|
.unwrap_or(file.clone());
|
|
let code = if d.opts.json {
|
|
d.json(out, dataset.as_deref())?
|
|
} else {
|
|
d.ddl(out, &fname, dataset.as_deref())?
|
|
};
|
|
if code != 0 {
|
|
return Ok(code);
|
|
}
|
|
Ok(if d.problems > 0 { 1 } else { 0 })
|
|
}
|
|
|
|
/// A full path for `name` inside the group at `base`.
|
|
fn join(base: &str, name: &str) -> String {
|
|
if base == "/" {
|
|
format!("/{name}")
|
|
} else {
|
|
format!("{base}/{name}")
|
|
}
|
|
}
|
|
|
|
fn quote(s: &str) -> String {
|
|
s.replace('\\', "\\\\").replace('"', "\\\"")
|
|
}
|
|
|
|
impl Dump<'_> {
|
|
/// Object address -> first path, for printing references.
|
|
fn paths(&self) -> &HashMap<u64, String> {
|
|
self.paths.get_or_init(|| {
|
|
let mut m = HashMap::new();
|
|
let _ = self.h5.walk(|it| {
|
|
if let (Some(a), None) = (it.addr, it.first_path) {
|
|
m.entry(a).or_insert_with(|| it.path.to_string());
|
|
}
|
|
});
|
|
m
|
|
})
|
|
}
|
|
|
|
fn problem(&mut self, out: &mut Out, what: &str, e: &Error) -> std::io::Result<()> {
|
|
self.problems += 1;
|
|
writeln!(out.e, "h5rs dump: {what}: {e}")
|
|
}
|
|
|
|
// ------------------------------------------------------------------
|
|
// DDL
|
|
// ------------------------------------------------------------------
|
|
|
|
fn ddl(&mut self, out: &mut Out, fname: &str, only: Option<&str>) -> std::io::Result<i32> {
|
|
writeln!(out.o, "HDF5 \"{}\" {{", quote(fname))?;
|
|
if let Some(p) = only {
|
|
let h = match self.h5.resolve(p).and_then(|a| self.h5.header(a)) {
|
|
Ok(h) => h,
|
|
Err(e) => {
|
|
writeln!(out.o, "}}")?;
|
|
writeln!(out.e, "h5rs dump: {p}: {e}")?;
|
|
return Ok(2);
|
|
}
|
|
};
|
|
if Kind::of(&h) != Kind::Dataset {
|
|
writeln!(out.o, "}}")?;
|
|
writeln!(out.e, "h5rs dump: {p}: not a dataset")?;
|
|
return Ok(2);
|
|
}
|
|
let full = if p.starts_with('/') {
|
|
p.to_string()
|
|
} else {
|
|
format!("/{p}")
|
|
};
|
|
self.ddl_dataset(out, &full, &full, &h, 0)?;
|
|
} else {
|
|
let root = self.h5.root();
|
|
let mut seen = HashMap::new();
|
|
match self.h5.header(root) {
|
|
Ok(h) => self.ddl_group(out, "/", "/", root, &h, 0, &mut seen)?,
|
|
Err(e) => {
|
|
writeln!(out.o, "GROUP \"/\" {{\n}}")?;
|
|
self.problem(out, "/", &e)?;
|
|
}
|
|
}
|
|
}
|
|
writeln!(out.o, "}}")?;
|
|
Ok(0)
|
|
}
|
|
|
|
#[allow(clippy::too_many_arguments)]
|
|
fn ddl_group(
|
|
&mut self,
|
|
out: &mut Out,
|
|
name: &str,
|
|
path: &str,
|
|
addr: u64,
|
|
h: &ObjectHeader,
|
|
ind: usize,
|
|
seen: &mut HashMap<u64, String>,
|
|
) -> std::io::Result<()> {
|
|
let pad = " ".repeat(ind);
|
|
seen.insert(addr, path.to_string());
|
|
writeln!(out.o, "{pad}GROUP \"{}\" {{", quote(name))?;
|
|
self.ddl_attributes(out, path, h, ind + 3)?;
|
|
let links = match self.h5.links(h) {
|
|
Ok(l) => l,
|
|
Err(e) => {
|
|
self.problem(out, path, &e)?;
|
|
Vec::new()
|
|
}
|
|
};
|
|
// The DDL recursion follows the group nesting; bound it (the walk used
|
|
// by the other commands is iterative).
|
|
if seen.len() > crate::h5::MAX_OBJECTS || ind > 3 * MAX_DDL_DEPTH {
|
|
let e = Error::new("too many objects, or groups nested too deeply; stopped");
|
|
self.problem(out, path, &e)?;
|
|
return writeln!(out.o, "{pad}}}");
|
|
}
|
|
for l in &links {
|
|
self.ddl_link(out, path, l, ind + 3, seen)?;
|
|
}
|
|
writeln!(out.o, "{pad}}}")
|
|
}
|
|
|
|
fn ddl_link(
|
|
&mut self,
|
|
out: &mut Out,
|
|
base: &str,
|
|
l: &Link,
|
|
ind: usize,
|
|
seen: &mut HashMap<u64, String>,
|
|
) -> std::io::Result<()> {
|
|
let pad = " ".repeat(ind);
|
|
let inner = " ".repeat(ind + 3);
|
|
let name = quote(&l.name);
|
|
let path = join(base, &l.name);
|
|
match &l.kind {
|
|
LinkKind::Soft(t) => writeln!(
|
|
out.o,
|
|
"{pad}SOFTLINK \"{name}\" {{\n{inner}LINKTARGET \"{}\"\n{pad}}}",
|
|
quote(t)
|
|
),
|
|
LinkKind::External { file, path: p } => writeln!(
|
|
out.o,
|
|
"{pad}EXTERNAL_LINK \"{name}\" {{\n{inner}TARGETFILE \"{}\"\n{inner}TARGETPATH \"{}\"\n{pad}}}",
|
|
quote(file),
|
|
quote(p)
|
|
),
|
|
LinkKind::UserDefined(t) => writeln!(
|
|
out.o,
|
|
"{pad}USERDEFINED_LINK \"{name}\" {{\n{inner}LINKCLASS {t}\n{pad}}}"
|
|
),
|
|
LinkKind::Hard(a) => {
|
|
let h = match self.h5.header(*a) {
|
|
Ok(h) => h,
|
|
Err(e) => {
|
|
self.problem(out, &path, &e)?;
|
|
return writeln!(out.o, "{pad}UNKNOWN_OBJECT \"{name}\" {{\n{pad}}}");
|
|
}
|
|
};
|
|
let kind = Kind::of(&h);
|
|
if let Some(first) = seen.get(a) {
|
|
let word = match kind {
|
|
Kind::Group => "GROUP",
|
|
Kind::Dataset => "DATASET",
|
|
Kind::Datatype => "DATATYPE",
|
|
Kind::Unknown => "OBJECT",
|
|
};
|
|
return writeln!(
|
|
out.o,
|
|
"{pad}{word} \"{name}\" {{\n{inner}HARDLINK \"{}\"\n{pad}}}",
|
|
quote(first)
|
|
);
|
|
}
|
|
match kind {
|
|
Kind::Group => self.ddl_group(out, &l.name, &path, *a, &h, ind, seen),
|
|
Kind::Dataset => {
|
|
seen.insert(*a, path.clone());
|
|
self.ddl_dataset(out, &l.name, &path, &h, ind)
|
|
}
|
|
Kind::Datatype => {
|
|
seen.insert(*a, path.clone());
|
|
match self.h5.datatype(&h) {
|
|
Ok(dt) => writeln!(
|
|
out.o,
|
|
"{pad}DATATYPE \"{name}\" {};",
|
|
crate::dtype::ddl(&dt, ind)
|
|
),
|
|
Err(e) => {
|
|
self.problem(out, &path, &e)?;
|
|
writeln!(out.o, "{pad}DATATYPE \"{name}\" ?;")
|
|
}
|
|
}
|
|
}
|
|
Kind::Unknown => {
|
|
seen.insert(*a, path.clone());
|
|
writeln!(out.o, "{pad}UNKNOWN_OBJECT \"{name}\" {{\n{pad}}}")
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
fn ddl_dataset(
|
|
&mut self,
|
|
out: &mut Out,
|
|
name: &str,
|
|
path: &str,
|
|
h: &ObjectHeader,
|
|
ind: usize,
|
|
) -> std::io::Result<()> {
|
|
let pad = " ".repeat(ind);
|
|
let inner = " ".repeat(ind + 3);
|
|
writeln!(out.o, "{pad}DATASET \"{}\" {{", quote(name))?;
|
|
let info = DsInfo::read(self.h5, path, h);
|
|
match &info.dt {
|
|
Ok(dt) => writeln!(out.o, "{inner}DATATYPE {}", crate::dtype::ddl(dt, ind + 3))?,
|
|
Err(e) => self.problem(out, path, e)?,
|
|
}
|
|
match &info.ds {
|
|
Ok(ds) => writeln!(out.o, "{inner}DATASPACE {}", info::dataspace_ddl(ds))?,
|
|
Err(e) => self.problem(out, path, e)?,
|
|
}
|
|
if self.opts.props {
|
|
self.ddl_properties(out, path, &info, ind + 3)?;
|
|
}
|
|
if !self.opts.header_only
|
|
&& let (Ok(dt), Ok(ds)) = (&info.dt, &info.ds)
|
|
{
|
|
match self.h5.read_dataset(path, dt, ds) {
|
|
Ok(raw) => self.ddl_data(out, dt, ds, &raw, ind + 3)?,
|
|
Err(e) => {
|
|
writeln!(out.o, "{inner}DATA {{\n{inner}<not read: {e}>\n{inner}}}")?;
|
|
self.problem(out, path, &e)?;
|
|
}
|
|
}
|
|
}
|
|
self.ddl_attributes(out, path, h, ind + 3)?;
|
|
writeln!(out.o, "{pad}}}")
|
|
}
|
|
|
|
fn ddl_properties(
|
|
&mut self,
|
|
out: &mut Out,
|
|
path: &str,
|
|
info: &DsInfo,
|
|
ind: usize,
|
|
) -> std::io::Result<()> {
|
|
let pad = " ".repeat(ind);
|
|
let inner = " ".repeat(ind + 3);
|
|
let Ok(layout) = &info.layout else {
|
|
if let Err(e) = &info.layout {
|
|
self.problem(out, path, e)?;
|
|
}
|
|
return Ok(());
|
|
};
|
|
writeln!(out.o, "{pad}STORAGE_LAYOUT {{")?;
|
|
match layout {
|
|
DataLayout::Chunked {
|
|
chunk_dimensions, ..
|
|
} => {
|
|
let rank = chunk_dimensions.len().saturating_sub(1);
|
|
writeln!(
|
|
out.o,
|
|
"{inner}CHUNKED ( {} )",
|
|
chunk_dimensions[..rank]
|
|
.iter()
|
|
.map(|d| d.to_string())
|
|
.collect::<Vec<_>>()
|
|
.join(", ")
|
|
)?;
|
|
writeln!(
|
|
out.o,
|
|
"{inner}INDEX {}",
|
|
info::chunk_index_name(layout).to_uppercase()
|
|
)?;
|
|
}
|
|
DataLayout::Contiguous { address, size } => {
|
|
writeln!(out.o, "{inner}CONTIGUOUS")?;
|
|
match address {
|
|
Some(a) => writeln!(out.o, "{inner}OFFSET {a}")?,
|
|
None => writeln!(out.o, "{inner}NOT ALLOCATED")?,
|
|
}
|
|
writeln!(out.o, "{inner}SIZE {size}")?;
|
|
}
|
|
DataLayout::Compact { data } => {
|
|
writeln!(out.o, "{inner}COMPACT")?;
|
|
writeln!(out.o, "{inner}SIZE {}", data.len())?;
|
|
}
|
|
DataLayout::Virtual { .. } => writeln!(out.o, "{inner}VIRTUAL")?,
|
|
}
|
|
if let Ok(a) = info::allocated_bytes(self.h5, info)
|
|
&& matches!(layout, DataLayout::Chunked { .. })
|
|
{
|
|
writeln!(out.o, "{inner}SIZE {a}")?;
|
|
}
|
|
writeln!(out.o, "{pad}}}")?;
|
|
writeln!(out.o, "{pad}FILTERS {{")?;
|
|
match &info.filters {
|
|
Ok(Some(p)) if !p.filters.is_empty() => {
|
|
for f in &p.filters {
|
|
writeln!(
|
|
out.o,
|
|
"{inner}{} {{ ID {}; PARAMS {{ {} }} }}",
|
|
info::filter_name(f).to_uppercase(),
|
|
f.filter_id,
|
|
f.client_data
|
|
.iter()
|
|
.map(|c| c.to_string())
|
|
.collect::<Vec<_>>()
|
|
.join(" ")
|
|
)?;
|
|
}
|
|
}
|
|
Ok(_) => writeln!(out.o, "{inner}NONE")?,
|
|
Err(e) => self.problem(out, path, e)?,
|
|
}
|
|
writeln!(out.o, "{pad}}}")
|
|
}
|
|
|
|
fn ddl_attributes(
|
|
&mut self,
|
|
out: &mut Out,
|
|
path: &str,
|
|
h: &ObjectHeader,
|
|
ind: usize,
|
|
) -> std::io::Result<()> {
|
|
let (attrs, errs) = match self.h5.attributes(h) {
|
|
Ok(x) => x,
|
|
Err(e) => return self.problem(out, path, &e),
|
|
};
|
|
for e in errs {
|
|
self.problem(out, path, &Error::new(format!("attribute: {e}")))?;
|
|
}
|
|
for a in &attrs {
|
|
self.ddl_attribute(out, path, a, ind)?;
|
|
}
|
|
Ok(())
|
|
}
|
|
|
|
fn ddl_attribute(
|
|
&mut self,
|
|
out: &mut Out,
|
|
path: &str,
|
|
a: &AttributeMessage,
|
|
ind: usize,
|
|
) -> std::io::Result<()> {
|
|
let pad = " ".repeat(ind);
|
|
let inner = " ".repeat(ind + 3);
|
|
writeln!(out.o, "{pad}ATTRIBUTE \"{}\" {{", quote(&a.name))?;
|
|
writeln!(
|
|
out.o,
|
|
"{inner}DATATYPE {}",
|
|
crate::dtype::ddl(&a.datatype, ind + 3)
|
|
)?;
|
|
writeln!(
|
|
out.o,
|
|
"{inner}DATASPACE {}",
|
|
info::dataspace_ddl(&a.dataspace)
|
|
)?;
|
|
{
|
|
match self.checked_attr(a) {
|
|
Ok(()) => self.ddl_data(out, &a.datatype, &a.dataspace, &a.raw_data, ind + 3)?,
|
|
Err(e) => {
|
|
writeln!(out.o, "{inner}DATA {{\n{inner}<not read: {e}>\n{inner}}}")?;
|
|
self.problem(out, &format!("{path} attribute {}", a.name), &e)?;
|
|
}
|
|
}
|
|
}
|
|
writeln!(out.o, "{pad}}}")
|
|
}
|
|
|
|
/// An attribute's value holds as many bytes as its dataspace says.
|
|
fn checked_attr(&self, a: &AttributeMessage) -> crate::h5::Result<()> {
|
|
let need = crate::h5::byte_len(&a.dataspace, &a.datatype)?;
|
|
if need > self.h5.max_bytes {
|
|
return Err(Error::new(format!(
|
|
"attribute is {need} bytes, over the {} byte limit (--max-bytes)",
|
|
self.h5.max_bytes
|
|
)));
|
|
}
|
|
if (a.raw_data.len() as u64) < need {
|
|
return Err(Error::new(format!(
|
|
"attribute holds {} bytes, its dataspace needs {need}",
|
|
a.raw_data.len()
|
|
)));
|
|
}
|
|
Ok(())
|
|
}
|
|
|
|
/// An h5dump DATA block: elements row by row, each line starting with
|
|
/// the index of its first element, wrapped where h5dump wraps (78 columns).
|
|
fn ddl_data(
|
|
&mut self,
|
|
out: &mut Out,
|
|
dt: &Datatype,
|
|
ds: &Dataspace,
|
|
raw: &[u8],
|
|
ind: usize,
|
|
) -> std::io::Result<()> {
|
|
let pad = " ".repeat(ind);
|
|
writeln!(out.o, "{pad}DATA {{")?;
|
|
let dims: Vec<u64> = match ds.space_type {
|
|
DataspaceType::Null => {
|
|
return writeln!(out.o, "{pad}}}");
|
|
}
|
|
DataspaceType::Scalar => vec![1],
|
|
DataspaceType::Simple => ds.dimensions.clone(),
|
|
};
|
|
let n = crate::h5::num_elements(ds).unwrap_or(0) as usize;
|
|
let dec = Decoder::new(self.h5);
|
|
let last = dims.last().copied().unwrap_or(1).max(1) as usize;
|
|
let paths = |a: u64| self.paths().get(&a).cloned();
|
|
let mut line = String::new();
|
|
let mut errors = 0usize;
|
|
for i in 0..n {
|
|
let v = dec.element(dt, raw, i);
|
|
if matches!(v, Value::Error(_)) {
|
|
errors += 1;
|
|
}
|
|
let comma = if i + 1 < n { "," } else { "" };
|
|
let esize = dt.type_size() as usize;
|
|
let eb = raw.get(i * esize..(i + 1) * esize).unwrap_or_default();
|
|
if let (Value::Compound(_), Datatype::Compound { members, .. }) = (&v, dt) {
|
|
// h5dump prints each compound element as a block, one
|
|
// member per line.
|
|
if !line.is_empty() {
|
|
writeln!(out.o, "{line}")?;
|
|
line.clear();
|
|
}
|
|
writeln!(out.o, "{pad}({}): {{", index_text(i as u64, &dims))?;
|
|
for (k, m) in members.iter().enumerate() {
|
|
let sep = if k + 1 < members.len() { "," } else { "" };
|
|
let mb = usize::try_from(m.byte_offset)
|
|
.ok()
|
|
.and_then(|o| eb.get(o..))
|
|
.unwrap_or_default();
|
|
let t = ddl_text(&dec, &m.datatype, mb, &paths, 1);
|
|
writeln!(out.o, "{pad} {t}{sep}")?;
|
|
}
|
|
writeln!(out.o, "{pad} }}{comma}")?;
|
|
continue;
|
|
}
|
|
let t = if matches!(v, Value::Error(_)) {
|
|
format!("{}{comma}", value::text(&v, &paths))
|
|
} else {
|
|
format!("{}{comma}", ddl_text(&dec, dt, eb, &paths, 0))
|
|
};
|
|
let row_start = i % last == 0;
|
|
if row_start || line.len() + 1 + t.len() > 77 {
|
|
if !line.is_empty() {
|
|
writeln!(out.o, "{line}")?;
|
|
}
|
|
line = format!("{pad}({}): {t}", index_text(i as u64, &dims));
|
|
} else {
|
|
line.push(' ');
|
|
line.push_str(&t);
|
|
}
|
|
}
|
|
if !line.is_empty() {
|
|
writeln!(out.o, "{line}")?;
|
|
}
|
|
if errors > 0 {
|
|
self.problems += 1;
|
|
writeln!(out.e, "h5rs dump: {errors} element(s) could not be decoded")?;
|
|
}
|
|
writeln!(out.o, "{pad}}}")
|
|
}
|
|
|
|
// ------------------------------------------------------------------
|
|
// JSON
|
|
// ------------------------------------------------------------------
|
|
|
|
fn json(&mut self, out: &mut Out, only: Option<&str>) -> std::io::Result<i32> {
|
|
let mut groups = Map::new();
|
|
let mut datasets = Map::new();
|
|
let mut datatypes = Map::new();
|
|
let mut doc = Map::new();
|
|
doc.insert("apiVersion".into(), json!("1.1.1"));
|
|
if let Some(p) = only {
|
|
let full = if p.starts_with('/') {
|
|
p.to_string()
|
|
} else {
|
|
format!("/{p}")
|
|
};
|
|
let r = self
|
|
.h5
|
|
.resolve(&full)
|
|
.and_then(|a| Ok((a, self.h5.header(a)?)));
|
|
match r {
|
|
Ok((a, h)) if Kind::of(&h) == Kind::Dataset => {
|
|
let obj = self.json_dataset(out, &full, &h)?;
|
|
datasets.insert(obj_id(Kind::Dataset, a), obj);
|
|
}
|
|
Ok(_) => {
|
|
writeln!(out.e, "h5rs dump: {p}: not a dataset")?;
|
|
return Ok(2);
|
|
}
|
|
Err(e) => {
|
|
writeln!(out.e, "h5rs dump: {p}: {e}")?;
|
|
return Ok(2);
|
|
}
|
|
}
|
|
} else {
|
|
doc.insert("root".into(), json!(obj_id(Kind::Group, self.h5.root())));
|
|
let (items, walk) = self.h5.walk_collect();
|
|
if let Err(e) = walk {
|
|
self.problem(out, "/", &e)?;
|
|
}
|
|
// Aliases: every path an object is reachable by.
|
|
let mut aliases: HashMap<u64, Vec<String>> = HashMap::new();
|
|
for it in &items {
|
|
if let Some(a) = it.addr {
|
|
aliases.entry(a).or_default().push(it.path.clone());
|
|
}
|
|
}
|
|
for it in items {
|
|
let path = it.path;
|
|
let (Some(addr), None, Some(header)) = (it.addr, it.first_path, it.header) else {
|
|
continue;
|
|
};
|
|
let h = match header {
|
|
Ok(h) => h,
|
|
Err(e) => {
|
|
self.problem(out, &path, &e)?;
|
|
continue;
|
|
}
|
|
};
|
|
let kind = Kind::of(&h);
|
|
let alias = aliases.remove(&addr).unwrap_or_default();
|
|
let mut obj = match kind {
|
|
Kind::Dataset => self.json_dataset(out, &path, &h)?,
|
|
Kind::Datatype => {
|
|
let mut m = Map::new();
|
|
match self.h5.datatype(&h) {
|
|
Ok(dt) => {
|
|
m.insert("type".into(), crate::dtype::json(&dt));
|
|
}
|
|
Err(e) => self.problem(out, &path, &e)?,
|
|
}
|
|
m.insert("attributes".into(), self.json_attributes(out, &path, &h)?);
|
|
J::Object(m)
|
|
}
|
|
_ => {
|
|
let mut m = Map::new();
|
|
m.insert("attributes".into(), self.json_attributes(out, &path, &h)?);
|
|
let links = match self.h5.links(&h) {
|
|
Ok(l) => l,
|
|
Err(e) => {
|
|
if kind == Kind::Group {
|
|
self.problem(out, &path, &e)?;
|
|
}
|
|
Vec::new()
|
|
}
|
|
};
|
|
let mut jl = Vec::new();
|
|
for l in &links {
|
|
jl.push(self.json_link(l));
|
|
}
|
|
m.insert("links".into(), J::Array(jl));
|
|
J::Object(m)
|
|
}
|
|
};
|
|
if let J::Object(m) = &mut obj {
|
|
m.insert("alias".into(), json!(alias));
|
|
}
|
|
let id = obj_id(kind, addr);
|
|
match kind {
|
|
Kind::Dataset => datasets.insert(id, obj),
|
|
Kind::Datatype => datatypes.insert(id, obj),
|
|
_ => groups.insert(id, obj),
|
|
};
|
|
}
|
|
doc.insert("groups".into(), J::Object(groups));
|
|
}
|
|
doc.insert("datasets".into(), J::Object(datasets));
|
|
if only.is_none() {
|
|
doc.insert("datatypes".into(), J::Object(datatypes));
|
|
}
|
|
let text = serde_json::to_string_pretty(&J::Object(doc)).map_err(std::io::Error::other)?;
|
|
writeln!(out.o, "{text}")?;
|
|
Ok(0)
|
|
}
|
|
|
|
fn json_link(&self, l: &Link) -> J {
|
|
match &l.kind {
|
|
LinkKind::Hard(a) => {
|
|
let kind = self
|
|
.h5
|
|
.header(*a)
|
|
.map(|h| Kind::of(&h))
|
|
.unwrap_or(Kind::Unknown);
|
|
let coll = match kind {
|
|
Kind::Dataset => "datasets",
|
|
Kind::Datatype => "datatypes",
|
|
_ => "groups",
|
|
};
|
|
json!({"class": "H5L_TYPE_HARD", "title": l.name, "collection": coll,
|
|
"id": obj_id(kind, *a)})
|
|
}
|
|
LinkKind::Soft(t) => json!({"class": "H5L_TYPE_SOFT", "title": l.name, "h5path": t}),
|
|
LinkKind::External { file, path } => json!({"class": "H5L_TYPE_EXTERNAL",
|
|
"title": l.name, "file": file, "h5path": path}),
|
|
LinkKind::UserDefined(t) => json!({"class": "H5L_TYPE_USER_DEFINED",
|
|
"title": l.name, "linkClass": t}),
|
|
}
|
|
}
|
|
|
|
fn json_dataset(&mut self, out: &mut Out, path: &str, h: &ObjectHeader) -> std::io::Result<J> {
|
|
let info = DsInfo::read(self.h5, path, h);
|
|
let mut m = Map::new();
|
|
match &info.dt {
|
|
Ok(dt) => {
|
|
m.insert("type".into(), crate::dtype::json(dt));
|
|
}
|
|
Err(e) => self.problem(out, path, e)?,
|
|
}
|
|
match &info.ds {
|
|
Ok(ds) => {
|
|
m.insert("shape".into(), shape_json(ds));
|
|
}
|
|
Err(e) => self.problem(out, path, e)?,
|
|
}
|
|
let mut cp = Map::new();
|
|
if let Ok(l) = &info.layout {
|
|
let mut lj = Map::new();
|
|
lj.insert(
|
|
"class".into(),
|
|
json!(match l {
|
|
DataLayout::Compact { .. } => "H5D_COMPACT",
|
|
DataLayout::Contiguous { .. } => "H5D_CONTIGUOUS",
|
|
DataLayout::Chunked { .. } => "H5D_CHUNKED",
|
|
DataLayout::Virtual { .. } => "H5D_VIRTUAL",
|
|
}),
|
|
);
|
|
if let DataLayout::Chunked {
|
|
chunk_dimensions, ..
|
|
} = l
|
|
{
|
|
let rank = chunk_dimensions.len().saturating_sub(1);
|
|
lj.insert("dims".into(), json!(chunk_dimensions[..rank]));
|
|
}
|
|
cp.insert("layout".into(), J::Object(lj));
|
|
} else if let Err(e) = &info.layout {
|
|
self.problem(out, path, e)?;
|
|
}
|
|
if let Ok(Some(p)) = &info.filters {
|
|
let fl: Vec<J> = p
|
|
.filters
|
|
.iter()
|
|
.map(|f| {
|
|
json!({"id": f.filter_id, "name": info::filter_name(f),
|
|
"class": filter_class(f.filter_id), "parameters": f.client_data})
|
|
})
|
|
.collect();
|
|
cp.insert("filters".into(), J::Array(fl));
|
|
}
|
|
m.insert("creationProperties".into(), J::Object(cp));
|
|
if !self.opts.header_only
|
|
&& let (Ok(dt), Ok(ds)) = (&info.dt, &info.ds)
|
|
{
|
|
// JSON values are built in memory, some 32+ bytes per element:
|
|
// hold them to the same budget as the raw data.
|
|
let n = crate::h5::num_elements(ds).unwrap_or(u64::MAX);
|
|
let read = if n.saturating_mul(JSON_BYTES_PER_ELEMENT) > self.h5.max_bytes {
|
|
Err(Error::new(format!(
|
|
"{n} elements are too many to hold as JSON within --max-bytes {}",
|
|
self.h5.max_bytes
|
|
))
|
|
.with_kind(crate::h5::ErrorKind::Limit))
|
|
} else {
|
|
self.h5.read_dataset(path, dt, ds)
|
|
};
|
|
match read {
|
|
Ok(raw) => {
|
|
let v = self.json_values(out, dt, ds, &raw)?;
|
|
m.insert("value".into(), v);
|
|
}
|
|
Err(e) => {
|
|
m.insert("value_error".into(), json!(e.to_string()));
|
|
self.problem(out, path, &e)?;
|
|
}
|
|
}
|
|
}
|
|
m.insert("attributes".into(), self.json_attributes(out, path, h)?);
|
|
Ok(J::Object(m))
|
|
}
|
|
|
|
fn json_attributes(
|
|
&mut self,
|
|
out: &mut Out,
|
|
path: &str,
|
|
h: &ObjectHeader,
|
|
) -> std::io::Result<J> {
|
|
let (attrs, errs) = match self.h5.attributes(h) {
|
|
Ok(x) => x,
|
|
Err(e) => {
|
|
self.problem(out, path, &e)?;
|
|
return Ok(J::Array(Vec::new()));
|
|
}
|
|
};
|
|
for e in errs {
|
|
self.problem(out, path, &Error::new(format!("attribute: {e}")))?;
|
|
}
|
|
let mut v = Vec::new();
|
|
for a in &attrs {
|
|
let mut m = Map::new();
|
|
m.insert("name".into(), json!(a.name));
|
|
m.insert("shape".into(), shape_json(&a.dataspace));
|
|
m.insert("type".into(), crate::dtype::json(&a.datatype));
|
|
{
|
|
match self.checked_attr(a) {
|
|
Ok(()) => {
|
|
let val = self.json_values(out, &a.datatype, &a.dataspace, &a.raw_data)?;
|
|
m.insert("value".into(), val);
|
|
}
|
|
Err(e) => {
|
|
m.insert("value_error".into(), json!(e.to_string()));
|
|
self.problem(out, &format!("{path} attribute {}", a.name), &e)?;
|
|
}
|
|
}
|
|
}
|
|
v.push(J::Object(m));
|
|
}
|
|
Ok(J::Array(v))
|
|
}
|
|
|
|
fn json_values(
|
|
&mut self,
|
|
out: &mut Out,
|
|
dt: &Datatype,
|
|
ds: &Dataspace,
|
|
raw: &[u8],
|
|
) -> std::io::Result<J> {
|
|
let n = crate::h5::num_elements(ds).unwrap_or(0) as usize;
|
|
let dec = Decoder::new(self.h5);
|
|
let paths = |a: u64| self.paths().get(&a).cloned();
|
|
let mut flat = Vec::with_capacity(n);
|
|
let mut errors = 0usize;
|
|
for i in 0..n {
|
|
let v = dec.element(dt, raw, i);
|
|
if matches!(v, Value::Error(_)) {
|
|
errors += 1;
|
|
}
|
|
flat.push(value::to_json(&v, &paths));
|
|
}
|
|
if errors > 0 {
|
|
self.problems += 1;
|
|
writeln!(out.e, "h5rs dump: {errors} element(s) could not be decoded")?;
|
|
}
|
|
Ok(match ds.space_type {
|
|
DataspaceType::Null => J::Null,
|
|
DataspaceType::Scalar => flat.into_iter().next().unwrap_or(J::Null),
|
|
DataspaceType::Simple => nest(flat, &ds.dimensions),
|
|
})
|
|
}
|
|
}
|
|
|
|
/// Deepest group nesting the DDL output follows.
|
|
const MAX_DDL_DEPTH: usize = 256;
|
|
|
|
/// Memory budgeted per element for a dataset's value held as JSON.
|
|
const JSON_BYTES_PER_ELEMENT: u64 = 64;
|
|
|
|
/// `(i,j,k)` index of flat element `i` of an array with `dims`.
|
|
/// One element as h5dump prints it in a DATA block: [`value::text`], except
|
|
/// that a null-padded fixed string shows its padding (every byte, NULs as
|
|
/// `\000`) at any depth, in compound members and array elements too.
|
|
fn ddl_text(
|
|
dec: &Decoder,
|
|
dt: &Datatype,
|
|
b: &[u8],
|
|
paths: &dyn Fn(u64) -> Option<String>,
|
|
depth: u32,
|
|
) -> String {
|
|
let err = || value::text(&Value::Error("short element".into()), paths);
|
|
if depth > 32 {
|
|
return value::text(&dec.decode(dt, b, depth), paths);
|
|
}
|
|
match dt {
|
|
Datatype::String {
|
|
padding: StringPadding::NullPad,
|
|
size,
|
|
..
|
|
} => match b.get(..*size as usize) {
|
|
Some(s) => value::quote_bytes(s),
|
|
None => err(),
|
|
},
|
|
Datatype::Compound { members, .. } => format!(
|
|
"{{ {} }}",
|
|
members
|
|
.iter()
|
|
.map(
|
|
|m| match usize::try_from(m.byte_offset).ok().and_then(|o| b.get(o..)) {
|
|
Some(mb) => ddl_text(dec, &m.datatype, mb, paths, depth + 1),
|
|
None => err(),
|
|
}
|
|
)
|
|
.collect::<Vec<_>>()
|
|
.join(", ")
|
|
),
|
|
Datatype::Array {
|
|
base_type,
|
|
dimensions,
|
|
} => {
|
|
let bs = base_type.type_size() as usize;
|
|
let n = dimensions
|
|
.iter()
|
|
.try_fold(1usize, |a, &d| a.checked_mul(d as usize));
|
|
match n.filter(|n| n.checked_mul(bs).is_some_and(|t| t <= b.len())) {
|
|
Some(n) => format!(
|
|
"[ {} ]",
|
|
(0..n)
|
|
.map(|k| ddl_text(dec, base_type, &b[k * bs..], paths, depth + 1))
|
|
.collect::<Vec<_>>()
|
|
.join(", ")
|
|
),
|
|
None => value::text(&dec.decode(dt, b, depth), paths),
|
|
}
|
|
}
|
|
_ => value::text(&dec.decode(dt, b, depth), paths),
|
|
}
|
|
}
|
|
|
|
fn index_text(mut i: u64, dims: &[u64]) -> String {
|
|
let mut idx = vec![0u64; dims.len()];
|
|
for (k, &d) in dims.iter().enumerate().rev() {
|
|
let d = d.max(1);
|
|
idx[k] = i % d;
|
|
i /= d;
|
|
}
|
|
idx.iter()
|
|
.map(|x| x.to_string())
|
|
.collect::<Vec<_>>()
|
|
.join(",")
|
|
}
|
|
|
|
/// Turn a flat row-major list into nested lists of shape `dims`.
|
|
fn nest(flat: Vec<J>, dims: &[u64]) -> J {
|
|
if dims.len() <= 1 {
|
|
return J::Array(flat);
|
|
}
|
|
let inner: usize = dims[1..].iter().map(|&d| d as usize).product();
|
|
if inner == 0 {
|
|
return J::Array((0..dims[0]).map(|_| nest(Vec::new(), &dims[1..])).collect());
|
|
}
|
|
let mut it = flat.into_iter();
|
|
let mut outer = Vec::with_capacity(dims[0] as usize);
|
|
for _ in 0..dims[0] {
|
|
let part: Vec<J> = it.by_ref().take(inner).collect();
|
|
outer.push(nest(part, &dims[1..]));
|
|
}
|
|
J::Array(outer)
|
|
}
|
|
|
|
fn shape_json(ds: &Dataspace) -> J {
|
|
match ds.space_type {
|
|
DataspaceType::Null => json!({"class": "H5S_NULL"}),
|
|
DataspaceType::Scalar => json!({"class": "H5S_SCALAR"}),
|
|
DataspaceType::Simple => {
|
|
let mut m = Map::new();
|
|
m.insert("class".into(), json!("H5S_SIMPLE"));
|
|
m.insert("dims".into(), json!(ds.dimensions));
|
|
if let Some(max) = &ds.max_dimensions {
|
|
let mx: Vec<J> = max
|
|
.iter()
|
|
.map(|&d| {
|
|
if d == u64::MAX {
|
|
json!("H5S_UNLIMITED")
|
|
} else {
|
|
json!(d)
|
|
}
|
|
})
|
|
.collect();
|
|
m.insert("maxdims".into(), J::Array(mx));
|
|
}
|
|
J::Object(m)
|
|
}
|
|
}
|
|
}
|
|
|
|
fn filter_class(id: u16) -> &'static str {
|
|
match id {
|
|
1 => "H5Z_FILTER_DEFLATE",
|
|
2 => "H5Z_FILTER_SHUFFLE",
|
|
3 => "H5Z_FILTER_FLETCHER32",
|
|
4 => "H5Z_FILTER_SZIP",
|
|
5 => "H5Z_FILTER_NBIT",
|
|
6 => "H5Z_FILTER_SCALEOFFSET",
|
|
_ => "H5Z_FILTER_USER",
|
|
}
|
|
}
|
|
|
|
/// Deterministic object id: kind prefix + header address (hdf5-json uses
|
|
/// UUIDs; these are stable for a given file instead).
|
|
pub fn obj_id(kind: Kind, addr: u64) -> String {
|
|
let p = match kind {
|
|
Kind::Dataset => 'd',
|
|
Kind::Datatype => 't',
|
|
_ => 'g',
|
|
};
|
|
format!("{p}-{addr:016x}")
|
|
}
|