//! Dataset facts shared by `ls`, `dump`, `stat` and `check`: shape text, //! layout, filters and storage. use clawhdf5_format::chunked_read::{ChunkInfo, list_chunks}; use clawhdf5_format::data_layout::DataLayout; use clawhdf5_format::dataspace::{Dataspace, DataspaceType}; use clawhdf5_format::datatype::Datatype; use clawhdf5_format::filter_pipeline::{FilterDescription, FilterPipeline}; use clawhdf5_format::message_type::MessageType; use clawhdf5_format::object_header::ObjectHeader; use crate::h5::{Error, H5, Result}; /// h5ls's `{10/Inf, 20}` shape text. `always_max` prints `cur/max` for every /// dimension (h5ls -v). pub fn shape_text(ds: &Dataspace, always_max: bool) -> String { match ds.space_type { DataspaceType::Null => "{NULL}".into(), DataspaceType::Scalar => "{SCALAR}".into(), DataspaceType::Simple => { let dims: Vec = ds .dimensions .iter() .enumerate() .map(|(i, &d)| { let m = ds.max_dimensions.as_ref().and_then(|m| m.get(i).copied()); match m { Some(u64::MAX) => format!("{d}/Inf"), Some(m) if m != d || always_max => format!("{d}/{m}"), None if always_max => format!("{d}/{d}"), _ => d.to_string(), } }) .collect(); format!("{{{}}}", dims.join(", ")) } } } /// h5dump's `SIMPLE { ( 3, 4 ) / ( 3, H5S_UNLIMITED ) }`. pub fn dataspace_ddl(ds: &Dataspace) -> String { match ds.space_type { DataspaceType::Null => "NULL".into(), DataspaceType::Scalar => "SCALAR".into(), DataspaceType::Simple => { let cur: Vec = ds.dimensions.iter().map(|d| d.to_string()).collect(); let max: Vec = ds .dimensions .iter() .enumerate() .map( |(i, &d)| match ds.max_dimensions.as_ref().and_then(|m| m.get(i).copied()) { Some(u64::MAX) => "H5S_UNLIMITED".into(), Some(m) => m.to_string(), None => d.to_string(), }, ) .collect(); format!( "SIMPLE {{ ( {} ) / ( {} ) }}", cur.join(", "), max.join(", ") ) } } } pub fn filter_name(f: &FilterDescription) -> String { let known = match f.filter_id { 1 => "deflate", 2 => "shuffle", 3 => "fletcher32", 4 => "szip", 5 => "nbit", 6 => "scaleoffset", 307 => "bzip2", 32000 => "lzf", 32001 => "blosc", 32004 => "lz4", 32008 => "bitshuffle", 32013 => "zfp", 32015 => "zstd", 32026 => "blosc2", _ => "", }; if !known.is_empty() { return known.into(); } match &f.name { Some(n) if !n.is_empty() => n.clone(), _ => "user-defined".into(), } } /// `deflate-1 OPT {4}` as h5ls prints a filter. pub fn filter_text(f: &FilterDescription) -> String { let opt = if f.flags & 1 != 0 { " OPT" } else { "" }; let cd = if f.client_data.is_empty() { String::new() } else { format!( " {{{}}}", f.client_data .iter() .map(|c| c.to_string()) .collect::>() .join(", ") ) }; format!("{}-{}{opt}{cd}", filter_name(f), f.filter_id) } pub fn layout_name(l: &DataLayout) -> &'static str { match l { DataLayout::Compact { .. } => "compact", DataLayout::Contiguous { .. } => "contiguous", DataLayout::Chunked { .. } => "chunked", DataLayout::Virtual { .. } => "virtual", } } /// Chunk index kind of a chunked layout. pub fn chunk_index_name(l: &DataLayout) -> &'static str { match l { DataLayout::Chunked { version, chunk_index_type, .. } => { if *version < 4 { return "v1 B-tree"; } match chunk_index_type { Some(1) => "single chunk", Some(2) => "implicit", Some(3) => "fixed array", Some(4) => "extensible array", Some(5) => "v2 B-tree", _ => "unknown", } } _ => "", } } /// Everything about one dataset that can be learned without reading its /// values. pub struct DsInfo { pub dt: Result, pub ds: Result, pub layout: Result, pub filters: Result>, pub external: bool, } impl DsInfo { pub fn read(h5: &H5, path: &str, h: &ObjectHeader) -> DsInfo { DsInfo { dt: h5.datatype(h), ds: h5.resolved_dataspace(path, h), layout: h5.layout(h), filters: h5.filters(h), external: h .messages .iter() .any(|m| m.msg_type == MessageType::ExternalDataFiles), } } pub fn logical_bytes(&self) -> Option { let (Ok(dt), Ok(ds)) = (&self.dt, &self.ds) else { return None; }; crate::h5::byte_len(ds, dt).ok() } } /// Every allocated chunk (empty when none are). Errors only for a corrupt /// chunk index. pub fn chunks( h5: &H5, layout: &DataLayout, ds: &Dataspace, dt: &Datatype, ) -> Result> { let DataLayout::Chunked { btree_address, .. } = layout else { return Ok(Vec::new()); }; let Some(addr) = *btree_address else { return Ok(Vec::new()); }; list_chunks( h5.data(), layout, ds, dt.type_size() as usize, h5.os(), h5.ls(), ) .map(|(c, _)| c) .map_err(|e| { Error::at( addr, format!("chunk index ({}): {e}", chunk_index_name(layout)), ) }) } /// Bytes of raw data the dataset has allocated in the file (what libhdf5's /// `H5Dget_storage_size` reports). pub fn allocated_bytes(h5: &H5, info: &DsInfo) -> Result { let layout = info.layout.as_ref().map_err(Clone::clone)?; Ok(match layout { DataLayout::Compact { data } => data.len() as u64, DataLayout::Contiguous { address, size } => { if address.is_some() { *size } else { 0 } } DataLayout::Chunked { .. } => { let dt = info.dt.as_ref().map_err(Clone::clone)?; let ds = info.ds.as_ref().map_err(Clone::clone)?; chunks(h5, layout, ds, dt)? .iter() .map(|c| u64::from(c.chunk_size)) .sum() } DataLayout::Virtual { .. } => 0, }) } /// Number of hard links to an object, as its header records it. pub fn link_count(h: &ObjectHeader) -> u64 { if let Some(rc) = h.reference_count { return u64::from(rc); } h.messages .iter() .find(|m| m.msg_type == MessageType::ObjectReferenceCount) .and_then(|m| m.data.get(1..5)) .map(|b| u64::from(u32::from_le_bytes([b[0], b[1], b[2], b[3]]))) .unwrap_or(1) }