//! h5py round-trip tests for the file writer.
//!
//! These tests write HDF5 files with our writer and verify h5py can read them
//! (and vice versa). They require python3 + h5py to be installed.

use clawhdf5_format::file_writer::{AttrValue, CompoundTypeBuilder, EnumTypeBuilder, FileWriter};

fn h5py_available() -> bool {
    std::process::Command::new("python3")
        .args(["-c", "import h5py"])
        .output()
        .map(|o| o.status.success())
        .unwrap_or(false)
}

fn h5py_read(_path: &std::path::Path, script: &str) -> String {
    if !h5py_available() {
        panic!("h5py not installed — skipping interop test");
    }
    let o = std::process::Command::new("python3")
        .args(["-c", script])
        .output()
        .expect("python3");
    if !o.status.success() {
        panic!("h5py: {}", String::from_utf8_lossy(&o.stderr));
    }
    String::from_utf8(o.stdout).unwrap().trim().to_string()
}

// ---- h5py round-trip: basic datasets ----

#[test]
#[ignore = "requires Python h5py module"]
fn h5py_reads_our_f64_dataset() {
    let mut fw = FileWriter::new();
    fw.create_dataset("data")
        .with_f64_data(&[1.0, 2.0, 3.0])
        .with_shape(&[3]);
    let bytes = fw.finish().unwrap();
    let path = std::env::temp_dir().join("clawhdf5_test_f64.h5");
    std::fs::write(&path, &bytes).unwrap();
    let script = format!(
        "import h5py, json; f=h5py.File('{}','r'); print(json.dumps(f['data'][:].tolist()))",
        path.display()
    );
    let stdout = h5py_read(&path, &script);
    let values: Vec<f64> = serde_json::from_str(&stdout).unwrap();
    assert_eq!(values, vec![1.0, 2.0, 3.0]);
}

#[test]
#[ignore = "requires Python h5py module"]
fn h5py_reads_our_i32_dataset() {
    let mut fw = FileWriter::new();
    fw.create_dataset("ints").with_i32_data(&[10, 20, 30]);
    let bytes = fw.finish().unwrap();
    let path = std::env::temp_dir().join("clawhdf5_test_i32.h5");
    std::fs::write(&path, &bytes).unwrap();
    let script = format!(
        "import h5py, json; f=h5py.File('{}','r'); print(json.dumps(f['ints'][:].tolist()))",
        path.display()
    );
    let stdout = h5py_read(&path, &script);
    let values: Vec<i32> = serde_json::from_str(&stdout).unwrap();
    assert_eq!(values, vec![10, 20, 30]);
}

#[test]
#[ignore = "requires Python h5py module"]
fn h5py_reads_dataset_with_attrs() {
    let mut fw = FileWriter::new();
    fw.create_dataset("data")
        .with_f64_data(&[1.0, 2.0])
        .set_attr("scale", AttrValue::F64(0.5));
    let bytes = fw.finish().unwrap();
    let path = std::env::temp_dir().join("clawhdf5_test_attrs.h5");
    std::fs::write(&path, &bytes).unwrap();
    let script = format!(
        "import h5py, json; f=h5py.File('{}','r'); d=f['data']; print(json.dumps({{'data': d[:].tolist(), 'scale': float(d.attrs['scale'])}}))",
        path.display()
    );
    let stdout = h5py_read(&path, &script);
    let v: serde_json::Value = serde_json::from_str(&stdout).unwrap();
    assert_eq!(v["data"], serde_json::json!([1.0, 2.0]));
    assert_eq!(v["scale"], serde_json::json!(0.5));
}

#[test]
#[ignore = "requires Python h5py module"]
fn h5py_reads_group_with_dataset() {
    let mut fw = FileWriter::new();
    let mut gb = fw.create_group("grp");
    gb.create_dataset("vals").with_f64_data(&[10.0, 20.0]);
    fw.add_group(gb.finish());
    let bytes = fw.finish().unwrap();
    let path = std::env::temp_dir().join("clawhdf5_test_grp.h5");
    std::fs::write(&path, &bytes).unwrap();
    let script = format!(
        "import h5py, json; f=h5py.File('{}','r'); print(json.dumps(f['grp/vals'][:].tolist()))",
        path.display()
    );
    let stdout = h5py_read(&path, &script);
    let values: Vec<f64> = serde_json::from_str(&stdout).unwrap();
    assert_eq!(values, vec![10.0, 20.0]);
}

#[test]
#[ignore = "requires Python h5py module"]
fn h5py_reads_root_attrs() {
    let mut fw = FileWriter::new();
    fw.set_root_attr("version", AttrValue::I64(42));
    fw.create_dataset("dummy").with_f64_data(&[0.0]);
    let bytes = fw.finish().unwrap();
    let path = std::env::temp_dir().join("clawhdf5_test_root_attrs.h5");
    std::fs::write(&path, &bytes).unwrap();
    let script = format!(
        "import h5py, json; f=h5py.File('{}','r'); print(int(f.attrs['version']))",
        path.display()
    );
    let stdout = h5py_read(&path, &script);
    assert_eq!(stdout, "42");
}

#[test]
#[ignore = "requires Python h5py module"]
fn h5py_reads_multiple_datasets() {
    let mut fw = FileWriter::new();
    fw.create_dataset("a").with_f64_data(&[1.0]);
    fw.create_dataset("b").with_f64_data(&[2.0]);
    fw.create_dataset("c").with_f64_data(&[3.0]);
    let bytes = fw.finish().unwrap();
    let path = std::env::temp_dir().join("clawhdf5_test_multi.h5");
    std::fs::write(&path, &bytes).unwrap();
    let script = format!(
        "import h5py, json; f=h5py.File('{}','r'); print(json.dumps({{k: f[k][:].tolist() for k in ['a','b','c']}}))",
        path.display()
    );
    let stdout = h5py_read(&path, &script);
    let v: serde_json::Value = serde_json::from_str(&stdout).unwrap();
    assert_eq!(v["a"], serde_json::json!([1.0]));
    assert_eq!(v["b"], serde_json::json!([2.0]));
    assert_eq!(v["c"], serde_json::json!([3.0]));
}

// ---- Compound / Enum / Array h5py round-trips ----

#[test]
#[ignore = "requires Python h5py module"]
fn h5py_reads_our_compound_dataset() {
    let ct = CompoundTypeBuilder::new()
        .f64_field("x")
        .f64_field("y")
        .i32_field("id")
        .build();
    let mut raw = Vec::new();
    raw.extend_from_slice(&1.5f64.to_le_bytes());
    raw.extend_from_slice(&2.5f64.to_le_bytes());
    raw.extend_from_slice(&10i32.to_le_bytes());
    raw.extend_from_slice(&3.5f64.to_le_bytes());
    raw.extend_from_slice(&4.5f64.to_le_bytes());
    raw.extend_from_slice(&20i32.to_le_bytes());

    let mut fw = FileWriter::new();
    fw.create_dataset("particles")
        .with_compound_data(ct, raw, 2);
    let bytes = fw.finish().unwrap();
    let path = std::env::temp_dir().join("clawhdf5_compound.h5");
    std::fs::write(&path, &bytes).unwrap();
    let script = format!(
        "import h5py,json; f=h5py.File('{}','r'); d=f['particles']; print(json.dumps({{'x':d['x'].tolist(),'y':d['y'].tolist(),'id':d['id'].tolist()}}))",
        path.display()
    );
    let stdout = h5py_read(&path, &script);
    let v: serde_json::Value = serde_json::from_str(&stdout).unwrap();
    assert_eq!(v["x"], serde_json::json!([1.5, 3.5]));
    assert_eq!(v["y"], serde_json::json!([2.5, 4.5]));
    assert_eq!(v["id"], serde_json::json!([10, 20]));
}

#[test]
#[ignore = "requires Python h5py module"]
fn h5py_reads_our_enum_dataset() {
    let et = EnumTypeBuilder::i32_based()
        .value("RED", 0)
        .value("GREEN", 1)
        .value("BLUE", 2)
        .build();

    let mut fw = FileWriter::new();
    fw.create_dataset("colors")
        .with_enum_i32_data(et, &[1, 0, 2]);
    let bytes = fw.finish().unwrap();
    let path = std::env::temp_dir().join("clawhdf5_enum.h5");
    std::fs::write(&path, &bytes).unwrap();
    let script = format!(
        "import h5py,json; f=h5py.File('{}','r'); d=f['colors']; print(json.dumps(d[:].tolist()))",
        path.display()
    );
    let stdout = h5py_read(&path, &script);
    let values: Vec<i32> = serde_json::from_str(&stdout).unwrap();
    assert_eq!(values, vec![1, 0, 2]);
}

#[test]
#[ignore = "requires Python h5py module"]
fn h5py_reads_our_array_dataset() {
    let mut raw = Vec::new();
    for v in &[1.0f64, 2.0, 3.0, 4.0, 5.0, 6.0] {
        raw.extend_from_slice(&v.to_le_bytes());
    }

    let mut fw = FileWriter::new();
    fw.create_dataset("vectors").with_array_data(
        clawhdf5_format::type_builders::make_f64_type(),
        &[3],
        raw,
        2,
    );
    let bytes = fw.finish().unwrap();
    let path = std::env::temp_dir().join("clawhdf5_array.h5");
    std::fs::write(&path, &bytes).unwrap();
    let script = format!(
        "import h5py,json; f=h5py.File('{}','r'); d=f['vectors']; print(json.dumps({{'shape':list(d.shape),'dtype':str(d.dtype),'values':d[:].tolist()}}))",
        path.display()
    );
    let stdout = h5py_read(&path, &script);
    let v: serde_json::Value = serde_json::from_str(&stdout).unwrap();
    assert_eq!(v["shape"], serde_json::json!([2]));
    assert_eq!(
        v["values"],
        serde_json::json!([[1.0, 2.0, 3.0], [4.0, 5.0, 6.0]])
    );
}

#[test]
#[ignore = "requires Python h5py module"]
fn read_h5py_generated_compound() {
    let path = std::env::temp_dir().join("clawhdf5_h5py_compound.h5");
    let gen_script = format!(
        r#"
import h5py, numpy as np
dt = np.dtype([('x', 'f8'), ('y', 'f8'), ('id', 'i4')])
data = np.array([(1.0, 2.0, 10), (3.0, 4.0, 20)], dtype=dt)
f = h5py.File('{}', 'w', libver='latest')
f.create_dataset('particles', data=data)
f.close()
"#,
        path.display()
    );
    h5py_read(&path, &gen_script);

    let bytes = std::fs::read(&path).unwrap();
    let sig = clawhdf5_format::signature::find_signature(&bytes).unwrap();
    let sb = clawhdf5_format::superblock::Superblock::parse(&bytes, sig).unwrap();
    let addr = clawhdf5_format::group_v2::resolve_path_any(&bytes, &sb, "particles").unwrap();
    let hdr = clawhdf5_format::object_header::ObjectHeader::parse(
        &bytes,
        addr as usize,
        sb.offset_size,
        sb.length_size,
    )
    .unwrap();
    let dt_data = &hdr
        .messages
        .iter()
        .find(|m| m.msg_type == clawhdf5_format::message_type::MessageType::Datatype)
        .unwrap()
        .data;
    let ds_data = &hdr
        .messages
        .iter()
        .find(|m| m.msg_type == clawhdf5_format::message_type::MessageType::Dataspace)
        .unwrap()
        .data;
    let dl_data = &hdr
        .messages
        .iter()
        .find(|m| m.msg_type == clawhdf5_format::message_type::MessageType::DataLayout)
        .unwrap()
        .data;
    let (dt, _) = clawhdf5_format::datatype::Datatype::parse(dt_data).unwrap();
    let ds = clawhdf5_format::dataspace::Dataspace::parse(ds_data, sb.length_size).unwrap();
    let dl =
        clawhdf5_format::data_layout::DataLayout::parse(dl_data, sb.offset_size, sb.length_size)
            .unwrap();
    let raw = clawhdf5_format::data_read::read_raw_data(&bytes, &dl, &ds, &dt).unwrap();
    let fields = clawhdf5_format::data_read::read_compound_fields(&raw, &dt).unwrap();
    assert_eq!(fields.len(), 3);
    assert_eq!(fields[0].name, "x");
    let x_vals =
        clawhdf5_format::data_read::read_as_f64(&fields[0].raw_data, &fields[0].datatype).unwrap();
    assert_eq!(x_vals, vec![1.0, 3.0]);
}

#[test]
#[ignore = "requires Python h5py module"]
fn read_h5py_generated_enum() {
    let path = std::env::temp_dir().join("clawhdf5_h5py_enum.h5");
    let gen_script = format!(
        r#"
import h5py, numpy as np
dt = h5py.enum_dtype({{"RED": 0, "GREEN": 1, "BLUE": 2}}, basetype=np.int32)
data = np.array([1, 0, 2, 1], dtype=np.int32)
f = h5py.File('{}', 'w', libver='latest')
f.create_dataset('colors', data=data, dtype=dt)
f.close()
"#,
        path.display()
    );
    h5py_read(&path, &gen_script);

    let bytes = std::fs::read(&path).unwrap();
    let sig = clawhdf5_format::signature::find_signature(&bytes).unwrap();
    let sb = clawhdf5_format::superblock::Superblock::parse(&bytes, sig).unwrap();
    let addr = clawhdf5_format::group_v2::resolve_path_any(&bytes, &sb, "colors").unwrap();
    let hdr = clawhdf5_format::object_header::ObjectHeader::parse(
        &bytes,
        addr as usize,
        sb.offset_size,
        sb.length_size,
    )
    .unwrap();
    let dt_data = &hdr
        .messages
        .iter()
        .find(|m| m.msg_type == clawhdf5_format::message_type::MessageType::Datatype)
        .unwrap()
        .data;
    let ds_data = &hdr
        .messages
        .iter()
        .find(|m| m.msg_type == clawhdf5_format::message_type::MessageType::Dataspace)
        .unwrap()
        .data;
    let dl_data = &hdr
        .messages
        .iter()
        .find(|m| m.msg_type == clawhdf5_format::message_type::MessageType::DataLayout)
        .unwrap()
        .data;
    let (dt, _) = clawhdf5_format::datatype::Datatype::parse(dt_data).unwrap();
    let ds = clawhdf5_format::dataspace::Dataspace::parse(ds_data, sb.length_size).unwrap();
    let dl =
        clawhdf5_format::data_layout::DataLayout::parse(dl_data, sb.offset_size, sb.length_size)
            .unwrap();
    let raw = clawhdf5_format::data_read::read_raw_data(&bytes, &dl, &ds, &dt).unwrap();
    let names = clawhdf5_format::data_read::read_enum_names(&raw, &dt).unwrap();
    assert_eq!(names, vec!["GREEN", "RED", "BLUE", "GREEN"]);
}

// ---- h5py chunked round-trip tests ----

#[test]
#[ignore = "requires Python h5py module"]
fn h5py_reads_chunked_no_compression() {
    let mut fw = FileWriter::new();
    let data: Vec<f64> = (0..100).map(|i| i as f64).collect();
    fw.create_dataset("data")
        .with_f64_data(&data)
        .with_shape(&[100])
        .with_chunks(&[20]);
    let bytes = fw.finish().unwrap();
    let path = std::env::temp_dir().join("clawhdf5_chunked_nocomp.h5");
    std::fs::write(&path, &bytes).unwrap();
    let script = format!(
        "import h5py,json; f=h5py.File('{}','r'); d=f['data']; print(json.dumps({{'values':d[:].tolist(),'chunks':list(d.chunks)}}))",
        path.display()
    );
    let stdout = h5py_read(&path, &script);
    let v: serde_json::Value = serde_json::from_str(&stdout).unwrap();
    let values: Vec<f64> = serde_json::from_value(v["values"].clone()).unwrap();
    assert_eq!(values, data);
    assert_eq!(v["chunks"], serde_json::json!([20]));
}

#[test]
#[ignore = "requires Python h5py module"]
fn h5py_reads_chunked_deflate() {
    let mut fw = FileWriter::new();
    let data: Vec<f64> = (0..100).map(|i| i as f64).collect();
    fw.create_dataset("data")
        .with_f64_data(&data)
        .with_shape(&[100])
        .with_chunks(&[20])
        .with_deflate(6);
    let bytes = fw.finish().unwrap();
    let path = std::env::temp_dir().join("clawhdf5_chunked_deflate.h5");
    std::fs::write(&path, &bytes).unwrap();
    let script = format!(
        "import h5py,json; f=h5py.File('{}','r'); d=f['data']; print(json.dumps({{'values':d[:].tolist(),'chunks':list(d.chunks),'compression':d.compression}}))",
        path.display()
    );
    let stdout = h5py_read(&path, &script);
    let v: serde_json::Value = serde_json::from_str(&stdout).unwrap();
    let values: Vec<f64> = serde_json::from_value(v["values"].clone()).unwrap();
    assert_eq!(values, data);
    assert_eq!(v["compression"], serde_json::json!("gzip"));
}

#[test]
#[ignore = "requires Python h5py module"]
fn h5py_reads_chunked_shuffle_deflate() {
    let mut fw = FileWriter::new();
    let data: Vec<f64> = (0..100).map(|i| i as f64).collect();
    fw.create_dataset("data")
        .with_f64_data(&data)
        .with_shape(&[100])
        .with_chunks(&[50])
        .with_shuffle()
        .with_deflate(6);
    let bytes = fw.finish().unwrap();
    let path = std::env::temp_dir().join("clawhdf5_chunked_shuffle_deflate.h5");
    std::fs::write(&path, &bytes).unwrap();
    let script = format!(
        "import h5py,json; f=h5py.File('{}','r'); d=f['data']; print(json.dumps({{'values':d[:].tolist(),'shuffle':bool(d.shuffle),'compression':d.compression}}))",
        path.display()
    );
    let stdout = h5py_read(&path, &script);
    let v: serde_json::Value = serde_json::from_str(&stdout).unwrap();
    let values: Vec<f64> = serde_json::from_value(v["values"].clone()).unwrap();
    assert_eq!(values, data);
    assert_eq!(v["shuffle"], serde_json::json!(true));
    assert_eq!(v["compression"], serde_json::json!("gzip"));
}

#[test]
#[ignore = "requires Python h5py module"]
fn h5py_reads_chunked_fletcher32() {
    let mut fw = FileWriter::new();
    let data: Vec<f64> = (0..100).map(|i| i as f64).collect();
    fw.create_dataset("data")
        .with_f64_data(&data)
        .with_shape(&[100])
        .with_chunks(&[100])
        .with_fletcher32();
    let bytes = fw.finish().unwrap();
    let path = std::env::temp_dir().join("clawhdf5_chunked_fletcher32.h5");
    std::fs::write(&path, &bytes).unwrap();
    let script = format!(
        "import h5py,json; f=h5py.File('{}','r'); d=f['data']; print(json.dumps({{'values':d[:].tolist(),'fletcher32':bool(d.fletcher32)}}))",
        path.display()
    );
    let stdout = h5py_read(&path, &script);
    let v: serde_json::Value = serde_json::from_str(&stdout).unwrap();
    let values: Vec<f64> = serde_json::from_value(v["values"].clone()).unwrap();
    assert_eq!(values, data);
    assert_eq!(v["fletcher32"], serde_json::json!(true));
}

#[test]
#[ignore = "requires Python h5py module"]
fn h5py_reads_chunked_2d() {
    let mut fw = FileWriter::new();
    let data: Vec<f64> = (0..24).map(|i| i as f64).collect();
    fw.create_dataset("data")
        .with_f64_data(&data)
        .with_shape(&[4, 6])
        .with_chunks(&[2, 3]);
    let bytes = fw.finish().unwrap();
    let path = std::env::temp_dir().join("clawhdf5_chunked_2d.h5");
    std::fs::write(&path, &bytes).unwrap();
    let script = format!(
        "import h5py,json; f=h5py.File('{}','r'); d=f['data']; print(json.dumps({{'shape':list(d.shape),'chunks':list(d.chunks),'values':d[:].flatten().tolist()}}))",
        path.display()
    );
    let stdout = h5py_read(&path, &script);
    let v: serde_json::Value = serde_json::from_str(&stdout).unwrap();
    let values: Vec<f64> = serde_json::from_value(v["values"].clone()).unwrap();
    assert_eq!(values, data);
    assert_eq!(v["shape"], serde_json::json!([4, 6]));
    assert_eq!(v["chunks"], serde_json::json!([2, 3]));
}

#[test]
#[ignore = "requires Python h5py module"]
fn h5py_reads_2d_data() {
    let mut fw = FileWriter::new();
    fw.create_dataset("matrix")
        .with_f64_data(&[1.0, 2.0, 3.0, 4.0, 5.0, 6.0])
        .with_shape(&[2, 3]);
    let bytes = fw.finish().unwrap();
    let path = std::env::temp_dir().join("clawhdf5_test_2d.h5");
    std::fs::write(&path, &bytes).unwrap();
    let script = format!(
        "import h5py, json; f=h5py.File('{}','r'); d=f['matrix']; print(json.dumps({{'shape': list(d.shape), 'data': d[:].flatten().tolist()}}))",
        path.display()
    );
    let stdout = h5py_read(&path, &script);
    let v: serde_json::Value = serde_json::from_str(&stdout).unwrap();
    assert_eq!(v["shape"], serde_json::json!([2, 3]));
    assert_eq!(v["data"], serde_json::json!([1.0, 2.0, 3.0, 4.0, 5.0, 6.0]));
}

// ---- Extensible Array / resizable dataset h5py tests ----

#[test]
#[ignore = "requires Python h5py module"]
fn h5py_reads_resizable_dataset() {
    let mut fw = FileWriter::new();
    let data: Vec<f64> = (0..50).map(|i| i as f64).collect();
    fw.create_dataset("data")
        .with_f64_data(&data)
        .with_shape(&[50])
        .with_chunks(&[10])
        .with_maxshape(&[u64::MAX]);
    let bytes = fw.finish().unwrap();
    let path = std::env::temp_dir().join("clawhdf5_ea_resizable.h5");
    std::fs::write(&path, &bytes).unwrap();
    let script = format!(
        "import h5py,json; f=h5py.File('{}','r'); d=f['data']; print(json.dumps({{'values':d[:].tolist(),'chunks':list(d.chunks),'maxshape':list(None if x is None else x for x in d.maxshape)}}))",
        path.display()
    );
    let stdout = h5py_read(&path, &script);
    let v: serde_json::Value = serde_json::from_str(&stdout).unwrap();
    let values: Vec<f64> = serde_json::from_value(v["values"].clone()).unwrap();
    assert_eq!(values, data);
    assert_eq!(v["chunks"], serde_json::json!([10]));
}

#[test]
#[ignore = "requires Python h5py module"]
fn read_h5py_generated_ea_file() {
    let path = std::env::temp_dir().join("clawhdf5_h5py_ea.h5");
    let gen_script = format!(
        "import h5py,numpy as np; f=h5py.File('{}','w'); d=f.create_dataset('data',data=np.arange(30,dtype='float64'),chunks=(10,),maxshape=(None,)); f.close()",
        path.display()
    );
    h5py_read(&path, &gen_script);

    let bytes = std::fs::read(&path).unwrap();
    let sig = clawhdf5_format::signature::find_signature(&bytes).unwrap();
    let sb = clawhdf5_format::superblock::Superblock::parse(&bytes, sig).unwrap();
    let addr = clawhdf5_format::group_v2::resolve_path_any(&bytes, &sb, "data").unwrap();
    let hdr = clawhdf5_format::object_header::ObjectHeader::parse(
        &bytes,
        addr as usize,
        sb.offset_size,
        sb.length_size,
    )
    .unwrap();
    let dt_data = &hdr
        .messages
        .iter()
        .find(|m| m.msg_type == clawhdf5_format::message_type::MessageType::Datatype)
        .unwrap()
        .data;
    let ds_data = &hdr
        .messages
        .iter()
        .find(|m| m.msg_type == clawhdf5_format::message_type::MessageType::Dataspace)
        .unwrap()
        .data;
    let dl_data = &hdr
        .messages
        .iter()
        .find(|m| m.msg_type == clawhdf5_format::message_type::MessageType::DataLayout)
        .unwrap()
        .data;
    let (dt, _) = clawhdf5_format::datatype::Datatype::parse(dt_data).unwrap();
    let ds = clawhdf5_format::dataspace::Dataspace::parse(ds_data, sb.length_size).unwrap();
    let dl =
        clawhdf5_format::data_layout::DataLayout::parse(dl_data, sb.offset_size, sb.length_size)
            .unwrap();
    let raw = match &dl {
        clawhdf5_format::data_layout::DataLayout::Chunked { .. } => {
            let pipeline = hdr
                .messages
                .iter()
                .find(|m| m.msg_type == clawhdf5_format::message_type::MessageType::FilterPipeline)
                .map(|m| clawhdf5_format::filter_pipeline::FilterPipeline::parse(&m.data).unwrap());
            clawhdf5_format::chunked_read::read_chunked_data(
                &bytes,
                &dl,
                &ds,
                &dt,
                pipeline.as_ref(),
                sb.offset_size,
                sb.length_size,
            )
            .unwrap()
        }
        _ => clawhdf5_format::data_read::read_raw_data(&bytes, &dl, &ds, &dt).unwrap(),
    };
    let result = clawhdf5_format::data_read::read_as_f64(&raw, &dt).unwrap();
    let expected: Vec<f64> = (0..30).map(|i| i as f64).collect();
    assert_eq!(result, expected);
}

#[test]
#[ignore = "requires Python h5py module"]
fn h5py_append_and_verify() {
    let mut fw = FileWriter::new();
    let initial: Vec<f64> = (0..10).map(|i| i as f64).collect();
    fw.create_dataset("data")
        .with_f64_data(&initial)
        .with_shape(&[10])
        .with_chunks(&[10])
        .with_maxshape(&[u64::MAX]);
    let bytes = fw.finish().unwrap();
    let path = std::env::temp_dir().join("clawhdf5_ea_append.h5");
    std::fs::write(&path, &bytes).unwrap();

    let script = format!(
        r#"
import h5py, json, numpy as np
f = h5py.File('{}', 'a')
d = f['data']
for batch in range(3):
    old_size = d.shape[0]
    new_data = np.arange(old_size, old_size + 10, dtype='float64')
    d.resize(old_size + 10, axis=0)
    d[old_size:] = new_data
f.close()
f = h5py.File('{}', 'r')
d = f['data']
result = d[:].tolist()
shape = list(d.shape)
f.close()
print(json.dumps({{'values': result, 'shape': shape}}))
"#,
        path.display(),
        path.display()
    );
    let stdout = h5py_read(&path, &script);
    let v: serde_json::Value = serde_json::from_str(&stdout).unwrap();
    let values: Vec<f64> = serde_json::from_value(v["values"].clone()).unwrap();
    let expected: Vec<f64> = (0..40).map(|i| i as f64).collect();
    assert_eq!(values, expected);
    assert_eq!(v["shape"], serde_json::json!([40]));
}

#[test]
#[ignore = "requires Python h5py module"]
fn h5py_reads_resizable_single_chunk() {
    let mut fw = FileWriter::new();
    let data: Vec<f64> = (0..5).map(|i| i as f64).collect();
    fw.create_dataset("data")
        .with_f64_data(&data)
        .with_shape(&[5])
        .with_chunks(&[10])
        .with_maxshape(&[u64::MAX]);
    let bytes = fw.finish().unwrap();
    let path = std::env::temp_dir().join("clawhdf5_ea_single.h5");
    std::fs::write(&path, &bytes).unwrap();
    let script = format!(
        "import h5py,json; f=h5py.File('{}','r'); d=f['data']; print(json.dumps(d[:].tolist()))",
        path.display()
    );
    let stdout = h5py_read(&path, &script);
    let values: Vec<f64> = serde_json::from_str(&stdout).unwrap();
    assert_eq!(values, data);
}

// ---- Dense attribute h5py round-trip ----

#[test]
#[ignore = "requires Python h5py module"]
fn h5py_reads_dense_attrs() {
    let mut fw = FileWriter::new();
    let ds = fw.create_dataset("data");
    ds.with_f64_data(&[1.0, 2.0, 3.0]);
    for i in 0..20 {
        ds.set_attr(&format!("attr_{i:03}"), AttrValue::F64(i as f64 * 1.5));
    }
    let bytes = fw.finish().unwrap();
    let path = std::env::temp_dir().join("clawhdf5_dense_attrs.h5");
    std::fs::write(&path, &bytes).unwrap();

    let script = format!(
        r#"
import h5py, json
f = h5py.File('{}', 'r')
d = f['data']
attrs = {{k: float(v) for k, v in d.attrs.items()}}
data = d[:].tolist()
f.close()
print(json.dumps({{'data': data, 'num_attrs': len(attrs), 'attr_000': attrs.get('attr_000'), 'attr_010': attrs.get('attr_010'), 'attr_019': attrs.get('attr_019')}}))
"#,
        path.display()
    );
    let stdout = h5py_read(&path, &script);
    let v: serde_json::Value = serde_json::from_str(&stdout).unwrap();
    assert_eq!(v["data"], serde_json::json!([1.0, 2.0, 3.0]));
    assert_eq!(v["num_attrs"], serde_json::json!(20));
    assert_eq!(v["attr_000"], serde_json::json!(0.0));
    assert_eq!(v["attr_010"], serde_json::json!(15.0));
    assert_eq!(v["attr_019"], serde_json::json!(28.5));
}

#[test]
#[ignore = "requires Python h5py module"]
fn h5py_reads_50_dense_attrs() {
    let mut fw = FileWriter::new();
    let ds = fw.create_dataset("data");
    ds.with_f64_data(&[42.0]);
    for i in 0..50 {
        ds.set_attr(&format!("attr_{i:03}"), AttrValue::F64(i as f64 * 1.5));
    }
    let bytes = fw.finish().unwrap();
    let path = std::env::temp_dir().join("clawhdf5_dense_50_attrs.h5");
    std::fs::write(&path, &bytes).unwrap();

    let script = format!(
        r#"
import h5py, json
f = h5py.File('{}', 'r')
d = f['data']
attrs = {{k: float(v) for k, v in d.attrs.items()}}
f.close()
print(json.dumps({{'num_attrs': len(attrs), 'first': attrs.get('attr_000'), 'last': attrs.get('attr_049')}}))
"#,
        path.display()
    );
    let stdout = h5py_read(&path, &script);
    let v: serde_json::Value = serde_json::from_str(&stdout).unwrap();
    assert_eq!(v["num_attrs"], serde_json::json!(50));
    assert_eq!(v["first"], serde_json::json!(0.0));
    assert_eq!(v["last"], serde_json::json!(73.5));
}

// ---- SHINES provenance h5py round-trips ----

#[test]
#[ignore = "requires Python h5py module"]
fn h5py_reads_provenance_attrs() {
    let mut fw = FileWriter::new();
    let ds = fw.create_dataset("sensor");
    ds.with_f64_data(&[1.0, 2.0, 3.0, 4.0]).with_provenance(
        "clawhdf5/test",
        "2026-02-19T12:00:00Z",
        Some("bench_42"),
    );
    let bytes = fw.finish().unwrap();
    let path = std::env::temp_dir().join("clawhdf5_provenance.h5");
    std::fs::write(&path, &bytes).unwrap();

    let script = format!(
        r#"
import h5py, json, hashlib, struct
f = h5py.File('{}', 'r')
d = f['sensor']
data = d[:].tolist()
sha = d.attrs['_provenance_sha256']
if isinstance(sha, bytes):
    sha = sha.decode('utf-8')
sha = sha.rstrip('\x00')
creator = d.attrs['_provenance_creator']
if isinstance(creator, bytes):
    creator = creator.decode('utf-8')
creator = creator.rstrip('\x00')
ts = d.attrs['_provenance_timestamp']
if isinstance(ts, bytes):
    ts = ts.decode('utf-8')
ts = ts.rstrip('\x00')
source = d.attrs['_provenance_source']
if isinstance(source, bytes):
    source = source.decode('utf-8')
source = source.rstrip('\x00')
# Verify SHA-256 matches the raw little-endian f64 bytes
raw = struct.pack('<4d', *data)
expected = hashlib.sha256(raw).hexdigest()
f.close()
print(json.dumps({{'data': data, 'sha256': sha, 'expected_sha256': expected, 'creator': creator, 'timestamp': ts, 'source': source}}))
"#,
        path.display()
    );
    let stdout = h5py_read(&path, &script);
    let v: serde_json::Value = serde_json::from_str(&stdout).unwrap();
    assert_eq!(v["data"], serde_json::json!([1.0, 2.0, 3.0, 4.0]));
    assert_eq!(v["sha256"], v["expected_sha256"]);
    assert_eq!(v["creator"], serde_json::json!("clawhdf5/test"));
    assert_eq!(v["timestamp"], serde_json::json!("2026-02-19T12:00:00Z"));
    assert_eq!(v["source"], serde_json::json!("bench_42"));
}

#[test]
#[ignore = "requires Python h5py module"]
fn h5py_reads_provenance_no_source() {
    let mut fw = FileWriter::new();
    let ds = fw.create_dataset("data");
    ds.with_i32_data(&[10, 20, 30])
        .with_provenance("test-writer", "2026-01-01T00:00:00Z", None);
    let bytes = fw.finish().unwrap();
    let path = std::env::temp_dir().join("clawhdf5_provenance_nosrc.h5");
    std::fs::write(&path, &bytes).unwrap();

    let script = format!(
        r#"
import h5py, json, hashlib, struct
f = h5py.File('{}', 'r')
d = f['data']
attr_names = sorted(d.attrs.keys())
sha = d.attrs['_provenance_sha256']
if isinstance(sha, bytes):
    sha = sha.decode('utf-8')
sha = sha.rstrip('\x00')
raw = struct.pack('<3i', *d[:].tolist())
expected = hashlib.sha256(raw).hexdigest()
has_source = '_provenance_source' in d.attrs
f.close()
print(json.dumps({{'attrs': attr_names, 'sha_ok': sha == expected, 'has_source': has_source}}))
"#,
        path.display()
    );
    let stdout = h5py_read(&path, &script);
    let v: serde_json::Value = serde_json::from_str(&stdout).unwrap();
    assert_eq!(v["sha_ok"], serde_json::json!(true));
    assert_eq!(v["has_source"], serde_json::json!(false));
}

#[test]
fn provenance_verify_written_file() {
    let mut fw = FileWriter::new();
    let ds = fw.create_dataset("values");
    ds.with_f64_data(&[100.0, 200.0, 300.0]).with_provenance(
        "integrity-test",
        "2026-02-19T00:00:00Z",
        None,
    );
    let bytes = fw.finish().unwrap();

    // Use our verification API to check integrity
    let sig = clawhdf5_format::signature::find_signature(&bytes).unwrap();
    let sb = clawhdf5_format::superblock::Superblock::parse(&bytes, sig).unwrap();
    let addr = clawhdf5_format::group_v2::resolve_path_any(&bytes, &sb, "values").unwrap();
    let hdr = clawhdf5_format::object_header::ObjectHeader::parse(
        &bytes,
        addr as usize,
        sb.offset_size,
        sb.length_size,
    )
    .unwrap();
    let result =
        clawhdf5_format::provenance::verify_dataset(&bytes, &hdr, sb.offset_size, sb.length_size)
            .unwrap();
    assert_eq!(result, clawhdf5_format::provenance::VerifyResult::Ok);
}