[package]
name = "rtx-tokenization"
version.workspace = true
edition.workspace = true
rust-version = "1.92"
authors.workspace = true
license.workspace = true
repository.workspace = true
description = "Advanced tokenization with BPE/SentencePiece trainers and multimodal support"

[dependencies]
# Core dependencies
anyhow.workspace = true
thiserror.workspace = true
tracing.workspace = true
serde.workspace = true
serde_json.workspace = true

# Text processing
unicode-segmentation.workspace = true
unicode-normalization.workspace = true
regex.workspace = true

# Collections and data structures
indexmap = { workspace = true, features = ["serde"] }
dashmap.workspace = true

# Async runtime
tokio.workspace = true
futures.workspace = true
async-trait.workspace = true

# Multimodal processing
image.workspace = true
symphonia.workspace = true

# Math and utilities
rand.workspace = true
base64.workspace = true
sha2.workspace = true
bincode.workspace = true
nalgebra.workspace = true

# Concurrency
parking_lot.workspace = true
rayon.workspace = true

# RTX dependencies
rtx-tensor.workspace = true

[dev-dependencies]
tokio-test.workspace = true
proptest.workspace = true
criterion.workspace = true
tempfile.workspace = true

[[bench]]
name = "tokenization_benchmarks"
harness = false

[features]
default = ["multimodal"]
multimodal = []
cjk-support = []
rtl-support = []
[lints]
workspace = true