68 lines
1.4 KiB
TOML
68 lines
1.4 KiB
TOML
[package]
|
|
name = "rtx-tokenization"
|
|
version.workspace = true
|
|
edition.workspace = true
|
|
rust-version = "1.92"
|
|
authors.workspace = true
|
|
license.workspace = true
|
|
repository.workspace = true
|
|
description = "Advanced tokenization with BPE/SentencePiece trainers and multimodal support"
|
|
|
|
[dependencies]
|
|
# Core dependencies
|
|
anyhow.workspace = true
|
|
thiserror.workspace = true
|
|
tracing.workspace = true
|
|
serde.workspace = true
|
|
serde_json.workspace = true
|
|
|
|
# Text processing
|
|
unicode-segmentation.workspace = true
|
|
unicode-normalization.workspace = true
|
|
regex.workspace = true
|
|
|
|
# Collections and data structures
|
|
indexmap = { workspace = true, features = ["serde"] }
|
|
dashmap.workspace = true
|
|
|
|
# Async runtime
|
|
tokio.workspace = true
|
|
futures.workspace = true
|
|
async-trait.workspace = true
|
|
|
|
# Multimodal processing
|
|
image.workspace = true
|
|
symphonia.workspace = true
|
|
|
|
# Math and utilities
|
|
rand.workspace = true
|
|
base64.workspace = true
|
|
sha2.workspace = true
|
|
bincode.workspace = true
|
|
nalgebra.workspace = true
|
|
|
|
# Concurrency
|
|
parking_lot.workspace = true
|
|
rayon.workspace = true
|
|
|
|
# RTX dependencies
|
|
rtx-tensor.workspace = true
|
|
|
|
[dev-dependencies]
|
|
tokio-test.workspace = true
|
|
proptest.workspace = true
|
|
criterion.workspace = true
|
|
tempfile.workspace = true
|
|
|
|
[[bench]]
|
|
name = "tokenization_benchmarks"
|
|
harness = false
|
|
|
|
[features]
|
|
default = ["multimodal"]
|
|
multimodal = []
|
|
cjk-support = []
|
|
rtl-support = []
|
|
[lints]
|
|
workspace = true
|