[package]
name = "charabia"
version = "0.10.0"
license = "MIT"
authors = ["Many <many@meilisearch.com>"]
edition = "2021"
description = "A simple library to detect the language, tokenize the text and normalize the tokens"
documentation = "https://docs.rs/charabia"
repository = "https://github.com/meilisearch/charabia"
keywords = ["segmenter", "tokenizer", "normalize", "language"]
categories = ["text-processing"]
exclude = ["dictionaries/txt/thai/words.txt"]

[dependencies]
aho-corasick = "1.1.5"
csv = "1.4.0"
either = "1.17.0"
finl_unicode = { version = "1.4.0", optional = true }
fst = "0.4"
jieba-rs = { version = "0.10.3", optional = true }
serde = "1.0.229"
slice-group-by = "0.3.1"
whatlang = "0.18.0"
lindera = { version = "5.1.0", default-features = false, optional = true }
pinyin = { version = "0.11", default-features = false, features = [
    "with_tone",
], optional = true }
wana_kana = { version = "5.0.0", optional = true }
unicode-normalization = "0.1.25"
irg-kvariants = { path = "../irg-kvariants", version = "=0.1.1" }

[features]
default = [
    "chinese",
    "hebrew",
    "japanese",
    "thai",
    "korean",
    "greek",
    "khmer",
    "vietnamese",
    "swedish-recomposition",
    "turkish",
    "german-segmentation",
]

# allow chinese specialized tokenization
chinese = ["chinese-segmentation", "chinese-normalization"]
chinese-segmentation = ["dep:jieba-rs"]
chinese-normalization = []
chinese-normalization-pinyin = ["dep:pinyin", "chinese-normalization"]

# allow hebrew specialized tokenization
hebrew = []

# allow japanese specialized tokenization
japanese = ["japanese-segmentation-unidic", "japanese-transliteration", "lindera/embed-unidic"]
japanese-segmentation-ipadic = ["lindera/lindera-ipadic", "lindera/embed-ipadic"]
japanese-segmentation-unidic = ["lindera/lindera-unidic", "lindera/embed-unidic"]
japanese-transliteration = ["dep:wana_kana"]

# allow korean specialized tokenization
korean = ["lindera/lindera-ko-dic", "lindera/embed-ko-dic"]

# allow thai specialized tokenization
thai = []

# allow greek specialized tokenization
greek = []

# allow splitting camelCase latin words
latin-camelcase = ["dep:finl_unicode"]

khmer = []

# allow vietnamese specialized tokenization
vietnamese = []

# allow splitting snake_case latin words
latin-snakecase = ["dep:finl_unicode"]

# force Charabia to recompose Swedish characters
swedish-recomposition = []

# allow turkish specialized tokenization
turkish = []

# allow decomposition of German composite words
german-segmentation = []

[dev-dependencies]
criterion = "0.8"
quickcheck = "1"
quickcheck_macros = "1"
mimalloc = "0.1.52"

[[bench]]
name = "bench"
harness = false