espnet2.text.hugging_face_tokenizer.HuggingFaceTokenizer

source

class espnet2.text.hugging_face_tokenizer.HuggingFaceTokenizer(model: Path | str)

Bases: AbsTokenizer

text2tokens(line: str) → List[str]

tokens2text(tokens: Iterable[str]) → str