espnet2.sds.end_to_end.mini_omni.litgpt.tokenizer.Tokenizer

source

class espnet2.sds.end_to_end.mini_omni.litgpt.tokenizer.Tokenizer(checkpoint_dir: Path | str)

Bases: object

check_if_bos_token_used(checkpoint_dir: Path) → bool

decode(tensor: Tensor) → str

encode(string: str, device: device | None = None, bos: bool | None = None, eos: bool = False, max_length: int = -1) → Tensor

token_to_id(token: str) → int

property vocab_size : int