espnet2.text.hugging_face_token_id_converter.HuggingFaceTokenIDConverter

source

class espnet2.text.hugging_face_token_id_converter.HuggingFaceTokenIDConverter(model_name_or_path: str)

Bases: object

get_num_vocabulary_size() → int

ids2tokens(integers: ndarray | Iterable[int]) → List[str]

tokens2ids(tokens: Iterable[str]) → List[int]