espnet2.sds.end_to_end.mini_omni.litgpt.config.Config

source

class espnet2.sds.end_to_end.mini_omni.litgpt.config.Config(name: str = '', hf_config: dict = <factory>, scale_embeddings: bool = False, block_size: int = 4096, vocab_size: int = 50254, padding_multiple: int = 512, padded_vocab_size: Optional[int] = None, n_layer: int = 16, n_head: int = 32, head_size: Optional[int] = None, n_embd: int = 4096, rotary_percentage: float = 0.25, parallel_residual: bool = True, bias: bool = True, lm_head_bias: bool = False, n_query_groups: Optional[int] = None, shared_attention_norm: bool = False, norm_class_name: Literal['LayerNorm', 'RMSNorm'] = 'LayerNorm', norm_eps: float = 1e-05, mlp_class_name: Literal['GptNeoxMLP', 'LLaMAMLP', 'GemmaMLP', 'LLaMAMoE'] = 'GptNeoxMLP', gelu_approximate: str = 'none', intermediate_size: Optional[int] = None, rope_condense_ratio: int = 1, rope_base: int = 10000, n_expert: int = 0, n_expert_per_token: int = 0, add_qkv_bias: Optional[bool] = None, prompt_vocab_size: Optional[int] = None, attn_dropout: float = 0.0, pos_type: str = 'rope', force_align: bool = False, use_pretrain_phoneme_emb: bool = False, tie_word_embeddings: bool = False, text_vocab_size: int = 152000, cat_audio_vocab_size: int = 29120, audio_vocab_size: int = 4160, whisper_adapter_dim: int = 768, post_adapter: bool = False, post_adapter_layers: int = 6, asr_adapter: str = 'llamamlp')

Bases: object

add_qkv_bias : bool | None = None

asr_adapter : str = 'llamamlp'

attn_dropout : float = 0.0

audio_vocab_size : int = 4160

bias : bool = True

block_size : int = 4096

cat_audio_vocab_size : int = 29120

force_align : bool = False

classmethod from_checkpoint(path: Path, **kwargs: Any) → Self

Automatically load model_config.yaml and if it doesn’t

exist - a matching config from litgpt/config.py.

classmethod from_file(path: str | Path, **kwargs: Any) → Self

classmethod from_name(name: str, **kwargs: Any) → Self | None

gelu_approximate : str = 'none'

head_size : int | None = None

hf_config : dict

intermediate_size : int | None = None

lm_head_bias : bool = False

property mlp_class : Type

mlp_class_name : Literal['GptNeoxMLP', 'LLaMAMLP', 'GemmaMLP', 'LLaMAMoE'] = 'GptNeoxMLP'

n_embd : int = 4096

n_expert : int = 0

n_expert_per_token : int = 0

n_head : int = 32

n_layer : int = 16

n_query_groups : int | None = None

name : str = ''

property norm_class : Type

norm_class_name : Literal['LayerNorm', 'RMSNorm'] = 'LayerNorm'

norm_eps : float = 1e-05

padded_vocab_size : int | None = None

padding_multiple : int = 512

parallel_residual : bool = True

pos_type : str = 'rope'

post_adapter : bool = False

post_adapter_layers : int = 6

prompt_vocab_size : int | None = None

rope_base : int = 10000

rope_condense_ratio : int = 1

rotary_percentage : float = 0.25

scale_embeddings : bool = False

shared_attention_norm : bool = False

text_vocab_size : int = 152000

tie_word_embeddings : bool = False

use_pretrain_phoneme_emb : bool = False

vocab_size : int = 50254

whisper_adapter_dim : int = 768