{ "date": "2026-09-22", "models": [ { "id": "Qwen/Qwen3-0.6B", "meta": { "sha": "c1899de289a04d12100db370d81485cdf75e47ca", "safetensors": { "parameters": { "BF16": 751632384 }, "total": 751632384 }, "modelType": "qwen3" }, "config": { "architectures": [ "Qwen3ForCausalLM" ], "attention_bias": false, "attention_dropout": 0, "bos_token_id": 151643, "eos_token_id": 151645, "head_dim": 128, "hidden_act": "silu", "hidden_size": 1024, "initializer_range": 0.02, "intermediate_size": 3072, "max_position_embeddings": 40960, "max_window_layers": 28, "model_type": "qwen3", "num_attention_heads": 16, "num_hidden_layers": 28, "num_key_value_heads": 8, "rms_norm_eps": 1e-06, "rope_scaling": null, "rope_theta": 1000000, "sliding_window": null, "tie_word_embeddings": true, "torch_dtype": "bfloat16", "transformers_version": "4.51.0", "use_cache": true, "use_sliding_window": false, "vocab_size": 151936 } }, { "id": "Qwen/Qwen3-8B", "meta": { "sha": "b968826d9c46dd6066d109eabc6255188de91218", "safetensors": { "parameters": { "BF16": 8190735360 }, "total": 8190735360 }, "modelType": "qwen3" }, "config": { "architectures": [ "Qwen3ForCausalLM" ], "attention_bias": false, "attention_dropout": 0, "bos_token_id": 151643, "eos_token_id": 151645, "head_dim": 128, "hidden_act": "silu", "hidden_size": 4096, "initializer_range": 0.02, "intermediate_size": 12288, "max_position_embeddings": 40960, "max_window_layers": 36, "model_type": "qwen3", "num_attention_heads": 32, "num_hidden_layers": 36, "num_key_value_heads": 8, "rms_norm_eps": 1e-06, "rope_scaling": null, "rope_theta": 1000000, "sliding_window": null, "tie_word_embeddings": false, "torch_dtype": "bfloat16", "transformers_version": "4.51.0", "use_cache": true, "use_sliding_window": false, "vocab_size": 151936 } }, { "id": "Qwen/Qwen2.5-7B-Instruct", "meta": { "sha": "a09a35458c702b33eeacc393d103063234e8bc28", "safetensors": { "parameters": { "BF16": 7615616512 }, "total": 7615616512 }, "modelType": "qwen2" }, "config": { "architectures": [ "Qwen2ForCausalLM" ], "attention_dropout": 0, "bos_token_id": 151643, "eos_token_id": 151645, "hidden_act": "silu", "hidden_size": 3584, "initializer_range": 0.02, "intermediate_size": 18944, "max_position_embeddings": 32768, "max_window_layers": 28, "model_type": "qwen2", "num_attention_heads": 28, "num_hidden_layers": 28, "num_key_value_heads": 4, "rms_norm_eps": 1e-06, "rope_theta": 1000000, "sliding_window": 131072, "tie_word_embeddings": false, "torch_dtype": "bfloat16", "transformers_version": "4.43.1", "use_cache": true, "use_sliding_window": false, "vocab_size": 152064 } }, { "id": "Qwen/Qwen2.5-0.5B-Instruct", "meta": { "sha": "7ae557604adf67be50417f59c2c2f167def9a775", "safetensors": { "parameters": { "BF16": 494032768 }, "total": 494032768 }, "modelType": "qwen2" }, "config": { "architectures": [ "Qwen2ForCausalLM" ], "attention_dropout": 0, "bos_token_id": 151643, "eos_token_id": 151645, "hidden_act": "silu", "hidden_size": 896, "initializer_range": 0.02, "intermediate_size": 4864, "max_position_embeddings": 32768, "max_window_layers": 21, "model_type": "qwen2", "num_attention_heads": 14, "num_hidden_layers": 24, "num_key_value_heads": 2, "rms_norm_eps": 1e-06, "rope_theta": 1000000, "sliding_window": 32768, "tie_word_embeddings": true, "torch_dtype": "bfloat16", "transformers_version": "4.43.1", "use_cache": true, "use_sliding_window": false, "vocab_size": 151936 } }, { "id": "Qwen/Qwen2.5-1.5B-Instruct", "meta": { "sha": "989aa7980e4cf806f80c7fef2b1adb7bc71aa306", "safetensors": { "parameters": { "BF16": 1543714304 }, "total": 1543714304 }, "modelType": "qwen2" }, "config": { "architectures": [ "Qwen2ForCausalLM" ], "attention_dropout": 0, "bos_token_id": 151643, "eos_token_id": 151645, "hidden_act": "silu", "hidden_size": 1536, "initializer_range": 0.02, "intermediate_size": 8960, "max_position_embeddings": 32768, "max_window_layers": 21, "model_type": "qwen2", "num_attention_heads": 12, "num_hidden_layers": 28, "num_key_value_heads": 2, "rms_norm_eps": 1e-06, "rope_theta": 1000000, "sliding_window": 32768, "tie_word_embeddings": true, "torch_dtype": "bfloat16", "transformers_version": "4.43.1", "use_cache": true, "use_sliding_window": false, "vocab_size": 151936 } }, { "id": "Qwen/Qwen3-4B", "meta": { "sha": "1cfa9a7208912126459214e8b04321603b3df60c", "safetensors": { "parameters": { "BF16": 4022468096 }, "total": 4022468096 }, "modelType": "qwen3" }, "config": { "architectures": [ "Qwen3ForCausalLM" ], "attention_bias": false, "attention_dropout": 0, "bos_token_id": 151643, "eos_token_id": 151645, "head_dim": 128, "hidden_act": "silu", "hidden_size": 2560, "initializer_range": 0.02, "intermediate_size": 9728, "max_position_embeddings": 40960, "max_window_layers": 36, "model_type": "qwen3", "num_attention_heads": 32, "num_hidden_layers": 36, "num_key_value_heads": 8, "rms_norm_eps": 1e-06, "rope_scaling": null, "rope_theta": 1000000, "sliding_window": null, "tie_word_embeddings": true, "torch_dtype": "bfloat16", "transformers_version": "4.51.0", "use_cache": true, "use_sliding_window": false, "vocab_size": 151936 } }, { "id": "Qwen/Qwen3-32B", "meta": { "sha": "9216db5781bf21249d130ec9da846c4624c16137", "safetensors": { "parameters": { "BF16": 32762123264 }, "total": 32762123264 }, "modelType": "qwen3" }, "config": { "architectures": [ "Qwen3ForCausalLM" ], "attention_bias": false, "attention_dropout": 0, "bos_token_id": 151643, "eos_token_id": 151645, "head_dim": 128, "hidden_act": "silu", "hidden_size": 5120, "initializer_range": 0.02, "intermediate_size": 25600, "max_position_embeddings": 40960, "max_window_layers": 64, "model_type": "qwen3", "num_attention_heads": 64, "num_hidden_layers": 64, "num_key_value_heads": 8, "rms_norm_eps": 1e-06, "rope_scaling": null, "rope_theta": 1000000, "sliding_window": null, "tie_word_embeddings": false, "torch_dtype": "bfloat16", "transformers_version": "4.51.0", "use_cache": true, "use_sliding_window": false, "vocab_size": 151936 } }, { "id": "Qwen/Qwen2.5-3B-Instruct", "meta": { "sha": "aa8e72537993ba99e69dfaafa59ed015b17504d1", "safetensors": { "parameters": { "BF16": 3085938688 }, "total": 3085938688 }, "modelType": "qwen2" }, "config": { "architectures": [ "Qwen2ForCausalLM" ], "attention_dropout": 0, "bos_token_id": 151643, "eos_token_id": 151645, "hidden_act": "silu", "hidden_size": 2048, "initializer_range": 0.02, "intermediate_size": 11008, "max_position_embeddings": 32768, "max_window_layers": 70, "model_type": "qwen2", "num_attention_heads": 16, "num_hidden_layers": 36, "num_key_value_heads": 2, "rms_norm_eps": 1e-06, "rope_theta": 1000000, "sliding_window": 32768, "tie_word_embeddings": true, "torch_dtype": "bfloat16", "transformers_version": "4.43.1", "use_cache": true, "use_sliding_window": false, "vocab_size": 151936 } }, { "id": "Qwen/Qwen-72B", "meta": { "sha": "b8e18ac61df64d35308695769ff46b976b6a00f4", "safetensors": { "parameters": { "BF16": 72287920128 }, "total": 72287920128 }, "modelType": "qwen" }, "config": { "architectures": [ "QWenLMHeadModel" ], "auto_map": { "AutoConfig": "configuration_qwen.QWenConfig", "AutoModelForCausalLM": "modeling_qwen.QWenLMHeadModel" }, "attn_dropout_prob": 0, "bf16": false, "emb_dropout_prob": 0, "fp16": false, "fp32": false, "hidden_size": 8192, "intermediate_size": 49152, "initializer_range": 0.02, "kv_channels": 128, "layer_norm_epsilon": 1e-06, "max_position_embeddings": 32768, "model_type": "qwen", "no_bias": true, "num_attention_heads": 64, "num_hidden_layers": 80, "onnx_safe": null, "rope_theta": 1000000, "rotary_emb_base": 1000000, "rotary_pct": 1, "scale_attn_weights": true, "seq_length": 32768, "tie_word_embeddings": false, "tokenizer_class": "QWenTokenizer", "transformers_version": "4.32.0", "use_cache": true, "use_dynamic_ntk": false, "use_flash_attn": "auto", "use_logn_attn": false, "vocab_size": 152064 } }, { "id": "Qwen/Qwen3-4B-Instruct-2507", "meta": { "sha": "cdbee75f17c01a7cc42f958dc650907174af0554", "safetensors": { "parameters": { "BF16": 4022468096 }, "total": 4022468096 }, "modelType": "qwen3" }, "config": { "architectures": [ "Qwen3ForCausalLM" ], "attention_bias": false, "attention_dropout": 0, "bos_token_id": 151643, "eos_token_id": 151645, "head_dim": 128, "hidden_act": "silu", "hidden_size": 2560, "initializer_range": 0.02, "intermediate_size": 9728, "max_position_embeddings": 262144, "max_window_layers": 36, "model_type": "qwen3", "num_attention_heads": 32, "num_hidden_layers": 36, "num_key_value_heads": 8, "rms_norm_eps": 1e-06, "rope_scaling": null, "rope_theta": 5000000, "sliding_window": null, "tie_word_embeddings": true, "torch_dtype": "bfloat16", "transformers_version": "4.51.0", "use_cache": true, "use_sliding_window": false, "vocab_size": 151936 } }, { "id": "Qwen/Qwen3-1.7B", "meta": { "sha": "70d244cc86ccca08cf5af4e1e306ecf908b1ad5e", "safetensors": { "parameters": { "BF16": 2031739904 }, "total": 2031739904 }, "modelType": "qwen3" }, "config": { "architectures": [ "Qwen3ForCausalLM" ], "attention_bias": false, "attention_dropout": 0, "bos_token_id": 151643, "eos_token_id": 151645, "head_dim": 128, "hidden_act": "silu", "hidden_size": 2048, "initializer_range": 0.02, "intermediate_size": 6144, "max_position_embeddings": 40960, "max_window_layers": 28, "model_type": "qwen3", "num_attention_heads": 16, "num_hidden_layers": 28, "num_key_value_heads": 8, "rms_norm_eps": 1e-06, "rope_scaling": null, "rope_theta": 1000000, "sliding_window": null, "tie_word_embeddings": true, "torch_dtype": "bfloat16", "transformers_version": "4.51.0", "use_cache": true, "use_sliding_window": false, "vocab_size": 151936 } }, { "id": "Qwen/Qwen2.5-Coder-7B-Instruct", "meta": { "sha": "c03e6d358207e414f1eca0bb1891e29f1db0e242", "safetensors": { "parameters": { "BF16": 7615616512 }, "total": 7615616512 }, "modelType": "qwen2" }, "config": { "architectures": [ "Qwen2ForCausalLM" ], "attention_dropout": 0, "bos_token_id": 151643, "eos_token_id": 151645, "hidden_act": "silu", "hidden_size": 3584, "initializer_range": 0.02, "intermediate_size": 18944, "max_position_embeddings": 32768, "max_window_layers": 28, "model_type": "qwen2", "num_attention_heads": 28, "num_hidden_layers": 28, "num_key_value_heads": 4, "rms_norm_eps": 1e-06, "rope_theta": 1000000, "sliding_window": 131072, "tie_word_embeddings": false, "torch_dtype": "bfloat16", "transformers_version": "4.44.0", "use_cache": true, "use_sliding_window": false, "vocab_size": 152064 } }, { "id": "Qwen/Qwen2.5-32B-Instruct", "meta": { "sha": "5ede1c97bbab6ce5cda5812749b4c0bdf79b18dd", "safetensors": { "parameters": { "BF16": 32763876352 }, "total": 32763876352 }, "modelType": "qwen2" }, "config": { "architectures": [ "Qwen2ForCausalLM" ], "attention_dropout": 0, "bos_token_id": 151643, "eos_token_id": 151645, "hidden_act": "silu", "hidden_size": 5120, "initializer_range": 0.02, "intermediate_size": 27648, "max_position_embeddings": 32768, "max_window_layers": 70, "model_type": "qwen2", "num_attention_heads": 40, "num_hidden_layers": 64, "num_key_value_heads": 8, "rms_norm_eps": 1e-06, "rope_theta": 1000000, "sliding_window": 131072, "tie_word_embeddings": false, "torch_dtype": "bfloat16", "transformers_version": "4.43.1", "use_cache": true, "use_sliding_window": false, "vocab_size": 152064 } }, { "id": "Qwen/Qwen2.5-14B-Instruct", "meta": { "sha": "cf98f3b3bbb457ad9e2bb7baf9a0125b6b88caa8", "safetensors": { "parameters": { "BF16": 14770033664 }, "total": 14770033664 }, "modelType": "qwen2" }, "config": { "architectures": [ "Qwen2ForCausalLM" ], "attention_dropout": 0, "bos_token_id": 151643, "eos_token_id": 151645, "hidden_act": "silu", "hidden_size": 5120, "initializer_range": 0.02, "intermediate_size": 13824, "max_position_embeddings": 32768, "max_window_layers": 70, "model_type": "qwen2", "num_attention_heads": 40, "num_hidden_layers": 48, "num_key_value_heads": 8, "rms_norm_eps": 1e-06, "rope_theta": 1000000, "sliding_window": 131072, "tie_word_embeddings": false, "torch_dtype": "bfloat16", "transformers_version": "4.43.1", "use_cache": true, "use_sliding_window": false, "vocab_size": 152064 } }, { "id": "Qwen/Qwen3-14B", "meta": { "sha": "40c069824f4251a91eefaf281ebe4c544efd3e18", "safetensors": { "parameters": { "BF16": 14768307200 }, "total": 14768307200 }, "modelType": "qwen3" }, "config": { "architectures": [ "Qwen3ForCausalLM" ], "attention_bias": false, "attention_dropout": 0, "bos_token_id": 151643, "eos_token_id": 151645, "head_dim": 128, "hidden_act": "silu", "hidden_size": 5120, "initializer_range": 0.02, "intermediate_size": 17408, "max_position_embeddings": 40960, "max_window_layers": 40, "model_type": "qwen3", "num_attention_heads": 40, "num_hidden_layers": 40, "num_key_value_heads": 8, "rms_norm_eps": 1e-06, "rope_scaling": null, "rope_theta": 1000000, "sliding_window": null, "tie_word_embeddings": false, "torch_dtype": "bfloat16", "transformers_version": "4.51.0", "use_cache": true, "use_sliding_window": false, "vocab_size": 151936 } }, { "id": "Qwen/Qwen2.5-Coder-14B-Instruct", "meta": { "sha": "aedcc2d42b622764e023cf882b6652e646b95671", "safetensors": { "parameters": { "BF16": 14770033664 }, "total": 14770033664 }, "modelType": "qwen2" }, "config": { "architectures": [ "Qwen2ForCausalLM" ], "attention_dropout": 0, "bos_token_id": 151643, "eos_token_id": 151645, "hidden_act": "silu", "hidden_size": 5120, "initializer_range": 0.02, "intermediate_size": 13824, "max_position_embeddings": 32768, "max_window_layers": 48, "model_type": "qwen2", "num_attention_heads": 40, "num_hidden_layers": 48, "num_key_value_heads": 8, "rms_norm_eps": 1e-06, "rope_theta": 1000000, "sliding_window": 131072, "tie_word_embeddings": false, "torch_dtype": "bfloat16", "transformers_version": "4.43.1", "use_cache": true, "use_sliding_window": false, "vocab_size": 152064 } }, { "id": "Qwen/Qwen3-30B-A3B", "meta": { "sha": "ad44e777bcd18fa416d9da3bd8f70d33ebb85d39", "safetensors": { "parameters": { "BF16": 30532122624 }, "total": 30532122624 }, "modelType": "qwen3_moe" }, "config": { "architectures": [ "Qwen3MoeForCausalLM" ], "attention_bias": false, "attention_dropout": 0, "bos_token_id": 151643, "decoder_sparse_step": 1, "eos_token_id": 151645, "head_dim": 128, "hidden_act": "silu", "hidden_size": 2048, "initializer_range": 0.02, "intermediate_size": 6144, "max_position_embeddings": 40960, "max_window_layers": 48, "mlp_only_layers": [], "model_type": "qwen3_moe", "moe_intermediate_size": 768, "norm_topk_prob": true, "num_attention_heads": 32, "num_experts": 128, "num_experts_per_tok": 8, "num_hidden_layers": 48, "num_key_value_heads": 4, "output_router_logits": false, "rms_norm_eps": 1e-06, "rope_scaling": null, "rope_theta": 1000000, "router_aux_loss_coef": 0.001, "sliding_window": null, "tie_word_embeddings": false, "torch_dtype": "bfloat16", "transformers_version": "4.51.0", "use_cache": true, "use_sliding_window": false, "vocab_size": 151936 } }, { "id": "Qwen/Qwen2.5-0.5B", "meta": { "sha": "060db6499f32faf8b98477b0a26969ef7d8b9987", "safetensors": { "parameters": { "BF16": 494032768 }, "total": 494032768 }, "modelType": "qwen2" }, "config": { "architectures": [ "Qwen2ForCausalLM" ], "attention_dropout": 0, "bos_token_id": 151643, "eos_token_id": 151643, "hidden_act": "silu", "hidden_size": 896, "initializer_range": 0.02, "intermediate_size": 4864, "max_position_embeddings": 32768, "max_window_layers": 24, "model_type": "qwen2", "num_attention_heads": 14, "num_hidden_layers": 24, "num_key_value_heads": 2, "rms_norm_eps": 1e-06, "rope_theta": 1000000, "sliding_window": 32768, "tie_word_embeddings": true, "torch_dtype": "bfloat16", "transformers_version": "4.40.1", "use_cache": true, "use_mrope": false, "use_sliding_window": false, "vocab_size": 151936 } }, { "id": "Qwen/Qwen3-1.7B-Base", "meta": { "sha": "ea980cb0a6c2ae4b936e82123acc929f1cec04c1", "safetensors": { "parameters": { "BF16": 1720574976 }, "total": 1720574976 }, "modelType": "qwen3" }, "config": { "architectures": [ "Qwen3ForCausalLM" ], "attention_bias": false, "attention_dropout": 0, "bos_token_id": 151643, "eos_token_id": 151643, "head_dim": 128, "hidden_act": "silu", "hidden_size": 2048, "initializer_range": 0.02, "intermediate_size": 6144, "max_position_embeddings": 32768, "max_window_layers": 28, "model_type": "qwen3", "num_attention_heads": 16, "num_hidden_layers": 28, "num_key_value_heads": 8, "rms_norm_eps": 1e-06, "rope_scaling": null, "rope_theta": 1000000, "sliding_window": null, "tie_word_embeddings": true, "torch_dtype": "bfloat16", "transformers_version": "4.51.0", "use_cache": true, "use_sliding_window": false, "vocab_size": 151936 } }, { "id": "Qwen/Qwen3-4B-Base", "meta": { "sha": "906bfd4b4dc7f14ee4320094d8b41684abff8539", "safetensors": { "parameters": { "BF16": 4022468096 }, "total": 4022468096 }, "modelType": "qwen3" }, "config": { "architectures": [ "Qwen3ForCausalLM" ], "attention_bias": false, "attention_dropout": 0, "bos_token_id": 151643, "eos_token_id": 151643, "head_dim": 128, "hidden_act": "silu", "hidden_size": 2560, "initializer_range": 0.02, "intermediate_size": 9728, "max_position_embeddings": 32768, "max_window_layers": 36, "model_type": "qwen3", "num_attention_heads": 32, "num_hidden_layers": 36, "num_key_value_heads": 8, "rms_norm_eps": 1e-06, "rope_scaling": null, "rope_theta": 1000000, "sliding_window": null, "tie_word_embeddings": true, "torch_dtype": "bfloat16", "transformers_version": "4.51.0", "use_cache": true, "use_sliding_window": false, "vocab_size": 151936 } }, { "id": "Qwen/Qwen2.5-Coder-32B-Instruct", "meta": { "sha": "381fc969f78efac66bc87ff7ddeadb7e73c218a7", "safetensors": { "parameters": { "BF16": 32763876352 }, "total": 32763876352 }, "modelType": "qwen2" }, "config": { "architectures": [ "Qwen2ForCausalLM" ], "attention_dropout": 0, "bos_token_id": 151643, "eos_token_id": 151645, "hidden_act": "silu", "hidden_size": 5120, "initializer_range": 0.02, "intermediate_size": 27648, "max_position_embeddings": 32768, "max_window_layers": 64, "model_type": "qwen2", "num_attention_heads": 40, "num_hidden_layers": 64, "num_key_value_heads": 8, "rms_norm_eps": 1e-06, "rope_theta": 1000000, "sliding_window": 131072, "tie_word_embeddings": false, "torch_dtype": "bfloat16", "transformers_version": "4.43.1", "use_cache": true, "use_sliding_window": false, "vocab_size": 152064 } }, { "id": "meta-llama/Llama-3.2-1B-Instruct", "meta": { "sha": "9213176726f574b556790deb65791e0c5aa438b6", "safetensors": { "parameters": { "BF16": 1235814400 }, "total": 1235814400 }, "modelType": "llama" }, "config": { "model_type": "llama", "num_hidden_layers": 16, "num_attention_heads": 32, "num_key_value_heads": 8, "hidden_size": 2048, "head_dim": 64, "vocab_size": 128256, "max_position_embeddings": 131072, "tie_word_embeddings": true, "sliding_window": null }, "previousConfigError": 401, "configSource": "https://github.com/meta-llama/llama-models/blob/0e0b8c519242d5833d8c11bffc1232b77ad7f301/models/sku_list.py", "configKind": "Normalized from official implementation, not the restricted Hugging Face config.json", "verifiedAt": "2026-09-22" }, { "id": "meta-llama/Llama-3.1-8B-Instruct", "meta": { "sha": "0e9e39f249a16976918f6564b8830bc894c89659", "safetensors": { "parameters": { "BF16": 8030261248 }, "total": 8030261248 }, "modelType": "llama" }, "config": { "model_type": "llama", "num_hidden_layers": 32, "num_attention_heads": 32, "num_key_value_heads": 8, "hidden_size": 4096, "head_dim": 128, "vocab_size": 128256, "max_position_embeddings": 131072, "tie_word_embeddings": false, "sliding_window": null }, "previousConfigError": 401, "configSource": "https://github.com/meta-llama/llama-models/blob/0e0b8c519242d5833d8c11bffc1232b77ad7f301/models/sku_list.py", "configKind": "Normalized from official implementation, not the restricted Hugging Face config.json", "verifiedAt": "2026-09-22" }, { "id": "meta-llama/Llama-3.2-3B-Instruct", "meta": { "sha": "0cb88a4f764b7a12671c53f0838cd831a0843b95", "safetensors": { "parameters": { "BF16": 3212749824 }, "total": 3212749824 }, "modelType": "llama" }, "config": { "model_type": "llama", "num_hidden_layers": 28, "num_attention_heads": 24, "num_key_value_heads": 8, "hidden_size": 3072, "head_dim": 128, "vocab_size": 128256, "max_position_embeddings": 131072, "tie_word_embeddings": true, "sliding_window": null }, "previousConfigError": 401, "configSource": "https://github.com/meta-llama/llama-models/blob/0e0b8c519242d5833d8c11bffc1232b77ad7f301/models/sku_list.py", "configKind": "Normalized from official implementation, not the restricted Hugging Face config.json", "verifiedAt": "2026-09-22" }, { "id": "meta-llama/Meta-Llama-3-8B-Instruct", "meta": { "sha": "8afb486c1db24fe5011ec46dfbe5b5dccdb575c2", "safetensors": { "parameters": { "BF16": 8030261248 }, "total": 8030261248 }, "modelType": "llama" }, "config": { "model_type": "llama", "num_hidden_layers": 32, "num_attention_heads": 32, "num_key_value_heads": 8, "hidden_size": 4096, "head_dim": 128, "vocab_size": 128256, "max_position_embeddings": 8192, "tie_word_embeddings": false, "sliding_window": null }, "previousConfigError": 401, "configSource": "https://github.com/meta-llama/llama-models/blob/0e0b8c519242d5833d8c11bffc1232b77ad7f301/models/sku_list.py", "configKind": "Normalized from official implementation, not the restricted Hugging Face config.json", "verifiedAt": "2026-09-22" }, { "id": "meta-llama/Llama-3.3-70B-Instruct", "meta": { "sha": "6f6073b423013f6a7d4d9f39144961bfbfbc386b", "safetensors": { "parameters": { "BF16": 70553706496 }, "total": 70553706496 }, "modelType": "llama" }, "config": { "model_type": "llama", "num_hidden_layers": 80, "num_attention_heads": 64, "num_key_value_heads": 8, "hidden_size": 8192, "head_dim": 128, "vocab_size": 128256, "max_position_embeddings": 131072, "tie_word_embeddings": false, "sliding_window": null }, "previousConfigError": 401, "configSource": "https://github.com/meta-llama/llama-models/blob/0e0b8c519242d5833d8c11bffc1232b77ad7f301/models/sku_list.py", "configKind": "Normalized from official implementation, not the restricted Hugging Face config.json", "verifiedAt": "2026-09-22" }, { "id": "meta-llama/Llama-3.2-1B", "meta": { "sha": "4e20de362430cd3b72f300e6b0f18e50e7166e08", "safetensors": { "parameters": { "BF16": 1235814400 }, "total": 1235814400 }, "modelType": "llama" }, "config": { "model_type": "llama", "num_hidden_layers": 16, "num_attention_heads": 32, "num_key_value_heads": 8, "hidden_size": 2048, "head_dim": 64, "vocab_size": 128256, "max_position_embeddings": 131072, "tie_word_embeddings": true, "sliding_window": null }, "previousConfigError": 401, "configSource": "https://github.com/meta-llama/llama-models/blob/0e0b8c519242d5833d8c11bffc1232b77ad7f301/models/sku_list.py", "configKind": "Normalized from official implementation, not the restricted Hugging Face config.json", "verifiedAt": "2026-09-22" }, { "id": "meta-llama/Llama-2-7b-hf", "meta": { "sha": "01c7f73d771dfac7d292323805ebc428287df4f9", "safetensors": { "parameters": { "F32": 2048, "F16": 6738415616 }, "total": 6738417664 }, "modelType": "llama" }, "config": { "error": 401 } }, { "id": "meta-llama/Llama-3.1-8B", "meta": { "sha": "d04e592bb4f6aa9cfee91e2e20afa771667e1d4b", "safetensors": { "parameters": { "BF16": 8030261248 }, "total": 8030261248 }, "modelType": "llama" }, "config": { "model_type": "llama", "num_hidden_layers": 32, "num_attention_heads": 32, "num_key_value_heads": 8, "hidden_size": 4096, "head_dim": 128, "vocab_size": 128256, "max_position_embeddings": 131072, "tie_word_embeddings": false, "sliding_window": null }, "previousConfigError": 401, "configSource": "https://github.com/meta-llama/llama-models/blob/0e0b8c519242d5833d8c11bffc1232b77ad7f301/models/sku_list.py", "configKind": "Normalized from official implementation, not the restricted Hugging Face config.json", "verifiedAt": "2026-09-22" }, { "id": "meta-llama/Llama-2-7b-chat-hf", "meta": { "sha": "f5db02db724555f92da89c216ac04704f23d4590", "safetensors": { "parameters": { "F16": 6738417664 }, "total": 6738417664 }, "modelType": "llama" }, "config": { "error": 401 } }, { "id": "meta-llama/Llama-3.2-3B", "meta": { "sha": "13afe5124825b4f3751f836b40dafda64c1ed062", "safetensors": { "parameters": { "BF16": 3212749824 }, "total": 3212749824 }, "modelType": "llama" }, "config": { "model_type": "llama", "num_hidden_layers": 28, "num_attention_heads": 24, "num_key_value_heads": 8, "hidden_size": 3072, "head_dim": 128, "vocab_size": 128256, "max_position_embeddings": 131072, "tie_word_embeddings": true, "sliding_window": null }, "previousConfigError": 401, "configSource": "https://github.com/meta-llama/llama-models/blob/0e0b8c519242d5833d8c11bffc1232b77ad7f301/models/sku_list.py", "configKind": "Normalized from official implementation, not the restricted Hugging Face config.json", "verifiedAt": "2026-09-22" }, { "id": "meta-llama/Meta-Llama-3-8B", "meta": { "sha": "8cde5ca8380496c9a6cc7ef3a8b46a0372a1d920", "safetensors": { "parameters": { "BF16": 8030261248 }, "total": 8030261248 }, "modelType": "llama" }, "config": { "model_type": "llama", "num_hidden_layers": 32, "num_attention_heads": 32, "num_key_value_heads": 8, "hidden_size": 4096, "head_dim": 128, "vocab_size": 128256, "max_position_embeddings": 8192, "tie_word_embeddings": false, "sliding_window": null }, "previousConfigError": 401, "configSource": "https://github.com/meta-llama/llama-models/blob/0e0b8c519242d5833d8c11bffc1232b77ad7f301/models/sku_list.py", "configKind": "Normalized from official implementation, not the restricted Hugging Face config.json", "verifiedAt": "2026-09-22" }, { "id": "meta-llama/Llama-2-13b-chat-hf", "meta": { "sha": "a2cb7a712bb6e5e736ca7f8cd98167f81a0b5bd8", "safetensors": { "parameters": { "F32": 2560, "F16": 13015864320 }, "total": 13015866880 }, "modelType": "llama" }, "config": { "error": 401 } }, { "id": "meta-llama/Llama-3.1-70B-Instruct", "meta": { "sha": "1605565b47bb9346c5515c34102e054115b4f98b", "safetensors": { "parameters": { "BF16": 70553706496 }, "total": 70553706496 }, "modelType": "llama" }, "config": { "model_type": "llama", "num_hidden_layers": 80, "num_attention_heads": 64, "num_key_value_heads": 8, "hidden_size": 8192, "head_dim": 128, "vocab_size": 128256, "max_position_embeddings": 131072, "tie_word_embeddings": false, "sliding_window": null }, "previousConfigError": 401, "configSource": "https://github.com/meta-llama/llama-models/blob/0e0b8c519242d5833d8c11bffc1232b77ad7f301/models/sku_list.py", "configKind": "Normalized from official implementation, not the restricted Hugging Face config.json", "verifiedAt": "2026-09-22" }, { "id": "meta-llama/Meta-Llama-3-70B-Instruct", "meta": { "sha": "50fd307e57011801c7833c87efa1984ddf2db42f", "safetensors": { "parameters": { "BF16": 70553706496 }, "total": 70553706496 }, "modelType": "llama" }, "config": { "model_type": "llama", "num_hidden_layers": 80, "num_attention_heads": 64, "num_key_value_heads": 8, "hidden_size": 8192, "head_dim": 128, "vocab_size": 128256, "max_position_embeddings": 8192, "tie_word_embeddings": false, "sliding_window": null }, "previousConfigError": 401, "configSource": "https://github.com/meta-llama/llama-models/blob/0e0b8c519242d5833d8c11bffc1232b77ad7f301/models/sku_list.py", "configKind": "Normalized from official implementation, not the restricted Hugging Face config.json", "verifiedAt": "2026-09-22" }, { "id": "meta-llama/Llama-3.1-405B", "meta": { "sha": "b906e4dc842aa489c962f9db26554dcfdde901fe", "safetensors": { "parameters": { "BF16": 405853388800 }, "total": 405853388800 }, "modelType": "llama" }, "config": { "model_type": "llama", "num_hidden_layers": 126, "num_attention_heads": 128, "num_key_value_heads": 8, "hidden_size": 16384, "head_dim": 128, "vocab_size": 128256, "max_position_embeddings": 131072, "tie_word_embeddings": false, "sliding_window": null }, "previousConfigError": 401, "configSource": "https://github.com/meta-llama/llama-models/blob/0e0b8c519242d5833d8c11bffc1232b77ad7f301/models/sku_list.py", "configKind": "Normalized from official implementation, not the restricted Hugging Face config.json", "verifiedAt": "2026-09-22" }, { "id": "meta-llama/Meta-Llama-3-70B", "meta": { "sha": "c82494877ce7f6d7d317c56ec081328e382c72fe", "safetensors": { "parameters": { "BF16": 70553706496 }, "total": 70553706496 }, "modelType": "llama" }, "config": { "model_type": "llama", "num_hidden_layers": 80, "num_attention_heads": 64, "num_key_value_heads": 8, "hidden_size": 8192, "head_dim": 128, "vocab_size": 128256, "max_position_embeddings": 8192, "tie_word_embeddings": false, "sliding_window": null }, "previousConfigError": 401, "configSource": "https://github.com/meta-llama/llama-models/blob/0e0b8c519242d5833d8c11bffc1232b77ad7f301/models/sku_list.py", "configKind": "Normalized from official implementation, not the restricted Hugging Face config.json", "verifiedAt": "2026-09-22" }, { "id": "meta-llama/Llama-3.1-70B", "meta": { "sha": "349b2ddb53ce8f2849a6c168a81980ab25258dac", "safetensors": { "parameters": { "BF16": 70553706496 }, "total": 70553706496 }, "modelType": "llama" }, "config": { "model_type": "llama", "num_hidden_layers": 80, "num_attention_heads": 64, "num_key_value_heads": 8, "hidden_size": 8192, "head_dim": 128, "vocab_size": 128256, "max_position_embeddings": 131072, "tie_word_embeddings": false, "sliding_window": null }, "previousConfigError": 401, "configSource": "https://github.com/meta-llama/llama-models/blob/0e0b8c519242d5833d8c11bffc1232b77ad7f301/models/sku_list.py", "configKind": "Normalized from official implementation, not the restricted Hugging Face config.json", "verifiedAt": "2026-09-22" }, { "id": "meta-llama/Llama-2-13b-hf", "meta": { "sha": "5c31dfb671ce7cfe2d7bb7c04375e44c55e815b1", "safetensors": { "parameters": { "F32": 2560, "F16": 13015864320 }, "total": 13015866880 }, "modelType": "llama" }, "config": { "error": 401 } }, { "id": "meta-llama/Llama-3.1-405B-Instruct", "meta": { "sha": "be673f326cab4cd22ccfef76109faf68e41aa5f1", "safetensors": { "parameters": { "BF16": 405853388800 }, "total": 405853388800 }, "modelType": "llama" }, "config": { "model_type": "llama", "num_hidden_layers": 126, "num_attention_heads": 128, "num_key_value_heads": 8, "hidden_size": 16384, "head_dim": 128, "vocab_size": 128256, "max_position_embeddings": 131072, "tie_word_embeddings": false, "sliding_window": null }, "previousConfigError": 401, "configSource": "https://github.com/meta-llama/llama-models/blob/0e0b8c519242d5833d8c11bffc1232b77ad7f301/models/sku_list.py", "configKind": "Normalized from official implementation, not the restricted Hugging Face config.json", "verifiedAt": "2026-09-22" }, { "id": "meta-llama/Llama-2-70b-chat-hf", "meta": { "sha": "e9149a12809580e8602995856f8098ce973d1080", "safetensors": { "parameters": { "F16": 68976648192, "F32": 5120 }, "total": 68976653312 }, "modelType": "llama" }, "config": { "error": 401 } }, { "id": "meta-llama/Llama-2-70b-hf", "meta": { "sha": "3aba440b59558f995867ba6e1f58f21d0336b5bb", "safetensors": { "parameters": { "F16": 68976648192, "F32": 5120 }, "total": 68976653312 }, "modelType": "llama" }, "config": { "error": 401 } }, { "id": "meta-llama/CodeLlama-7b-Instruct-hf", "meta": { "sha": "4ce0c40b2ea823bd1d8f1f3fd5bc8a7e80d749bc", "safetensors": { "parameters": { "BF16": 6738546688 }, "total": 6738546688 }, "modelType": "llama" }, "config": { "error": 401 } }, { "id": "meta-llama/CodeLlama-34b-Instruct-hf", "meta": { "sha": "edf62e7a0389eafc71b8fb1da96008817c0c9edb", "safetensors": { "parameters": { "BF16": 33743970304 }, "total": 33743970304 }, "modelType": "llama" }, "config": { "error": 401 } }, { "id": "meta-llama/CodeLlama-7b-hf", "meta": { "sha": "b462c3c99b077d341db691ec780a33156f3c1472", "safetensors": { "parameters": { "BF16": 6738546688 }, "total": 6738546688 }, "modelType": "llama" }, "config": { "error": 401 } }, { "id": "meta-llama/CodeLlama-13b-Instruct-hf", "meta": { "sha": "c24689b733f841ab1a6cce35bccb729f3f7b23fd", "safetensors": { "parameters": { "BF16": 13016028160 }, "total": 13016028160 }, "modelType": "llama" }, "config": { "error": 401 } }, { "id": "deepseek-ai/DeepSeek-V4-Flash-0731", "meta": { "sha": "7872f01b1d1fe23eabc4c98b48bffcef5a386062", "safetensors": { "parameters": { "BF16": 1483567488, "I64": 2327040, "F32": 37741630, "F8_E4M3": 6304038912, "I8": 296352743424 }, "total": 304180418494 }, "modelType": "deepseek_v4" }, "config": { "architectures": [ "DeepseekV4ForCausalLM" ], "attention_bias": false, "attention_dropout": 0, "bos_token_id": 0, "eos_token_id": 1, "expert_dtype": "fp4", "hc_eps": 1e-06, "hc_mult": 4, "hc_sinkhorn_iters": 20, "head_dim": 512, "hidden_act": "silu", "hidden_size": 4096, "index_head_dim": 128, "index_n_heads": 64, "index_topk": 512, "initializer_range": 0.02, "max_position_embeddings": 1048576, "model_type": "deepseek_v4", "moe_intermediate_size": 2048, "n_routed_experts": 256, "n_shared_experts": 1, "norm_topk_prob": true, "num_attention_heads": 64, "num_experts_per_tok": 6, "num_hidden_layers": 43, "num_hash_layers": 3, "num_key_value_heads": 1, "num_nextn_predict_layers": 1, "o_groups": 8, "o_lora_rank": 1024, "q_lora_rank": 1024, "qk_rope_head_dim": 64, "quantization_config": { "activation_scheme": "dynamic", "fmt": "e4m3", "quant_method": "fp8", "scale_fmt": "ue8m0", "weight_block_size": [ 128, 128 ] }, "rms_norm_eps": 1e-06, "rope_scaling": { "beta_fast": 32, "beta_slow": 1, "factor": 16, "original_max_position_embeddings": 65536, "type": "yarn" }, "rope_theta": 10000, "routed_scaling_factor": 1.5, "scoring_func": "sqrtsoftplus", "sliding_window": 128, "swiglu_limit": 10, "tie_word_embeddings": false, "topk_method": "noaux_tc", "torch_dtype": "bfloat16", "transformers_version": "4.57.1", "use_cache": true, "vocab_size": 129280, "compress_rope_theta": 160000, "compress_ratios": [ 0, 0, 4, 128, 4, 128, 4, 128, 4, 128, 4, 128, 4, 128, 4, 128, 4, 128, 4, 128, 4, 128, 4, 128, 4, 128, 4, 128, 4, 128, 4, 128, 4, 128, 4, 128, 4, 128, 4, 128, 4, 128, 4, 0, 0, 0 ], "dspark_block_size": 5, "dspark_noise_token_id": 128799, "dspark_target_layer_ids": [ 40, 41, 42 ], "dspark_markov_rank": 256 } }, { "id": "deepseek-ai/DeepSeek-V3.2", "meta": { "sha": "a7e62ac04ecb2c0a54d736dc46601c5606cf10a6", "safetensors": { "parameters": { "BF16": 3946184704, "F8_E4M3": 681408069632, "F32": 1075456 }, "total": 685355329792 }, "modelType": "deepseek_v32" }, "config": { "architectures": [ "DeepseekV32ForCausalLM" ], "attention_bias": false, "attention_dropout": 0, "bos_token_id": 0, "eos_token_id": 1, "ep_size": 1, "first_k_dense_replace": 3, "hidden_act": "silu", "hidden_size": 7168, "index_head_dim": 128, "index_n_heads": 64, "index_topk": 2048, "initializer_range": 0.02, "intermediate_size": 18432, "kv_lora_rank": 512, "max_position_embeddings": 163840, "model_type": "deepseek_v32", "moe_intermediate_size": 2048, "moe_layer_freq": 1, "n_group": 8, "n_routed_experts": 256, "n_shared_experts": 1, "norm_topk_prob": true, "num_attention_heads": 128, "num_experts_per_tok": 8, "num_hidden_layers": 61, "num_key_value_heads": 128, "num_nextn_predict_layers": 1, "q_lora_rank": 1536, "qk_nope_head_dim": 128, "qk_rope_head_dim": 64, "quantization_config": { "activation_scheme": "dynamic", "fmt": "e4m3", "quant_method": "fp8", "scale_fmt": "ue8m0", "weight_block_size": [ 128, 128 ] }, "rms_norm_eps": 1e-06, "rope_scaling": { "beta_fast": 32, "beta_slow": 1, "factor": 40, "mscale": 1, "mscale_all_dim": 1, "original_max_position_embeddings": 4096, "type": "yarn" }, "rope_theta": 10000, "routed_scaling_factor": 2.5, "scoring_func": "sigmoid", "tie_word_embeddings": false, "topk_group": 4, "topk_method": "noaux_tc", "torch_dtype": "bfloat16", "transformers_version": "4.44.2", "use_cache": true, "v_head_dim": 128, "vocab_size": 129280 } }, { "id": "deepseek-ai/DeepSeek-V4-Flash", "meta": { "sha": "60d8d70770c6776ff598c94bb586a859a38244f1", "safetensors": { "parameters": { "BF16": 1415259264, "I64": 2327040, "F32": 36168018, "F8_E4M3": 6023020544, "I8": 283467841536 }, "total": 290944616402 }, "modelType": "deepseek_v4" }, "config": { "architectures": [ "DeepseekV4ForCausalLM" ], "attention_bias": false, "attention_dropout": 0, "bos_token_id": 0, "eos_token_id": 1, "expert_dtype": "fp4", "hc_eps": 1e-06, "hc_mult": 4, "hc_sinkhorn_iters": 20, "head_dim": 512, "hidden_act": "silu", "hidden_size": 4096, "index_head_dim": 128, "index_n_heads": 64, "index_topk": 512, "initializer_range": 0.02, "max_position_embeddings": 1048576, "model_type": "deepseek_v4", "moe_intermediate_size": 2048, "n_routed_experts": 256, "n_shared_experts": 1, "norm_topk_prob": true, "num_attention_heads": 64, "num_experts_per_tok": 6, "num_hidden_layers": 43, "num_hash_layers": 3, "num_key_value_heads": 1, "num_nextn_predict_layers": 1, "o_groups": 8, "o_lora_rank": 1024, "q_lora_rank": 1024, "qk_rope_head_dim": 64, "quantization_config": { "activation_scheme": "dynamic", "fmt": "e4m3", "quant_method": "fp8", "scale_fmt": "ue8m0", "weight_block_size": [ 128, 128 ] }, "rms_norm_eps": 1e-06, "rope_scaling": { "beta_fast": 32, "beta_slow": 1, "factor": 16, "original_max_position_embeddings": 65536, "type": "yarn" }, "rope_theta": 10000, "routed_scaling_factor": 1.5, "scoring_func": "sqrtsoftplus", "sliding_window": 128, "swiglu_limit": 10, "tie_word_embeddings": false, "topk_method": "noaux_tc", "torch_dtype": "bfloat16", "transformers_version": "4.57.1", "use_cache": true, "vocab_size": 129280, "compress_rope_theta": 160000, "compress_ratios": [ 0, 0, 4, 128, 4, 128, 4, 128, 4, 128, 4, 128, 4, 128, 4, 128, 4, 128, 4, 128, 4, 128, 4, 128, 4, 128, 4, 128, 4, 128, 4, 128, 4, 128, 4, 128, 4, 128, 4, 128, 4, 128, 4, 0 ] } }, { "id": "deepseek-ai/DeepSeek-V3-0324", "meta": { "sha": "e9b33add76883f293d6bf61f6bd89b497e80e335", "safetensors": { "parameters": { "BF16": 3918786560, "F8_E4M3": 680571043840, "F32": 41555600 }, "total": 684531386000 }, "modelType": "deepseek_v3" }, "config": { "architectures": [ "DeepseekV3ForCausalLM" ], "attention_bias": false, "attention_dropout": 0.0, "auto_map": { "AutoConfig": "configuration_deepseek.DeepseekV3Config", "AutoModel": "modeling_deepseek.DeepseekV3Model", "AutoModelForCausalLM": "modeling_deepseek.DeepseekV3ForCausalLM" }, "bos_token_id": 0, "eos_token_id": 1, "ep_size": 1, "first_k_dense_replace": 3, "hidden_act": "silu", "hidden_size": 7168, "initializer_range": 0.02, "intermediate_size": 18432, "kv_lora_rank": 512, "max_position_embeddings": 163840, "model_type": "deepseek_v3", "moe_intermediate_size": 2048, "moe_layer_freq": 1, "n_group": 8, "n_routed_experts": 256, "n_shared_experts": 1, "norm_topk_prob": true, "num_attention_heads": 128, "num_experts_per_tok": 8, "num_hidden_layers": 61, "num_key_value_heads": 128, "num_nextn_predict_layers": 1, "q_lora_rank": 1536, "qk_nope_head_dim": 128, "qk_rope_head_dim": 64, "quantization_config": { "activation_scheme": "dynamic", "fmt": "e4m3", "quant_method": "fp8", "weight_block_size": [ 128, 128 ] }, "rms_norm_eps": 1e-06, "rope_scaling": { "beta_fast": 32, "beta_slow": 1, "factor": 40, "mscale": 1.0, "mscale_all_dim": 1.0, "original_max_position_embeddings": 4096, "type": "yarn" }, "rope_theta": 10000, "routed_scaling_factor": 2.5, "scoring_func": "sigmoid", "tie_word_embeddings": false, "topk_group": 4, "topk_method": "noaux_tc", "torch_dtype": "bfloat16", "transformers_version": "4.46.3", "use_cache": true, "v_head_dim": 128, "vocab_size": 129280 }, "verifiedAt": "2026-09-22" }, { "id": "deepseek-ai/DeepSeek-V3", "meta": { "sha": "e815299b0bcbac849fa540c768ef21845365c9eb", "safetensors": { "parameters": { "BF16": 3918786560, "F8_E4M3": 680571043840, "F32": 41555600 }, "total": 684531386000 }, "modelType": "deepseek_v3" }, "config": { "architectures": [ "DeepseekV3ForCausalLM" ], "attention_bias": false, "attention_dropout": 0.0, "auto_map": { "AutoConfig": "configuration_deepseek.DeepseekV3Config", "AutoModel": "modeling_deepseek.DeepseekV3Model", "AutoModelForCausalLM": "modeling_deepseek.DeepseekV3ForCausalLM" }, "bos_token_id": 0, "eos_token_id": 1, "ep_size": 1, "first_k_dense_replace": 3, "hidden_act": "silu", "hidden_size": 7168, "initializer_range": 0.02, "intermediate_size": 18432, "kv_lora_rank": 512, "max_position_embeddings": 163840, "model_type": "deepseek_v3", "moe_intermediate_size": 2048, "moe_layer_freq": 1, "n_group": 8, "n_routed_experts": 256, "n_shared_experts": 1, "norm_topk_prob": true, "num_attention_heads": 128, "num_experts_per_tok": 8, "num_hidden_layers": 61, "num_key_value_heads": 128, "num_nextn_predict_layers": 1, "q_lora_rank": 1536, "qk_nope_head_dim": 128, "qk_rope_head_dim": 64, "quantization_config": { "activation_scheme": "dynamic", "fmt": "e4m3", "quant_method": "fp8", "weight_block_size": [ 128, 128 ] }, "rms_norm_eps": 1e-06, "rope_scaling": { "beta_fast": 32, "beta_slow": 1, "factor": 40, "mscale": 1.0, "mscale_all_dim": 1.0, "original_max_position_embeddings": 4096, "type": "yarn" }, "rope_theta": 10000, "routed_scaling_factor": 2.5, "scoring_func": "sigmoid", "tie_word_embeddings": false, "topk_group": 4, "topk_method": "noaux_tc", "torch_dtype": "bfloat16", "transformers_version": "4.33.1", "use_cache": true, "v_head_dim": 128, "vocab_size": 129280 }, "verifiedAt": "2026-09-22" }, { "id": "deepseek-ai/DeepSeek-V4-Flash-DSpark", "meta": { "sha": "62af8fffb2f7030cac4de2f0169f5b8d1101b646", "safetensors": { "parameters": { "BF16": 1483567488, "I64": 2327040, "F32": 37741630, "F8_E8M0": 9261408000, "F8_E4M3": 6304038912, "I8": 148176371712 }, "total": 165265454782 }, "modelType": "deepseek_v4" }, "config": { "architectures": [ "DeepseekV4ForCausalLM" ], "attention_bias": false, "attention_dropout": 0, "bos_token_id": 0, "eos_token_id": 1, "expert_dtype": "fp4", "hc_eps": 1e-06, "hc_mult": 4, "hc_sinkhorn_iters": 20, "head_dim": 512, "hidden_act": "silu", "hidden_size": 4096, "index_head_dim": 128, "index_n_heads": 64, "index_topk": 512, "initializer_range": 0.02, "max_position_embeddings": 1048576, "model_type": "deepseek_v4", "moe_intermediate_size": 2048, "n_routed_experts": 256, "n_shared_experts": 1, "norm_topk_prob": true, "num_attention_heads": 64, "num_experts_per_tok": 6, "num_hidden_layers": 43, "num_hash_layers": 3, "num_key_value_heads": 1, "num_nextn_predict_layers": 1, "o_groups": 8, "o_lora_rank": 1024, "q_lora_rank": 1024, "qk_rope_head_dim": 64, "quantization_config": { "activation_scheme": "dynamic", "fmt": "e4m3", "quant_method": "fp8", "scale_fmt": "ue8m0", "weight_block_size": [ 128, 128 ] }, "rms_norm_eps": 1e-06, "rope_scaling": { "beta_fast": 32, "beta_slow": 1, "factor": 16, "original_max_position_embeddings": 65536, "type": "yarn" }, "rope_theta": 10000, "routed_scaling_factor": 1.5, "scoring_func": "sqrtsoftplus", "sliding_window": 128, "swiglu_limit": 10, "tie_word_embeddings": false, "topk_method": "noaux_tc", "torch_dtype": "bfloat16", "transformers_version": "4.57.1", "use_cache": true, "vocab_size": 129280, "compress_rope_theta": 160000, "compress_ratios": [ 0, 0, 4, 128, 4, 128, 4, 128, 4, 128, 4, 128, 4, 128, 4, 128, 4, 128, 4, 128, 4, 128, 4, 128, 4, 128, 4, 128, 4, 128, 4, 128, 4, 128, 4, 128, 4, 128, 4, 128, 4, 128, 4, 0, 0, 0 ], "dspark_block_size": 5, "dspark_noise_token_id": 128799, "dspark_target_layer_ids": [ 40, 41, 42 ], "dspark_markov_rank": 256 } }, { "id": "deepseek-ai/DeepSeek-Coder-V2-Lite-Instruct", "meta": { "sha": "e434a23f91ba5b4923cf6c9d9a238eb4a08e3a11", "safetensors": { "parameters": { "BF16": 15706484224 }, "total": 15706484224 }, "modelType": "deepseek_v2" }, "config": { "architectures": [ "DeepseekV2ForCausalLM" ], "attention_bias": false, "attention_dropout": 0, "auto_map": { "AutoConfig": "configuration_deepseek.DeepseekV2Config", "AutoModel": "modeling_deepseek.DeepseekV2Model", "AutoModelForCausalLM": "modeling_deepseek.DeepseekV2ForCausalLM" }, "aux_loss_alpha": 0.001, "bos_token_id": 100000, "eos_token_id": 100001, "first_k_dense_replace": 1, "hidden_act": "silu", "hidden_size": 2048, "initializer_range": 0.02, "intermediate_size": 10944, "kv_lora_rank": 512, "max_position_embeddings": 163840, "model_type": "deepseek_v2", "moe_intermediate_size": 1408, "moe_layer_freq": 1, "n_group": 1, "n_routed_experts": 64, "n_shared_experts": 2, "norm_topk_prob": false, "num_attention_heads": 16, "num_experts_per_tok": 6, "num_hidden_layers": 27, "num_key_value_heads": 16, "pretraining_tp": 1, "q_lora_rank": null, "qk_nope_head_dim": 128, "qk_rope_head_dim": 64, "rms_norm_eps": 1e-06, "rope_scaling": { "beta_fast": 32, "beta_slow": 1, "factor": 40, "mscale": 0.707, "mscale_all_dim": 0.707, "original_max_position_embeddings": 4096, "type": "yarn" }, "rope_theta": 10000, "routed_scaling_factor": 1, "scoring_func": "softmax", "seq_aux": true, "tie_word_embeddings": false, "topk_group": 1, "topk_method": "greedy", "torch_dtype": "bfloat16", "transformers_version": "4.39.3", "use_cache": true, "v_head_dim": 128, "vocab_size": 102400 } }, { "id": "deepseek-ai/DeepSeek-R1-0528-Qwen3-8B", "meta": { "sha": "6e8885a6ff5c1dc5201574c8fd700323f23c25fa", "safetensors": { "parameters": { "BF16": 8190735360 }, "total": 8190735360 }, "modelType": "qwen3" }, "config": { "architectures": [ "Qwen3ForCausalLM" ], "attention_bias": false, "attention_dropout": 0, "bos_token_id": 151643, "eos_token_id": 151645, "head_dim": 128, "hidden_act": "silu", "hidden_size": 4096, "initializer_range": 0.02, "intermediate_size": 12288, "max_position_embeddings": 131072, "max_window_layers": 36, "model_type": "qwen3", "num_attention_heads": 32, "num_hidden_layers": 36, "num_key_value_heads": 8, "rms_norm_eps": 1e-06, "rope_scaling": { "rope_type": "yarn", "factor": 4, "original_max_position_embeddings": 32768, "attn_factor": 0.8782488562869419 }, "rope_theta": 1000000, "sliding_window": null, "tie_word_embeddings": false, "torch_dtype": "bfloat16", "transformers_version": "4.51.0", "use_cache": true, "use_sliding_window": false, "vocab_size": 151936 } }, { "id": "deepseek-ai/DeepSeek-R1", "meta": { "sha": "56d4cbbb4d29f4355bab4b9a39ccb717a14ad5ad", "safetensors": { "parameters": { "BF16": 3918786560, "F8_E4M3": 680571043840, "F32": 15104 }, "total": 684489845504 }, "modelType": "deepseek_v3" }, "config": { "architectures": [ "DeepseekV3ForCausalLM" ], "attention_bias": false, "attention_dropout": 0.0, "auto_map": { "AutoConfig": "configuration_deepseek.DeepseekV3Config", "AutoModel": "modeling_deepseek.DeepseekV3Model", "AutoModelForCausalLM": "modeling_deepseek.DeepseekV3ForCausalLM" }, "bos_token_id": 0, "eos_token_id": 1, "ep_size": 1, "first_k_dense_replace": 3, "hidden_act": "silu", "hidden_size": 7168, "initializer_range": 0.02, "intermediate_size": 18432, "kv_lora_rank": 512, "max_position_embeddings": 163840, "model_type": "deepseek_v3", "moe_intermediate_size": 2048, "moe_layer_freq": 1, "n_group": 8, "n_routed_experts": 256, "n_shared_experts": 1, "norm_topk_prob": true, "num_attention_heads": 128, "num_experts_per_tok": 8, "num_hidden_layers": 61, "num_key_value_heads": 128, "num_nextn_predict_layers": 1, "q_lora_rank": 1536, "qk_nope_head_dim": 128, "qk_rope_head_dim": 64, "quantization_config": { "activation_scheme": "dynamic", "fmt": "e4m3", "quant_method": "fp8", "weight_block_size": [ 128, 128 ] }, "rms_norm_eps": 1e-06, "rope_scaling": { "beta_fast": 32, "beta_slow": 1, "factor": 40, "mscale": 1.0, "mscale_all_dim": 1.0, "original_max_position_embeddings": 4096, "type": "yarn" }, "rope_theta": 10000, "routed_scaling_factor": 2.5, "scoring_func": "sigmoid", "tie_word_embeddings": false, "topk_group": 4, "topk_method": "noaux_tc", "torch_dtype": "bfloat16", "transformers_version": "4.46.3", "use_cache": true, "v_head_dim": 128, "vocab_size": 129280 }, "verifiedAt": "2026-09-22" }, { "id": "deepseek-ai/deepseek-coder-7b-instruct-v1.5", "meta": { "sha": "2a050a4c59d687a85324d32e147517992117ed30", "safetensors": { "parameters": { "BF16": 6910365696 }, "total": 6910365696 }, "modelType": "llama" }, "config": { "architectures": [ "LlamaForCausalLM" ], "attention_bias": false, "bos_token_id": 100000, "eos_token_id": 100015, "hidden_act": "silu", "hidden_size": 4096, "initializer_range": 0.02, "intermediate_size": 11008, "max_position_embeddings": 4096, "model_type": "llama", "num_attention_heads": 32, "num_hidden_layers": 30, "num_key_value_heads": 32, "pretraining_tp": 1, "rms_norm_eps": 1e-06, "rope_scaling": null, "rope_theta": 10000, "tie_word_embeddings": false, "torch_dtype": "bfloat16", "transformers_version": "4.35.2", "use_cache": true, "vocab_size": 102400 } }, { "id": "deepseek-ai/DeepSeek-V4-Pro", "meta": { "sha": "b5968e9190ef611bbf34a7229255be88a0e937c1", "safetensors": { "parameters": { "BF16": 2816899328, "I64": 2327040, "F32": 87776414, "F8_E4M3": 23169335296, "I8": 1572763336704 }, "total": 1598839674782 }, "modelType": "deepseek_v4" }, "config": { "architectures": [ "DeepseekV4ForCausalLM" ], "attention_bias": false, "attention_dropout": 0, "bos_token_id": 0, "eos_token_id": 1, "expert_dtype": "fp4", "hc_eps": 1e-06, "hc_mult": 4, "hc_sinkhorn_iters": 20, "head_dim": 512, "hidden_act": "silu", "hidden_size": 7168, "index_head_dim": 128, "index_n_heads": 64, "index_topk": 1024, "initializer_range": 0.02, "max_position_embeddings": 1048576, "model_type": "deepseek_v4", "moe_intermediate_size": 3072, "n_routed_experts": 384, "n_shared_experts": 1, "norm_topk_prob": true, "num_attention_heads": 128, "num_experts_per_tok": 6, "num_hidden_layers": 61, "num_hash_layers": 3, "num_key_value_heads": 1, "num_nextn_predict_layers": 1, "o_groups": 16, "o_lora_rank": 1024, "q_lora_rank": 1536, "qk_rope_head_dim": 64, "quantization_config": { "activation_scheme": "dynamic", "fmt": "e4m3", "quant_method": "fp8", "scale_fmt": "ue8m0", "weight_block_size": [ 128, 128 ] }, "rms_norm_eps": 1e-06, "rope_scaling": { "beta_fast": 32, "beta_slow": 1, "factor": 16, "original_max_position_embeddings": 65536, "type": "yarn" }, "rope_theta": 10000, "routed_scaling_factor": 2.5, "scoring_func": "sqrtsoftplus", "sliding_window": 128, "swiglu_limit": 10, "tie_word_embeddings": false, "topk_method": "noaux_tc", "torch_dtype": "bfloat16", "transformers_version": "4.57.1", "use_cache": true, "vocab_size": 129280, "compress_rope_theta": 160000, "compress_ratios": [ 128, 128, 4, 128, 4, 128, 4, 128, 4, 128, 4, 128, 4, 128, 4, 128, 4, 128, 4, 128, 4, 128, 4, 128, 4, 128, 4, 128, 4, 128, 4, 128, 4, 128, 4, 128, 4, 128, 4, 128, 4, 128, 4, 128, 4, 128, 4, 128, 4, 128, 4, 128, 4, 128, 4, 128, 4, 128, 4, 128, 4, 0 ] } }, { "id": "deepseek-ai/DeepSeek-V4.1-Flash", "meta": { "sha": "dba1be0a40aa45a94ad051997016db3960a90277", "safetensors": { "parameters": { "BF16": 1976441856, "F32": 42307282, "F8_E4M3": 204015223296, "I8": 557171343360 }, "total": 763205315794 }, "modelType": "deepseek_v41" }, "config": { "architectures": [ "DeepseekV41ForCausalLM" ], "model_type": "deepseek_v41", "dtype": "bfloat16", "transformers_version": "5.6.0", "bos_token_id": 0, "eos_token_id": 1, "pad_token_id": 2, "image_token_id": 129264, "quantization_config": { "quant_method": "fp8", "activation_scheme": "dynamic", "weight_block_size": [ 32, 32 ], "scale_fmt": "ue8m0", "expert_dtype": "fp4" }, "text_config": { "model_type": "deepseek_v41_text", "vocab_size": 129280, "hidden_size": 5120, "moe_intermediate_size": 2304, "num_hidden_layers": 40, "num_attention_heads": 64, "num_key_value_heads": 1, "head_dim": 512, "qk_rope_head_dim": 64, "q_lora_rank": 1280, "o_lora_rank": 1024, "o_groups": 8, "hidden_act": "silu", "swiglu_limit": 10, "rms_norm_eps": 1e-20, "attention_bias": false, "attention_dropout": 0, "initializer_range": 0.02, "use_cache": true, "tie_word_embeddings": false, "max_position_embeddings": 1048576, "rope_theta": 10000, "rope_scaling": { "rope_type": "yarn", "factor": 16, "beta_fast": 32, "beta_slow": 1, "original_max_position_embeddings": 65536 }, "n_routed_experts": 384, "n_shared_experts": 1, "num_experts_per_tok": 6, "scoring_func": "sqrtsoftplus", "topk_method": "noaux_tc", "norm_topk_prob": true, "routed_scaling_factor": 1.5, "sliding_window": 128, "compress_ratios": [ 0, 0, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0 ], "compress_rope_theta": 160000, "kv_source_layer_ids": [ 2, 8, 14, 20 ], "index_source_layer_ids": [ 2, 8, 14, 20, 24, 28, 32, 36 ], "index_n_heads": 32, "index_head_dim": 128, "index_topk": 512, "candidate_source_layer_id": 20, "candidate_topk_blocks": 2048, "candidate_block_size": 8, "hc_mult": 4, "hc_sinkhorn_iters": 20, "hc_eps": 1e-06, "engram_layer_ids": [ 1, 14 ], "engram_num_embeddings": [ 384006168, 384016682 ], "engram_max_ngram_size": 4, "engram_vocab_size": 16000000, "engram_n_heads": 8, "engram_head_dim": 256, "engram_pad_token_id": 2, "engram_compressed_vocab_size": 99092, "num_nextn_predict_layers": 3, "dspark_block_size": 5, "dspark_noise_token_id": 128799, "dspark_target_layer_ids": [ 37, 38, 39 ], "dspark_markov_rank": 256, "dspark_n_routed_experts": 128, "dspark_num_experts_per_tok": 3 }, "vision_config": { "model_type": "deepseek_v41_vision", "num_hidden_layers": 32, "hidden_size": 1024, "num_attention_heads": 16, "intermediate_size": 2816, "patch_size": 14, "rope_theta": 10000, "downsample_ratio": 3, "max_image_tokens": 1024, "min_pixels": 295936, "max_wh_ratio": null } } }, { "id": "deepseek-ai/DeepSeek-R1-Distill-Qwen-32B", "meta": { "sha": "711ad2ea6aa40cfca18895e8aca02ab92df1a746", "safetensors": { "parameters": { "BF16": 32763876352 }, "total": 32763876352 }, "modelType": "qwen2" }, "config": { "architectures": [ "Qwen2ForCausalLM" ], "attention_dropout": 0, "bos_token_id": 151643, "eos_token_id": 151643, "hidden_act": "silu", "hidden_size": 5120, "initializer_range": 0.02, "intermediate_size": 27648, "max_position_embeddings": 131072, "max_window_layers": 64, "model_type": "qwen2", "num_attention_heads": 40, "num_hidden_layers": 64, "num_key_value_heads": 8, "rms_norm_eps": 1e-05, "rope_theta": 1000000, "sliding_window": 131072, "tie_word_embeddings": false, "torch_dtype": "bfloat16", "transformers_version": "4.43.1", "use_cache": true, "use_sliding_window": false, "vocab_size": 152064 } }, { "id": "deepseek-ai/DeepSeek-R1-Distill-Qwen-1.5B", "meta": { "sha": "ad9f0ae0864d7fbcd1cd905e3c6c5b069cc8b562", "safetensors": { "parameters": { "BF16": 1777088000 }, "total": 1777088000 }, "modelType": "qwen2" }, "config": { "architectures": [ "Qwen2ForCausalLM" ], "attention_dropout": 0, "bos_token_id": 151643, "eos_token_id": 151643, "hidden_act": "silu", "hidden_size": 1536, "initializer_range": 0.02, "intermediate_size": 8960, "max_position_embeddings": 131072, "max_window_layers": 21, "model_type": "qwen2", "num_attention_heads": 12, "num_hidden_layers": 28, "num_key_value_heads": 2, "rms_norm_eps": 1e-06, "rope_theta": 10000, "sliding_window": 4096, "tie_word_embeddings": false, "torch_dtype": "bfloat16", "transformers_version": "4.44.0", "use_cache": true, "use_mrope": false, "use_sliding_window": false, "vocab_size": 151936 } }, { "id": "deepseek-ai/DeepSeek-R1-Distill-Qwen-14B", "meta": { "sha": "1df8507178afcc1bef68cd8c393f61a886323761", "safetensors": { "parameters": { "BF16": 14770033664 }, "total": 14770033664 }, "modelType": "qwen2" }, "config": { "architectures": [ "Qwen2ForCausalLM" ], "attention_dropout": 0, "bos_token_id": 151643, "eos_token_id": 151643, "hidden_act": "silu", "hidden_size": 5120, "initializer_range": 0.02, "intermediate_size": 13824, "max_position_embeddings": 131072, "max_window_layers": 48, "model_type": "qwen2", "num_attention_heads": 40, "num_hidden_layers": 48, "num_key_value_heads": 8, "rms_norm_eps": 1e-05, "rope_theta": 1000000, "sliding_window": 131072, "tie_word_embeddings": false, "torch_dtype": "bfloat16", "transformers_version": "4.43.1", "use_cache": true, "use_sliding_window": false, "vocab_size": 152064 } }, { "id": "deepseek-ai/DeepSeek-V3.1", "meta": { "sha": "c0781d039fb7a1ba2abc4add0bdc293e92d2b8db", "safetensors": { "parameters": { "BF16": 3918786560, "F8_E4M3": 680571043840, "F32": 41555600 }, "total": 684531386000 }, "modelType": "deepseek_v3" }, "config": { "architectures": [ "DeepseekV3ForCausalLM" ], "attention_bias": false, "attention_dropout": 0.0, "auto_map": { "AutoConfig": "configuration_deepseek.DeepseekV3Config", "AutoModel": "modeling_deepseek.DeepseekV3Model", "AutoModelForCausalLM": "modeling_deepseek.DeepseekV3ForCausalLM" }, "bos_token_id": 0, "eos_token_id": 1, "ep_size": 1, "first_k_dense_replace": 3, "hidden_act": "silu", "hidden_size": 7168, "initializer_range": 0.02, "intermediate_size": 18432, "kv_lora_rank": 512, "max_position_embeddings": 163840, "model_type": "deepseek_v3", "moe_intermediate_size": 2048, "moe_layer_freq": 1, "n_group": 8, "n_routed_experts": 256, "n_shared_experts": 1, "norm_topk_prob": true, "num_attention_heads": 128, "num_experts_per_tok": 8, "num_hidden_layers": 61, "num_key_value_heads": 128, "num_nextn_predict_layers": 1, "q_lora_rank": 1536, "qk_nope_head_dim": 128, "qk_rope_head_dim": 64, "quantization_config": { "activation_scheme": "dynamic", "fmt": "e4m3", "quant_method": "fp8", "weight_block_size": [ 128, 128 ], "scale_fmt": "ue8m0" }, "rms_norm_eps": 1e-06, "rope_scaling": { "beta_fast": 32, "beta_slow": 1, "factor": 40, "mscale": 1.0, "mscale_all_dim": 1.0, "original_max_position_embeddings": 4096, "type": "yarn" }, "rope_theta": 10000, "routed_scaling_factor": 2.5, "scoring_func": "sigmoid", "tie_word_embeddings": false, "topk_group": 4, "topk_method": "noaux_tc", "torch_dtype": "bfloat16", "transformers_version": "4.44.2", "use_cache": true, "v_head_dim": 128, "vocab_size": 129280 }, "verifiedAt": "2026-09-22" }, { "id": "deepseek-ai/DeepSeek-V2-Lite", "meta": { "sha": "604d5664dddd88a0433dbae533b7fe9472482de0", "safetensors": { "parameters": { "BF16": 15706484224 }, "total": 15706484224 }, "modelType": "deepseek_v2" }, "config": { "architectures": [ "DeepseekV2ForCausalLM" ], "attention_bias": false, "attention_dropout": 0, "auto_map": { "AutoConfig": "configuration_deepseek.DeepseekV2Config", "AutoModel": "modeling_deepseek.DeepseekV2Model", "AutoModelForCausalLM": "modeling_deepseek.DeepseekV2ForCausalLM" }, "aux_loss_alpha": 0.001, "bos_token_id": 100000, "eos_token_id": 100001, "first_k_dense_replace": 1, "hidden_act": "silu", "hidden_size": 2048, "initializer_range": 0.02, "intermediate_size": 10944, "kv_lora_rank": 512, "max_position_embeddings": 163840, "model_type": "deepseek_v2", "moe_intermediate_size": 1408, "moe_layer_freq": 1, "n_group": 1, "n_routed_experts": 64, "n_shared_experts": 2, "norm_topk_prob": false, "num_attention_heads": 16, "num_experts_per_tok": 6, "num_hidden_layers": 27, "num_key_value_heads": 16, "pretraining_tp": 1, "q_lora_rank": null, "qk_nope_head_dim": 128, "qk_rope_head_dim": 64, "rms_norm_eps": 1e-06, "rope_scaling": { "beta_fast": 32, "beta_slow": 1, "factor": 40, "mscale": 0.707, "mscale_all_dim": 0.707, "original_max_position_embeddings": 4096, "type": "yarn" }, "rope_theta": 10000, "routed_scaling_factor": 1, "scoring_func": "softmax", "seq_aux": true, "tie_word_embeddings": false, "topk_group": 1, "topk_method": "greedy", "torch_dtype": "bfloat16", "transformers_version": "4.33.1", "use_cache": true, "v_head_dim": 128, "vocab_size": 102400 } }, { "id": "deepseek-ai/deepseek-coder-6.7b-instruct", "meta": { "sha": "e5d64addd26a6a1db0f9b863abf6ee3141936807", "safetensors": { "parameters": { "BF16": 6740512768 }, "total": 6740512768 }, "modelType": "llama" }, "config": { "_name_or_path": "/3fs-jd/prod/deepseek/shared/zhuqihao/public_model/deepseek-coder-7b-instruct2", "architectures": [ "LlamaForCausalLM" ], "attention_bias": false, "bos_token_id": 32013, "eos_token_id": 32021, "hidden_act": "silu", "hidden_size": 4096, "initializer_range": 0.02, "intermediate_size": 11008, "max_position_embeddings": 16384, "model_type": "llama", "num_attention_heads": 32, "num_hidden_layers": 32, "num_key_value_heads": 32, "pretraining_tp": 1, "rms_norm_eps": 1e-06, "rope_scaling": { "factor": 4, "type": "linear" }, "rope_theta": 100000, "tie_word_embeddings": false, "torch_dtype": "bfloat16", "transformers_version": "4.34.1", "use_cache": true, "vocab_size": 32256 } }, { "id": "deepseek-ai/DeepSeek-R1-Distill-Qwen-7B", "meta": { "sha": "916b56a44061fd5cd7d6a8fb632557ed4f724f60", "safetensors": { "parameters": { "BF16": 7615616512 }, "total": 7615616512 }, "modelType": "qwen2" }, "config": { "architectures": [ "Qwen2ForCausalLM" ], "attention_dropout": 0, "bos_token_id": 151643, "eos_token_id": 151643, "hidden_act": "silu", "hidden_size": 3584, "initializer_range": 0.02, "intermediate_size": 18944, "max_position_embeddings": 131072, "max_window_layers": 28, "model_type": "qwen2", "num_attention_heads": 28, "num_hidden_layers": 28, "num_key_value_heads": 4, "rms_norm_eps": 1e-06, "rope_theta": 10000, "sliding_window": 4096, "tie_word_embeddings": false, "torch_dtype": "bfloat16", "transformers_version": "4.44.0", "use_cache": true, "use_mrope": false, "use_sliding_window": false, "vocab_size": 152064 } }, { "id": "deepseek-ai/DeepSeek-R1-Distill-Llama-8B", "meta": { "sha": "6a6f4aa4197940add57724a7707d069478df56b1", "safetensors": { "parameters": { "BF16": 8030261248 }, "total": 8030261248 }, "modelType": "llama" }, "config": { "architectures": [ "LlamaForCausalLM" ], "attention_bias": false, "attention_dropout": 0, "bos_token_id": 128000, "eos_token_id": 128001, "hidden_act": "silu", "hidden_size": 4096, "initializer_range": 0.02, "intermediate_size": 14336, "max_position_embeddings": 131072, "mlp_bias": false, "model_type": "llama", "num_attention_heads": 32, "num_hidden_layers": 32, "num_key_value_heads": 8, "pretraining_tp": 1, "rms_norm_eps": 1e-05, "rope_scaling": { "factor": 8, "low_freq_factor": 1, "high_freq_factor": 4, "original_max_position_embeddings": 8192, "rope_type": "llama3" }, "rope_theta": 500000, "tie_word_embeddings": false, "torch_dtype": "bfloat16", "transformers_version": "4.43.0.dev0", "use_cache": true, "vocab_size": 128256 } }, { "id": "deepseek-ai/DeepSeek-R1-0528", "meta": { "sha": "4236a6af538feda4548eca9ab308586007567f52", "safetensors": { "parameters": { "BF16": 3918786560, "F8_E4M3": 680571043840, "F32": 41555600 }, "total": 684531386000 }, "modelType": "deepseek_v3" }, "config": { "architectures": [ "DeepseekV3ForCausalLM" ], "attention_bias": false, "attention_dropout": 0.0, "auto_map": { "AutoConfig": "configuration_deepseek.DeepseekV3Config", "AutoModel": "modeling_deepseek.DeepseekV3Model", "AutoModelForCausalLM": "modeling_deepseek.DeepseekV3ForCausalLM" }, "bos_token_id": 0, "eos_token_id": 1, "ep_size": 1, "first_k_dense_replace": 3, "hidden_act": "silu", "hidden_size": 7168, "initializer_range": 0.02, "intermediate_size": 18432, "kv_lora_rank": 512, "max_position_embeddings": 163840, "model_type": "deepseek_v3", "moe_intermediate_size": 2048, "moe_layer_freq": 1, "n_group": 8, "n_routed_experts": 256, "n_shared_experts": 1, "norm_topk_prob": true, "num_attention_heads": 128, "num_experts_per_tok": 8, "num_hidden_layers": 61, "num_key_value_heads": 128, "num_nextn_predict_layers": 1, "q_lora_rank": 1536, "qk_nope_head_dim": 128, "qk_rope_head_dim": 64, "quantization_config": { "activation_scheme": "dynamic", "fmt": "e4m3", "quant_method": "fp8", "weight_block_size": [ 128, 128 ] }, "rms_norm_eps": 1e-06, "rope_scaling": { "beta_fast": 32, "beta_slow": 1, "factor": 40, "mscale": 1.0, "mscale_all_dim": 1.0, "original_max_position_embeddings": 4096, "type": "yarn" }, "rope_theta": 10000, "routed_scaling_factor": 2.5, "scoring_func": "sigmoid", "tie_word_embeddings": false, "topk_group": 4, "topk_method": "noaux_tc", "torch_dtype": "bfloat16", "transformers_version": "4.46.3", "use_cache": true, "v_head_dim": 128, "vocab_size": 129280 }, "verifiedAt": "2026-09-22" }, { "id": "deepseek-ai/DeepSeek-V4-Pro-0813", "meta": { "sha": "72e1d3230f6c080a530b0a1d46f8eb4602340597", "safetensors": { "parameters": { "BF16": 2954820352, "I64": 2327040, "F32": 90530058, "F8_E4M3": 23952621568, "I8": 1623497637888 }, "total": 1650497936906 }, "modelType": "deepseek_v4" }, "config": { "architectures": [ "DeepseekV4ForCausalLM" ], "attention_bias": false, "attention_dropout": 0, "bos_token_id": 0, "eos_token_id": 1, "expert_dtype": "fp4", "hc_eps": 1e-06, "hc_mult": 4, "hc_sinkhorn_iters": 20, "head_dim": 512, "hidden_act": "silu", "hidden_size": 7168, "index_head_dim": 128, "index_n_heads": 64, "index_topk": 1024, "initializer_range": 0.02, "max_position_embeddings": 1048576, "model_type": "deepseek_v4", "moe_intermediate_size": 3072, "n_routed_experts": 384, "n_shared_experts": 1, "norm_topk_prob": true, "num_attention_heads": 128, "num_experts_per_tok": 6, "num_hidden_layers": 61, "num_hash_layers": 3, "num_key_value_heads": 1, "num_nextn_predict_layers": 1, "o_groups": 16, "o_lora_rank": 1024, "q_lora_rank": 1536, "qk_rope_head_dim": 64, "quantization_config": { "activation_scheme": "dynamic", "fmt": "e4m3", "quant_method": "fp8", "scale_fmt": "ue8m0", "weight_block_size": [ 128, 128 ] }, "rms_norm_eps": 1e-06, "rope_scaling": { "beta_fast": 32, "beta_slow": 1, "factor": 16, "original_max_position_embeddings": 65536, "type": "yarn" }, "rope_theta": 10000, "routed_scaling_factor": 2.5, "scoring_func": "sqrtsoftplus", "sliding_window": 128, "swiglu_limit": 10, "tie_word_embeddings": false, "topk_method": "noaux_tc", "torch_dtype": "bfloat16", "transformers_version": "4.57.1", "use_cache": true, "vocab_size": 129280, "compress_rope_theta": 160000, "compress_ratios": [ 128, 128, 4, 128, 4, 128, 4, 128, 4, 128, 4, 128, 4, 128, 4, 128, 4, 128, 4, 128, 4, 128, 4, 128, 4, 128, 4, 128, 4, 128, 4, 128, 4, 128, 4, 128, 4, 128, 4, 128, 4, 128, 4, 128, 4, 128, 4, 128, 4, 128, 4, 128, 4, 128, 4, 128, 4, 128, 4, 128, 4, 0, 0, 0 ], "dspark_block_size": 5, "dspark_noise_token_id": 128799, "dspark_target_layer_ids": [ 58, 59, 60 ], "dspark_markov_rank": 512 } }, { "id": "deepseek-ai/DeepSeek-V2-Lite-Chat", "meta": { "sha": "85864749cd611b4353ce1decdb286193298f64c7", "safetensors": { "parameters": { "BF16": 15706484224 }, "total": 15706484224 }, "modelType": "deepseek_v2" }, "config": { "architectures": [ "DeepseekV2ForCausalLM" ], "attention_bias": false, "attention_dropout": 0, "auto_map": { "AutoConfig": "configuration_deepseek.DeepseekV2Config", "AutoModel": "modeling_deepseek.DeepseekV2Model", "AutoModelForCausalLM": "modeling_deepseek.DeepseekV2ForCausalLM" }, "aux_loss_alpha": 0.001, "bos_token_id": 100000, "eos_token_id": 100001, "first_k_dense_replace": 1, "hidden_act": "silu", "hidden_size": 2048, "initializer_range": 0.02, "intermediate_size": 10944, "kv_lora_rank": 512, "max_position_embeddings": 163840, "model_type": "deepseek_v2", "moe_intermediate_size": 1408, "moe_layer_freq": 1, "n_group": 1, "n_routed_experts": 64, "n_shared_experts": 2, "norm_topk_prob": false, "num_attention_heads": 16, "num_experts_per_tok": 6, "num_hidden_layers": 27, "num_key_value_heads": 16, "pretraining_tp": 1, "q_lora_rank": null, "qk_nope_head_dim": 128, "qk_rope_head_dim": 64, "rms_norm_eps": 1e-06, "rope_scaling": { "beta_fast": 32, "beta_slow": 1, "factor": 40, "mscale": 0.707, "mscale_all_dim": 0.707, "original_max_position_embeddings": 4096, "type": "yarn" }, "rope_theta": 10000, "routed_scaling_factor": 1, "scoring_func": "softmax", "seq_aux": true, "tie_word_embeddings": false, "topk_group": 1, "topk_method": "greedy", "torch_dtype": "bfloat16", "transformers_version": "4.33.1", "use_cache": true, "v_head_dim": 128, "vocab_size": 102400 } }, { "id": "deepseek-ai/deepseek-coder-6.7b-base", "meta": { "sha": "ce2207a8bfef3ee92bd7dd4cc31c52cfa0046912", "safetensors": { "parameters": { "BF16": 6740512768 }, "total": 6740512768 }, "modelType": "llama" }, "config": { "architectures": [ "LlamaForCausalLM" ], "bos_token_id": 32013, "eos_token_id": 32014, "hidden_act": "silu", "hidden_size": 4096, "initializer_range": 0.02, "intermediate_size": 11008, "max_position_embeddings": 16384, "model_type": "llama", "num_attention_heads": 32, "num_hidden_layers": 32, "num_key_value_heads": 32, "pretraining_tp": 1, "rms_norm_eps": 1e-06, "rope_scaling": { "factor": 4, "type": "linear" }, "rope_theta": 100000, "tie_word_embeddings": false, "torch_dtype": "bfloat16", "transformers_version": "4.34.1", "use_cache": true, "vocab_size": 32256 } }, { "id": "deepseek-ai/DeepSeek-R1-Distill-Llama-70B", "meta": { "sha": "b1c0b44b4369b597ad119a196caf79a9c40e141e", "safetensors": { "parameters": { "BF16": 70553706496 }, "total": 70553706496 }, "modelType": "llama" }, "config": { "architectures": [ "LlamaForCausalLM" ], "attention_bias": false, "attention_dropout": 0, "bos_token_id": 128000, "eos_token_id": [ 128001, 128008, 128009 ], "head_dim": 128, "hidden_act": "silu", "hidden_size": 8192, "initializer_range": 0.02, "intermediate_size": 28672, "max_position_embeddings": 131072, "mlp_bias": false, "model_type": "llama", "num_attention_heads": 64, "num_hidden_layers": 80, "num_key_value_heads": 8, "pretraining_tp": 1, "rms_norm_eps": 1e-05, "rope_scaling": { "factor": 8, "high_freq_factor": 4, "low_freq_factor": 1, "original_max_position_embeddings": 8192, "rope_type": "llama3" }, "rope_theta": 500000, "tie_word_embeddings": false, "torch_dtype": "bfloat16", "transformers_version": "4.47.0.dev0", "use_cache": true, "vocab_size": 128256 } }, { "id": "deepseek-ai/DeepSeek-V3.2-Exp", "meta": { "sha": "194c67e12b1b0d6df0ef373ddcf215bc84027409", "safetensors": { "parameters": { "BF16": 3946184704, "F8_E4M3": 681408069632, "F32": 42667040 }, "total": 685396921376 }, "modelType": "deepseek_v32" }, "config": { "architectures": [ "DeepseekV32ForCausalLM" ], "attention_bias": false, "attention_dropout": 0, "bos_token_id": 0, "eos_token_id": 1, "ep_size": 1, "first_k_dense_replace": 3, "hidden_act": "silu", "hidden_size": 7168, "index_head_dim": 128, "index_n_heads": 64, "index_topk": 2048, "initializer_range": 0.02, "intermediate_size": 18432, "kv_lora_rank": 512, "max_position_embeddings": 163840, "model_type": "deepseek_v32", "moe_intermediate_size": 2048, "moe_layer_freq": 1, "n_group": 8, "n_routed_experts": 256, "n_shared_experts": 1, "norm_topk_prob": true, "num_attention_heads": 128, "num_experts_per_tok": 8, "num_hidden_layers": 61, "num_key_value_heads": 128, "num_nextn_predict_layers": 1, "q_lora_rank": 1536, "qk_nope_head_dim": 128, "qk_rope_head_dim": 64, "quantization_config": { "activation_scheme": "dynamic", "fmt": "e4m3", "quant_method": "fp8", "scale_fmt": "ue8m0", "weight_block_size": [ 128, 128 ] }, "rms_norm_eps": 1e-06, "rope_scaling": { "beta_fast": 32, "beta_slow": 1, "factor": 40, "mscale": 1, "mscale_all_dim": 1, "original_max_position_embeddings": 4096, "type": "yarn" }, "rope_theta": 10000, "routed_scaling_factor": 2.5, "scoring_func": "sigmoid", "tie_word_embeddings": false, "topk_group": 4, "topk_method": "noaux_tc", "torch_dtype": "bfloat16", "transformers_version": "4.44.2", "use_cache": true, "v_head_dim": 128, "vocab_size": 129280 } }, { "id": "deepseek-ai/deepseek-coder-1.3b-instruct", "meta": { "sha": "e063262dac8366fc1f28a4da0ff3c50ea66259ca", "safetensors": { "parameters": { "BF16": 1346471936 }, "total": 1346471936 }, "modelType": "llama" }, "config": { "architectures": [ "LlamaForCausalLM" ], "bos_token_id": 32013, "eos_token_id": 32021, "hidden_act": "silu", "hidden_size": 2048, "initializer_range": 0.02, "intermediate_size": 5504, "max_position_embeddings": 16384, "model_type": "llama", "num_attention_heads": 16, "num_hidden_layers": 24, "num_key_value_heads": 16, "pretraining_tp": 1, "rms_norm_eps": 1e-06, "rope_scaling": { "factor": 4, "type": "linear" }, "rope_theta": 100000, "tie_word_embeddings": false, "torch_dtype": "bfloat16", "transformers_version": "4.33.1", "use_cache": true, "vocab_size": 32256 } }, { "id": "deepseek-ai/DeepSeek-V2", "meta": { "sha": "4461458f186c35188585855f28f77af5661ad489", "safetensors": { "parameters": { "BF16": 235741434880 }, "total": 235741434880 }, "modelType": "deepseek_v2" }, "config": { "architectures": [ "DeepseekV2ForCausalLM" ], "attention_bias": false, "attention_dropout": 0, "auto_map": { "AutoConfig": "configuration_deepseek.DeepseekV2Config", "AutoModel": "modeling_deepseek.DeepseekV2Model", "AutoModelForCausalLM": "modeling_deepseek.DeepseekV2ForCausalLM" }, "aux_loss_alpha": 0.001, "bos_token_id": 100000, "eos_token_id": 100001, "first_k_dense_replace": 1, "hidden_act": "silu", "hidden_size": 5120, "initializer_range": 0.02, "intermediate_size": 12288, "kv_lora_rank": 512, "max_position_embeddings": 163840, "model_type": "deepseek_v2", "moe_intermediate_size": 1536, "moe_layer_freq": 1, "n_group": 8, "n_routed_experts": 160, "n_shared_experts": 2, "norm_topk_prob": false, "num_attention_heads": 128, "num_experts_per_tok": 6, "num_hidden_layers": 60, "num_key_value_heads": 128, "pretraining_tp": 1, "q_lora_rank": 1536, "qk_nope_head_dim": 128, "qk_rope_head_dim": 64, "rms_norm_eps": 1e-06, "rope_scaling": { "beta_fast": 32, "beta_slow": 1, "factor": 40, "mscale": 0.707, "mscale_all_dim": 0.707, "original_max_position_embeddings": 4096, "type": "yarn" }, "rope_theta": 10000, "routed_scaling_factor": 16, "scoring_func": "softmax", "seq_aux": true, "tie_word_embeddings": false, "topk_group": 3, "topk_method": "group_limited_greedy", "torch_dtype": "bfloat16", "transformers_version": "4.39.3", "use_cache": true, "v_head_dim": 128, "vocab_size": 102400 } }, { "id": "deepseek-ai/deepseek-coder-1.3b-base", "meta": { "sha": "c919139c3a9b4070729c8b2cca4847ab29ca8d94", "modelType": "llama" }, "config": { "architectures": [ "LlamaForCausalLM" ], "bos_token_id": 32013, "eos_token_id": 32014, "hidden_act": "silu", "hidden_size": 2048, "initializer_range": 0.02, "intermediate_size": 5504, "max_position_embeddings": 16384, "model_type": "llama", "num_attention_heads": 16, "num_hidden_layers": 24, "num_key_value_heads": 16, "pretraining_tp": 1, "rms_norm_eps": 1e-06, "rope_scaling": { "factor": 4, "type": "linear" }, "rope_theta": 100000, "tie_word_embeddings": false, "torch_dtype": "bfloat16", "transformers_version": "4.34.1", "use_cache": true, "vocab_size": 32256 } }, { "id": "deepseek-ai/DeepSeek-V3.1-Base", "meta": { "sha": "d3d4eafdc470de44bbf6f0a74f852eb522357be8", "safetensors": { "parameters": { "BF16": 3918786560, "F8_E4M3": 680571043840, "F32": 41555600 }, "total": 684531386000 }, "modelType": "deepseek_v3" }, "config": { "architectures": [ "DeepseekV3ForCausalLM" ], "attention_bias": false, "attention_dropout": 0.0, "auto_map": { "AutoConfig": "configuration_deepseek.DeepseekV3Config", "AutoModel": "modeling_deepseek.DeepseekV3Model", "AutoModelForCausalLM": "modeling_deepseek.DeepseekV3ForCausalLM" }, "bos_token_id": 0, "eos_token_id": 1, "ep_size": 1, "first_k_dense_replace": 3, "hidden_act": "silu", "hidden_size": 7168, "initializer_range": 0.02, "intermediate_size": 18432, "kv_lora_rank": 512, "max_position_embeddings": 163840, "model_type": "deepseek_v3", "moe_intermediate_size": 2048, "moe_layer_freq": 1, "n_group": 8, "n_routed_experts": 256, "n_shared_experts": 1, "norm_topk_prob": true, "num_attention_heads": 128, "num_experts_per_tok": 8, "num_hidden_layers": 61, "num_key_value_heads": 128, "num_nextn_predict_layers": 1, "q_lora_rank": 1536, "qk_nope_head_dim": 128, "qk_rope_head_dim": 64, "quantization_config": { "activation_scheme": "dynamic", "fmt": "e4m3", "quant_method": "fp8", "weight_block_size": [ 128, 128 ], "scale_fmt": "ue8m0" }, "rms_norm_eps": 1e-06, "rope_scaling": { "beta_fast": 32, "beta_slow": 1, "factor": 40, "mscale": 1.0, "mscale_all_dim": 1.0, "original_max_position_embeddings": 4096, "type": "yarn" }, "rope_theta": 10000, "routed_scaling_factor": 2.5, "scoring_func": "sigmoid", "tie_word_embeddings": false, "topk_group": 4, "topk_method": "noaux_tc", "torch_dtype": "bfloat16", "transformers_version": "4.44.2", "use_cache": true, "v_head_dim": 128, "vocab_size": 129280 }, "verifiedAt": "2026-09-22" }, { "id": "deepseek-ai/deepseek-llm-7b-chat", "meta": { "sha": "afbda8b347ec881666061fa67447046fc5164ec8", "modelType": "llama" }, "config": { "architectures": [ "LlamaForCausalLM" ], "bos_token_id": 100000, "eos_token_id": 100001, "hidden_act": "silu", "hidden_size": 4096, "initializer_range": 0.02, "intermediate_size": 11008, "max_position_embeddings": 4096, "model_type": "llama", "num_attention_heads": 32, "num_hidden_layers": 30, "num_key_value_heads": 32, "pretraining_tp": 1, "rms_norm_eps": 1e-06, "rope_scaling": null, "rope_theta": 10000, "tie_word_embeddings": false, "torch_dtype": "bfloat16", "transformers_version": "4.33.1", "use_cache": true, "vocab_size": 102400 } }, { "id": "deepseek-ai/deepseek-llm-7b-base", "meta": { "sha": "7683fea62db869066ddaff6a41d032262c490d4f", "modelType": "llama" }, "config": { "architectures": [ "LlamaForCausalLM" ], "bos_token_id": 1, "eos_token_id": 2, "hidden_act": "silu", "hidden_size": 4096, "initializer_range": 0.02, "intermediate_size": 11008, "max_position_embeddings": 4096, "model_type": "llama", "num_attention_heads": 32, "num_hidden_layers": 30, "num_key_value_heads": 32, "pretraining_tp": 1, "rms_norm_eps": 1e-06, "rope_scaling": null, "rope_theta": 10000, "tie_word_embeddings": false, "torch_dtype": "bfloat16", "transformers_version": "4.33.1", "use_cache": true, "vocab_size": 102400 } }, { "id": "deepseek-ai/DeepSeek-V3.1-Terminus", "meta": { "sha": "19510d6dc61f79dbd925bd51ee8a9081c509a4b6", "safetensors": { "parameters": { "BF16": 3917742080, "F8_E4M3": 680571043840, "F32": 42600080 }, "total": 684531386000 }, "modelType": "deepseek_v3" }, "config": { "architectures": [ "DeepseekV3ForCausalLM" ], "attention_bias": false, "attention_dropout": 0.0, "attn_module_list_cfg": [], "auto_map": { "AutoConfig": "configuration_deepseek.DeepseekV3Config", "AutoModel": "modeling_deepseek.DeepseekV3Model", "AutoModelForCausalLM": "modeling_deepseek.DeepseekV3ForCausalLM" }, "bos_token_id": 0, "eos_token_id": 1, "ep_size": 1, "first_k_dense_replace": 3, "hidden_act": "silu", "hidden_size": 7168, "initializer_range": 0.02, "intermediate_size": 18432, "kv_lora_rank": 512, "max_position_embeddings": 163840, "model_type": "deepseek_v3", "moe_intermediate_size": 2048, "moe_layer_freq": 1, "n_group": 8, "n_routed_experts": 256, "n_shared_experts": 1, "norm_topk_prob": true, "num_attention_heads": 128, "num_experts_per_tok": 8, "num_hidden_layers": 61, "num_key_value_heads": 128, "num_nextn_predict_layers": 1, "q_lora_rank": 1536, "qk_nope_head_dim": 128, "qk_rope_head_dim": 64, "quantization_config": { "activation_scheme": "dynamic", "fmt": "e4m3", "quant_method": "fp8", "weight_block_size": [ 128, 128 ] }, "rms_norm_eps": 1e-06, "rope_scaling": { "beta_fast": 32, "beta_slow": 1, "factor": 40, "mscale": 1.0, "mscale_all_dim": 1.0, "original_max_position_embeddings": 4096, "type": "yarn" }, "rope_theta": 10000, "routed_scaling_factor": 2.5, "scoring_func": "sigmoid", "tie_word_embeddings": false, "topk_group": 4, "topk_method": "noaux_tc", "torch_dtype": "bfloat16", "transformers_version": "4.44.2", "use_cache": true, "v_head_dim": 128, "vocab_size": 129280 }, "verifiedAt": "2026-09-22" }, { "id": "deepseek-ai/deepseek-moe-16b-chat", "meta": { "sha": "eefd8ac7e8dc90e095129fe1a537d5e236b2e57c", "safetensors": { "parameters": { "BF16": 16375728128 }, "total": 16375728128 }, "modelType": "deepseek" }, "config": { "architectures": [ "DeepseekForCausalLM" ], "attention_bias": false, "attention_dropout": 0, "auto_map": { "AutoConfig": "configuration_deepseek.DeepseekConfig", "AutoModel": "modeling_deepseek.DeepseekModel", "AutoModelForCausalLM": "modeling_deepseek.DeepseekForCausalLM" }, "aux_loss_alpha": 0.001, "bos_token_id": 100000, "eos_token_id": 100001, "first_k_dense_replace": 1, "hidden_act": "silu", "hidden_size": 2048, "initializer_range": 0.02, "intermediate_size": 10944, "max_position_embeddings": 4096, "model_type": "deepseek", "moe_intermediate_size": 1408, "moe_layer_freq": 1, "n_routed_experts": 64, "n_shared_experts": 2, "norm_topk_prob": false, "num_attention_heads": 16, "num_experts_per_tok": 6, "num_hidden_layers": 28, "num_key_value_heads": 16, "pretraining_tp": 1, "rms_norm_eps": 1e-06, "rope_scaling": null, "rope_theta": 10000, "scoring_func": "softmax", "seq_aux": true, "tie_word_embeddings": false, "torch_dtype": "bfloat16", "transformers_version": "4.36.2", "use_cache": true, "vocab_size": 102400 } }, { "id": "mistralai/Mistral-7B-Instruct-v0.2", "meta": { "sha": "63a8b081895390a26e140280378bc85ec8bce07a", "safetensors": { "parameters": { "BF16": 7241732096 }, "total": 7241732096 }, "modelType": "mistral" }, "config": { "architectures": [ "MistralForCausalLM" ], "attention_dropout": 0, "bos_token_id": 1, "eos_token_id": 2, "hidden_act": "silu", "hidden_size": 4096, "initializer_range": 0.02, "intermediate_size": 14336, "max_position_embeddings": 32768, "model_type": "mistral", "num_attention_heads": 32, "num_hidden_layers": 32, "num_key_value_heads": 8, "rms_norm_eps": 1e-05, "rope_theta": 1000000, "sliding_window": null, "tie_word_embeddings": false, "torch_dtype": "bfloat16", "transformers_version": "4.36.0", "use_cache": true, "vocab_size": 32000 } }, { "id": "mistralai/Mistral-7B-v0.1", "meta": { "sha": "27d67f1b5f57dc0953326b2601d68371d40ea8da", "safetensors": { "parameters": { "BF16": 7241732096 }, "total": 7241732096 }, "modelType": "mistral" }, "config": { "architectures": [ "MistralForCausalLM" ], "bos_token_id": 1, "eos_token_id": 2, "hidden_act": "silu", "hidden_size": 4096, "initializer_range": 0.02, "intermediate_size": 14336, "max_position_embeddings": 32768, "model_type": "mistral", "num_attention_heads": 32, "num_hidden_layers": 32, "num_key_value_heads": 8, "rms_norm_eps": 1e-05, "rope_theta": 10000, "sliding_window": 4096, "tie_word_embeddings": false, "torch_dtype": "bfloat16", "transformers_version": "4.34.0.dev0", "use_cache": true, "vocab_size": 32000 } }, { "id": "mistralai/Mistral-7B-Instruct-v0.1", "meta": { "sha": "ec5deb64f2c6e6fa90c1abf74a91d5c93a9669ca", "safetensors": { "parameters": { "BF16": 7241732096 }, "total": 7241732096 }, "modelType": "mistral" }, "config": { "architectures": [ "MistralForCausalLM" ], "bos_token_id": 1, "eos_token_id": 2, "hidden_act": "silu", "hidden_size": 4096, "initializer_range": 0.02, "intermediate_size": 14336, "max_position_embeddings": 32768, "model_type": "mistral", "num_attention_heads": 32, "num_hidden_layers": 32, "num_key_value_heads": 8, "rms_norm_eps": 1e-05, "rope_theta": 10000, "sliding_window": 4096, "tie_word_embeddings": false, "torch_dtype": "bfloat16", "transformers_version": "4.34.0.dev0", "use_cache": true, "vocab_size": 32000 } }, { "id": "google/gemma-3-1b-it", "meta": { "sha": "dcc83ea841ab6100d6b47a070329e1ba4cf78752", "safetensors": { "parameters": { "BF16": 999885952 }, "total": 999885952 }, "modelType": "gemma3" }, "config": { "num_hidden_layers": 26, "num_attention_heads": 4, "num_key_value_heads": 1, "hidden_size": 1152, "head_dim": 256, "vocab_size": 262144, "intermediate_size": 6912, "model_type": "gemma3", "max_position_embeddings": 32768, "tie_word_embeddings": true, "sliding_window": 512, "window_layers": 22 }, "previousConfigError": 401, "configSource": "https://github.com/google/gemma_pytorch/blob/014acb7ac4563a5f77c76d7ff98f31b568c16508/gemma/config.py", "configKind": "Normalized from official implementation, not the restricted Hugging Face config.json", "verifiedAt": "2026-09-22" }, { "id": "google/gemma-2-9b-it", "meta": { "sha": "11c9b309abf73637e4b6f9a3fa1e92e615547819", "safetensors": { "parameters": { "BF16": 9241705984 }, "total": 9241705984 }, "modelType": "gemma2" }, "config": { "num_hidden_layers": 42, "num_attention_heads": 16, "num_key_value_heads": 8, "hidden_size": 3584, "head_dim": 256, "vocab_size": 256000, "intermediate_size": 14336, "model_type": "gemma2", "max_position_embeddings": 8192, "tie_word_embeddings": true, "sliding_window": 4096, "window_layers": 21 }, "previousConfigError": 401, "configSource": "https://github.com/google/gemma_pytorch/blob/014acb7ac4563a5f77c76d7ff98f31b568c16508/gemma/config.py", "configKind": "Normalized from official implementation, not the restricted Hugging Face config.json", "verifiedAt": "2026-09-22" }, { "id": "google/gemma-2-2b-it", "meta": { "sha": "299a8560bedf22ed1c72a8a11e7dce4a7f9f51f8", "safetensors": { "parameters": { "BF16": 2614341888 }, "total": 2614341888 }, "modelType": "gemma2" }, "config": { "num_hidden_layers": 26, "num_attention_heads": 8, "num_key_value_heads": 4, "hidden_size": 2304, "head_dim": 256, "vocab_size": 256000, "intermediate_size": 9216, "model_type": "gemma2", "max_position_embeddings": 8192, "tie_word_embeddings": true, "sliding_window": 4096, "window_layers": 13 }, "previousConfigError": 401, "configSource": "https://github.com/google/gemma_pytorch/blob/014acb7ac4563a5f77c76d7ff98f31b568c16508/gemma/config.py", "configKind": "Normalized from official implementation, not the restricted Hugging Face config.json", "verifiedAt": "2026-09-22" }, { "id": "google/gemma-3-270m", "meta": { "sha": "9b0cfec892e2bc2afd938c98eabe4e4a7b1e0ca1", "safetensors": { "parameters": { "BF16": 268098176 }, "total": 268098176 }, "modelType": "gemma3_text" }, "config": { "model_type": "gemma3_text", "num_hidden_layers": 18, "num_attention_heads": 4, "num_key_value_heads": 1, "hidden_size": 640, "head_dim": 256, "intermediate_size": 2048, "vocab_size": 262144, "tie_word_embeddings": true, "sliding_window": 512, "max_position_embeddings": 32768, "layer_types": [ "sliding_attention", "sliding_attention", "sliding_attention", "sliding_attention", "sliding_attention", "full_attention", "sliding_attention", "sliding_attention", "sliding_attention", "sliding_attention", "sliding_attention", "full_attention", "sliding_attention", "sliding_attention", "sliding_attention", "sliding_attention", "sliding_attention", "full_attention" ] }, "configSource": "https://github.com/google-deepmind/gemma/blob/0513283af5afffa27390b6ede2facc35d0f16e08/gemma/gm/nn/_gemma.py", "configKind": "Normalized from official Gemma3_270M implementation; context from Google model card", "contextSource": "https://ai.google.dev/gemma/docs/core/model_card_3", "verifiedAt": "2026-09-22" }, { "id": "google/gemma-2-2b", "meta": { "sha": "c5ebcd40d208330abc697524c919956e692655cf", "safetensors": { "parameters": { "F32": 2614341888 }, "total": 2614341888 }, "modelType": "gemma2" }, "config": { "num_hidden_layers": 26, "num_attention_heads": 8, "num_key_value_heads": 4, "hidden_size": 2304, "head_dim": 256, "vocab_size": 256000, "intermediate_size": 9216, "model_type": "gemma2", "max_position_embeddings": 8192, "tie_word_embeddings": true, "sliding_window": 4096, "window_layers": 13 }, "previousConfigError": 401, "configSource": "https://github.com/google/gemma_pytorch/blob/014acb7ac4563a5f77c76d7ff98f31b568c16508/gemma/config.py", "configKind": "Normalized from official implementation, not the restricted Hugging Face config.json", "verifiedAt": "2026-09-22" }, { "id": "google/gemma-2b", "meta": { "sha": "9cf48e52b224239de00d483ec8eb84fb8d0f3a3a", "safetensors": { "parameters": { "BF16": 2506172416 }, "total": 2506172416 }, "modelType": "gemma" }, "config": { "num_hidden_layers": 18, "num_attention_heads": 8, "num_key_value_heads": 1, "hidden_size": 2048, "head_dim": 256, "vocab_size": 256000, "intermediate_size": 16384, "model_type": "gemma", "max_position_embeddings": 8192, "tie_word_embeddings": true, "sliding_window": null, "window_layers": 0 }, "previousConfigError": 401, "configSource": "https://github.com/google/gemma_pytorch/blob/014acb7ac4563a5f77c76d7ff98f31b568c16508/gemma/config.py", "configKind": "Normalized from official implementation, not the restricted Hugging Face config.json", "verifiedAt": "2026-09-22" }, { "id": "google/gemma-1.1-2b-it", "meta": { "sha": "d750f5eceb83e978c09e2b3597c2a8784e381022", "safetensors": { "parameters": { "BF16": 2506172416 }, "total": 2506172416 }, "modelType": "gemma" }, "config": { "num_hidden_layers": 18, "num_attention_heads": 8, "num_key_value_heads": 1, "hidden_size": 2048, "head_dim": 256, "vocab_size": 256000, "intermediate_size": 16384, "model_type": "gemma", "max_position_embeddings": 8192, "tie_word_embeddings": true, "sliding_window": null, "window_layers": 0 }, "previousConfigError": 401, "configSource": "https://github.com/google/gemma_pytorch/blob/014acb7ac4563a5f77c76d7ff98f31b568c16508/gemma/config.py", "configKind": "Normalized from official implementation, not the restricted Hugging Face config.json", "verifiedAt": "2026-09-22" }, { "id": "google/gemma-3-270m-it", "meta": { "sha": "ac82b4e820549b854eebf28ce6dedaf9fdfa17b3", "safetensors": { "parameters": { "BF16": 268098176 }, "total": 268098176 }, "modelType": "gemma3_text" }, "config": { "model_type": "gemma3_text", "num_hidden_layers": 18, "num_attention_heads": 4, "num_key_value_heads": 1, "hidden_size": 640, "head_dim": 256, "intermediate_size": 2048, "vocab_size": 262144, "tie_word_embeddings": true, "sliding_window": 512, "max_position_embeddings": 32768, "layer_types": [ "sliding_attention", "sliding_attention", "sliding_attention", "sliding_attention", "sliding_attention", "full_attention", "sliding_attention", "sliding_attention", "sliding_attention", "sliding_attention", "sliding_attention", "full_attention", "sliding_attention", "sliding_attention", "sliding_attention", "sliding_attention", "sliding_attention", "full_attention" ] }, "configSource": "https://github.com/google-deepmind/gemma/blob/0513283af5afffa27390b6ede2facc35d0f16e08/gemma/gm/nn/_gemma.py", "configKind": "Normalized from official Gemma3_270M implementation; context from Google model card", "contextSource": "https://ai.google.dev/gemma/docs/core/model_card_3", "verifiedAt": "2026-09-22" }, { "id": "google/gemma-3-1b-pt", "meta": { "sha": "fcf18a2a879aab110ca39f8bffbccd5d49d8eb29", "safetensors": { "parameters": { "BF16": 999885952 }, "total": 999885952 }, "modelType": "gemma3" }, "config": { "num_hidden_layers": 26, "num_attention_heads": 4, "num_key_value_heads": 1, "hidden_size": 1152, "head_dim": 256, "vocab_size": 262144, "intermediate_size": 6912, "model_type": "gemma3", "max_position_embeddings": 32768, "tie_word_embeddings": true, "sliding_window": 512, "window_layers": 22 }, "previousConfigError": 401, "configSource": "https://github.com/google/gemma_pytorch/blob/014acb7ac4563a5f77c76d7ff98f31b568c16508/gemma/config.py", "configKind": "Normalized from official implementation, not the restricted Hugging Face config.json", "verifiedAt": "2026-09-22" }, { "id": "google/gemma-2-9b", "meta": { "sha": "33c193028431c2fde6c6e51f29e6f17b60cbfac6", "safetensors": { "parameters": { "F32": 9241705984 }, "total": 9241705984 }, "modelType": "gemma2" }, "config": { "num_hidden_layers": 42, "num_attention_heads": 16, "num_key_value_heads": 8, "hidden_size": 3584, "head_dim": 256, "vocab_size": 256000, "intermediate_size": 14336, "model_type": "gemma2", "max_position_embeddings": 8192, "tie_word_embeddings": true, "sliding_window": 4096, "window_layers": 21 }, "previousConfigError": 401, "configSource": "https://github.com/google/gemma_pytorch/blob/014acb7ac4563a5f77c76d7ff98f31b568c16508/gemma/config.py", "configKind": "Normalized from official implementation, not the restricted Hugging Face config.json", "verifiedAt": "2026-09-22" }, { "id": "google/gemma-7b-it", "meta": { "sha": "9c5798d27f588501ce1e108079d2a19e4c3a2353", "safetensors": { "parameters": { "BF16": 8537680896 }, "total": 8537680896 }, "modelType": "gemma" }, "config": { "num_hidden_layers": 28, "num_attention_heads": 16, "num_key_value_heads": 16, "hidden_size": 3072, "head_dim": 256, "vocab_size": 256000, "intermediate_size": 24576, "model_type": "gemma", "max_position_embeddings": 8192, "tie_word_embeddings": true, "sliding_window": null, "window_layers": 0 }, "previousConfigError": 401, "configSource": "https://github.com/google/gemma_pytorch/blob/014acb7ac4563a5f77c76d7ff98f31b568c16508/gemma/config.py", "configKind": "Normalized from official implementation, not the restricted Hugging Face config.json", "verifiedAt": "2026-09-22" }, { "id": "google/gemma-2-27b-it", "meta": { "sha": "aaf20e6b9f4c0fcf043f6fb2a2068419086d77b0", "safetensors": { "parameters": { "BF16": 27227128320 }, "total": 27227128320 }, "modelType": "gemma2" }, "config": { "num_hidden_layers": 46, "num_attention_heads": 32, "num_key_value_heads": 16, "hidden_size": 4608, "head_dim": 128, "vocab_size": 256000, "intermediate_size": 36864, "model_type": "gemma2", "max_position_embeddings": 8192, "tie_word_embeddings": true, "sliding_window": 4096, "window_layers": 23 }, "previousConfigError": 401, "configSource": "https://github.com/google/gemma_pytorch/blob/014acb7ac4563a5f77c76d7ff98f31b568c16508/gemma/config.py", "configKind": "Normalized from official implementation, not the restricted Hugging Face config.json", "verifiedAt": "2026-09-22" }, { "id": "google/medgemma-27b-text-it", "meta": { "sha": "5b667cf2ddcf064085bc90952edb35a0edbfb79c", "safetensors": { "parameters": { "BF16": 27009002240 }, "total": 27009002240 }, "modelType": "gemma3_text" }, "config": { "error": 401 } }, { "id": "google/gemma-2b-it", "meta": { "sha": "96988410cbdaeb8d5093d1ebdc5a8fb563e02bad", "safetensors": { "parameters": { "BF16": 2506172416 }, "total": 2506172416 }, "modelType": "gemma" }, "config": { "num_hidden_layers": 18, "num_attention_heads": 8, "num_key_value_heads": 1, "hidden_size": 2048, "head_dim": 256, "vocab_size": 256000, "intermediate_size": 16384, "model_type": "gemma", "max_position_embeddings": 8192, "tie_word_embeddings": true, "sliding_window": null, "window_layers": 0 }, "previousConfigError": 401, "configSource": "https://github.com/google/gemma_pytorch/blob/014acb7ac4563a5f77c76d7ff98f31b568c16508/gemma/config.py", "configKind": "Normalized from official implementation, not the restricted Hugging Face config.json", "verifiedAt": "2026-09-22" }, { "id": "google/gemma-7b", "meta": { "sha": "ff6768d9368919a1f025a54f9f5aa0ee591730bb", "safetensors": { "parameters": { "BF16": 8537680896 }, "total": 8537680896 }, "modelType": "gemma" }, "config": { "num_hidden_layers": 28, "num_attention_heads": 16, "num_key_value_heads": 16, "hidden_size": 3072, "head_dim": 256, "vocab_size": 256000, "intermediate_size": 24576, "model_type": "gemma", "max_position_embeddings": 8192, "tie_word_embeddings": true, "sliding_window": null, "window_layers": 0 }, "previousConfigError": 401, "configSource": "https://github.com/google/gemma_pytorch/blob/014acb7ac4563a5f77c76d7ff98f31b568c16508/gemma/config.py", "configKind": "Normalized from official implementation, not the restricted Hugging Face config.json", "verifiedAt": "2026-09-22" }, { "id": "google/functiongemma-270m-it", "meta": { "sha": "39eccb091651513a5dfb56892d3714c1b5b8276c", "safetensors": { "parameters": { "BF16": 268098176 }, "total": 268098176 }, "modelType": "gemma3_text" }, "config": { "error": 401 } }, { "id": "google/gemma-1.1-7b-it", "meta": { "sha": "065a528791af6f57f013e8e42b7276992b45ef71", "safetensors": { "parameters": { "BF16": 8537680896 }, "total": 8537680896 }, "modelType": "gemma" }, "config": { "num_hidden_layers": 28, "num_attention_heads": 16, "num_key_value_heads": 16, "hidden_size": 3072, "head_dim": 256, "vocab_size": 256000, "intermediate_size": 24576, "model_type": "gemma", "max_position_embeddings": 8192, "tie_word_embeddings": true, "sliding_window": null, "window_layers": 0 }, "previousConfigError": 401, "configSource": "https://github.com/google/gemma_pytorch/blob/014acb7ac4563a5f77c76d7ff98f31b568c16508/gemma/config.py", "configKind": "Normalized from official implementation, not the restricted Hugging Face config.json", "verifiedAt": "2026-09-22" }, { "id": "google/recurrentgemma-2b-it", "meta": { "sha": "2766eb5d4264c6c0357803990791f9ab9cd50f8e", "safetensors": { "parameters": { "BF16": 2682862080 }, "total": 2682862080 }, "modelType": "recurrent_gemma" }, "config": { "error": 401 } }, { "id": "google/vaultgemma-1b", "meta": { "sha": "f9624dafc1760cb2f6039e86e12055d6559d7abb", "safetensors": { "parameters": { "BF16": 1038741120 }, "total": 1038741120 }, "modelType": "vaultgemma" }, "config": { "error": 401 } }, { "id": "google/gemma-2-2b-jpn-it", "meta": { "sha": "6b046bbc091084a1ec89fe03e58871fde10868eb", "safetensors": { "parameters": { "BF16": 2614341888 }, "total": 2614341888 }, "modelType": "gemma2" }, "config": { "error": 401 } }, { "id": "google/codegemma-2b", "meta": { "sha": "e7583edabd3dd48a9c705974a9456852f1205ab1", "safetensors": { "parameters": { "BF16": 2506172416 }, "total": 2506172416 }, "modelType": "gemma" }, "config": { "error": 401 } }, { "id": "google/gemma-2-27b", "meta": { "sha": "938270f5272feb02779b55c2bb2fffdd0f53ff0c", "safetensors": { "parameters": { "F32": 27227128320 }, "total": 27227128320 }, "modelType": "gemma2" }, "config": { "num_hidden_layers": 46, "num_attention_heads": 32, "num_key_value_heads": 16, "hidden_size": 4608, "head_dim": 128, "vocab_size": 256000, "intermediate_size": 36864, "model_type": "gemma2", "max_position_embeddings": 8192, "tie_word_embeddings": true, "sliding_window": 4096, "window_layers": 23 }, "previousConfigError": 401, "configSource": "https://github.com/google/gemma_pytorch/blob/014acb7ac4563a5f77c76d7ff98f31b568c16508/gemma/config.py", "configKind": "Normalized from official implementation, not the restricted Hugging Face config.json", "verifiedAt": "2026-09-22" }, { "id": "google/codegemma-7b-it", "meta": { "sha": "078cdc51070553d1636d645c9a238f3b0914459a", "safetensors": { "parameters": { "BF16": 8537680896 }, "total": 8537680896 }, "modelType": "gemma" }, "config": { "error": 401 } }, { "id": "microsoft/phi-4", "meta": { "sha": "2db69c1c3e91a05d2c64a3185acfbaf36f744e25", "safetensors": { "parameters": { "BF16": 14659507200 }, "total": 14659507200 }, "modelType": "phi3" }, "config": { "_name_or_path": "microsoft/phi-4", "architectures": [ "Phi3ForCausalLM" ], "attention_bias": false, "attention_dropout": 0, "bos_token_id": 100257, "embd_pdrop": 0, "eos_token_id": 100265, "hidden_act": "silu", "hidden_size": 5120, "initializer_range": 0.02, "intermediate_size": 17920, "max_position_embeddings": 16384, "model_type": "phi3", "num_attention_heads": 40, "num_hidden_layers": 40, "num_key_value_heads": 10, "original_max_position_embeddings": 16384, "pad_token_id": 100349, "resid_pdrop": 0, "rms_norm_eps": 1e-05, "rope_scaling": null, "rope_theta": 250000, "sliding_window": null, "tie_word_embeddings": false, "torch_dtype": "bfloat16", "transformers_version": "4.47.0", "use_cache": true, "vocab_size": 100352 } }, { "id": "microsoft/phi-2", "meta": { "sha": "810d367871c1d460086d9f82db8696f2e0a0fcd0", "safetensors": { "parameters": { "F16": 2779683840 }, "total": 2779683840 }, "modelType": "phi" }, "config": { "_name_or_path": "microsoft/phi-2", "architectures": [ "PhiForCausalLM" ], "attention_dropout": 0, "bos_token_id": 50256, "embd_pdrop": 0, "eos_token_id": 50256, "hidden_act": "gelu_new", "hidden_size": 2560, "initializer_range": 0.02, "intermediate_size": 10240, "layer_norm_eps": 1e-05, "max_position_embeddings": 2048, "model_type": "phi", "num_attention_heads": 32, "num_hidden_layers": 32, "num_key_value_heads": 32, "partial_rotary_factor": 0.4, "qk_layernorm": false, "resid_pdrop": 0.1, "rope_scaling": null, "rope_theta": 10000, "tie_word_embeddings": false, "torch_dtype": "float16", "transformers_version": "4.37.0", "use_cache": true, "vocab_size": 51200 } }, { "id": "microsoft/Phi-4-mini-instruct", "meta": { "sha": "cfbefacb99257ffa30c83adab238a50856ac3083", "safetensors": { "parameters": { "BF16": 3836021760 }, "total": 3836021760 }, "modelType": "phi3" }, "config": { "_name_or_path": "Phi-4-mini-instruct", "architectures": [ "Phi3ForCausalLM" ], "attention_bias": false, "attention_dropout": 0, "auto_map": { "AutoConfig": "configuration_phi3.Phi3Config", "AutoModelForCausalLM": "modeling_phi3.Phi3ForCausalLM", "AutoTokenizer": "Xenova/gpt-4o" }, "bos_token_id": 199999, "embd_pdrop": 0, "eos_token_id": 199999, "full_attn_mod": 1, "hidden_act": "silu", "hidden_size": 3072, "initializer_range": 0.02, "intermediate_size": 8192, "interpolate_factor": 1, "lm_head_bias": false, "max_position_embeddings": 131072, "mlp_bias": false, "model_type": "phi3", "num_attention_heads": 24, "num_hidden_layers": 32, "num_key_value_heads": 8, "original_max_position_embeddings": 4096, "pad_token_id": 199999, "partial_rotary_factor": 0.75, "resid_pdrop": 0, "rms_norm_eps": 1e-05, "rope_scaling": { "long_factor": [ 1, 1.118320672, 1.250641126, 1.398617824, 1.564103225, 1.74916897, 1.956131817, 2.187582649, 2.446418898, 2.735880826, 3.059592084, 3.421605075, 3.826451687, 4.279200023, 4.785517845, 5.351743533, 5.984965424, 6.693110555, 7.485043894, 8.370679318, 9.36110372, 10.4687158, 11.70738129, 13.09260651, 14.64173252, 16.37415215, 18.31155283, 20.47818807, 22.90118105, 25.61086418, 28.64115884, 32.03, 32.1, 32.13, 32.23, 32.6, 32.61, 32.64, 32.66, 32.7, 32.71, 32.93, 32.97, 33.28, 33.49, 33.5, 44.16, 47.77 ], "short_factor": [ 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1 ], "type": "longrope" }, "rope_theta": 10000, "sliding_window": 262144, "tie_word_embeddings": true, "torch_dtype": "bfloat16", "transformers_version": "4.45.0", "use_cache": true, "vocab_size": 200064 } }, { "id": "microsoft/Phi-3-mini-4k-instruct", "meta": { "sha": "f39ac1d28e925b323eae81227eaba4464caced4e", "safetensors": { "parameters": { "BF16": 3821079552 }, "total": 3821079552 }, "modelType": "phi3" }, "config": { "_name_or_path": "Phi-3-mini-4k-instruct", "architectures": [ "Phi3ForCausalLM" ], "attention_dropout": 0, "auto_map": { "AutoConfig": "configuration_phi3.Phi3Config", "AutoModelForCausalLM": "modeling_phi3.Phi3ForCausalLM" }, "bos_token_id": 1, "embd_pdrop": 0, "eos_token_id": 32000, "hidden_act": "silu", "hidden_size": 3072, "initializer_range": 0.02, "intermediate_size": 8192, "max_position_embeddings": 4096, "model_type": "phi3", "num_attention_heads": 32, "num_hidden_layers": 32, "num_key_value_heads": 32, "original_max_position_embeddings": 4096, "pad_token_id": 32000, "resid_pdrop": 0, "rms_norm_eps": 1e-05, "rope_scaling": null, "rope_theta": 10000, "sliding_window": 2047, "tie_word_embeddings": false, "torch_dtype": "bfloat16", "transformers_version": "4.40.2", "use_cache": true, "attention_bias": false, "vocab_size": 32064 } }, { "id": "microsoft/Phi-3.5-mini-instruct", "meta": { "sha": "2fe192450127e6a83f7441aef6e3ca586c338b77", "safetensors": { "parameters": { "BF16": 3821079552 }, "total": 3821079552 }, "modelType": "phi3" }, "config": { "_name_or_path": "Phi-3.5-mini-instruct", "architectures": [ "Phi3ForCausalLM" ], "attention_dropout": 0, "auto_map": { "AutoConfig": "configuration_phi3.Phi3Config", "AutoModelForCausalLM": "modeling_phi3.Phi3ForCausalLM" }, "bos_token_id": 1, "embd_pdrop": 0, "eos_token_id": 32000, "hidden_act": "silu", "hidden_size": 3072, "initializer_range": 0.02, "intermediate_size": 8192, "max_position_embeddings": 131072, "model_type": "phi3", "num_attention_heads": 32, "num_hidden_layers": 32, "num_key_value_heads": 32, "original_max_position_embeddings": 4096, "pad_token_id": 32000, "resid_pdrop": 0, "rms_norm_eps": 1e-05, "rope_scaling": { "long_factor": [ 1.0800000429153442, 1.1100000143051147, 1.1399999856948853, 1.340000033378601, 1.5899999141693115, 1.600000023841858, 1.6200000047683716, 2.620000123977661, 3.2300000190734863, 3.2300000190734863, 4.789999961853027, 7.400000095367432, 7.700000286102295, 9.09000015258789, 12.199999809265137, 17.670000076293945, 24.46000099182129, 28.57000160217285, 30.420001983642578, 30.840002059936523, 32.590003967285156, 32.93000411987305, 42.320003509521484, 44.96000289916992, 50.340003967285156, 50.45000457763672, 57.55000305175781, 57.93000411987305, 58.21000289916992, 60.1400032043457, 62.61000442504883, 62.62000274658203, 62.71000289916992, 63.1400032043457, 63.1400032043457, 63.77000427246094, 63.93000411987305, 63.96000289916992, 63.970001220703125, 64.02999877929688, 64.06999969482422, 64.08000183105469, 64.12000274658203, 64.41000366210938, 64.4800033569336, 64.51000213623047, 64.52999877929688, 64.83999633789062 ], "short_factor": [ 1, 1.0199999809265137, 1.0299999713897705, 1.0299999713897705, 1.0499999523162842, 1.0499999523162842, 1.0499999523162842, 1.0499999523162842, 1.0499999523162842, 1.0699999332427979, 1.0999999046325684, 1.1099998950958252, 1.1599998474121094, 1.1599998474121094, 1.1699998378753662, 1.2899998426437378, 1.339999794960022, 1.679999828338623, 1.7899998426437378, 1.8199998140335083, 1.8499997854232788, 1.8799997568130493, 1.9099997282028198, 1.9399996995925903, 1.9899996519088745, 2.0199997425079346, 2.0199997425079346, 2.0199997425079346, 2.0199997425079346, 2.0199997425079346, 2.0199997425079346, 2.0299997329711914, 2.0299997329711914, 2.0299997329711914, 2.0299997329711914, 2.0299997329711914, 2.0299997329711914, 2.0299997329711914, 2.0299997329711914, 2.0299997329711914, 2.0799996852874756, 2.0899996757507324, 2.189999580383301, 2.2199995517730713, 2.5899994373321533, 2.729999542236328, 2.749999523162842, 2.8399994373321533 ], "type": "longrope" }, "rope_theta": 10000, "sliding_window": 262144, "tie_word_embeddings": false, "torch_dtype": "bfloat16", "transformers_version": "4.43.3", "use_cache": true, "attention_bias": false, "vocab_size": 32064 } }, { "id": "microsoft/Phi-4-multimodal-instruct", "meta": { "sha": "93f923e1a7727d1c4f446756212d9d3e8fcc5d81", "safetensors": { "parameters": { "BF16": 5574460384 }, "total": 5574460384 }, "modelType": "phi4mm" }, "config": { "_name_or_path": "Phi-4-multimodal-instruct", "architectures": [ "Phi4MMForCausalLM" ], "attention_bias": false, "attention_dropout": 0, "audio_processor": { "config": { "activation": "swish", "activation_checkpointing": { "interval": 1, "module": "transformer", "offload": false }, "attention_dim": 1024, "attention_heads": 16, "batch_norm": false, "bias_in_glu": true, "causal": true, "chunk_size": -1, "cnn_layer_norm": true, "conv_activation": "swish", "conv_glu_type": "swish", "depthwise_multiplier": 1, "depthwise_seperable_out_channel": 1024, "dropout_rate": 0, "encoder_embedding_config": { "input_size": 80 }, "ext_pw_kernel_size": 1, "ext_pw_out_channel": 1024, "input_layer": "nemo_conv", "input_size": 80, "kernel_size": 3, "left_chunk": 18, "linear_units": 1536, "nemo_conv_settings": { "conv_channels": 1024 }, "num_blocks": 24, "relative_attention_bias_args": { "t5_bias_max_distance": 500, "type": "t5" }, "time_reduction": 8 }, "name": "cascades" }, "auto_map": { "AutoConfig": "configuration_phi4mm.Phi4MMConfig", "AutoModelForCausalLM": "modeling_phi4mm.Phi4MMForCausalLM", "AutoTokenizer": "Xenova/gpt-4o" }, "bos_token_id": 199999, "embd_layer": { "audio_embd_layer": { "compression_rate": 8, "downsample_rate": 1, "embedding_cls": "audio", "enable_gradient_checkpointing": true, "projection_cls": "mlp", "use_conv_downsample": false, "use_qformer": false }, "embedding_cls": "image_audio", "image_embd_layer": { "crop_size": 448, "embedding_cls": "tune_image", "enable_gradient_checkpointing": true, "hd_transform_order": "sub_glb", "image_token_compression_cls": "avg_pool_2d", "projection_cls": "mlp", "use_hd_transform": true, "with_learnable_separator": true } }, "embd_pdrop": 0, "eos_token_id": 199999, "full_attn_mod": 1, "hidden_act": "silu", "hidden_size": 3072, "initializer_range": 0.02, "intermediate_size": 8192, "interpolate_factor": 1, "lm_head_bias": false, "vision_lora": { "dp": 0, "layer": "layers.*((self_attn\\.(qkv_proj|o_proj))|(mlp\\.(gate_up|down)_proj))", "lora_alpha": 512, "r": 256 }, "speech_lora": { "dp": 0.01, "layer": "((layers.*self_attn\\.(qkv|o)_proj)|(layers.*mlp\\.(gate_up|down)_proj))", "lora_alpha": 640, "r": 320 }, "max_position_embeddings": 131072, "mlp_bias": false, "model_type": "phi4mm", "num_attention_heads": 24, "num_hidden_layers": 32, "num_key_value_heads": 8, "original_max_position_embeddings": 4096, "pad_token_id": 199999, "partial_rotary_factor": 0.75, "resid_pdrop": 0, "rms_norm_eps": 1e-05, "rope_scaling": { "long_factor": [ 1, 1.118320672, 1.250641126, 1.398617824, 1.564103225, 1.74916897, 1.956131817, 2.187582649, 2.446418898, 2.735880826, 3.059592084, 3.421605075, 3.826451687, 4.279200023, 4.785517845, 5.351743533, 5.984965424, 6.693110555, 7.485043894, 8.370679318, 9.36110372, 10.4687158, 11.70738129, 13.09260651, 14.64173252, 16.37415215, 18.31155283, 20.47818807, 22.90118105, 25.61086418, 28.64115884, 32.03, 32.1, 32.13, 32.23, 32.6, 32.61, 32.64, 32.66, 32.7, 32.71, 32.93, 32.97, 33.28, 33.49, 33.5, 44.16, 47.77 ], "short_factor": [ 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1 ], "type": "longrope" }, "rope_theta": 10000, "sliding_window": 262144, "tie_word_embeddings": true, "torch_dtype": "bfloat16", "transformers_version": "4.46.1", "use_cache": true, "vocab_size": 200064, "_attn_implementation": "flash_attention_2" } }, { "id": "microsoft/Phi-3-mini-128k-instruct", "meta": { "sha": "f3c06aed622e14ca0abf5115094e4fc9a9948f36", "safetensors": { "parameters": { "BF16": 3821079552 }, "total": 3821079552 }, "modelType": "phi3" }, "config": { "_name_or_path": "Phi-3-mini-128k-instruct", "architectures": [ "Phi3ForCausalLM" ], "attention_dropout": 0, "auto_map": { "AutoConfig": "configuration_phi3.Phi3Config", "AutoModelForCausalLM": "modeling_phi3.Phi3ForCausalLM" }, "bos_token_id": 1, "embd_pdrop": 0, "eos_token_id": 32000, "hidden_act": "silu", "hidden_size": 3072, "initializer_range": 0.02, "intermediate_size": 8192, "max_position_embeddings": 131072, "model_type": "phi3", "num_attention_heads": 32, "num_hidden_layers": 32, "num_key_value_heads": 32, "original_max_position_embeddings": 4096, "pad_token_id": 32000, "resid_pdrop": 0, "rms_norm_eps": 1e-05, "rope_scaling": { "long_factor": [ 1.0700000524520874, 1.1200000047683716, 1.149999976158142, 1.4199999570846558, 1.5699999332427979, 1.7999999523162842, 2.129999876022339, 2.129999876022339, 3.009999990463257, 5.910000324249268, 6.950000286102295, 9.070000648498535, 9.930000305175781, 10.710000038146973, 11.130000114440918, 14.609999656677246, 15.409998893737793, 19.809999465942383, 37.279998779296875, 38.279998779296875, 38.599998474121094, 40.12000274658203, 46.20000457763672, 50.940006256103516, 53.66000747680664, 54.9373893737793, 56.89738845825195, 57.28738784790039, 59.98738479614258, 60.86738586425781, 60.887386322021484, 61.71739196777344, 62.91739273071289, 62.957393646240234, 63.41739273071289, 63.8173942565918, 63.83739471435547, 63.897396087646484, 63.93739700317383, 64.06739807128906, 64.11434936523438, 64.12435150146484, 64.15435028076172, 64.19435119628906, 64.24435424804688, 64.57435607910156, 64.69000244140625, 64.76000213623047 ], "short_factor": [ 1.1, 1.1, 1.1, 1.3000000000000003, 1.3500000000000003, 1.3500000000000003, 1.4000000000000004, 1.5500000000000005, 2.000000000000001, 2.000000000000001, 2.000000000000001, 2.000000000000001, 2.000000000000001, 2.000000000000001, 2.000000000000001, 2.000000000000001, 2.000000000000001, 2.000000000000001, 2.000000000000001, 2.000000000000001, 2.000000000000001, 2.000000000000001, 2.000000000000001, 2.000000000000001, 2.000000000000001, 2.0500000000000007, 2.0500000000000007, 2.0500000000000007, 2.0500000000000007, 2.0500000000000007, 2.0500000000000007, 2.1000000000000005, 2.1000000000000005, 2.1500000000000004, 2.25, 2.25, 2.25, 2.25, 2.25, 2.3999999999999995, 2.4499999999999993, 2.499999999999999, 2.6999999999999984, 2.6999999999999984, 2.7499999999999982, 2.799999999999998, 2.8999999999999977, 3.049999999999997 ], "type": "longrope" }, "rope_theta": 10000, "sliding_window": 262144, "tie_word_embeddings": false, "torch_dtype": "bfloat16", "transformers_version": "4.40.2", "use_cache": true, "attention_bias": false, "vocab_size": 32064 } }, { "id": "microsoft/Phi-3.5-MoE-instruct", "meta": { "sha": "43688451b462a3351d8580625ebe1931adb3986d", "safetensors": { "parameters": { "BF16": 41873153344 }, "total": 41873153344 }, "modelType": "phimoe" }, "config": { "_name_or_path": "Phi-3.5-MoE-instruct", "architectures": [ "PhiMoEForCausalLM" ], "attention_bias": true, "attention_dropout": 0, "auto_map": { "AutoConfig": "configuration_phimoe.PhiMoEConfig", "AutoModelForCausalLM": "modeling_phimoe.PhiMoEForCausalLM" }, "bos_token_id": 1, "eos_token_id": 32000, "hidden_act": "silu", "hidden_dropout": 0, "hidden_size": 4096, "initializer_range": 0.02, "input_jitter_noise": 0.01, "intermediate_size": 6400, "lm_head_bias": true, "max_position_embeddings": 131072, "model_type": "phimoe", "num_attention_heads": 32, "num_experts_per_tok": 2, "num_hidden_layers": 32, "num_key_value_heads": 8, "num_local_experts": 16, "original_max_position_embeddings": 4096, "output_router_logits": false, "rms_norm_eps": 1e-05, "rope_scaling": { "long_factor": [ 1.0199999809265137, 1.0299999713897705, 1.0399999618530273, 1.0499999523162842, 1.0499999523162842, 1.0499999523162842, 1.059999942779541, 1.059999942779541, 1.059999942779541, 1.059999942779541, 1.059999942779541, 1.059999942779541, 1.0999999046325684, 1.1799999475479126, 1.1799999475479126, 1.3700000047683716, 1.4899998903274536, 2.109999895095825, 2.8899998664855957, 3.9499998092651367, 4.299999713897705, 6.429999828338623, 8.09000015258789, 10.690000534057617, 12.050000190734863, 18.229999542236328, 18.84000015258789, 19.899999618530273, 21.420000076293945, 26.200000762939453, 34.28000259399414, 34.590003967285156, 38.730003356933594, 40.22000503540039, 42.54000473022461, 44.000003814697266, 47.590003967285156, 54.750003814697266, 56.19000244140625, 57.44000244140625, 57.4900016784668, 61.20000076293945, 61.540000915527344, 61.75, 61.779998779296875, 62.06999969482422, 63.11000061035156, 63.43000030517578, 63.560001373291016, 63.71000289916992, 63.92000198364258, 63.94000244140625, 63.94000244140625, 63.96000289916992, 63.980003356933594, 64.0300064086914, 64.0300064086914, 64.0300064086914, 64.04000854492188, 64.10000610351562, 64.19000244140625, 64.20999908447266, 64.75, 64.95999908447266 ], "long_mscale": 1.243163121016122, "original_max_position_embeddings": 4096, "short_factor": [ 1, 1.0399999618530273, 1.0399999618530273, 1.0399999618530273, 1.0499999523162842, 1.0499999523162842, 1.0499999523162842, 1.0499999523162842, 1.0499999523162842, 1.0499999523162842, 1.0499999523162842, 1.0499999523162842, 1.0499999523162842, 1.0499999523162842, 1.059999942779541, 1.059999942779541, 1.0699999332427979, 1.0699999332427979, 1.0699999332427979, 1.0699999332427979, 1.1399999856948853, 1.159999966621399, 1.159999966621399, 1.159999966621399, 1.159999966621399, 1.1799999475479126, 1.1999999284744263, 1.3199999332427979, 1.3399999141693115, 1.3499999046325684, 1.3999998569488525, 1.4799998998641968, 1.4999998807907104, 1.589999794960022, 1.6499998569488525, 1.71999990940094, 1.8999998569488525, 1.9099998474121094, 1.9099998474121094, 1.9899998903274536, 1.9999998807907104, 1.9999998807907104, 2.009999990463257, 2.009999990463257, 2.009999990463257, 2.009999990463257, 2.009999990463257, 2.009999990463257, 2.009999990463257, 2.009999990463257, 2.009999990463257, 2.009999990463257, 2.009999990463257, 2.009999990463257, 2.009999990463257, 2.009999990463257, 2.009999990463257, 2.009999990463257, 2.009999990463257, 2.0999999046325684, 2.319999933242798, 2.419999837875366, 2.5899999141693115, 2.7899999618530273 ], "short_mscale": 1.243163121016122, "type": "longrope" }, "rope_theta": 10000, "router_aux_loss_coef": 0, "router_jitter_noise": 0.01, "sliding_window": 131072, "tie_word_embeddings": false, "torch_dtype": "bfloat16", "transformers_version": "4.43.3", "use_cache": true, "vocab_size": 32064 } }, { "id": "microsoft/Phi-tiny-MoE-instruct", "meta": { "sha": "2fe50e88d0e2a5a132563815686ea0dcc8e252b5", "safetensors": { "parameters": { "BF16": 3755220288 }, "total": 3755220288 }, "modelType": "phimoe" }, "config": { "_name_or_path": "Phi-tiny-MoE", "architectures": [ "PhiMoEForCausalLM" ], "attention_bias": true, "attention_dropout": 0, "auto_map": { "AutoConfig": "configuration_slimmoe.PhiMoEConfig", "AutoModelForCausalLM": "modeling_slimmoe.PhiMoEForCausalLM" }, "bos_token_id": 1, "eos_token_id": 32000, "expert_dropout": 0, "head_dim": 128, "hidden_act": "silu", "hidden_dropout": 0, "hidden_size": 4096, "initializer_range": 0.02, "input_jitter_noise": 0.01, "intermediate_size": 448, "lm_head_bias": true, "max_position_embeddings": 4096, "model_type": "phimoe", "num_attention_heads": 16, "num_experts_per_tok": 2, "num_hidden_layers": 32, "num_key_value_heads": 4, "num_local_experts": 16, "output_router_logits": false, "rms_norm_eps": 1e-05, "rope_theta": 10000, "router_aux_loss_coef": 0, "router_jitter_noise": 0.01, "sliding_window": 2047, "tie_word_embeddings": false, "torch_dtype": "bfloat16", "transformers_version": "4.43.3", "use_cache": true, "vocab_size": 32064 } }, { "id": "microsoft/phi-1_5", "meta": { "sha": "77aa61eeac94fbf33d492b9f2744c98b42d5b5eb", "safetensors": { "parameters": { "F16": 1418270720 }, "total": 1418270720 }, "modelType": "phi" }, "config": { "_name_or_path": "microsoft/phi-1_5", "architectures": [ "PhiForCausalLM" ], "attention_dropout": 0, "bos_token_id": null, "embd_pdrop": 0, "eos_token_id": null, "hidden_act": "gelu_new", "hidden_size": 2048, "initializer_range": 0.02, "intermediate_size": 8192, "layer_norm_eps": 1e-05, "max_position_embeddings": 2048, "model_type": "phi", "num_attention_heads": 32, "num_hidden_layers": 24, "num_key_value_heads": null, "partial_rotary_factor": 0.5, "qk_layernorm": false, "resid_pdrop": 0, "rope_scaling": null, "rope_theta": 10000, "tie_word_embeddings": false, "torch_dtype": "float16", "transformers_version": "4.37.0", "use_cache": true, "vocab_size": 51200 } }, { "id": "microsoft/Phi-4-mini-reasoning", "meta": { "sha": "0e3b1e2d02ee478a3743abe3f629e9c0cb722e0a", "safetensors": { "parameters": { "BF16": 3836021760 }, "total": 3836021760 }, "modelType": "phi3" }, "config": { "architectures": [ "Phi3ForCausalLM" ], "attention_bias": false, "attention_dropout": 0, "bos_token_id": 199999, "embd_pdrop": 0, "eos_token_id": 199999, "full_attn_mod": 1, "hidden_act": "silu", "hidden_size": 3072, "initializer_range": 0.02, "intermediate_size": 8192, "interpolate_factor": 1, "lm_head_bias": false, "max_position_embeddings": 131072, "mlp_bias": false, "model_type": "phi3", "num_attention_heads": 24, "num_hidden_layers": 32, "num_key_value_heads": 8, "original_max_position_embeddings": 4096, "pad_token_id": 199999, "partial_rotary_factor": 0.75, "resid_pdrop": 0, "rms_norm_eps": 1e-05, "rope_scaling": { "long_factor": [ 1, 1.118320672, 1.250641126, 1.398617824, 1.564103225, 1.74916897, 1.956131817, 2.187582649, 2.446418898, 2.735880826, 3.059592084, 3.421605075, 3.826451687, 4.279200023, 4.785517845, 5.351743533, 5.984965424, 6.693110555, 7.485043894, 8.370679318, 9.36110372, 10.4687158, 11.70738129, 13.09260651, 14.64173252, 16.37415215, 18.31155283, 20.47818807, 22.90118105, 25.61086418, 28.64115884, 32.03, 32.1, 32.13, 32.23, 32.6, 32.61, 32.64, 32.66, 32.7, 32.71, 32.93, 32.97, 33.28, 33.49, 33.5, 44.16, 47.77 ], "short_factor": [ 1, 1.118320672, 1.250641126, 1.398617824, 1.564103225, 1.74916897, 1.956131817, 2.187582649, 2.446418898, 2.735880826, 3.059592084, 3.421605075, 3.826451687, 4.279200023, 4.785517845, 5.351743533, 5.984965424, 6.693110555, 7.485043894, 8.370679318, 9.36110372, 10.4687158, 11.70738129, 13.09260651, 14.64173252, 16.37415215, 18.31155283, 20.47818807, 22.90118105, 25.61086418, 28.64115884, 32.03, 32.1, 32.13, 32.23, 32.6, 32.61, 32.64, 32.66, 32.7, 32.71, 32.93, 32.97, 33.28, 33.49, 33.5, 44.16, 47.77 ], "type": "longrope" }, "rope_theta": 10000, "sliding_window": 262144, "tie_word_embeddings": true, "torch_dtype": "bfloat16", "transformers_version": "4.50.0", "use_cache": true, "vocab_size": 200064 } }, { "id": "microsoft/Phi-4-reasoning", "meta": { "sha": "1de18ec97600877ce63dbf60c73b998da99f0195", "safetensors": { "parameters": { "BF16": 14659507200 }, "total": 14659507200 }, "modelType": "phi3" }, "config": { "architectures": [ "Phi3ForCausalLM" ], "attention_bias": false, "attention_dropout": 0, "bos_token_id": 100257, "embd_pdrop": 0, "eos_token_id": 100265, "hidden_act": "silu", "hidden_size": 5120, "initializer_range": 0.02, "intermediate_size": 17920, "max_position_embeddings": 32768, "model_type": "phi3", "num_attention_heads": 40, "num_hidden_layers": 40, "num_key_value_heads": 10, "original_max_position_embeddings": 32768, "pad_token_id": 100349, "partial_rotary_factor": 1, "resid_pdrop": 0, "rms_norm_eps": 1e-05, "rope_scaling": null, "rope_theta": 500000, "sliding_window": null, "tie_word_embeddings": false, "torch_dtype": "bfloat16", "transformers_version": "4.51.1", "use_cache": true, "vocab_size": 100352 } }, { "id": "microsoft/Phi-mini-MoE-instruct", "meta": { "sha": "f620b32c0d3e8f7e76f57ccdaa88e0df8bc8bfcd", "safetensors": { "parameters": { "BF16": 7647632704 }, "total": 7647632704 }, "modelType": "phimoe" }, "config": { "_name_or_path": "Phi-mini-MoE", "architectures": [ "PhiMoEForCausalLM" ], "attention_bias": true, "attention_dropout": 0, "auto_map": { "AutoConfig": "configuration_slimmoe.PhiMoEConfig", "AutoModelForCausalLM": "modeling_slimmoe.PhiMoEForCausalLM" }, "bos_token_id": 1, "eos_token_id": 32000, "expert_dropout": 0, "head_dim": 128, "hidden_act": "silu", "hidden_dropout": 0, "hidden_size": 4096, "initializer_range": 0.02, "input_jitter_noise": 0.01, "intermediate_size": 960, "lm_head_bias": true, "max_position_embeddings": 4096, "model_type": "phimoe", "num_attention_heads": 32, "num_experts_per_tok": 2, "num_hidden_layers": 32, "num_key_value_heads": 8, "num_local_experts": 16, "output_router_logits": false, "rms_norm_eps": 1e-05, "rope_theta": 10000, "router_aux_loss_coef": 0, "router_jitter_noise": 0.01, "sliding_window": 2047, "tie_word_embeddings": false, "torch_dtype": "bfloat16", "transformers_version": "4.43.3", "use_cache": true, "vocab_size": 32064 } }, { "id": "microsoft/Phi-3-small-8k-instruct", "meta": { "sha": "188b876e997ee4055df01afe46274c14c400fe40", "safetensors": { "parameters": { "BF16": 7392274432 }, "total": 7392274432 }, "modelType": "phi3small" }, "config": { "_name_or_path": "Phi-3-small-8k-instruct", "architectures": [ "Phi3SmallForCausalLM" ], "attention_dropout_prob": 0, "auto_map": { "AutoConfig": "configuration_phi3_small.Phi3SmallConfig", "AutoModelForCausalLM": "modeling_phi3_small.Phi3SmallForCausalLM", "AutoModelForSequenceClassification": "modeling_phi3_small.Phi3SmallForSequenceClassification", "AutoTokenizer": "tokenization_phi3_small.Phi3SmallTokenizer" }, "blocksparse_block_size": 64, "blocksparse_homo_head_pattern": false, "blocksparse_num_local_blocks": 16, "blocksparse_triton_kernel_block_size": 64, "blocksparse_vert_stride": 8, "bos_token_id": 100257, "dense_attention_every_n_layers": 2, "embedding_dropout_prob": 0.1, "eos_token_id": 100257, "ff_dim_multiplier": null, "ff_intermediate_size": 14336, "ffn_dropout_prob": 0.1, "gegelu_limit": 20, "gegelu_pad_to_256": true, "hidden_act": "gegelu", "hidden_size": 4096, "initializer_range": 0.02, "layer_norm_epsilon": 1e-05, "max_position_embeddings": 8192, "model_type": "phi3small", "mup_attn_multiplier": 1, "mup_embedding_multiplier": 10, "mup_use_scaling": true, "mup_width_multiplier": 8, "num_attention_heads": 32, "num_hidden_layers": 32, "num_key_value_heads": 8, "pad_sequence_to_multiple_of_64": true, "reorder_and_upcast_attn": false, "rope_embedding_base": 1000000, "rope_position_scale": 1, "torch_dtype": "bfloat16", "transformers_version": "4.38.1", "use_cache": true, "attention_bias": false, "vocab_size": 100352 } }, { "id": "microsoft/bitnet-b1.58-2B-4T", "meta": { "sha": "04c3b9ad9361b824064a1f25ea60a8be9599b127", "safetensors": { "parameters": { "BF16": 328775890, "U8": 521011200 }, "total": 849787090 }, "modelType": "bitnet" }, "config": { "architectures": [ "BitNetForCausalLM" ], "auto_map": { "AutoConfig": "configuration_bitnet.BitNetConfig", "AutoModelForCausalLM": "modeling_bitnet.BitNetForCausalLM" }, "bos_token_id": 128000, "eos_token_id": 128001, "hidden_act": "relu2", "hidden_size": 2560, "initializer_range": 0.02, "intermediate_size": 6912, "max_position_embeddings": 4096, "model_type": "bitnet", "rms_norm_eps": 1e-05, "num_attention_heads": 20, "num_hidden_layers": 30, "num_key_value_heads": 5, "rope_theta": 500000, "tie_word_embeddings": true, "torch_dtype": "bfloat16", "use_cache": true, "vocab_size": 128256, "quantization_config": { "quant_method": "bitnet", "linear_class": "autobitlinear", "quantization_mode": "offline" } } }, { "id": "microsoft/Phi-3-medium-4k-instruct", "meta": { "sha": "48d87cd5e0523b430d77e93becd3655cd6897230", "safetensors": { "parameters": { "BF16": 13960238080 }, "total": 13960238080 }, "modelType": "phi3" }, "config": { "_name_or_path": "Phi-3-medium-4k-instruct", "architectures": [ "Phi3ForCausalLM" ], "attention_dropout": 0, "auto_map": { "AutoConfig": "configuration_phi3.Phi3Config", "AutoModelForCausalLM": "modeling_phi3.Phi3ForCausalLM" }, "bos_token_id": 1, "embd_pdrop": 0, "eos_token_id": 32000, "hidden_act": "silu", "hidden_size": 5120, "initializer_range": 0.02, "intermediate_size": 17920, "max_position_embeddings": 4096, "model_type": "phi3", "num_attention_heads": 40, "num_hidden_layers": 40, "num_key_value_heads": 10, "original_max_position_embeddings": 4096, "pad_token_id": 32000, "resid_pdrop": 0, "rms_norm_eps": 1e-05, "rope_scaling": null, "rope_theta": 10000, "sliding_window": 2047, "tie_word_embeddings": false, "torch_dtype": "bfloat16", "transformers_version": "4.39.3", "use_cache": true, "attention_bias": false, "vocab_size": 32064 } }, { "id": "microsoft/phi-1", "meta": { "sha": "d4c0adcb065e84e00ca814e35cba3012ea9841ab", "safetensors": { "parameters": { "F16": 1418270720 }, "total": 1418270720 }, "modelType": "phi" }, "config": { "_name_or_path": "microsoft/phi-1", "architectures": [ "PhiForCausalLM" ], "attention_dropout": 0, "bos_token_id": null, "embd_pdrop": 0, "eos_token_id": null, "hidden_act": "gelu_new", "hidden_size": 2048, "initializer_range": 0.02, "intermediate_size": 8192, "layer_norm_eps": 1e-05, "max_position_embeddings": 2048, "model_type": "phi", "num_attention_heads": 32, "num_hidden_layers": 24, "num_key_value_heads": null, "partial_rotary_factor": 0.5, "qk_layernorm": false, "resid_pdrop": 0, "rope_scaling": null, "rope_theta": 10000, "tie_word_embeddings": false, "torch_dtype": "float32", "transformers_version": "4.37.0", "use_cache": true, "vocab_size": 51200 } }, { "id": "microsoft/Phi-4-reasoning-plus", "meta": { "sha": "69baf8528e1bcf05f475034d9e5dd32875ed125f", "safetensors": { "parameters": { "BF16": 14659507200 }, "total": 14659507200 }, "modelType": "phi3" }, "config": { "architectures": [ "Phi3ForCausalLM" ], "attention_bias": false, "attention_dropout": 0, "bos_token_id": 100257, "embd_pdrop": 0, "eos_token_id": 100265, "hidden_act": "silu", "hidden_size": 5120, "initializer_range": 0.02, "intermediate_size": 17920, "max_position_embeddings": 32768, "model_type": "phi3", "num_attention_heads": 40, "num_hidden_layers": 40, "num_key_value_heads": 10, "original_max_position_embeddings": 32768, "pad_token_id": 100349, "partial_rotary_factor": 1, "resid_pdrop": 0, "rms_norm_eps": 1e-05, "rope_scaling": null, "rope_theta": 500000, "sliding_window": null, "tie_word_embeddings": false, "torch_dtype": "bfloat16", "transformers_version": "4.51.1", "use_cache": true, "vocab_size": 100352 } }, { "id": "microsoft/Phi-3-medium-128k-instruct", "meta": { "sha": "a088b37c71d441ab6d862bb3fcfe6165b3014702", "safetensors": { "parameters": { "BF16": 13960238080 }, "total": 13960238080 }, "modelType": "phi3" }, "config": { "_name_or_path": "Phi-3-medium-128k-instruct", "architectures": [ "Phi3ForCausalLM" ], "attention_dropout": 0, "auto_map": { "AutoConfig": "configuration_phi3.Phi3Config", "AutoModelForCausalLM": "modeling_phi3.Phi3ForCausalLM" }, "bos_token_id": 1, "embd_pdrop": 0, "eos_token_id": 32000, "hidden_act": "silu", "hidden_size": 5120, "initializer_range": 0.02, "intermediate_size": 17920, "max_position_embeddings": 131072, "model_type": "phi3", "num_attention_heads": 40, "num_hidden_layers": 40, "num_key_value_heads": 10, "original_max_position_embeddings": 4096, "pad_token_id": null, "resid_pdrop": 0, "rms_norm_eps": 1e-05, "rope_scaling": { "long_factor": [ 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1.25, 1.25, 1.5, 2, 2.75, 5.75, 5.75, 6.5, 9.25, 11, 13.25, 19.25, 19.75, 19.75, 21.25, 21.5, 26.5, 30, 33.75, 35.25, 38.5, 42, 42.25, 46, 47, 50, 50.5, 51, 52, 52.75, 53.75, 54.75, 57, 57.25, 58.5, 59.25, 59.5, 62, 62.5, 62.75, 63.25, 63.25, 63.25, 63.75, 64, 64, 64.25, 64.5, 64.5, 65, 65 ], "short_factor": [ 1, 1, 1, 1, 1, 1, 1.01, 1.02, 1.02, 1.04, 1.04, 1.07, 1.07, 1.1, 1.3000000000000003, 1.3000000000000003, 1.5000000000000004, 1.5700000000000005, 1.9000000000000008, 2.3100000000000014, 2.759999999999992, 3.3899999999999784, 3.9399999999999666, 4.009999999999965, 4.289999999999959, 4.349999999999958, 5.349999999999937, 6.659999999999909, 7.029999999999901, 7.51999999999989, 8.00999999999988, 8.249999999999876, 8.279999999999875, 9.629999999999846, 9.89999999999984, 10.589999999999826, 11.049999999999816, 11.7899999999998, 12.189999999999792, 12.889999999999777, 13.129999999999772, 13.16999999999977, 13.20999999999977, 13.479999999999764, 13.539999999999763, 13.779999999999758, 13.929999999999755, 14.429999999999744, 14.759999999999737, 15.149999999999729, 15.419999999999723, 15.53999999999972, 15.659999999999718, 15.749999999999716, 15.759999999999716, 15.799999999999715, 16.05999999999971, 16.079999999999714, 16.11999999999972, 16.11999999999972, 16.18999999999973, 16.31999999999975, 16.539999999999786, 16.799999999999827 ], "type": "su" }, "rope_theta": 10000, "sliding_window": 131072, "tie_word_embeddings": false, "torch_dtype": "bfloat16", "transformers_version": "4.39.3", "use_cache": true, "attention_bias": false, "vocab_size": 32064 } }, { "id": "microsoft/bitnet-b1.58-2B-4T-bf16", "meta": { "sha": "276681394656abdadb8e80e5b2c3db5e5d7fcaff", "safetensors": { "parameters": { "BF16": 2412820480 }, "total": 2412820480 }, "modelType": "bitnet" }, "config": { "architectures": [ "BitNetForCausalLM" ], "auto_map": { "AutoConfig": "configuration_bitnet.BitNetConfig", "AutoModelForCausalLM": "modeling_bitnet.BitNetForCausalLM" }, "bos_token_id": 128000, "eos_token_id": 128001, "hidden_act": "relu2", "hidden_size": 2560, "initializer_range": 0.02, "intermediate_size": 6912, "max_position_embeddings": 4096, "model_type": "bitnet", "rms_norm_eps": 1e-05, "num_attention_heads": 20, "num_hidden_layers": 30, "num_key_value_heads": 5, "rope_theta": 500000, "tie_word_embeddings": true, "torch_dtype": "bfloat16", "use_cache": true, "vocab_size": 128256, "quantization_config": { "quant_method": "bitnet", "linear_class": "autobitlinear", "quantization_mode": "online" } } }, { "id": "NousResearch/Hermes-3-Llama-3.1-8B", "meta": { "sha": "896ea440e5a9e6070e3d8a2774daf2b481ab425b", "safetensors": { "parameters": { "BF16": 8030261248 }, "total": 8030261248 }, "modelType": "llama" }, "config": { "_name_or_path": "NousResearch/Hermes-3-Llama-3.1-8B", "architectures": [ "LlamaForCausalLM" ], "attention_bias": false, "attention_dropout": 0, "bos_token_id": 128000, "eos_token_id": 128040, "hidden_act": "silu", "hidden_size": 4096, "initializer_range": 0.02, "intermediate_size": 14336, "max_position_embeddings": 131072, "mlp_bias": false, "model_type": "llama", "num_attention_heads": 32, "num_hidden_layers": 32, "num_key_value_heads": 8, "pretraining_tp": 1, "rms_norm_eps": 1e-05, "rope_scaling": { "factor": 8, "high_freq_factor": 4, "low_freq_factor": 1, "original_max_position_embeddings": 8192, "rope_type": "llama3" }, "rope_theta": 500000, "tie_word_embeddings": false, "torch_dtype": "bfloat16", "transformers_version": "4.44.0.dev0", "use_cache": true, "vocab_size": 128256 } }, { "id": "NousResearch/Hermes-2-Pro-Mistral-7B", "meta": { "sha": "24dbda51d986efa05201eec7a093ac91fc5519cf", "safetensors": { "parameters": { "BF16": 7241994240 }, "total": 7241994240 }, "modelType": "mistral" }, "config": { "_name_or_path": "bagel-hermes-2-pro-7b", "architectures": [ "MistralForCausalLM" ], "attention_dropout": 0, "bos_token_id": 1, "eos_token_id": 32000, "hidden_act": "silu", "hidden_size": 4096, "initializer_range": 0.02, "intermediate_size": 14336, "max_position_embeddings": 32768, "model_type": "mistral", "num_attention_heads": 32, "num_hidden_layers": 32, "num_key_value_heads": 8, "rms_norm_eps": 1e-05, "rope_theta": 10000, "sliding_window": 4096, "tie_word_embeddings": false, "torch_dtype": "bfloat16", "transformers_version": "4.38.2", "use_cache": false, "vocab_size": 32032 } }, { "id": "NousResearch/Nous-Hermes-llama-2-7b", "meta": { "sha": "cbd525ad0d5012e470c7696d5905345dc082e432", "safetensors": { "parameters": { "BF16": 6738415616 }, "total": 6738415616 }, "modelType": "llama" }, "config": { "_name_or_path": "output/hermes-llama2-4k/checkpoint-2259", "architectures": [ "LlamaForCausalLM" ], "bos_token_id": 1, "eos_token_id": 2, "hidden_act": "silu", "hidden_size": 4096, "initializer_range": 0.02, "intermediate_size": 11008, "max_position_embeddings": 4096, "model_type": "llama", "num_attention_heads": 32, "num_hidden_layers": 32, "num_key_value_heads": 32, "pad_token_id": 0, "pretraining_tp": 1, "rms_norm_eps": 1e-05, "rope_scaling": null, "tie_word_embeddings": false, "torch_dtype": "bfloat16", "transformers_version": "4.32.0.dev0", "use_cache": false, "vocab_size": 32000 } }, { "id": "NousResearch/Nous-Hermes-2-Mixtral-8x7B-DPO", "meta": { "sha": "286ae6737d048ad1d965c2e830864df02db50f2f", "safetensors": { "parameters": { "BF16": 46702809088 }, "total": 46702809088 }, "modelType": "mixtral" }, "config": { "_name_or_path": "NousResearch/OpenHermes-2.5-Mixtral-8x7B-epoch4", "architectures": [ "MixtralForCausalLM" ], "attention_dropout": 0, "bos_token_id": 1, "eos_token_id": 32000, "hidden_act": "silu", "hidden_size": 4096, "initializer_range": 0.02, "intermediate_size": 14336, "max_position_embeddings": 32768, "model_type": "mixtral", "num_attention_heads": 32, "num_experts_per_tok": 2, "num_hidden_layers": 32, "num_key_value_heads": 8, "num_local_experts": 8, "output_router_logits": false, "rms_norm_eps": 1e-05, "rope_theta": 1000000, "router_aux_loss_coef": 0.02, "sliding_window": null, "tie_word_embeddings": false, "torch_dtype": "bfloat16", "transformers_version": "4.37.0.dev0", "use_cache": false, "vocab_size": 32002 } }, { "id": "NousResearch/Hermes-2-Theta-Llama-3-8B", "meta": { "sha": "57a73110702e7b05ba3f39fef36297454c680725", "safetensors": { "parameters": { "BF16": 8030261248 }, "total": 8030261248 }, "modelType": "llama" }, "config": { "_name_or_path": "goddardexperiments/HermesProInstructV10", "architectures": [ "LlamaForCausalLM" ], "attention_bias": false, "attention_dropout": 0, "bos_token_id": 128000, "eos_token_id": 128003, "hidden_act": "silu", "hidden_size": 4096, "initializer_range": 0.02, "intermediate_size": 14336, "max_position_embeddings": 8192, "model_type": "llama", "num_attention_heads": 32, "num_hidden_layers": 32, "num_key_value_heads": 8, "pretraining_tp": 1, "rms_norm_eps": 1e-05, "rope_scaling": null, "rope_theta": 500000, "tie_word_embeddings": false, "torch_dtype": "bfloat16", "transformers_version": "4.40.0.dev0", "use_cache": true, "vocab_size": 128256 } }, { "id": "NousResearch/Nous-Hermes-2-SOLAR-10.7B", "meta": { "sha": "14c1fbe2f71acdcd58247b30d5439bd572d52386", "safetensors": { "parameters": { "BF16": 10731540480 }, "total": 10731540480 }, "modelType": "llama" }, "config": { "_name_or_path": "upstage/SOLAR-10.7B-v1.0", "architectures": [ "LlamaForCausalLM" ], "attention_bias": false, "attention_dropout": 0, "bos_token_id": 1, "eos_token_id": 32000, "hidden_act": "silu", "hidden_size": 4096, "initializer_range": 0.02, "intermediate_size": 14336, "max_position_embeddings": 4096, "model_type": "llama", "num_attention_heads": 32, "num_hidden_layers": 48, "num_key_value_heads": 8, "pretraining_tp": 1, "rms_norm_eps": 1e-05, "rope_scaling": null, "rope_theta": 10000, "tie_word_embeddings": false, "torch_dtype": "bfloat16", "transformers_version": "4.37.0.dev0", "use_cache": false, "vocab_size": 32002 } }, { "id": "NousResearch/Nous-Hermes-2-Yi-34B", "meta": { "sha": "fcb0a8847e76aea14aba9aa44009d4418ad7c18f", "safetensors": { "parameters": { "BF16": 34388917248 }, "total": 34388917248 }, "modelType": "llama" }, "config": { "_name_or_path": "01-ai/Yi-34B", "architectures": [ "LlamaForCausalLM" ], "attention_bias": false, "attention_dropout": 0, "bos_token_id": 1, "eos_token_id": 7, "hidden_act": "silu", "hidden_size": 7168, "initializer_range": 0.02, "intermediate_size": 20480, "max_position_embeddings": 4096, "model_type": "llama", "num_attention_heads": 56, "num_hidden_layers": 60, "num_key_value_heads": 8, "pad_token_id": 0, "pretraining_tp": 1, "rms_norm_eps": 1e-05, "rope_scaling": null, "rope_theta": 5000000, "tie_word_embeddings": false, "torch_dtype": "bfloat16", "transformers_version": "4.37.0.dev0", "use_cache": false, "vocab_size": 64000 } }, { "id": "NousResearch/Hermes-4-70B", "meta": { "sha": "d5dec2bd6b3930a09ddefd0b7fc6523fe0720d09", "safetensors": { "parameters": { "BF16": 70553706496 }, "total": 70553706496 }, "modelType": "llama" }, "config": { "architectures": [ "LlamaForCausalLM" ], "attention_bias": false, "attention_dropout": 0, "bos_token_id": 128000, "eos_token_id": 128001, "head_dim": 128, "hidden_act": "silu", "hidden_size": 8192, "initializer_range": 0.02, "intermediate_size": 28672, "max_position_embeddings": 131072, "mlp_bias": false, "model_type": "llama", "num_attention_heads": 64, "num_hidden_layers": 80, "num_key_value_heads": 8, "pretraining_tp": 1, "rms_norm_eps": 1e-05, "rope_scaling": { "factor": 8, "high_freq_factor": 4, "low_freq_factor": 1, "original_max_position_embeddings": 8192, "rope_type": "llama3" }, "rope_theta": 500000, "tie_word_embeddings": false, "torch_dtype": "bfloat16", "transformers_version": "4.52.4", "use_cache": true, "vocab_size": 128256 } }, { "id": "NousResearch/Hermes-3-Llama-3.1-70B", "meta": { "sha": "1fc86da0ce9cdb14cd775ad270bc7d1b4bf70ede", "safetensors": { "parameters": { "BF16": 70553706496 }, "total": 70553706496 }, "modelType": "llama" }, "config": { "_name_or_path": "out/hermes-3-8k-llama-3.1-70b-chatml-1/checkpoint-3075/merged", "architectures": [ "LlamaForCausalLM" ], "attention_bias": false, "attention_dropout": 0, "bos_token_id": 128000, "eos_token_id": 128039, "hidden_act": "silu", "hidden_size": 8192, "initializer_range": 0.02, "intermediate_size": 28672, "max_position_embeddings": 131072, "mlp_bias": false, "model_type": "llama", "num_attention_heads": 64, "num_hidden_layers": 80, "num_key_value_heads": 8, "pretraining_tp": 1, "rms_norm_eps": 1e-05, "rope_scaling": { "factor": 8, "high_freq_factor": 4, "low_freq_factor": 1, "original_max_position_embeddings": 8192, "rope_type": "llama3" }, "rope_theta": 500000, "tie_word_embeddings": false, "torch_dtype": "bfloat16", "transformers_version": "4.44.0.dev0", "use_cache": true, "vocab_size": 128256 } }, { "id": "NousResearch/Hermes-2-Theta-Llama-3-70B", "meta": { "sha": "8948247fa83f3f970ad24b3394d5656507642922", "safetensors": { "parameters": { "BF16": 70553706496 }, "total": 70553706496 }, "modelType": "llama" }, "config": { "_name_or_path": "NousResearch/Hermes-2-Theta-Llama-3-70B", "architectures": [ "LlamaForCausalLM" ], "attention_bias": false, "attention_dropout": 0, "bos_token_id": 128000, "eos_token_id": 128003, "hidden_act": "silu", "hidden_size": 8192, "initializer_range": 0.02, "intermediate_size": 28672, "max_position_embeddings": 8192, "mlp_bias": false, "model_type": "llama", "num_attention_heads": 64, "num_hidden_layers": 80, "num_key_value_heads": 8, "pretraining_tp": 1, "rms_norm_eps": 1e-05, "rope_scaling": null, "rope_theta": 500000, "tie_word_embeddings": false, "torch_dtype": "bfloat16", "transformers_version": "4.40.0.dev0", "use_cache": false, "vocab_size": 128256 } }, { "id": "NousResearch/Hermes-3-Llama-3.2-3B", "meta": { "sha": "7f1a6bec8cdce6551014fd5bbeb4cd8c0f1fbeab", "safetensors": { "parameters": { "BF16": 3212749824 }, "total": 3212749824 }, "modelType": "llama" }, "config": { "_name_or_path": "Nous-Research-Core/Hermes-3-Llama-3.2-3b-base-fft-chatml-rerun-8xh100-e4-2024-11-24-4k-steps", "architectures": [ "LlamaForCausalLM" ], "attention_bias": false, "attention_dropout": 0, "bos_token_id": 128000, "eos_token_id": 128039, "head_dim": 128, "hidden_act": "silu", "hidden_size": 3072, "initializer_range": 0.02, "intermediate_size": 8192, "max_position_embeddings": 131072, "mlp_bias": false, "model_type": "llama", "num_attention_heads": 24, "num_hidden_layers": 28, "num_key_value_heads": 8, "pretraining_tp": 1, "rms_norm_eps": 1e-05, "rope_scaling": { "factor": 32, "high_freq_factor": 4, "low_freq_factor": 1, "original_max_position_embeddings": 8192, "rope_type": "llama3" }, "rope_theta": 500000, "tie_word_embeddings": true, "torch_dtype": "bfloat16", "transformers_version": "4.46.3", "use_cache": false, "vocab_size": 128256 } }, { "id": "NousResearch/Hermes-2-Pro-Llama-3-8B", "meta": { "sha": "f798274b30e7d2d4797c369edcc0cc7473b6e6f2", "safetensors": { "parameters": { "F16": 8030523392 }, "total": 8030523392 }, "modelType": "llama" }, "config": { "_name_or_path": "./hermes-2-pro-llama-3-8b-DPO", "architectures": [ "LlamaForCausalLM" ], "attention_bias": false, "attention_dropout": 0, "bos_token_id": 128000, "eos_token_id": 128003, "hidden_act": "silu", "hidden_size": 4096, "initializer_range": 0.02, "intermediate_size": 14336, "max_position_embeddings": 8192, "model_type": "llama", "num_attention_heads": 32, "num_hidden_layers": 32, "num_key_value_heads": 8, "pretraining_tp": 1, "rms_norm_eps": 1e-05, "rope_scaling": null, "rope_theta": 500000, "tie_word_embeddings": false, "torch_dtype": "float16", "transformers_version": "4.40.1", "use_cache": true, "vocab_size": 128288 } }, { "id": "NousResearch/Hermes-4.3-36B", "meta": { "sha": "3899db2b6c4b35f16bde3b570bb7dd2775d56161", "safetensors": { "parameters": { "BF16": 36151104512 }, "total": 36151104512 }, "modelType": "seed_oss" }, "config": { "architectures": [ "SeedOssForCausalLM" ], "attention_bias": true, "attention_dropout": 0, "attention_out_bias": false, "bos_token_id": 0, "pad_token_id": 1, "eos_token_id": 2, "head_dim": 128, "hidden_act": "silu", "hidden_size": 5120, "initializer_range": 0.02, "intermediate_size": 27648, "max_position_embeddings": 524288, "mlp_bias": false, "model_type": "seed_oss", "num_attention_heads": 80, "num_hidden_layers": 64, "num_key_value_heads": 8, "residual_dropout": 0, "rms_norm_eps": 1e-06, "rope_scaling": { "rope_type": "default" }, "rope_theta": 10000000, "tie_word_embeddings": false, "torch_dtype": "bfloat16", "transformers_version": "4.55.0", "use_cache": true, "vocab_size": 155136 } }, { "id": "NousResearch/Hermes-4-14B", "meta": { "sha": "d6ce765c8b83f847357b98254be079afa0c6ca76", "safetensors": { "parameters": { "BF16": 14768307200 }, "total": 424960 }, "modelType": "qwen3" }, "config": { "architectures": [ "Qwen3ForCausalLM" ], "attention_bias": false, "attention_dropout": 0, "eos_token_id": 151645, "head_dim": 128, "hidden_act": "silu", "hidden_size": 5120, "initializer_range": 0.02, "intermediate_size": 17408, "layer_types": [ "full_attention", "full_attention", "full_attention", "full_attention", "full_attention", "full_attention", "full_attention", "full_attention", "full_attention", "full_attention", "full_attention", "full_attention", "full_attention", "full_attention", "full_attention", "full_attention", "full_attention", "full_attention", "full_attention", "full_attention", "full_attention", "full_attention", "full_attention", "full_attention", "full_attention", "full_attention", "full_attention", "full_attention", "full_attention", "full_attention", "full_attention", "full_attention", "full_attention", "full_attention", "full_attention", "full_attention", "full_attention", "full_attention", "full_attention", "full_attention" ], "max_position_embeddings": 40960, "max_window_layers": 40, "model_type": "qwen3", "num_attention_heads": 40, "num_hidden_layers": 40, "num_key_value_heads": 8, "rms_norm_eps": 1e-06, "rope_scaling": null, "rope_theta": 1000000, "sliding_window": null, "tie_word_embeddings": false, "torch_dtype": "bfloat16", "transformers_version": "4.54.0", "use_cache": false, "use_sliding_window": false, "vocab_size": 151936 } }, { "id": "NousResearch/Nous-Hermes-Llama2-13b", "meta": { "sha": "a5787bbb8ed8d322f3b1f91f9afd7fe07e7f041a", "safetensors": { "parameters": { "F32": 2560, "BF16": 13016192000 }, "total": 13016194560 }, "modelType": "llama" }, "config": { "_name_or_path": "output/hermes-llama2-4k/checkpoint-2259", "architectures": [ "LlamaForCausalLM" ], "bos_token_id": 1, "eos_token_id": 2, "hidden_act": "silu", "hidden_size": 5120, "initializer_range": 0.02, "intermediate_size": 13824, "max_position_embeddings": 4096, "model_type": "llama", "num_attention_heads": 40, "num_hidden_layers": 40, "num_key_value_heads": 40, "pad_token_id": 0, "pretraining_tp": 1, "rms_norm_eps": 1e-05, "rope_scaling": null, "tie_word_embeddings": false, "torch_dtype": "bfloat16", "transformers_version": "4.32.0.dev0", "use_cache": true, "vocab_size": 32032 } }, { "id": "NousResearch/Nous-Hermes-2-Mistral-7B-DPO", "meta": { "sha": "ebec0a691037d38955727d6949798429a63929dd", "safetensors": { "parameters": { "BF16": 7241748480 }, "total": 7241748480 }, "modelType": "mistral" }, "config": { "_name_or_path": "teknium/OpenHermes-2.5-Mistral-7B", "architectures": [ "MistralForCausalLM" ], "attention_dropout": 0, "bos_token_id": 1, "eos_token_id": 32000, "hidden_act": "silu", "hidden_size": 4096, "initializer_range": 0.02, "intermediate_size": 14336, "max_position_embeddings": 32768, "model_type": "mistral", "num_attention_heads": 32, "num_hidden_layers": 32, "num_key_value_heads": 8, "rms_norm_eps": 1e-05, "rope_theta": 10000, "sliding_window": 4096, "tie_word_embeddings": false, "torch_dtype": "bfloat16", "transformers_version": "4.37.0.dev0", "use_cache": false, "vocab_size": 32002 } }, { "id": "NousResearch/Hermes-3-Llama-3.1-405B", "meta": { "sha": "b9fe8ede1f5075f171b2c0a235e20ae4aba33767", "safetensors": { "parameters": { "BF16": 405853388800 }, "total": 405853388800 }, "modelType": "llama" }, "config": { "_name_or_path": "NousResearch/Hermes-3-Llama-3.1-405B", "architectures": [ "LlamaForCausalLM" ], "attention_bias": false, "attention_dropout": 0, "bos_token_id": 128000, "eos_token_id": 128039, "hidden_act": "silu", "hidden_size": 16384, "initializer_range": 0.02, "intermediate_size": 53248, "max_position_embeddings": 131072, "mlp_bias": false, "model_type": "llama", "num_attention_heads": 128, "num_hidden_layers": 126, "num_key_value_heads": 8, "pretraining_tp": 1, "rms_norm_eps": 1e-05, "rope_scaling": { "factor": 8, "high_freq_factor": 4, "low_freq_factor": 1, "original_max_position_embeddings": 8192, "rope_type": "llama3" }, "rope_theta": 500000, "tie_word_embeddings": false, "torch_dtype": "bfloat16", "transformers_version": "4.44.0.dev0", "use_cache": true, "vocab_size": 128256 } }, { "id": "tiiuae/falcon-7b", "meta": { "sha": "ec89142b67d748a1865ea4451372db8313ada0d8", "safetensors": { "parameters": { "BF16": 7217189760 }, "total": 7217189760 }, "modelType": "falcon" }, "config": { "alibi": false, "apply_residual_connection_post_layernorm": false, "architectures": [ "FalconForCausalLM" ], "attention_dropout": 0.0, "auto_map": { "AutoConfig": "configuration_falcon.FalconConfig", "AutoModel": "modeling_falcon.FalconModel", "AutoModelForSequenceClassification": "modeling_falcon.FalconForSequenceClassification", "AutoModelForTokenClassification": "modeling_falcon.FalconForTokenClassification", "AutoModelForQuestionAnswering": "modeling_falcon.FalconForQuestionAnswering", "AutoModelForCausalLM": "modeling_falcon.FalconForCausalLM" }, "bias": false, "bos_token_id": 11, "eos_token_id": 11, "hidden_dropout": 0.0, "hidden_size": 4544, "initializer_range": 0.02, "layer_norm_epsilon": 1e-05, "model_type": "falcon", "multi_query": true, "new_decoder_architecture": false, "num_attention_heads": 71, "num_hidden_layers": 32, "parallel_attn": true, "torch_dtype": "bfloat16", "transformers_version": "4.27.4", "use_cache": true, "vocab_size": 65024 }, "contextSource": "https://huggingface.co/tiiuae/falcon-7b", "verifiedAt": "2026-09-22" }, { "id": "tiiuae/Falcon-H1-0.5B-Base", "meta": { "sha": "59fb76e8c5d3fc7441b062be638e1ba0afd5c687", "safetensors": { "parameters": { "BF16": 521411104 }, "total": 521411104 }, "modelType": "falcon_h1" }, "config": { "architectures": [ "FalconH1ForCausalLM" ], "attention_bias": false, "attention_dropout": 0, "attention_in_multiplier": 1, "attention_out_multiplier": 0.9375, "attn_layer_indices": null, "bos_token_id": 1, "embedding_multiplier": 5.656854249492381, "eos_token_id": 11, "head_dim": 64, "hidden_act": "silu", "hidden_size": 1024, "initializer_range": 0.02, "intermediate_size": 2048, "key_multiplier": 0.39062499999999994, "lm_head_multiplier": 0.0390625, "mamba_chunk_size": 128, "mamba_conv_bias": true, "mamba_d_conv": 4, "mamba_d_head": 64, "mamba_d_ssm": 1536, "mamba_d_state": 128, "mamba_expand": 2, "mamba_n_groups": 1, "mamba_n_heads": 24, "mamba_norm_before_gate": false, "mamba_proj_bias": false, "mamba_rms_norm": false, "mamba_use_mlp": true, "max_position_embeddings": 16384, "mlp_bias": false, "mlp_expansion_factor": 8, "mlp_multipliers": [ 0.8838834764831844, 0.5859375 ], "model_type": "falcon_h1", "num_attention_heads": 8, "num_hidden_layers": 36, "num_key_value_heads": 2, "num_logits_to_keep": 1, "pad_token_id": 0, "projectors_bias": false, "rms_norm_eps": 1e-05, "rope_scaling": null, "rope_theta": 100000000000, "ssm_in_multiplier": 1.25, "ssm_multipliers": [ 0.3535533905932738, 0.25, 0.3535533905932738, 0.5, 0.3535533905932738 ], "ssm_out_multiplier": 0.23570226039551587, "tie_word_embeddings": false, "torch_dtype": "bfloat16", "transformers_version": "4.52.0.dev0", "use_cache": true, "vocab_size": 32784 } }, { "id": "tiiuae/falcon-mamba-7b", "meta": { "sha": "080ad94b3619e2c2d0afa59bafdc6113465b7006", "safetensors": { "parameters": { "BF16": 7272665088 }, "total": 7272665088 }, "modelType": "falcon_mamba" }, "config": { "_name_or_path": "./", "architectures": [ "FalconMambaForCausalLM" ], "bos_token_id": 0, "conv_kernel": 4, "eos_token_id": 11, "expand": 16, "hidden_act": "silu", "hidden_size": 4096, "initializer_range": 0.1, "intermediate_size": 8192, "layer_norm_epsilon": 1e-05, "model_type": "falcon_mamba", "num_hidden_layers": 64, "pad_token_id": 11, "rescale_prenorm_residual": false, "residual_in_fp32": true, "state_size": 16, "tie_word_embeddings": false, "time_step_floor": 0.0001, "time_step_init_scheme": "random", "time_step_max": 0.1, "time_step_min": 0.001, "time_step_rank": 256, "time_step_scale": 1, "torch_dtype": "bfloat16", "transformers_version": "4.43.0.dev0", "use_bias": false, "use_cache": true, "use_conv_bias": true, "vocab_size": 65024 } }, { "id": "tiiuae/falcon-7b-instruct", "meta": { "sha": "8782b5c5d8c9290412416618f36a133653e85285", "safetensors": { "parameters": { "BF16": 7217189760 }, "total": 7217189760 }, "modelType": "falcon" }, "config": { "alibi": false, "apply_residual_connection_post_layernorm": false, "architectures": [ "FalconForCausalLM" ], "attention_dropout": 0.0, "auto_map": { "AutoConfig": "configuration_falcon.FalconConfig", "AutoModel": "modeling_falcon.FalconModel", "AutoModelForSequenceClassification": "modeling_falcon.FalconForSequenceClassification", "AutoModelForTokenClassification": "modeling_falcon.FalconForTokenClassification", "AutoModelForQuestionAnswering": "modeling_falcon.FalconForQuestionAnswering", "AutoModelForCausalLM": "modeling_falcon.FalconForCausalLM" }, "bias": false, "bos_token_id": 11, "eos_token_id": 11, "hidden_dropout": 0.0, "hidden_size": 4544, "initializer_range": 0.02, "layer_norm_epsilon": 1e-05, "model_type": "falcon", "multi_query": true, "new_decoder_architecture": false, "num_attention_heads": 71, "num_hidden_layers": 32, "parallel_attn": true, "torch_dtype": "bfloat16", "transformers_version": "4.27.4", "use_cache": true, "vocab_size": 65024 }, "contextSource": "https://huggingface.co/tiiuae/falcon-7b-instruct", "verifiedAt": "2026-09-22" }, { "id": "tiiuae/falcon-mamba-7b-instruct", "meta": { "sha": "b250fc9399d14f56aca18e9ea70bbfb1f73479eb", "safetensors": { "parameters": { "BF16": 7272665088 }, "total": 7272665088 }, "modelType": "falcon_mamba" }, "config": { "_name_or_path": "tiiuae/falcon-mamba-7b-chat", "architectures": [ "FalconMambaForCausalLM" ], "bos_token_id": 8, "conv_kernel": 4, "eos_token_id": 11, "expand": 16, "hidden_act": "silu", "hidden_size": 4096, "initializer_range": 0.1, "intermediate_size": 8192, "layer_norm_epsilon": 1e-05, "model_type": "falcon_mamba", "num_hidden_layers": 64, "pad_token_id": 0, "rescale_prenorm_residual": false, "residual_in_fp32": true, "state_size": 16, "tie_word_embeddings": false, "time_step_floor": 0.0001, "time_step_init_scheme": "random", "time_step_max": 0.1, "time_step_min": 0.001, "time_step_rank": 256, "time_step_scale": 1, "torch_dtype": "bfloat16", "transformers_version": "4.44.0.dev0", "use_bias": false, "use_cache": true, "use_conv_bias": true, "use_mambapy": false, "vocab_size": 65024 } }, { "id": "tiiuae/Falcon-H1-7B-Instruct", "meta": { "sha": "41e72f27effbab80cd45b6e884688452253a3686", "safetensors": { "parameters": { "BF16": 7585654880 }, "total": 7585654880 }, "modelType": "falcon_h1" }, "config": { "architectures": [ "FalconH1ForCausalLM" ], "attention_bias": false, "attention_dropout": 0, "attention_in_multiplier": 1, "attention_out_multiplier": 0.10416666666666667, "attn_layer_indices": null, "bos_token_id": 1, "embedding_multiplier": 5.656854249492381, "eos_token_id": 11, "head_dim": 128, "hidden_act": "silu", "hidden_size": 3072, "initializer_range": 0.02, "intermediate_size": 12288, "key_multiplier": 0.030690398488999456, "lm_head_multiplier": 0.013020833333333334, "mamba_chunk_size": 256, "mamba_conv_bias": true, "mamba_d_conv": 4, "mamba_d_head": 128, "mamba_d_ssm": 3072, "mamba_d_state": 256, "mamba_expand": 2, "mamba_n_groups": 1, "mamba_n_heads": 24, "mamba_norm_before_gate": false, "mamba_proj_bias": false, "mamba_rms_norm": true, "mamba_use_mlp": true, "max_position_embeddings": 262144, "mlp_bias": false, "mlp_expansion_factor": 8, "mlp_multipliers": [ 0.2946278254943948, 0.032552083333333336 ], "model_type": "falcon_h1", "num_attention_heads": 12, "num_hidden_layers": 44, "num_key_value_heads": 2, "num_logits_to_keep": 1, "pad_token_id": 0, "projectors_bias": false, "rms_norm_eps": 1e-05, "rope_scaling": null, "rope_theta": 100000000000, "ssm_in_multiplier": 0.4166666666666667, "ssm_multipliers": [ 0.3535533905932738, 0.25, 0.1767766952966369, 0.5, 0.3535533905932738 ], "ssm_out_multiplier": 0.11785113019775793, "tie_word_embeddings": false, "torch_dtype": "bfloat16", "transformers_version": "4.52.0.dev0", "use_cache": true, "vocab_size": 130049 } }, { "id": "tiiuae/Falcon3-7B-Instruct", "meta": { "sha": "1e57a0ecd176c7c139f289c60a74e57f887c3dfb", "safetensors": { "parameters": { "BF16": 7455550464 }, "total": 7455550464 }, "modelType": "llama" }, "config": { "architectures": [ "LlamaForCausalLM" ], "attention_bias": false, "attention_dropout": 0, "bos_token_id": 11, "eos_token_id": 11, "head_dim": 256, "hidden_act": "silu", "hidden_size": 3072, "intermediate_size": 23040, "max_position_embeddings": 32768, "mlp_bias": false, "model_type": "llama", "num_attention_heads": 12, "num_hidden_layers": 28, "num_key_value_heads": 4, "pretraining_tp": 1, "rms_norm_eps": 1e-06, "rope_scaling": null, "rope_theta": 1000042, "tie_word_embeddings": false, "torch_dtype": "bfloat16", "transformers_version": "4.46.1", "use_cache": true, "vocab_size": 131072 } }, { "id": "tiiuae/Falcon-H1-1.5B-Deep-Instruct", "meta": { "sha": "b6648636ddc906688974282de6e7a243395f5423", "safetensors": { "parameters": { "BF16": 1554874768 }, "total": 1554874768 }, "modelType": "falcon_h1" }, "config": { "architectures": [ "FalconH1ForCausalLM" ], "attention_bias": false, "attention_dropout": 0, "attention_in_multiplier": 1, "attention_out_multiplier": 0.5, "attn_layer_indices": null, "bos_token_id": 1, "embedding_multiplier": 5.656854249492381, "eos_token_id": 11, "head_dim": 128, "hidden_act": "silu", "hidden_size": 1280, "initializer_range": 0.02, "intermediate_size": 3072, "key_multiplier": 0.17677669529663687, "lm_head_multiplier": 0.03125, "mamba_chunk_size": 128, "mamba_conv_bias": true, "mamba_d_conv": 4, "mamba_d_head": 64, "mamba_d_ssm": 1536, "mamba_d_state": 256, "mamba_expand": 2, "mamba_n_groups": 1, "mamba_n_heads": 24, "mamba_norm_before_gate": false, "mamba_proj_bias": false, "mamba_rms_norm": true, "mamba_use_mlp": true, "max_position_embeddings": 131072, "mlp_bias": false, "mlp_expansion_factor": 8, "mlp_multipliers": [ 0.7071067811865476, 0.3125 ], "model_type": "falcon_h1", "num_attention_heads": 6, "num_hidden_layers": 66, "num_key_value_heads": 2, "num_logits_to_keep": 1, "pad_token_id": 0, "projectors_bias": false, "rms_norm_eps": 1e-05, "rope_scaling": null, "rope_theta": 100000000000, "ssm_in_multiplier": 1, "ssm_multipliers": [ 0.3535533905932738, 0.25, 0.1767766952966369, 0.5, 0.3535533905932738 ], "ssm_out_multiplier": 0.23570226039551587, "tie_word_embeddings": false, "torch_dtype": "bfloat16", "transformers_version": "4.52.0.dev0", "use_cache": true, "vocab_size": 65537 } }, { "id": "tiiuae/Falcon-H1-1.5B-Base", "meta": { "sha": "06d7330266253c20784f58b0a846dc09a9d12cee", "safetensors": { "parameters": { "BF16": 1554859392 }, "total": 1554859392 }, "modelType": "falcon_h1" }, "config": { "architectures": [ "FalconH1ForCausalLM" ], "attention_bias": false, "attention_dropout": 0, "attention_in_multiplier": 1, "attention_out_multiplier": 0.234375, "attn_layer_indices": null, "bos_token_id": 1, "embedding_multiplier": 5.656854249492381, "eos_token_id": 11, "head_dim": 128, "hidden_act": "silu", "hidden_size": 2048, "initializer_range": 0.02, "intermediate_size": 4608, "key_multiplier": 0.06905339660024876, "lm_head_multiplier": 0.01953125, "mamba_chunk_size": 128, "mamba_conv_bias": true, "mamba_d_conv": 4, "mamba_d_head": 64, "mamba_d_ssm": 3072, "mamba_d_state": 256, "mamba_expand": 2, "mamba_n_groups": 1, "mamba_n_heads": 48, "mamba_norm_before_gate": false, "mamba_proj_bias": false, "mamba_rms_norm": true, "mamba_use_mlp": true, "max_position_embeddings": 131072, "mlp_bias": false, "mlp_expansion_factor": 8, "mlp_multipliers": [ 0.4419417382415922, 0.13020833333333331 ], "model_type": "falcon_h1", "num_attention_heads": 8, "num_hidden_layers": 24, "num_key_value_heads": 2, "num_logits_to_keep": 1, "pad_token_id": 0, "projectors_bias": false, "rms_norm_eps": 1e-05, "rope_scaling": null, "rope_theta": 100000000000, "ssm_in_multiplier": 0.625, "ssm_multipliers": [ 0.3535533905932738, 0.25, 0.1767766952966369, 0.5, 0.3535533905932738 ], "ssm_out_multiplier": 0.11785113019775793, "tie_word_embeddings": false, "torch_dtype": "bfloat16", "transformers_version": "4.52.0.dev0", "use_cache": true, "vocab_size": 65536 } }, { "id": "tiiuae/Falcon3-10B-Instruct", "meta": { "sha": "8799bc6aec0152757221dc6b272d824642db6202", "safetensors": { "parameters": { "BF16": 10305653760 }, "total": 10305653760 }, "modelType": "llama" }, "config": { "architectures": [ "LlamaForCausalLM" ], "attention_bias": false, "attention_dropout": 0, "bos_token_id": 11, "eos_token_id": 11, "head_dim": 256, "hidden_act": "silu", "hidden_size": 3072, "intermediate_size": 23040, "max_position_embeddings": 32768, "mlp_bias": false, "model_type": "llama", "num_attention_heads": 12, "num_hidden_layers": 40, "num_key_value_heads": 4, "pretraining_tp": 1, "rms_norm_eps": 1e-06, "rope_scaling": null, "rope_theta": 1000042, "tie_word_embeddings": false, "torch_dtype": "bfloat16", "transformers_version": "4.46.1", "use_cache": true, "vocab_size": 131072 } }, { "id": "tiiuae/Falcon-H1-0.5B-Instruct", "meta": { "sha": "8f2587ca06bff78d8fa1adfccbe8c24d5f86b368", "safetensors": { "parameters": { "BF16": 521411104 }, "total": 521411104 }, "modelType": "falcon_h1" }, "config": { "architectures": [ "FalconH1ForCausalLM" ], "attention_bias": false, "attention_dropout": 0, "attention_in_multiplier": 1, "attention_out_multiplier": 0.9375, "attn_layer_indices": null, "bos_token_id": 1, "embedding_multiplier": 5.656854249492381, "eos_token_id": 11, "head_dim": 64, "hidden_act": "silu", "hidden_size": 1024, "initializer_range": 0.02, "intermediate_size": 2048, "key_multiplier": 0.39062499999999994, "lm_head_multiplier": 0.0390625, "mamba_chunk_size": 128, "mamba_conv_bias": true, "mamba_d_conv": 4, "mamba_d_head": 64, "mamba_d_ssm": 1536, "mamba_d_state": 128, "mamba_expand": 2, "mamba_n_groups": 1, "mamba_n_heads": 24, "mamba_norm_before_gate": false, "mamba_proj_bias": false, "mamba_rms_norm": false, "mamba_use_mlp": true, "max_position_embeddings": 16384, "mlp_bias": false, "mlp_expansion_factor": 8, "mlp_multipliers": [ 0.8838834764831844, 0.5859375 ], "model_type": "falcon_h1", "num_attention_heads": 8, "num_hidden_layers": 36, "num_key_value_heads": 2, "num_logits_to_keep": 1, "pad_token_id": 0, "projectors_bias": false, "rms_norm_eps": 1e-05, "rope_scaling": null, "rope_theta": 100000000000, "ssm_in_multiplier": 1.25, "ssm_multipliers": [ 0.3535533905932738, 0.25, 0.3535533905932738, 0.5, 0.3535533905932738 ], "ssm_out_multiplier": 0.23570226039551587, "tie_word_embeddings": false, "torch_dtype": "bfloat16", "transformers_version": "4.52.0.dev0", "use_cache": true, "vocab_size": 32784 } }, { "id": "tiiuae/falcon-rw-1b", "meta": { "sha": "e4b9872bb803165eb22f0a867d4e6a64d34fce19", "modelType": "falcon" }, "config": { "alibi": true, "apply_residual_connection_post_layernorm": false, "architectures": [ "FalconForCausalLM" ], "attention_dropout": 0, "auto_map": { "AutoConfig": "configuration_falcon.FalconConfig", "AutoModel": "modeling_falcon.FalconModel", "AutoModelForSequenceClassification": "modeling_falcon.FalconForSequenceClassification", "AutoModelForTokenClassification": "modeling_falcon.FalconForTokenClassification", "AutoModelForQuestionAnswering": "modeling_falcon.FalconForQuestionAnswering", "AutoModelForCausalLM": "modeling_falcon.FalconForCausalLM" }, "bias": true, "bos_token_id": 1, "eos_token_id": 2, "hidden_dropout": 0, "hidden_size": 2048, "initializer_range": 0.02, "layer_norm_epsilon": 1e-05, "model_type": "falcon", "multi_query": false, "new_decoder_architecture": false, "num_attention_heads": 32, "num_hidden_layers": 24, "parallel_attn": false, "torch_dtype": "bfloat16", "transformers_version": "4.27.4", "use_cache": true, "vocab_size": 50304 } }, { "id": "tiiuae/Falcon3-1B-Instruct", "meta": { "sha": "28ba2251970a01dd1edc7ba7dad2eb71216ccfdf", "safetensors": { "parameters": { "BF16": 1669408768 }, "total": 1669408768 }, "modelType": "llama" }, "config": { "architectures": [ "LlamaForCausalLM" ], "attention_bias": false, "attention_dropout": 0, "eos_token_id": 11, "head_dim": 256, "hidden_act": "silu", "hidden_size": 2048, "initializer_range": 0.02, "intermediate_size": 8192, "max_position_embeddings": 8192, "mlp_bias": false, "model_type": "llama", "num_attention_heads": 8, "num_hidden_layers": 18, "num_key_value_heads": 4, "pretraining_tp": 1, "rms_norm_eps": 1e-06, "rope_scaling": null, "rope_theta": 1000042, "tie_word_embeddings": false, "torch_dtype": "bfloat16", "transformers_version": "4.46.1", "use_cache": true, "vocab_size": 131072 } }, { "id": "tiiuae/Falcon3-3B-Instruct", "meta": { "sha": "411bb94318f94f7a5735b77109f456b1e74b42a1", "safetensors": { "parameters": { "BF16": 3227655168 }, "total": 3227655168 }, "modelType": "llama" }, "config": { "architectures": [ "LlamaForCausalLM" ], "attention_bias": false, "attention_dropout": 0, "eos_token_id": 11, "head_dim": 256, "hidden_act": "silu", "hidden_size": 3072, "initializer_range": 0.02, "intermediate_size": 9216, "max_position_embeddings": 32768, "mlp_bias": false, "model_type": "llama", "num_attention_heads": 12, "num_hidden_layers": 22, "num_key_value_heads": 4, "pretraining_tp": 1, "rms_norm_eps": 1e-06, "rope_scaling": null, "rope_theta": 1000042, "tie_word_embeddings": false, "torch_dtype": "bfloat16", "transformers_version": "4.46.1", "use_cache": true, "vocab_size": 131072 } }, { "id": "tiiuae/Falcon-H1-3B-Base", "meta": { "sha": "c096902c69be0eed2e5369d3420a8bb960d293e1", "safetensors": { "parameters": { "BF16": 3149387264 }, "total": 3149387264 }, "modelType": "falcon_h1" }, "config": { "architectures": [ "FalconH1ForCausalLM" ], "attention_bias": false, "attention_dropout": 0, "attention_in_multiplier": 1, "attention_out_multiplier": 0.15, "attn_layer_indices": null, "bos_token_id": 1, "embedding_multiplier": 5.656854249492381, "eos_token_id": 11, "head_dim": 128, "hidden_act": "silu", "hidden_size": 2560, "initializer_range": 0.02, "intermediate_size": 6144, "key_multiplier": 0.044194173824159216, "lm_head_multiplier": 0.015625, "mamba_chunk_size": 128, "mamba_conv_bias": true, "mamba_d_conv": 4, "mamba_d_head": 128, "mamba_d_ssm": 4096, "mamba_d_state": 256, "mamba_expand": 2, "mamba_n_groups": 1, "mamba_n_heads": 32, "mamba_norm_before_gate": false, "mamba_proj_bias": false, "mamba_rms_norm": true, "mamba_use_mlp": true, "max_position_embeddings": 131072, "mlp_bias": false, "mlp_expansion_factor": 8, "mlp_multipliers": [ 0.3535533905932738, 0.078125 ], "model_type": "falcon_h1", "num_attention_heads": 10, "num_hidden_layers": 32, "num_key_value_heads": 2, "num_logits_to_keep": 1, "pad_token_id": 0, "projectors_bias": false, "rms_norm_eps": 1e-05, "rope_scaling": null, "rope_theta": 100000000000, "ssm_in_multiplier": 0.5, "ssm_multipliers": [ 0.3535533905932738, 0.25, 0.1767766952966369, 0.5, 0.3535533905932738 ], "ssm_out_multiplier": 0.08838834764831845, "tie_word_embeddings": false, "torch_dtype": "bfloat16", "transformers_version": "4.52.0.dev0", "use_cache": true, "vocab_size": 65536 } }, { "id": "tiiuae/Falcon3-1B-Base", "meta": { "sha": "cb37ef3559b157b5c9d9226296ba01a5162da1f7", "safetensors": { "parameters": { "BF16": 1669408768 }, "total": 1669408768 }, "modelType": "llama" }, "config": { "architectures": [ "LlamaForCausalLM" ], "activation": "swiglu", "attention_bias": false, "attention_dropout": 0, "eos_token_id": 11, "head_dim": 256, "hidden_act": "silu", "hidden_size": 2048, "intermediate_size": 8192, "max_position_embeddings": 4096, "mlp_bias": false, "model_type": "llama", "num_attention_heads": 8, "num_hidden_layers": 18, "num_key_value_heads": 4, "pretraining_tp": 1, "rms_norm_eps": 1e-06, "rope_scaling": null, "rope_theta": 1000042, "tie_word_embeddings": false, "torch_dtype": "bfloat16", "transformers_version": "4.46.1", "use_cache": true, "vocab_size": 131072 } }, { "id": "tiiuae/falcon-40b", "meta": { "sha": "05ab2ee8d6b593bdbab17d728de5c028a7a94d83", "safetensors": { "parameters": { "BF16": 41835970560 }, "total": 41835970560 }, "modelType": "falcon" }, "config": { "alibi": false, "apply_residual_connection_post_layernorm": false, "architectures": [ "FalconForCausalLM" ], "attention_dropout": 0.0, "auto_map": { "AutoConfig": "configuration_falcon.FalconConfig", "AutoModel": "modeling_falcon.FalconModel", "AutoModelForSequenceClassification": "modeling_falcon.FalconForSequenceClassification", "AutoModelForTokenClassification": "modeling_falcon.FalconForTokenClassification", "AutoModelForQuestionAnswering": "modeling_falcon.FalconForQuestionAnswering", "AutoModelForCausalLM": "modeling_falcon.FalconForCausalLM" }, "bias": false, "bos_token_id": 11, "eos_token_id": 11, "hidden_dropout": 0.0, "hidden_size": 8192, "initializer_range": 0.02, "layer_norm_epsilon": 1e-05, "model_type": "falcon", "new_decoder_architecture": true, "num_attention_heads": 128, "num_hidden_layers": 60, "num_kv_heads": 8, "parallel_attn": true, "torch_dtype": "bfloat16", "transformers_version": "4.27.4", "use_cache": true, "vocab_size": 65024 }, "contextSource": "https://huggingface.co/tiiuae/falcon-40b", "verifiedAt": "2026-09-22" }, { "id": "tiiuae/Falcon-H1-3B-Instruct", "meta": { "sha": "01087ec4c132d7f186908716b3530ea187f062a1", "safetensors": { "parameters": { "BF16": 3149392384 }, "total": 3149392384 }, "modelType": "falcon_h1" }, "config": { "architectures": [ "FalconH1ForCausalLM" ], "attention_bias": false, "attention_dropout": 0, "attention_in_multiplier": 1, "attention_out_multiplier": 0.15, "attn_layer_indices": null, "bos_token_id": 1, "embedding_multiplier": 5.656854249492381, "eos_token_id": 11, "head_dim": 128, "hidden_act": "silu", "hidden_size": 2560, "initializer_range": 0.02, "intermediate_size": 6144, "key_multiplier": 0.044194173824159216, "lm_head_multiplier": 0.015625, "mamba_chunk_size": 128, "mamba_conv_bias": true, "mamba_d_conv": 4, "mamba_d_head": 128, "mamba_d_ssm": 4096, "mamba_d_state": 256, "mamba_expand": 2, "mamba_n_groups": 1, "mamba_n_heads": 32, "mamba_norm_before_gate": false, "mamba_proj_bias": false, "mamba_rms_norm": true, "mamba_use_mlp": true, "max_position_embeddings": 131072, "mlp_bias": false, "mlp_expansion_factor": 8, "mlp_multipliers": [ 0.3535533905932738, 0.078125 ], "model_type": "falcon_h1", "num_attention_heads": 10, "num_hidden_layers": 32, "num_key_value_heads": 2, "num_logits_to_keep": 1, "pad_token_id": 0, "projectors_bias": false, "rms_norm_eps": 1e-05, "rope_scaling": null, "rope_theta": 100000000000, "ssm_in_multiplier": 0.5, "ssm_multipliers": [ 0.3535533905932738, 0.25, 0.1767766952966369, 0.5, 0.3535533905932738 ], "ssm_out_multiplier": 0.08838834764831845, "tie_word_embeddings": false, "torch_dtype": "bfloat16", "transformers_version": "4.52.0.dev0", "use_cache": true, "vocab_size": 65537 } }, { "id": "tiiuae/falcon-40b-instruct", "meta": { "sha": "ecb78d97ac356d098e79f0db222c9ce7c5d9ee5f", "modelType": "falcon" }, "config": { "alibi": false, "apply_residual_connection_post_layernorm": false, "architectures": [ "FalconForCausalLM" ], "attention_dropout": 0, "auto_map": { "AutoConfig": "configuration_falcon.FalconConfig", "AutoModel": "modeling_falcon.FalconModel", "AutoModelForSequenceClassification": "modeling_falcon.FalconForSequenceClassification", "AutoModelForTokenClassification": "modeling_falcon.FalconForTokenClassification", "AutoModelForQuestionAnswering": "modeling_falcon.FalconForQuestionAnswering", "AutoModelForCausalLM": "modeling_falcon.FalconForCausalLM" }, "bias": false, "bos_token_id": 11, "eos_token_id": 11, "hidden_dropout": 0, "hidden_size": 8192, "initializer_range": 0.02, "layer_norm_epsilon": 1e-05, "model_type": "falcon", "new_decoder_architecture": true, "num_attention_heads": 128, "num_hidden_layers": 60, "num_kv_heads": 8, "parallel_attn": true, "torch_dtype": "bfloat16", "transformers_version": "4.26.0", "use_cache": true, "vocab_size": 65024 } }, { "id": "tiiuae/Falcon3-7B-Base", "meta": { "sha": "bf3d7ed586cb22a921520e2d681a9d3d7642cde8", "safetensors": { "parameters": { "BF16": 7455550464 }, "total": 7455550464 }, "modelType": "llama" }, "config": { "architectures": [ "LlamaForCausalLM" ], "attention_bias": false, "attention_dropout": 0, "eos_token_id": 11, "head_dim": 256, "hidden_act": "silu", "hidden_size": 3072, "initializer_range": 0.02, "intermediate_size": 23040, "max_position_embeddings": 32768, "mlp_bias": false, "model_type": "llama", "num_attention_heads": 12, "num_hidden_layers": 28, "num_key_value_heads": 4, "pretraining_tp": 1, "rms_norm_eps": 1e-06, "rope_scaling": null, "rope_theta": 1000042, "tie_word_embeddings": false, "torch_dtype": "bfloat16", "transformers_version": "4.46.1", "use_cache": true, "vocab_size": 131072 } }, { "id": "tiiuae/falcon-11B", "meta": { "sha": "a4ab868d7e8d53844009657bf7e755933ccccf8f", "safetensors": { "parameters": { "BF16": 11102822400 }, "total": 11102822400 }, "modelType": "falcon" }, "config": { "_name_or_path": "tiiuae/falcon-11B", "activation": "gelu", "alibi": false, "architectures": [ "FalconForCausalLM" ], "attention_dropout": 0.0, "auto_map": { "AutoConfig": "tiiuae/falcon-11B--configuration_falcon.FalconConfig", "AutoModel": "tiiuae/falcon-11B--modeling_falcon.FalconModel", "AutoModelForCausalLM": "tiiuae/falcon-11B--modeling_falcon.FalconForCausalLM", "AutoModelForQuestionAnswering": "tiiuae/falcon-11B--modeling_falcon.FalconForQuestionAnswering", "AutoModelForSequenceClassification": "tiiuae/falcon-11B--modeling_falcon.FalconForSequenceClassification", "AutoModelForTokenClassification": "tiiuae/falcon-11B--modeling_falcon.FalconForTokenClassification" }, "bias": false, "bos_token_id": 11, "eos_token_id": 11, "ff_factor": 4, "ffn_hidden_size": 16384, "hidden_dropout": 0.0, "hidden_size": 4096, "initializer_range": 0.02, "layer_norm_epsilon": 1e-05, "max_position_embeddings": 8192, "model_type": "falcon", "multi_query": true, "new_decoder_architecture": true, "num_attention_heads": 32, "num_hidden_layers": 60, "num_kv_heads": 8, "num_ln_in_parallel_attn": 1, "parallel_attn": true, "rope_scaling": null, "rope_theta": 500042.0, "tie_word_embeddings": false, "torch_dtype": "bfloat16", "transformers_version": "4.39.2", "use_cache": true, "vocab_size": 65024 }, "verifiedAt": "2026-09-22" }, { "id": "tiiuae/Falcon-H1-7B-Base", "meta": { "sha": "c9a4cbb95c01b1ede39f69eda083d03d8903b8f0", "safetensors": { "parameters": { "BF16": 7585648736 }, "total": 7585648736 }, "modelType": "falcon_h1" }, "config": { "architectures": [ "FalconH1ForCausalLM" ], "attention_bias": false, "attention_dropout": 0, "attention_in_multiplier": 1, "attention_out_multiplier": 0.10416666666666669, "attn_layer_indices": null, "bos_token_id": 1, "embedding_multiplier": 5.656854249492381, "eos_token_id": 11, "head_dim": 128, "hidden_act": "silu", "hidden_size": 3072, "initializer_range": 0.02, "intermediate_size": 12288, "key_multiplier": 0.030690398488999456, "lm_head_multiplier": 0.013020833333333334, "mamba_chunk_size": 128, "mamba_conv_bias": true, "mamba_d_conv": 4, "mamba_d_head": 128, "mamba_d_ssm": 3072, "mamba_d_state": 256, "mamba_expand": 2, "mamba_n_groups": 1, "mamba_n_heads": 24, "mamba_norm_before_gate": false, "mamba_proj_bias": false, "mamba_rms_norm": true, "mamba_use_mlp": true, "max_position_embeddings": 262144, "mlp_bias": false, "mlp_expansion_factor": 8, "mlp_multipliers": [ 0.2946278254943948, 0.032552083333333336 ], "model_type": "falcon_h1", "num_attention_heads": 12, "num_hidden_layers": 44, "num_key_value_heads": 2, "num_logits_to_keep": 1, "pad_token_id": 0, "projectors_bias": false, "rms_norm_eps": 1e-05, "rope_scaling": null, "rope_theta": 100000000000, "ssm_in_multiplier": 0.4166666666666667, "ssm_multipliers": [ 0.3535533905932738, 0.25, 0.1767766952966369, 0.5, 0.3535533905932738 ], "ssm_out_multiplier": 0.11785113019775792, "tie_word_embeddings": false, "torch_dtype": "bfloat16", "transformers_version": "4.52.0.dev0", "use_cache": true, "vocab_size": 130048 } }, { "id": "tiiuae/Falcon3-Mamba-7B-Instruct", "meta": { "sha": "79268d5c8e650ec0ec24aad2729bfc906f569580", "safetensors": { "parameters": { "BF16": 7272665088 }, "total": 7272665088 }, "modelType": "falcon_mamba" }, "config": { "architectures": [ "FalconMambaForCausalLM" ], "bos_token_id": 0, "conv_kernel": 4, "eos_token_id": 0, "expand": 2, "hidden_act": "silu", "hidden_size": 4096, "initializer_range": 0.1, "intermediate_size": 8192, "layer_norm_epsilon": 1e-05, "mixer_rms_eps": 1e-05, "model_type": "falcon_mamba", "num_hidden_layers": 64, "pad_token_id": 0, "rescale_prenorm_residual": false, "residual_in_fp32": true, "state_size": 16, "tie_word_embeddings": false, "time_step_floor": 0.0001, "time_step_init_scheme": "random", "time_step_max": 0.1, "time_step_min": 0.001, "time_step_rank": 256, "time_step_scale": 1, "torch_dtype": "bfloat16", "transformers_version": "4.47.0", "use_bias": false, "use_cache": true, "use_conv_bias": true, "use_mambapy": false, "vocab_size": 65024 } }, { "id": "tiiuae/Falcon3-3B-Base", "meta": { "sha": "092c29e3114ff551bd06b93b85424cdb525d227c", "safetensors": { "parameters": { "BF16": 3227655168 }, "total": 3227655168 }, "modelType": "llama" }, "config": { "architectures": [ "LlamaForCausalLM" ], "activation": "swiglu", "attention_bias": false, "attention_dropout": 0, "eos_token_id": 11, "head_dim": 256, "hidden_act": "silu", "hidden_size": 3072, "intermediate_size": 9216, "max_position_embeddings": 32768, "mlp_bias": false, "model_type": "llama", "num_attention_heads": 12, "num_hidden_layers": 22, "num_key_value_heads": 4, "pretraining_tp": 1, "rms_norm_eps": 1e-06, "rope_scaling": null, "rope_theta": 1000042, "tie_word_embeddings": false, "torch_dtype": "bfloat16", "transformers_version": "4.46.1", "use_cache": true, "vocab_size": 131072 } }, { "id": "tiiuae/Falcon3-10B-Base", "meta": { "sha": "34bb99a889fe0426412da3dd2b46e6f64c8fd003", "safetensors": { "parameters": { "BF16": 10305653760 }, "total": 10305653760 }, "modelType": "llama" }, "config": { "architectures": [ "LlamaForCausalLM" ], "attention_bias": false, "attention_dropout": 0, "eos_token_id": 11, "head_dim": 256, "hidden_act": "silu", "hidden_size": 3072, "initializer_range": 0.02, "intermediate_size": 23040, "max_position_embeddings": 32768, "mlp_bias": false, "model_type": "llama", "num_attention_heads": 12, "num_hidden_layers": 40, "num_key_value_heads": 4, "pretraining_tp": 1, "rms_norm_eps": 1e-06, "rope_scaling": null, "rope_theta": 1000042, "tie_word_embeddings": false, "torch_dtype": "bfloat16", "transformers_version": "4.46.1", "use_cache": true, "vocab_size": 131072 } }, { "id": "tiiuae/Falcon-H1-34B-Instruct", "meta": { "sha": "6e6890e26cba1f58b1e6ee70d654ddf054f9fce8", "safetensors": { "parameters": { "BF16": 33642516224 }, "total": 33642516224 }, "modelType": "falcon_h1" }, "config": { "architectures": [ "FalconH1ForCausalLM" ], "attention_bias": false, "attention_dropout": 0, "attention_in_multiplier": 1, "attention_out_multiplier": 0.0375, "attn_layer_indices": null, "bos_token_id": 1, "embedding_multiplier": 5.656854249492381, "eos_token_id": 11, "head_dim": 128, "hidden_act": "silu", "hidden_size": 5120, "initializer_range": 0.02, "intermediate_size": 21504, "key_multiplier": 0.011048543456039804, "lm_head_multiplier": 0.0078125, "mamba_chunk_size": 128, "mamba_conv_bias": true, "mamba_d_conv": 4, "mamba_d_head": 128, "mamba_d_ssm": 4096, "mamba_d_state": 256, "mamba_expand": 2, "mamba_n_groups": 2, "mamba_n_heads": 32, "mamba_norm_before_gate": false, "mamba_proj_bias": false, "mamba_rms_norm": true, "mamba_use_mlp": true, "max_position_embeddings": 262144, "mlp_bias": false, "mlp_expansion_factor": 8, "mlp_multipliers": [ 0.1767766952966369, 0.011160714285714284 ], "model_type": "falcon_h1", "num_attention_heads": 20, "num_hidden_layers": 72, "num_key_value_heads": 4, "num_logits_to_keep": 1, "pad_token_id": 0, "projectors_bias": false, "rms_norm_eps": 1e-05, "rope_scaling": null, "rope_theta": 100000000000, "ssm_in_multiplier": 0.25, "ssm_multipliers": [ 0.3535533905932738, 0.25, 0.1767766952966369, 0.5, 0.3535533905932738 ], "ssm_out_multiplier": 0.08838834764831845, "tie_word_embeddings": false, "torch_dtype": "bfloat16", "transformers_version": "4.52.0.dev0", "use_cache": true, "vocab_size": 261120 } }, { "id": "tiiuae/Falcon-H1-1.5B-Instruct", "meta": { "sha": "80ebc50d7799a440b96c93bb6686a3924a09b0cb", "safetensors": { "parameters": { "BF16": 1554863488 }, "total": 1554863488 }, "modelType": "falcon_h1" }, "config": { "architectures": [ "FalconH1ForCausalLM" ], "attention_bias": false, "attention_dropout": 0, "attention_in_multiplier": 1, "attention_out_multiplier": 0.234375, "attn_layer_indices": null, "bos_token_id": 1, "embedding_multiplier": 5.656854249492381, "eos_token_id": 11, "head_dim": 128, "hidden_act": "silu", "hidden_size": 2048, "initializer_range": 0.02, "intermediate_size": 4608, "key_multiplier": 0.06905339660024876, "lm_head_multiplier": 0.01953125, "mamba_chunk_size": 128, "mamba_conv_bias": true, "mamba_d_conv": 4, "mamba_d_head": 64, "mamba_d_ssm": 3072, "mamba_d_state": 256, "mamba_expand": 2, "mamba_n_groups": 1, "mamba_n_heads": 48, "mamba_norm_before_gate": false, "mamba_proj_bias": false, "mamba_rms_norm": true, "mamba_use_mlp": true, "max_position_embeddings": 131072, "mlp_bias": false, "mlp_expansion_factor": 8, "mlp_multipliers": [ 0.4419417382415922, 0.13020833333333331 ], "model_type": "falcon_h1", "num_attention_heads": 8, "num_hidden_layers": 24, "num_key_value_heads": 2, "num_logits_to_keep": 1, "pad_token_id": 0, "projectors_bias": false, "rms_norm_eps": 1e-05, "rope_scaling": null, "rope_theta": 100000000000, "ssm_in_multiplier": 0.625, "ssm_multipliers": [ 0.3535533905932738, 0.25, 0.1767766952966369, 0.5, 0.3535533905932738 ], "ssm_out_multiplier": 0.11785113019775793, "tie_word_embeddings": false, "torch_dtype": "bfloat16", "transformers_version": "4.52.0.dev0", "use_cache": true, "vocab_size": 65537 } }, { "id": "tiiuae/Falcon-H1R-7B", "meta": { "sha": "a6f74bf181389908efd6970878d7ee2b42f5d417", "safetensors": { "parameters": { "BF16": 7585648736 }, "total": 7585648736 }, "modelType": "falcon_h1" }, "config": { "architectures": [ "FalconH1ForCausalLM" ], "attention_bias": false, "attention_dropout": 0, "attention_in_multiplier": 1, "attention_out_multiplier": 0.10416666666666669, "attn_layer_indices": null, "bos_token_id": 1, "embedding_multiplier": 5.656854249492381, "eos_token_id": 11, "head_dim": 128, "hidden_act": "silu", "hidden_size": 3072, "initializer_range": 0.02, "intermediate_size": 12288, "key_multiplier": 0.030690398488999456, "lm_head_multiplier": 0.013020833333333334, "mamba_chunk_size": 128, "mamba_conv_bias": true, "mamba_d_conv": 4, "mamba_d_head": 128, "mamba_d_ssm": 3072, "mamba_d_state": 256, "mamba_expand": 2, "mamba_n_groups": 1, "mamba_n_heads": 24, "mamba_norm_before_gate": false, "mamba_proj_bias": false, "mamba_rms_norm": true, "mamba_use_mlp": true, "max_position_embeddings": 262144, "mlp_bias": false, "mlp_expansion_factor": 8, "mlp_multipliers": [ 0.2946278254943948, 0.032552083333333336 ], "model_type": "falcon_h1", "num_attention_heads": 12, "num_hidden_layers": 44, "num_key_value_heads": 2, "num_logits_to_keep": 1, "pad_token_id": 0, "projectors_bias": false, "rms_norm_eps": 1e-05, "rope_scaling": null, "rope_theta": 100000000000, "ssm_in_multiplier": 0.4166666666666667, "ssm_multipliers": [ 0.3535533905932738, 0.25, 0.1767766952966369, 0.5, 0.3535533905932738 ], "ssm_out_multiplier": 0.11785113019775792, "tie_word_embeddings": false, "torch_dtype": "bfloat16", "transformers_version": "4.52.0.dev0", "use_cache": true, "vocab_size": 130048 } }, { "id": "tiiuae/Falcon-H1-Tiny-90M-Instruct", "meta": { "sha": "e6389502a0b12cd8da894b395ba5bf7436873b16", "safetensors": { "parameters": { "BF16": 91131072 }, "total": 91131072 }, "modelType": "falcon_h1" }, "config": { "architectures": [ "FalconH1ForCausalLM" ], "attention_bias": false, "attention_dropout": 0, "attention_in_multiplier": 1, "attention_out_multiplier": 1, "attn_layer_indices": null, "bos_token_id": 1, "dtype": "bfloat16", "embedding_multiplier": 0.11083984375, "eos_token_id": 11, "expansion_factor": 1.5, "head_dim": 64, "hidden_act": "silu", "hidden_size": 512, "initializer_range": 0.02, "intermediate_size": 768, "key_multiplier": 1, "lm_head_multiplier": 0.078125, "mamba_chunk_size": 128, "mamba_conv_bias": true, "mamba_d_conv": 4, "mamba_d_head": 32, "mamba_d_ssm": 768, "mamba_d_state": 64, "mamba_expand": 2, "mamba_n_groups": 1, "mamba_n_heads": 24, "mamba_norm_before_gate": false, "mamba_proj_bias": false, "mamba_rms_norm": false, "mamba_use_mlp": true, "max_position_embeddings": 262144, "mlp_bias": false, "mlp_expansion_factor": 8, "mlp_multipliers": [ 1, 1 ], "model_type": "falcon_h1", "num_attention_heads": 8, "num_hidden_layers": 24, "num_key_value_heads": 2, "num_logits_to_keep": 1, "pad_token_id": 0, "projectors_bias": false, "rms_norm_eps": 1e-05, "rope_scaling": null, "rope_theta": 100000000000, "sliding_window": null, "ssm_in_multiplier": 1, "ssm_multipliers": [ 1, 1, 1, 1, 1 ], "ssm_out_multiplier": 1, "tie_word_embeddings": true, "time_step_floor": 0.0001, "time_step_max": 0.1, "time_step_min": 0.001, "time_step_rank": "auto", "transformers_version": "4.57.0", "use_cache": true, "vocab_size": 32768 } }, { "id": "01-ai/Yi-34B-Chat", "meta": { "sha": "cf02cb50f2a03dead2fe205766a1c5598a90bf80", "safetensors": { "parameters": { "BF16": 34388917248 }, "total": 34388917248 }, "modelType": "llama" }, "config": { "architectures": [ "LlamaForCausalLM" ], "attention_bias": false, "bos_token_id": 1, "eos_token_id": 2, "hidden_act": "silu", "hidden_size": 7168, "initializer_range": 0.02, "intermediate_size": 20480, "max_position_embeddings": 4096, "model_type": "llama", "num_attention_heads": 56, "num_hidden_layers": 60, "num_key_value_heads": 8, "pretraining_tp": 1, "rms_norm_eps": 1e-05, "rope_scaling": null, "rope_theta": 5000000, "tie_word_embeddings": false, "torch_dtype": "bfloat16", "transformers_version": "4.35.0", "use_cache": true, "vocab_size": 64000 } }, { "id": "01-ai/Yi-6B-Chat", "meta": { "sha": "2dbf63b0cb7bc493c0243502c6e6111a36e3a093", "safetensors": { "parameters": { "BF16": 6061035520 }, "total": 6061035520 }, "modelType": "llama" }, "config": { "architectures": [ "LlamaForCausalLM" ], "attention_bias": false, "bos_token_id": 1, "eos_token_id": 2, "hidden_act": "silu", "hidden_size": 4096, "initializer_range": 0.02, "intermediate_size": 11008, "max_position_embeddings": 4096, "model_type": "llama", "num_attention_heads": 32, "num_hidden_layers": 32, "num_key_value_heads": 4, "pretraining_tp": 1, "rms_norm_eps": 1e-05, "rope_scaling": null, "rope_theta": 5000000, "tie_word_embeddings": false, "torch_dtype": "bfloat16", "transformers_version": "4.35.0", "use_cache": true, "vocab_size": 64000 } }, { "id": "01-ai/Yi-6B-200K", "meta": { "sha": "46b2762d8a78344292598f3ced6af746cec84bec", "safetensors": { "parameters": { "BF16": 6061037568 }, "total": 6061037568 }, "modelType": "llama" }, "config": { "architectures": [ "LlamaForCausalLM" ], "bos_token_id": 1, "eos_token_id": 2, "hidden_act": "silu", "hidden_size": 4096, "initializer_range": 0.02, "intermediate_size": 11008, "max_position_embeddings": 200000, "model_type": "llama", "num_attention_heads": 32, "num_hidden_layers": 32, "num_key_value_heads": 4, "pad_token_id": 0, "pretraining_tp": 1, "rms_norm_eps": 1e-05, "rope_scaling": null, "rope_theta": 10000000, "tie_word_embeddings": false, "torch_dtype": "bfloat16", "transformers_version": "4.34.0", "use_cache": true, "vocab_size": 64000 } }, { "id": "01-ai/Yi-1.5-9B-Chat", "meta": { "sha": "1a0fc698cf883c4f5c325f026ca79f0ebd9955a5", "safetensors": { "parameters": { "BF16": 8829407232 }, "total": 8829407232 }, "modelType": "llama" }, "config": { "architectures": [ "LlamaForCausalLM" ], "attention_bias": false, "attention_dropout": 0, "bos_token_id": 1, "eos_token_id": 2, "hidden_act": "silu", "hidden_size": 4096, "initializer_range": 0.02, "intermediate_size": 11008, "max_position_embeddings": 4096, "model_type": "llama", "num_attention_heads": 32, "num_hidden_layers": 48, "num_key_value_heads": 4, "pad_token_id": 0, "pretraining_tp": 1, "rms_norm_eps": 1e-06, "rope_scaling": null, "rope_theta": 5000000, "tie_word_embeddings": false, "torch_dtype": "bfloat16", "transformers_version": "4.40.0", "use_cache": false, "vocab_size": 64000 } }, { "id": "01-ai/Yi-1.5-34B-Chat", "meta": { "sha": "fa4ffba162f20948bf77c2a30eca952bf0812b7f", "safetensors": { "parameters": { "BF16": 34388917248 }, "total": 34388917248 }, "modelType": "llama" }, "config": { "architectures": [ "LlamaForCausalLM" ], "attention_bias": false, "attention_dropout": 0, "bos_token_id": 1, "eos_token_id": 2, "hidden_act": "silu", "hidden_size": 7168, "initializer_range": 0.02, "intermediate_size": 20480, "max_position_embeddings": 4096, "model_type": "llama", "num_attention_heads": 56, "num_hidden_layers": 60, "num_key_value_heads": 8, "pad_token_id": 0, "pretraining_tp": 1, "rms_norm_eps": 1e-06, "rope_scaling": null, "rope_theta": 5000000, "tie_word_embeddings": false, "torch_dtype": "bfloat16", "transformers_version": "4.40.0", "use_cache": false, "vocab_size": 64000 } }, { "id": "01-ai/Yi-6B", "meta": { "sha": "80080be87ec5a0103f643195f2d9003b8068941b", "safetensors": { "parameters": { "BF16": 6061035520 }, "total": 6061035520 }, "modelType": "llama" }, "config": { "architectures": [ "LlamaForCausalLM" ], "bos_token_id": 1, "eos_token_id": 2, "hidden_act": "silu", "hidden_size": 4096, "initializer_range": 0.02, "intermediate_size": 11008, "max_position_embeddings": 4096, "model_type": "llama", "num_attention_heads": 32, "num_hidden_layers": 32, "num_key_value_heads": 4, "pad_token_id": 0, "pretraining_tp": 1, "rms_norm_eps": 1e-05, "rope_scaling": null, "rope_theta": 5000000, "tie_word_embeddings": false, "torch_dtype": "bfloat16", "transformers_version": "4.34.0", "use_cache": true, "vocab_size": 64000 } }, { "id": "01-ai/Yi-1.5-9B", "meta": { "sha": "80d5471b1eae28beae33e06eadbd4b48e74d4ce1", "safetensors": { "parameters": { "BF16": 8829407232 }, "total": 8829407232 }, "modelType": "llama" }, "config": { "architectures": [ "LlamaForCausalLM" ], "attention_bias": false, "attention_dropout": 0, "bos_token_id": 1, "eos_token_id": 2, "hidden_act": "silu", "hidden_size": 4096, "initializer_range": 0.02, "intermediate_size": 11008, "max_position_embeddings": 4096, "model_type": "llama", "num_attention_heads": 32, "num_hidden_layers": 48, "num_key_value_heads": 4, "pad_token_id": 0, "pretraining_tp": 1, "rms_norm_eps": 1e-06, "rope_scaling": null, "rope_theta": 5000000, "tie_word_embeddings": false, "torch_dtype": "bfloat16", "transformers_version": "4.37.2", "use_cache": true, "vocab_size": 64000 } }, { "id": "01-ai/Yi-1.5-6B", "meta": { "sha": "157a3d77ab5a8a8f079510eaa25169b547481f06", "safetensors": { "parameters": { "BF16": 6061035520 }, "total": 6061035520 }, "modelType": "llama" }, "config": { "architectures": [ "LlamaForCausalLM" ], "attention_bias": false, "attention_dropout": 0, "bos_token_id": 1, "eos_token_id": 2, "hidden_act": "silu", "hidden_size": 4096, "initializer_range": 0.02, "intermediate_size": 11008, "max_position_embeddings": 4096, "model_type": "llama", "num_attention_heads": 32, "num_hidden_layers": 32, "num_key_value_heads": 4, "pad_token_id": 0, "pretraining_tp": 1, "rms_norm_eps": 1e-06, "rope_scaling": null, "rope_theta": 5000000, "tie_word_embeddings": false, "torch_dtype": "bfloat16", "transformers_version": "4.37.2", "use_cache": true, "vocab_size": 64000 } }, { "id": "01-ai/Yi-34B", "meta": { "sha": "e1fa7c83283f2e1d5a12cb4adea37c3f829e28f8", "safetensors": { "parameters": { "BF16": 34388917248 }, "total": 34388917248 }, "modelType": "llama" }, "config": { "architectures": [ "LlamaForCausalLM" ], "bos_token_id": 1, "eos_token_id": 2, "hidden_act": "silu", "hidden_size": 7168, "initializer_range": 0.02, "intermediate_size": 20480, "max_position_embeddings": 4096, "model_type": "llama", "num_attention_heads": 56, "num_hidden_layers": 60, "num_key_value_heads": 8, "pad_token_id": 0, "pretraining_tp": 1, "rms_norm_eps": 1e-05, "rope_scaling": null, "rope_theta": 5000000, "tie_word_embeddings": false, "torch_dtype": "bfloat16", "transformers_version": "4.34.0", "use_cache": true, "vocab_size": 64000 } }, { "id": "01-ai/Yi-Coder-9B-Chat", "meta": { "sha": "356a1f8d4e4a606d0b879e54191ca809918576b8", "safetensors": { "parameters": { "BF16": 8829407232 }, "total": 8829407232 }, "modelType": "llama" }, "config": { "architectures": [ "LlamaForCausalLM" ], "attention_bias": false, "attention_dropout": 0, "bos_token_id": 1, "eos_token_id": 7, "hidden_act": "silu", "hidden_size": 4096, "initializer_range": 0.02, "intermediate_size": 11008, "max_position_embeddings": 131072, "mlp_bias": false, "model_type": "llama", "num_attention_heads": 32, "num_hidden_layers": 48, "num_key_value_heads": 4, "pad_token_id": 0, "pretraining_tp": 1, "rms_norm_eps": 1e-05, "rope_scaling": null, "rope_theta": 10000000, "tie_word_embeddings": false, "torch_dtype": "bfloat16", "transformers_version": "4.44.0", "use_cache": false, "vocab_size": 64000 } }, { "id": "01-ai/Yi-1.5-9B-Chat-16K", "meta": { "sha": "2f8753bed95bd4fede3fd64f8305428620b4b212", "safetensors": { "parameters": { "BF16": 8829407232 }, "total": 8829407232 }, "modelType": "llama" }, "config": { "architectures": [ "LlamaForCausalLM" ], "attention_bias": false, "attention_dropout": 0, "bos_token_id": 1, "eos_token_id": 2, "hidden_act": "silu", "hidden_size": 4096, "initializer_range": 0.02, "intermediate_size": 11008, "max_position_embeddings": 16384, "model_type": "llama", "num_attention_heads": 32, "num_hidden_layers": 48, "num_key_value_heads": 4, "pretraining_tp": 1, "rms_norm_eps": 1e-06, "rope_scaling": null, "rope_theta": 5000000, "tie_word_embeddings": false, "torch_dtype": "bfloat16", "transformers_version": "4.40.0", "use_cache": false, "vocab_size": 64000 } }, { "id": "01-ai/Yi-1.5-34B", "meta": { "sha": "58a29f6dca2a4eda38821c086c29c1906114aabb", "safetensors": { "parameters": { "BF16": 34388917248 }, "total": 34388917248 }, "modelType": "llama" }, "config": { "architectures": [ "LlamaForCausalLM" ], "attention_bias": false, "attention_dropout": 0, "bos_token_id": 1, "eos_token_id": 2, "hidden_act": "silu", "hidden_size": 7168, "initializer_range": 0.02, "intermediate_size": 20480, "max_position_embeddings": 4096, "model_type": "llama", "num_attention_heads": 56, "num_hidden_layers": 60, "num_key_value_heads": 8, "pad_token_id": 0, "pretraining_tp": 1, "rms_norm_eps": 1e-06, "rope_scaling": null, "rope_theta": 5000000, "tie_word_embeddings": false, "torch_dtype": "bfloat16", "transformers_version": "4.37.2", "use_cache": true, "vocab_size": 64000 } }, { "id": "01-ai/Yi-34B-200K", "meta": { "sha": "e0ae1afac6b69f604556efd441ab4befafb2a835", "safetensors": { "parameters": { "BF16": 34388921088 }, "total": 34388921088 }, "modelType": "llama" }, "config": { "architectures": [ "LlamaForCausalLM" ], "bos_token_id": 1, "eos_token_id": 2, "hidden_act": "silu", "hidden_size": 7168, "initializer_range": 0.02, "intermediate_size": 20480, "max_position_embeddings": 200000, "model_type": "llama", "num_attention_heads": 56, "num_hidden_layers": 60, "num_key_value_heads": 8, "pad_token_id": 0, "pretraining_tp": 1, "rms_norm_eps": 1e-05, "rope_scaling": null, "rope_theta": 10000000, "tie_word_embeddings": false, "torch_dtype": "bfloat16", "transformers_version": "4.34.0", "use_cache": true, "vocab_size": 64000 } }, { "id": "01-ai/Yi-9B", "meta": { "sha": "7b7ca5cf23753f488859321e2439dc4bcbc0eca0", "safetensors": { "parameters": { "BF16": 8829407232 }, "total": 8829407232 }, "modelType": "llama" }, "config": { "architectures": [ "LlamaForCausalLM" ], "attention_bias": false, "attention_dropout": 0, "bos_token_id": 1, "eos_token_id": 2, "hidden_act": "silu", "hidden_size": 4096, "initializer_range": 0.02, "intermediate_size": 11008, "max_position_embeddings": 4096, "model_type": "llama", "num_attention_heads": 32, "num_hidden_layers": 48, "num_key_value_heads": 4, "pad_token_id": 0, "pretraining_tp": 1, "rms_norm_eps": 1e-06, "rope_scaling": null, "rope_theta": 10000, "tie_word_embeddings": false, "torch_dtype": "bfloat16", "transformers_version": "4.37.2", "use_cache": true, "vocab_size": 64000 } }, { "id": "01-ai/Yi-9B-200K", "meta": { "sha": "8efb8b93f2afdd04ec6884352cd0ea727064a95b", "safetensors": { "parameters": { "BF16": 8829407232 }, "total": 8829407232 }, "modelType": "llama" }, "config": { "architectures": [ "LlamaForCausalLM" ], "attention_bias": false, "attention_dropout": 0, "bos_token_id": 1, "eos_token_id": 2, "hidden_act": "silu", "hidden_size": 4096, "initializer_range": 0.02, "intermediate_size": 11008, "max_position_embeddings": 262144, "model_type": "llama", "num_attention_heads": 32, "num_hidden_layers": 48, "num_key_value_heads": 4, "pad_token_id": 0, "pretraining_tp": 1, "rms_norm_eps": 1e-06, "rope_scaling": null, "rope_theta": 10000000, "tie_word_embeddings": false, "torch_dtype": "bfloat16", "transformers_version": "4.37.2", "use_cache": true, "vocab_size": 64000 } }, { "id": "01-ai/Yi-Coder-9B", "meta": { "sha": "bc6e63fd26a654ee42dba737a553310bfa01dc5d", "safetensors": { "parameters": { "BF16": 8829407232 }, "total": 8829407232 }, "modelType": "llama" }, "config": { "architectures": [ "LlamaForCausalLM" ], "attention_bias": false, "attention_dropout": 0, "bos_token_id": 1, "eos_token_id": 2, "hidden_act": "silu", "hidden_size": 4096, "initializer_range": 0.02, "intermediate_size": 11008, "max_position_embeddings": 131072, "mlp_bias": false, "model_type": "llama", "num_attention_heads": 32, "num_hidden_layers": 48, "num_key_value_heads": 4, "pad_token_id": 0, "pretraining_tp": 1, "rms_norm_eps": 1e-05, "rope_scaling": null, "rope_theta": 10000000, "tie_word_embeddings": false, "torch_dtype": "bfloat16", "transformers_version": "4.44.0", "use_cache": true, "vocab_size": 64000 } }, { "id": "01-ai/Yi-1.5-9B-32K", "meta": { "sha": "c0239dbc923b8a2b5ca849763bdd592d39c60850", "safetensors": { "parameters": { "BF16": 8829407232 }, "total": 8829407232 }, "modelType": "llama" }, "config": { "architectures": [ "LlamaForCausalLM" ], "attention_bias": false, "attention_dropout": 0, "bos_token_id": 1, "eos_token_id": 2, "hidden_act": "silu", "hidden_size": 4096, "initializer_range": 0.02, "intermediate_size": 11008, "max_position_embeddings": 32768, "model_type": "llama", "num_attention_heads": 32, "num_hidden_layers": 48, "num_key_value_heads": 4, "pretraining_tp": 1, "rms_norm_eps": 1e-06, "rope_scaling": null, "rope_theta": 5000000, "tie_word_embeddings": false, "torch_dtype": "bfloat16", "transformers_version": "4.37.1", "use_cache": true, "vocab_size": 64000 } }, { "id": "01-ai/Yi-1.5-34B-32K", "meta": { "sha": "05a747585797c119aca9c49965003e99e9490b60", "safetensors": { "parameters": { "BF16": 34388917248 }, "total": 34388917248 }, "modelType": "llama" }, "config": { "architectures": [ "LlamaForCausalLM" ], "attention_bias": false, "attention_dropout": 0, "bos_token_id": 1, "eos_token_id": 2, "hidden_act": "silu", "hidden_size": 7168, "initializer_range": 0.02, "intermediate_size": 20480, "max_position_embeddings": 32768, "model_type": "llama", "num_attention_heads": 56, "num_hidden_layers": 60, "num_key_value_heads": 8, "pretraining_tp": 1, "rms_norm_eps": 1e-06, "rope_scaling": null, "rope_theta": 5000000, "tie_word_embeddings": false, "torch_dtype": "bfloat16", "transformers_version": "4.37.1", "use_cache": true, "vocab_size": 64000 } }, { "id": "01-ai/Yi-1.5-34B-Chat-16K", "meta": { "sha": "ff74452e11f0f749ab872dc19b1dd3813c25c4d8", "safetensors": { "parameters": { "BF16": 34388917248 }, "total": 34388917248 }, "modelType": "llama" }, "config": { "architectures": [ "LlamaForCausalLM" ], "attention_bias": false, "attention_dropout": 0, "bos_token_id": 1, "eos_token_id": 2, "hidden_act": "silu", "hidden_size": 7168, "initializer_range": 0.02, "intermediate_size": 20480, "max_position_embeddings": 16384, "model_type": "llama", "num_attention_heads": 56, "num_hidden_layers": 60, "num_key_value_heads": 8, "pretraining_tp": 1, "rms_norm_eps": 1e-06, "rope_scaling": null, "rope_theta": 5000000, "tie_word_embeddings": false, "torch_dtype": "bfloat16", "transformers_version": "4.40.0", "use_cache": false, "vocab_size": 64000 } }, { "id": "01-ai/Yi-1.5-6B-Chat", "meta": { "sha": "771924d1c83d67527d665913415d7086f11ea9c0", "safetensors": { "parameters": { "BF16": 6061035520 }, "total": 6061035520 }, "modelType": "llama" }, "config": { "architectures": [ "LlamaForCausalLM" ], "attention_bias": false, "attention_dropout": 0, "bos_token_id": 1, "eos_token_id": 2, "hidden_act": "silu", "hidden_size": 4096, "initializer_range": 0.02, "intermediate_size": 11008, "max_position_embeddings": 4096, "model_type": "llama", "num_attention_heads": 32, "num_hidden_layers": 32, "num_key_value_heads": 4, "pad_token_id": 0, "pretraining_tp": 1, "rms_norm_eps": 1e-06, "rope_scaling": null, "rope_theta": 5000000, "tie_word_embeddings": false, "torch_dtype": "bfloat16", "transformers_version": "4.40.0", "use_cache": false, "vocab_size": 64000 } }, { "id": "01-ai/Yi-Coder-1.5B-Chat", "meta": { "sha": "92fdd1b2f1539ac990e7f4a921db5601da2f0299", "safetensors": { "parameters": { "BF16": 1476495360 }, "total": 1476495360 }, "modelType": "llama" }, "config": { "architectures": [ "LlamaForCausalLM" ], "attention_bias": false, "attention_dropout": 0, "bos_token_id": 1, "eos_token_id": 7, "hidden_act": "silu", "hidden_size": 2048, "initializer_range": 0.02, "intermediate_size": 5504, "max_position_embeddings": 131072, "mlp_bias": false, "model_type": "llama", "num_attention_heads": 16, "num_hidden_layers": 24, "num_key_value_heads": 16, "pad_token_id": 0, "pretraining_tp": 1, "rms_norm_eps": 1e-05, "rope_scaling": null, "rope_theta": 10000000, "tie_word_embeddings": false, "torch_dtype": "bfloat16", "transformers_version": "4.44.0", "use_cache": false, "vocab_size": 64000 } }, { "id": "HuggingFaceTB/SmolLM2-135M", "meta": { "sha": "93efa2f097d58c2a74874c7e644dbc9b0cee75a2", "safetensors": { "parameters": { "BF16": 134515008 }, "total": 134515008 }, "modelType": "llama" }, "config": { "architectures": [ "LlamaForCausalLM" ], "attention_bias": false, "attention_dropout": 0, "bos_token_id": 0, "eos_token_id": 0, "hidden_act": "silu", "hidden_size": 576, "initializer_range": 0.041666666666666664, "intermediate_size": 1536, "is_llama_config": true, "max_position_embeddings": 8192, "model_type": "llama", "num_attention_heads": 9, "num_hidden_layers": 30, "num_key_value_heads": 3, "pretraining_tp": 1, "rms_norm_eps": 1e-05, "rope_interleaved": false, "rope_scaling": null, "rope_theta": 100000, "tie_word_embeddings": true, "torch_dtype": "bfloat16", "transformers_version": "4.40.1", "use_cache": true, "vocab_size": 49152 } }, { "id": "HuggingFaceTB/SmolLM2-135M-Instruct", "meta": { "sha": "12fd25f77366fa6b3b4b768ec3050bf629380bac", "safetensors": { "parameters": { "BF16": 134515008 }, "total": 134515008 }, "modelType": "llama" }, "config": { "architectures": [ "LlamaForCausalLM" ], "attention_bias": false, "attention_dropout": 0, "bos_token_id": 1, "eos_token_id": 2, "hidden_act": "silu", "hidden_size": 576, "initializer_range": 0.041666666666666664, "intermediate_size": 1536, "is_llama_config": true, "max_position_embeddings": 8192, "mlp_bias": false, "model_type": "llama", "num_attention_heads": 9, "num_hidden_layers": 30, "num_key_value_heads": 3, "pad_token_id": 2, "pretraining_tp": 1, "rms_norm_eps": 1e-05, "rope_interleaved": false, "rope_scaling": null, "rope_theta": 100000, "tie_word_embeddings": true, "torch_dtype": "bfloat16", "transformers_version": "4.42.3", "transformers.js_config": { "kv_cache_dtype": { "q4f16": "float16", "fp16": "float16" } }, "use_cache": true, "vocab_size": 49152 } }, { "id": "HuggingFaceTB/SmolLM3-3B", "meta": { "sha": "a07cc9a04f16550a088caea529712d1d335b0ac1", "safetensors": { "parameters": { "BF16": 3075098624 }, "total": 3075098624 }, "modelType": "smollm3" }, "config": { "architectures": [ "SmolLM3ForCausalLM" ], "attention_bias": false, "attention_dropout": 0, "bos_token_id": 128000, "eos_token_id": 128012, "hidden_act": "silu", "hidden_size": 2048, "initializer_range": 0.02, "intermediate_size": 11008, "layer_types": [ "full_attention", "full_attention", "full_attention", "full_attention", "full_attention", "full_attention", "full_attention", "full_attention", "full_attention", "full_attention", "full_attention", "full_attention", "full_attention", "full_attention", "full_attention", "full_attention", "full_attention", "full_attention", "full_attention", "full_attention", "full_attention", "full_attention", "full_attention", "full_attention", "full_attention", "full_attention", "full_attention", "full_attention", "full_attention", "full_attention", "full_attention", "full_attention", "full_attention", "full_attention", "full_attention", "full_attention" ], "max_position_embeddings": 65536, "max_window_layers": 28, "mlp_bias": false, "model_type": "smollm3", "no_rope_layer_interval": 4, "no_rope_layers": [ 1, 1, 1, 0, 1, 1, 1, 0, 1, 1, 1, 0, 1, 1, 1, 0, 1, 1, 1, 0, 1, 1, 1, 0, 1, 1, 1, 0, 1, 1, 1, 0, 1, 1, 1, 0 ], "num_attention_heads": 16, "num_hidden_layers": 36, "num_key_value_heads": 4, "pad_token_id": 128004, "pretraining_tp": 2, "rms_norm_eps": 1e-06, "rope_scaling": null, "rope_theta": 5000000, "sliding_window": null, "tie_word_embeddings": true, "torch_dtype": "bfloat16", "transformers_version": "4.54.0.dev0", "use_cache": false, "use_sliding_window": false, "vocab_size": 128256 } }, { "id": "HuggingFaceTB/SmolLM3-3B-Base", "meta": { "sha": "d78a42f79198603e614095753484a04c10c2b940", "safetensors": { "parameters": { "BF16": 3075098624 }, "total": 3075098624 }, "modelType": "smollm3" }, "config": { "architectures": [ "SmolLM3ForCausalLM" ], "attention_bias": false, "attention_dropout": 0, "bos_token_id": null, "eos_token_id": 128001, "hidden_act": "silu", "hidden_size": 2048, "initializer_range": 0.02, "intermediate_size": 11008, "max_position_embeddings": 65536, "model_type": "smollm3", "no_rope_layer_interval": 4, "num_attention_heads": 16, "num_hidden_layers": 36, "num_key_value_heads": 4, "pad_token_id": 128004, "pretraining_tp": 2, "rms_norm_eps": 1e-06, "rope_scaling": null, "rope_theta": 5000000, "sliding_window": null, "tie_word_embeddings": true, "torch_dtype": "bfloat16", "transformers_version": "4.53.0.dev0", "transformers.js_config": { "use_external_data_format": true, "dtype": "q4", "kv_cache_dtype": { "q4f16": "float16", "fp16": "float16" } }, "use_cache": true, "use_sliding_window": false, "vocab_size": 128256 } }, { "id": "HuggingFaceTB/SmolLM2-360M", "meta": { "sha": "f8027fd0eaeea54caa13c31d31b9fdc459c38b49", "safetensors": { "parameters": { "BF16": 361821120 }, "total": 361821120 }, "modelType": "llama" }, "config": { "architectures": [ "LlamaForCausalLM" ], "attention_bias": false, "attention_dropout": 0, "bos_token_id": 0, "eos_token_id": 0, "hidden_act": "silu", "hidden_size": 960, "initializer_range": 0.02, "intermediate_size": 2560, "is_llama_config": true, "max_position_embeddings": 8192, "model_type": "llama", "num_attention_heads": 15, "num_hidden_layers": 32, "num_key_value_heads": 5, "pretraining_tp": 1, "rms_norm_eps": 1e-05, "rope_interleaved": false, "rope_scaling": null, "rope_theta": 100000, "tie_word_embeddings": true, "torch_dtype": "bfloat16", "transformers_version": "4.40.1", "use_cache": true, "vocab_size": 49152 } }, { "id": "HuggingFaceTB/SmolLM2-360M-Instruct", "meta": { "sha": "a10cc1512eabd3dde888204e902eca88bddb4951", "safetensors": { "parameters": { "BF16": 361821120 }, "total": 361821120 }, "modelType": "llama" }, "config": { "architectures": [ "LlamaForCausalLM" ], "attention_bias": false, "attention_dropout": 0, "bos_token_id": 1, "eos_token_id": 2, "hidden_act": "silu", "hidden_size": 960, "initializer_range": 0.02, "intermediate_size": 2560, "is_llama_config": true, "max_position_embeddings": 8192, "mlp_bias": false, "model_type": "llama", "num_attention_heads": 15, "num_hidden_layers": 32, "num_key_value_heads": 5, "pad_token_id": 2, "pretraining_tp": 1, "rms_norm_eps": 1e-05, "rope_interleaved": false, "rope_scaling": null, "rope_theta": 100000, "tie_word_embeddings": true, "torch_dtype": "bfloat16", "transformers_version": "4.42.3", "transformers.js_config": { "kv_cache_dtype": { "q4f16": "float16", "fp16": "float16" } }, "use_cache": true, "vocab_size": 49152 } }, { "id": "HuggingFaceTB/SmolLM2-1.7B-Instruct", "meta": { "sha": "31b70e2e869a7173562077fd711b654946d38674", "safetensors": { "parameters": { "BF16": 1711376384 }, "total": 1711376384 }, "modelType": "llama" }, "config": { "architectures": [ "LlamaForCausalLM" ], "attention_bias": false, "attention_dropout": 0, "bos_token_id": 1, "eos_token_id": 2, "hidden_act": "silu", "hidden_size": 2048, "initializer_range": 0.02, "intermediate_size": 8192, "max_position_embeddings": 8192, "mlp_bias": false, "model_type": "llama", "num_attention_heads": 32, "num_hidden_layers": 24, "num_key_value_heads": 32, "pad_token_id": 2, "pretraining_tp": 1, "rms_norm_eps": 1e-05, "rope_scaling": null, "rope_theta": 130000, "tie_word_embeddings": true, "torch_dtype": "bfloat16", "transformers_version": "4.42.3", "transformers.js_config": { "dtype": "q4", "kv_cache_dtype": { "q4f16": "float16", "fp16": "float16" }, "use_external_data_format": { "model.onnx": true, "model_fp16.onnx": true } }, "use_cache": true, "vocab_size": 49152 } }, { "id": "HuggingFaceTB/SmolLM2-1.7B", "meta": { "sha": "effd688a12921b4cc83e3312b6feb579f70f9c71", "safetensors": { "parameters": { "BF16": 1711376384 }, "total": 1711376384 }, "modelType": "llama" }, "config": { "architectures": [ "LlamaForCausalLM" ], "attention_bias": false, "attention_dropout": 0, "bos_token_id": 0, "eos_token_id": 0, "hidden_act": "silu", "hidden_size": 2048, "initializer_range": 0.02, "intermediate_size": 8192, "max_position_embeddings": 8192, "model_type": "llama", "num_attention_heads": 32, "num_hidden_layers": 24, "num_key_value_heads": 32, "pretraining_tp": 1, "rms_norm_eps": 1e-05, "rope_scaling": null, "rope_theta": 130000, "tie_word_embeddings": true, "torch_dtype": "bfloat16", "transformers_version": "4.40.1", "use_cache": true, "vocab_size": 49152 } }, { "id": "HuggingFaceTB/SmolLM-135M", "meta": { "sha": "1d461723eec654e65efdc40cf49301c89c0c92f4", "safetensors": { "parameters": { "F32": 134515008 }, "total": 134515008 }, "modelType": "llama" }, "config": { "_name_or_path": "/fsx/elie_bakouch/checkpoints/final-149M/600000", "architectures": [ "LlamaForCausalLM" ], "attention_bias": false, "attention_dropout": 0, "bos_token_id": 0, "eos_token_id": 0, "hidden_act": "silu", "hidden_size": 576, "initializer_range": 0.02, "intermediate_size": 1536, "max_position_embeddings": 2048, "mlp_bias": false, "model_type": "llama", "num_attention_heads": 9, "num_hidden_layers": 30, "num_key_value_heads": 3, "pretraining_tp": 1, "rms_norm_eps": 1e-05, "rope_scaling": null, "rope_theta": 10000, "tie_word_embeddings": true, "torch_dtype": "bfloat16", "transformers_version": "4.41.2", "use_cache": true, "vocab_size": 49152 } }, { "id": "HuggingFaceTB/SmolLM-1.7B", "meta": { "sha": "d7449ff7241c863f3e8accc475155f0f97afa011", "safetensors": { "parameters": { "F32": 1711376384 }, "total": 1711376384 }, "modelType": "llama" }, "config": { "_name_or_path": "/fsx/loubna/checkpoints/cosmo2_1T/500000", "architectures": [ "LlamaForCausalLM" ], "attention_bias": false, "attention_dropout": 0, "bos_token_id": 0, "eos_token_id": 0, "hidden_act": "silu", "hidden_size": 2048, "initializer_range": 0.02, "intermediate_size": 8192, "max_position_embeddings": 2048, "model_type": "llama", "num_attention_heads": 32, "num_hidden_layers": 24, "num_key_value_heads": 32, "pretraining_tp": 1, "rms_norm_eps": 1e-05, "rope_scaling": null, "rope_theta": 10000, "tie_word_embeddings": true, "torch_dtype": "float32", "transformers_version": "4.39.3", "use_cache": true, "vocab_size": 49152 } }, { "id": "HuggingFaceTB/SmolLM-135M-Instruct", "meta": { "sha": "fcc320f490e08fdb4b99d935b2c58d40bf35b0d0", "safetensors": { "parameters": { "BF16": 134515008 }, "total": 134515008 }, "modelType": "llama" }, "config": { "_name_or_path": "HuggingFaceTB/SmolLM-135M", "architectures": [ "LlamaForCausalLM" ], "attention_bias": false, "attention_dropout": 0, "bos_token_id": 1, "eos_token_id": 2, "hidden_act": "silu", "hidden_size": 576, "initializer_range": 0.02, "intermediate_size": 1536, "max_position_embeddings": 2048, "mlp_bias": false, "model_type": "llama", "num_attention_heads": 9, "num_hidden_layers": 30, "num_key_value_heads": 3, "pad_token_id": 2, "pretraining_tp": 1, "rms_norm_eps": 1e-05, "rope_scaling": null, "rope_theta": 10000, "tie_word_embeddings": true, "torch_dtype": "bfloat16", "transformers_version": "4.42.3", "use_cache": true, "vocab_size": 49152 } }, { "id": "HuggingFaceTB/SmolLM-1.7B-Instruct", "meta": { "sha": "69f49d9c36434d6a3f319dadc5bd3b812752b98b", "safetensors": { "parameters": { "BF16": 1711376384 }, "total": 1711376384 }, "modelType": "llama" }, "config": { "_name_or_path": "HuggingFaceTB/cosmo2-1.7B-webinst-sc2", "architectures": [ "LlamaForCausalLM" ], "attention_bias": false, "attention_dropout": 0, "bos_token_id": 1, "eos_token_id": 2, "hidden_act": "silu", "hidden_size": 2048, "initializer_range": 0.02, "intermediate_size": 8192, "max_position_embeddings": 2048, "mlp_bias": false, "model_type": "llama", "num_attention_heads": 32, "num_hidden_layers": 24, "num_key_value_heads": 32, "pad_token_id": 2, "pretraining_tp": 1, "rms_norm_eps": 1e-05, "rope_scaling": null, "rope_theta": 10000, "tie_word_embeddings": true, "torch_dtype": "bfloat16", "transformers_version": "4.42.3", "use_cache": true, "vocab_size": 49152 } }, { "id": "HuggingFaceTB/SmolLM-360M", "meta": { "sha": "59f7ef243ee09a72cbc14cb054393a3e3b771d41", "safetensors": { "parameters": { "F32": 361821120 }, "total": 361821120 }, "modelType": "llama" }, "config": { "_name_or_path": "/fsx/elie_bakouch/checkpoints/final-362M/600000", "architectures": [ "LlamaForCausalLM" ], "attention_bias": false, "attention_dropout": 0, "bos_token_id": 0, "eos_token_id": 0, "hidden_act": "silu", "hidden_size": 960, "initializer_range": 0.02, "intermediate_size": 2560, "max_position_embeddings": 2048, "mlp_bias": false, "model_type": "llama", "num_attention_heads": 15, "num_hidden_layers": 32, "num_key_value_heads": 5, "pretraining_tp": 1, "rms_norm_eps": 1e-05, "rope_scaling": null, "rope_theta": 10000, "tie_word_embeddings": true, "torch_dtype": "bfloat16", "transformers_version": "4.41.2", "use_cache": true, "vocab_size": 49152 } }, { "id": "HuggingFaceTB/SmolLM-360M-Instruct", "meta": { "sha": "73b7144f76331266f5f45d5642fd8da653583b13", "safetensors": { "parameters": { "BF16": 361821120 }, "total": 361821120 }, "modelType": "llama" }, "config": { "_name_or_path": "HuggingFaceTB/SmolLM-360M", "architectures": [ "LlamaForCausalLM" ], "attention_bias": false, "attention_dropout": 0, "bos_token_id": 1, "eos_token_id": 2, "hidden_act": "silu", "hidden_size": 960, "initializer_range": 0.02, "intermediate_size": 2560, "max_position_embeddings": 2048, "mlp_bias": false, "model_type": "llama", "num_attention_heads": 15, "num_hidden_layers": 32, "num_key_value_heads": 5, "pad_token_id": 2, "pretraining_tp": 1, "rms_norm_eps": 1e-05, "rope_scaling": null, "rope_theta": 10000, "tie_word_embeddings": true, "torch_dtype": "bfloat16", "transformers_version": "4.42.3", "use_cache": true, "vocab_size": 49152 } }, { "id": "HuggingFaceTB/SmolLM2-1.7B-Instruct-16k", "meta": { "sha": "e6176739e9ab04f510137531200981d8401263f9", "safetensors": { "parameters": { "BF16": 1711376384 }, "total": 1711376384 }, "modelType": "llama" }, "config": { "_name_or_path": "/fsx/loubna/data/SmolLM2-1.7B-Instruct-rope500k", "architectures": [ "LlamaForCausalLM" ], "attention_bias": false, "attention_dropout": 0, "bos_token_id": 1, "eos_token_id": 2, "head_dim": 64, "hidden_act": "silu", "hidden_size": 2048, "initializer_range": 0.02, "intermediate_size": 8192, "max_position_embeddings": 16384, "mlp_bias": false, "model_type": "llama", "num_attention_heads": 32, "num_hidden_layers": 24, "num_key_value_heads": 32, "pad_token_id": 2, "pretraining_tp": 1, "rms_norm_eps": 1e-05, "rope_scaling": null, "rope_theta": 500000, "tie_word_embeddings": true, "torch_dtype": "bfloat16", "transformers.js_config": { "kv_cache_dtype": { "fp16": "float16", "q4f16": "float16" } }, "transformers_version": "4.48.1", "use_cache": false, "vocab_size": 49152 } }, { "id": "ibm-granite/granite-4.1-30b", "meta": { "sha": "4fae6278f7132abf5e971f9de49ebbad09c54cce", "safetensors": { "parameters": { "BF16": 28865728512 }, "total": 28865728512 }, "modelType": "granite" }, "config": { "architectures": [ "GraniteForCausalLM" ], "attention_bias": false, "attention_dropout": 0, "attention_multiplier": 0.0078125, "bos_token_id": 100257, "embedding_multiplier": 12, "eos_token_id": 100257, "hidden_act": "silu", "hidden_size": 4096, "init_method": "mup", "initializer_range": 0.1, "intermediate_size": 32768, "logits_scaling": 16, "max_position_embeddings": 131072, "mlp_bias": false, "model_type": "granite", "num_attention_heads": 32, "num_hidden_layers": 64, "num_key_value_heads": 8, "pad_token_id": 100256, "residual_multiplier": 0.175, "rms_norm_eps": 1e-05, "rope_scaling": null, "rope_theta": 50000000, "tie_word_embeddings": true, "torch_dtype": "bfloat16", "transformers_version": "4.53.3", "use_cache": true, "vocab_size": 100352 } }, { "id": "ibm-granite/granite-4.1-3b", "meta": { "sha": "c0650403e44e78ec0262dab1c90914c65b196c4e", "safetensors": { "parameters": { "BF16": 3402836480 }, "total": 3402836480 }, "modelType": "granite" }, "config": { "architectures": [ "GraniteForCausalLM" ], "attention_bias": false, "attention_dropout": 0, "attention_multiplier": 0.015625, "bos_token_id": 100257, "embedding_multiplier": 12, "eos_token_id": 100257, "hidden_act": "silu", "hidden_size": 2560, "initializer_range": 0.1, "intermediate_size": 8192, "logits_scaling": 10, "max_position_embeddings": 131072, "mlp_bias": false, "model_type": "granite", "num_attention_heads": 40, "num_hidden_layers": 40, "num_key_value_heads": 8, "pad_token_id": 100256, "residual_multiplier": 0.22, "rms_norm_eps": 1e-05, "rope_scaling": null, "rope_theta": 10000000, "tie_word_embeddings": true, "torch_dtype": "bfloat16", "transformers_version": "4.53.3", "use_cache": true, "vocab_size": 100352 } }, { "id": "ibm-granite/granite-4.1-8b", "meta": { "sha": "1504002f650e656a0a3789d99574df12e3e94ed0", "safetensors": { "parameters": { "BF16": 8791592960 }, "total": 8791592960 }, "modelType": "granite" }, "config": { "architectures": [ "GraniteForCausalLM" ], "attention_bias": false, "attention_dropout": 0, "attention_multiplier": 0.0078125, "bos_token_id": 100257, "embedding_multiplier": 12, "eos_token_id": 100257, "hidden_act": "silu", "hidden_size": 4096, "initializer_range": 0.1, "intermediate_size": 12800, "logits_scaling": 16, "max_position_embeddings": 131072, "mlp_bias": false, "model_type": "granite", "num_attention_heads": 32, "num_hidden_layers": 40, "num_key_value_heads": 8, "pad_token_id": 100256, "residual_multiplier": 0.22, "rms_norm_eps": 1e-05, "rope_scaling": null, "rope_theta": 10000000, "tie_word_embeddings": true, "torch_dtype": "bfloat16", "transformers_version": "4.53.3", "use_cache": true, "vocab_size": 100352 } }, { "id": "ibm-granite/granite-4.2-8b", "meta": { "sha": "f8de16cdcdbc6c779ca517604e050d82cc119e44", "safetensors": { "parameters": { "BF16": 8791592960 }, "total": 8791592960 }, "modelType": "granite" }, "config": { "architectures": [ "GraniteForCausalLM" ], "attention_bias": false, "attention_dropout": 0, "attention_multiplier": 0.0078125, "bos_token_id": 100283, "dtype": "bfloat16", "embedding_multiplier": 1, "eos_token_id": 100257, "hidden_act": "silu", "hidden_size": 4096, "initializer_range": 0.1, "intermediate_size": 12800, "logits_scaling": 1, "max_position_embeddings": 131072, "mlp_bias": false, "model_type": "granite", "num_attention_heads": 32, "num_hidden_layers": 40, "num_key_value_heads": 8, "pad_token_id": 100257, "residual_multiplier": 1, "rms_norm_eps": 1e-05, "rope_parameters": { "rope_theta": 10000000, "rope_type": "default" }, "rope_scaling": null, "rope_theta": 10000000, "tie_word_embeddings": false, "transformers_version": "4.57.1", "use_cache": true, "vocab_size": 100352, "torch_dtype": "bfloat16" } }, { "id": "ibm-granite/granite-3.0-1b-a400m-instruct", "meta": { "sha": "ffec3c35bdfd97a06f0b4cd5fcc92cd9b1584445", "safetensors": { "parameters": { "BF16": 1334628352 }, "total": 1334628352 }, "modelType": "granitemoe" }, "config": { "architectures": [ "GraniteMoeForCausalLM" ], "attention_bias": false, "attention_dropout": 0, "attention_multiplier": 0.015625, "bos_token_id": 0, "embedding_multiplier": 12, "eos_token_id": 0, "hidden_act": "silu", "hidden_size": 1024, "initializer_range": 0.02, "intermediate_size": 512, "logits_scaling": 6, "max_position_embeddings": 4096, "model_type": "granitemoe", "num_attention_heads": 16, "num_experts_per_tok": 8, "num_hidden_layers": 24, "num_key_value_heads": 8, "num_local_experts": 32, "output_router_logits": false, "pad_token_id": 0, "residual_multiplier": 0.22, "rms_norm_eps": 1e-06, "rope_scaling": null, "rope_theta": 10000, "router_aux_loss_coef": 0, "tie_word_embeddings": true, "torch_dtype": "bfloat16", "transformers_version": "4.46.0.dev0", "use_cache": true, "vocab_size": 49155 } }, { "id": "ibm-granite/granite-3.3-8b-instruct", "meta": { "sha": "51dd4bc2ade4059a6bd87649d68aa11e4fb2529b", "safetensors": { "parameters": { "BF16": 8170864640 }, "total": 8170864640 }, "modelType": "granite" }, "config": { "architectures": [ "GraniteForCausalLM" ], "attention_bias": false, "attention_dropout": 0, "attention_multiplier": 0.0078125, "bos_token_id": 0, "embedding_multiplier": 12, "eos_token_id": 0, "hidden_act": "silu", "hidden_size": 4096, "initializer_range": 0.02, "intermediate_size": 12800, "logits_scaling": 16, "max_position_embeddings": 131072, "mlp_bias": false, "model_type": "granite", "num_attention_heads": 32, "num_hidden_layers": 40, "num_key_value_heads": 8, "pad_token_id": 0, "residual_multiplier": 0.22, "rms_norm_eps": 1e-05, "rope_scaling": null, "rope_theta": 10000000, "tie_word_embeddings": true, "torch_dtype": "bfloat16", "transformers_version": "4.49.0", "use_cache": true, "vocab_size": 49159 } }, { "id": "ibm-granite/granite-4.0-tiny-preview", "meta": { "sha": "4ec5f963443b2e15e9be82595289e96ff082e280", "safetensors": { "parameters": { "BF16": 6671539776 }, "total": 6671539776 }, "modelType": "granitemoehybrid" }, "config": { "architectures": [ "GraniteMoeHybridForCausalLM" ], "attention_bias": false, "attention_dropout": 0, "attention_multiplier": 0.0078125, "bos_token_id": 0, "embedding_multiplier": 12, "eos_token_id": 0, "hidden_act": "silu", "hidden_size": 1536, "init_method": "mup", "initializer_range": 0.1, "intermediate_size": 512, "layer_types": [ "mamba", "mamba", "mamba", "mamba", "mamba", "attention", "mamba", "mamba", "mamba", "mamba", "mamba", "mamba", "mamba", "mamba", "mamba", "attention", "mamba", "mamba", "mamba", "mamba", "mamba", "mamba", "mamba", "mamba", "mamba", "attention", "mamba", "mamba", "mamba", "mamba", "mamba", "mamba", "mamba", "mamba", "mamba", "attention", "mamba", "mamba", "mamba", "mamba" ], "logits_scaling": 6, "logits_to_keep": 1, "mamba_chunk_size": 256, "mamba_conv_bias": true, "mamba_d_conv": 4, "mamba_d_head": 64, "mamba_d_state": 128, "mamba_expand": 2, "mamba_n_groups": 1, "mamba_n_heads": 48, "mamba_proj_bias": false, "max_position_embeddings": 131072, "model_type": "granitemoehybrid", "normalization_function": "rmsnorm", "num_attention_heads": 12, "num_experts_per_tok": 6, "num_hidden_layers": 40, "num_key_value_heads": 4, "num_local_experts": 62, "output_router_logits": false, "pad_token_id": 0, "position_embedding_type": "nope", "residual_multiplier": 0.22, "rms_norm_eps": 1e-05, "rope_scaling": null, "rope_theta": 10000, "router_aux_loss_coef": 0, "shared_intermediate_size": 1024, "tie_word_embeddings": true, "transformers_version": "4.51.0.dev0", "use_cache": true, "vocab_size": 49160 } }, { "id": "ibm-granite/granite-4.0-h-tiny", "meta": { "sha": "791e0d3d28c86e106c9b6e0b4cecdee0375b6124", "safetensors": { "parameters": { "BF16": 6939037248 }, "total": 6939037248 }, "modelType": "granitemoehybrid" }, "config": { "architectures": [ "GraniteMoeHybridForCausalLM" ], "attention_bias": false, "attention_dropout": 0, "attention_multiplier": 0.0078125, "bos_token_id": 100257, "embedding_multiplier": 12, "eos_token_id": 100257, "hidden_act": "silu", "hidden_size": 1536, "initializer_range": 0.1, "intermediate_size": 512, "layer_types": [ "mamba", "mamba", "mamba", "mamba", "mamba", "attention", "mamba", "mamba", "mamba", "mamba", "mamba", "mamba", "mamba", "mamba", "mamba", "attention", "mamba", "mamba", "mamba", "mamba", "mamba", "mamba", "mamba", "mamba", "mamba", "attention", "mamba", "mamba", "mamba", "mamba", "mamba", "mamba", "mamba", "mamba", "mamba", "attention", "mamba", "mamba", "mamba", "mamba" ], "logits_scaling": 6, "mamba_chunk_size": 256, "mamba_conv_bias": true, "mamba_d_conv": 4, "mamba_d_head": 64, "mamba_d_state": 128, "mamba_expand": 2, "mamba_n_groups": 1, "mamba_n_heads": 48, "mamba_proj_bias": false, "max_position_embeddings": 131072, "model_type": "granitemoehybrid", "normalization_function": "rmsnorm", "num_attention_heads": 12, "num_experts_per_tok": 6, "num_hidden_layers": 40, "num_key_value_heads": 4, "num_local_experts": 64, "output_router_logits": false, "pad_token_id": 100256, "position_embedding_type": "nope", "residual_multiplier": 0.22, "rms_norm_eps": 1e-05, "rope_scaling": null, "rope_theta": 10000, "router_aux_loss_coef": 0, "shared_intermediate_size": 1024, "tie_word_embeddings": true, "torch_dtype": "bfloat16", "transformers_version": "4.56.0", "use_cache": true, "vocab_size": 100352 } }, { "id": "ibm-granite/granite-3.3-2b-instruct", "meta": { "sha": "707f574c62054322f6b5b04b6d075f0a8f05e0f0", "safetensors": { "parameters": { "BF16": 2533539840 }, "total": 2533539840 }, "modelType": "granite" }, "config": { "architectures": [ "GraniteForCausalLM" ], "attention_bias": false, "attention_dropout": 0, "attention_multiplier": 0.015625, "bos_token_id": 0, "embedding_multiplier": 12, "eos_token_id": 0, "hidden_act": "silu", "hidden_size": 2048, "initializer_range": 0.02, "intermediate_size": 8192, "logits_scaling": 8, "max_position_embeddings": 131072, "mlp_bias": false, "model_type": "granite", "num_attention_heads": 32, "num_hidden_layers": 40, "num_key_value_heads": 8, "pad_token_id": 0, "residual_multiplier": 0.22, "rms_norm_eps": 1e-05, "rope_scaling": null, "rope_theta": 10000000, "tie_word_embeddings": true, "torch_dtype": "bfloat16", "transformers_version": "4.49.0", "use_cache": true, "vocab_size": 49159 } }, { "id": "ibm-granite/granite-docling-258M", "meta": { "sha": "982fe3b40f2fa73c365bdb1bcacf6c81b7184bfe", "safetensors": { "parameters": { "BF16": 257517120 }, "total": 257517120 }, "modelType": "idefics3" }, "config": { "architectures": [ "Idefics3ForConditionalGeneration" ], "bos_token_id": 100264, "dtype": "bfloat16", "eos_token_id": 100257, "image_token_id": 100270, "model_type": "idefics3", "pad_token_id": 100257, "scale_factor": 4, "text_config": { "_name_or_path": "models/granitev06_hf_ai4k_sft_data_v4", "architectures": [ "LlamaForCausalLM" ], "attention_bias": false, "attention_dropout": 0, "bos_token_id": 100264, "dtype": "bfloat16", "eos_token_id": 100257, "head_dim": 64, "hidden_act": "silu", "hidden_size": 576, "initializer_range": 0.02, "intermediate_size": 1536, "max_position_embeddings": 8192, "mlp_bias": false, "model_type": "llama", "num_attention_heads": 9, "num_hidden_layers": 30, "num_key_value_heads": 3, "pad_token_id": 100257, "pretraining_tp": 1, "rms_norm_eps": 1e-05, "rope_scaling": null, "rope_theta": 100000, "tie_word_embeddings": true, "use_cache": false, "vocab_size": 100352 }, "tie_word_embeddings": true, "transformers_version": "4.56.1", "use_cache": true, "vision_config": { "attention_dropout": 0, "hidden_act": "gelu_pytorch_tanh", "hidden_size": 768, "image_size": 512, "initializer_range": 0.02, "intermediate_size": 3072, "layer_norm_eps": 1e-06, "max_image_size": { "longest_edge": 512 }, "model_type": "idefics3_vision", "num_attention_heads": 12, "num_channels": 3, "num_hidden_layers": 12, "patch_size": 16, "size": { "longest_edge": 512 } }, "vocab_size": 100352 } }, { "id": "ibm-granite/granite-4.0-micro", "meta": { "sha": "56111ae135df9c53a78c99028e7bc24035a9e979", "safetensors": { "parameters": { "BF16": 3402836480 }, "total": 3402836480 }, "modelType": "granitemoehybrid" }, "config": { "architectures": [ "GraniteMoeHybridForCausalLM" ], "attention_bias": false, "attention_dropout": 0, "attention_multiplier": 0.015625, "bos_token_id": 100257, "embedding_multiplier": 12, "eos_token_id": 100257, "hidden_act": "silu", "hidden_size": 2560, "initializer_range": 0.1, "intermediate_size": 8192, "layer_types": [ "attention", "attention", "attention", "attention", "attention", "attention", "attention", "attention", "attention", "attention", "attention", "attention", "attention", "attention", "attention", "attention", "attention", "attention", "attention", "attention", "attention", "attention", "attention", "attention", "attention", "attention", "attention", "attention", "attention", "attention", "attention", "attention", "attention", "attention", "attention", "attention", "attention", "attention", "attention", "attention" ], "logits_scaling": 10, "mamba_chunk_size": 256, "mamba_conv_bias": true, "mamba_d_conv": 4, "mamba_d_head": 40, "mamba_d_state": 256, "mamba_expand": 2, "mamba_n_groups": 1, "mamba_n_heads": 128, "mamba_proj_bias": false, "max_position_embeddings": 131072, "model_type": "granitemoehybrid", "normalization_function": "rmsnorm", "num_attention_heads": 40, "num_experts_per_tok": 0, "num_hidden_layers": 40, "num_key_value_heads": 8, "num_local_experts": 0, "output_router_logits": false, "pad_token_id": 100256, "position_embedding_type": "rope", "residual_multiplier": 0.22, "rms_norm_eps": 1e-05, "rope_scaling": null, "rope_theta": 10000000, "router_aux_loss_coef": 0.01, "shared_intermediate_size": 8192, "tie_word_embeddings": true, "torch_dtype": "bfloat16", "transformers_version": "4.56.0", "use_cache": true, "vocab_size": 100352 } }, { "id": "ibm-granite/granite-4.2-3b", "meta": { "sha": "e459acceac81e5fe67c07d9cfc72329a332e7eb1", "safetensors": { "parameters": { "BF16": 3659737600 }, "total": 3659737600 }, "modelType": "granite" }, "config": { "architectures": [ "GraniteForCausalLM" ], "attention_bias": false, "attention_dropout": 0, "attention_multiplier": 0.015625, "bos_token_id": 100283, "dtype": "bfloat16", "embedding_multiplier": 1, "eos_token_id": 100257, "hidden_act": "silu", "hidden_size": 2560, "initializer_range": 0.1, "intermediate_size": 8192, "logits_scaling": 1, "max_position_embeddings": 131072, "mlp_bias": false, "model_type": "granite", "num_attention_heads": 40, "num_hidden_layers": 40, "num_key_value_heads": 8, "pad_token_id": 100257, "residual_multiplier": 1, "rms_norm_eps": 1e-05, "rope_parameters": { "rope_theta": 10000000, "rope_type": "default" }, "rope_scaling": null, "rope_theta": 10000000, "tie_word_embeddings": false, "transformers_version": "4.57.1", "use_cache": true, "vocab_size": 100352, "torch_dtype": "bfloat16" } }, { "id": "ibm-granite/granite-3.1-8b-instruct", "meta": { "sha": "4009206d5fc95d2e65a7b7633e159d6e97e25d35", "safetensors": { "parameters": { "BF16": 8170848256 }, "total": 8170848256 }, "modelType": "granite" }, "config": { "architectures": [ "GraniteForCausalLM" ], "attention_bias": false, "attention_dropout": 0.1, "attention_multiplier": 0.0078125, "bos_token_id": 0, "embedding_multiplier": 12, "eos_token_id": 0, "hidden_act": "silu", "hidden_size": 4096, "initializer_range": 0.02, "intermediate_size": 12800, "logits_scaling": 16, "max_position_embeddings": 131072, "mlp_bias": false, "model_type": "granite", "num_attention_heads": 32, "num_hidden_layers": 40, "num_key_value_heads": 8, "pad_token_id": 0, "residual_multiplier": 0.22, "rms_norm_eps": 1e-05, "rope_scaling": null, "rope_theta": 10000000, "tie_word_embeddings": true, "torch_dtype": "bfloat16", "transformers_version": "4.47.0", "use_cache": true, "vocab_size": 49155 } }, { "id": "ibm-granite/granite-3b-code-base-2k", "meta": { "sha": "b67f3dabc0b6d00ae477bd76732c7f3458a2caf3", "safetensors": { "parameters": { "BF16": 3482503680 }, "total": 3482503680 }, "modelType": "llama" }, "config": { "architectures": [ "LlamaForCausalLM" ], "attention_bias": true, "attention_dropout": 0.1, "bos_token_id": 0, "eos_token_id": 0, "hidden_act": "silu", "hidden_size": 2560, "initializer_range": 0.02, "intermediate_size": 10240, "max_position_embeddings": 2048, "mlp_bias": true, "model_type": "llama", "num_attention_heads": 32, "num_hidden_layers": 32, "num_key_value_heads": 32, "pad_token_id": 0, "pretraining_tp": 1, "rms_norm_eps": 1e-05, "rope_scaling": null, "rope_theta": 10000, "tie_word_embeddings": true, "torch_dtype": "bfloat16", "transformers_version": "4.41.0.dev0", "use_cache": true, "vocab_size": 49152 } }, { "id": "ibm-granite/granite-4.0-1b-base", "meta": { "sha": "15148e99f8ca689325e2067ba282b23dec6670c7", "safetensors": { "parameters": { "BF16": 1631750144 }, "total": 1631750144 }, "modelType": "granitemoehybrid" }, "config": { "architectures": [ "GraniteMoeHybridForCausalLM" ], "attention_bias": false, "attention_dropout": 0, "attention_multiplier": 0.0078125, "bos_token_id": 100257, "embedding_multiplier": 12, "eos_token_id": 100257, "hidden_act": "silu", "hidden_size": 2048, "init_method": "mup", "initializer_range": 0.1, "intermediate_size": 4096, "layer_types": [ "attention", "attention", "attention", "attention", "attention", "attention", "attention", "attention", "attention", "attention", "attention", "attention", "attention", "attention", "attention", "attention", "attention", "attention", "attention", "attention", "attention", "attention", "attention", "attention", "attention", "attention", "attention", "attention", "attention", "attention", "attention", "attention", "attention", "attention", "attention", "attention", "attention", "attention", "attention", "attention" ], "logits_scaling": 8, "mamba_chunk_size": 256, "mamba_conv_bias": true, "mamba_d_conv": 4, "mamba_d_head": 32, "mamba_d_state": 256, "mamba_expand": 2, "mamba_n_groups": 1, "mamba_n_heads": 128, "mamba_proj_bias": false, "max_position_embeddings": 131072, "model_type": "granitemoehybrid", "normalization_function": "rmsnorm", "num_attention_heads": 16, "num_experts_per_tok": 0, "num_hidden_layers": 40, "num_key_value_heads": 4, "num_local_experts": 0, "output_router_logits": false, "pad_token_id": 100256, "position_embedding_type": "rope", "residual_multiplier": 0.22, "rms_norm_eps": 1e-05, "rope_scaling": null, "rope_theta": 10000000, "router_aux_loss_coef": 0.01, "shared_intermediate_size": 4096, "tie_word_embeddings": true, "torch_dtype": "bfloat16", "transformers_version": "4.56.0", "use_cache": true, "vocab_size": 100352 } }, { "id": "ibm-granite/granite-3.2-8b-instruct", "meta": { "sha": "610d8c6ee9c84ce51f6dfd7bc5c0215d95d49695", "safetensors": { "parameters": { "BF16": 8170848256 }, "total": 8170848256 }, "modelType": "granite" }, "config": { "architectures": [ "GraniteForCausalLM" ], "attention_bias": false, "attention_dropout": 0.1, "attention_multiplier": 0.0078125, "bos_token_id": 0, "embedding_multiplier": 12, "eos_token_id": 0, "hidden_act": "silu", "hidden_size": 4096, "initializer_range": 0.02, "intermediate_size": 12800, "logits_scaling": 16, "max_position_embeddings": 131072, "mlp_bias": false, "model_type": "granite", "num_attention_heads": 32, "num_hidden_layers": 40, "num_key_value_heads": 8, "pad_token_id": 0, "residual_multiplier": 0.22, "rms_norm_eps": 1e-05, "rope_scaling": null, "rope_theta": 10000000, "tie_word_embeddings": true, "torch_dtype": "bfloat16", "transformers_version": "4.47.0", "use_cache": true, "vocab_size": 49155 } }, { "id": "ibm-granite/granite-3.0-8b-instruct", "meta": { "sha": "5af56291a1c6ca9056df597aab0cff53edabddb0", "safetensors": { "parameters": { "BF16": 8170848256 }, "total": 8170848256 }, "modelType": "granite" }, "config": { "architectures": [ "GraniteForCausalLM" ], "attention_bias": false, "attention_dropout": 0.1, "attention_multiplier": 0.0078125, "bos_token_id": 0, "embedding_multiplier": 12, "eos_token_id": 0, "hidden_act": "silu", "hidden_size": 4096, "initializer_range": 0.02, "intermediate_size": 12800, "logits_scaling": 16, "max_position_embeddings": 4096, "mlp_bias": false, "model_type": "granite", "num_attention_heads": 32, "num_hidden_layers": 40, "num_key_value_heads": 8, "pad_token_id": 0, "residual_multiplier": 0.22, "rms_norm_eps": 1e-05, "rope_scaling": null, "rope_theta": 10000, "tie_word_embeddings": true, "torch_dtype": "bfloat16", "transformers_version": "4.46.0.dev0", "use_cache": true, "vocab_size": 49155 } }, { "id": "ibm-granite/granite-4.2-30b", "meta": { "sha": "9e668ce1c538387ef24d3644e9b0606647762636", "safetensors": { "parameters": { "BF16": 29276770304 }, "total": 29276770304 }, "modelType": "granite" }, "config": { "architectures": [ "GraniteForCausalLM" ], "attention_bias": false, "attention_dropout": 0, "attention_multiplier": 0.0078125, "bos_token_id": 100283, "dtype": "bfloat16", "embedding_multiplier": 1, "eos_token_id": 100257, "hidden_act": "silu", "hidden_size": 4096, "initializer_range": 0.1, "intermediate_size": 32768, "logits_scaling": 1, "max_position_embeddings": 131072, "mlp_bias": false, "model_type": "granite", "num_attention_heads": 32, "num_hidden_layers": 64, "num_key_value_heads": 8, "pad_token_id": 100257, "residual_multiplier": 1, "rms_norm_eps": 1e-05, "rope_parameters": { "rope_theta": 50000000, "rope_type": "default" }, "rope_scaling": null, "rope_theta": 50000000, "tie_word_embeddings": false, "transformers_version": "4.57.1", "use_cache": true, "vocab_size": 100352, "torch_dtype": "bfloat16" } }, { "id": "ibm-granite/granite-4.0-350m", "meta": { "sha": "bd8a1497065c0d6ba1ef19af6b0d2b14bacf71c2", "safetensors": { "parameters": { "BF16": 352379904 }, "total": 352379904 }, "modelType": "granitemoehybrid" }, "config": { "architectures": [ "GraniteMoeHybridForCausalLM" ], "attention_bias": false, "attention_dropout": 0, "attention_multiplier": 0.015625, "bos_token_id": 100257, "embedding_multiplier": 12, "eos_token_id": 100257, "hidden_act": "silu", "hidden_size": 1024, "init_method": "mup", "initializer_range": 0.1, "intermediate_size": 2048, "layer_types": [ "attention", "attention", "attention", "attention", "attention", "attention", "attention", "attention", "attention", "attention", "attention", "attention", "attention", "attention", "attention", "attention", "attention", "attention", "attention", "attention", "attention", "attention", "attention", "attention", "attention", "attention", "attention", "attention" ], "logits_scaling": 4, "mamba_chunk_size": 256, "mamba_conv_bias": true, "mamba_d_conv": 4, "mamba_d_head": 16, "mamba_d_state": 256, "mamba_expand": 2, "mamba_n_groups": 1, "mamba_n_heads": 128, "mamba_proj_bias": false, "max_position_embeddings": 32768, "model_type": "granitemoehybrid", "normalization_function": "rmsnorm", "num_attention_heads": 16, "num_experts_per_tok": 0, "num_hidden_layers": 28, "num_key_value_heads": 4, "num_local_experts": 0, "output_router_logits": false, "pad_token_id": 100256, "position_embedding_type": "rope", "residual_multiplier": 0.263, "rms_norm_eps": 1e-05, "rope_scaling": null, "rope_theta": 10000000, "router_aux_loss_coef": 0.01, "shared_intermediate_size": 2048, "tie_word_embeddings": true, "torch_dtype": "bfloat16", "transformers_version": "4.56.0", "use_cache": true, "vocab_size": 100352 } }, { "id": "ibm-granite/granite-4.0-1b", "meta": { "sha": "6a7381ba1f54d684ff508d991aeb7dc580157103", "safetensors": { "parameters": { "BF16": 1631750144 }, "total": 1631750144 }, "modelType": "granitemoehybrid" }, "config": { "architectures": [ "GraniteMoeHybridForCausalLM" ], "attention_bias": false, "attention_dropout": 0, "attention_multiplier": 0.0078125, "bos_token_id": 100257, "embedding_multiplier": 12, "eos_token_id": 100257, "hidden_act": "silu", "hidden_size": 2048, "init_method": "mup", "initializer_range": 0.1, "intermediate_size": 4096, "layer_types": [ "attention", "attention", "attention", "attention", "attention", "attention", "attention", "attention", "attention", "attention", "attention", "attention", "attention", "attention", "attention", "attention", "attention", "attention", "attention", "attention", "attention", "attention", "attention", "attention", "attention", "attention", "attention", "attention", "attention", "attention", "attention", "attention", "attention", "attention", "attention", "attention", "attention", "attention", "attention", "attention" ], "logits_scaling": 8, "mamba_chunk_size": 256, "mamba_conv_bias": true, "mamba_d_conv": 4, "mamba_d_head": 32, "mamba_d_state": 256, "mamba_expand": 2, "mamba_n_groups": 1, "mamba_n_heads": 128, "mamba_proj_bias": false, "max_position_embeddings": 131072, "model_type": "granitemoehybrid", "normalization_function": "rmsnorm", "num_attention_heads": 16, "num_experts_per_tok": 0, "num_hidden_layers": 40, "num_key_value_heads": 4, "num_local_experts": 0, "output_router_logits": false, "pad_token_id": 100256, "position_embedding_type": "rope", "residual_multiplier": 0.22, "rms_norm_eps": 1e-05, "rope_scaling": null, "rope_theta": 10000000, "router_aux_loss_coef": 0.01, "shared_intermediate_size": 4096, "tie_word_embeddings": true, "torch_dtype": "bfloat16", "transformers_version": "4.56.0", "use_cache": true, "vocab_size": 100352 } }, { "id": "ibm-granite/granite-swash-2b", "meta": { "sha": "af1e3227100b61088eead48389ab5409b5d0e39c", "safetensors": { "parameters": { "BF16": 2144463840 }, "total": 2144463840 }, "modelType": "granite_swa" }, "config": { "model_type": "granite_swa", "architectures": [ "GraniteSWAForCausalLM" ], "vocab_size": 100352, "hidden_size": 2560, "intermediate_size": 8192, "num_hidden_layers": 24, "num_attention_heads": 20, "num_key_value_heads": 4, "hidden_act": "silu", "max_position_embeddings": 8192, "rms_norm_eps": 1e-05, "attention_bias": false, "mlp_bias": false, "tie_word_embeddings": true, "initializer_range": 0.1, "bos_token_id": 100257, "eos_token_id": 100257, "pad_token_id": 100256, "sliding_window": 128, "layer_types": [ "full_attention", "sliding_attention", "sliding_attention", "full_attention", "sliding_attention", "sliding_attention", "sliding_attention", "full_attention", "sliding_attention", "sliding_attention", "sliding_attention", "full_attention", "sliding_attention", "sliding_attention", "sliding_attention", "full_attention", "sliding_attention", "sliding_attention", "sliding_attention", "full_attention", "sliding_attention", "sliding_attention", "sliding_attention", "full_attention" ], "embedding_multiplier": 12, "residual_multiplier": 0.28, "logits_scaling": 10, "attention_multiplier": 0.0078125, "rope_theta": 10000 } }, { "id": "ibm-granite/granite-3.0-1b-a400m-base", "meta": { "sha": "d91cbed802d85eb1b32374623331f6ab2b37403a", "safetensors": { "parameters": { "F32": 1384956928 }, "total": 1384956928 }, "modelType": "granitemoe" }, "config": { "architectures": [ "GraniteMoeForCausalLM" ], "attention_bias": false, "attention_dropout": 0, "attention_multiplier": 0.015625, "bos_token_id": 0, "embedding_multiplier": 12, "eos_token_id": 0, "hidden_act": "silu", "hidden_size": 1024, "initializer_range": 0.02, "intermediate_size": 512, "logits_scaling": 6, "max_position_embeddings": 4096, "model_type": "granitemoe", "num_attention_heads": 16, "num_experts_per_tok": 8, "num_hidden_layers": 24, "num_key_value_heads": 8, "num_local_experts": 32, "output_router_logits": false, "pad_token_id": 0, "residual_multiplier": 0.22, "rms_norm_eps": 1e-06, "rope_scaling": null, "rope_theta": 10000, "router_aux_loss_coef": 0.001, "tie_word_embeddings": true, "transformers_version": "4.45.0.dev0", "use_cache": true, "vocab_size": 49152 } }, { "id": "ibm-granite/granite-swash-3b-a600m", "meta": { "sha": "568a94fcfeff42a8cab9ee2ffea89fde12071282", "safetensors": { "parameters": { "BF16": 3020382000 }, "total": 3020382000 }, "modelType": "granitemoe_swa" }, "config": { "architectures": [ "GraniteMoeSWAForCausalLM" ], "attention_bias": false, "attention_dropout": 0, "attention_multiplier": 0.015625, "bos_token_id": 100257, "dtype": "bfloat16", "embedding_multiplier": 12, "eos_token_id": 100257, "hidden_act": "silu", "hidden_size": 1280, "initializer_range": 0.1, "intermediate_size": 512, "layer_types": [ "full_attention", "sliding_attention", "sliding_attention", "full_attention", "sliding_attention", "sliding_attention", "sliding_attention", "full_attention", "sliding_attention", "sliding_attention", "sliding_attention", "full_attention", "sliding_attention", "sliding_attention", "sliding_attention", "full_attention", "sliding_attention", "sliding_attention", "sliding_attention", "full_attention", "sliding_attention", "sliding_attention", "sliding_attention", "full_attention", "sliding_attention", "sliding_attention", "sliding_attention", "full_attention" ], "logits_scaling": 5, "max_position_embeddings": 8192, "mlp_bias": false, "model_type": "granitemoe_swa", "num_attention_heads": 20, "num_experts_per_tok": 4, "num_hidden_layers": 28, "num_key_value_heads": 4, "num_local_experts": 48, "output_router_logits": false, "pad_token_id": 100256, "residual_multiplier": 0.26, "rms_norm_eps": 1e-05, "rope_parameters": { "rope_theta": 10000, "rope_type": "default" }, "router_aux_loss_coef": 0.01, "shared_intermediate_size": 1280, "sliding_window": 128, "tie_word_embeddings": true, "transformers_version": "5.8.0.dev0", "use_cache": true, "vocab_size": 100352 } }, { "id": "ibm-granite/granite-4.0-h-small", "meta": { "sha": "b8c0982bab7fde4eb48110f5a069527c008fab39", "safetensors": { "parameters": { "BF16": 32207337984 }, "total": 32207337984 }, "modelType": "granitemoehybrid" }, "config": { "architectures": [ "GraniteMoeHybridForCausalLM" ], "attention_bias": false, "attention_dropout": 0, "attention_multiplier": 0.0078125, "bos_token_id": 100257, "embedding_multiplier": 12, "eos_token_id": 100257, "hidden_act": "silu", "hidden_size": 4096, "initializer_range": 0.1, "intermediate_size": 768, "layer_types": [ "mamba", "mamba", "mamba", "mamba", "mamba", "attention", "mamba", "mamba", "mamba", "mamba", "mamba", "mamba", "mamba", "mamba", "mamba", "attention", "mamba", "mamba", "mamba", "mamba", "mamba", "mamba", "mamba", "mamba", "mamba", "attention", "mamba", "mamba", "mamba", "mamba", "mamba", "mamba", "mamba", "mamba", "mamba", "attention", "mamba", "mamba", "mamba", "mamba" ], "logits_scaling": 16, "mamba_chunk_size": 256, "mamba_conv_bias": true, "mamba_d_conv": 4, "mamba_d_head": 64, "mamba_d_state": 128, "mamba_expand": 2, "mamba_n_groups": 1, "mamba_n_heads": 128, "mamba_proj_bias": false, "max_position_embeddings": 131072, "model_type": "granitemoehybrid", "normalization_function": "rmsnorm", "num_attention_heads": 32, "num_experts_per_tok": 10, "num_hidden_layers": 40, "num_key_value_heads": 8, "num_local_experts": 72, "output_router_logits": false, "pad_token_id": 100256, "position_embedding_type": "nope", "residual_multiplier": 0.22, "rms_norm_eps": 1e-05, "rope_scaling": null, "rope_theta": 10000, "router_aux_loss_coef": 0, "shared_intermediate_size": 1536, "tie_word_embeddings": true, "torch_dtype": "bfloat16", "transformers_version": "4.56.0", "use_cache": true, "vocab_size": 100352 } }, { "id": "ibm-granite/granite-4.1-8b-fp8", "meta": { "sha": "070021b3608433b6107a00733d561c9779b9937e", "safetensors": { "parameters": { "BF16": 824012800, "F8_E4M3": 7969177600 }, "total": 8793190400 }, "modelType": "granite" }, "config": { "architectures": [ "GraniteForCausalLM" ], "attention_bias": false, "attention_dropout": 0, "attention_multiplier": 0.0078125, "bos_token_id": 100257, "dtype": "bfloat16", "embedding_multiplier": 12, "eos_token_id": 100257, "hidden_act": "silu", "hidden_size": 4096, "initializer_range": 0.1, "intermediate_size": 12800, "logits_scaling": 16, "max_position_embeddings": 131072, "mlp_bias": false, "model_type": "granite", "num_attention_heads": 32, "num_hidden_layers": 40, "num_key_value_heads": 8, "pad_token_id": 100256, "quantization_config": { "config_groups": { "group_0": { "format": "float-quantized", "input_activations": { "actorder": null, "block_structure": null, "dynamic": true, "group_size": null, "num_bits": 8, "observer": null, "observer_kwargs": {}, "scale_dtype": null, "strategy": "token", "symmetric": true, "type": "float", "zp_dtype": null }, "output_activations": null, "targets": [ "Linear" ], "weights": { "actorder": null, "block_structure": null, "dynamic": false, "group_size": null, "num_bits": 8, "observer": "memoryless_minmax", "observer_kwargs": {}, "scale_dtype": null, "strategy": "channel", "symmetric": true, "type": "float", "zp_dtype": null } } }, "format": "float-quantized", "global_compression_ratio": null, "ignore": [ "lm_head" ], "kv_cache_scheme": null, "quant_method": "compressed-tensors", "quantization_status": "compressed", "sparsity_config": {}, "transform_config": {}, "version": "0.14.0.1" }, "residual_multiplier": 0.22, "rms_norm_eps": 1e-05, "rope_scaling": null, "rope_theta": 10000000, "tie_word_embeddings": true, "transformers_version": "4.57.6", "use_cache": true, "vocab_size": 100352 } }, { "id": "ibm-granite/granite-4.1-3b-base", "meta": { "sha": "dacb9cb9157bec98e99b09f285c92a4d58405c96", "safetensors": { "parameters": { "BF16": 3402836480 }, "total": 3402836480 }, "modelType": "granite" }, "config": { "architectures": [ "GraniteForCausalLM" ], "attention_bias": false, "attention_dropout": 0, "attention_multiplier": 0.015625, "bos_token_id": 100257, "embedding_multiplier": 12, "eos_token_id": 100257, "hidden_act": "silu", "hidden_size": 2560, "initializer_range": 0.1, "intermediate_size": 8192, "logits_scaling": 10, "max_position_embeddings": 131072, "mlp_bias": false, "model_type": "granite", "num_attention_heads": 40, "num_hidden_layers": 40, "num_key_value_heads": 8, "pad_token_id": 100256, "residual_multiplier": 0.22, "rms_norm_eps": 1e-05, "rope_scaling": null, "rope_theta": 10000000, "tie_word_embeddings": true, "torch_dtype": "bfloat16", "transformers_version": "4.53.3", "use_cache": true, "vocab_size": 100352 } }, { "id": "ibm-granite/granite-3.1-2b-instruct", "meta": { "sha": "bbc2aed595bd38bd770263dc3ab831db9794441d", "safetensors": { "parameters": { "BF16": 2533531648 }, "total": 2533531648 }, "modelType": "granite" }, "config": { "architectures": [ "GraniteForCausalLM" ], "attention_bias": false, "attention_dropout": 0.1, "attention_multiplier": 0.015625, "bos_token_id": 0, "embedding_multiplier": 12, "eos_token_id": 0, "hidden_act": "silu", "hidden_size": 2048, "initializer_range": 0.02, "intermediate_size": 8192, "logits_scaling": 8, "max_position_embeddings": 131072, "mlp_bias": false, "model_type": "granite", "num_attention_heads": 32, "num_hidden_layers": 40, "num_key_value_heads": 8, "pad_token_id": 0, "residual_multiplier": 0.22, "rms_norm_eps": 1e-05, "rope_scaling": null, "rope_theta": 5000000, "tie_word_embeddings": true, "torch_dtype": "bfloat16", "transformers_version": "4.47.0", "use_cache": true, "vocab_size": 49155 } }, { "id": "ibm-granite/granite-4.0-h-micro", "meta": { "sha": "d5f01a3ea75f088947be3aae039f4ad52837dfde", "safetensors": { "parameters": { "BF16": 3191396096 }, "total": 3191396096 }, "modelType": "granitemoehybrid" }, "config": { "architectures": [ "GraniteMoeHybridForCausalLM" ], "attention_bias": false, "attention_dropout": 0, "attention_multiplier": 0.015625, "bos_token_id": 100257, "embedding_multiplier": 12, "eos_token_id": 100257, "hidden_act": "silu", "hidden_size": 2048, "initializer_range": 0.1, "intermediate_size": 8192, "layer_types": [ "mamba", "mamba", "mamba", "mamba", "mamba", "attention", "mamba", "mamba", "mamba", "mamba", "mamba", "mamba", "mamba", "mamba", "mamba", "attention", "mamba", "mamba", "mamba", "mamba", "mamba", "mamba", "mamba", "mamba", "mamba", "attention", "mamba", "mamba", "mamba", "mamba", "mamba", "mamba", "mamba", "mamba", "mamba", "attention", "mamba", "mamba", "mamba", "mamba" ], "logits_scaling": 8, "mamba_chunk_size": 256, "mamba_conv_bias": true, "mamba_d_conv": 4, "mamba_d_head": 64, "mamba_d_state": 128, "mamba_expand": 2, "mamba_n_groups": 1, "mamba_n_heads": 64, "mamba_proj_bias": false, "max_position_embeddings": 131072, "model_type": "granitemoehybrid", "normalization_function": "rmsnorm", "num_attention_heads": 32, "num_experts_per_tok": 0, "num_hidden_layers": 40, "num_key_value_heads": 8, "num_local_experts": 0, "output_router_logits": false, "pad_token_id": 100256, "position_embedding_type": "nope", "residual_multiplier": 0.22, "rms_norm_eps": 1e-05, "rope_scaling": null, "rope_theta": 10000, "router_aux_loss_coef": 0.01, "shared_intermediate_size": 8192, "tie_word_embeddings": true, "torch_dtype": "bfloat16", "transformers_version": "4.56.0", "use_cache": true, "vocab_size": 100352 } }, { "id": "ibm-granite/granite-3.0-8b-base", "meta": { "sha": "4fad7f8ad56393dcef4e34e37a35962bd091f320", "safetensors": { "parameters": { "F32": 8170835968 }, "total": 8170835968 }, "modelType": "granite" }, "config": { "architectures": [ "GraniteForCausalLM" ], "attention_bias": false, "attention_dropout": 0.1, "attention_multiplier": 0.0078125, "bos_token_id": 0, "embedding_multiplier": 12, "eos_token_id": 0, "hidden_act": "silu", "hidden_size": 4096, "initializer_range": 0.02, "intermediate_size": 12800, "logits_scaling": 16, "max_position_embeddings": 4096, "mlp_bias": false, "model_type": "granite", "num_attention_heads": 32, "num_hidden_layers": 40, "num_key_value_heads": 8, "pad_token_id": 0, "residual_multiplier": 0.22, "rms_norm_eps": 1e-05, "rope_scaling": null, "rope_theta": 10000, "tie_word_embeddings": true, "transformers_version": "4.45.0.dev0", "use_cache": true, "vocab_size": 49152 } }, { "id": "ibm-granite/granite-3.1-1b-a400m-instruct", "meta": { "sha": "0da7a48b0276d500ce5922fd2b33944091fc6c09", "safetensors": { "parameters": { "BF16": 1334628352 }, "total": 1334628352 }, "modelType": "granitemoe" }, "config": { "architectures": [ "GraniteMoeForCausalLM" ], "attention_bias": false, "attention_dropout": 0, "attention_multiplier": 0.015625, "bos_token_id": 0, "embedding_multiplier": 12, "eos_token_id": 0, "hidden_act": "silu", "hidden_size": 1024, "initializer_range": 0.02, "intermediate_size": 512, "logits_scaling": 6, "max_position_embeddings": 131072, "model_type": "granitemoe", "num_attention_heads": 16, "num_experts_per_tok": 8, "num_hidden_layers": 24, "num_key_value_heads": 8, "num_local_experts": 32, "output_router_logits": false, "pad_token_id": 0, "residual_multiplier": 0.22, "rms_norm_eps": 1e-06, "rope_scaling": null, "rope_theta": 1500000, "router_aux_loss_coef": 0, "tie_word_embeddings": true, "torch_dtype": "bfloat16", "transformers_version": "4.47.0", "use_cache": true, "vocab_size": 49155 } }, { "id": "ibm-granite/granite-3.1-3b-a800m-instruct", "meta": { "sha": "a02780686e08a03fe0d2679a293b5c74a90efa89", "safetensors": { "parameters": { "BF16": 3298793472 }, "total": 3298793472 }, "modelType": "granitemoe" }, "config": { "architectures": [ "GraniteMoeForCausalLM" ], "attention_bias": false, "attention_dropout": 0, "attention_multiplier": 0.015625, "bos_token_id": 0, "embedding_multiplier": 12, "eos_token_id": 0, "hidden_act": "silu", "hidden_size": 1536, "initializer_range": 0.02, "intermediate_size": 512, "logits_scaling": 6, "max_position_embeddings": 131072, "model_type": "granitemoe", "num_attention_heads": 24, "num_experts_per_tok": 8, "num_hidden_layers": 32, "num_key_value_heads": 8, "num_local_experts": 40, "output_router_logits": false, "pad_token_id": 0, "residual_multiplier": 0.22, "rms_norm_eps": 1e-06, "rope_scaling": null, "rope_theta": 10000000, "router_aux_loss_coef": 0, "tie_word_embeddings": true, "torch_dtype": "bfloat16", "transformers_version": "4.47.0", "use_cache": true, "vocab_size": 49155 } }, { "id": "zai-org/GLM-4.7-Flash", "meta": { "sha": "7dd20894a642a0aa287e9827cb1a1f7f91386b67", "safetensors": { "parameters": { "BF16": 31221485568, "F32": 3008 }, "total": 31221488576 }, "modelType": "glm4_moe_lite" }, "config": { "architectures": [ "Glm4MoeLiteForCausalLM" ], "attention_bias": false, "attention_dropout": 0, "pad_token_id": 154820, "eos_token_id": [ 154820, 154827, 154829 ], "hidden_act": "silu", "hidden_size": 2048, "intermediate_size": 10240, "max_position_embeddings": 202752, "model_type": "glm4_moe_lite", "moe_intermediate_size": 1536, "topk_method": "noaux_tc", "norm_topk_prob": true, "num_attention_heads": 20, "n_group": 1, "topk_group": 1, "n_routed_experts": 64, "n_shared_experts": 1, "routed_scaling_factor": 1.8, "num_experts_per_tok": 4, "first_k_dense_replace": 1, "num_hidden_layers": 47, "num_key_value_heads": 20, "num_nextn_predict_layers": 1, "partial_rotary_factor": 1, "rms_norm_eps": 1e-05, "rope_scaling": null, "rope_theta": 1000000, "tie_word_embeddings": false, "dtype": "bfloat16", "transformers_version": "5.0.0rc0", "q_lora_rank": 768, "kv_lora_rank": 512, "qk_nope_head_dim": 192, "qk_rope_head_dim": 64, "v_head_dim": 256, "vocab_size": 154880 } }, { "id": "zai-org/GLM-5.3", "meta": { "sha": "aca966e4e02791568aa6a4ced368624b3d897f42", "safetensors": { "parameters": { "BF16": 2103729152, "F8_E4M3": 751226191872, "F32": 19456 }, "total": 753329940480 }, "modelType": "glm_moe_dsa" }, "config": { "architectures": [ "GlmMoeDsaForCausalLM" ], "attention_bias": false, "attention_dropout": 0, "dtype": "bfloat16", "eos_token_id": [ 154820, 154827, 154829 ], "ep_size": 1, "first_k_dense_replace": 3, "head_dim": 192, "hidden_act": "silu", "hidden_size": 6144, "index_head_dim": 128, "index_n_heads": 32, "index_share_for_mtp_iteration": true, "index_skip_topk_offset": 3, "index_topk": 2048, "index_topk_freq": 4, "index_topk_pattern": null, "indexer_rope_interleave": true, "indexer_types": [ "full", "full", "full", "shared", "shared", "shared", "full", "shared", "shared", "shared", "full", "shared", "shared", "shared", "full", "shared", "shared", "shared", "full", "shared", "shared", "shared", "full", "shared", "shared", "shared", "full", "shared", "shared", "shared", "full", "shared", "shared", "shared", "full", "shared", "shared", "shared", "full", "shared", "shared", "shared", "full", "shared", "shared", "shared", "full", "shared", "shared", "shared", "full", "shared", "shared", "shared", "full", "shared", "shared", "shared", "full", "shared", "shared", "shared", "full", "shared", "shared", "shared", "full", "shared", "shared", "shared", "full", "shared", "shared", "shared", "full", "shared", "shared", "shared" ], "initializer_range": 0.02, "intermediate_size": 12288, "kv_lora_rank": 512, "max_position_embeddings": 1048576, "mlp_layer_types": [ "dense", "dense", "dense", "sparse", "sparse", "sparse", "sparse", "sparse", "sparse", "sparse", "sparse", "sparse", "sparse", "sparse", "sparse", "sparse", "sparse", "sparse", "sparse", "sparse", "sparse", "sparse", "sparse", "sparse", "sparse", "sparse", "sparse", "sparse", "sparse", "sparse", "sparse", "sparse", "sparse", "sparse", "sparse", "sparse", "sparse", "sparse", "sparse", "sparse", "sparse", "sparse", "sparse", "sparse", "sparse", "sparse", "sparse", "sparse", "sparse", "sparse", "sparse", "sparse", "sparse", "sparse", "sparse", "sparse", "sparse", "sparse", "sparse", "sparse", "sparse", "sparse", "sparse", "sparse", "sparse", "sparse", "sparse", "sparse", "sparse", "sparse", "sparse", "sparse", "sparse", "sparse", "sparse", "sparse", "sparse", "sparse" ], "model_type": "glm_moe_dsa", "moe_intermediate_size": 2048, "moe_layer_freq": 1, "moe_router_dtype": "float32", "n_group": 1, "n_routed_experts": 256, "n_shared_experts": 1, "norm_topk_prob": true, "num_attention_heads": 64, "num_experts_per_tok": 8, "num_hidden_layers": 78, "num_key_value_heads": 64, "num_nextn_predict_layers": 1, "pad_token_id": 154820, "pretraining_tp": 1, "q_lora_rank": 2048, "qk_head_dim": 256, "qk_nope_head_dim": 192, "qk_rope_head_dim": 64, "rms_norm_eps": 1e-05, "rope_interleave": true, "rope_parameters": { "rope_theta": 8000000, "rope_type": "default" }, "routed_scaling_factor": 2.5, "scoring_func": "sigmoid", "tie_word_embeddings": false, "topk_group": 1, "topk_method": "noaux_tc", "transformers_version": "5.15.0", "use_cache": true, "v_head_dim": 256, "vocab_size": 154880, "quantization_config": { "activation_scheme": "dynamic", "fmt": "e4m3", "quant_method": "fp8", "weight_block_size": [ 128, 128 ], "modules_to_not_convert": [ "model.layers.59.post_attention_layernorm", "model.layers.47.mlp.gate.e_score_correction_bias", "model.layers.71.self_attn.kv_a_layernorm", "model.layers.46.mlp.gate.e_score_correction_bias", "model.layers.30.self_attn.kv_a_layernorm", "model.layers.23.self_attn.kv_a_layernorm", "model.layers.73.mlp.gate.e_score_correction_bias", "model.layers.36.mlp.gate", "model.layers.12.self_attn.q_a_layernorm", "model.layers.51.mlp.gate", "model.layers.47.self_attn.kv_a_layernorm", "model.layers.45.self_attn.q_a_layernorm", "model.layers.15.self_attn.q_a_layernorm", "model.layers.39.input_layernorm", "model.layers.50.self_attn.kv_a_layernorm", "model.layers.19.post_attention_layernorm", "model.layers.71.input_layernorm", "model.layers.72.self_attn.kv_a_layernorm", "model.layers.70.mlp.gate", "model.layers.52.input_layernorm", "model.layers.78.self_attn.kv_a_layernorm", "model.layers.48.post_attention_layernorm", "model.layers.4.self_attn.q_a_layernorm", "model.layers.62.self_attn.q_a_layernorm", "model.layers.4.post_attention_layernorm", "model.layers.38.self_attn.indexers_proj", "model.layers.61.mlp.gate.e_score_correction_bias", "model.layers.14.mlp.gate.e_score_correction_bias", "model.layers.36.self_attn.kv_a_layernorm", "model.layers.65.mlp.gate", "model.layers.13.self_attn.kv_a_layernorm", "model.layers.57.mlp.gate", "model.layers.65.self_attn.kv_a_layernorm", "model.layers.68.input_layernorm", "model.layers.70.self_attn.kv_a_layernorm", "model.layers.24.mlp.gate.e_score_correction_bias", "model.layers.4.input_layernorm", "model.layers.20.input_layernorm", "model.layers.65.self_attn.q_a_layernorm", "model.layers.38.mlp.gate", "model.layers.76.mlp.gate.e_score_correction_bias", "model.layers.62.input_layernorm", "model.layers.45.input_layernorm", "model.layers.68.post_attention_layernorm", "model.layers.72.input_layernorm", "model.layers.32.mlp.gate.e_score_correction_bias", "model.layers.27.input_layernorm", "model.layers.16.post_attention_layernorm", "model.layers.35.self_attn.q_a_layernorm", "model.layers.58.self_attn.indexers_proj", "model.layers.3.mlp.gate.e_score_correction_bias", "model.layers.49.self_attn.q_a_layernorm", "model.layers.66.mlp.gate.e_score_correction_bias", "model.layers.10.self_attn.q_a_layernorm", "model.layers.51.mlp.gate.e_score_correction_bias", "model.layers.64.self_attn.q_a_layernorm", "model.layers.70.input_layernorm", "model.layers.71.post_attention_layernorm", "model.layers.67.post_attention_layernorm", "model.layers.6.self_attn.indexer.k_norm", "model.layers.3.mlp.gate", "model.layers.11.mlp.gate", "model.layers.17.input_layernorm", "model.layers.13.mlp.gate", "model.layers.15.mlp.gate", "model.layers.71.self_attn.q_a_layernorm", "model.layers.33.mlp.gate.e_score_correction_bias", "model.layers.72.mlp.gate.e_score_correction_bias", "model.layers.65.post_attention_layernorm", "model.layers.78.hnorm", "model.layers.40.post_attention_layernorm", "model.layers.12.mlp.gate.e_score_correction_bias", "model.layers.8.input_layernorm", "model.layers.73.mlp.gate", "model.layers.11.post_attention_layernorm", "model.layers.61.input_layernorm", "model.layers.14.self_attn.kv_a_layernorm", "model.layers.39.self_attn.kv_a_layernorm", "model.layers.70.self_attn.indexer.k_norm", "model.layers.45.mlp.gate", "model.layers.51.self_attn.q_a_layernorm", "model.layers.64.post_attention_layernorm", "model.layers.20.post_attention_layernorm", "model.layers.66.self_attn.kv_a_layernorm", "model.layers.10.self_attn.indexer.k_norm.bias", "model.layers.23.input_layernorm", "model.layers.38.self_attn.indexer.k_norm", "model.layers.19.self_attn.kv_a_layernorm", "model.layers.40.self_attn.q_a_layernorm", "model.layers.10.self_attn.indexer.k_norm", "model.layers.44.post_attention_layernorm", "model.layers.70.post_attention_layernorm", "model.layers.41.mlp.gate.e_score_correction_bias", "model.layers.36.post_attention_layernorm", "model.layers.30.self_attn.indexer.k_norm.bias", "model.layers.54.self_attn.q_a_layernorm", "model.layers.59.mlp.gate", "model.layers.69.post_attention_layernorm", "lm_head", "model.layers.77.self_attn.kv_a_layernorm", "model.layers.75.self_attn.q_a_layernorm", "model.layers.56.self_attn.q_a_layernorm", "model.layers.50.self_attn.indexer.k_norm.bias", "model.layers.54.mlp.gate", "model.layers.6.self_attn.indexer.k_norm.bias", "model.layers.15.input_layernorm", "model.layers.29.self_attn.q_a_layernorm", "model.layers.54.self_attn.indexers_proj", "model.layers.73.self_attn.kv_a_layernorm", "model.layers.0.self_attn.indexer.k_norm", "model.layers.58.self_attn.indexer.k_norm", "model.layers.36.input_layernorm", "model.layers.47.self_attn.q_a_layernorm", "model.layers.26.mlp.gate.e_score_correction_bias", "model.layers.50.input_layernorm", "model.layers.16.self_attn.q_a_layernorm", "model.layers.19.mlp.gate.e_score_correction_bias", "model.layers.78.input_layernorm", "model.layers.27.mlp.gate", "model.layers.19.self_attn.q_a_layernorm", "model.layers.25.input_layernorm", "model.layers.33.self_attn.kv_a_layernorm", "model.layers.32.mlp.gate", "model.layers.61.self_attn.kv_a_layernorm", "model.layers.50.self_attn.indexers_proj", "model.layers.34.self_attn.indexers_proj", "model.layers.4.mlp.gate.e_score_correction_bias", "model.layers.76.post_attention_layernorm", "model.layers.57.post_attention_layernorm", "model.layers.73.post_attention_layernorm", "model.layers.68.mlp.gate.e_score_correction_bias", "model.layers.34.self_attn.kv_a_layernorm", "model.layers.78.enorm", "model.layers.30.mlp.gate", "model.layers.10.self_attn.indexers_proj", "model.layers.34.input_layernorm", "model.layers.34.post_attention_layernorm", "model.layers.20.mlp.gate.e_score_correction_bias", "model.layers.54.self_attn.indexer.k_norm.bias", "model.layers.44.mlp.gate", "model.layers.25.mlp.gate.e_score_correction_bias", "model.layers.26.self_attn.indexers_proj", "model.layers.66.self_attn.indexer.k_norm.bias", "model.layers.41.post_attention_layernorm", "model.layers.51.post_attention_layernorm", "model.layers.22.self_attn.indexer.k_norm.bias", "model.layers.21.mlp.gate.e_score_correction_bias", "model.layers.15.self_attn.kv_a_layernorm", "model.layers.25.self_attn.kv_a_layernorm", "model.layers.55.mlp.gate", "model.layers.11.self_attn.q_a_layernorm", "model.layers.72.mlp.gate", "model.layers.21.self_attn.kv_a_layernorm", "model.layers.59.mlp.gate.e_score_correction_bias", "model.layers.2.input_layernorm", "model.layers.5.mlp.gate", "model.layers.54.self_attn.kv_a_layernorm", "model.layers.0.self_attn.indexers_proj", "model.layers.71.mlp.gate.e_score_correction_bias", "model.layers.6.self_attn.q_a_layernorm", "model.layers.14.mlp.gate", "model.layers.44.self_attn.kv_a_layernorm", "model.layers.61.post_attention_layernorm", "model.layers.64.self_attn.kv_a_layernorm", "model.layers.1.post_attention_layernorm", "model.layers.4.mlp.gate", "model.layers.22.post_attention_layernorm", "model.layers.21.self_attn.q_a_layernorm", "model.layers.74.self_attn.q_a_layernorm", "model.layers.18.self_attn.indexers_proj", "model.layers.75.input_layernorm", "model.layers.70.self_attn.q_a_layernorm", "model.layers.22.self_attn.indexer.k_norm", "model.layers.42.mlp.gate.e_score_correction_bias", "model.layers.5.self_attn.kv_a_layernorm", "model.layers.31.mlp.gate.e_score_correction_bias", "model.layers.62.self_attn.indexer.k_norm.bias", "model.layers.29.post_attention_layernorm", "model.layers.6.self_attn.kv_a_layernorm", "model.layers.48.input_layernorm", "model.layers.74.self_attn.indexer.k_norm.bias", "model.layers.7.input_layernorm", "model.layers.39.self_attn.q_a_layernorm", "model.layers.41.input_layernorm", "model.layers.54.self_attn.indexer.k_norm", "model.layers.32.self_attn.q_a_layernorm", "model.layers.39.mlp.gate.e_score_correction_bias", "model.layers.42.input_layernorm", "model.layers.43.mlp.gate", "model.layers.72.self_attn.q_a_layernorm", "model.layers.37.mlp.gate.e_score_correction_bias", "model.layers.5.post_attention_layernorm", "model.layers.28.mlp.gate", "model.layers.2.post_attention_layernorm", "model.layers.49.post_attention_layernorm", "model.layers.43.self_attn.kv_a_layernorm", "model.layers.12.input_layernorm", "model.layers.56.mlp.gate", "model.layers.5.input_layernorm", "model.layers.29.mlp.gate.e_score_correction_bias", "model.layers.4.self_attn.kv_a_layernorm", "model.layers.20.self_attn.q_a_layernorm", "model.layers.59.self_attn.q_a_layernorm", "model.layers.68.mlp.gate", "model.layers.21.post_attention_layernorm", "model.layers.26.self_attn.q_a_layernorm", "model.layers.52.self_attn.q_a_layernorm", "model.layers.29.self_attn.kv_a_layernorm", "model.layers.24.input_layernorm", "model.layers.34.self_attn.indexer.k_norm.bias", "model.layers.43.input_layernorm", "model.layers.27.self_attn.kv_a_layernorm", "model.layers.31.input_layernorm", "model.layers.69.mlp.gate", "model.layers.26.post_attention_layernorm", "model.layers.3.self_attn.q_a_layernorm", "model.layers.37.post_attention_layernorm", "model.layers.52.mlp.gate", "model.layers.73.input_layernorm", "model.layers.19.input_layernorm", "model.layers.13.self_attn.q_a_layernorm", "model.layers.76.input_layernorm", "model.layers.8.mlp.gate", "model.layers.63.post_attention_layernorm", "model.layers.58.mlp.gate", "model.layers.31.post_attention_layernorm", "model.layers.23.mlp.gate.e_score_correction_bias", "model.layers.34.self_attn.q_a_layernorm", "model.layers.78.self_attn.indexer.k_norm.bias", "model.layers.13.mlp.gate.e_score_correction_bias", "model.layers.71.mlp.gate", "model.layers.14.input_layernorm", "model.layers.17.mlp.gate.e_score_correction_bias", "model.layers.8.post_attention_layernorm", "model.layers.45.self_attn.kv_a_layernorm", "model.layers.46.self_attn.indexer.k_norm", "model.layers.20.self_attn.kv_a_layernorm", "model.layers.78.self_attn.indexer.k_norm", "model.layers.37.mlp.gate", "model.layers.7.mlp.gate", "model.layers.30.mlp.gate.e_score_correction_bias", "model.layers.67.mlp.gate", "model.layers.73.self_attn.q_a_layernorm", "model.layers.10.self_attn.kv_a_layernorm", "model.layers.61.mlp.gate", "model.layers.22.mlp.gate.e_score_correction_bias", "model.layers.58.input_layernorm", "model.layers.8.self_attn.kv_a_layernorm", "model.layers.33.self_attn.q_a_layernorm", "model.layers.55.self_attn.q_a_layernorm", "model.layers.32.input_layernorm", "model.layers.17.self_attn.kv_a_layernorm", "model.layers.46.self_attn.indexer.k_norm.bias", "model.layers.11.mlp.gate.e_score_correction_bias", "model.layers.9.input_layernorm", "model.layers.77.input_layernorm", "model.layers.16.self_attn.kv_a_layernorm", "model.layers.45.post_attention_layernorm", "model.layers.74.mlp.gate", "model.layers.68.self_attn.q_a_layernorm", "model.layers.55.self_attn.kv_a_layernorm", "model.layers.44.input_layernorm", "model.layers.60.self_attn.kv_a_layernorm", "model.layers.14.post_attention_layernorm", "model.layers.62.self_attn.indexers_proj", "model.layers.0.self_attn.q_a_layernorm", "model.layers.72.post_attention_layernorm", "model.layers.78.post_attention_layernorm", "model.layers.60.mlp.gate.e_score_correction_bias", "model.layers.26.self_attn.indexer.k_norm.bias", "model.layers.38.post_attention_layernorm", "model.layers.50.post_attention_layernorm", "model.layers.28.input_layernorm", "model.layers.30.self_attn.indexer.k_norm", "model.layers.37.self_attn.q_a_layernorm", "model.layers.56.input_layernorm", "model.layers.27.post_attention_layernorm", "model.layers.74.input_layernorm", "model.layers.78.self_attn.q_a_layernorm", "model.layers.42.self_attn.q_a_layernorm", "model.layers.26.self_attn.indexer.k_norm", "model.layers.42.self_attn.kv_a_layernorm", "model.layers.1.self_attn.indexer.k_norm", "model.layers.39.post_attention_layernorm", "model.layers.48.mlp.gate.e_score_correction_bias", "model.layers.74.post_attention_layernorm", "model.layers.69.self_attn.q_a_layernorm", "model.layers.13.input_layernorm", "model.layers.74.self_attn.indexer.k_norm", "model.layers.29.mlp.gate", "model.layers.7.mlp.gate.e_score_correction_bias", "model.layers.3.self_attn.kv_a_layernorm", "model.layers.35.self_attn.kv_a_layernorm", "model.layers.46.self_attn.indexers_proj", "model.layers.0.self_attn.kv_a_layernorm", "model.layers.43.self_attn.q_a_layernorm", "model.norm", "model.layers.1.self_attn.kv_a_layernorm", "model.layers.38.self_attn.indexer.k_norm.bias", "model.layers.30.self_attn.q_a_layernorm", "model.layers.0.input_layernorm", "model.layers.47.input_layernorm", "model.layers.8.self_attn.q_a_layernorm", "model.layers.61.self_attn.q_a_layernorm", "model.layers.1.self_attn.indexer.k_norm.bias", "model.layers.62.mlp.gate.e_score_correction_bias", "model.layers.66.self_attn.q_a_layernorm", "model.layers.46.self_attn.kv_a_layernorm", "model.layers.66.mlp.gate", "model.layers.69.input_layernorm", "model.layers.69.self_attn.kv_a_layernorm", "model.layers.22.input_layernorm", "model.layers.30.post_attention_layernorm", "model.layers.24.self_attn.q_a_layernorm", "model.layers.48.mlp.gate", "model.layers.7.self_attn.kv_a_layernorm", "model.layers.67.input_layernorm", "model.layers.27.self_attn.q_a_layernorm", "model.layers.23.mlp.gate", "model.layers.45.mlp.gate.e_score_correction_bias", "model.layers.76.self_attn.kv_a_layernorm", "model.layers.22.self_attn.q_a_layernorm", "model.layers.78.mlp.gate", "model.layers.9.mlp.gate", "model.layers.57.self_attn.q_a_layernorm", "model.layers.78.mlp.gate.e_score_correction_bias", "model.layers.23.post_attention_layernorm", "model.layers.38.self_attn.q_a_layernorm", "model.layers.53.mlp.gate", "model.layers.77.self_attn.q_a_layernorm", "model.layers.62.self_attn.kv_a_layernorm", "model.layers.12.mlp.gate", "model.layers.34.self_attn.indexer.k_norm", "model.layers.46.self_attn.q_a_layernorm", "model.layers.36.self_attn.q_a_layernorm", "model.layers.58.self_attn.indexer.k_norm.bias", "model.layers.60.post_attention_layernorm", "model.layers.28.post_attention_layernorm", "model.layers.2.self_attn.kv_a_layernorm", "model.layers.70.self_attn.indexers_proj", "model.layers.16.mlp.gate.e_score_correction_bias", "model.layers.2.self_attn.indexer.k_norm", "model.layers.31.self_attn.kv_a_layernorm", "model.layers.64.input_layernorm", "model.layers.6.mlp.gate.e_score_correction_bias", "model.layers.50.self_attn.indexer.k_norm", "model.layers.58.self_attn.q_a_layernorm", "model.layers.57.mlp.gate.e_score_correction_bias", "model.layers.13.post_attention_layernorm", "model.layers.28.self_attn.kv_a_layernorm", "model.layers.31.self_attn.q_a_layernorm", "model.layers.1.self_attn.q_a_layernorm", "model.layers.17.mlp.gate", "model.layers.15.post_attention_layernorm", "model.layers.18.self_attn.indexer.k_norm.bias", "model.layers.18.self_attn.indexer.k_norm", "model.layers.43.mlp.gate.e_score_correction_bias", "model.layers.11.input_layernorm", "model.layers.9.post_attention_layernorm", "model.layers.33.input_layernorm", "model.layers.66.input_layernorm", "model.layers.78.self_attn.indexers_proj", "model.layers.54.post_attention_layernorm", "model.layers.53.mlp.gate.e_score_correction_bias", "model.layers.5.mlp.gate.e_score_correction_bias", "model.layers.19.mlp.gate", "model.layers.49.mlp.gate", "model.layers.66.self_attn.indexers_proj", "model.layers.14.self_attn.indexer.k_norm.bias", "model.layers.41.mlp.gate", "model.layers.42.self_attn.indexer.k_norm.bias", "model.layers.65.input_layernorm", "model.layers.75.mlp.gate", "model.layers.12.self_attn.kv_a_layernorm", "model.layers.53.input_layernorm", "model.layers.58.self_attn.kv_a_layernorm", "model.layers.60.self_attn.q_a_layernorm", "model.layers.6.mlp.gate", "model.layers.49.self_attn.kv_a_layernorm", "model.layers.16.mlp.gate", "model.layers.22.mlp.gate", "model.layers.58.post_attention_layernorm", "model.layers.15.mlp.gate.e_score_correction_bias", "model.layers.26.input_layernorm", "model.layers.3.post_attention_layernorm", "model.layers.6.input_layernorm", "model.layers.22.self_attn.kv_a_layernorm", "model.layers.16.input_layernorm", "model.layers.33.post_attention_layernorm", "model.layers.17.post_attention_layernorm", "model.layers.35.mlp.gate", "model.layers.56.post_attention_layernorm", "model.layers.7.self_attn.q_a_layernorm", "model.layers.24.mlp.gate", "model.layers.24.post_attention_layernorm", "model.layers.36.mlp.gate.e_score_correction_bias", "model.layers.56.mlp.gate.e_score_correction_bias", "model.layers.32.post_attention_layernorm", "model.layers.47.post_attention_layernorm", "model.layers.40.mlp.gate", "model.layers.41.self_attn.kv_a_layernorm", "model.layers.52.self_attn.kv_a_layernorm", "model.layers.9.self_attn.kv_a_layernorm", "model.layers.77.mlp.gate.e_score_correction_bias", "model.layers.78.shared_head.norm", "model.layers.53.post_attention_layernorm", "model.layers.63.self_attn.kv_a_layernorm", "model.layers.48.self_attn.kv_a_layernorm", "model.layers.28.mlp.gate.e_score_correction_bias", "model.layers.8.mlp.gate.e_score_correction_bias", "model.layers.52.mlp.gate.e_score_correction_bias", "model.layers.52.post_attention_layernorm", "model.layers.18.self_attn.q_a_layernorm", "model.layers.50.mlp.gate.e_score_correction_bias", "model.layers.64.mlp.gate.e_score_correction_bias", "model.layers.38.mlp.gate.e_score_correction_bias", "model.layers.59.input_layernorm", "model.layers.21.input_layernorm", "model.layers.39.mlp.gate", "model.layers.77.post_attention_layernorm", "model.layers.76.self_attn.q_a_layernorm", "model.layers.12.post_attention_layernorm", "model.layers.43.post_attention_layernorm", "model.layers.35.input_layernorm", "model.layers.62.mlp.gate", "model.layers.38.self_attn.kv_a_layernorm", "model.layers.63.mlp.gate", "model.layers.27.mlp.gate.e_score_correction_bias", "model.layers.18.mlp.gate", "model.layers.25.post_attention_layernorm", "model.layers.55.mlp.gate.e_score_correction_bias", "model.layers.66.self_attn.indexer.k_norm", "model.layers.48.self_attn.q_a_layernorm", "model.layers.78.eh_proj", "model.layers.77.mlp.gate", "model.layers.6.self_attn.indexers_proj", "model.layers.14.self_attn.q_a_layernorm", "model.layers.2.self_attn.q_a_layernorm", "model.layers.42.mlp.gate", "model.layers.54.mlp.gate.e_score_correction_bias", "model.layers.60.input_layernorm", "model.layers.40.self_attn.kv_a_layernorm", "model.layers.51.self_attn.kv_a_layernorm", "model.layers.18.input_layernorm", "model.layers.30.self_attn.indexers_proj", "model.layers.53.self_attn.kv_a_layernorm", "model.layers.6.post_attention_layernorm", "model.layers.70.self_attn.indexer.k_norm.bias", "model.layers.54.input_layernorm", "model.layers.41.self_attn.q_a_layernorm", "model.layers.18.mlp.gate.e_score_correction_bias", "model.layers.34.mlp.gate.e_score_correction_bias", "model.layers.47.mlp.gate", "model.layers.46.mlp.gate", "model.layers.25.self_attn.q_a_layernorm", "model.layers.63.mlp.gate.e_score_correction_bias", "model.layers.66.post_attention_layernorm", "model.layers.76.mlp.gate", "model.embed_tokens", "model.layers.67.self_attn.kv_a_layernorm", "model.layers.67.mlp.gate.e_score_correction_bias", "model.layers.26.self_attn.kv_a_layernorm", "model.layers.31.mlp.gate", "model.layers.18.self_attn.kv_a_layernorm", "model.layers.34.mlp.gate", "model.layers.55.post_attention_layernorm", "model.layers.30.input_layernorm", "model.layers.50.mlp.gate", "model.layers.63.input_layernorm", "model.layers.11.self_attn.kv_a_layernorm", "model.layers.24.self_attn.kv_a_layernorm", "model.layers.75.mlp.gate.e_score_correction_bias", "model.layers.10.post_attention_layernorm", "model.layers.1.self_attn.indexers_proj", "model.layers.42.self_attn.indexers_proj", "model.layers.46.input_layernorm", "model.layers.23.self_attn.q_a_layernorm", "model.layers.63.self_attn.q_a_layernorm", "model.layers.14.self_attn.indexer.k_norm", "model.layers.14.self_attn.indexers_proj", "model.layers.60.mlp.gate", "model.layers.70.mlp.gate.e_score_correction_bias", "model.layers.53.self_attn.q_a_layernorm", "model.layers.0.post_attention_layernorm", "model.layers.35.mlp.gate.e_score_correction_bias", "model.layers.75.self_attn.kv_a_layernorm", "model.layers.1.input_layernorm", "model.layers.42.post_attention_layernorm", "model.layers.37.input_layernorm", "model.layers.9.mlp.gate.e_score_correction_bias", "model.layers.46.post_attention_layernorm", "model.layers.26.mlp.gate", "model.layers.65.mlp.gate.e_score_correction_bias", "model.layers.67.self_attn.q_a_layernorm", "model.layers.44.mlp.gate.e_score_correction_bias", "model.layers.33.mlp.gate", "model.layers.29.input_layernorm", "model.layers.18.post_attention_layernorm", "model.layers.75.post_attention_layernorm", "model.layers.44.self_attn.q_a_layernorm", "model.layers.10.mlp.gate", "model.layers.21.mlp.gate", "model.layers.58.mlp.gate.e_score_correction_bias", "model.layers.7.post_attention_layernorm", "model.layers.50.self_attn.q_a_layernorm", "model.layers.9.self_attn.q_a_layernorm", "model.layers.2.self_attn.indexers_proj", "model.layers.0.self_attn.indexer.k_norm.bias", "model.layers.74.self_attn.kv_a_layernorm", "model.layers.74.mlp.gate.e_score_correction_bias", "model.layers.49.mlp.gate.e_score_correction_bias", "model.layers.62.post_attention_layernorm", "model.layers.57.self_attn.kv_a_layernorm", "model.layers.40.input_layernorm", "model.layers.69.mlp.gate.e_score_correction_bias", "model.layers.35.post_attention_layernorm", "model.layers.74.self_attn.indexers_proj", "model.layers.55.input_layernorm", "model.layers.49.input_layernorm", "model.layers.40.mlp.gate.e_score_correction_bias", "model.layers.25.mlp.gate", "model.layers.2.self_attn.indexer.k_norm.bias", "model.layers.68.self_attn.kv_a_layernorm", "model.layers.51.input_layernorm", "model.layers.22.self_attn.indexers_proj", "model.layers.10.input_layernorm", "model.layers.42.self_attn.indexer.k_norm", "model.layers.56.self_attn.kv_a_layernorm", "model.layers.17.self_attn.q_a_layernorm", "model.layers.59.self_attn.kv_a_layernorm", "model.layers.3.input_layernorm", "model.layers.57.input_layernorm", "model.layers.38.input_layernorm", "model.layers.62.self_attn.indexer.k_norm", "model.layers.20.mlp.gate", "model.layers.64.mlp.gate", "model.layers.28.self_attn.q_a_layernorm", "model.layers.32.self_attn.kv_a_layernorm", "model.layers.10.mlp.gate.e_score_correction_bias", "model.layers.5.self_attn.q_a_layernorm", "model.layers.37.self_attn.kv_a_layernorm" ] } } }, { "id": "zai-org/GLM-5.2", "meta": { "sha": "cf457fa734ab149ffef225f80893eb38c6ff5cdc", "safetensors": { "parameters": { "BF16": 753329921024, "F32": 19456 }, "total": 753329940480 }, "modelType": "glm_moe_dsa" }, "config": { "architectures": [ "GlmMoeDsaForCausalLM" ], "attention_bias": false, "attention_dropout": 0, "dtype": "bfloat16", "eos_token_id": [ 154820, 154827, 154829 ], "ep_size": 1, "first_k_dense_replace": 3, "head_dim": 192, "hidden_act": "silu", "hidden_size": 6144, "index_head_dim": 128, "index_n_heads": 32, "index_share_for_mtp_iteration": true, "index_skip_topk_offset": 3, "index_topk": 2048, "index_topk_freq": 4, "index_topk_pattern": null, "indexer_rope_interleave": true, "indexer_types": [ "full", "full", "full", "shared", "shared", "shared", "full", "shared", "shared", "shared", "full", "shared", "shared", "shared", "full", "shared", "shared", "shared", "full", "shared", "shared", "shared", "full", "shared", "shared", "shared", "full", "shared", "shared", "shared", "full", "shared", "shared", "shared", "full", "shared", "shared", "shared", "full", "shared", "shared", "shared", "full", "shared", "shared", "shared", "full", "shared", "shared", "shared", "full", "shared", "shared", "shared", "full", "shared", "shared", "shared", "full", "shared", "shared", "shared", "full", "shared", "shared", "shared", "full", "shared", "shared", "shared", "full", "shared", "shared", "shared", "full", "shared", "shared", "shared" ], "initializer_range": 0.02, "intermediate_size": 12288, "kv_lora_rank": 512, "max_position_embeddings": 1048576, "mlp_layer_types": [ "dense", "dense", "dense", "sparse", "sparse", "sparse", "sparse", "sparse", "sparse", "sparse", "sparse", "sparse", "sparse", "sparse", "sparse", "sparse", "sparse", "sparse", "sparse", "sparse", "sparse", "sparse", "sparse", "sparse", "sparse", "sparse", "sparse", "sparse", "sparse", "sparse", "sparse", "sparse", "sparse", "sparse", "sparse", "sparse", "sparse", "sparse", "sparse", "sparse", "sparse", "sparse", "sparse", "sparse", "sparse", "sparse", "sparse", "sparse", "sparse", "sparse", "sparse", "sparse", "sparse", "sparse", "sparse", "sparse", "sparse", "sparse", "sparse", "sparse", "sparse", "sparse", "sparse", "sparse", "sparse", "sparse", "sparse", "sparse", "sparse", "sparse", "sparse", "sparse", "sparse", "sparse", "sparse", "sparse", "sparse", "sparse" ], "model_type": "glm_moe_dsa", "moe_intermediate_size": 2048, "moe_layer_freq": 1, "moe_router_dtype": "float32", "n_group": 1, "n_routed_experts": 256, "n_shared_experts": 1, "norm_topk_prob": true, "num_attention_heads": 64, "num_experts_per_tok": 8, "num_hidden_layers": 78, "num_key_value_heads": 64, "num_nextn_predict_layers": 1, "pad_token_id": 154820, "pretraining_tp": 1, "q_lora_rank": 2048, "qk_head_dim": 256, "qk_nope_head_dim": 192, "qk_rope_head_dim": 64, "rms_norm_eps": 1e-05, "rope_interleave": true, "rope_parameters": { "rope_theta": 8000000, "rope_type": "default" }, "routed_scaling_factor": 2.5, "scoring_func": "sigmoid", "tie_word_embeddings": false, "topk_group": 1, "topk_method": "noaux_tc", "transformers_version": "5.12.0", "use_cache": true, "v_head_dim": 256, "vocab_size": 154880 } }, { "id": "zai-org/GLM-4.7", "meta": { "sha": "602d01efcdd332c5238ca4bcede555defbe83eb7", "safetensors": { "parameters": { "BF16": 358337776896, "F32": 14400 }, "total": 358337791296 }, "modelType": "glm4_moe" }, "config": { "architectures": [ "Glm4MoeForCausalLM" ], "attention_bias": true, "attention_dropout": 0, "pad_token_id": 151329, "eos_token_id": [ 151329, 151336, 151338 ], "head_dim": 128, "hidden_act": "silu", "hidden_size": 5120, "partial_rotary_factor": 0.5, "initializer_range": 0.02, "intermediate_size": 12288, "max_position_embeddings": 202752, "model_type": "glm4_moe", "moe_intermediate_size": 1536, "norm_topk_prob": true, "num_attention_heads": 96, "n_group": 1, "topk_group": 1, "n_routed_experts": 160, "n_shared_experts": 1, "routed_scaling_factor": 2.5, "num_experts_per_tok": 8, "first_k_dense_replace": 3, "num_hidden_layers": 92, "num_key_value_heads": 8, "rms_norm_eps": 1e-05, "rope_scaling": null, "rope_theta": 1000000, "num_nextn_predict_layers": 1, "tie_word_embeddings": false, "torch_dtype": "bfloat16", "transformers_version": "4.54.0", "use_cache": true, "use_qk_norm": true, "vocab_size": 151552 } }, { "id": "zai-org/GLM-4.5", "meta": { "sha": "cbb2c7cfb52fa128a9660cb1a7a78e017899e115", "safetensors": { "parameters": { "BF16": 358337776896, "F32": 14400 }, "total": 358337791296 }, "modelType": "glm4_moe" }, "config": { "architectures": [ "Glm4MoeForCausalLM" ], "attention_bias": true, "attention_dropout": 0, "pad_token_id": 151329, "eos_token_id": [ 151329, 151336, 151338 ], "head_dim": 128, "hidden_act": "silu", "hidden_size": 5120, "partial_rotary_factor": 0.5, "initializer_range": 0.02, "intermediate_size": 12288, "max_position_embeddings": 131072, "model_type": "glm4_moe", "moe_intermediate_size": 1536, "norm_topk_prob": true, "num_attention_heads": 96, "n_group": 1, "topk_group": 1, "n_routed_experts": 160, "n_shared_experts": 1, "routed_scaling_factor": 2.5, "num_experts_per_tok": 8, "first_k_dense_replace": 3, "num_hidden_layers": 92, "num_key_value_heads": 8, "rms_norm_eps": 1e-05, "rope_scaling": null, "rope_theta": 1000000, "num_nextn_predict_layers": 1, "tie_word_embeddings": false, "torch_dtype": "bfloat16", "transformers_version": "4.54.0", "use_cache": true, "use_qk_norm": true, "vocab_size": 151552 } }, { "id": "zai-org/GLM-4.5-Air", "meta": { "sha": "a24ceef6ce4f3536971efe9b778bdaa1bab18daa", "safetensors": { "parameters": { "BF16": 110468818944, "F32": 5888 }, "total": 110468824832 }, "modelType": "glm4_moe" }, "config": { "architectures": [ "Glm4MoeForCausalLM" ], "attention_bias": true, "attention_dropout": 0, "pad_token_id": 151329, "eos_token_id": [ 151329, 151336, 151338 ], "head_dim": 128, "hidden_act": "silu", "hidden_size": 4096, "partial_rotary_factor": 0.5, "initializer_range": 0.02, "intermediate_size": 10944, "max_position_embeddings": 131072, "model_type": "glm4_moe", "moe_intermediate_size": 1408, "norm_topk_prob": true, "num_attention_heads": 96, "n_group": 1, "topk_group": 1, "n_routed_experts": 128, "n_shared_experts": 1, "routed_scaling_factor": 1, "num_experts_per_tok": 8, "first_k_dense_replace": 1, "num_hidden_layers": 46, "num_key_value_heads": 8, "rms_norm_eps": 1e-05, "rope_scaling": null, "rope_theta": 1000000, "num_nextn_predict_layers": 1, "tie_word_embeddings": false, "torch_dtype": "bfloat16", "transformers_version": "4.54.0", "use_cache": true, "use_qk_norm": false, "vocab_size": 151552 } }, { "id": "zai-org/GLM-5.1", "meta": { "sha": "26e1bd6e011feb778d25ae34b09b07074139d92d", "safetensors": { "parameters": { "BF16": 753864119552, "F32": 19456 }, "total": 753864139008 }, "modelType": "glm_moe_dsa" }, "config": { "architectures": [ "GlmMoeDsaForCausalLM" ], "attention_bias": false, "attention_dropout": 0, "dtype": "bfloat16", "eos_token_id": [ 154820, 154827, 154829 ], "ep_size": 1, "first_k_dense_replace": 3, "hidden_act": "silu", "head_dim": 64, "hidden_size": 6144, "index_head_dim": 128, "index_n_heads": 32, "index_topk": 2048, "indexer_rope_interleave": true, "initializer_range": 0.02, "intermediate_size": 12288, "kv_lora_rank": 512, "max_position_embeddings": 202752, "moe_intermediate_size": 2048, "moe_layer_freq": 1, "model_type": "glm_moe_dsa", "n_group": 1, "n_routed_experts": 256, "n_shared_experts": 1, "norm_topk_prob": true, "num_attention_heads": 64, "num_experts_per_tok": 8, "num_hidden_layers": 78, "num_key_value_heads": 64, "num_nextn_predict_layers": 1, "pad_token_id": 154820, "pretraining_tp": 1, "q_lora_rank": 2048, "qk_head_dim": 256, "qk_nope_head_dim": 192, "qk_rope_head_dim": 64, "rms_norm_eps": 1e-05, "rope_interleave": true, "rope_parameters": { "rope_theta": 1000000, "rope_type": "default" }, "routed_scaling_factor": 2.5, "scoring_func": "sigmoid", "tie_word_embeddings": false, "topk_group": 1, "topk_method": "noaux_tc", "transformers_version": "5.4.0", "use_cache": true, "v_head_dim": 256, "vocab_size": 154880 } }, { "id": "zai-org/GLM-5", "meta": { "sha": "c183ef8c61faee82855eca1ed9bb3a9a7ce3b0b2", "safetensors": { "parameters": { "BF16": 753864119552, "F32": 19456 }, "total": 753864139008 }, "modelType": "glm_moe_dsa" }, "config": { "architectures": [ "GlmMoeDsaForCausalLM" ], "attention_bias": false, "attention_dropout": 0, "dtype": "bfloat16", "eos_token_id": [ 154820, 154827, 154829 ], "ep_size": 1, "first_k_dense_replace": 3, "hidden_act": "silu", "head_dim": 64, "hidden_size": 6144, "index_head_dim": 128, "index_n_heads": 32, "index_topk": 2048, "indexer_rope_interleave": true, "initializer_range": 0.02, "intermediate_size": 12288, "kv_lora_rank": 512, "max_position_embeddings": 202752, "moe_intermediate_size": 2048, "moe_layer_freq": 1, "model_type": "glm_moe_dsa", "n_group": 1, "n_routed_experts": 256, "n_shared_experts": 1, "norm_topk_prob": true, "num_attention_heads": 64, "num_experts_per_tok": 8, "num_hidden_layers": 78, "num_key_value_heads": 64, "num_nextn_predict_layers": 1, "pad_token_id": 154820, "pretraining_tp": 1, "q_lora_rank": 2048, "qk_head_dim": 256, "qk_nope_head_dim": 192, "qk_rope_head_dim": 64, "rms_norm_eps": 1e-05, "rope_interleave": true, "rope_parameters": { "rope_theta": 1000000, "rope_type": "default" }, "routed_scaling_factor": 2.5, "scoring_func": "sigmoid", "tie_word_embeddings": false, "topk_group": 1, "topk_method": "noaux_tc", "transformers_version": "5.0.2.dev0", "use_cache": true, "v_head_dim": 256, "vocab_size": 154880 } }, { "id": "zai-org/GLM-Z1-32B-0414", "meta": { "sha": "8eb2858992c1f749e2a6d4075455decc2484722d", "safetensors": { "parameters": { "BF16": 32566081536 }, "total": 32566081536 }, "modelType": "glm4" }, "config": { "architectures": [ "Glm4ForCausalLM" ], "attention_bias": false, "attention_dropout": 0, "eos_token_id": [ 151329, 151336, 151338 ], "head_dim": 128, "hidden_act": "silu", "hidden_size": 6144, "initializer_range": 0.02, "intermediate_size": 23040, "max_position_embeddings": 32768, "model_type": "glm4", "num_attention_heads": 48, "num_hidden_layers": 61, "num_key_value_heads": 2, "pad_token_id": 151329, "partial_rotary_factor": 0.5, "rms_norm_eps": 1e-05, "rope_theta": 10000, "tie_word_embeddings": false, "torch_dtype": "bfloat16", "transformers_version": "4.52.0.dev0", "use_cache": true, "vocab_size": 151552 } }, { "id": "zai-org/GLM-5.3-BF16", "meta": { "sha": "9d2398f478cab2de883137db3a36ad2c96205e24", "safetensors": { "parameters": { "BF16": 753329921024, "F32": 19456 }, "total": 753329940480 }, "modelType": "glm_moe_dsa" }, "config": { "architectures": [ "GlmMoeDsaForCausalLM" ], "attention_bias": false, "attention_dropout": 0, "dtype": "bfloat16", "eos_token_id": [ 154820, 154827, 154829 ], "ep_size": 1, "first_k_dense_replace": 3, "head_dim": 192, "hidden_act": "silu", "hidden_size": 6144, "index_head_dim": 128, "index_n_heads": 32, "index_share_for_mtp_iteration": true, "index_skip_topk_offset": 3, "index_topk": 2048, "index_topk_freq": 4, "index_topk_pattern": null, "indexer_rope_interleave": true, "indexer_types": [ "full", "full", "full", "shared", "shared", "shared", "full", "shared", "shared", "shared", "full", "shared", "shared", "shared", "full", "shared", "shared", "shared", "full", "shared", "shared", "shared", "full", "shared", "shared", "shared", "full", "shared", "shared", "shared", "full", "shared", "shared", "shared", "full", "shared", "shared", "shared", "full", "shared", "shared", "shared", "full", "shared", "shared", "shared", "full", "shared", "shared", "shared", "full", "shared", "shared", "shared", "full", "shared", "shared", "shared", "full", "shared", "shared", "shared", "full", "shared", "shared", "shared", "full", "shared", "shared", "shared", "full", "shared", "shared", "shared", "full", "shared", "shared", "shared" ], "initializer_range": 0.02, "intermediate_size": 12288, "kv_lora_rank": 512, "max_position_embeddings": 1048576, "mlp_layer_types": [ "dense", "dense", "dense", "sparse", "sparse", "sparse", "sparse", "sparse", "sparse", "sparse", "sparse", "sparse", "sparse", "sparse", "sparse", "sparse", "sparse", "sparse", "sparse", "sparse", "sparse", "sparse", "sparse", "sparse", "sparse", "sparse", "sparse", "sparse", "sparse", "sparse", "sparse", "sparse", "sparse", "sparse", "sparse", "sparse", "sparse", "sparse", "sparse", "sparse", "sparse", "sparse", "sparse", "sparse", "sparse", "sparse", "sparse", "sparse", "sparse", "sparse", "sparse", "sparse", "sparse", "sparse", "sparse", "sparse", "sparse", "sparse", "sparse", "sparse", "sparse", "sparse", "sparse", "sparse", "sparse", "sparse", "sparse", "sparse", "sparse", "sparse", "sparse", "sparse", "sparse", "sparse", "sparse", "sparse", "sparse", "sparse" ], "model_type": "glm_moe_dsa", "moe_intermediate_size": 2048, "moe_layer_freq": 1, "moe_router_dtype": "float32", "n_group": 1, "n_routed_experts": 256, "n_shared_experts": 1, "norm_topk_prob": true, "num_attention_heads": 64, "num_experts_per_tok": 8, "num_hidden_layers": 78, "num_key_value_heads": 64, "num_nextn_predict_layers": 1, "pad_token_id": 154820, "pretraining_tp": 1, "q_lora_rank": 2048, "qk_head_dim": 256, "qk_nope_head_dim": 192, "qk_rope_head_dim": 64, "rms_norm_eps": 1e-05, "rope_interleave": true, "rope_parameters": { "rope_theta": 8000000, "rope_type": "default" }, "routed_scaling_factor": 2.5, "scoring_func": "sigmoid", "tie_word_embeddings": false, "topk_group": 1, "topk_method": "noaux_tc", "transformers_version": "5.15.0", "use_cache": true, "v_head_dim": 256, "vocab_size": 154880 } }, { "id": "zai-org/GLM-4-9B-0414", "meta": { "sha": "645b8482494e31b6b752272bf7f7f273ef0f3caf", "safetensors": { "parameters": { "BF16": 9400279040 }, "total": 9400279040 }, "modelType": "glm4" }, "config": { "architectures": [ "Glm4ForCausalLM" ], "attention_bias": true, "attention_dropout": 0, "eos_token_id": [ 151329, 151336, 151338 ], "head_dim": 128, "hidden_act": "silu", "hidden_size": 4096, "initializer_range": 0.02, "intermediate_size": 13696, "max_position_embeddings": 32768, "model_type": "glm4", "num_attention_heads": 32, "num_hidden_layers": 40, "num_key_value_heads": 2, "pad_token_id": 151329, "partial_rotary_factor": 0.5, "rms_norm_eps": 1e-05, "rope_theta": 10000, "tie_word_embeddings": false, "torch_dtype": "bfloat16", "transformers_version": "4.52.0.dev0", "use_cache": true, "vocab_size": 151552 } }, { "id": "zai-org/GLM-4.6", "meta": { "sha": "be72194883d968d7923a07e2f61681ea9a2826d1", "safetensors": { "parameters": { "BF16": 356785884416, "F32": 14400 }, "total": 356785898816 }, "modelType": "glm4_moe" }, "config": { "architectures": [ "Glm4MoeForCausalLM" ], "attention_bias": true, "attention_dropout": 0, "pad_token_id": 151329, "eos_token_id": [ 151329, 151336, 151338 ], "head_dim": 128, "hidden_act": "silu", "hidden_size": 5120, "partial_rotary_factor": 0.5, "initializer_range": 0.02, "intermediate_size": 12288, "max_position_embeddings": 202752, "model_type": "glm4_moe", "moe_intermediate_size": 1536, "norm_topk_prob": true, "num_attention_heads": 96, "n_group": 1, "topk_group": 1, "n_routed_experts": 160, "n_shared_experts": 1, "routed_scaling_factor": 2.5, "num_experts_per_tok": 8, "first_k_dense_replace": 3, "num_hidden_layers": 92, "num_key_value_heads": 8, "rms_norm_eps": 1e-05, "rope_scaling": null, "rope_theta": 1000000, "num_nextn_predict_layers": 1, "tie_word_embeddings": false, "torch_dtype": "bfloat16", "transformers_version": "4.54.0", "use_cache": true, "use_qk_norm": true, "vocab_size": 151552 } }, { "id": "zai-org/codegeex4-all-9b", "meta": { "sha": "6ee90cf42fbd24807825b5ff6bed9830a5a4cfb2", "safetensors": { "parameters": { "BF16": 9399951392 }, "total": 9399951392 }, "modelType": "chatglm" }, "config": { "_name_or_path": "THUDM/codegeex4-all-9b", "add_bias_linear": false, "add_qkv_bias": true, "apply_query_key_layer_scaling": true, "apply_residual_connection_post_layernorm": false, "architectures": [ "ChatGLMModel" ], "attention_dropout": 0, "attention_softmax_in_fp32": true, "auto_map": { "AutoConfig": "configuration_chatglm.ChatGLMConfig", "AutoModel": "modeling_chatglm.ChatGLMForConditionalGeneration", "AutoModelForCausalLM": "modeling_chatglm.ChatGLMForConditionalGeneration", "AutoModelForSeq2SeqLM": "modeling_chatglm.ChatGLMForConditionalGeneration", "AutoModelForSequenceClassification": "modeling_chatglm.ChatGLMForSequenceClassification" }, "bias_dropout_fusion": true, "classifier_dropout": null, "eos_token_id": [ 151329, 151336, 151338 ], "ffn_hidden_size": 13696, "fp32_residual_connection": false, "hidden_dropout": 0, "hidden_size": 4096, "kv_channels": 128, "layernorm_epsilon": 1e-05, "model_type": "chatglm", "multi_query_attention": true, "multi_query_group_num": 2, "num_attention_heads": 32, "num_hidden_layers": 40, "num_layers": 40, "original_rope": true, "pad_token_id": 151329, "padded_vocab_size": 151552, "post_layer_norm": true, "rmsnorm": true, "rope_ratio": 500, "seq_length": 131072, "tie_word_embeddings": false, "torch_dtype": "bfloat16", "transformers_version": "4.40.2", "use_cache": true, "vocab_size": 151552 } }, { "id": "zai-org/GLM-4.5-Air-Base", "meta": { "sha": "888c873d4eca81f28d0ef420aa2d96457c28b959", "safetensors": { "parameters": { "BF16": 110468818944, "F32": 5888 }, "total": 110468824832 }, "modelType": "glm4_moe" }, "config": { "architectures": [ "Glm4MoeForCausalLM" ], "attention_bias": true, "attention_dropout": 0, "pad_token_id": 151329, "eos_token_id": [ 151329, 151336, 151338 ], "head_dim": 128, "hidden_act": "silu", "hidden_size": 4096, "partial_rotary_factor": 0.5, "initializer_range": 0.02, "intermediate_size": 10944, "max_position_embeddings": 131072, "model_type": "glm4_moe", "moe_intermediate_size": 1408, "norm_topk_prob": true, "num_attention_heads": 96, "n_group": 1, "topk_group": 1, "n_routed_experts": 128, "n_shared_experts": 1, "routed_scaling_factor": 1, "num_experts_per_tok": 8, "first_k_dense_replace": 1, "num_hidden_layers": 46, "num_key_value_heads": 8, "rms_norm_eps": 1e-05, "rope_scaling": null, "rope_theta": 1000000, "num_nextn_predict_layers": 1, "tie_word_embeddings": false, "torch_dtype": "bfloat16", "transformers_version": "4.54.0", "use_cache": true, "use_qk_norm": false, "vocab_size": 151552 } }, { "id": "zai-org/glm-4-9b", "meta": { "sha": "8cd2b585357ba9e702647ac4e6fa4fafe5cc7bee", "safetensors": { "parameters": { "BF16": 9399951392 }, "total": 9399951392 }, "modelType": "chatglm" }, "config": { "_name_or_path": "THUDM/glm-4-9b", "model_type": "chatglm", "architectures": [ "ChatGLMModel" ], "auto_map": { "AutoConfig": "configuration_chatglm.ChatGLMConfig", "AutoModel": "modeling_chatglm.ChatGLMForConditionalGeneration", "AutoModelForCausalLM": "modeling_chatglm.ChatGLMForConditionalGeneration", "AutoModelForSeq2SeqLM": "modeling_chatglm.ChatGLMForConditionalGeneration", "AutoModelForSequenceClassification": "modeling_chatglm.ChatGLMForSequenceClassification" }, "add_bias_linear": false, "add_qkv_bias": true, "apply_query_key_layer_scaling": true, "apply_residual_connection_post_layernorm": false, "attention_dropout": 0, "attention_softmax_in_fp32": true, "attn_implementation": "sdpa", "bias_dropout_fusion": true, "ffn_hidden_size": 13696, "fp32_residual_connection": false, "hidden_dropout": 0, "hidden_size": 4096, "kv_channels": 128, "layernorm_epsilon": 1.5625e-07, "multi_query_attention": true, "multi_query_group_num": 2, "num_attention_heads": 32, "num_layers": 40, "original_rope": true, "padded_vocab_size": 151552, "post_layer_norm": true, "rmsnorm": true, "seq_length": 8192, "use_cache": true, "torch_dtype": "bfloat16", "transformers_version": "4.44.0", "tie_word_embeddings": false, "eos_token_id": [ 151329, 151336, 151338 ], "pad_token_id": 151329 } }, { "id": "zai-org/GLM-4-32B-0414", "meta": { "sha": "077b5c2f5c43bd3239fd605a0600229e8facbd4a", "safetensors": { "parameters": { "BF16": 32566081536 }, "total": 32566081536 }, "modelType": "glm4" }, "config": { "architectures": [ "Glm4ForCausalLM" ], "attention_bias": false, "attention_dropout": 0, "eos_token_id": [ 151329, 151336, 151338 ], "head_dim": 128, "hidden_act": "silu", "hidden_size": 6144, "initializer_range": 0.02, "intermediate_size": 23040, "max_position_embeddings": 32768, "model_type": "glm4", "num_attention_heads": 48, "num_hidden_layers": 61, "num_key_value_heads": 2, "pad_token_id": 151329, "partial_rotary_factor": 0.5, "rms_norm_eps": 1e-05, "rope_theta": 10000, "tie_word_embeddings": false, "torch_dtype": "bfloat16", "transformers_version": "4.52.0.dev0", "use_cache": true, "vocab_size": 151552 } }, { "id": "zai-org/glm-4-9b-chat-hf", "meta": { "sha": "8599336fc6c125203efb2360bfaf4c80eef1d1bf", "safetensors": { "parameters": { "BF16": 9399951360 }, "total": 9399951360 }, "modelType": "glm" }, "config": { "architectures": [ "GlmForCausalLM" ], "attention_bias": true, "attention_dropout": 0, "eos_token_id": [ 151329, 151336, 151338 ], "head_dim": 128, "hidden_act": "silu", "hidden_size": 4096, "initializer_range": 0.02, "intermediate_size": 13696, "max_position_embeddings": 131072, "model_type": "glm", "num_attention_heads": 32, "num_hidden_layers": 40, "num_key_value_heads": 2, "pad_token_id": 151329, "rms_norm_eps": 1.5625e-07, "rope_theta": 10000, "tie_word_embeddings": false, "torch_dtype": "bfloat16", "transformers_version": "4.46.0.dev0", "use_cache": true, "vocab_size": 151552 } }, { "id": "zai-org/glm-4-9b-hf", "meta": { "sha": "b44e98fcc8df0faba03a48b405356af6b91821e7", "safetensors": { "parameters": { "BF16": 9399951360 }, "total": 9399951360 }, "modelType": "glm" }, "config": { "architectures": [ "GlmForCausalLM" ], "attention_bias": true, "attention_dropout": 0, "eos_token_id": [ 151329, 151336, 151338 ], "head_dim": 128, "hidden_act": "silu", "hidden_size": 4096, "initializer_range": 0.02, "intermediate_size": 13696, "max_position_embeddings": 8192, "model_type": "glm", "num_attention_heads": 32, "num_hidden_layers": 40, "num_key_value_heads": 2, "pad_token_id": 151329, "rms_norm_eps": 1.5625e-07, "rope_theta": 10000, "tie_word_embeddings": false, "torch_dtype": "bfloat16", "transformers_version": "4.46.0.dev0", "use_cache": true, "vocab_size": 151552 } }, { "id": "zai-org/glm-edge-4b-chat", "meta": { "sha": "a1817f2ab339ecdd8497c4d752ea71a65299f29a", "safetensors": { "parameters": { "BF16": 4327984128 }, "total": 4327984128 }, "modelType": "glm" }, "config": { "architectures": [ "GlmForCausalLM" ], "partial_rotary_factor": 1, "attention_bias": false, "attention_dropout": 0, "eos_token_id": [ 59246, 59253, 59255 ], "head_dim": 128, "hidden_act": "silu", "hidden_size": 3072, "initializer_range": 0.02, "intermediate_size": 8192, "max_position_embeddings": 8192, "model_type": "glm", "num_attention_heads": 24, "num_hidden_layers": 40, "num_key_value_heads": 6, "pad_token_id": 59246, "rms_norm_eps": 1e-05, "rope_theta": 10000, "tie_word_embeddings": false, "torch_dtype": "bfloat16", "transformers_version": "4.47.0.dev0", "use_cache": true, "vocab_size": 59264 } }, { "id": "zai-org/GLM-Z1-9B-0414", "meta": { "sha": "b221b06fefb23ca320922cf6e68ab5f2fb82de81", "safetensors": { "parameters": { "BF16": 9400279040 }, "total": 9400279040 }, "modelType": "glm4" }, "config": { "architectures": [ "Glm4ForCausalLM" ], "attention_bias": true, "attention_dropout": 0, "eos_token_id": [ 151329, 151336, 151338 ], "head_dim": 128, "hidden_act": "silu", "hidden_size": 4096, "initializer_range": 0.02, "intermediate_size": 13696, "max_position_embeddings": 32768, "model_type": "glm4", "num_attention_heads": 32, "num_hidden_layers": 40, "num_key_value_heads": 2, "pad_token_id": 151329, "partial_rotary_factor": 0.5, "rms_norm_eps": 1e-05, "rope_theta": 10000, "tie_word_embeddings": false, "torch_dtype": "bfloat16", "transformers_version": "4.52.0.dev0", "use_cache": true, "vocab_size": 151552 } }, { "id": "zai-org/glm-edge-1.5b-chat", "meta": { "sha": "7b201d3c160c25beda4cf0d107617ad975cd1ca8", "safetensors": { "parameters": { "BF16": 1593427968 }, "total": 1593427968 }, "modelType": "glm" }, "config": { "architectures": [ "GlmForCausalLM" ], "partial_rotary_factor": 1, "attention_bias": false, "attention_dropout": 0, "eos_token_id": [ 59246, 59253, 59255 ], "head_dim": 128, "hidden_act": "silu", "hidden_size": 2048, "initializer_range": 0.02, "intermediate_size": 6144, "max_position_embeddings": 8192, "model_type": "glm", "num_attention_heads": 16, "num_hidden_layers": 28, "num_key_value_heads": 4, "pad_token_id": 59246, "rms_norm_eps": 1e-05, "rope_theta": 10000, "tie_word_embeddings": true, "torch_dtype": "bfloat16", "transformers_version": "4.47.0.dev0", "use_cache": true, "vocab_size": 59264 } }, { "id": "moonshotai/Kimi-K2-Instruct", "meta": { "sha": "fd1984e2b7a3350dbf7305fe73a4ede25c14de50", "safetensors": { "parameters": { "F32": 23040, "BF16": 2514970968, "F8_E4M3": 1023893241856 }, "total": 1026408235864 }, "modelType": "kimi_k2" }, "config": { "architectures": [ "DeepseekV3ForCausalLM" ], "attention_bias": false, "attention_dropout": 0.0, "auto_map": { "AutoConfig": "configuration_deepseek.DeepseekV3Config", "AutoModel": "modeling_deepseek.DeepseekV3Model", "AutoModelForCausalLM": "modeling_deepseek.DeepseekV3ForCausalLM" }, "aux_loss_alpha": 0.001, "bos_token_id": 163584, "eos_token_id": 163585, "first_k_dense_replace": 1, "hidden_act": "silu", "hidden_size": 7168, "initializer_range": 0.02, "intermediate_size": 18432, "kv_lora_rank": 512, "max_position_embeddings": 131072, "model_type": "kimi_k2", "moe_intermediate_size": 2048, "moe_layer_freq": 1, "n_group": 1, "n_routed_experts": 384, "n_shared_experts": 1, "norm_topk_prob": true, "num_attention_heads": 64, "num_experts_per_tok": 8, "num_hidden_layers": 61, "num_key_value_heads": 64, "num_nextn_predict_layers": 0, "pretraining_tp": 1, "q_lora_rank": 1536, "qk_nope_head_dim": 128, "qk_rope_head_dim": 64, "quantization_config": { "activation_scheme": "dynamic", "fmt": "e4m3", "quant_method": "fp8", "weight_block_size": [ 128, 128 ] }, "rms_norm_eps": 1e-06, "rope_theta": 50000.0, "routed_scaling_factor": 2.827, "rope_scaling": { "beta_fast": 1.0, "beta_slow": 1.0, "factor": 32.0, "mscale": 1.0, "mscale_all_dim": 1.0, "original_max_position_embeddings": 4096, "type": "yarn" }, "scoring_func": "sigmoid", "seq_aux": true, "tie_word_embeddings": false, "topk_group": 1, "topk_method": "noaux_tc", "torch_dtype": "bfloat16", "transformers_version": "4.48.3", "use_cache": true, "v_head_dim": 128, "vocab_size": 163840 }, "verifiedAt": "2026-09-22" }, { "id": "moonshotai/Kimi-Linear-48B-A3B-Instruct", "meta": { "sha": "e1df551a447157d4658b573f9a695d57658590e9", "safetensors": { "parameters": { "BF16": 49122599168, "F32": 82560 }, "total": 49122681728 }, "modelType": "kimi_linear" }, "config": { "architectures": [ "KimiLinearForCausalLM" ], "auto_map": { "AutoConfig": "configuration_kimi.KimiLinearConfig", "AutoModel": "modeling_kimi.KimiLinearModel", "AutoModelForCausalLM": "modeling_kimi.KimiLinearForCausalLM" }, "bos_token_id": 163584, "dtype": "bfloat16", "eos_token_id": 163586, "first_k_dense_replace": 1, "head_dim": 72, "hidden_act": "silu", "hidden_size": 2304, "initializer_range": 0.02, "intermediate_size": 9216, "kv_lora_rank": 512, "linear_attn_config": { "full_attn_layers": [ 4, 8, 12, 16, 20, 24, 27 ], "head_dim": 128, "kda_layers": [ 1, 2, 3, 5, 6, 7, 9, 10, 11, 13, 14, 15, 17, 18, 19, 21, 22, 23, 25, 26 ], "num_heads": 32, "short_conv_kernel_size": 4 }, "mla_use_nope": true, "model_max_length": 1048576, "model_type": "kimi_linear", "moe_intermediate_size": 1024, "moe_layer_freq": 1, "moe_renormalize": true, "moe_router_activation_func": "sigmoid", "num_attention_heads": 32, "num_expert_group": 1, "num_experts": 256, "num_experts_per_token": 8, "num_hidden_layers": 27, "num_key_value_heads": 32, "num_nextn_predict_layers": 0, "num_shared_experts": 1, "pad_token_id": 163839, "q_lora_rank": null, "qk_nope_head_dim": 128, "qk_rope_head_dim": 64, "rms_norm_eps": 1e-05, "rope_scaling": null, "rope_theta": 10000, "routed_scaling_factor": 2.446, "tie_word_embeddings": false, "topk_group": 1, "transformers_version": "4.57.1", "use_cache": true, "use_grouped_topk": true, "v_head_dim": 128, "vocab_size": 163840 } }, { "id": "moonshotai/Kimi-Linear-48B-A3B-Base", "meta": { "sha": "3b171c17bfc4ee348599b6781a2ca8715c21c8dc", "safetensors": { "parameters": { "BF16": 49122599168, "F32": 82560 }, "total": 49122681728 }, "modelType": "kimi_linear" }, "config": { "architectures": [ "KimiLinearForCausalLM" ], "auto_map": { "AutoConfig": "configuration_kimi.KimiLinearConfig", "AutoModel": "modeling_kimi.KimiLinearModel", "AutoModelForCausalLM": "modeling_kimi.KimiLinearForCausalLM" }, "bos_token_id": 163584, "dtype": "bfloat16", "eos_token_id": 163586, "first_k_dense_replace": 1, "head_dim": 72, "hidden_act": "silu", "hidden_size": 2304, "initializer_range": 0.02, "intermediate_size": 9216, "kv_lora_rank": 512, "linear_attn_config": { "full_attn_layers": [ 4, 8, 12, 16, 20, 24, 27 ], "head_dim": 128, "kda_layers": [ 1, 2, 3, 5, 6, 7, 9, 10, 11, 13, 14, 15, 17, 18, 19, 21, 22, 23, 25, 26 ], "num_heads": 32, "short_conv_kernel_size": 4 }, "mla_use_nope": true, "model_max_length": 1048576, "model_type": "kimi_linear", "moe_intermediate_size": 1024, "moe_layer_freq": 1, "moe_renormalize": true, "moe_router_activation_func": "sigmoid", "num_attention_heads": 32, "num_expert_group": 1, "num_experts": 256, "num_experts_per_token": 8, "num_hidden_layers": 27, "num_key_value_heads": 32, "num_nextn_predict_layers": 0, "num_shared_experts": 1, "pad_token_id": 163839, "q_lora_rank": null, "qk_nope_head_dim": 128, "qk_rope_head_dim": 64, "rms_norm_eps": 1e-05, "rope_scaling": null, "rope_theta": 10000, "routed_scaling_factor": 2.446, "tie_word_embeddings": false, "topk_group": 1, "transformers_version": "4.57.0", "use_cache": true, "use_grouped_topk": true, "v_head_dim": 128, "vocab_size": 163840 } }, { "id": "moonshotai/Kimi-K2-Thinking", "meta": { "sha": "a51ccc050d73dab088bf7b0e2dd9b30ae85a4e55", "safetensors": { "parameters": { "F32": 23040, "I32": 1014686023680, "BF16": 11722193536 }, "total": 1026408240256 }, "modelType": "kimi_k2" }, "config": { "_attn_implementation_autoset": false, "_name_or_path": "", "add_cross_attention": false, "architectures": [ "DeepseekV3ForCausalLM" ], "attention_bias": false, "attention_dropout": 0, "auto_map": { "AutoConfig": "configuration_deepseek.DeepseekV3Config", "AutoModel": "modeling_deepseek.DeepseekV3Model", "AutoModelForCausalLM": "modeling_deepseek.DeepseekV3ForCausalLM" }, "aux_loss_alpha": 0.001, "bad_words_ids": null, "begin_suppress_tokens": null, "bos_token_id": 163584, "chunk_size_feed_forward": 0, "cross_attention_hidden_size": null, "decoder_start_token_id": null, "diversity_penalty": 0, "do_sample": false, "early_stopping": false, "encoder_no_repeat_ngram_size": 0, "eos_token_id": 163586, "ep_size": 1, "exponential_decay_length_penalty": null, "finetuning_task": null, "first_k_dense_replace": 1, "forced_bos_token_id": null, "forced_eos_token_id": null, "hidden_act": "silu", "hidden_size": 7168, "id2label": { "0": "LABEL_0", "1": "LABEL_1" }, "initializer_range": 0.02, "intermediate_size": 18432, "is_decoder": false, "is_encoder_decoder": false, "kv_lora_rank": 512, "label2id": { "LABEL_0": 0, "LABEL_1": 1 }, "length_penalty": 1, "max_length": 20, "max_position_embeddings": 262144, "min_length": 0, "model_type": "kimi_k2", "moe_intermediate_size": 2048, "moe_layer_freq": 1, "n_group": 1, "n_routed_experts": 384, "n_shared_experts": 1, "no_repeat_ngram_size": 0, "norm_topk_prob": true, "num_attention_heads": 64, "num_beam_groups": 1, "num_beams": 1, "num_experts_per_tok": 8, "num_hidden_layers": 61, "num_key_value_heads": 64, "num_nextn_predict_layers": 0, "num_return_sequences": 1, "output_attentions": false, "output_hidden_states": false, "output_scores": false, "pad_token_id": 163839, "prefix": null, "pretraining_tp": 1, "problem_type": null, "pruned_heads": {}, "q_lora_rank": 1536, "qk_nope_head_dim": 128, "qk_rope_head_dim": 64, "quantization_config": { "config_groups": { "group_0": { "input_activations": null, "output_activations": null, "targets": [ "Linear" ], "weights": { "actorder": null, "block_structure": null, "dynamic": false, "group_size": 32, "num_bits": 4, "observer": "minmax", "observer_kwargs": {}, "strategy": "group", "symmetric": true, "type": "int" } } }, "format": "pack-quantized", "ignore": [ "lm_head", "re:.*self_attn.*", "re:.*shared_experts.*", "re:.*mlp\\.(gate|up|gate_up|down)_proj.*" ], "kv_cache_scheme": null, "quant_method": "compressed-tensors", "quantization_status": "compressed" }, "remove_invalid_values": false, "repetition_penalty": 1, "return_dict": true, "return_dict_in_generate": false, "rms_norm_eps": 1e-05, "rope_scaling": { "beta_fast": 1, "beta_slow": 1, "factor": 64, "mscale": 1, "mscale_all_dim": 1, "original_max_position_embeddings": 4096, "type": "yarn" }, "rope_theta": 50000, "routed_scaling_factor": 2.827, "scoring_func": "sigmoid", "sep_token_id": null, "seq_aux": true, "suppress_tokens": null, "task_specific_params": null, "temperature": 1, "tf_legacy_loss": false, "tie_encoder_decoder": false, "tie_word_embeddings": false, "tokenizer_class": null, "top_k": 50, "top_p": 1, "topk_group": 1, "topk_method": "noaux_tc", "torch_dtype": "bfloat16", "torchscript": false, "transformers_version": "4.51.3", "typical_p": 1, "use_bfloat16": false, "use_cache": true, "v_head_dim": 128, "vocab_size": 163840 } }, { "id": "moonshotai/Kimi-K2-Instruct-0905", "meta": { "sha": "ac6c49f04883bd0a0598b790693a72061c676629", "safetensors": { "parameters": { "F32": 62518232, "BF16": 2514975360, "F8_E4M3": 1023893241856 }, "total": 1026470735448 }, "modelType": "kimi_k2" }, "config": { "_attn_implementation_autoset": false, "_name_or_path": "", "add_cross_attention": false, "architectures": [ "DeepseekV3ForCausalLM" ], "attention_bias": false, "attention_dropout": 0.0, "auto_map": { "AutoConfig": "configuration_deepseek.DeepseekV3Config", "AutoModel": "modeling_deepseek.DeepseekV3Model", "AutoModelForCausalLM": "modeling_deepseek.DeepseekV3ForCausalLM" }, "aux_loss_alpha": 0.001, "bad_words_ids": null, "begin_suppress_tokens": null, "bos_token_id": 163584, "chunk_size_feed_forward": 0, "cross_attention_hidden_size": null, "decoder_start_token_id": null, "diversity_penalty": 0.0, "do_sample": false, "early_stopping": false, "encoder_no_repeat_ngram_size": 0, "eos_token_id": 163585, "ep_size": 1, "exponential_decay_length_penalty": null, "finetuning_task": null, "first_k_dense_replace": 1, "forced_bos_token_id": null, "forced_eos_token_id": null, "hidden_act": "silu", "hidden_size": 7168, "id2label": { "0": "LABEL_0", "1": "LABEL_1" }, "initializer_range": 0.02, "intermediate_size": 18432, "is_decoder": false, "is_encoder_decoder": false, "kv_lora_rank": 512, "label2id": { "LABEL_0": 0, "LABEL_1": 1 }, "length_penalty": 1.0, "max_length": 20, "max_position_embeddings": 262144, "min_length": 0, "model_type": "kimi_k2", "moe_intermediate_size": 2048, "moe_layer_freq": 1, "n_group": 1, "n_routed_experts": 384, "n_shared_experts": 1, "no_repeat_ngram_size": 0, "norm_topk_prob": true, "num_attention_heads": 64, "num_beam_groups": 1, "num_beams": 1, "num_experts_per_tok": 8, "num_hidden_layers": 61, "num_key_value_heads": 64, "num_nextn_predict_layers": 0, "num_return_sequences": 1, "output_attentions": false, "output_hidden_states": false, "output_scores": false, "pad_token_id": 163839, "prefix": null, "pretraining_tp": 1, "problem_type": null, "pruned_heads": {}, "q_lora_rank": 1536, "qk_nope_head_dim": 128, "qk_rope_head_dim": 64, "quantization_config": { "activation_scheme": "dynamic", "fmt": "e4m3", "quant_method": "fp8", "weight_block_size": [ 128, 128 ] }, "remove_invalid_values": false, "repetition_penalty": 1.0, "return_dict": true, "return_dict_in_generate": false, "rms_norm_eps": 1e-05, "rope_scaling": { "beta_fast": 1.0, "beta_slow": 1.0, "factor": 64.0, "mscale": 1.0, "mscale_all_dim": 1.0, "original_max_position_embeddings": 4096, "type": "yarn" }, "rope_theta": 50000.0, "routed_scaling_factor": 2.827, "scoring_func": "sigmoid", "sep_token_id": null, "seq_aux": true, "suppress_tokens": null, "task_specific_params": null, "temperature": 1.0, "tf_legacy_loss": false, "tie_encoder_decoder": false, "tie_word_embeddings": false, "tokenizer_class": null, "top_k": 50, "top_p": 1.0, "topk_group": 1, "topk_method": "noaux_tc", "torch_dtype": "bfloat16", "torchscript": false, "transformers_version": "4.51.3", "typical_p": 1.0, "use_bfloat16": false, "use_cache": true, "v_head_dim": 128, "vocab_size": 163840 }, "verifiedAt": "2026-09-22" }, { "id": "moonshotai/Moonlight-16B-A3B-Instruct", "meta": { "sha": "4e735b07a89f73647dfab71ab91b840f362ede5b", "safetensors": { "parameters": { "BF16": 15960111936 }, "total": 15960111936 }, "modelType": "deepseek_v3" }, "config": { "architectures": [ "DeepseekV3ForCausalLM" ], "attention_bias": false, "attention_dropout": 0, "auto_map": { "AutoConfig": "configuration_deepseek.DeepseekV3Config", "AutoModel": "modeling_deepseek.DeepseekV3Model", "AutoModelForCausalLM": "modeling_deepseek.DeepseekV3ForCausalLM" }, "aux_loss_alpha": 0.001, "bos_token_id": 163584, "eos_token_id": 163586, "ep_size": 1, "first_k_dense_replace": 1, "hidden_act": "silu", "hidden_size": 2048, "initializer_range": 0.02, "intermediate_size": 11264, "kv_lora_rank": 512, "max_position_embeddings": 8192, "model_type": "deepseek_v3", "moe_intermediate_size": 1408, "moe_layer_freq": 1, "n_group": 1, "n_routed_experts": 64, "n_shared_experts": 2, "norm_topk_prob": true, "num_attention_heads": 16, "num_experts_per_tok": 6, "num_hidden_layers": 27, "num_key_value_heads": 16, "num_nextn_predict_layers": 0, "pretraining_tp": 1, "q_lora_rank": null, "qk_nope_head_dim": 128, "qk_rope_head_dim": 64, "rms_norm_eps": 1e-05, "rope_theta": 50000, "routed_scaling_factor": 2.446, "scoring_func": "sigmoid", "seq_aux": true, "tie_word_embeddings": false, "topk_group": 1, "topk_method": "noaux_tc", "torch_dtype": "bfloat16", "transformers_version": "4.46.3", "use_cache": true, "v_head_dim": 128, "vocab_size": 163840 } }, { "id": "moonshotai/Moonlight-16B-A3B", "meta": { "sha": "476b36a473d4467f94469414bef6cee75c9c8172", "safetensors": { "parameters": { "BF16": 15960111936 }, "total": 15960111936 }, "modelType": "deepseek_v3" }, "config": { "architectures": [ "DeepseekV3ForCausalLM" ], "attention_bias": false, "attention_dropout": 0, "auto_map": { "AutoConfig": "configuration_deepseek.DeepseekV3Config", "AutoModel": "modeling_deepseek.DeepseekV3Model", "AutoModelForCausalLM": "modeling_deepseek.DeepseekV3ForCausalLM" }, "aux_loss_alpha": 0.001, "bos_token_id": 163584, "eos_token_id": 163585, "ep_size": 1, "first_k_dense_replace": 1, "hidden_act": "silu", "hidden_size": 2048, "initializer_range": 0.02, "intermediate_size": 11264, "kv_lora_rank": 512, "max_position_embeddings": 8192, "model_type": "deepseek_v3", "moe_intermediate_size": 1408, "moe_layer_freq": 1, "n_group": 1, "n_routed_experts": 64, "n_shared_experts": 2, "norm_topk_prob": true, "num_attention_heads": 16, "num_experts_per_tok": 6, "num_hidden_layers": 27, "num_key_value_heads": 16, "num_nextn_predict_layers": 0, "pretraining_tp": 1, "q_lora_rank": null, "qk_nope_head_dim": 128, "qk_rope_head_dim": 64, "rms_norm_eps": 1e-05, "rope_theta": 50000, "routed_scaling_factor": 2.446, "scoring_func": "sigmoid", "seq_aux": true, "tie_word_embeddings": false, "topk_group": 1, "topk_method": "noaux_tc", "torch_dtype": "bfloat16", "transformers_version": "4.46.3", "use_cache": true, "v_head_dim": 128, "vocab_size": 163840 } }, { "id": "moonshotai/Kimi-K2-Base", "meta": { "sha": "ce72df012259dcc55d945e890f815fe7ef69159c", "safetensors": { "parameters": { "F32": 62518232, "BF16": 2514970968, "F8_E4M3": 1023893241856 }, "total": 1026470731056 }, "modelType": "kimi_k2" }, "config": { "architectures": [ "DeepseekV3ForCausalLM" ], "attention_bias": false, "attention_dropout": 0.0, "auto_map": { "AutoConfig": "configuration_deepseek.DeepseekV3Config", "AutoModel": "modeling_deepseek.DeepseekV3Model", "AutoModelForCausalLM": "modeling_deepseek.DeepseekV3ForCausalLM" }, "aux_loss_alpha": 0.001, "bos_token_id": 163584, "eos_token_id": 163585, "first_k_dense_replace": 1, "hidden_act": "silu", "hidden_size": 7168, "initializer_range": 0.02, "intermediate_size": 18432, "kv_lora_rank": 512, "max_position_embeddings": 131072, "model_type": "kimi_k2", "moe_intermediate_size": 2048, "moe_layer_freq": 1, "n_group": 1, "n_routed_experts": 384, "n_shared_experts": 1, "norm_topk_prob": true, "num_attention_heads": 64, "num_experts_per_tok": 8, "num_hidden_layers": 61, "num_key_value_heads": 64, "num_nextn_predict_layers": 0, "pretraining_tp": 1, "q_lora_rank": 1536, "qk_nope_head_dim": 128, "qk_rope_head_dim": 64, "quantization_config": { "activation_scheme": "dynamic", "fmt": "e4m3", "quant_method": "fp8", "weight_block_size": [ 128, 128 ] }, "rms_norm_eps": 1e-06, "rope_theta": 50000.0, "routed_scaling_factor": 2.827, "rope_scaling": { "beta_fast": 1.0, "beta_slow": 1.0, "factor": 32.0, "mscale": 1.0, "mscale_all_dim": 1.0, "original_max_position_embeddings": 4096, "type": "yarn" }, "scoring_func": "sigmoid", "seq_aux": true, "tie_word_embeddings": false, "topk_group": 1, "topk_method": "noaux_tc", "torch_dtype": "bfloat16", "transformers_version": "4.48.3", "use_cache": true, "v_head_dim": 128, "vocab_size": 163840 }, "verifiedAt": "2026-09-22" }, { "id": "moonshotai/Kimi-Dev-72B", "meta": { "sha": "8791d7981945752a51f692d66f2bbfb3573c9722", "safetensors": { "parameters": { "BF16": 72706203648 }, "total": 72706203648 }, "modelType": "qwen2" }, "config": { "architectures": [ "Qwen2ForCausalLM" ], "attention_dropout": 0, "bos_token_id": 151643, "eos_token_id": 151645, "hidden_act": "silu", "hidden_size": 8192, "initializer_range": 0.02, "intermediate_size": 29568, "max_position_embeddings": 131072, "max_window_layers": 70, "model_type": "qwen2", "num_attention_heads": 64, "num_hidden_layers": 80, "num_key_value_heads": 8, "rms_norm_eps": 1e-06, "rope_theta": 1000000, "sliding_window": 131072, "tie_word_embeddings": false, "torch_dtype": "bfloat16", "transformers_version": "4.43.1", "use_cache": true, "use_sliding_window": false, "vocab_size": 152064 } }, { "id": "Qwen/Qwen2.5-72B-Instruct", "meta": { "sha": "495f39366efef23836d0cfae4fbe635880d2be31", "safetensors": { "parameters": { "BF16": 72706203648 }, "total": 72706203648 }, "modelType": "qwen2" }, "config": { "architectures": [ "Qwen2ForCausalLM" ], "attention_dropout": 0, "bos_token_id": 151643, "eos_token_id": 151645, "hidden_act": "silu", "hidden_size": 8192, "initializer_range": 0.02, "intermediate_size": 29568, "max_position_embeddings": 32768, "max_window_layers": 70, "model_type": "qwen2", "num_attention_heads": 64, "num_hidden_layers": 80, "num_key_value_heads": 8, "rms_norm_eps": 1e-06, "rope_theta": 1000000, "sliding_window": 131072, "tie_word_embeddings": false, "torch_dtype": "bfloat16", "transformers_version": "4.43.1", "use_cache": true, "use_sliding_window": false, "vocab_size": 152064 } }, { "id": "Qwen/Qwen3-235B-A22B", "meta": { "sha": "8efa61729e24bd65b1d152b5ab5409052aa80e65", "safetensors": { "parameters": { "BF16": 235093634560 }, "total": 235093634560 }, "modelType": "qwen3_moe" }, "config": { "architectures": [ "Qwen3MoeForCausalLM" ], "attention_bias": false, "attention_dropout": 0, "bos_token_id": 151643, "decoder_sparse_step": 1, "eos_token_id": 151645, "head_dim": 128, "hidden_act": "silu", "hidden_size": 4096, "initializer_range": 0.02, "intermediate_size": 12288, "max_position_embeddings": 40960, "max_window_layers": 94, "mlp_only_layers": [], "model_type": "qwen3_moe", "moe_intermediate_size": 1536, "norm_topk_prob": true, "num_attention_heads": 64, "num_experts": 128, "num_experts_per_tok": 8, "num_hidden_layers": 94, "num_key_value_heads": 4, "output_router_logits": false, "rms_norm_eps": 1e-06, "rope_scaling": null, "rope_theta": 1000000, "router_aux_loss_coef": 0.001, "sliding_window": null, "tie_word_embeddings": false, "torch_dtype": "bfloat16", "transformers_version": "4.51.0", "use_cache": true, "use_sliding_window": false, "vocab_size": 151936 } }, { "id": "Qwen/Qwen3-235B-A22B-Instruct-2507", "meta": { "sha": "ac9c66cc9b46af7306746a9250f23d47083d689e", "safetensors": { "parameters": { "BF16": 235093634560 }, "total": 235093634560 }, "modelType": "qwen3_moe" }, "config": { "architectures": [ "Qwen3MoeForCausalLM" ], "attention_bias": false, "attention_dropout": 0, "bos_token_id": 151643, "decoder_sparse_step": 1, "eos_token_id": 151645, "head_dim": 128, "hidden_act": "silu", "hidden_size": 4096, "initializer_range": 0.02, "intermediate_size": 12288, "max_position_embeddings": 262144, "max_window_layers": 94, "mlp_only_layers": [], "model_type": "qwen3_moe", "moe_intermediate_size": 1536, "norm_topk_prob": true, "num_attention_heads": 64, "num_experts": 128, "num_experts_per_tok": 8, "num_hidden_layers": 94, "num_key_value_heads": 4, "output_router_logits": false, "rms_norm_eps": 1e-06, "rope_scaling": null, "rope_theta": 5000000, "router_aux_loss_coef": 0.001, "sliding_window": null, "tie_word_embeddings": false, "torch_dtype": "bfloat16", "transformers_version": "4.51.0", "use_cache": true, "use_sliding_window": false, "vocab_size": 151936 } }, { "id": "Qwen/Qwen3-30B-A3B-Instruct-2507", "meta": { "sha": "0d7cf23991f47feeb3a57ecb4c9cee8ea4a17bfe", "safetensors": { "parameters": { "BF16": 30532122624 }, "total": 30532122624 }, "modelType": "qwen3_moe" }, "config": { "architectures": [ "Qwen3MoeForCausalLM" ], "attention_bias": false, "attention_dropout": 0, "bos_token_id": 151643, "decoder_sparse_step": 1, "eos_token_id": 151645, "head_dim": 128, "hidden_act": "silu", "hidden_size": 2048, "initializer_range": 0.02, "intermediate_size": 6144, "max_position_embeddings": 262144, "max_window_layers": 48, "mlp_only_layers": [], "model_type": "qwen3_moe", "moe_intermediate_size": 768, "norm_topk_prob": true, "num_attention_heads": 32, "num_experts": 128, "num_experts_per_tok": 8, "num_hidden_layers": 48, "num_key_value_heads": 4, "output_router_logits": false, "rms_norm_eps": 1e-06, "rope_scaling": null, "rope_theta": 10000000, "router_aux_loss_coef": 0.001, "sliding_window": null, "tie_word_embeddings": false, "torch_dtype": "bfloat16", "transformers_version": "4.51.0", "use_cache": true, "use_sliding_window": false, "vocab_size": 151936 } }, { "id": "Qwen/Qwen3-Coder-480B-A35B-Instruct", "meta": { "sha": "9d90cf8fca1bf7b7acca42d3fc9ae694a2194069", "safetensors": { "parameters": { "BF16": 480154875392 }, "total": 480154875392 }, "modelType": "qwen3_moe" }, "config": { "architectures": [ "Qwen3MoeForCausalLM" ], "attention_dropout": 0, "decoder_sparse_step": 1, "eos_token_id": 151645, "head_dim": 128, "hidden_act": "silu", "hidden_size": 6144, "initializer_range": 0.02, "intermediate_size": 8192, "max_position_embeddings": 262144, "max_window_layers": 62, "mlp_only_layers": [], "model_type": "qwen3_moe", "moe_intermediate_size": 2560, "norm_topk_prob": true, "num_attention_heads": 96, "num_experts": 160, "num_experts_per_tok": 8, "num_hidden_layers": 62, "num_key_value_heads": 8, "output_router_logits": false, "qkv_bias": false, "rms_norm_eps": 1e-06, "rope_scaling": null, "rope_theta": 10000000, "router_aux_loss_coef": 0, "shared_expert_intermediate_size": 0, "sliding_window": null, "tie_word_embeddings": false, "torch_dtype": "bfloat16", "transformers_version": "4.51.0", "use_cache": true, "use_qk_norm": true, "use_sliding_window": false, "vocab_size": 151936 } }, { "id": "Qwen/Qwen3-Coder-30B-A3B-Instruct", "meta": { "sha": "b2cff646eb4bb1d68355c01b18ae02e7cf42d120", "safetensors": { "parameters": { "BF16": 30532122624 }, "total": 30532122624 }, "modelType": "qwen3_moe" }, "config": { "architectures": [ "Qwen3MoeForCausalLM" ], "attention_bias": false, "attention_dropout": 0, "bos_token_id": 151643, "decoder_sparse_step": 1, "eos_token_id": 151645, "head_dim": 128, "hidden_act": "silu", "hidden_size": 2048, "initializer_range": 0.02, "intermediate_size": 6144, "max_position_embeddings": 262144, "max_window_layers": 48, "mlp_only_layers": [], "model_type": "qwen3_moe", "moe_intermediate_size": 768, "norm_topk_prob": true, "num_attention_heads": 32, "num_experts": 128, "num_experts_per_tok": 8, "num_hidden_layers": 48, "num_key_value_heads": 4, "output_router_logits": false, "rms_norm_eps": 1e-06, "rope_scaling": null, "rope_theta": 10000000, "router_aux_loss_coef": 0.001, "sliding_window": null, "tie_word_embeddings": false, "torch_dtype": "bfloat16", "transformers_version": "4.52.3", "use_cache": true, "use_sliding_window": false, "vocab_size": 151936 } }, { "id": "Qwen/Qwen3.5-9B", "meta": { "sha": "c202236235762e1c871ad0ccb60c8ee5ba337b9a", "safetensors": { "parameters": { "BF16": 9653100528, "F32": 3840 }, "total": 9653104368 }, "modelType": "qwen3_5" }, "config": { "architectures": [ "Qwen3_5ForConditionalGeneration" ], "image_token_id": 248056, "model_type": "qwen3_5", "text_config": { "attention_bias": false, "attention_dropout": 0, "attn_output_gate": true, "dtype": "bfloat16", "eos_token_id": 248044, "full_attention_interval": 4, "head_dim": 256, "hidden_act": "silu", "hidden_size": 4096, "initializer_range": 0.02, "intermediate_size": 12288, "layer_types": [ "linear_attention", "linear_attention", "linear_attention", "full_attention", "linear_attention", "linear_attention", "linear_attention", "full_attention", "linear_attention", "linear_attention", "linear_attention", "full_attention", "linear_attention", "linear_attention", "linear_attention", "full_attention", "linear_attention", "linear_attention", "linear_attention", "full_attention", "linear_attention", "linear_attention", "linear_attention", "full_attention", "linear_attention", "linear_attention", "linear_attention", "full_attention", "linear_attention", "linear_attention", "linear_attention", "full_attention" ], "linear_conv_kernel_dim": 4, "linear_key_head_dim": 128, "linear_num_key_heads": 16, "linear_num_value_heads": 32, "linear_value_head_dim": 128, "max_position_embeddings": 262144, "mlp_only_layers": [], "model_type": "qwen3_5_text", "mtp_num_hidden_layers": 1, "mtp_use_dedicated_embeddings": false, "num_attention_heads": 16, "num_hidden_layers": 32, "num_key_value_heads": 4, "rms_norm_eps": 1e-06, "use_cache": true, "vocab_size": 248320, "mamba_ssm_dtype": "float32", "rope_parameters": { "mrope_interleaved": true, "mrope_section": [ 11, 11, 10 ], "rope_type": "default", "rope_theta": 10000000, "partial_rotary_factor": 0.25 } }, "tie_word_embeddings": false, "transformers_version": "4.57.0.dev0", "video_token_id": 248057, "vision_config": { "deepstack_visual_indexes": [], "depth": 27, "hidden_act": "gelu_pytorch_tanh", "hidden_size": 1152, "in_channels": 3, "initializer_range": 0.02, "intermediate_size": 4304, "model_type": "qwen3_5", "num_heads": 16, "num_position_embeddings": 2304, "out_hidden_size": 4096, "patch_size": 16, "spatial_merge_size": 2, "temporal_patch_size": 2 }, "vision_end_token_id": 248054, "vision_start_token_id": 248053 } }, { "id": "Qwen/Qwen3.5-27B", "meta": { "sha": "fc05daec18b0a78c049392ed2e771dde82bdf654", "safetensors": { "parameters": { "BF16": 27781419504, "F32": 8448 }, "total": 27781427952 }, "modelType": "qwen3_5" }, "config": { "architectures": [ "Qwen3_5ForConditionalGeneration" ], "image_token_id": 248056, "model_type": "qwen3_5", "text_config": { "attention_bias": false, "attention_dropout": 0, "attn_output_gate": true, "dtype": "bfloat16", "eos_token_id": 248044, "full_attention_interval": 4, "head_dim": 256, "hidden_act": "silu", "hidden_size": 5120, "initializer_range": 0.02, "intermediate_size": 17408, "layer_types": [ "linear_attention", "linear_attention", "linear_attention", "full_attention", "linear_attention", "linear_attention", "linear_attention", "full_attention", "linear_attention", "linear_attention", "linear_attention", "full_attention", "linear_attention", "linear_attention", "linear_attention", "full_attention", "linear_attention", "linear_attention", "linear_attention", "full_attention", "linear_attention", "linear_attention", "linear_attention", "full_attention", "linear_attention", "linear_attention", "linear_attention", "full_attention", "linear_attention", "linear_attention", "linear_attention", "full_attention", "linear_attention", "linear_attention", "linear_attention", "full_attention", "linear_attention", "linear_attention", "linear_attention", "full_attention", "linear_attention", "linear_attention", "linear_attention", "full_attention", "linear_attention", "linear_attention", "linear_attention", "full_attention", "linear_attention", "linear_attention", "linear_attention", "full_attention", "linear_attention", "linear_attention", "linear_attention", "full_attention", "linear_attention", "linear_attention", "linear_attention", "full_attention", "linear_attention", "linear_attention", "linear_attention", "full_attention" ], "linear_conv_kernel_dim": 4, "linear_key_head_dim": 128, "linear_num_key_heads": 16, "linear_num_value_heads": 48, "linear_value_head_dim": 128, "max_position_embeddings": 262144, "mlp_only_layers": [], "model_type": "qwen3_5_text", "mtp_num_hidden_layers": 1, "mtp_use_dedicated_embeddings": false, "num_attention_heads": 24, "num_hidden_layers": 64, "num_key_value_heads": 4, "rms_norm_eps": 1e-06, "use_cache": true, "vocab_size": 248320, "mamba_ssm_dtype": "float32", "rope_parameters": { "mrope_interleaved": true, "mrope_section": [ 11, 11, 10 ], "rope_type": "default", "rope_theta": 10000000, "partial_rotary_factor": 0.25 } }, "tie_word_embeddings": false, "transformers_version": "4.57.0.dev0", "video_token_id": 248057, "vision_config": { "deepstack_visual_indexes": [], "depth": 27, "hidden_act": "gelu_pytorch_tanh", "hidden_size": 1152, "in_channels": 3, "initializer_range": 0.02, "intermediate_size": 4304, "model_type": "qwen3_5", "num_heads": 16, "num_position_embeddings": 2304, "out_hidden_size": 5120, "patch_size": 16, "spatial_merge_size": 2, "temporal_patch_size": 2 }, "vision_end_token_id": 248054, "vision_start_token_id": 248053 } }, { "id": "Qwen/Qwen3.5-4B", "meta": { "sha": "851bf6e806efd8d0a36b00ddf55e13ccb7b8cd0a", "safetensors": { "parameters": { "BF16": 4659861248, "F32": 3840 }, "total": 4659865088 }, "modelType": "qwen3_5" }, "config": { "architectures": [ "Qwen3_5ForConditionalGeneration" ], "image_token_id": 248056, "model_type": "qwen3_5", "text_config": { "attention_bias": false, "attention_dropout": 0, "attn_output_gate": true, "dtype": "bfloat16", "eos_token_id": 248044, "full_attention_interval": 4, "head_dim": 256, "hidden_act": "silu", "hidden_size": 2560, "initializer_range": 0.02, "intermediate_size": 9216, "layer_types": [ "linear_attention", "linear_attention", "linear_attention", "full_attention", "linear_attention", "linear_attention", "linear_attention", "full_attention", "linear_attention", "linear_attention", "linear_attention", "full_attention", "linear_attention", "linear_attention", "linear_attention", "full_attention", "linear_attention", "linear_attention", "linear_attention", "full_attention", "linear_attention", "linear_attention", "linear_attention", "full_attention", "linear_attention", "linear_attention", "linear_attention", "full_attention", "linear_attention", "linear_attention", "linear_attention", "full_attention" ], "linear_conv_kernel_dim": 4, "linear_key_head_dim": 128, "linear_num_key_heads": 16, "linear_num_value_heads": 32, "linear_value_head_dim": 128, "max_position_embeddings": 262144, "mlp_only_layers": [], "model_type": "qwen3_5_text", "mtp_num_hidden_layers": 1, "mtp_use_dedicated_embeddings": false, "num_attention_heads": 16, "num_hidden_layers": 32, "num_key_value_heads": 4, "rms_norm_eps": 1e-06, "tie_word_embeddings": true, "use_cache": true, "vocab_size": 248320, "mamba_ssm_dtype": "float32", "rope_parameters": { "mrope_interleaved": true, "mrope_section": [ 11, 11, 10 ], "rope_type": "default", "rope_theta": 10000000, "partial_rotary_factor": 0.25 } }, "tie_word_embeddings": true, "transformers_version": "4.57.0.dev0", "video_token_id": 248057, "vision_config": { "deepstack_visual_indexes": [], "depth": 24, "hidden_act": "gelu_pytorch_tanh", "hidden_size": 1024, "in_channels": 3, "initializer_range": 0.02, "intermediate_size": 4096, "model_type": "qwen3_5", "num_heads": 16, "num_position_embeddings": 2304, "out_hidden_size": 2560, "patch_size": 16, "spatial_merge_size": 2, "temporal_patch_size": 2 }, "vision_end_token_id": 248054, "vision_start_token_id": 248053 } }, { "id": "Qwen/Qwen3.5-35B-A3B", "meta": { "sha": "59d61f3ce65a6d9863b86d2e96597125219dc754", "safetensors": { "parameters": { "BF16": 35951817904, "F32": 4800 }, "total": 35951822704 }, "modelType": "qwen3_5_moe" }, "config": { "architectures": [ "Qwen3_5MoeForConditionalGeneration" ], "image_token_id": 248056, "model_type": "qwen3_5_moe", "text_config": { "attention_bias": false, "attention_dropout": 0, "attn_output_gate": true, "dtype": "bfloat16", "eos_token_id": 248044, "full_attention_interval": 4, "head_dim": 256, "hidden_act": "silu", "hidden_size": 2048, "initializer_range": 0.02, "layer_types": [ "linear_attention", "linear_attention", "linear_attention", "full_attention", "linear_attention", "linear_attention", "linear_attention", "full_attention", "linear_attention", "linear_attention", "linear_attention", "full_attention", "linear_attention", "linear_attention", "linear_attention", "full_attention", "linear_attention", "linear_attention", "linear_attention", "full_attention", "linear_attention", "linear_attention", "linear_attention", "full_attention", "linear_attention", "linear_attention", "linear_attention", "full_attention", "linear_attention", "linear_attention", "linear_attention", "full_attention", "linear_attention", "linear_attention", "linear_attention", "full_attention", "linear_attention", "linear_attention", "linear_attention", "full_attention" ], "linear_conv_kernel_dim": 4, "linear_key_head_dim": 128, "linear_num_key_heads": 16, "linear_num_value_heads": 32, "linear_value_head_dim": 128, "max_position_embeddings": 262144, "mlp_only_layers": [], "model_type": "qwen3_5_moe_text", "moe_intermediate_size": 512, "mtp_num_hidden_layers": 1, "mtp_use_dedicated_embeddings": false, "num_attention_heads": 16, "num_experts": 256, "num_experts_per_tok": 8, "num_hidden_layers": 40, "num_key_value_heads": 2, "rms_norm_eps": 1e-06, "router_aux_loss_coef": 0.001, "shared_expert_intermediate_size": 512, "use_cache": true, "vocab_size": 248320, "mamba_ssm_dtype": "float32", "rope_parameters": { "mrope_interleaved": true, "mrope_section": [ 11, 11, 10 ], "rope_type": "default", "rope_theta": 10000000, "partial_rotary_factor": 0.25 } }, "tie_word_embeddings": false, "transformers_version": "4.57.0.dev0", "video_token_id": 248057, "vision_config": { "deepstack_visual_indexes": [], "depth": 27, "hidden_act": "gelu_pytorch_tanh", "hidden_size": 1152, "in_channels": 3, "initializer_range": 0.02, "intermediate_size": 4304, "model_type": "qwen3_5_moe", "num_heads": 16, "num_position_embeddings": 2304, "out_hidden_size": 2048, "patch_size": 16, "spatial_merge_size": 2, "temporal_patch_size": 2 }, "vision_end_token_id": 248054, "vision_start_token_id": 248053 } }, { "id": "mistralai/Mistral-7B-Instruct-v0.3", "meta": { "sha": "c170c708c41dac9275d15a8fff4eca08d52bab71", "safetensors": { "parameters": { "BF16": 7248023552 }, "total": 7248023552 }, "modelType": "mistral" }, "config": { "architectures": [ "MistralForCausalLM" ], "attention_dropout": 0, "bos_token_id": 1, "eos_token_id": 2, "hidden_act": "silu", "hidden_size": 4096, "initializer_range": 0.02, "intermediate_size": 14336, "max_position_embeddings": 32768, "model_type": "mistral", "num_attention_heads": 32, "num_hidden_layers": 32, "num_key_value_heads": 8, "rms_norm_eps": 1e-05, "rope_theta": 1000000, "sliding_window": null, "tie_word_embeddings": false, "torch_dtype": "bfloat16", "transformers_version": "4.42.0.dev0", "use_cache": true, "vocab_size": 32768 } }, { "id": "mistralai/Mixtral-8x7B-Instruct-v0.1", "meta": { "sha": "eba92302a2861cdc0098cc54bc9f17cb2c47eb61", "safetensors": { "parameters": { "BF16": 46702792704 }, "total": 46702792704 }, "modelType": "mixtral" }, "config": { "architectures": [ "MixtralForCausalLM" ], "attention_dropout": 0, "bos_token_id": 1, "eos_token_id": 2, "hidden_act": "silu", "hidden_size": 4096, "initializer_range": 0.02, "intermediate_size": 14336, "max_position_embeddings": 32768, "model_type": "mixtral", "num_attention_heads": 32, "num_experts_per_tok": 2, "num_hidden_layers": 32, "num_key_value_heads": 8, "num_local_experts": 8, "output_router_logits": false, "rms_norm_eps": 1e-05, "rope_theta": 1000000, "router_aux_loss_coef": 0.02, "sliding_window": null, "tie_word_embeddings": false, "torch_dtype": "bfloat16", "transformers_version": "4.36.0.dev0", "use_cache": true, "vocab_size": 32000 } }, { "id": "mistralai/Mixtral-8x22B-Instruct-v0.1", "meta": { "sha": "cc88a6cc19fbd17d9f1c0ee0b0d70a748dce698d", "safetensors": { "parameters": { "BF16": 140630071296 }, "total": 140630071296 }, "modelType": "mixtral" }, "config": { "architectures": [ "MixtralForCausalLM" ], "attention_dropout": 0, "bos_token_id": 1, "eos_token_id": 2, "hidden_act": "silu", "hidden_size": 6144, "initializer_range": 0.02, "intermediate_size": 16384, "max_position_embeddings": 65536, "model_type": "mixtral", "num_attention_heads": 48, "num_experts_per_tok": 2, "num_hidden_layers": 56, "num_key_value_heads": 8, "num_local_experts": 8, "output_router_logits": false, "rms_norm_eps": 1e-05, "rope_theta": 1000000, "router_aux_loss_coef": 0.001, "sliding_window": null, "tie_word_embeddings": false, "torch_dtype": "bfloat16", "transformers_version": "4.38.0", "use_cache": true, "vocab_size": 32768 } }, { "id": "mistralai/Mistral-Nemo-Instruct-2407", "meta": { "sha": "04d8a90549d23fc6bd7f642064003592df51e9b3", "safetensors": { "parameters": { "BF16": 12247782400 }, "total": 12247782400 }, "modelType": "mistral" }, "config": { "architectures": [ "MistralForCausalLM" ], "attention_dropout": 0, "bos_token_id": 1, "eos_token_id": 2, "head_dim": 128, "hidden_act": "silu", "hidden_size": 5120, "initializer_range": 0.02, "intermediate_size": 14336, "max_position_embeddings": 131072, "model_type": "mistral", "num_attention_heads": 32, "num_hidden_layers": 40, "num_key_value_heads": 8, "rms_norm_eps": 1e-05, "rope_theta": 1000000, "sliding_window": null, "tie_word_embeddings": false, "torch_dtype": "bfloat16", "transformers_version": "4.43.0.dev0", "use_cache": true, "vocab_size": 131072 } }, { "id": "mistralai/Mistral-Small-24B-Instruct-2501", "meta": { "sha": "9527884be6e5616bdd54de542f9ae13384489724", "safetensors": { "parameters": { "BF16": 23572403200 }, "total": 23572403200 }, "modelType": "mistral" }, "config": { "architectures": [ "MistralForCausalLM" ], "attention_dropout": 0, "bos_token_id": 1, "eos_token_id": 2, "head_dim": 128, "hidden_act": "silu", "hidden_size": 5120, "initializer_range": 0.02, "intermediate_size": 32768, "max_position_embeddings": 32768, "model_type": "mistral", "num_attention_heads": 32, "num_hidden_layers": 40, "num_key_value_heads": 8, "rms_norm_eps": 1e-05, "rope_theta": 100000000, "sliding_window": null, "tie_word_embeddings": false, "torch_dtype": "bfloat16", "transformers_version": "4.49.0.dev0", "use_cache": true, "vocab_size": 131072 } }, { "id": "mistralai/Mistral-Small-3.1-24B-Instruct-2503", "meta": { "sha": "68faf511d618ef198fef186659617cfd2eb8e33a", "safetensors": { "parameters": { "BF16": 24011361280 }, "total": 24011361280 }, "modelType": "mistral3" }, "config": { "architectures": [ "Mistral3ForConditionalGeneration" ], "image_token_index": 10, "model_type": "mistral3", "multimodal_projector_bias": false, "projector_hidden_act": "gelu", "spatial_merge_size": 2, "text_config": { "attention_dropout": 0, "head_dim": 128, "hidden_act": "silu", "hidden_size": 5120, "initializer_range": 0.02, "intermediate_size": 32768, "max_position_embeddings": 131072, "model_type": "mistral", "num_attention_heads": 32, "num_hidden_layers": 40, "num_key_value_heads": 8, "rms_norm_eps": 1e-05, "rope_theta": 1000000000, "sliding_window": null, "use_cache": true, "vocab_size": 131072 }, "torch_dtype": "bfloat16", "transformers_version": "4.50.0.dev0", "vision_config": { "attention_dropout": 0, "head_dim": 64, "hidden_act": "silu", "hidden_size": 1024, "image_size": 1540, "initializer_range": 0.02, "intermediate_size": 4096, "model_type": "pixtral", "num_attention_heads": 16, "num_channels": 3, "num_hidden_layers": 24, "patch_size": 14, "rope_theta": 10000 }, "vision_feature_layer": -1 } }, { "id": "mistralai/Mistral-Large-Instruct-2407", "meta": { "sha": "a286006d554cb37a61d13c7ae61bc90cc1d372fc", "safetensors": { "parameters": { "BF16": 122610069504 }, "total": 122610069504 }, "modelType": "mistral" }, "config": { "error": 401 } }, { "id": "mistralai/Ministral-8B-Instruct-2410", "meta": { "sha": "2f494a194c5b980dfb9772cb92d26cbb671fce5a", "safetensors": { "parameters": { "BF16": 8019808256 }, "total": 8019808256 }, "modelType": "mistral" }, "config": { "architectures": [ "MistralForCausalLM" ], "attention_dropout": 0, "bos_token_id": 1, "eos_token_id": 2, "head_dim": 128, "hidden_act": "silu", "hidden_size": 4096, "initializer_range": 0.02, "intermediate_size": 12288, "layer_types": [ "full_attention", "sliding_attention", "sliding_attention", "sliding_attention", "full_attention", "sliding_attention", "sliding_attention", "sliding_attention", "full_attention", "sliding_attention", "sliding_attention", "sliding_attention", "full_attention", "sliding_attention", "sliding_attention", "sliding_attention", "full_attention", "sliding_attention", "sliding_attention", "sliding_attention", "full_attention", "sliding_attention", "sliding_attention", "sliding_attention", "full_attention", "sliding_attention", "sliding_attention", "sliding_attention", "full_attention", "sliding_attention", "sliding_attention", "sliding_attention", "full_attention", "sliding_attention", "sliding_attention", "sliding_attention" ], "max_position_embeddings": 32768, "model_type": "mistral", "num_attention_heads": 32, "num_hidden_layers": 36, "num_key_value_heads": 8, "rms_norm_eps": 1e-05, "rope_theta": 100000000, "sliding_window": 32768, "tie_word_embeddings": false, "torch_dtype": "bfloat16", "transformers_version": "4.46.0.dev0", "use_cache": true, "vocab_size": 131072 } }, { "id": "mistralai/Magistral-Small-2509", "meta": { "sha": "a31cc96ab10cf19bc42c628fedf1e359e0853c49", "safetensors": { "parameters": { "BF16": 24011361280 }, "total": 24011361280 }, "modelType": "mistral3" }, "config": { "architectures": [ "Mistral3ForConditionalGeneration" ], "dtype": "bfloat16", "image_token_index": 10, "model_type": "mistral3", "multimodal_projector_bias": false, "projector_hidden_act": "gelu", "spatial_merge_size": 2, "text_config": { "attention_dropout": 0, "head_dim": 128, "hidden_act": "silu", "hidden_size": 5120, "initializer_range": 0.02, "intermediate_size": 32768, "max_position_embeddings": 131072, "model_type": "mistral", "num_attention_heads": 32, "num_hidden_layers": 40, "num_key_value_heads": 8, "rms_norm_eps": 1e-05, "rope_theta": 1000000000, "sliding_window": null, "use_cache": true, "vocab_size": 131072 }, "transformers_version": "4.56.1", "vision_config": { "attention_dropout": 0, "head_dim": 64, "hidden_act": "silu", "hidden_size": 1024, "image_size": 1540, "initializer_range": 0.02, "intermediate_size": 4096, "model_type": "pixtral", "num_attention_heads": 16, "num_channels": 3, "num_hidden_layers": 24, "patch_size": 14, "rope_theta": 10000 }, "vision_feature_layer": -1 } }, { "id": "google/gemma-3-4b-it", "meta": { "sha": "093f9f388b31de276ce2de164bdc2081324b9767", "safetensors": { "parameters": { "BF16": 4300079472 }, "total": 4300079472 }, "modelType": "gemma3" }, "config": { "num_hidden_layers": 34, "num_attention_heads": 8, "num_key_value_heads": 4, "hidden_size": 2560, "head_dim": 256, "vocab_size": 262144, "intermediate_size": 10240, "model_type": "gemma3", "max_position_embeddings": 131072, "tie_word_embeddings": true, "sliding_window": 1024, "window_layers": 29 }, "previousConfigError": 401, "configSource": "https://github.com/google/gemma_pytorch/blob/014acb7ac4563a5f77c76d7ff98f31b568c16508/gemma/config.py", "configKind": "Normalized from official implementation, not the restricted Hugging Face config.json", "verifiedAt": "2026-09-22" }, { "id": "google/gemma-3-12b-it", "meta": { "sha": "96b6f1eccf38110c56df3a15bffe176da04bfd80", "safetensors": { "parameters": { "BF16": 12187325040 }, "total": 12187325040 }, "modelType": "gemma3" }, "config": { "num_hidden_layers": 48, "num_attention_heads": 16, "num_key_value_heads": 8, "hidden_size": 3840, "head_dim": 256, "vocab_size": 262144, "intermediate_size": 15360, "model_type": "gemma3", "max_position_embeddings": 131072, "tie_word_embeddings": true, "sliding_window": 1024, "window_layers": 40 }, "previousConfigError": 401, "configSource": "https://github.com/google/gemma_pytorch/blob/014acb7ac4563a5f77c76d7ff98f31b568c16508/gemma/config.py", "configKind": "Normalized from official implementation, not the restricted Hugging Face config.json", "verifiedAt": "2026-09-22" }, { "id": "google/gemma-3-27b-it", "meta": { "sha": "005ad3404e59d6023443cb575daa05336842228a", "safetensors": { "parameters": { "BF16": 27432406640 }, "total": 27432406640 }, "modelType": "gemma3" }, "config": { "num_hidden_layers": 62, "num_attention_heads": 32, "num_key_value_heads": 16, "hidden_size": 5376, "head_dim": 128, "vocab_size": 262144, "intermediate_size": 21504, "model_type": "gemma3", "max_position_embeddings": 131072, "tie_word_embeddings": true, "sliding_window": 1024, "window_layers": 52 }, "previousConfigError": 401, "configSource": "https://github.com/google/gemma_pytorch/blob/014acb7ac4563a5f77c76d7ff98f31b568c16508/gemma/config.py", "configKind": "Normalized from official implementation, not the restricted Hugging Face config.json", "verifiedAt": "2026-09-22" }, { "id": "openai/gpt-oss-20b", "meta": { "sha": "6cee5e81ee83917806bbde320786a8fb61efebee", "safetensors": { "parameters": { "BF16": 1804459584, "U8": 19110297600 }, "total": 20914757184 }, "modelType": "gpt_oss" }, "config": { "architectures": [ "GptOssForCausalLM" ], "attention_bias": true, "attention_dropout": 0.0, "eos_token_id": 200002, "experts_per_token": 4, "head_dim": 64, "hidden_act": "silu", "hidden_size": 2880, "initial_context_length": 4096, "initializer_range": 0.02, "intermediate_size": 2880, "layer_types": [ "sliding_attention", "full_attention", "sliding_attention", "full_attention", "sliding_attention", "full_attention", "sliding_attention", "full_attention", "sliding_attention", "full_attention", "sliding_attention", "full_attention", "sliding_attention", "full_attention", "sliding_attention", "full_attention", "sliding_attention", "full_attention", "sliding_attention", "full_attention", "sliding_attention", "full_attention", "sliding_attention", "full_attention" ], "max_position_embeddings": 131072, "model_type": "gpt_oss", "num_attention_heads": 64, "num_experts_per_tok": 4, "num_hidden_layers": 24, "num_key_value_heads": 8, "num_local_experts": 32, "output_router_logits": false, "pad_token_id": 199999, "quantization_config": { "modules_to_not_convert": [ "model.layers.*.self_attn", "model.layers.*.mlp.router", "model.embed_tokens", "lm_head" ], "quant_method": "mxfp4" }, "rms_norm_eps": 1e-05, "rope_scaling": { "beta_fast": 32.0, "beta_slow": 1.0, "factor": 32.0, "original_max_position_embeddings": 4096, "rope_type": "yarn", "truncate": false }, "rope_theta": 150000, "router_aux_loss_coef": 0.9, "sliding_window": 128, "swiglu_limit": 7.0, "tie_word_embeddings": false, "transformers_version": "4.55.0.dev0", "use_cache": true, "vocab_size": 201088 }, "quantizationSource": "https://github.com/openai/gpt-oss#quantization", "logicalExpertParameters": 19110297600, "verifiedAt": "2026-09-22" }, { "id": "openai/gpt-oss-120b", "meta": { "sha": "b5c939de8f754692c1647ca79fbf85e8c1e70f8a", "safetensors": { "parameters": { "BF16": 2167371072, "U8": 114661785600 }, "total": 116829156672 }, "modelType": "gpt_oss" }, "config": { "architectures": [ "GptOssForCausalLM" ], "attention_bias": true, "attention_dropout": 0.0, "eos_token_id": 200002, "experts_per_token": 4, "head_dim": 64, "hidden_act": "silu", "hidden_size": 2880, "initial_context_length": 4096, "initializer_range": 0.02, "intermediate_size": 2880, "layer_types": [ "sliding_attention", "full_attention", "sliding_attention", "full_attention", "sliding_attention", "full_attention", "sliding_attention", "full_attention", "sliding_attention", "full_attention", "sliding_attention", "full_attention", "sliding_attention", "full_attention", "sliding_attention", "full_attention", "sliding_attention", "full_attention", "sliding_attention", "full_attention", "sliding_attention", "full_attention", "sliding_attention", "full_attention", "sliding_attention", "full_attention", "sliding_attention", "full_attention", "sliding_attention", "full_attention", "sliding_attention", "full_attention", "sliding_attention", "full_attention", "sliding_attention", "full_attention" ], "max_position_embeddings": 131072, "model_type": "gpt_oss", "num_attention_heads": 64, "num_experts_per_tok": 4, "num_hidden_layers": 36, "num_key_value_heads": 8, "num_local_experts": 128, "output_router_logits": false, "pad_token_id": 199999, "quantization_config": { "modules_to_not_convert": [ "model.layers.*.self_attn", "model.layers.*.mlp.router", "model.embed_tokens", "lm_head" ], "quant_method": "mxfp4" }, "rms_norm_eps": 1e-05, "rope_scaling": { "beta_fast": 32.0, "beta_slow": 1.0, "factor": 32.0, "original_max_position_embeddings": 4096, "rope_type": "yarn", "truncate": false }, "rope_theta": 150000, "router_aux_loss_coef": 0.9, "sliding_window": 128, "swiglu_limit": 7.0, "tie_word_embeddings": false, "transformers_version": "4.55.0.dev0", "use_cache": true, "vocab_size": 201088 }, "quantizationSource": "https://github.com/openai/gpt-oss#quantization", "logicalExpertParameters": 114661785600, "verifiedAt": "2026-09-22" }, { "id": "CohereForAI/c4ai-command-r-v01", "meta": { "sha": "760ddb6c203d87ebdbe3c9785b49570e1bf95585", "safetensors": { "parameters": { "F16": 34980831232 }, "total": 34980831232 }, "modelType": "cohere" }, "config": { "error": 401 } }, { "id": "CohereForAI/c4ai-command-r-plus", "meta": { "sha": "3de12c88b4c6c882389a6f424cc2074383aa17d8", "safetensors": { "parameters": { "F16": 103810674688 }, "total": 103810674688 }, "modelType": "cohere" }, "config": { "error": 401 } }, { "id": "CohereForAI/c4ai-command-r7b-12-2024", "meta": { "sha": "4f3d0aa6856e322f2f4480fe65420d5d53d297b8", "safetensors": { "parameters": { "BF16": 8028033024 }, "total": 8028033024 }, "modelType": "cohere2" }, "config": { "error": 401 } }, { "id": "nvidia/Llama-3.1-Nemotron-70B-Instruct", "meta": { "sha": "a83af1f4968437064635f6726fb745e5b615e863" }, "config": { "error": 404 } }, { "id": "nvidia/NVIDIA-Nemotron-Nano-9B-v2", "meta": { "sha": "6533e8de2c68e4536bf7c411d7a3ce5734111476", "safetensors": { "parameters": { "BF16": 8888227328 }, "total": 8888227328 }, "modelType": "nemotron_h" }, "config": { "architectures": [ "NemotronHForCausalLM" ], "attention_bias": false, "attention_dropout": 0, "auto_map": { "AutoConfig": "configuration_nemotron_h.NemotronHConfig", "AutoModelForCausalLM": "modeling_nemotron_h.NemotronHForCausalLM" }, "bos_token_id": 1, "chunk_size": 128, "conv_kernel": 4, "eos_token_id": 12, "head_dim": 128, "hidden_dropout": 0, "hidden_size": 4480, "hybrid_override_pattern": "M-M-M-MM-M-M-M*-M-M-M*-M-M-M-M*-M-M-M-M*-M-MM-M-M-M-M-M-", "initializer_range": 0.02, "intermediate_size": 15680, "layer_norm_epsilon": 1e-05, "mamba_head_dim": 80, "mamba_hidden_act": "silu", "mamba_num_groups": 8, "mamba_num_heads": 128, "mamba_proj_bias": false, "mamba_state_dim": 128, "max_position_embeddings": 131072, "mlp_bias": false, "mlp_hidden_act": "relu2", "model_type": "nemotron_h", "n_groups": 8, "num_attention_heads": 40, "num_hidden_layers": 56, "num_key_value_heads": 8, "num_logits_to_keep": 1, "num_query_groups": 8, "pad_token_id": 0, "rescale_prenorm_residual": true, "residual_in_fp32": false, "rms_norm_eps": 1e-05, "sliding_window": null, "ssm_state_size": 128, "tie_word_embeddings": false, "time_step_floor": 0.0001, "time_step_max": 0.1, "time_step_min": 0.001, "time_step_rank": 256, "torch_dtype": "bfloat16", "transformers_version": "4.51.3", "use_bias": false, "use_cache": true, "use_conv_bias": true, "use_mamba_kernels": true, "vocab_size": 131072 } }, { "id": "allenai/OLMo-2-1124-7B-Instruct", "meta": { "sha": "470b1fba1ae01581f270116362ee4aa1b97f4c84", "safetensors": { "parameters": { "BF16": 7298617344 }, "total": 7298617344 }, "modelType": "olmo2" }, "config": { "_name_or_path": "allenai/open_instruct_dev", "architectures": [ "Olmo2ForCausalLM" ], "attention_bias": false, "attention_dropout": 0, "eos_token_id": 100257, "hidden_act": "silu", "hidden_size": 4096, "initializer_range": 0.02, "intermediate_size": 11008, "max_position_embeddings": 4096, "model_type": "olmo2", "num_attention_heads": 32, "num_hidden_layers": 32, "num_key_value_heads": 32, "pad_token_id": 100277, "rms_norm_eps": 1e-06, "rope_scaling": null, "rope_theta": 500000, "tie_word_embeddings": false, "torch_dtype": "bfloat16", "transformers_version": "4.47.0.dev0", "use_cache": false, "vocab_size": 100352 } }, { "id": "allenai/OLMo-2-1124-13B-Instruct", "meta": { "sha": "3a5c85baefbb1896a54d56fe2e76c0395627ddf4", "safetensors": { "parameters": { "BF16": 13716198400 }, "total": 13716198400 }, "modelType": "olmo2" }, "config": { "_name_or_path": "/weka/oe-adapt-default/costah/models/olmo2/1210_13b_gsm_beta_0.03_lr_3e-7_7470_checkpoints/step_180/", "architectures": [ "Olmo2ForCausalLM" ], "attention_bias": false, "attention_dropout": 0, "eos_token_id": 100257, "hidden_act": "silu", "hidden_size": 5120, "initializer_range": 0.02, "intermediate_size": 13824, "max_position_embeddings": 4096, "model_type": "olmo2", "num_attention_heads": 40, "num_hidden_layers": 40, "num_key_value_heads": 40, "pad_token_id": 100277, "rms_norm_eps": 1e-06, "rope_scaling": null, "rope_theta": 500000, "tie_word_embeddings": false, "torch_dtype": "bfloat16", "transformers_version": "4.47.0.dev0", "use_cache": false, "vocab_size": 100352 } }, { "id": "allenai/OLMo-2-0325-32B-Instruct", "meta": { "sha": "b96024342a77a69aa0dda815c3454a671f477463", "safetensors": { "parameters": { "BF16": 32234279936 }, "total": 32234279936 }, "modelType": "olmo2" }, "config": { "_name_or_path": "allenai/open_instruct_dev", "architectures": [ "Olmo2ForCausalLM" ], "attention_bias": false, "attention_dropout": 0, "bos_token_id": 100257, "eos_token_id": 100257, "hidden_act": "silu", "hidden_size": 5120, "initializer_range": 0.02, "intermediate_size": 27648, "max_position_embeddings": 4096, "model_type": "olmo2", "num_attention_heads": 40, "num_hidden_layers": 64, "num_key_value_heads": 8, "pad_token_id": 100277, "rms_norm_eps": 1e-06, "rope_scaling": null, "rope_theta": 500000, "tie_word_embeddings": false, "torch_dtype": "bfloat16", "transformers_version": "4.49.0", "use_cache": false, "vocab_size": 100352 } }, { "id": "bigcode/starcoder2-3b", "meta": { "sha": "733247c55e3f73af49ce8e9c7949bf14af205928", "safetensors": { "parameters": { "F32": 3030371328 }, "total": 3030371328 }, "modelType": "starcoder2" }, "config": { "architectures": [ "Starcoder2ForCausalLM" ], "attention_dropout": 0.1, "residual_dropout": 0.1, "embedding_dropout": 0.1, "bos_token_id": 0, "eos_token_id": 0, "hidden_act": "gelu_pytorch_tanh", "hidden_size": 3072, "initializer_range": 0.018042, "intermediate_size": 12288, "max_position_embeddings": 16384, "mlp_type": "default", "model_type": "starcoder2", "norm_epsilon": 1e-05, "norm_type": "layer_norm", "num_attention_heads": 24, "num_hidden_layers": 30, "num_key_value_heads": 2, "rope_theta": 999999.4420358813, "sliding_window": 4096, "transformers_version": "4.37.0.dev0", "use_bias": true, "use_cache": true, "vocab_size": 49152 } }, { "id": "bigcode/starcoder2-7b", "meta": { "sha": "bb9afde76d7945da5745592525db122d4d729eb1", "safetensors": { "parameters": { "BF16": 7173923840 }, "total": 7173923840 }, "modelType": "starcoder2" }, "config": { "activation_function": "gelu", "architectures": [ "Starcoder2ForCausalLM" ], "attention_dropout": 0.1, "residual_dropout": 0.1, "embedding_dropout": 0.1, "attention_softmax_in_fp32": true, "bos_token_id": 0, "eos_token_id": 0, "hidden_act": "gelu_pytorch_tanh", "hidden_size": 4608, "initializer_range": 0.018042, "intermediate_size": 18432, "layer_norm_epsilon": 1e-05, "max_position_embeddings": 16384, "mlp_type": "default", "model_type": "starcoder2", "norm_epsilon": 1e-05, "norm_type": "layer_norm", "num_attention_heads": 36, "num_hidden_layers": 32, "num_key_value_heads": 4, "rope_theta": 1000000, "scale_attention_softmax_in_fp32": true, "scale_attn_weights": true, "sliding_window": 4096, "torch_dtype": "bfloat16", "transformers_version": "4.37.0.dev0", "use_bias": true, "use_cache": true, "vocab_size": 49152 } }, { "id": "bigcode/starcoder2-15b", "meta": { "sha": "46d44742909c03ac8cee08eb03fdebce02e193ec", "safetensors": { "parameters": { "F32": 15957889024 }, "total": 15957889024 }, "modelType": "starcoder2" }, "config": { "_name_or_path": "/aot/checkpoints/15b/config.json", "architectures": [ "Starcoder2ForCausalLM" ], "attention_dropout": 0.1, "residual_dropout": 0.1, "embedding_dropout": 0.1, "bos_token_id": 0, "eos_token_id": 0, "hidden_act": "gelu_pytorch_tanh", "hidden_size": 6144, "initializer_range": 0.01275, "intermediate_size": 24576, "max_position_embeddings": 16384, "mlp_type": "default", "model_type": "starcoder2", "norm_epsilon": 1e-05, "norm_type": "layer_norm", "num_attention_heads": 48, "num_hidden_layers": 40, "num_key_value_heads": 4, "rope_theta": 100000, "sliding_window": 4096, "tie_word_embeddings": false, "torch_dtype": "float32", "transformers_version": "4.37.0.dev0", "use_bias": true, "use_cache": true, "vocab_size": 49152 } }, { "id": "Qwen/Qwen2.5-Coder-0.5B-Instruct", "meta": { "sha": "ea3f2471cf1b1f0db85067f1ef93848e38e88c25", "safetensors": { "parameters": { "BF16": 494032768 }, "total": 494032768 }, "modelType": "qwen2" }, "config": { "architectures": [ "Qwen2ForCausalLM" ], "attention_dropout": 0, "bos_token_id": 151643, "eos_token_id": 151645, "hidden_act": "silu", "hidden_size": 896, "initializer_range": 0.02, "intermediate_size": 4864, "max_position_embeddings": 32768, "max_window_layers": 24, "model_type": "qwen2", "num_attention_heads": 14, "num_hidden_layers": 24, "num_key_value_heads": 2, "rms_norm_eps": 1e-06, "rope_theta": 1000000, "sliding_window": 32768, "tie_word_embeddings": true, "torch_dtype": "bfloat16", "transformers_version": "4.43.1", "use_cache": true, "use_sliding_window": false, "vocab_size": 151936 } }, { "id": "Qwen/Qwen2.5-Coder-1.5B-Instruct", "meta": { "sha": "2e1fd397ee46e1388853d2af2c993145b0f1098a", "safetensors": { "parameters": { "BF16": 1543714304 }, "total": 1543714304 }, "modelType": "qwen2" }, "config": { "architectures": [ "Qwen2ForCausalLM" ], "attention_dropout": 0, "bos_token_id": 151643, "eos_token_id": 151645, "hidden_act": "silu", "hidden_size": 1536, "initializer_range": 0.02, "intermediate_size": 8960, "max_position_embeddings": 32768, "max_window_layers": 28, "model_type": "qwen2", "num_attention_heads": 12, "num_hidden_layers": 28, "num_key_value_heads": 2, "rms_norm_eps": 1e-06, "rope_theta": 1000000, "sliding_window": 32768, "tie_word_embeddings": true, "torch_dtype": "bfloat16", "transformers_version": "4.44.0", "use_cache": true, "use_sliding_window": false, "vocab_size": 151936 } }, { "id": "Qwen/Qwen2.5-Coder-3B-Instruct", "meta": { "sha": "488639f1ff808d1d3d0ba301aef8c11461451ec5", "safetensors": { "parameters": { "BF16": 3085938688 }, "total": 3085938688 }, "modelType": "qwen2" }, "config": { "architectures": [ "Qwen2ForCausalLM" ], "attention_dropout": 0, "bos_token_id": 151643, "eos_token_id": 151645, "hidden_act": "silu", "hidden_size": 2048, "initializer_range": 0.02, "intermediate_size": 11008, "max_position_embeddings": 32768, "max_window_layers": 36, "model_type": "qwen2", "num_attention_heads": 16, "num_hidden_layers": 36, "num_key_value_heads": 2, "rms_norm_eps": 1e-06, "rope_theta": 1000000, "sliding_window": 32768, "tie_word_embeddings": true, "torch_dtype": "bfloat16", "transformers_version": "4.43.1", "use_cache": true, "use_sliding_window": false, "vocab_size": 151936 } }, { "id": "mistralai/Mixtral-8x7B-v0.1", "meta": { "sha": "fc7ac94680e38d7348cfa806e51218e6273104b0", "safetensors": { "parameters": { "BF16": 46702792704 }, "total": 46702792704 }, "modelType": "mixtral" }, "config": { "architectures": [ "MixtralForCausalLM" ], "attention_dropout": 0.0, "bos_token_id": 1, "eos_token_id": 2, "hidden_act": "silu", "hidden_size": 4096, "initializer_range": 0.02, "intermediate_size": 14336, "max_position_embeddings": 32768, "model_type": "mixtral", "num_attention_heads": 32, "num_experts_per_tok": 2, "num_hidden_layers": 32, "num_key_value_heads": 8, "num_local_experts": 8, "output_router_logits": false, "rms_norm_eps": 1e-05, "rope_theta": 1000000.0, "router_aux_loss_coef": 0.02, "sliding_window": null, "tie_word_embeddings": false, "torch_dtype": "bfloat16", "transformers_version": "4.36.0.dev0", "use_cache": true, "vocab_size": 32000 }, "verifiedAt": "2026-09-22" }, { "id": "mistralai/Mixtral-8x22B-v0.1", "meta": { "sha": "e1cd34ff1747406fb2277635ed25242803009bc2", "safetensors": { "parameters": { "BF16": 140620634112 }, "total": 140620634112 }, "modelType": "mixtral" }, "config": { "architectures": [ "MixtralForCausalLM" ], "attention_dropout": 0.0, "bos_token_id": 1, "eos_token_id": 2, "hidden_act": "silu", "hidden_size": 6144, "initializer_range": 0.02, "intermediate_size": 16384, "max_position_embeddings": 65536, "model_type": "mixtral", "num_attention_heads": 48, "num_experts_per_tok": 2, "num_hidden_layers": 56, "num_key_value_heads": 8, "num_local_experts": 8, "output_router_logits": false, "rms_norm_eps": 1e-05, "rope_theta": 1000000, "router_aux_loss_coef": 0.001, "sliding_window": null, "tie_word_embeddings": false, "torch_dtype": "bfloat16", "transformers_version": "4.38.0", "use_cache": true, "vocab_size": 32000 }, "verifiedAt": "2026-09-22" }, { "id": "mistralai/Devstral-Small-2505", "meta": { "sha": "c2a9d81a2989af566682b4cecc828c84556076c5", "safetensors": { "parameters": { "BF16": 23572403200 }, "total": 23572403200 }, "modelType": "mistral" }, "config": { "architectures": [ "MistralForCausalLM" ], "attention_dropout": 0.0, "bos_token_id": 1, "eos_token_id": 2, "head_dim": 128, "hidden_act": "silu", "hidden_size": 5120, "initializer_range": 0.02, "intermediate_size": 32768, "max_position_embeddings": 131072, "model_type": "mistral", "num_attention_heads": 32, "num_hidden_layers": 40, "num_key_value_heads": 8, "rms_norm_eps": 1e-05, "rope_theta": 1000000000.0, "sliding_window": null, "tie_word_embeddings": false, "torch_dtype": "bfloat16", "transformers_version": "4.51.3", "use_cache": true, "vocab_size": 131072 }, "verifiedAt": "2026-09-22" }, { "id": "mistralai/Magistral-Small-2506", "meta": { "sha": "ad2fce5b4113139e1877dcadfd826c9262ad8e8c", "safetensors": { "parameters": { "BF16": 23572403200 }, "total": 23572403200 }, "modelType": "mistral" }, "config": { "architectures": [ "MistralForCausalLM" ], "attention_dropout": 0.0, "bos_token_id": 1, "eos_token_id": 2, "head_dim": 128, "hidden_act": "silu", "hidden_size": 5120, "initializer_range": 0.02, "intermediate_size": 32768, "max_position_embeddings": 40960, "model_type": "mistral", "num_attention_heads": 32, "num_hidden_layers": 40, "num_key_value_heads": 8, "rms_norm_eps": 1e-05, "rope_theta": 1000000000.0, "sliding_window": null, "tie_word_embeddings": false, "torch_dtype": "bfloat16", "transformers_version": "4.52.4", "use_cache": true, "vocab_size": 131072 }, "verifiedAt": "2026-09-22" } ] }