{
  "id": "llm-models",
  "title": "Open LLM architecture parameters",
  "description": "Parameter counts, active parameters, layers, KV heads, head dimension, derived KV-cache bytes per token and context limits for Qwen, Llama, DeepSeek and MiniMax models.",
  "version": "1.0",
  "updatedAt": "2026-11-06",
  "license": "https://creativecommons.org/licenses/by/4.0/",
  "publisher": "Cloud Provider Benchmarks",
  "url": "https://www.cloudproviderbenchmarks.com/datasets/llm-models/",
  "columns": [
    {
      "key": "model",
      "description": "Model id"
    },
    {
      "key": "name",
      "description": "Model name"
    },
    {
      "key": "family",
      "description": "Model family"
    },
    {
      "key": "architecture",
      "description": "dense | moe"
    },
    {
      "key": "attention",
      "description": "gqa | mla | hybrid-linear"
    },
    {
      "key": "params_b",
      "description": "Total parameters, billions"
    },
    {
      "key": "active_params_b",
      "description": "Active parameters per token, billions"
    },
    {
      "key": "layers",
      "description": "Transformer layers"
    },
    {
      "key": "kv_heads",
      "description": "KV heads"
    },
    {
      "key": "head_dim",
      "description": "Head dimension"
    },
    {
      "key": "kv_bytes_per_token_16bit",
      "description": "Derived KV-cache bytes per token at 16-bit (null if not modelled)"
    },
    {
      "key": "max_context",
      "description": "Maximum context, tokens"
    },
    {
      "key": "hf_id",
      "description": "Hugging Face repository"
    }
  ],
  "rows": [
    {
      "model": "llama-3-1-8b",
      "name": "Llama 3.1 8B Instruct",
      "family": "llama",
      "architecture": "dense",
      "attention": "gqa",
      "params_b": 8.03,
      "active_params_b": 8.03,
      "layers": 32,
      "kv_heads": 8,
      "head_dim": 128,
      "kv_bytes_per_token_16bit": 131072,
      "max_context": 131072,
      "hf_id": "meta-llama/Llama-3.1-8B-Instruct"
    },
    {
      "model": "llama-3-3-70b",
      "name": "Llama 3.3 70B Instruct",
      "family": "llama",
      "architecture": "dense",
      "attention": "gqa",
      "params_b": 70.6,
      "active_params_b": 70.6,
      "layers": 80,
      "kv_heads": 8,
      "head_dim": 128,
      "kv_bytes_per_token_16bit": 327680,
      "max_context": 131072,
      "hf_id": "meta-llama/Llama-3.3-70B-Instruct"
    },
    {
      "model": "qwen2-5-7b",
      "name": "Qwen2.5 7B Instruct",
      "family": "qwen",
      "architecture": "dense",
      "attention": "gqa",
      "params_b": 7.61,
      "active_params_b": 7.61,
      "layers": 28,
      "kv_heads": 4,
      "head_dim": 128,
      "kv_bytes_per_token_16bit": 57344,
      "max_context": 131072,
      "hf_id": "Qwen/Qwen2.5-7B-Instruct"
    },
    {
      "model": "qwen3-32b",
      "name": "Qwen3 32B",
      "family": "qwen",
      "architecture": "dense",
      "attention": "gqa",
      "params_b": 32.8,
      "active_params_b": 32.8,
      "layers": 64,
      "kv_heads": 8,
      "head_dim": 128,
      "kv_bytes_per_token_16bit": 262144,
      "max_context": 131072,
      "hf_id": "Qwen/Qwen3-32B"
    },
    {
      "model": "qwen3-30b-a3b",
      "name": "Qwen3 30B-A3B",
      "family": "qwen",
      "architecture": "moe",
      "attention": "gqa",
      "params_b": 30.5,
      "active_params_b": 3.3,
      "layers": 48,
      "kv_heads": 4,
      "head_dim": 128,
      "kv_bytes_per_token_16bit": 98304,
      "max_context": 131072,
      "hf_id": "Qwen/Qwen3-30B-A3B"
    },
    {
      "model": "qwen2-5-72b",
      "name": "Qwen2.5 72B Instruct",
      "family": "qwen",
      "architecture": "dense",
      "attention": "gqa",
      "params_b": 72.7,
      "active_params_b": 72.7,
      "layers": 80,
      "kv_heads": 8,
      "head_dim": 128,
      "kv_bytes_per_token_16bit": 327680,
      "max_context": 131072,
      "hf_id": "Qwen/Qwen2.5-72B-Instruct"
    },
    {
      "model": "deepseek-r1-distill-qwen-32b",
      "name": "DeepSeek-R1-Distill-Qwen-32B",
      "family": "deepseek",
      "architecture": "dense",
      "attention": "gqa",
      "params_b": 32.8,
      "active_params_b": 32.8,
      "layers": 64,
      "kv_heads": 8,
      "head_dim": 128,
      "kv_bytes_per_token_16bit": 262144,
      "max_context": 131072,
      "hf_id": "deepseek-ai/DeepSeek-R1-Distill-Qwen-32B"
    },
    {
      "model": "deepseek-v3",
      "name": "DeepSeek-V3",
      "family": "deepseek",
      "architecture": "moe",
      "attention": "mla",
      "params_b": 671,
      "active_params_b": 37,
      "layers": 61,
      "kv_heads": null,
      "head_dim": null,
      "kv_bytes_per_token_16bit": 70272,
      "max_context": 131072,
      "hf_id": "deepseek-ai/DeepSeek-V3"
    },
    {
      "model": "minimax-m1-80k",
      "name": "MiniMax-M1-80k",
      "family": "minimax",
      "architecture": "moe",
      "attention": "hybrid-linear",
      "params_b": 456,
      "active_params_b": 45.9,
      "layers": null,
      "kv_heads": null,
      "head_dim": null,
      "kv_bytes_per_token_16bit": null,
      "max_context": 1000000,
      "hf_id": "MiniMaxAI/MiniMax-M1-80k"
    }
  ]
}