{
 "generated_at": "2026-11-12",
 "site": "https://www.onvergeai.com/tools/perf-benchmarks",
 "inclusion_rule": "只收写明引擎、显卡/实例、模型、量化与工作负载的测量；厂商自报与第三方复测分别标注；单来源博客不入库。",
 "throughput_measured": {
  "source": "https://nvidia.github.io/TensorRT-LLM/performance/perf-overview.html",
  "engine": "NVIDIA TensorRT-LLM",
  "weights": "ModelOpt 量化权重（NVIDIA 发布）",
  "metric": "total output throughput (tokens/sec) at infinite request rate（最大负载吞吐，非单请求延迟）",
  "caveats": [
   "不含 RTX 消费级显卡",
   "不含 AMD",
   "无 并发×P99 延迟 曲线"
  ],
  "rows": [
   {
    "model": "Llama 3.3 70B",
    "quant": "FP4",
    "gpu": "B200",
    "tp": 1,
    "isl_osl_tokens_per_sec": {
     "128/128": 10613.84,
     "1024/2048": 6546.61,
     "2048/2048": 4528.48,
     "5000/500": 1427.44
    }
   },
   {
    "model": "Llama 3.3 70B",
    "quant": "FP4",
    "gpu": "GB200",
    "tp": 1,
    "isl_osl_tokens_per_sec": {
     "128/128": 11100.97,
     "1024/2048": 7922.88,
     "2048/2048": 5326.77,
     "5000/500": 1502.44
    }
   },
   {
    "model": "Llama 3.3 70B",
    "quant": "FP8",
    "gpu": "H100",
    "tp": 2,
    "isl_osl_tokens_per_sec": {
     "128/128": 6092.28,
     "1024/2048": 3785.04,
     "2048/2048": 2785.53,
     "5000/500": 865.55
    }
   },
   {
    "model": "Llama 3.3 70B",
    "quant": "FP8",
    "gpu": "H200",
    "tp": 2,
    "isl_osl_tokens_per_sec": {
     "128/128": 6327.98,
     "1024/2048": 5480.44,
     "2048/2048": 3775.8,
     "5000/500": 978.28
    }
   },
   {
    "model": "Llama 3.1 8B",
    "quant": "FP8",
    "gpu": "H100",
    "tp": 1,
    "isl_osl_tokens_per_sec": {
     "128/128": 26401.48,
     "1024/2048": 13165.91,
     "2048/2048": 9462.43,
     "5000/500": 3276.47
    }
   },
   {
    "model": "Llama 3.1 8B",
    "quant": "FP8",
    "gpu": "H200",
    "tp": 1,
    "isl_osl_tokens_per_sec": {
     "128/128": 27027.8,
     "1024/2048": 16057.66,
     "2048/2048": 11822.14,
     "5000/500": 3758.08
    }
   },
   {
    "model": "Llama 3.1 405B",
    "quant": "FP8",
    "gpu": "H100",
    "tp": 8,
    "isl_osl_tokens_per_sec": {
     "1024/2048": 3237.41,
     "2048/2048": 2216.55,
     "5000/500": 579.05
    }
   },
   {
    "model": "Llama 3.1 405B",
    "quant": "FP8",
    "gpu": "H200",
    "tp": 8,
    "isl_osl_tokens_per_sec": {
     "1024/2048": 3609.5,
     "2048/2048": 2840.86,
     "5000/500": 645.26
    }
   },
   {
    "model": "Llama 4 Maverick",
    "quant": "FP8",
    "gpu": "H200",
    "tp": 8,
    "isl_osl_tokens_per_sec": {
     "1024/2048": 16859.45,
     "2048/2048": 12800.89,
     "5000/500": 5128.6
    }
   }
  ]
 },
 "quantization_accuracy_loss": {
  "status": "no_public_dataset",
  "source": "https://arxiv.org/abs/2611.04380",
  "source_title": "Twice as Fast, Until It Isn't: Quantized Serving of Mid-Size Open LLMs (arXiv:2611.04380, 2026-11-03)",
  "findings": [
   "同时具备 vLLM 计时与精度分数的 checkpoint 只有两个，均为 Gemma 3 12B INT4 构建，且精度只覆盖单一 MMLU 学科（AWQ 校准科目）",
   "batch-1 计时早于当前引擎版本；AMD 与 Intel Gaudi 在该参数带无任何计时；FP4 无对 BF16 的计时",
   "多数公开精度差（含全部 HumanEval）落在基准采样噪声内",
   "小 batch 解码受带宽限制：4bit 约 2 倍速（Qwen2.5-14B on A100 = 2.12x）；大 batch 转 compute-bound 后增益消失（Gemma 3 12B bs=256 仅 1.02x），此时 FP8 仍有效"
  ],
  "site_policy": "不引用拼接而成的精度差数字；该列数据需固定 harness 三档权重重跑自测，暂无对外数据"
 },
 "chunk_recall": {
  "source": "https://arxiv.org/abs/2505.21700",
  "source_title": "Rethinking Chunk Size for Long-Document Retrieval (arXiv:2505.21700v2, 2025-05-29)",
  "rows": [
   {
    "dataset": "TechQA",
    "doc_tokens_avg": 7597,
    "recall_at_1": {
     "64": 0.048,
     "1024": 0.715
    }
   }
  ],
  "caveats": [
   "英文数据集",
   "固定大小切分",
   "单篇论文，中文文档需自测"
  ]
 },
 "vector_db": {
  "source": "https://zilliz.com/vdbbench-leaderboard",
  "tool": "VectorDBBench (open source, reproducible)",
  "disclosure": "榜单由 Zilliz（Milvus 商业方）维护，Milvus 侧存在主场优势",
  "dataset_1m": {
   "desc": "Cohere Medium 1M x 768d, $1,000/月成本档",
   "rows": [
    {
     "system": "ZillizCloud-8cu-perf",
     "qps": 13316.23,
     "p99_ms": 2,
     "recall": 0.9383
    },
    {
     "system": "Milvus-16c64g-sq4u-fp16",
     "qps": 9575.69,
     "p99_ms": 2.3,
     "recall": 0.9189
    },
    {
     "system": "Milvus-16c64g-sq8",
     "qps": 5973,
     "p99_ms": 2.4,
     "recall": 0.9192
    },
    {
     "system": "OpenSearch-16c128g",
     "qps": 3055.01,
     "p99_ms": 7.2,
     "recall": 0.9066
    },
    {
     "system": "ElasticCloud-8c60g",
     "qps": 2353.89,
     "p99_ms": 17.1,
     "recall": 0.9143
    },
    {
     "system": "QdrantCloud-16c64g",
     "qps": 1242.43,
     "p99_ms": 6.4,
     "recall": 0.9474
    },
    {
     "system": "Pinecone-p2.x8-1node",
     "qps": 1146.53,
     "p99_ms": 13.7,
     "recall": 0.9262
    }
   ]
  },
  "dataset_10m_streaming": {
   "desc": "Cohere Large 10M, 90% 容量, 持续写入",
   "rows": [
    {
     "system": "ZillizCloud-8cu-perf",
     "static_qps": 7385,
     "qps_at_500_rows_s": 2119,
     "qps_at_1000_rows_s": 1860,
     "recall": 0.9384
    },
    {
     "system": "Milvus-16c64g-sq8",
     "static_qps": 2747,
     "qps_at_500_rows_s": 306,
     "qps_at_1000_rows_s": 156,
     "recall": 0.9204
    },
    {
     "system": "ElasticCloud-8c60g",
     "static_qps": 1452,
     "qps_at_500_rows_s": 61.67,
     "qps_at_1000_rows_s": 61.82,
     "recall": 0.9042
    },
    {
     "system": "Pinecone-p2.x8-1node",
     "static_qps": 1131,
     "qps_at_500_rows_s": 367.4,
     "qps_at_1000_rows_s": 369.7,
     "recall": 0.9024
    },
    {
     "system": "OpenSearch-16c128g",
     "static_qps": 505.7,
     "qps_at_500_rows_s": 161.7,
     "qps_at_1000_rows_s": 149.7,
     "recall": 0.9068
    },
    {
     "system": "QdrantCloud-16c64g",
     "static_qps": 446.9,
     "qps_at_500_rows_s": 393.8,
     "qps_at_1000_rows_s": 347.6,
     "recall": 0.9357
    }
   ]
  }
 },
 "poc_gates": {
  "accuracy": {
   "threshold": "问答准确率 >= 85%",
   "sample": "业务真实题集 >=200 题人工标注"
  },
  "retrieval": {
   "threshold": "Recall@5 >= 90% 且 MRR >= 0.75",
   "sample": ">=100 条问题-片段标注"
  },
  "latency": {
   "threshold": "首 token P95 <= 2s 且单请求 >= 20 tok/s",
   "env": "内网无外网出口，目标并发下测"
  },
  "stability": {
   "threshold": "连续 8 小时零 OOM，错误率 < 0.5%",
   "include": "长上下文与超长输入用例"
  }
 },
 "cost_model": {
  "status": "self_hosted_is_not_token_billed",
  "statement": "私有化部署不按 token 收费：token 输出没有价格，自部署买的是容量不是用量。与云 API 只能按「每百万 token 当量成本」比，且必须写清利用率假设。",
  "cost_items": [
   {
    "item": "GPU / 服务器 / 机房",
    "nature": "资产，不随 token 变动",
    "note": "一次性或分期，3-5 年折旧；采购价实时变动"
   },
   {
    "item": "电费",
    "nature": "唯一随用量线性变化",
    "note": "GPU 功耗 × PUE × 电价 = token 边际成本"
   },
   {
    "item": "运维人力",
    "nature": "固定投入",
    "note": "部署运维、监控、每年 2 次微调（本站运维费占合同总价 50%）"
   },
   {
    "item": "存储 / 带宽",
    "nature": "半固定",
    "note": "内网为主，公网出口按业务"
   }
  ],
  "electricity_model": {
   "gpu_watt": 700,
   "pue": 1.3,
   "price_cny_per_kwh": 0.7,
   "kwh_per_day": 21.84,
   "cny_per_day": 15.29
  },
  "examples": [
   {
    "scenario": "Llama 3.3 70B FP8",
    "config": "2xH100 TP2",
    "throughput_tok_s": 6092,
    "tokens_per_day_m": 526.3,
    "power_cost_cny_per_day": 30.58,
    "cny_per_million_tokens_electricity_only": 0.058
   },
   {
    "scenario": "Llama 3.1 8B FP8",
    "config": "1xH100 TP1",
    "throughput_tok_s": 26401,
    "tokens_per_day_m": 2281.0,
    "power_cost_cny_per_day": 15.29,
    "cny_per_million_tokens_electricity_only": 0.0067
   }
  ],
  "full_cost_example": {
   "assumptions": {
    "cards": 2,
    "card_price_cny": 240000,
    "depreciation_years": 3,
    "ops_cny_per_year_cluster": 400000,
    "cluster_cards": 8
   },
   "cny_per_day": 743,
   "cny_per_million_tokens_full_load": 1.41,
   "cny_per_million_tokens_50pct_util": 2.82
  },
  "comparison_with_api": {
   "marginal_cost_vs_api": "0.0067 元/百万 vs 10 元/百万 API ≈ 1/1500（8B 高吞吐）；0.058 元/百万 vs 10 元 API ≈ 1/170，vs 30 元 API ≈ 1/500",
   "full_cost_vs_api": "1.4-2.8 元/百万 = 主流 API 的 1/3 ~ 1/10，不是千分之一",
   "break_even_utilization": {
    "vs_api_10_cny": 0.15,
    "vs_api_30_cny": 0.05,
    "note": "利用率低于临界点时自部署当量单价反而高于 API"
   }
  },
  "decision_note": "选型结论不是「自部署 token 更便宜」，而是利用率高+用量大时便宜一个数量级；真正决定自部署的是数据不出内网、可微调、可审计、无锁定、延迟可控。",
  "price_entry": "https://www.onvergeai.com/hardware"
 }
}