mirror of
https://github.com/kvcache-ai/ktransformers.git
synced 2026-10-22 01:27:10 +08:00
2194 lines
156 KiB
HTML
2194 lines
156 KiB
HTML
<!DOCTYPE HTML>
|
||
<html lang="zh-CN" class="light sidebar-visible" dir="ltr">
|
||
<head>
|
||
<!-- Book generated using mdBook -->
|
||
<meta charset="UTF-8">
|
||
<title>Ktransformers</title>
|
||
<meta name="robots" content="noindex">
|
||
|
||
|
||
<!-- Custom HTML head -->
|
||
|
||
<meta name="description" content="">
|
||
<meta name="viewport" content="width=device-width, initial-scale=1">
|
||
<meta name="theme-color" content="#ffffff">
|
||
|
||
<link rel="icon" href="favicon.svg">
|
||
<link rel="shortcut icon" href="favicon.png">
|
||
<link rel="stylesheet" href="css/variables.css">
|
||
<link rel="stylesheet" href="css/general.css">
|
||
<link rel="stylesheet" href="css/chrome.css">
|
||
<link rel="stylesheet" href="css/print.css" media="print">
|
||
|
||
<!-- Fonts -->
|
||
<link rel="stylesheet" href="FontAwesome/css/font-awesome.css">
|
||
<link rel="stylesheet" href="fonts/fonts.css">
|
||
|
||
<!-- Highlight.js Stylesheets -->
|
||
<link rel="stylesheet" id="highlight-css" href="highlight.css">
|
||
<link rel="stylesheet" id="tomorrow-night-css" href="tomorrow-night.css">
|
||
<link rel="stylesheet" id="ayu-highlight-css" href="ayu-highlight.css">
|
||
|
||
<!-- Custom theme stylesheets -->
|
||
|
||
|
||
<!-- Provide site root and default themes to javascript -->
|
||
<script>
|
||
const path_to_root = "";
|
||
const default_light_theme = "light";
|
||
const default_dark_theme = "navy";
|
||
window.path_to_searchindex_js = "searchindex.js";
|
||
</script>
|
||
<!-- Start loading toc.js asap -->
|
||
<script src="toc.js"></script>
|
||
</head>
|
||
<body>
|
||
<div id="mdbook-help-container">
|
||
<div id="mdbook-help-popup">
|
||
<h2 class="mdbook-help-title">Keyboard shortcuts</h2>
|
||
<div>
|
||
<p>Press <kbd>←</kbd> or <kbd>→</kbd> to navigate between chapters</p>
|
||
<p>Press <kbd>S</kbd> or <kbd>/</kbd> to search in the book</p>
|
||
<p>Press <kbd>?</kbd> to show this help</p>
|
||
<p>Press <kbd>Esc</kbd> to hide this help</p>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
<div id="body-container">
|
||
<!-- Work around some values being stored in localStorage wrapped in quotes -->
|
||
<script>
|
||
try {
|
||
let theme = localStorage.getItem('mdbook-theme');
|
||
let sidebar = localStorage.getItem('mdbook-sidebar');
|
||
|
||
if (theme.startsWith('"') && theme.endsWith('"')) {
|
||
localStorage.setItem('mdbook-theme', theme.slice(1, theme.length - 1));
|
||
}
|
||
|
||
if (sidebar.startsWith('"') && sidebar.endsWith('"')) {
|
||
localStorage.setItem('mdbook-sidebar', sidebar.slice(1, sidebar.length - 1));
|
||
}
|
||
} catch (e) { }
|
||
</script>
|
||
|
||
<!-- Set the theme before any content is loaded, prevents flash -->
|
||
<script>
|
||
const default_theme = window.matchMedia("(prefers-color-scheme: dark)").matches ? default_dark_theme : default_light_theme;
|
||
let theme;
|
||
try { theme = localStorage.getItem('mdbook-theme'); } catch(e) { }
|
||
if (theme === null || theme === undefined) { theme = default_theme; }
|
||
const html = document.documentElement;
|
||
html.classList.remove('light')
|
||
html.classList.add(theme);
|
||
html.classList.add("js");
|
||
</script>
|
||
|
||
<input type="checkbox" id="sidebar-toggle-anchor" class="hidden">
|
||
|
||
<!-- Hide / unhide sidebar before it is displayed -->
|
||
<script>
|
||
let sidebar = null;
|
||
const sidebar_toggle = document.getElementById("sidebar-toggle-anchor");
|
||
if (document.body.clientWidth >= 1080) {
|
||
try { sidebar = localStorage.getItem('mdbook-sidebar'); } catch(e) { }
|
||
sidebar = sidebar || 'visible';
|
||
} else {
|
||
sidebar = 'hidden';
|
||
sidebar_toggle.checked = false;
|
||
}
|
||
if (sidebar === 'visible') {
|
||
sidebar_toggle.checked = true;
|
||
} else {
|
||
html.classList.remove('sidebar-visible');
|
||
}
|
||
</script>
|
||
|
||
<nav id="sidebar" class="sidebar" aria-label="Table of contents">
|
||
<!-- populated by js -->
|
||
<mdbook-sidebar-scrollbox class="sidebar-scrollbox"></mdbook-sidebar-scrollbox>
|
||
<noscript>
|
||
<iframe class="sidebar-iframe-outer" src="toc.html"></iframe>
|
||
</noscript>
|
||
<div id="sidebar-resize-handle" class="sidebar-resize-handle">
|
||
<div class="sidebar-resize-indicator"></div>
|
||
</div>
|
||
</nav>
|
||
|
||
<div id="page-wrapper" class="page-wrapper">
|
||
|
||
<div class="page">
|
||
<div id="menu-bar-hover-placeholder"></div>
|
||
<div id="menu-bar" class="menu-bar sticky">
|
||
<div class="left-buttons">
|
||
<label id="sidebar-toggle" class="icon-button" for="sidebar-toggle-anchor" title="Toggle Table of Contents" aria-label="Toggle Table of Contents" aria-controls="sidebar">
|
||
<i class="fa fa-bars"></i>
|
||
</label>
|
||
<button id="theme-toggle" class="icon-button" type="button" title="Change theme" aria-label="Change theme" aria-haspopup="true" aria-expanded="false" aria-controls="theme-list">
|
||
<i class="fa fa-paint-brush"></i>
|
||
</button>
|
||
<ul id="theme-list" class="theme-popup" aria-label="Themes" role="menu">
|
||
<li role="none"><button role="menuitem" class="theme" id="default_theme">Auto</button></li>
|
||
<li role="none"><button role="menuitem" class="theme" id="light">Light</button></li>
|
||
<li role="none"><button role="menuitem" class="theme" id="rust">Rust</button></li>
|
||
<li role="none"><button role="menuitem" class="theme" id="coal">Coal</button></li>
|
||
<li role="none"><button role="menuitem" class="theme" id="navy">Navy</button></li>
|
||
<li role="none"><button role="menuitem" class="theme" id="ayu">Ayu</button></li>
|
||
</ul>
|
||
<button id="search-toggle" class="icon-button" type="button" title="Search (`/`)" aria-label="Toggle Searchbar" aria-expanded="false" aria-keyshortcuts="/ s" aria-controls="searchbar">
|
||
<i class="fa fa-search"></i>
|
||
</button>
|
||
</div>
|
||
|
||
<h1 class="menu-title">Ktransformers</h1>
|
||
|
||
<div class="right-buttons">
|
||
<a href="print.html" title="Print this book" aria-label="Print this book">
|
||
<i id="print-button" class="fa fa-print"></i>
|
||
</a>
|
||
<a href="https://github.com/kvcache-ai/ktransformers" title="Git repository" aria-label="Git repository">
|
||
<i id="git-repository-button" class="fa fa-github"></i>
|
||
</a>
|
||
|
||
</div>
|
||
</div>
|
||
|
||
<div id="search-wrapper" class="hidden">
|
||
<form id="searchbar-outer" class="searchbar-outer">
|
||
<div class="search-wrapper">
|
||
<input type="search" id="searchbar" name="searchbar" placeholder="Search this book ..." aria-controls="searchresults-outer" aria-describedby="searchresults-header">
|
||
<div class="spinner-wrapper">
|
||
<i class="fa fa-spinner fa-spin"></i>
|
||
</div>
|
||
</div>
|
||
</form>
|
||
<div id="searchresults-outer" class="searchresults-outer hidden">
|
||
<div id="searchresults-header" class="searchresults-header"></div>
|
||
<ul id="searchresults">
|
||
</ul>
|
||
</div>
|
||
</div>
|
||
|
||
<!-- Apply ARIA attributes after the sidebar and the sidebar toggle button are added to the DOM -->
|
||
<script>
|
||
document.getElementById('sidebar-toggle').setAttribute('aria-expanded', sidebar === 'visible');
|
||
document.getElementById('sidebar').setAttribute('aria-hidden', sidebar !== 'visible');
|
||
Array.from(document.querySelectorAll('#sidebar a')).forEach(function(link) {
|
||
link.setAttribute('tabIndex', sidebar === 'visible' ? 0 : -1);
|
||
});
|
||
</script>
|
||
|
||
<div id="content" class="content">
|
||
<main>
|
||
<div align="center">
|
||
<p align="center">
|
||
<picture>
|
||
<img alt="KTransformers" src="https://github.com/user-attachments/assets/d5a2492f-a415-4456-af99-4ab102f13f8b" width=50%>
|
||
</picture>
|
||
</p>
|
||
<h3 id="a-flexible-framework-for-experiencing-cutting-edge-llm-inferencefine-tune-optimizations"><a class="header" href="#a-flexible-framework-for-experiencing-cutting-edge-llm-inferencefine-tune-optimizations">A Flexible Framework for Experiencing Cutting-edge LLM Inference/Fine-tune Optimizations</a></h3>
|
||
<strong><a href="index.html#-overview">🎯 Overview</a> | <a href="index.html#-kt-kernel---high-performance-inference-kernels">🚀 kt-kernel</a> | <a href="index.html#-kt-sft---fine-tuning-framework">🎓 KT-SFT</a> | <a href="index.html#-citation">🔥 Citation</a> | <a href="https://github.com/kvcache-ai/ktransformers/issues/1582">🚀 Roadmap(2025Q4)</a> </strong>
|
||
</div>
|
||
<h2 id="-overview"><a class="header" href="#-overview">🎯 Overview</a></h2>
|
||
<p>KTransformers is a research project focused on efficient inference and fine-tuning of large language models through CPU-GPU heterogeneous computing. The project has evolved into <strong>two core modules</strong>: <a href="./kt-kernel/">kt-kernel</a> and <a href="./KT-SFT/">KT-SFT</a>.</p>
|
||
<h2 id="-updates"><a class="header" href="#-updates">🔥 Updates</a></h2>
|
||
<ul>
|
||
<li><strong>Nov 6, 2025</strong>: Support Kimi-K2-Thinking inference (<a href="./doc/en/Kimi-K2-Thinking.html">Tutorial</a>) and fine-tune (<a href="./doc/en/SFT_Installation_Guide_KimiK2.html">Tutorial</a>)</li>
|
||
<li><strong>Nov 4, 2025</strong>: KTransformers Fine-Tuning × LLaMA-Factory Integration. (<a href="./doc/en/KTransformers-Fine-Tuning_User-Guide.html">Tutorial</a>)</li>
|
||
<li><strong>Oct 27, 2025</strong>: Support Ascend NPU. (<a href="./doc/zh/DeepseekR1_V3_tutorial_zh_for_Ascend_NPU.html">Tutorial</a>)</li>
|
||
<li><strong>Oct 10, 2025</strong>: Integrating into SGLang. (<a href="https://github.com/sgl-project/sglang/issues/11425">Roadmap</a>, <a href="https://lmsys.org/blog/2025-10-22-KTransformers/">Blog</a>)</li>
|
||
<li><strong>Sept 11, 2025</strong>: Support Qwen3-Next. (<a href="./doc/en/Qwen3-Next.html">Tutorial</a>)</li>
|
||
<li><strong>Sept 05, 2025</strong>: Support Kimi-K2-0905. (<a href="./doc/en/Kimi-K2.html">Tutorial</a>)</li>
|
||
<li><strong>July 26, 2025</strong>: Support SmallThinker and GLM4-MoE. (<a href="./doc/en/SmallThinker_and_Glm4moe.html">Tutorial</a>)</li>
|
||
<li><strong>July 11, 2025</strong>: Support Kimi-K2. (<a href="./doc/en/Kimi-K2.html">Tutorial</a>)</li>
|
||
<li><strong>June 30, 2025</strong>: Support 3-layer (GPU-CPU-Disk) <a href="./doc/en/prefix_cache.html">prefix cache</a> reuse.</li>
|
||
<li><strong>May 14, 2025</strong>: Support Intel Arc GPU (<a href="./doc/en/xpu.html">Tutorial</a>).</li>
|
||
<li><strong>Apr 29, 2025</strong>: Support AMX-Int8、 AMX-BF16 and Qwen3MoE (<a href="./doc/en/AMX.html">Tutorial</a>)</li>
|
||
<li><strong>Apr 9, 2025</strong>: Experimental support for LLaMA 4 models (<a href="./doc/en/llama4.html">Tutorial</a>).</li>
|
||
<li><strong>Apr 2, 2025</strong>: Support Multi-concurrency. (<a href="./doc/en/balance-serve.html">Tutorial</a>).</li>
|
||
<li><strong>Mar 15, 2025</strong>: Support ROCm on AMD GPU (<a href="./doc/en/ROCm.html">Tutorial</a>).</li>
|
||
<li><strong>Mar 5, 2025</strong>: Support unsloth 1.58/2.51 bits weights and <a href="./doc/en/fp8_kernel.html">IQ1_S/FP8 hybrid</a> weights. Support 139K <a href="./doc/en/DeepseekR1_V3_tutorial.html#v022--v023-longer-context--fp8-kernel">Longer Context</a> for DeepSeek-V3 and R1 in 24GB VRAM.</li>
|
||
<li><strong>Feb 25, 2025</strong>: Support <a href="./doc/en/fp8_kernel.html">FP8 GPU kernel</a> for DeepSeek-V3 and R1; <a href="./doc/en/DeepseekR1_V3_tutorial.html#v022-longer-context">Longer Context</a>.</li>
|
||
<li><strong>Feb 15, 2025</strong>: Longer Context (from 4K to 8K for 24GB VRAM) & Slightly Faster Speed (+15%, up to 16 Tokens/s), update <a href="./doc/en/DeepseekR1_V3_tutorial.html">docs</a> and <a href="https://kvcache-ai.github.io/ktransformers/">online books</a>.</li>
|
||
<li><strong>Feb 10, 2025</strong>: Support Deepseek-R1 and V3 on single (24GB VRAM)/multi gpu and 382G DRAM, up to 3~28x speedup. For detailed show case and reproduction tutorial, see <a href="./doc/en/DeepseekR1_V3_tutorial.html">here</a>.</li>
|
||
<li><strong>Aug 28, 2024</strong>: Decrease DeepseekV2's required VRAM from 21G to 11G.</li>
|
||
<li><strong>Aug 15, 2024</strong>: Update detailed <a href="doc/en/injection_tutorial.html">tutorial</a> for injection and multi-GPU.</li>
|
||
<li><strong>Aug 14, 2024</strong>: Support llamfile as linear backend.</li>
|
||
<li><strong>Aug 12, 2024</strong>: Support multiple GPU; Support new model: mixtral 8*7B and 8*22B; Support q2k, q3k, q5k dequant on gpu.</li>
|
||
<li><strong>Aug 9, 2024</strong>: Support windows native.</li>
|
||
</ul>
|
||
<hr />
|
||
<h2 id="-core-modules"><a class="header" href="#-core-modules">📦 Core Modules</a></h2>
|
||
<h3 id="-kt-kernel---high-performance-inference-kernels"><a class="header" href="#-kt-kernel---high-performance-inference-kernels">🚀 <a href="./kt-kernel/">kt-kernel</a> - High-Performance Inference Kernels</a></h3>
|
||
<p>CPU-optimized kernel operations for heterogeneous LLM inference.</p>
|
||
<img width="1049" height="593" alt="image" src="https://github.com/user-attachments/assets/68f423da-3f55-4025-bdc9-9ceaa554f00b" />
|
||
<p><strong>Key Features:</strong></p>
|
||
<ul>
|
||
<li><strong>AMX/AVX Acceleration</strong>: Intel AMX and AVX512/AVX2 optimized kernels for INT4/INT8 quantized inference</li>
|
||
<li><strong>MoE Optimization</strong>: Efficient Mixture-of-Experts inference with NUMA-aware memory management</li>
|
||
<li><strong>Quantization Support</strong>: CPU-side INT4/INT8 quantized weights, GPU-side GPTQ support</li>
|
||
<li><strong>Easy Integration</strong>: Clean Python API for SGLang and other frameworks</li>
|
||
</ul>
|
||
<p><strong>Quick Start:</strong></p>
|
||
<pre><code class="language-bash">cd kt-kernel
|
||
pip install .
|
||
</code></pre>
|
||
<p><strong>Use Cases:</strong></p>
|
||
<ul>
|
||
<li>CPU-GPU hybrid inference for large MoE models</li>
|
||
<li>Integration with SGLang for production serving</li>
|
||
<li>Heterogeneous expert placement (hot experts on GPU, cold experts on CPU)</li>
|
||
</ul>
|
||
<p><strong>Performance Examples:</strong></p>
|
||
<div class="table-wrapper"><table><thead><tr><th>Model</th><th>Hardware Configuration</th><th>Total Throughput</th><th>Output Throughput</th></tr></thead><tbody>
|
||
<tr><td>DeepSeek-R1-0528 (FP8)</td><td>8×L20 GPU + Xeon Gold 6454S</td><td>227.85 tokens/s</td><td>87.58 tokens/s (8-way concurrency)</td></tr>
|
||
</tbody></table>
|
||
</div>
|
||
<p>👉 <strong><a href="./kt-kernel/README.html">Full Documentation →</a></strong></p>
|
||
<hr />
|
||
<h3 id="-kt-sft---fine-tuning-framework"><a class="header" href="#-kt-sft---fine-tuning-framework">🎓 <a href="./KT-SFT/">KT-SFT</a> - Fine-Tuning Framework</a></h3>
|
||
<p>KTransformers × LLaMA-Factory integration for ultra-large MoE model fine-tuning.</p>
|
||
<p><img src="./doc/assets/image-20251011010558909.png" alt="image-20251011010558909" /></p>
|
||
<p><strong>Key Features:</strong></p>
|
||
<ul>
|
||
<li><strong>Resource Efficient</strong>: Fine-tune 671B DeepSeek-V3 with just <strong>70GB GPU memory</strong> + 1.3TB RAM</li>
|
||
<li><strong>LoRA Support</strong>: Full LoRA fine-tuning with heterogeneous acceleration</li>
|
||
<li><strong>LLaMA-Factory Integration</strong>: Seamless integration with popular fine-tuning framework</li>
|
||
<li><strong>Production Ready</strong>: Chat, batch inference, and metrics evaluation</li>
|
||
</ul>
|
||
<p><strong>Performance Examples:</strong></p>
|
||
<div class="table-wrapper"><table><thead><tr><th>Model</th><th>Configuration</th><th>Throughput</th><th>GPU Memory</th></tr></thead><tbody>
|
||
<tr><td>DeepSeek-V3 (671B)</td><td>LoRA + AMX</td><td>~40 tokens/s</td><td>70GB (multi-GPU)</td></tr>
|
||
<tr><td>DeepSeek-V2-Lite (14B)</td><td>LoRA + AMX</td><td>~530 tokens/s</td><td>6GB</td></tr>
|
||
</tbody></table>
|
||
</div>
|
||
<p><strong>Quick Start:</strong></p>
|
||
<pre><code class="language-bash">cd KT-SFT
|
||
# Install environment following KT-SFT/README.md
|
||
USE_KT=1 llamafactory-cli train examples/train_lora/deepseek3_lora_sft_kt.yaml
|
||
</code></pre>
|
||
<p>👉 <strong><a href="./KT-SFT/README.html">Full Documentation →</a></strong></p>
|
||
<hr />
|
||
<h2 id="-citation"><a class="header" href="#-citation">🔥 Citation</a></h2>
|
||
<p>If you use KTransformers in your research, please cite our paper:</p>
|
||
<pre><code class="language-bibtex">@inproceedings{10.1145/3731569.3764843,
|
||
title = {KTransformers: Unleashing the Full Potential of CPU/GPU Hybrid Inference for MoE Models},
|
||
author = {Chen, Hongtao and Xie, Weiyu and Zhang, Boxin and Tang, Jingqi and Wang, Jiahao and Dong, Jianwei and Chen, Shaoyuan and Yuan, Ziwei and Lin, Chen and Qiu, Chengyu and Zhu, Yuening and Ou, Qingliang and Liao, Jiaqi and Chen, Xianglin and Ai, Zhiyuan and Wu, Yongwei and Zhang, Mingxing},
|
||
booktitle = {Proceedings of the ACM SIGOPS 31st Symposium on Operating Systems Principles},
|
||
year = {2025}
|
||
}
|
||
</code></pre>
|
||
<h2 id="-contributors--team"><a class="header" href="#-contributors--team">👥 Contributors & Team</a></h2>
|
||
<p>Developed and maintained by:</p>
|
||
<ul>
|
||
<li><a href="https://madsys.cs.tsinghua.edu.cn/">MADSys Lab</a> @ Tsinghua University</li>
|
||
<li><a href="http://approaching.ai/">Approaching.AI</a></li>
|
||
<li>Community contributors</li>
|
||
</ul>
|
||
<p>We welcome contributions! Please feel free to submit issues and pull requests.</p>
|
||
<h2 id="-community--support"><a class="header" href="#-community--support">💬 Community & Support</a></h2>
|
||
<ul>
|
||
<li><strong>GitHub Issues</strong>: <a href="https://github.com/kvcache-ai/ktransformers/issues">Report bugs or request features</a></li>
|
||
<li><strong>GitHub Discussions</strong>: <a href="https://github.com/kvcache-ai/ktransformers/discussions">Ask questions and share ideas</a></li>
|
||
<li><strong>WeChat Group</strong>: See <a href="./archive/WeChatGroup.png">archive/WeChatGroup.png</a></li>
|
||
</ul>
|
||
<h2 id="-kt-original-code"><a class="header" href="#-kt-original-code">📦 KT original Code</a></h2>
|
||
<p>The original integrated KTransformers framework has been archived to the <a href="./archive/"><code>archive/</code></a> directory for reference. The project now focuses on the two core modules above for better modularity and maintainability.</p>
|
||
<p>For the original documentation with full quick-start guides and examples, see:</p>
|
||
<ul>
|
||
<li><a href="./archive/README.html">archive/README.md</a> (English)</li>
|
||
<li><a href="./archive/README_ZH.html">archive/README_ZH.md</a> (中文)</li>
|
||
</ul>
|
||
<div style="break-before: page; page-break-before: always;"></div><h1 id="kt-kernel"><a class="header" href="#kt-kernel">KT-Kernel</a></h1>
|
||
<p>High-performance kernel operations for KTransformers, featuring CPU-optimized MoE inference with AMX, AVX, KML and blis (amd library) support.</p>
|
||
<ul>
|
||
<li><a href="en/kt-kernel_intro.html#note">Note</a></li>
|
||
<li><a href="en/kt-kernel_intro.html#features">Features</a></li>
|
||
<li><a href="en/kt-kernel_intro.html#installation">Installation</a>
|
||
<ul>
|
||
<li><a href="en/kt-kernel_intro.html#prerequisites">Prerequisites</a></li>
|
||
<li><a href="en/kt-kernel_intro.html#quick-installation-recommended">Quick Installation (Recommended)</a></li>
|
||
<li><a href="en/kt-kernel_intro.html#manual-configuration-advanced">Manual Configuration (Advanced)</a></li>
|
||
</ul>
|
||
</li>
|
||
<li><a href="en/kt-kernel_intro.html#verification">Verification</a></li>
|
||
<li><a href="en/kt-kernel_intro.html#integration-with-sglang">Integration with SGLang</a>
|
||
<ul>
|
||
<li><a href="en/kt-kernel_intro.html#installation-steps">Installation Steps</a></li>
|
||
<li><a href="en/kt-kernel_intro.html#complete-example-qwen3-30b-a3b">Complete Example: Qwen3-30B-A3B</a></li>
|
||
<li><a href="en/kt-kernel_intro.html#kt-kernel-parameters">KT-Kernel Parameters</a></li>
|
||
</ul>
|
||
</li>
|
||
<li><a href="en/kt-kernel_intro.html#direct-python-api-usage">Direct Python API Usage</a>
|
||
<ul>
|
||
<li><a href="en/kt-kernel_intro.html#advanced-options">Advanced Options</a></li>
|
||
</ul>
|
||
</li>
|
||
<li><a href="en/kt-kernel_intro.html#build-configuration">Build Configuration</a>
|
||
<ul>
|
||
<li><a href="en/kt-kernel_intro.html#manual-installation">Manual Installation</a></li>
|
||
</ul>
|
||
</li>
|
||
<li><a href="en/kt-kernel_intro.html#error-troubleshooting">Error Troubleshooting</a>
|
||
<ul>
|
||
<li><a href="en/kt-kernel_intro.html#cuda-not-found">CUDA Not Found</a></li>
|
||
<li><a href="en/kt-kernel_intro.html#hwloc-not-found">hwloc Not Found</a></li>
|
||
</ul>
|
||
</li>
|
||
<li><a href="en/kt-kernel_intro.html#weight-quantization">Weight Quantization</a>
|
||
<ul>
|
||
<li><a href="en/kt-kernel_intro.html#cpu-weights-for-cold-experts-on-cpu">CPU Weights (for "cold" experts on CPU)</a></li>
|
||
<li><a href="en/kt-kernel_intro.html#gpu-weights-for-hot-experts-on-gpu">GPU Weights (for "hot" experts on GPU)</a></li>
|
||
</ul>
|
||
</li>
|
||
<li><a href="en/kt-kernel_intro.html#before-commit">Before Commit!</a></li>
|
||
</ul>
|
||
<h2 id="note"><a class="header" href="#note">Note</a></h2>
|
||
<p><strong>Current Support Status:</strong></p>
|
||
<ul>
|
||
<li>✅ <strong>Intel CPUs with AMX</strong>: Fully supported</li>
|
||
<li>⚠️ <strong>Universal CPU with llamafile</strong>: In preview, not yet fully complete</li>
|
||
<li>⚠️ <strong>AMD CPUs with BLIS</strong>: Upcoming, not yet fully integrated</li>
|
||
</ul>
|
||
<h2 id="features"><a class="header" href="#features">Features</a></h2>
|
||
<ul>
|
||
<li><strong>AMX Optimization</strong>: Intel AMX (Advanced Matrix Extensions) support for INT4/INT8 quantized MoE inference</li>
|
||
<li><strong>Multi-Backend</strong>: Unified <code>KTMoEWrapper</code> API supporting multiple backends (AMXINT4, AMXINT8, LLAMAFILE*)</li>
|
||
<li><strong>Flexible Backends</strong>: AVX512, AVX2 via pluggable backend architecture</li>
|
||
<li><strong>Efficient MoE</strong>: Optimized Mixture-of-Experts operations with NUMA-aware memory management</li>
|
||
<li><strong>Async Execution</strong>: Non-blocking <code>submit_forward</code> / <code>sync_forward</code> API for improved pipelining</li>
|
||
<li><strong>Easy Integration</strong>: Clean Python API with automatic backend selection</li>
|
||
</ul>
|
||
<p><strong>Note</strong>: *LLAMAFILE backend support is currently in <em>preview</em> and not yet fully complete.</p>
|
||
<h2 id="installation"><a class="header" href="#installation">Installation</a></h2>
|
||
<h3 id="prerequisites"><a class="header" href="#prerequisites">Prerequisites</a></h3>
|
||
<p>First, initialize git submodules:</p>
|
||
<pre><code class="language-bash">git submodule update --init --recursive
|
||
</code></pre>
|
||
<h3 id="quick-installation-recommended"><a class="header" href="#quick-installation-recommended">Quick Installation (Recommended)</a></h3>
|
||
<p>Step 0: Create and activate a conda environment (recommended):</p>
|
||
<pre><code class="language-bash">conda create -n kt-kernel python=3.11 -y
|
||
conda activate kt-kernel
|
||
</code></pre>
|
||
<p>You can now install in two clear steps using the same script.</p>
|
||
<p>Option A: Two-step (explicit)</p>
|
||
<pre><code class="language-bash"># 1) Install system prerequisites (cmake, hwloc, pkg-config)
|
||
./install.sh deps
|
||
|
||
# 2) Build and install kt-kernel (auto-detects CPU)
|
||
# By default, the script cleans the local ./build directory before compiling.
|
||
./install.sh build
|
||
</code></pre>
|
||
<p>Option B: One-step (deps + build)</p>
|
||
<pre><code class="language-bash">./install.sh
|
||
</code></pre>
|
||
<p>The install script will:</p>
|
||
<ul>
|
||
<li>Auto-detect CPU capabilities (AMX support)</li>
|
||
<li>Install <code>cmake</code> via conda (if available)</li>
|
||
<li>Install system dependencies (<code>libhwloc-dev</code>, <code>pkg-config</code>) based on your OS</li>
|
||
</ul>
|
||
<p><strong>What gets configured automatically:</strong></p>
|
||
<ul>
|
||
<li>AMX CPU detected → <code>NATIVE + AMX=ON</code></li>
|
||
<li>No AMX detected → <code>NATIVE + AMX=OFF</code></li>
|
||
</ul>
|
||
<p>⚠️ <strong>Important for LLAMAFILE backend users:</strong> If you have an AMX-capable CPU and plan to use the LLAMAFILE backend, do NOT use auto-detection. Use manual mode with <code>AVX512</code> or <code>AVX2</code> instead of <code>NATIVE</code> to avoid compilation issues (see below).</p>
|
||
<h3 id="manual-configuration-advanced"><a class="header" href="#manual-configuration-advanced">Manual Configuration (Advanced)</a></h3>
|
||
<p>If you need specific build options (e.g., for LLAMAFILE backend, compatibility, or binary distribution):</p>
|
||
<pre><code class="language-bash"># Example for LLAMAFILE backend on AMX CPU with AVX512
|
||
export CPUINFER_CPU_INSTRUCT=AVX512 # Options: NATIVE, AVX512, AVX2
|
||
export CPUINFER_ENABLE_AMX=OFF # Options: ON, OFF
|
||
|
||
# Run with manual mode (build only)
|
||
./install.sh build --manual
|
||
</code></pre>
|
||
<p>For advanced build options and binary distribution, see the <a href="en/kt-kernel_intro.html#build-configuration">Build Configuration</a> section. If you encounter issues, refer to <a href="en/kt-kernel_intro.html#error-troubleshooting">Error Troubleshooting</a>.</p>
|
||
<h2 id="verification"><a class="header" href="#verification">Verification</a></h2>
|
||
<pre><code class="language-bash">python -c "from kt_kernel import KTMoEWrapper; print('✓ kt-kernel installed successfully')"
|
||
</code></pre>
|
||
<h2 id="integration-with-sglang"><a class="header" href="#integration-with-sglang">Integration with SGLang</a></h2>
|
||
<p>KT-Kernel can be used standalone via <a href="en/kt-kernel_intro.html#direct-python-api-usage">Direct Python API</a> or integrated with SGLang for production deployment. This section describes SGLang integration to enable CPU-GPU heterogeneous inference, where "hot" experts run on GPU and "cold" experts run on CPU for optimal resource utilization.</p>
|
||
<h3 id="installation-steps"><a class="header" href="#installation-steps">Installation Steps</a></h3>
|
||
<h4 id="1-install-sglang"><a class="header" href="#1-install-sglang">1. Install SGLang</a></h4>
|
||
<pre><code class="language-bash">git clone https://github.com/sgl-project/sglang.git
|
||
cd sglang
|
||
pip install -e "python[all]"
|
||
</code></pre>
|
||
<h4 id="2-prepare-weights"><a class="header" href="#2-prepare-weights">2. Prepare Weights</a></h4>
|
||
<p>You need both GPU weights and CPU weights for heterogeneous inference:</p>
|
||
<p><strong>GPU Weights:</strong> Use the original / quantized model weights.</p>
|
||
<p><strong>CPU Weights:</strong> Quantize to AMX-optimized format using the conversion script:</p>
|
||
<pre><code class="language-bash">python scripts/convert_cpu_weights.py \
|
||
--input-path /path/to/model \
|
||
--input-type bf16 \ # Depends on your GPU weights type: fp8, fp16, or bf16
|
||
--output /path/to/cpu-weights \
|
||
--quant-method int8 # or int4
|
||
</code></pre>
|
||
<p><strong>Supported input formats:</strong> FP8, FP16, BF16 → INT4/INT8.</p>
|
||
<p>For more details, see:</p>
|
||
<ul>
|
||
<li><a href="en/kt-kernel_intro.html#cpu-weights-for-cold-experts-on-cpu">CPU Weights conversion</a></li>
|
||
<li><a href="en/kt-kernel_intro.html#gpu-weights-for-hot-experts-on-gpu">GPU Weights quantization</a></li>
|
||
</ul>
|
||
<p><strong>Note:</strong> LLAMAFILE backend supports GGUF format directly, but this feature is still in preview.</p>
|
||
<h4 id="3-launch-sglang-server"><a class="header" href="#3-launch-sglang-server">3. Launch SGLang Server</a></h4>
|
||
<p>Start the SGLang server with your normal SGLang parameters, and add the following KT-Kernel specific parameters to enable CPU-GPU heterogeneous inference:</p>
|
||
<p><strong>KT-Kernel Parameters to Add:</strong></p>
|
||
<ul>
|
||
<li><code>--kt-method</code>: Backend method (AMXINT4, AMXINT8, or LLAMAFILE)</li>
|
||
<li><code>--kt-weight-path</code>: Path to the converted CPU weights</li>
|
||
<li><code>--kt-cpuinfer</code>: Number of CPU inference threads (set to physical cores)</li>
|
||
<li><code>--kt-threadpool-count</code>: Number of thread pools (set to NUMA node count)</li>
|
||
<li><code>--kt-num-gpu-experts</code>: Number of experts to keep on GPU</li>
|
||
<li><code>--kt-max-deferred-experts-per-token</code>: Deferred experts for pipelined execution</li>
|
||
</ul>
|
||
<p>Example:</p>
|
||
<pre><code class="language-bash">python -m sglang.launch_server \
|
||
[your normal SGLang parameters...] \
|
||
--kt-method AMXINT8 \
|
||
--kt-weight-path /path/to/cpu-weights \
|
||
--kt-cpuinfer 64 \
|
||
--kt-threadpool-count 2 \
|
||
--kt-num-gpu-experts 32 \
|
||
--kt-max-deferred-experts-per-token 2
|
||
</code></pre>
|
||
<p>See <a href="en/kt-kernel_intro.html#kt-kernel-parameters">KT-Kernel Parameters</a> section below for detailed parameter tuning guidelines.</p>
|
||
<h3 id="complete-example-qwen3-30b-a3b"><a class="header" href="#complete-example-qwen3-30b-a3b">Complete Example: Qwen3-30B-A3B</a></h3>
|
||
<p>This example demonstrates the full workflow from downloading weights to launching the server.</p>
|
||
<p><strong>Hardware Configuration:</strong></p>
|
||
<ul>
|
||
<li><strong>GPU</strong>: NVIDIA RTX 4090 24GB</li>
|
||
<li><strong>CPU</strong>: 2x Intel Xeon Gold 6454S (64 physical cores total, 128 threads, 2 NUMA nodes)</li>
|
||
<li><strong>Model</strong>: <a href="https://huggingface.co/Qwen/Qwen3-30B-A3B">Qwen3-30B-A3B</a></li>
|
||
<li><strong>GPU Weights</strong>: BF16 original weights</li>
|
||
<li><strong>CPU Weights</strong>: AMXINT8 quantized</li>
|
||
</ul>
|
||
<p><strong>How to verify your system configuration:</strong></p>
|
||
<pre><code class="language-bash"># Check CPU configuration
|
||
lscpu | grep -E "^CPU\(s\)|Thread\(s\) per core|Socket\(s\)|NUMA node\(s\)"
|
||
# Expected output example:
|
||
CPU(s): 128
|
||
Thread(s) per core: 2
|
||
Socket(s): 2
|
||
NUMA node(s): 2
|
||
# → Physical cores = CPU(s) / Thread(s) per core = 128 / 2 = 64
|
||
</code></pre>
|
||
<p><strong>Parameter Rationale:</strong></p>
|
||
<ul>
|
||
<li><code>--kt-cpuinfer 64</code>: Set to physical cores (64), not hyperthreads (128)</li>
|
||
<li><code>--kt-threadpool-count 2</code>: 2 NUMA nodes detected (dual-socket system)</li>
|
||
<li><code>--kt-num-gpu-experts 32</code>: With 24GB GPU memory, we can fit ~32 experts on GPU for this model (varies by model architecture and actual memory usage)</li>
|
||
<li><code>--kt-max-deferred-experts-per-token 2</code>: Enable pipelined execution - allows CPU to process next batch while GPU completes current batch</li>
|
||
</ul>
|
||
<h4 id="step-1-download-model-weights"><a class="header" href="#step-1-download-model-weights">Step 1: Download model weights</a></h4>
|
||
<pre><code class="language-bash"># Install huggingface-cli if not already installed
|
||
pip install huggingface-hub
|
||
|
||
# Download model from Hugging Face
|
||
hf download Qwen/Qwen3-30B-A3B --local-dir /mnt/data/models/Qwen3-30B-A3B
|
||
</code></pre>
|
||
<h4 id="step-2-convert-to-cpu-weights-amxint8"><a class="header" href="#step-2-convert-to-cpu-weights-amxint8">Step 2: Convert to CPU weights (AMXINT8)</a></h4>
|
||
<pre><code class="language-bash">python scripts/convert_cpu_weights.py \
|
||
--input-path /mnt/data/models/Qwen3-30B-A3B \
|
||
--input-type bf16 \
|
||
--output /mnt/data/models/Qwen3-30B-A3B-INT8 \
|
||
--quant-method int8
|
||
</code></pre>
|
||
<h4 id="step-3-launch-sglang-server"><a class="header" href="#step-3-launch-sglang-server">Step 3: Launch SGLang server</a></h4>
|
||
<pre><code class="language-bash">python -m sglang.launch_server \
|
||
--host 0.0.0.0 \
|
||
--port 8000 \
|
||
--model /mnt/data/models/Qwen3-30B-A3B \
|
||
--trust-remote-code \
|
||
--mem-fraction-static 0.92 \
|
||
--chunked-prefill-size 4096 \
|
||
--served-model-name Qwen3-30B-A3B \
|
||
--enable-mixed-chunk \
|
||
--kt-method AMXINT8 \
|
||
--kt-weight-path /mnt/data/models/Qwen3-30B-A3B-INT8 \
|
||
--kt-cpuinfer 64 \
|
||
--kt-threadpool-count 2 \
|
||
--kt-num-gpu-experts 32 \
|
||
--kt-max-deferred-experts-per-token 2
|
||
</code></pre>
|
||
<h3 id="kt-kernel-parameters"><a class="header" href="#kt-kernel-parameters">KT-Kernel Parameters</a></h3>
|
||
<div class="table-wrapper"><table><thead><tr><th>Parameter</th><th>Description</th><th>Example Value</th></tr></thead><tbody>
|
||
<tr><td><code>--kt-method</code></td><td>CPU inference backend method</td><td><code>AMXINT4</code>, <code>AMXINT8</code>, or <code>LLAMAFILE</code> (preview)</td></tr>
|
||
<tr><td><code>--kt-weight-path</code></td><td>Path to quantized CPU weights</td><td><code>/path/to/cpu-weights</code></td></tr>
|
||
<tr><td><code>--kt-cpuinfer</code></td><td>Number of CPU inference threads</td><td><code>64</code> (adjust based on CPU cores)</td></tr>
|
||
<tr><td><code>--kt-threadpool-count</code></td><td>Number of thread pools for parallel execution</td><td><code>2</code> (typically 1-4)</td></tr>
|
||
<tr><td><code>--kt-num-gpu-experts</code></td><td>Number of experts to keep on GPU</td><td><code>32</code> (remaining experts go to CPU)</td></tr>
|
||
<tr><td><code>--kt-max-deferred-experts-per-token</code></td><td>Number of experts per token to defer for pipelined execution</td><td><code>2</code> (0 to disable, 1-2 recommended)</td></tr>
|
||
</tbody></table>
|
||
</div>
|
||
<p><strong>Parameter Guidelines:</strong></p>
|
||
<ul>
|
||
<li>
|
||
<p><strong><code>kt-method</code></strong>: Choose based on your CPU and weight format:</p>
|
||
<ul>
|
||
<li><code>AMXINT4</code>: Best performance on AMX CPUs with INT4 quantized weights (May cause huge accuracy drop for some models, e.g., Qwen3-30B-A3B)</li>
|
||
<li><code>AMXINT8</code>: Higher accuracy with INT8 quantized weights on AMX CPUs</li>
|
||
<li><code>LLAMAFILE</code>: Preview support for GGUF format (not fully complete)</li>
|
||
</ul>
|
||
</li>
|
||
<li>
|
||
<p><strong><code>kt-cpuinfer</code></strong>: Set to the number of <strong>physical CPU cores</strong> (not hyperthreads).</p>
|
||
<ul>
|
||
<li>Check physical cores: <code>lscpu | grep -E "^CPU\(s\)|Thread\(s\) per core"</code></li>
|
||
<li>Physical cores = CPU(s) / Thread(s) per core</li>
|
||
<li>Example: If CPU(s)=128 and Thread(s) per core=2, then physical cores = 64</li>
|
||
<li><strong>Important</strong>: Do NOT set to hyperthread count - this will degrade performance</li>
|
||
</ul>
|
||
</li>
|
||
<li>
|
||
<p><strong><code>kt-threadpool-count</code></strong>: Set to the number of <strong>NUMA nodes</strong>.</p>
|
||
<ul>
|
||
<li>Check NUMA count: <code>lscpu | grep "NUMA node(s)"</code></li>
|
||
<li>Or use: <code>numactl --hardware | grep "available"</code></li>
|
||
<li><strong>Note</strong>: NUMA node count is NOT necessarily the number of physical CPUs
|
||
<ul>
|
||
<li>It represents memory domains, which may be divided within a single CPU or across multiple CPUs</li>
|
||
<li>Use the NUMA node count from <code>lscpu</code>, regardless of physical CPU count</li>
|
||
</ul>
|
||
</li>
|
||
<li>Typical values: 1-2 for single-socket, 2-4 for dual-socket systems</li>
|
||
<li>This enables better memory bandwidth utilization across NUMA domains</li>
|
||
</ul>
|
||
</li>
|
||
<li>
|
||
<p><strong><code>kt-num-gpu-experts</code></strong>: Determine based on GPU memory and profiling:</p>
|
||
<ul>
|
||
<li>More GPU experts = lower latency but higher GPU memory usage (May cause OOM)</li>
|
||
</ul>
|
||
</li>
|
||
<li>
|
||
<p><strong><code>kt-max-deferred-experts-per-token</code></strong>: Enables pipelined execution:</p>
|
||
<ul>
|
||
<li><code>0</code>: Synchronous execution (simpler, higher latency)</li>
|
||
<li><code>1-2</code>: Deferred execution (better latency, requires tuning) - recommended</li>
|
||
<li><code>3-4</code>: Higher deferred count (possible but rarely beneficial)</li>
|
||
</ul>
|
||
</li>
|
||
</ul>
|
||
<h2 id="direct-python-api-usage"><a class="header" href="#direct-python-api-usage">Direct Python API Usage</a></h2>
|
||
<p>For standalone usage without SGLang, you can use KT-Kernel directly via Python API:</p>
|
||
<pre><code class="language-python">from kt_kernel import KTMoEWrapper
|
||
|
||
# Initialize the MoE wrapper
|
||
wrapper = KTMoEWrapper(
|
||
layer_idx=0,
|
||
num_experts=8,
|
||
num_experts_per_tok=2,
|
||
hidden_size=4096,
|
||
moe_intermediate_size=14336,
|
||
num_gpu_experts=2,
|
||
cpuinfer_threads=32,
|
||
threadpool_count=2,
|
||
weight_path="/path/to/weights",
|
||
chunked_prefill_size=512,
|
||
method="AMXINT4" # Options: "AMXINT4", "AMXINT8", "LLAMAFILE" (preview)
|
||
)
|
||
|
||
# Load weights (from disk - pre-quantized)
|
||
wrapper.load_weights(physical_to_logical_map)
|
||
|
||
# Or load weights from tensors (online quantization)
|
||
wrapper.load_weights_from_tensors(gate_proj, up_proj, down_proj, physical_to_logical_map)
|
||
|
||
# Run inference
|
||
output = wrapper.forward(hidden_states, topk_ids, topk_weights, cuda_stream)
|
||
|
||
# Or use async API for better performance
|
||
wrapper.submit_forward(hidden_states, topk_ids, topk_weights, cuda_stream)
|
||
# ... do other work ...
|
||
output = wrapper.sync_forward(hidden_states, cuda_stream)
|
||
</code></pre>
|
||
<h3 id="advanced-options"><a class="header" href="#advanced-options">Advanced Options</a></h3>
|
||
<pre><code class="language-python"># Initialize with additional options
|
||
wrapper = KTMoEWrapper(
|
||
layer_idx=0,
|
||
num_experts=8,
|
||
num_experts_per_tok=2,
|
||
hidden_size=4096,
|
||
moe_intermediate_size=14336,
|
||
num_gpu_experts=2,
|
||
cpuinfer_threads=32,
|
||
threadpool_count=2,
|
||
weight_path="/path/to/weights",
|
||
chunked_prefill_size=512,
|
||
method="AMXINT4",
|
||
cpu_save=False, # Keep weights in CPU memory after loading
|
||
max_deferred_experts_per_token=0 # Number of experts to defer (for pipelined execution)
|
||
)
|
||
|
||
# Pre-allocate buffers for specific batch sizes (improves performance)
|
||
KTMoEWrapper.set_capture_batch_sizes([1, 2, 4, 8, 16])
|
||
|
||
# Query captured batch sizes
|
||
batch_sizes = KTMoEWrapper.get_capture_batch_sizes()
|
||
|
||
# Clear buffer cache to free memory
|
||
KTMoEWrapper.clear_buffer_cache()
|
||
</code></pre>
|
||
<h2 id="build-configuration"><a class="header" href="#build-configuration">Build Configuration</a></h2>
|
||
<h3 id="manual-installation"><a class="header" href="#manual-installation">Manual Installation</a></h3>
|
||
<p>If you prefer manual installation without the <code>install.sh</code> script, follow these steps:</p>
|
||
<h4 id="1-install-system-dependencies"><a class="header" href="#1-install-system-dependencies">1. Install System Dependencies</a></h4>
|
||
<p><strong>Prerequisites:</strong></p>
|
||
<ul>
|
||
<li><code>cmake</code> (recommended: <code>conda install -y cmake</code>)</li>
|
||
<li><code>libhwloc-dev</code> and <code>pkg-config</code></li>
|
||
</ul>
|
||
<h4 id="2-set-build-configuration"><a class="header" href="#2-set-build-configuration">2. Set Build Configuration</a></h4>
|
||
<p><strong>Core Options:</strong></p>
|
||
<div class="table-wrapper"><table><thead><tr><th>Variable</th><th>Options</th><th>Description</th></tr></thead><tbody>
|
||
<tr><td><code>CPUINFER_CPU_INSTRUCT</code></td><td><code>NATIVE</code>, <code>AVX512</code>, <code>AVX2</code>, <code>FANCY</code></td><td>CPU instruction set to use</td></tr>
|
||
<tr><td><code>CPUINFER_ENABLE_AMX</code></td><td><code>ON</code>, <code>OFF</code></td><td>Enable Intel AMX support</td></tr>
|
||
<tr><td><code>CPUINFER_BUILD_TYPE</code></td><td><code>Release</code>, <code>Debug</code>, <code>RelWithDebInfo</code></td><td>Build type (default: <code>Release</code>)</td></tr>
|
||
<tr><td><code>CPUINFER_PARALLEL</code></td><td>Number</td><td>Parallel build jobs (default: auto-detect)</td></tr>
|
||
<tr><td><code>CPUINFER_VERBOSE</code></td><td><code>0</code>, <code>1</code></td><td>Verbose build output (default: <code>0</code>)</td></tr>
|
||
</tbody></table>
|
||
</div>
|
||
<p><strong>Instruction Set Details:</strong></p>
|
||
<ul>
|
||
<li><strong><code>NATIVE</code></strong>: Auto-detect and use all available CPU instructions (<code>-march=native</code>) - <strong>Recommended for best performance</strong></li>
|
||
<li><strong><code>AVX512</code></strong>: Explicit AVX512 support for Skylake-SP and Cascade Lake</li>
|
||
<li><strong><code>AVX2</code></strong>: AVX2 support for maximum compatibility</li>
|
||
<li><strong><code>FANCY</code></strong>: AVX512 with full extensions (AVX512F/BW/DQ/VL/VNNI) for Ice Lake+ and Zen 4+. Use this when building pre-compiled binaries to distribute to users with modern CPUs. For local builds, prefer <code>NATIVE</code> for better performance.</li>
|
||
</ul>
|
||
<p><strong>Example Configurations:</strong></p>
|
||
<pre><code class="language-bash"># Maximum performance on AMX CPU
|
||
export CPUINFER_CPU_INSTRUCT=NATIVE
|
||
export CPUINFER_ENABLE_AMX=ON
|
||
|
||
# AVX512 CPU without AMX
|
||
export CPUINFER_CPU_INSTRUCT=AVX512
|
||
export CPUINFER_ENABLE_AMX=OFF
|
||
|
||
# Compatibility build
|
||
export CPUINFER_CPU_INSTRUCT=AVX2
|
||
export CPUINFER_ENABLE_AMX=OFF
|
||
|
||
# Debug build for development
|
||
export CPUINFER_BUILD_TYPE=Debug
|
||
export CPUINFER_VERBOSE=1
|
||
</code></pre>
|
||
<h4 id="3-build-and-install"><a class="header" href="#3-build-and-install">3. Build and Install</a></h4>
|
||
<pre><code class="language-bash"># Editable installation (for development)
|
||
pip install -e .
|
||
|
||
# Standard installation
|
||
pip install .
|
||
</code></pre>
|
||
<h2 id="error-troubleshooting"><a class="header" href="#error-troubleshooting">Error Troubleshooting</a></h2>
|
||
<h3 id="cuda-not-found"><a class="header" href="#cuda-not-found">CUDA Not Found</a></h3>
|
||
<pre><code> -- Looking for a CUDA compiler - NOTFOUND
|
||
CMake Error at CMakeLists.txt:389 (message):
|
||
KTRANSFORMERS_USE_CUDA=ON but CUDA compiler not found
|
||
</code></pre>
|
||
<p>Make sure you have the CUDA toolkit installed and <code>nvcc</code> is in your system PATH.</p>
|
||
<p>Try <code>export CMAKE_ARGS="-D CMAKE_CUDA_COMPILER=$(which nvcc)"</code> and reinstall again.</p>
|
||
<h3 id="hwloc-not-found"><a class="header" href="#hwloc-not-found">hwloc Not Found</a></h3>
|
||
<p>Run <code>sudo apt install libhwloc-dev</code> if on a Debian-based system or build from source: https://www.open-mpi.org/projects/hwloc/.</p>
|
||
<pre><code>wget https://download.open-mpi.org/release/hwloc/v2.12/hwloc-2.12.2.tar.gz
|
||
tar -xzf hwloc-2.12.2.tar.gz
|
||
cd hwloc-2.12.2
|
||
./configure
|
||
make
|
||
sudo make install
|
||
</code></pre>
|
||
<h2 id="weight-quantization"><a class="header" href="#weight-quantization">Weight Quantization</a></h2>
|
||
<p>KT-Kernel provides weight quantization tools for CPU-GPU hybrid inference (e.g., integrating with SGLang). Both tools work together to enable heterogeneous expert placement across CPUs and GPUs.</p>
|
||
<h3 id="cpu-weights-for-cold-experts-on-cpu"><a class="header" href="#cpu-weights-for-cold-experts-on-cpu">CPU Weights (for "cold" experts on CPU)</a></h3>
|
||
<p>Quantize weights to INT4/INT8 format optimized for AMX inference:</p>
|
||
<pre><code class="language-bash">python scripts/convert_cpu_weights.py \
|
||
--input-path /path/to/model \
|
||
--input-type bf16 \
|
||
--output /path/to/output \
|
||
--quant-method int4
|
||
</code></pre>
|
||
<p><strong>Supported formats:</strong> FP8, FP16, BF16 → INT4/INT8</p>
|
||
<h3 id="gpu-weights-for-hot-experts-on-gpu"><a class="header" href="#gpu-weights-for-hot-experts-on-gpu">GPU Weights (for "hot" experts on GPU)</a></h3>
|
||
<p>Apply GPTQ quantization to model weights:</p>
|
||
<pre><code class="language-bash"># Install additional dependencies first
|
||
pip install accelerate transformers llmcompressor datasets
|
||
|
||
# Quantize GPU weights
|
||
python scripts/convert_gpu_weights.py \
|
||
--model_id /path/to/model \
|
||
--output_dir /path/to/output \
|
||
--quant_type W4A16
|
||
</code></pre>
|
||
<p><strong>Supported types:</strong> W4A16 (GPTQ4), W8A16 (GPTQ8)</p>
|
||
<hr />
|
||
<p>For detailed documentation, advanced options, and low-memory mode, see <a href="en/scripts/README.html">scripts/README.md</a>.</p>
|
||
<h2 id="before-commit"><a class="header" href="#before-commit">Before Commit!</a></h2>
|
||
<p>your msg should match: Conventional Commits (https://www.conventionalcommits.org/) <br>and format your code before commit:</p>
|
||
<pre><code class="language-shell">cmake -B build
|
||
cd build
|
||
make format
|
||
</code></pre>
|
||
<p>and you may need a new clang-format at least 18, use this command in conda env:</p>
|
||
<pre><code class="language-shell">conda install -c conda-forge clang-format=18
|
||
rm -rf build
|
||
</code></pre>
|
||
<p>and you may need black for python format:</p>
|
||
<pre><code class="language-shell">conda install black
|
||
</code></pre>
|
||
<div style="break-before: page; page-break-before: always;"></div><ul>
|
||
<li>
|
||
<p><a href="en/KTransformers-Fine-Tuning_User-Guide.html#ktransformers-fine-tuning-x-llama-factory-integration-%E2%80%93-user-guide">KTransformers Fine-Tuning × LLaMA-Factory Integration – User Guide</a></p>
|
||
</li>
|
||
<li>
|
||
<p><a href="en/KTransformers-Fine-Tuning_User-Guide.html#introduction">Introduction</a></p>
|
||
</li>
|
||
<li>
|
||
<p><a href="en/KTransformers-Fine-Tuning_User-Guide.html#fine-tuning-results-examples">Fine-Tuning Results (Examples)</a></p>
|
||
<ul>
|
||
<li><a href="en/KTransformers-Fine-Tuning_User-Guide.html#stylized-dialogue-catgirl-tone">Stylized Dialogue (CatGirl tone)</a></li>
|
||
<li><a href="en/KTransformers-Fine-Tuning_User-Guide.html#benchmarks">Benchmarks</a>
|
||
<ul>
|
||
<li><a href="en/KTransformers-Fine-Tuning_User-Guide.html#translational-style-dataset">Translational-Style dataset</a></li>
|
||
<li><a href="en/KTransformers-Fine-Tuning_User-Guide.html#afrimed-qa-short-answer">AfriMed-QA (short answer)</a></li>
|
||
<li><a href="en/KTransformers-Fine-Tuning_User-Guide.html#afrimed-qa-multiple-choice">AfriMed-QA (multiple choice)</a></li>
|
||
</ul>
|
||
</li>
|
||
</ul>
|
||
</li>
|
||
<li>
|
||
<p><a href="en/KTransformers-Fine-Tuning_User-Guide.html#quick-to-start">Quick to Start</a></p>
|
||
<ul>
|
||
<li><a href="en/KTransformers-Fine-Tuning_User-Guide.html#environment-setup">Environment Setup</a></li>
|
||
<li><a href="en/KTransformers-Fine-Tuning_User-Guide.html#core-feature-1-use-ktransformers-backend-to-fine-tune-ultra-large-moe-models">Core Feature 1: Use KTransformers backend to fine-tune ultra-large MoE models</a></li>
|
||
<li><a href="en/KTransformers-Fine-Tuning_User-Guide.html#core-feature-2-chat-with-the-fine-tuned-model-base--lora-adapter">Core Feature 2: Chat with the fine-tuned model (base + LoRA adapter)</a></li>
|
||
<li><a href="en/KTransformers-Fine-Tuning_User-Guide.html#core-feature-3-batch-inference--metrics-base--lora-adapter">Core Feature 3: Batch inference + metrics (base + LoRA adapter)</a></li>
|
||
</ul>
|
||
</li>
|
||
<li>
|
||
<p><a href="en/KTransformers-Fine-Tuning_User-Guide.html#kt-fine-tuning-speed-user-side-view">KT Fine-Tuning Speed (User-Side View)</a></p>
|
||
<ul>
|
||
<li><a href="en/KTransformers-Fine-Tuning_User-Guide.html#end-to-end-performance">End-to-End Performance</a></li>
|
||
<li><a href="en/KTransformers-Fine-Tuning_User-Guide.html#gpucpu-memory-footprint">GPU/CPU Memory Footprint</a></li>
|
||
</ul>
|
||
</li>
|
||
<li>
|
||
<p><a href="en/KTransformers-Fine-Tuning_User-Guide.html#conclusion">Conclusion</a></p>
|
||
</li>
|
||
</ul>
|
||
<h1 id="ktransformers-fine-tuning--llama-factory-integration--user-guide"><a class="header" href="#ktransformers-fine-tuning--llama-factory-integration--user-guide">KTransformers Fine-Tuning × LLaMA-Factory Integration – User Guide</a></h1>
|
||
<p><strong>MadSys Lab, KVCache-AI Team, Approaching AI, LLaMA-Factory Team</strong></p>
|
||
<h2 id="introduction"><a class="header" href="#introduction">Introduction</a></h2>
|
||
<p>From <strong>DeepSeek-V3/R1</strong> to <strong>Qwen3-MoE</strong> and <strong>Kimi-K2</strong>, each wave of open-sourced large models brings leaps in performance and scale. However, many researchers and developers are constrained by expensive GPUs and models with tens or even hundreds of billions of parameters, making it <strong>hard to fine-tune very large models under limited resources</strong>. To bridge this gap, we propose a practical approach: combining <strong>KTransformers</strong> with <strong>LLaMA-Factory</strong>. With just <strong>2–4 RTX 4090s</strong> and a high-memory CPU, you can fine-tune ultra-large MoE models like DeepSeek-671B.</p>
|
||
<p>Our goal is to give resource-constrained researchers a <strong>local path to explore fine-tuning ultra-large models</strong>, and also a fast way to customize smaller models (e.g., 14B/30B) for specific scenarios. We validate the setup using <strong>stylized dialogue</strong>, <strong>Westernized translation tone</strong>, and <strong>medical Q&A</strong> as representative tasks, showing that <strong>personalized adaptation can be achieved within hours</strong>.</p>
|
||
<p>As shown below, LLaMA-Factory is the unified orchestration/configuration layer for the whole fine-tuning workflow—handling data, training scheduling, LoRA injection, and inference interfaces. <strong>KTransformers</strong> acts as a pluggable high-performance backend that takes over core operators like Attention/MoE under the same training configs, enabling efficient <strong>GPU+CPU heterogeneous cooperation</strong>.</p>
|
||
<p><img src="en/../assets/image-20251011010558909.png" alt="image-20251011010558909" /></p>
|
||
<p>Within LLaMA-Factory, we compared LoRA fine-tuning with <strong>HuggingFace</strong>, <strong>Unsloth</strong>, and <strong>KTransformers</strong> backends. KTransformers is the <strong>only workable 4090-class solution</strong> for ultra-large MoE models (e.g., 671B) and also delivers higher throughput and lower GPU memory on smaller MoE models (e.g., DeepSeek-14B).</p>
|
||
<div class="table-wrapper"><table><thead><tr><th>Under LoRA (BF16) + <a href="https://github.com/mindsRiverPonder/LLM-practice">NekoQA-10K stylized dialogue</a></th><th>HuggingFace Backend</th><th>Unsloth Backend</th><th>KTransformers Backend</th></tr></thead><tbody>
|
||
<tr><td>[14B-DeepSeekV2-Lite] LoRA fine-tuning throughput</td><td>303.58 token/s</td><td>455.37 token/s</td><td>530.38 token/s</td></tr>
|
||
<tr><td>[14B-DeepSeekV2-Lite] GPU memory</td><td>32.12 GB</td><td>9.64 GB</td><td>6.08 GB</td></tr>
|
||
<tr><td>[671B-DeepSeekV3] LoRA fine-tuning throughput</td><td><font color='red'>Too Huge to run</font></td><td><font color='red'>NOT SUPPORT</font></td><td>40.35 token/s</td></tr>
|
||
<tr><td>[671B-DeepSeekV3] GPU memory (sum across GPUs)</td><td>theoretical 1400 GB †</td><td><font color='red'>NOT SUPPORT</font></td><td>70 GB †</td></tr>
|
||
</tbody></table>
|
||
</div>
|
||
<p>† <strong>1400 GB</strong> is a <strong>theoretical</strong> FP16 full-parameter resident footprint (not runnable). <strong>70 GB</strong> is the <strong>measured peak</strong> with KT strategy (Attention on GPU + layered MoE offload).</p>
|
||
<p><img src="en/../assets/image-compare_model.png" alt="按照模型划分的对比图_02" /></p>
|
||
<h3 id="fine-tuning-results-examples"><a class="header" href="#fine-tuning-results-examples">Fine-Tuning Results (Examples)</a></h3>
|
||
<h4 id="stylized-dialogue-catgirl-tone"><a class="header" href="#stylized-dialogue-catgirl-tone">Stylized Dialogue (CatGirl tone)</a></h4>
|
||
<p>Dataset: <a href="https://zhuanlan.zhihu.com/p/1934983798233231689">NekoQA-10K</a>. Goal: improve style consistency and recognizability.</p>
|
||
<p>The figure compares responses from the base vs. fine-tuned models. The fine-tuned model maintains the target tone and address terms more consistently (red boxes), validating the effectiveness of <strong>style-transfer fine-tuning</strong>.</p>
|
||
<p><img src="en/../assets/image-20251016175046882.png" alt="image-20251016175046882" /></p>
|
||
<h4 id="benchmarks"><a class="header" href="#benchmarks">Benchmarks</a></h4>
|
||
<p>We use:</p>
|
||
<p>(1) <a href="https://github.com/Benson114/Translational-Style-ChatLLM">Translational-Style-ChatLLM</a>, which asks for an exaggerated, Westernized translation tone—clear, stylized customization.</p>
|
||
<p>(2) <a href="https://aclanthology.org/2025.acl-long.96/">AfriMed-QA</a> (ACL 2025), a medical dataset for African contexts with strong domain specificity, including multiple-choice and short-answer sub-tasks—well-suited for vertical fine-tuning evaluation.</p>
|
||
<p>The tables show metrics before vs. after LoRA fine-tuning. We observe <strong>large improvements</strong> across metrics, verifying fine-tuning effectiveness:</p>
|
||
<div class="table-wrapper"><table><thead><tr><th>Translational-Style dataset</th><th>BLEU-1</th><th>BLEU-2</th><th>BLEU-3</th><th>BLEU-4</th><th>ROUGE-1</th><th>ROUGE-2</th><th>ROUGE-L</th></tr></thead><tbody>
|
||
<tr><td>V2-Lite (no LoRA)</td><td>20.66</td><td>8.33</td><td>4.54</td><td>2.89</td><td>22.71</td><td>4.52</td><td>19.19</td></tr>
|
||
<tr><td><strong>KT-LoRA fine-tuned V2-Lite</strong></td><td><strong>35.41</strong></td><td><strong>22.44</strong></td><td><strong>15.42</strong></td><td><strong>11.18</strong></td><td><strong>42.03</strong></td><td><strong>18.38</strong></td><td><strong>33.10</strong></td></tr>
|
||
<tr><td>V3 base (no LoRA)</td><td>8.49</td><td>3.34</td><td>1.62</td><td>0.96</td><td>15.91</td><td>2.55</td><td>10.07</td></tr>
|
||
<tr><td><strong>KT-LoRA fine-tuned V3</strong></td><td><strong>37.02</strong></td><td><strong>23.70</strong></td><td><strong>16.21</strong></td><td><strong>11.49</strong></td><td><strong>43.43</strong></td><td><strong>18.96</strong></td><td><strong>34.54</strong></td></tr>
|
||
</tbody></table>
|
||
</div><div class="table-wrapper"><table><thead><tr><th>AfriMed-QA (short answer)</th><th>BLEU-1</th><th>BLEU-2</th><th>BLEU-3</th><th>BLEU-4</th><th>ROUGE-1</th><th>ROUGE-2</th><th>ROUGE-L</th></tr></thead><tbody>
|
||
<tr><td>V2-Lite (no LoRA)</td><td>13.58</td><td>11.12</td><td>9.10</td><td>7.23</td><td>22.48</td><td>7.81</td><td>11.73</td></tr>
|
||
<tr><td><strong>KT-LoRA fine-tuned V2-Lite</strong></td><td><strong>35.90</strong></td><td><strong>27.63</strong></td><td><strong>22.99</strong></td><td><strong>19.15</strong></td><td><strong>35.25</strong></td><td><strong>17.50</strong></td><td><strong>28.44</strong></td></tr>
|
||
<tr><td>V3 base (no LoRA)</td><td>12.75</td><td>10.27</td><td>8.05</td><td>5.99</td><td>20.33</td><td>5.65</td><td>10.11</td></tr>
|
||
<tr><td><strong>KT-LoRA fine-tuned V3</strong></td><td><strong>42.42</strong></td><td><strong>34.12</strong></td><td><strong>28.95</strong></td><td><strong>24.54</strong></td><td><strong>41.97</strong></td><td><strong>22.37</strong></td><td><strong>33.28</strong></td></tr>
|
||
</tbody></table>
|
||
</div><div class="table-wrapper"><table><thead><tr><th>AfriMed-QA (multiple choice)</th><th>Accuracy</th></tr></thead><tbody>
|
||
<tr><td>V2-Lite (no LoRA)</td><td>0.0645</td></tr>
|
||
<tr><td><strong>KT-LoRA fine-tuned V2-Lite</strong></td><td><strong>0.4812</strong></td></tr>
|
||
<tr><td>V3 base (no LoRA)</td><td>0.5833</td></tr>
|
||
<tr><td><strong>KT-LoRA fine-tuned V3</strong></td><td><strong>0.7930</strong></td></tr>
|
||
</tbody></table>
|
||
</div>
|
||
<p>Even for ultra-large MoE models, <strong>KTransformers-backed fine-tuning</strong> achieves strong task performance quickly.</p>
|
||
<h2 id="quick-to-start"><a class="header" href="#quick-to-start">Quick to Start</a></h2>
|
||
<p>This section shows how to install and use <strong>LLaMA-Factory + KTransformers</strong> for fine-tuning and inference:</p>
|
||
<ul>
|
||
<li>Environment setup</li>
|
||
<li>Fine-tune ultra-large MoE models with KTransformers backend</li>
|
||
<li>Load the fine-tuned model (base + LoRA adapter) for chat/inference</li>
|
||
<li>Batch inference and metric evaluation</li>
|
||
</ul>
|
||
<h3 id="environment-setup"><a class="header" href="#environment-setup">Environment Setup</a></h3>
|
||
<p>According to the following example, install both the <strong>KTransformers</strong> and <strong>LLaMA-Factory</strong> environments simultaneously.
|
||
This time, to simplify the installation process of KTransformers, we have specially packaged a wheel file to avoid local compilation.
|
||
The detailed installation steps are as follows:
|
||
(Note: Make sure your local <strong>Python version</strong>, <strong>Torch version</strong>, <strong>CUDA version</strong>, and the <strong>KTransformers wheel filename</strong> correspond correctly.)</p>
|
||
<pre><code class="language-shell"># 1. Create a conda environment
|
||
conda create -n Kllama python=3.12 # choose from : [3.10, 3.11, 3.12, 3.13]
|
||
conda install -y -c conda-forge libstdcxx-ng gcc_impl_linux-64
|
||
conda install -y -c nvidia/label/cuda-11.8.0 cuda-runtime
|
||
|
||
# 2. Install the LLaMA-Factory environment
|
||
git clone --depth 1 https://github.com/hiyouga/LLaMA-Factory.git
|
||
cd LLaMA-Factory
|
||
pip install -e ".[torch,metrics]" --no-build-isolation
|
||
|
||
# 3. Install the KTransformers wheel that matches your Torch and Python versions, from https://github.com/kvcache-ai/ktransformers/releases/tag/v0.4.1 (Note: The CUDA version can differ from that in the wheel filename.)
|
||
pip install ktransformers-0.4.1+cu128torch27fancy-cp312-cp312-linux_x86_64.whl
|
||
|
||
# 4. Install flash-attention, download the corresponding file based on your Python and Torch versions from: https://github.com/Dao-AILab/flash-attention/releases
|
||
pip install flash_attn-2.8.3+cu12torch2.7cxx11abiTRUE-cp312-cp312-linux_x86_64.whl
|
||
# abi=True/False can find from below
|
||
# import torch
|
||
# print(torch._C._GLIBCXX_USE_CXX11_ABI)
|
||
|
||
# 5. (Optional) If you want to use flash_infer (otherwise it defaults to triton)
|
||
git clone https://github.com/kvcache-ai/custom_flashinfer.git
|
||
pip install custom_flashinfer/
|
||
</code></pre>
|
||
<p><strong>Usage tip:</strong> In LLaMA-Factory YAML, set <code>use_kt: true</code> and pick a <code>kt_optimize_rule</code> file to have KTransformers handle the core compute. The features below show typical configs.</p>
|
||
<h3 id="core-feature-1-use-ktransformers-backend-to-fine-tune-ultra-large-moe-models"><a class="header" href="#core-feature-1-use-ktransformers-backend-to-fine-tune-ultra-large-moe-models">Core Feature 1: Use KTransformers backend to fine-tune ultra-large MoE models</a></h3>
|
||
<p>Run the command: <code>USE_KT=1 llamafactory-cli train examples/train_lora/deepseek3_lora_sft_kt.yaml</code>.</p>
|
||
<p>Note: You <strong>must</strong> provide a <strong>BF16</strong> model. DeepSeek-V3-671B is released in FP8 by default; convert with <a href="https://github.com/deepseek-ai/DeepSeek-V3/blob/main/inference/fp8_cast_bf16.py">DeepSeek-V3/inference/fp8_cast_bf16.py</a>.</p>
|
||
<pre><code class="language-yaml">### model
|
||
model_name_or_path: opensourcerelease/DeepSeek-V3-bf16
|
||
trust_remote_code: true
|
||
|
||
### method
|
||
stage: sft
|
||
do_train: true
|
||
finetuning_type: lora
|
||
lora_rank: 8
|
||
lora_target: all
|
||
|
||
### dataset
|
||
dataset: identity
|
||
template: deepseek
|
||
cutoff_len: 2048
|
||
max_samples: 100000
|
||
overwrite_cache: true
|
||
preprocessing_num_workers: 16
|
||
dataloader_num_workers: 4
|
||
|
||
### output
|
||
output_dir: saves/Kllama_deepseekV3
|
||
logging_steps: 10
|
||
save_steps: 500
|
||
plot_loss: true
|
||
overwrite_output_dir: true
|
||
save_only_model: false
|
||
report_to: none # choices: [none, wandb, tensorboard, swanlab, mlflow]
|
||
|
||
### train
|
||
per_device_train_batch_size: 1
|
||
gradient_accumulation_steps: 8
|
||
learning_rate: 1.0e-4
|
||
num_train_epochs: 3.0
|
||
lr_scheduler_type: cosine
|
||
warmup_ratio: 0.1
|
||
bf16: true
|
||
ddp_timeout: 180000000
|
||
resume_from_checkpoint: null
|
||
|
||
### ktransformers
|
||
use_kt: true # use KTransformers as LoRA sft backend
|
||
kt_optimize_rule: examples/kt_optimize_rules/DeepSeek-V3-Chat-sft-amx-multi-gpu.yaml
|
||
cpu_infer: 32
|
||
chunk_size: 8192
|
||
</code></pre>
|
||
<p><code>kt_optimize_rule</code> controls <strong>placement strategy</strong>. See also <a href="https://github.com/kvcache-ai/ktransformers/tree/main/ktransformers/optimize/optimize_rules">ktransformers/optimize_rules</a>. Naming hints (<code>*</code> = wildcard):</p>
|
||
<div class="table-wrapper"><table><thead><tr><th>Pattern</th><th>Meaning</th></tr></thead><tbody>
|
||
<tr><td>DeepSeek-V2-Lite-Chat-* / DeepSeek-V3-Chat-*</td><td>Target model variants</td></tr>
|
||
<tr><td><em>-sft-</em></td><td>Strategy for fine-tuning; others are for inference</td></tr>
|
||
<tr><td><em>-amx-</em></td><td>Use AMX on CPU; otherwise use <strong>llamafile</strong></td></tr>
|
||
<tr><td><em>-multi-gpu-X</em></td><td>Model parallel on X GPUs (X omitted → default 2 GPUs)</td></tr>
|
||
</tbody></table>
|
||
</div>
|
||
<p>Example: <code>DeepSeek-V3-Chat-sft-amx-multi-gpu.yaml</code> = V3-Chat fine-tuning with AMX and 2-GPU model parallel.</p>
|
||
<p>We recommend <strong>AMX acceleration</strong> where available (<code>lscpu | grep amx</code>). AMX supports BF16/INT8. Example:</p>
|
||
<pre><code class="language-yaml">- match:
|
||
name: "^model\\.layers\\..*\\.mlp\\.experts$"
|
||
replace:
|
||
class: ktransformers.operators.experts.KTransformersExperts # custom MoE Kernel with expert parallelism
|
||
kwargs:
|
||
prefill_device: "cpu"
|
||
prefill_op: "KExpertsTorch"
|
||
generate_device: "cpu"
|
||
generate_op: "KSFTExpertsCPU"
|
||
out_device: "cuda"
|
||
backend: "AMXInt8" # or "AMXBF16" or "llamafile" (default)
|
||
</code></pre>
|
||
<p>Outputs go to <code>output_dir</code> in safetensors format plus adapter metadata for later loading.</p>
|
||
<p><img src="en/../assets/image-20251016171537997.png" alt="image-20251016171537997" /></p>
|
||
<h3 id="core-feature-2-chat-with-the-fine-tuned-model-base--lora-adapter"><a class="header" href="#core-feature-2-chat-with-the-fine-tuned-model-base--lora-adapter">Core Feature 2: Chat with the fine-tuned model (base + LoRA adapter)</a></h3>
|
||
<p>Run the command: <code>llamafactory-cli chat examples/inference/deepseek3_lora_sft_kt.yaml</code>.</p>
|
||
<p>Use the safetensors adapter trained with KT for inference.</p>
|
||
<pre><code class="language-yaml">model_name_or_path: opensourcerelease/DeepSeek-V3-bf16
|
||
adapter_name_or_path: saves/Kllama_deepseekV3
|
||
template: deepseek
|
||
infer_backend: ktransformers # choices: [huggingface, vllm, sglang, ktransformers]
|
||
trust_remote_code: true
|
||
|
||
use_kt: true # use KTransformers as LoRA sft backend to inference
|
||
kt_optimize_rule: examples/kt_optimize_rules/DeepSeek-V3-Chat-sft-amx-multi-gpu.yaml
|
||
cpu_infer: 32
|
||
chunk_size: 8192
|
||
</code></pre>
|
||
<p>We also support <strong>GGUF</strong> adapters: for safetensors, set the <strong>directory</strong>; for GGUF, set the <strong>file path</strong> in <code>adapter_name_or_path</code>.</p>
|
||
<p>During loading, LLaMA-Factory maps layer names to KT’s naming. You’ll see logs like <code>Loaded adapter weight: XXX -> XXX</code>:</p>
|
||
<p><img src="en/../assets/image-20251016171526210.png" alt="image-20251016171526210" /></p>
|
||
<h3 id="core-feature-3-batch-inference--metrics-base--lora-adapter"><a class="header" href="#core-feature-3-batch-inference--metrics-base--lora-adapter">Core Feature 3: Batch inference + metrics (base + LoRA adapter)</a></h3>
|
||
<p>Run the command: <code>API_PORT=8000 llamafactory-cli api examples/inference/deepseek3_lora_sft_kt.yaml</code>.
|
||
Invoke the KT fine-tuned adapter to provide the API; the usage logic of other APIs is consistent with the native LLaMA-Factory approach.</p>
|
||
<pre><code class="language-yaml">model_name_or_path: opensourcerelease/DeepSeek-V3-bf16
|
||
adapter_name_or_path: saves/Kllama_deepseekV3
|
||
template: deepseek
|
||
infer_backend: ktransformers # choices: [huggingface, vllm, sglang, ktransformers]
|
||
trust_remote_code: true
|
||
|
||
use_kt: true # use KTransformers as LoRA sft backend to inference
|
||
kt_optimize_rule: examples/kt_optimize_rules/DeepSeek-V3-Chat-sft-amx-multi-gpu.yaml
|
||
cpu_infer: 32
|
||
chunk_size: 8192
|
||
</code></pre>
|
||
<h2 id="kt-fine-tuning-speed-user-side-view"><a class="header" href="#kt-fine-tuning-speed-user-side-view">KT Fine-Tuning Speed (User-Side View)</a></h2>
|
||
<h3 id="end-to-end-performance"><a class="header" href="#end-to-end-performance">End-to-End Performance</a></h3>
|
||
<p><strong>Definitions</strong></p>
|
||
<ul>
|
||
<li><code>step_time</code>: wall-clock time for a full optimization step (tensor movement + Attention + MoE + other compute).</li>
|
||
<li><code>tokens_per_step = GAS × qlen</code>; <code>token/s = tokens_per_step / step_time</code>.</li>
|
||
</ul>
|
||
<p><strong>Settings:</strong> <code>GAS=16</code>, <code>qlen=512</code> (→ <code>tokens_per_step = 8192</code>); LoRA (<code>r=8, alpha=32, dropout=0.1</code>); <strong>AMX</strong> enabled; GPU: RTX 4090, CPU: Intel Xeon Platinum 8488C.</p>
|
||
<p><strong>Measured</strong></p>
|
||
<ul>
|
||
<li><strong>DeepSeek-V3-671B:</strong> <code>step_time = 203 s</code> → <code>token/s ≈ 8192 / 203 ≈ 40.35</code></li>
|
||
<li><strong>DeepSeek-V2-Lite-14B:</strong> <code>step_time = 36 s</code> → <code>token/s ≈ 8192 / 36 ≈ 227.6</code></li>
|
||
</ul>
|
||
<h3 id="gpucpu-memory-footprint"><a class="header" href="#gpucpu-memory-footprint">GPU/CPU Memory Footprint</a></h3>
|
||
<ul>
|
||
<li>DeepSeek-V3 (671B; 61 layers with 58 MoE): ~<strong>70 GB</strong> total GPU VRAM (multi-GPU), ~<strong>1.2–1.3 TB</strong> CPU RAM.</li>
|
||
<li>DeepSeek-V2-Lite (14B; 27 layers with 26 MoE): ~<strong>5.5 GB</strong> GPU VRAM, ~<strong>30 GB</strong> CPU RAM.</li>
|
||
</ul>
|
||
<h2 id="conclusion"><a class="header" href="#conclusion">Conclusion</a></h2>
|
||
<p>By integrating <strong>KTransformers LoRA fine-tuning</strong> into <strong>LLaMA-Factory</strong>, we provide a practical guide for efficient training and deployment of MoE LLMs. KT brings cutting-edge optimizations (DeepSeek/Qwen/Kimi support with AMX-accelerated kernels), and LoRA enables customization under very low GPU memory. LLaMA-Factory offers a friendly, unified interface.</p>
|
||
<p>This integration (akin to Unsloth-style speedups) means even models with tens to hundreds of billions of parameters can be fine-tuned and deployed with low latency on commodity hardware. You get <strong>memory savings, speed-ups, and usability</strong> together. We encourage you to try LLaMA-Factory + KT for your next MoE project and follow this guide. Feedback is welcome!</p>
|
||
<div style="break-before: page; page-break-before: always;"></div><!-- omit in toc -->
|
||
<h1 id="gpt-4o1-level-local-vscode-copilot-on-a-desktop-with-only-24gb-vram"><a class="header" href="#gpt-4o1-level-local-vscode-copilot-on-a-desktop-with-only-24gb-vram">GPT-4/o1-level Local VSCode Copilot on a Desktop with only 24GB VRAM</a></h1>
|
||
<ul>
|
||
<li><a href="en/DeepseekR1_V3_tutorial.html#summary">SUMMARY</a>
|
||
<ul>
|
||
<li><a href="en/DeepseekR1_V3_tutorial.html#show-case-environment">Show Case Environment</a></li>
|
||
<li><a href="en/DeepseekR1_V3_tutorial.html#bench-result">Bench Result</a>
|
||
<ul>
|
||
<li><a href="en/DeepseekR1_V3_tutorial.html#v021">V0.2.1</a>
|
||
<ul>
|
||
<li><a href="en/DeepseekR1_V3_tutorial.html#memory-consumption">Memory consumption:</a></li>
|
||
<li><a href="en/DeepseekR1_V3_tutorial.html#change-log">Change Log</a></li>
|
||
<li><a href="en/DeepseekR1_V3_tutorial.html#benchmark-results">Benchmark Results</a></li>
|
||
</ul>
|
||
</li>
|
||
<li><a href="en/DeepseekR1_V3_tutorial.html#v02">V0.2</a>
|
||
<ul>
|
||
<li><a href="en/DeepseekR1_V3_tutorial.html#settings">Settings</a></li>
|
||
<li><a href="en/DeepseekR1_V3_tutorial.html#memory-consumption-1">Memory consumption:</a></li>
|
||
<li><a href="en/DeepseekR1_V3_tutorial.html#benchmark-results-1">Benchmark Results</a></li>
|
||
</ul>
|
||
</li>
|
||
<li><a href="en/DeepseekR1_V3_tutorial.html#v03-preview">V0.3-Preview</a>
|
||
<ul>
|
||
<li><a href="en/DeepseekR1_V3_tutorial.html#settings-1">Settings</a></li>
|
||
<li><a href="en/DeepseekR1_V3_tutorial.html#memory-consumptions">Memory consumptions:</a></li>
|
||
<li><a href="en/DeepseekR1_V3_tutorial.html#benchmark-results-2">Benchmark results</a></li>
|
||
</ul>
|
||
</li>
|
||
</ul>
|
||
</li>
|
||
<li><a href="en/DeepseekR1_V3_tutorial.html#how-to-run">How to Run</a>
|
||
<ul>
|
||
<li><a href="en/DeepseekR1_V3_tutorial.html#v022--v023-longer-context--fp8-kernel">v0.2.2 & v0.2.3 longer context & FP8 kernel</a>
|
||
<ul>
|
||
<li><a href="en/DeepseekR1_V3_tutorial.html#longer-context">longer context</a></li>
|
||
<li><a href="en/DeepseekR1_V3_tutorial.html#fp8-kernel">FP8 kernel</a></li>
|
||
</ul>
|
||
</li>
|
||
<li><a href="en/DeepseekR1_V3_tutorial.html#v02--v021-showcase">V0.2 & V0.2.1 Showcase</a>
|
||
<ul>
|
||
<li><a href="en/DeepseekR1_V3_tutorial.html#single-socket-version-32-cores">Single socket version (32 cores)</a></li>
|
||
<li><a href="en/DeepseekR1_V3_tutorial.html#dual-socket-version-64-cores">Dual socket version (64 cores)</a></li>
|
||
</ul>
|
||
</li>
|
||
<li><a href="en/DeepseekR1_V3_tutorial.html#v03-showcase">V0.3 Showcase</a>
|
||
<ul>
|
||
<li><a href="en/DeepseekR1_V3_tutorial.html#dual-socket-version-64-cores-1">Dual socket version (64 cores)</a></li>
|
||
</ul>
|
||
</li>
|
||
</ul>
|
||
</li>
|
||
<li><a href="en/DeepseekR1_V3_tutorial.html#some-explanations">Some Explanations</a></li>
|
||
<li><a href="en/DeepseekR1_V3_tutorial.html#next">Next</a>
|
||
<ul>
|
||
<li><a href="en/DeepseekR1_V3_tutorial.html#faster">Faster</a></li>
|
||
<li><a href="en/DeepseekR1_V3_tutorial.html#easier">Easier</a></li>
|
||
</ul>
|
||
</li>
|
||
<li><a href="en/DeepseekR1_V3_tutorial.html#faq">FAQ</a>
|
||
<ul>
|
||
<li><a href="en/DeepseekR1_V3_tutorial.html#r1-no-thinking">R1 No Thinking</a></li>
|
||
<li><a href="en/DeepseekR1_V3_tutorial.html#more-faq">More FAQ</a></li>
|
||
</ul>
|
||
</li>
|
||
</ul>
|
||
</li>
|
||
</ul>
|
||
<h1 id="summary"><a class="header" href="#summary">SUMMARY</a></h1>
|
||
<blockquote>
|
||
<p><strong>Feb 10, 2025</strong>: Support DeepseekR1 and V3 on single (24GB VRAM)/multi gpu and 382G DRAM, up to 3~28x speedup.<br></p>
|
||
</blockquote>
|
||
<p>Hi, we're the KTransformers team (formerly known for our local CPU/GPU hybrid inference open source project with DeepSeek-V2).</p>
|
||
<p>We've heard your requests for DeepSeek-R1/V3 support—and we're excited to finally deliver!
|
||
Apologies for the wait, but we've been cooking up something truly amazing!</p>
|
||
<p>Today, we're proud to announce that we not only support DeepSeek-R1/V3, as showcased in the video below:</p>
|
||
<p>https://github.com/user-attachments/assets/ebd70bfa-b2c1-4abb-ae3b-296ed38aa285</p>
|
||
</p>
|
||
<ul>
|
||
<li><strong>[NEW!!!] Local 671B DeepSeek-Coder-V3/R1:</strong> Running its Q4_K_M version using only 14GB VRAM and 382GB DRAM.
|
||
<ul>
|
||
<li>Prefill Speed (tokens/s):
|
||
<ul>
|
||
<li>KTransformers: 54.21 (32 cores) → 74.362 (dual-socket, 2×32 cores) → 255.26 (optimized AMX-based MoE kernel, V0.3 only) → 286.55 (selectively using 6 experts, V0.3 only)</li>
|
||
<li>Compared to 10.31 tokens/s in llama.cpp with 2×32 cores, achieving up to <strong>27.79× speedup</strong>.</li>
|
||
</ul>
|
||
</li>
|
||
<li>Decode Speed (tokens/s):
|
||
<ul>
|
||
<li>KTransformers: 8.73 (32 cores) → 11.26 (dual-socket, 2×32 cores) → 13.69 (selectively using 6 experts, V0.3 only)</li>
|
||
<li>Compared to 4.51 tokens/s in llama.cpp with 2×32 cores, achieving up to <strong>3.03× speedup</strong>.</li>
|
||
</ul>
|
||
</li>
|
||
</ul>
|
||
</li>
|
||
</ul>
|
||
<p>We also give our upcoming optimizations previews, including an Intel AMX-accelerated kernel and a selective expert activation method, which will significantly enhance performance. With V0.3-preview, we achieve up to 286 tokens/s for prefill, making it up to <strong>28× faster than llama.cpp</strong> for local inference.
|
||
The binary distribution is available now and the source code will come ASAP! Check out the wheel package <a href="https://github.com/kvcache-ai/ktransformers/releases/download/v0.1.4/ktransformers-0.3.0rc0+cu126torch26fancy-cp311-cp311-linux_x86_64.whl">here</a></p>
|
||
<blockquote>
|
||
<p><strong>Feb 15, 2025</strong>: KTransformers V0.2.1: Longer Context (from 4K to 8K for 24GB VRAM) & Slightly Faster Speed (+15%) (Up to 16 Tokens/s), update docs <a href="en/./doc/en/DeepseekR1_V3_tutorial.html">here</a> and <a href="https://kvcache-ai.github.io/ktransformers/">online books</a>.</p>
|
||
</blockquote>
|
||
<p>We speed up the decode and prefill speed a littlt bit. The reason for the limited performance improvement mainly lies in the fact that the inference process is still constrained by the CPU's computational speed and memory bandwidth. The MLA part handled by the GPU accounts for a relatively small proportion.</p>
|
||
<p>Besides the improvements in speed, we've also significantly updated the documentation to enhance usability, including:<br></p>
|
||
<ul>
|
||
<li>Added Multi-GPU configuration tutorial.</li>
|
||
<li>Consolidated installation guide.</li>
|
||
<li>Add a detailed tutorial on registering extra GPU memory with ExpertMarlin;</li>
|
||
</ul>
|
||
<h2 id="show-case-environment"><a class="header" href="#show-case-environment">Show Case Environment</a></h2>
|
||
<p>We run our best performance tests (V0.2) on <br>
|
||
CPU: Intel (R) Xeon (R) Gold 6454S 1T DRAM (2 NUMA nodes) <br>
|
||
GPU: 4090D 24G VRAM <br>
|
||
Memory: standard DDR5-4800 server DRAM (1 TB), each socket with 8×DDR5-4800</p>
|
||
<h2 id="bench-result"><a class="header" href="#bench-result">Bench Result</a></h2>
|
||
<h3 id="v021"><a class="header" href="#v021">V0.2.1</a></h3>
|
||
<ul>
|
||
<li>Model: DeepseekV3-q4km (int4)<br></li>
|
||
<li>CPU: cpu_model_name: Intel (R) Xeon (R) Gold 6454S, 32 cores per socket, 2 sockets, 2 numa nodes</li>
|
||
<li>GPU: 4090 24G VRAM</li>
|
||
<li>We test after enough warm up</li>
|
||
</ul>
|
||
<h4 id="memory-consumption"><a class="header" href="#memory-consumption">Memory consumption:</a></h4>
|
||
<ul>
|
||
<li>Single socket: 382G DRAM, at least 14GB VRAM</li>
|
||
<li>Dual socket: 1T DRAM, at least 14GB VRAM</li>
|
||
</ul>
|
||
<h4 id="change-log"><a class="header" href="#change-log">Change Log</a></h4>
|
||
<ul>
|
||
<li>Longer Context (from 4K to 8K for 24GB VRAM) and Slightly Faster Speed (+15%):<br>
|
||
Integrated the highly efficient Triton MLA Kernel from the fantastic sglang project, enable much longer context length and slightly faster prefill/decode speed</li>
|
||
<li>We suspect that some of the improvements come from the change of hardware platform (4090D->4090)</li>
|
||
</ul>
|
||
<h4 id="benchmark-results"><a class="header" href="#benchmark-results">Benchmark Results</a></h4>
|
||
<p>"6 experts" case is part of V0.3's preview</p>
|
||
<div class="table-wrapper"><table><thead><tr><th>Prompt</th><th>hi (2)</th><th>1K (969)</th><th>2K (1930)</th><th>4K (3846)</th><th>8K (7678)</th></tr></thead><tbody>
|
||
<tr><td>Output length</td><td>10tokens</td><td>300tokens</td><td>300tokens</td><td>300tokens</td><td>300tokens</td></tr>
|
||
<tr><td><strong>6 experts V0.2.0</strong></td><td></td><td></td><td></td><td></td><td></td></tr>
|
||
<tr><td>Prefill token/s</td><td>13</td><td>105</td><td>102</td><td>88</td><td>CUDA OOM</td></tr>
|
||
<tr><td>decode token/s</td><td>16.8</td><td>15.4</td><td>14.2</td><td>13.0</td><td>CUDA OOM</td></tr>
|
||
<tr><td><strong>6 experts V0.2.1</strong></td><td></td><td></td><td></td><td></td><td></td></tr>
|
||
<tr><td>Prefill token/s</td><td>13</td><td>111</td><td>112.5</td><td>102**(1.16x speedup)**</td><td>101</td></tr>
|
||
<tr><td>decode token/s</td><td>16.8</td><td>15.9</td><td>15.4</td><td>14.9**(1.15x speedup)**</td><td>13.9</td></tr>
|
||
<tr><td><strong>8 experts V0.2.1</strong></td><td></td><td></td><td></td><td></td><td></td></tr>
|
||
<tr><td>Prefill token/s</td><td>12.2</td><td>88.2</td><td>88.5</td><td>81.9</td><td>80</td></tr>
|
||
<tr><td>Decode token/s</td><td>13.4</td><td>13.5</td><td>13.4</td><td>13.2</td><td>12.4</td></tr>
|
||
</tbody></table>
|
||
</div>
|
||
<h3 id="v02"><a class="header" href="#v02">V0.2</a></h3>
|
||
<h4 id="settings"><a class="header" href="#settings">Settings</a></h4>
|
||
<ul>
|
||
<li>Model: DeepseekV3-q4km (int4)<br></li>
|
||
<li>CPU: cpu_model_name: Intel (R) Xeon (R) Gold 6454S, 32 cores per socket, 2 sockets, 2 numa nodes</li>
|
||
<li>GPU: 4090D 24G VRAM</li>
|
||
<li>We test after enough warm up</li>
|
||
</ul>
|
||
<h4 id="memory-consumption-1"><a class="header" href="#memory-consumption-1">Memory consumption:</a></h4>
|
||
<ul>
|
||
<li>Single socket: 382G DRAM, at least 14GB VRAM</li>
|
||
<li>Dual socket: 1T DRAM, at least 14GB VRAM</li>
|
||
</ul>
|
||
<h4 id="benchmark-results-1"><a class="header" href="#benchmark-results-1">Benchmark Results</a></h4>
|
||
<p>"6 experts" case is part of V0.3's preview</p>
|
||
<div class="table-wrapper"><table><thead><tr><th>Prompt<br>(500 tokens)</th><th>Dual socket Ktrans (6 experts)</th><th>Dual socket Ktrans (8 experts)</th><th>Single socket Ktrans (6 experts)</th><th>Single socket Ktrans (8 experts)</th><th>llama.cpp (8 experts)</th></tr></thead><tbody>
|
||
<tr><td>Prefill token/s</td><td>97.32</td><td>82.94</td><td>65.14</td><td>54.21</td><td>10.31</td></tr>
|
||
<tr><td>Decode token/s</td><td>13.69</td><td>12.208</td><td>10.303</td><td>8.73</td><td>4.51</td></tr>
|
||
</tbody></table>
|
||
</div>
|
||
<p><strong>The highest speedup reaches up to <u>3.03x</u> in decoding and <u>9.44x</u> in prefill.</strong></p>
|
||
<h3 id="v03-preview"><a class="header" href="#v03-preview">V0.3-Preview</a></h3>
|
||
<h4 id="settings-1"><a class="header" href="#settings-1">Settings</a></h4>
|
||
<ul>
|
||
<li>Model: DeepseekV3-BF16 (online quant into int8 for CPU and int4 for GPU)</li>
|
||
<li>CPU: cpu_model_name: Intel (R) Xeon (R) Gold 6454S, 32 cores per socket, 2 socket, 2 numa nodes</li>
|
||
<li>GPU: (1~4)x 4090D 24GVRAM (requires more VRAM for longer prompt)</li>
|
||
</ul>
|
||
<h4 id="memory-consumptions"><a class="header" href="#memory-consumptions">Memory consumptions:</a></h4>
|
||
<ul>
|
||
<li>644GB DRAM, at least 14GB VRAM</li>
|
||
</ul>
|
||
<h4 id="benchmark-results-2"><a class="header" href="#benchmark-results-2">Benchmark results</a></h4>
|
||
<div class="table-wrapper"><table><thead><tr><th>Prompt length</th><th>1K</th><th>2K</th><th>4K</th><th>8K</th></tr></thead><tbody>
|
||
<tr><td>KTrans (8 experts) Prefill token/s</td><td>185.96</td><td>255.26</td><td>252.58</td><td>195.62</td></tr>
|
||
<tr><td>KTrans (6 experts) Prefill token/s</td><td>203.70</td><td>286.55</td><td>271.08</td><td>207.20</td></tr>
|
||
</tbody></table>
|
||
</div>
|
||
<p><strong>The prefill of KTrans V0.3 is up to <u>3.45x</u> times faster than KTrans V0.2, and is up to <u>27.79x</u> times faster than llama.cpp.</strong>
|
||
<strong>The decoding speed is the same as KTrans V0.2 (6 experts version) so it is omitted</strong></p>
|
||
<p>The main acceleration comes from</p>
|
||
<ul>
|
||
<li>Intel AMX instruction set and our specially designed cache friendly memory layout</li>
|
||
<li>Expert selection strategy that selects fewer experts based on offline profile results of out of domain data</li>
|
||
</ul>
|
||
<p><em>From our research on DeepSeekV2, DeepSeekV3 and DeepSeekR1,
|
||
when we slightly decrease the activation experts num in inference,
|
||
the output quality doesn't change. But the speed of decoding and prefill
|
||
is speed up which is inspiring. So our showcase makes use of this finding</em></p>
|
||
<h2 id="how-to-run"><a class="header" href="#how-to-run">How to Run</a></h2>
|
||
<h3 id="v024"><a class="header" href="#v024">v0.2.4</a></h3>
|
||
<p>We provide a server script, which supports multi-concurrency functionality in version v0.2.4.</p>
|
||
<pre><code>python ktransformers/server/main.py --model_path /mnt/data/models/DeepSeek-V3 --gguf_path /mnt/data/models/DeepSeek-V3-GGUF/DeepSeek-V3-Q4_K_M/ --cpu_infer 62 --optimize_config_path ktransformers/optimize/optimize_rules/DeepSeek-V3-Chat-serve.yaml --port 10002 --chunk_size 256 --max_new_tokens 1024 --max_batch_size 4 --port 10002 --cache_lens 32768 --backend_type balance_serve
|
||
</code></pre>
|
||
<p>It features the following arguments:</p>
|
||
<ul>
|
||
<li><code>--chunk_size</code>: Maximum number of tokens processed in a single run by the engine.</li>
|
||
<li><code>--cache_lens</code>: Total length of kvcache allocated by the scheduler. All requests share a kvcache space corresponding to 32768 tokens, and the space occupied will be released after the requests are completed.</li>
|
||
<li><code>--backend_type</code>: <code>balance_serve</code> is a multi-concurrency backend engine introduced in version v0.2.4. The original single-concurrency engine is <code>ktransformers</code>.</li>
|
||
<li><code>--max_batch_size</code>: Maximum number of requests (prefill + decode) processed in a single run by the engine. (Supported only by <code>balance_serve</code>)</li>
|
||
</ul>
|
||
<h3 id="v022--v023-longer-context--fp8-kernel"><a class="header" href="#v022--v023-longer-context--fp8-kernel">v0.2.2 & v0.2.3 longer context & FP8 kernel</a></h3>
|
||
<h4 id="longer-context"><a class="header" href="#longer-context">longer context</a></h4>
|
||
<p>To use this feature, <a href="https://github.com/flashinfer-ai/flashinfer">install flashinfer</a> first.</p>
|
||
<p>Note: The latest MLA kernel in FlashInfer still has a few minor issues. They are continuously fixing them on the main branch. If you are using FlashInfer, please install it from the main source code.</p>
|
||
<p>If you want to use long context(longer than 20K) for prefill, enable the matrix absorption MLA during the prefill phase, which will significantly reduce the size of the kv cache. Modify yaml file like this:</p>
|
||
<pre><code>- match:
|
||
name: "^model\\.layers\\..*\\.self_attn$"
|
||
replace:
|
||
class: ktransformers.operators.attention.KDeepseekV2Attention # optimized MLA implementation
|
||
kwargs:
|
||
generate_device: "cuda"
|
||
prefill_device: "cuda"
|
||
absorb_for_prefill: True # change this to True to enable long context(prefill may slower).
|
||
</code></pre>
|
||
<p>If the VRAM is still insufficient, try reducing the <code>chunk_size</code> parameter (default is 8192) to further decrease the intermediate results during chunk prefill.</p>
|
||
<h4 id="fp8-kernel"><a class="header" href="#fp8-kernel">FP8 kernel</a></h4>
|
||
<p>The DeepSeek-AI team provides FP8 safetensors for DeepSeek-R1/V3 models. We achieve performance optimization through the following works:</p>
|
||
<ul>
|
||
<li><strong>FP8 GPU Kernel Integration</strong>: FP8 linear layer acceleration kernels integrated in KTransformers</li>
|
||
<li><strong>Hybrid Quantization Architecture</strong>:
|
||
<ul>
|
||
<li>Attention and Shared-Expert modules use FP8 precision (enhances computational accuracy)</li>
|
||
<li>Experts modules retain GGML quantization (GGUF format, reside in CPU to save GPU memory)</li>
|
||
</ul>
|
||
</li>
|
||
</ul>
|
||
<p>So those who are persuing the best performance can use the FP8 linear kernel for DeepSeek-V3/R1.</p>
|
||
<p>The detailed guide is <a href="en/./fp8_kernel.html">here</a>.</p>
|
||
<h3 id="v02--v021-showcase"><a class="header" href="#v02--v021-showcase">V0.2 & V0.2.1 Showcase</a></h3>
|
||
<h4 id="single-socket-version-32-cores"><a class="header" href="#single-socket-version-32-cores">Single socket version (32 cores)</a></h4>
|
||
<p>Our local_chat test command is:</p>
|
||
<pre><code class="language-shell">numactl -N 1 -m 1 python ./ktransformers/local_chat.py --model_path <your model path> --gguf_path <your gguf path> --prompt_file <your prompt txt file> --cpu_infer 33 --max_new_tokens 1000
|
||
<when you see chat, then press enter to load the text prompt_file>
|
||
</code></pre>
|
||
<p><code><your model path></code> can be local or set from online huggingface like deepseek-ai/DeepSeek-V3. If online encounters connection problem, try use mirror (hf-mirror.com) <br>
|
||
<code><your gguf path></code> can also be online, but as its large we recommend you download it and quantize the model to what you want (notice it's the dir path) <br>
|
||
<code>--max_new_tokens 1000</code> is the max output token length. If you find the answer is truncated, you
|
||
can increase the number for longer answer (But be aware of OOM, and increase it will slow down the generation rate.).</p>
|
||
<p>The command <code>numactl -N 1 -m 1</code> aims to avoid data transfer between numa nodes<br>
|
||
Attention! If you are testing R1 and it may skip thinking. So you can add arg: <code>--force_think true</code>. This is explained in <a href="en/DeepseekR1_V3_tutorial.html#faq">FAQ</a> part</p>
|
||
<h4 id="dual-socket-version-64-cores"><a class="header" href="#dual-socket-version-64-cores">Dual socket version (64 cores)</a></h4>
|
||
<p>Make sure before you install (use install.sh or <code>make dev_install</code>), setting the env var <code>USE_NUMA=1</code> by <code>export USE_NUMA=1</code> (if already installed, reinstall it with this env var set). You may check the doc <a href="en/./install.html">here</a> for install details. <br></p>
|
||
<p>Test Command:</p>
|
||
<pre><code class="language-shell"># ---For those who have not installed ktransformers---
|
||
# git clone https://github.com/kvcache-ai/ktransformers.git
|
||
# cd ktransformers
|
||
# git submodule init
|
||
# git submodule update
|
||
# export USE_NUMA=1
|
||
# make dev_install # or sh ./install.sh
|
||
# ----------------------------------------------------
|
||
python ./ktransformers/local_chat.py --model_path <your model path> --gguf_path <your gguf path> --prompt_file <your prompt txt file> --cpu_infer 65 --max_new_tokens 1000
|
||
<when you see chat, then press enter to load the text prompt_file>
|
||
</code></pre>
|
||
<p>The parameters' meaning is the same. But As we use dual socket, we set cpu_infer to 65</p>
|
||
<h3 id="v03-showcase"><a class="header" href="#v03-showcase">V0.3 Showcase</a></h3>
|
||
<h4 id="dual-socket-version-64-cores-1"><a class="header" href="#dual-socket-version-64-cores-1">Dual socket version (64 cores)</a></h4>
|
||
<p>Our local_chat test command is:</p>
|
||
<pre><code class="language-shell">wget https://github.com/kvcache-ai/ktransformers/releases/download/v0.1.4/ktransformers-0.3.0rc0+cu126torch26fancy-cp311-cp311-linux_x86_64.whl
|
||
pip install ./ktransformers-0.3.0rc0+cu126torch26fancy-cp311-cp311-linux_x86_64.whl
|
||
python -m ktransformers.local_chat --model_path <your model path> --gguf_path <your gguf path> --prompt_file <your prompt txt file> --cpu_infer 65 --max_new_tokens 1000
|
||
<when you see chat, then press enter to load the text prompt_file>
|
||
</code></pre>
|
||
<p>The parameters' meaning is the same with V0.2. But As we use dual socket, we set cpu_infer to 65</p>
|
||
<h2 id="some-explanations"><a class="header" href="#some-explanations">Some Explanations</a></h2>
|
||
<ol>
|
||
<li>
|
||
<p>Also we want to make further use of our two NUMA nodes on Xeon Gold cpu.
|
||
To avoid the cost of data transfer between nodes, we "copy" the critical matrix on
|
||
both nodes which takes more memory consumption but accelerates the prefill and decoding process.
|
||
But this method takes huge memory and slow when loading weights, So be patient when loading
|
||
and monitor the memory usage. We are going to optimize this huge memory overhead. Stay tuned~ <br></p>
|
||
</li>
|
||
<li>
|
||
<p>The command args <code>--cpu_infer 65</code> specifies how many cores to use (it's ok that it exceeds the physical number,
|
||
but it's not the more the better. Adjust it slightly lower to your actual number of cores)<br></p>
|
||
</li>
|
||
<li>
|
||
<p>Why CPU/GPU Hybrid Inference?
|
||
DeepSeek's MLA operators are highly computationally intensive. While running everything on CPU is possible, offloading the heavy computations to the GPU results in a massive performance boost.</p>
|
||
</li>
|
||
<li>
|
||
<p>Where Does the Speedup Come From?</p>
|
||
<ul>
|
||
<li>Expert Offload: Unlike traditional layer-based or KVCache offloading (as seen in llama.cpp), we offload the expert computation to the CPU and MLA/KVCache to GPU, aligning perfectly with DeepSeek’s architecture for optimal efficiency.</li>
|
||
<li>Intel AMX Optimization – Our AMX-accelerated kernel is meticulously tuned, running several times faster than existing llama.cpp implementations. We plan to open-source this kernel after cleansing and are considering upstream contributions to llama.cpp.</li>
|
||
</ul>
|
||
</li>
|
||
<li>
|
||
<p>Why Intel CPUs?
|
||
Intel is currently the only CPU vendor that supports AMX-like instructions, which delivers significantly better performance compared to AVX-only alternatives.</p>
|
||
</li>
|
||
</ol>
|
||
<h2 id="next"><a class="header" href="#next">Next</a></h2>
|
||
<h3 id="faster"><a class="header" href="#faster">Faster</a></h3>
|
||
<ul>
|
||
<li>The FlashInfer (https://github.com/flashinfer-ai/flashinfer) project is releasing an even more efficient fused MLA operator, promising further speedups</li>
|
||
<li>vLLM has explored multi-token prediction in DeepSeek-V3, and support is on our roadmap for even better performance</li>
|
||
<li>We are collaborating with Intel to enhance the AMX kernel (v0.3) and optimize for Xeon6/MRDIMM</li>
|
||
</ul>
|
||
<h3 id="easier"><a class="header" href="#easier">Easier</a></h3>
|
||
<ul>
|
||
<li>Official Docker images to simplify installation</li>
|
||
<li>Fix the server integration for web API access</li>
|
||
<li>Fix the local chat only accepting a single line prompt (currently \n begins generating prompt)</li>
|
||
<li>Support for more quantization types, including the highly requested dynamic quantization from unsloth</li>
|
||
</ul>
|
||
<p>Stay tuned for more updates!</p>
|
||
<h2 id="faq"><a class="header" href="#faq">FAQ</a></h2>
|
||
<h3 id="r1-no-thinking"><a class="header" href="#r1-no-thinking">R1 No Thinking</a></h3>
|
||
<p>Attention! If you are testing R1 and it may skip thinking. So you can add arg: <code>--force_think true</code>. The detail is in <a href="en/./FAQ.html">FAQ</a> part <br></p>
|
||
<h3 id="more-faq"><a class="header" href="#more-faq">More FAQ</a></h3>
|
||
<p><a href="en/./FAQ.html">See detail</a></p>
|
||
<div style="break-before: page; page-break-before: always;"></div><h1 id="tutorial-heterogeneous-and-local-moe-inference"><a class="header" href="#tutorial-heterogeneous-and-local-moe-inference">Tutorial: Heterogeneous and Local MoE Inference</a></h1>
|
||
<p>DeepSeek-(Code)-V2 is a series of strong mixture-of-experts (MoE) models, featuring a total of 236 billion parameters, with 21 billion parameters activated per token. This model has demonstrated remarkable reasoning capabilities across various benchmarks, positioning it as one of the SOTA open models and nearly comparable in performance to GPT-4. DeepSeek-R1 uses a similar architecture to DeepSeek-V2, but with a bigger number of parameters.</p>
|
||
<p align="center">
|
||
<picture>
|
||
<img alt="DeepSeek-Coder-V2 Score" src="en/../assets/BigCodeBench.png" width=80%>
|
||
</picture>
|
||
</p>
|
||
<p>Moreover, unlike previous models that employed traditional attention mechanisms like Grouped-Query Attention (GQA), DeepSeek-V2 incorporates a novel Multi-head Latent Attention (MLA). This innovation significantly reduces the size of the KV cache required during inference, enhancing efficiency.</p>
|
||
<p>However, despite its efficiency, the practicality of running such a large model on personal computing setups seems impractical. Official documentation for DeepSeek-V2 indicates that eight 80GB GPUs are necessary for standard inference operations, and even the scaled-down Q4_k_m version requires at least two 80GB GPUs. These requirements are beyond the reach of most individual researchers and small teams.</p>
|
||
<p>Nonetheless, by employing several cutting-edge optimization techniques, we have successfully operated this colossal model on a desktop computer with only 21GB of VRAM and 136GB of DRAM. In this document, we outline the specific optimizations utilized and provide a detailed tutorial on how to implement these strategies using KTransformers.</p>
|
||
<h2 id="applied-optimizations"><a class="header" href="#applied-optimizations">Applied Optimizations</a></h2>
|
||
<h3 id="optimized-mla-operator"><a class="header" href="#optimized-mla-operator">Optimized MLA Operator</a></h3>
|
||
<p>The following figure provides a brief overview of DeepSeek-V2 architecture. At the heart of its attention layer, DeepSeek-V2 introduces a novel MLA operator that represents the heads of key-value pairs using a common, joint compressed representation, which holds significant potential for efficiency improvements. However, the official open-source implementation of the MLA operator explicitly decompresses this compressed representation and caches the decompressed key-value pairs. This process not only enlarges the KV cache size but also diminishes inference performance.</p>
|
||
<p align="center">
|
||
<picture>
|
||
<img alt="DeepSeek on KTransformers" src="en/../assets/DeepSeek-on-KTransformers.png" width=80%>
|
||
</picture>
|
||
</p>
|
||
<p>To truly capitalize on the benefits of MLA, we have implemented an optimized version for inference. According to its original paper, we absorb the decompression matrices directly into the q_proj and out_proj weights. Consequently, the compressed representation does not need to be decompressed to compute the attention. This adjustment significantly reduces the KV cache size and increases the arithmetic intensity of this operator, which greatly optimizes the utilization of GPU computational power.</p>
|
||
<h3 id="advanced-quantization-kernels"><a class="header" href="#advanced-quantization-kernels">Advanced Quantization Kernels</a></h3>
|
||
<p>The original DeepSeek-V2 model stores its parameters in BF16 format, consuming approximately 470GB of raw storage. This exceeds the RAM capacity available on mainstream desktop computers. To address this, we leverage the well-established GGUF community's quantized weights to simplify the process for users.
|
||
However, quantized data types are not typically supported by highly-optimized BLAS packages. As a result, the original HuggingFace Transformers' Torch implementation must dequantize these tensors to supported data types before processing, which introduces unnecessary computational overhead and increases memory traffic. To overcome this, we have incorporated advanced kernels that operate directly on quantized data types, thereby optimizing inference performance.</p>
|
||
<p>In the current version of KTransformers, we utilize Marlin for GPU kernels and llamafile for CPU kernels. These kerenls are specially designed to benefit from modern GPU architecture and modern CPU instruction extensions such as AVX512-BF16 (AMD Zen4 or newer) and AVX-VNNI (Intel Alder Lake or newer), that are tailored for quantized data types and machine learning workloads. We also use expert parallelism and other optimization for MOE inferencem on CPU based on llamafile, and call them as CPUInfer. As demonstrated in Figure 2(cite from Marlin), Marlin can achieve near ideal 3.87x speedup compare to corresponding Torch counterparts. As demonstrated in the following figure, our micro benchmarks show that inference using CPUInfer performs several times faster than Torch in low bits representation. Note that in practical inference such as using transformers, the Torch baseline use BF16 or FP16 as linear weights, and will occupy more memory resources, or it will be more slower due to dequantization when using quanted weights.</p>
|
||
<p align="center">
|
||
<picture>
|
||
<img alt="CPUInfer Performance" src="en/../assets/cpuinfer.png" width=80%>
|
||
</picture>
|
||
</p>
|
||
<p align="center">
|
||
<picture>
|
||
<img alt="marlin performance" src="https://github.com/IST-DASLab/marlin/blob/master/assets/sustained.png?raw=true" width=80%>
|
||
</picture>
|
||
</p>
|
||
<h3 id="arithmetic-intensity-guided-offloading"><a class="header" href="#arithmetic-intensity-guided-offloading">Arithmetic Intensity Guided Offloading</a></h3>
|
||
<p>Storing all 236 billion parameters of a model in GPU VRAM is clearly impractical for local users. Therefore, we strategically store only the most computationally intensive parameters on the GPU. For instance, after our optimizations, the MLA operator, which contains 128 heads with a shared compressed key-value representation, shows an arithmetic intensity of 512. This makes it the most intensive operator, particularly during smaller inference batch sizes. Hence, it is allocated to the GPU to leverage the power of tensor cores.</p>
|
||
<p>On the other hand, as shown in Figure 1, each transformer block in DeepSeek-V2 includes 160 mixture-of-experts (MoE) experts, comprising 96% of the total parameters. However, the MoE router activates only 6 out of these 160 experts for each token, which means that only 3.75% of the MoE parameters are utilized during the decoding phase. With a batch size of one, the arithmetic intensity of the MoE operation is roughly 0.075. This operation, primarily involving a batched General Matrix-Vector Multiplication (GEMV), can thus be efficiently handled by the CPU.</p>
|
||
<p>Following this principle of arranging all operators by their arithmetic intensity and placing the most intensive ones in the GPU as much as possible, we prioritize positioning the MoE parameters and word embeddings computations on the CPU side to utilize its larger memory capacity. Meanwhile, the remaining parameters, including shared experts, projections in the attention module, and MLA, are stored in the GPU VRAM. As these parameters are accessed by every token, their placement on the GPU maximizes the benefits of high memory bandwidth. This configuration leads to approximately 20.7 GB of VRAM usage and 136GB DRAM memory requests if the Q4_K_M version is used, which is feasible even on a local desktop. Additionally, the placement can be adjusted according to the actual configuration, adhering to the same principle.</p>
|
||
<p>Moreover, as an extensible framework, KTransformers is set to support more advanced operators in future releases, continually enhancing its capability to handle diverse workloads efficiently.</p>
|
||
<h2 id="yaml-template"><a class="header" href="#yaml-template">YAML Template</a></h2>
|
||
<p>To implement the above optimizations in KTransformers, users need to write a YAML file containing the optimized rules.
|
||
KTransformers will iterate through all sub-modules of the model, match rules specified in the YAML rule file, and replace them with advanced modules as specified.</p>
|
||
<p align="center">
|
||
<picture>
|
||
<img alt="Inject-Struction" src="en/../assets/InjectStruction.png" width=80%>
|
||
</picture>
|
||
</p>
|
||
<p>Specifically, the following rules are used:</p>
|
||
<ul>
|
||
<li>Replace the Attention module with our <a href="en/deepseek-v2-injection.html#mla">optimized MLA Operator</a>.</li>
|
||
<li>Replace routed experts with <a href="en/deepseek-v2-injection.html#experts">CPUInfer kernels</a> that use Llamafile.</li>
|
||
<li>Replace all Linear modules not belonging to attention with <a href="en/deepseek-v2-injection.html#linear">Marlin</a> kernels.</li>
|
||
</ul>
|
||
<h3 id="mla"><a class="header" href="#mla">MLA</a></h3>
|
||
<p>For attention module injection, we only need to match the module name used in Transformers using a regular expression and replace it with our pre-implemented module.
|
||
The YAML rule is listed below.</p>
|
||
<pre><code class="language-yaml">- match:
|
||
name: "^model\\.layers\\..*\\.self_attn$" # regular expression
|
||
replace:
|
||
class: ktransformers.operators.attention.KDeepseekV2Attention # optimized MLA implementation
|
||
</code></pre>
|
||
<p>As we can see, each rule in the YAML file has two parts: <code>match</code> and <code>replace</code>.
|
||
The match part specifies which module should be replaced, and the replace part specifies the module to be injected into the model along with the initialization keywords.</p>
|
||
<h3 id="experts"><a class="header" href="#experts">Routed Experts </a></h3>
|
||
<p>For routed experts, the module we inject is a wrapper of CPUInfer, KTransformersExperts. There are several implementations within a wrapper, and we need to specify keywords to tell the wrapper which implementation we want to use and how we intend to use it.</p>
|
||
<p>In KTransformers, some models exhibit different behaviors during prefilling and generation for better performance. KTransformersExperts is one of them. All these special modules have a <code>device</code> keyword describing which device the module should be initialized on. Other keywords specify the behaviors during prefilling and generation and may be differ when using different injection modules. Here, we specify which implementation on which device we want to use during prefilling and generation, and which device the output should be on.
|
||
Note that we only use these parameters when layer-wise prefilling is enabled; otherwise, prefilling is conducted with the same configuration as generation.</p>
|
||
<p>In the original implementation of Transformers, MoE is implemented using <code>nn.ModuleList</code>. We don't want KTransformers to iterate through all the sub-modules in the list, so we set <code>recursive: False</code> in this rule to prevent recursive injection into submodules of the current module. Here is the YAML rule:</p>
|
||
<pre><code class="language-yaml">- match:
|
||
name: "^model\\.layers\\..*\\.mlp\\.experts$"
|
||
replace:
|
||
class: ktransformers.operators.experts.KTransformersExperts # custom MoE Kernel with expert parallelism
|
||
device: "cpu" # device to load this module on initialization
|
||
kwargs:
|
||
prefill_device: "cuda"
|
||
prefill_op: "KExpertsTorch"
|
||
generate_device: "cpu"
|
||
generate_op: "KExpertsCPU"
|
||
out_device: "cuda"
|
||
recursive: False # don't recursively inject submodules of this module
|
||
</code></pre>
|
||
<p>If we inject the expert list as a custom module, we can't use the interface in <code>nn.ModuleList</code> as default. We need to change the forward function in the FFN module. The simplest way is implementing a new module using custom forward function and inject it. We have implemented the new module, and the injection can be done by simply adding an injection rule. We can use the <code>class</code> instead of <code>name</code> to match a module that will be replaced. Here is the YAML rule:</p>
|
||
<pre><code class="language-yaml">- match:
|
||
class: ktransformers.models.modeling_deepseek.DeepseekV2MoE
|
||
replace:
|
||
class: ktransformers.operators.experts.KDeepseekV2MoE # MLP module with custom forward function
|
||
</code></pre>
|
||
<h3 id="linear"><a class="header" href="#linear">Other Linear Modules</a></h3>
|
||
<p>For the remained linear modules, we want to use our quantization kernels. However, we don't want to inject linear in the MLA operator because we currently don't know the effect of using quantization in MLA.
|
||
So, we can change our regular expression and add a class check in the match part of the rule. Only modules matching both name and class simultaneously will be injected.
|
||
We also need to transfer some keywords similar to the injection of experts. Here is the YAML rule:</p>
|
||
<pre><code class="language-yaml">- match:
|
||
name: "^model\\.layers\\.(?!.*self_attn).*$" # regular expression
|
||
class: torch.nn.Linear # only match modules matching name and class simultaneously
|
||
replace:
|
||
class: ktransformers.operators.linear.KTransformersLinear # optimized Kernel on quantized data types
|
||
kwargs:
|
||
generate_device: "cuda"
|
||
prefill_device: "cuda"
|
||
generate_op: "KLinearMarlin"
|
||
prefill_op: "KLinearTorch"
|
||
</code></pre>
|
||
<h3 id="Pre-compute Buffers"><a class="header" href="#Pre-compute Buffers">Pre-compute Buffers </a></h3>
|
||
<p>The original model is initialized on the meta device. The rotary embedding module pre-computes some buffers when initializing, which has no effect and doesn't compute anything when using the meta device. Therefore, we need to compute the buffers when loading the model. For convenience, we inject the rotary embedding module with our custom module, which performs pre-computations when loading. Here is the YAML rule:</p>
|
||
<pre><code class="language-yaml">- match:
|
||
class: ktransformers.models.modeling_deepseek.DeepseekV2YarnRotaryEmbedding
|
||
replace:
|
||
class: ktransformers.operators.RoPE.YarnRotaryEmbedding
|
||
</code></pre>
|
||
<h2 id="wrap-your-custom-module"><a class="header" href="#wrap-your-custom-module">Wrap Your Custom Module</a></h2>
|
||
<p>We have implemented some modules, but you may need to inject your custom module using KTransformers.
|
||
The only thing you need to do is wrap your custom module and write YAML files. We provide a base operator specifying interfaces an injection module should have. You only need to inherit from that module and change the <code>__init__</code>, <code>forward</code>, or <code>load</code> function as needed.</p>
|
||
<ul>
|
||
<li>The <code>__init__</code> function of the base operator maintains the necessary information for injection and execution of the KTransformers framework. To override this function, subclass modules need to call the base operator's <code>__init__</code> function in their own initializer.</li>
|
||
<li>The <code>forward</code> function is a function in torch that will be called during inference, where the module author has the freedom to achieve higher performance.</li>
|
||
<li>The <code>load</code> function is used to load all parameters of this module. The default implementation is to call the <code>load</code> function of all submodules. You can modify this function to customize its loading method and explicitly control the loading of its submodules.</li>
|
||
</ul>
|
||
<div style="break-before: page; page-break-before: always;"></div><h1 id="tutorial-inject-operator-step-by-step"><a class="header" href="#tutorial-inject-operator-step-by-step">Tutorial: Inject Operator Step by Step</a></h1>
|
||
<blockquote>
|
||
<p>Author: Azure-Tang</p>
|
||
</blockquote>
|
||
<h2 id="tldr"><a class="header" href="#tldr">TL;DR</a></h2>
|
||
<p>This tutorial will guide you through the process of injecting custom operators into a model using the KTransformers framework. We will use the DeepSeekV2-Chat model as an example to demonstrate how to inject custom operators into the model step by step. The tutorial will cover the following topics:</p>
|
||
<ul>
|
||
<li><a href="en/injection_tutorial.html#how-to-write-injection-rules">How to write injection rules</a>
|
||
<ul>
|
||
<li><a href="en/injection_tutorial.html#understanding-model-structure">Understanding the structure of the model</a></li>
|
||
</ul>
|
||
</li>
|
||
<li><a href="en/injection_tutorial.html#muti-gpu">Multi-GPU</a></li>
|
||
<li><a href="en/injection_tutorial.html#how-to-write-a-new-operator-and-inject-into-the-model">How to write a new operator and inject it into the model</a></li>
|
||
</ul>
|
||
<h2 id="how-to-write-injection-rules"><a class="header" href="#how-to-write-injection-rules">How to Write Injection Rules</a></h2>
|
||
<p>The basic form of the injection rules for the Inject framework is as follows:</p>
|
||
<pre><code class="language-yaml">- match:
|
||
name: "^model\\.layers\\..*\\.*$" # Target module name
|
||
class: torch.nn.Linear # Target module
|
||
replace:
|
||
class: "default"
|
||
kwargs:
|
||
generate_device: "cuda:0"
|
||
# your_op_param_1: 1234
|
||
# your_op_param_2: 5678
|
||
recursive: True
|
||
</code></pre>
|
||
<ul>
|
||
<li>match: This field marks the matching rules, which can appear in two forms, name and class. These two matching rules can appear together or separately; they only match when both criteria are met.</li>
|
||
<li>replace:
|
||
<ul>
|
||
<li>class: Python class that can be imported to replace the target module. If no replacement is desired, set to default.</li>
|
||
<li>kwargs: List of parameters needed for module initialization.
|
||
<ul>
|
||
<li>generate_device: The device for this module, can be set to “cpu”, “cuda”, “cuda:1”, etc.</li>
|
||
</ul>
|
||
</li>
|
||
</ul>
|
||
</li>
|
||
<li>recursive: Whether to recursively inject this module’s submodules, default is True.</li>
|
||
</ul>
|
||
<p>For the recursive field: Some modules contain multiple submodules, such as the Self-attention module typically includes q/k/v/o four linear modules. If we replace the self-attention module but do not want the internal linear modules to be covered by other rules, set this rule to False.</p>
|
||
<h2 id="understanding-model-structure"><a class="header" href="#understanding-model-structure">Understanding Model Structure</a></h2>
|
||
<p>Using <a href="https://huggingface.co/deepseek-ai/DeepSeek-V2-Lite-Chat">deepseek-ai/DeepSeek-V2-Lite-Chat</a> as an example, we can follow the above rules step by step to inject our custom module and run it. KTransformers offers a high degree of flexibility, allowing you to replace/experiment with basic operators. However, it also requires users to clearly understand the structure of the model they are running.</p>
|
||
<p>Fortunately, knowing the structure of a model is very simple. Open the file list on the <a href="https://huggingface.co/deepseek-ai/DeepSeek-V2-Lite-Chat/tree/main">deepseek-ai/DeepSeek-V2-Lite</a> homepage, and you can see the following files:</p>
|
||
<p align="center">
|
||
<picture>
|
||
<img alt="Inject-Struction" src="en/../assets/model_structure_guild.png" width=60%>
|
||
</picture>
|
||
</p>
|
||
<p>From the <code>.saftensors</code> file, we can see the name of each layer’s weights, corresponding to the match.name attribute in the injection rules.
|
||
From the <code>modeling_deepseek.py</code> file, we can see the specific implementation of each module class, with the class name corresponding to the match.class attribute in the injection rules.</p>
|
||
<p>The structure of the DeepSeekV2 model from the <code>.saftensors</code> and <code>modeling_deepseek.py</code> files is as follows:</p>
|
||
<p align="center">
|
||
<picture>
|
||
<img alt="Inject-Struction" src="en/../assets/deepseekv2_structure.png" width=60%>
|
||
</picture>
|
||
</p>
|
||
<p>Supported operators and their corresponding classes are as follows:</p>
|
||
<div class="table-wrapper"><table><thead><tr><th>match</th><th>replace</th><th>backends</th><th>descriptions</th></tr></thead><tbody>
|
||
<tr><td>Linear</td><td>KTransformersLinear</td><td>KLinearMarlin</td><td>Marlin as backend</td></tr>
|
||
<tr><td></td><td></td><td>KLinearTorch</td><td>pytorch as backend</td></tr>
|
||
<tr><td></td><td></td><td>KLinearCPUInfer</td><td>llamafile as backend</td></tr>
|
||
<tr><td></td><td></td><td>KLinearFP8</td><td>Triton fp8_gemm kernel. Requires GPU be able to caluculate fp8 data</td></tr>
|
||
<tr><td>experts</td><td>KTransformersExperts</td><td>KExpertsTorch</td><td>pytorch as backend</td></tr>
|
||
<tr><td></td><td></td><td>KExpertsMarlin</td><td>Marlin as backend</td></tr>
|
||
<tr><td></td><td></td><td>KExpertsCPU</td><td>llamafile as backend</td></tr>
|
||
<tr><td>Attention</td><td>KDeepseekV2Attention</td><td>KDeepseekV2Attention</td><td>MLA implementation</td></tr>
|
||
<tr><td>MoE</td><td>KMistralSparseMoEBlock</td><td>KQwen2MoeSparseMoeBlock</td><td>MoE for Qwen2</td></tr>
|
||
<tr><td></td><td>KDeepseekV2MoE</td><td>KDeepseekV2MoE</td><td>MoE for DeepseekV2</td></tr>
|
||
<tr><td>Model</td><td>KQwen2MoeModel</td><td>KQwen2MoeModel</td><td>Model for Qwen2</td></tr>
|
||
<tr><td></td><td>KDeepseekV2Model</td><td>KDeepseekV2Model</td><td>Model for DeepseekV2</td></tr>
|
||
<tr><td>RoPE</td><td>RotaryEmbedding</td><td>RotaryEmbedding</td><td>RoPE module</td></tr>
|
||
<tr><td></td><td>YarnRotaryEmbedding</td><td>YarnRotaryEmbedding</td><td>RoPE module</td></tr>
|
||
</tbody></table>
|
||
</div>
|
||
<p>Then we start step-by-step injection of custom modules, our targets are:</p>
|
||
<ul>
|
||
<li>Replace the linear module with custom Marlin linear module.</li>
|
||
<li>Replace the self-attention module with a custom Absorption-based MLA module.</li>
|
||
<li>Replace the experts module with a custom Experts module.</li>
|
||
<li>Replace the MoE module with a custom MoE module.</li>
|
||
<li>Replace the RoPE module with a custom RoPE module.</li>
|
||
<li>Set the running device for each module.</li>
|
||
</ul>
|
||
<p>The full implementation of the injection rules can be found in the <a href="https://github.com/kvcache-ai/ktransformers/blob/main/ktransformers/optimize/optimize_rules/DeepSeek-V2-Chat.yaml">here</a>.</p>
|
||
<h2 id="matrix-absorption-based-mla-injection"><a class="header" href="#matrix-absorption-based-mla-injection">Matrix Absorption-based MLA Injection</a></h2>
|
||
<p>For the injection of the Attention module, we only need to use a regular expression to match the module names used in transformers and replace them with our own MLA module implementation. The YAML injection rule is as follows:</p>
|
||
<pre><code class="language-yaml">- match:
|
||
name: "^model\\.layers\\..*\\.self_attn$" # Regular expression
|
||
replace:
|
||
class: ktransformers.operators.attention.KDeepseekV2Attention # Optimized MLA implementation
|
||
</code></pre>
|
||
<p>As you can see, each rule in the YAML file has two parts: match and replace. The match part specifies the module to be replaced, and the replace part specifies the module to be injected into the model along with the initialization keywords.</p>
|
||
<h2 id="injection-of-routed-experts"><a class="header" href="#injection-of-routed-experts">Injection of Routed Experts</a></h2>
|
||
<p>For Routed Experts (corresponding to the exps in the diagram), the module we inject is CPUInfer, which is wrapped in the wrapper module KTransformersExperts. KTransformersExperts has multiple implementations, and we need to specify keywords to tell the wrapper module which implementation we want to use and how we plan to use it.</p>
|
||
<p>In the source code of the transformer, MoE is implemented using nn.ModuleList. We do not want KTransformers to traverse all submodules in the list and inject them one by one, so in this rule, we set recursive: False to prevent recursive injection into the submodules of this module. The YAML rule is as follows:</p>
|
||
<pre><code class="language-yaml">- match:
|
||
name: "^model\\.layers\\..*\\.mlp\\.experts$"
|
||
replace:
|
||
class: ktransformers.operators.experts.KTransformersExperts # Custom MoE kernel with expert parallelism
|
||
kwargs:
|
||
generate_device: "cpu"
|
||
generate_op: "MLPCPUExperts"
|
||
out_device: "cuda"
|
||
recursive: False # Don't recursively inject submodules of this module
|
||
</code></pre>
|
||
<p>If we inject Routed Experts as a custom module, we cannot use the interfaces in the original <code>nn.ModuleList</code>. Therefore, it is necessary to modify the forward function in the FFN module. The simplest method is to implement a new module with a custom forward function and inject it.</p>
|
||
<pre><code class="language-yaml">- match:
|
||
class: ktransformers.models.modeling_deepseek.DeepseekV2MoE
|
||
replace:
|
||
class: ktransformers.operators.experts.KDeepseekV2MoE # MLP module with custom forward function
|
||
</code></pre>
|
||
<h2 id="injection-of-linear-layers"><a class="header" href="#injection-of-linear-layers">Injection of Linear Layers</a></h2>
|
||
<p>For the remaining linear layer modules, we aim to use quantized operators to save storage space while improving performance. Since there is no current research on using MLA and quantization together, we do not want to inject linear into the MLA operator. Therefore, we can modify the regular expression and add a type check in the match part of the rule. Only modules that match both the name and class simultaneously will be injected. We also need to pass some keywords similar to the injection of Routed Experts. The YAML rule is as follows:</p>
|
||
<pre><code class="language-yaml">- match:
|
||
name: "^model\\.layers\\.(?!.*self_attn).*$" # Regular expression
|
||
class: torch.nn.Linear # Only match modules matching name and class simultaneously
|
||
replace:
|
||
class: ktransformers.operators.linear.KTransformersLinear # Optimized kernel on quantized data types
|
||
kwargs:
|
||
generate_device: "cuda"
|
||
generate_op: "QuantizedLinearMarlin"
|
||
</code></pre>
|
||
<h2 id="injection-of-modules-with-pre-calculated-buffers"><a class="header" href="#injection-of-modules-with-pre-calculated-buffers">Injection of Modules with Pre-calculated Buffers</a></h2>
|
||
<p>To avoid occupying resources when initializing the injected original model, we use torch’s meta device to initialize the original model. The RoPE module pre-calculates some buffers during initialization, but no calculations are performed when using the meta device. Therefore, we need to compensate for the calculation of the buffer when loading the model. Simply, we inject a custom module into the rotary embedding module, which performs pre-calculation during loading. The YAML rule is as follows:</p>
|
||
<pre><code class="language-yaml">- match:
|
||
class: ktransformers.models.modeling_deepseek.DeepseekV2YarnRotaryEmbedding
|
||
replace:
|
||
class: ktransformers.operators.RoPE.YarnRotaryEmbedding
|
||
</code></pre>
|
||
<h2 id="specifying-running-devices-for-modules"><a class="header" href="#specifying-running-devices-for-modules">Specifying Running Devices for Modules</a></h2>
|
||
<p>Finally, we set a fallback basic attribute generate_device for all modules:</p>
|
||
<pre><code class="language-yaml">- match:
|
||
name: "^model\\.layers\\..*\\.|^lm_head"
|
||
replace:
|
||
class: "default"
|
||
kwargs:
|
||
generate_device: "cuda"
|
||
|
||
- match:
|
||
name: "^model.embed_tokens"
|
||
replace:
|
||
class: "default"
|
||
kwargs:
|
||
generate_device: "cpu"
|
||
</code></pre>
|
||
<p>Through these two rules, we place all previously unmatched layers (and their submodules) and lm_head on cuda, and the embedding on cpu. Note that the properties of a module will be determined by the first rule it matches. For example, if you later set a new replace.kwargs.generate_device in an injected module, the device set earlier will take precedence. If your computer has multiple cards, you can also configure the model to multiple cards.</p>
|
||
<h2 id="muti-gpu"><a class="header" href="#muti-gpu">Muti-GPU</a></h2>
|
||
<p>If you have multiple GPUs, you can set the device for each module to different GPUs.
|
||
DeepseekV2-Chat got 60 layers, if we got 2 GPUs, we can allocate 30 layers to each GPU. Complete multi GPU rule examples <a href="https://github.com/kvcache-ai/ktransformers/blob/main/ktransformers/optimize/optimize_rules/DeepSeek-V2-Chat-multi-gpu.yaml">here</a>.</p>
|
||
<p align="center">
|
||
<picture>
|
||
<img alt="Inject-Struction" src="en/../assets/multi_gpu.png" width=60%>
|
||
</picture>
|
||
</p>
|
||
<p>First of all, for multi-GPU, we have to inject an new operator <code>KDeepseekV2Model</code>. And set division of the layers to different GPUs. For our case, we have to set the <code>transfer_map</code> in the <code>KDeepseekV2Model</code> operatoras as follows:</p>
|
||
<pre><code class="language-yaml">- match:
|
||
name: "^model$"
|
||
replace:
|
||
class: "ktransformers.operators.models.KDeepseekV2Model"
|
||
kwargs:
|
||
transfer_map:
|
||
30: "cuda:1"
|
||
</code></pre>
|
||
<p>And we have to set the device for each module in the model.</p>
|
||
<p>For example, for <code>routed experts</code>, the yaml for one GPU is:</p>
|
||
<pre><code class="language-yaml">- match:
|
||
name: "^model\\.layers\\..*\\.mlp\\.experts$"
|
||
replace:
|
||
class: ktransformers.operators.experts.KTransformersExperts # Custom MoE kernel with expert parallelism
|
||
kwargs:
|
||
generate_device: "cuda:0"
|
||
generate_op: "MLPCUDAExperts"
|
||
out_device: "cuda:0"
|
||
recursive: False # Don't recursively inject submodules of this module
|
||
</code></pre>
|
||
<p>But for two GPUs, we need to set the device for each module in the model.</p>
|
||
<pre><code class="language-yaml"># allcate 0-29 layers‘s out_device to cuda:0
|
||
- match:
|
||
name: "^model\\.layers\\.(0|[1-9]|[12][0-9])\\.mlp\\.experts$"
|
||
replace:
|
||
class: ktransformers.operators.experts.KTransformersExperts # custom MoE Kernel with expert paralleism
|
||
kwargs:
|
||
generate_device: "cpu"
|
||
generate_op: "KExpertsCPU"
|
||
out_device: "cuda:0"
|
||
recursive: False # don't recursively inject submodules of this module
|
||
|
||
# allocate 30-59 layers‘s out_device to cuda:1
|
||
- match:
|
||
name: "^model\\.layers\\.([345][0-9])\\.mlp\\.experts$"
|
||
replace:
|
||
class: ktransformers.operators.experts.KTransformersExperts # custom MoE Kernel with expert paralleism
|
||
kwargs:
|
||
generate_device: "cpu"
|
||
generate_op: "KExpertsCPU"
|
||
out_device: "cuda:1"
|
||
recursive: False # don't recursively inject submodules of this module
|
||
</code></pre>
|
||
<p>For other modules, we can set the device in the same way.</p>
|
||
<h2 id="how-to-write-a-new-operator-and-inject-into-the-model"><a class="header" href="#how-to-write-a-new-operator-and-inject-into-the-model">How to Write a New Operator and Inject into the Model</a></h2>
|
||
<p>In this section, we will explain how to write an operator that can be injected, using the implementation of a new linear as an example.</p>
|
||
<p>First, all injectable operators need to inherit from the BaseInjectedModule class, which inherits some attributes required by our injection framework. Its initialization function needs to meet the following basic format:</p>
|
||
<pre><code class="language-python">class LinearTorchInject(BaseInjectedModule):
|
||
def __init__(
|
||
self,
|
||
key: str,
|
||
gguf_loader: GGUFLoader,
|
||
config: PretrainedConfig,
|
||
orig_module: nn.Module = None,
|
||
generate_device: str = "cuda",
|
||
**kwargs,
|
||
):
|
||
super().__init__(key, gguf_loader, config, orig_module, generate_device, **kwargs)
|
||
</code></pre>
|
||
<p>If users have other parameters that need to be passed to this class, they can also be included in the init function and re-passed in the kwargs parameter in the yaml file. For example, if our operator wants to pass a parameter <code>my_param</code>, the init function can be written as:</p>
|
||
<pre><code class="language-python">class LinearTorchInject(BaseInjectedModule):
|
||
def __init__(
|
||
self,
|
||
key: str,
|
||
gguf_loader: GGUFLoader,
|
||
config: PretrainedConfig,
|
||
orig_module: nn.Module = None,
|
||
generate_device: str = "cuda",
|
||
my_param: bool = True,
|
||
**kwargs,
|
||
):
|
||
super().__init__(key, gguf_loader, config, orig_module, generate_device, **kwargs)
|
||
self.my_param = my_param
|
||
</code></pre>
|
||
<p>Then our injection rule can be written as:</p>
|
||
<pre><code class="language-yaml">- match:
|
||
name: "^model\\.layers\\..*$" # Regular expression matches the module name.
|
||
class: torch.nn.Linear # Type restrictions can be added.
|
||
replace:
|
||
class: ktransformers.operators.linear.LinearTorchInject # Inject module path
|
||
kwargs: # Extra parameters
|
||
generate_device: "cuda"
|
||
my_param: True
|
||
</code></pre>
|
||
<p>For the linear module, it is also necessary to read weights from a gguf file. We provide the <code>KLinearBase</code> class to help users read weights from gguf files. Users only need to inherit and implement the load, unload, and forward functions. Therefore, a fully injectable linear class would look like this:</p>
|
||
<pre><code class="language-python">class LinearTorchInject(BaseInjectedModule, KLinearBase):
|
||
def __init__(
|
||
self,
|
||
key: str,
|
||
gguf_loader: GGUFLoader,
|
||
config: PretrainedConfig,
|
||
orig_module: nn.Module = None,
|
||
generate_device: str = "cuda",
|
||
**kwargs,
|
||
):
|
||
super().__init__(key, gguf_loader, config, orig_module, generate_device, **kwargs)
|
||
KLinearBase.__init__(self)
|
||
self.has_bias = False
|
||
self.dtype = torch.get_default_dtype()
|
||
self.w = None
|
||
self.has_bias = False
|
||
|
||
def load(self, w: dict | nn.Parameter | tuple | None = None, device: str|None = None):
|
||
if device is None: device = self.device
|
||
if w is None: w = self.load_weight(device=device)
|
||
|
||
if isinstance(w, nn.Parameter):
|
||
self.w = w.to(dtype=self.dtype).view(self.out_features, self.in_features).T
|
||
self.has_bias = False
|
||
elif isinstance(w, tuple):
|
||
self.w = w[0].to(dtype=self.dtype).view(self.out_features, self.in_features).T
|
||
self.bias = w[1].to(dtype=self.dtype)
|
||
self.has_bias = True
|
||
else:
|
||
raise ValueError("Invalid weight type")
|
||
self.w = self.w.to(device)
|
||
if self.has_bias:
|
||
self.bias = self.bias.to(device)
|
||
|
||
def unload(self):
|
||
if self.w is not None:
|
||
self.w = None
|
||
if self.has_bias:
|
||
self.bias = None
|
||
|
||
def forward(self, x: torch.Tensor) -> torch.Tensor:
|
||
dtype = x.dtype
|
||
out_device = x.device
|
||
x = x.to(device=self.device, dtype=self.dtype)
|
||
x = x @ self.w
|
||
if self.has_bias:
|
||
x = x + self.bias
|
||
x = x.to(dtype=dtype, device=out_device)
|
||
return x
|
||
</code></pre>
|
||
<p>Note that the <code>self.load_weight</code> function is provided by the KLinearBase class to help users load weights from a gguf file into the module. The implementation details of KLinearBase can be found on <a href="https://github.com/kvcache-ai/ktransformers/blob/44f57270c9514d79fab224186d90ccf61059331a/ktransformers/operators/linear.py#L31">GITHUB</a>.</p>
|
||
<div style="break-before: page; page-break-before: always;"></div><h1 id="muti-gpu-1"><a class="header" href="#muti-gpu-1">Muti-GPU</a></h1>
|
||
<p>Assume you have read the <a href="en/./injection_tutorial.html">Injection Tutorial</a> and have a basic understanding of how to inject a model. In this tutorial, we will show you how to use KTransformers to run a model on multiple GPUs.</p>
|
||
<p>If you have multiple GPUs, you can set the device for each module to different GPUs.
|
||
DeepseekV2-Chat got 60 layers, if we got 2 GPUs, we can allocate 30 layers to each GPU. Complete multi GPU rule examples <a href="https://github.com/kvcache-ai/ktransformers/blob/main/ktransformers/optimize/optimize_rules/DeepSeek-V3-Chat-multi-gpu.yaml">here</a>.</p>
|
||
<p align="center">
|
||
<picture>
|
||
<img alt="Inject-Struction" src="en/../assets/multi_gpu.png" width=60%>
|
||
</picture>
|
||
</p>
|
||
<p>First of all, for multi-GPU, we have to inject an new operator <code>KDeepseekV2Model</code>. And set division of the layers to different GPUs. For our case, we have to set the <code>transfer_map</code> in the <code>KDeepseekV2Model</code> operatoras as follows:</p>
|
||
<pre><code class="language-yaml">- match:
|
||
name: "^model$"
|
||
replace:
|
||
class: "ktransformers.operators.models.KDeepseekV2Model"
|
||
kwargs:
|
||
transfer_map:
|
||
30: "cuda:1"
|
||
</code></pre>
|
||
<p>And we have to set the device for each module in the model.</p>
|
||
<p>For example, for <code>routed experts</code>, the yaml for one GPU is:</p>
|
||
<pre><code class="language-yaml">- match:
|
||
name: "^model\\.layers\\..*\\.mlp\\.experts$"
|
||
replace:
|
||
class: ktransformers.operators.experts.KTransformersExperts # Custom MoE kernel with expert parallelism
|
||
kwargs:
|
||
generate_device: "cuda:0"
|
||
generate_op: "MLPCUDAExperts"
|
||
out_device: "cuda:0"
|
||
recursive: False # Don't recursively inject submodules of this module
|
||
</code></pre>
|
||
<p>But for two GPUs, we need to set the device for each module in the model.</p>
|
||
<pre><code class="language-yaml"># allcate 0-29 layers‘s out_device to cuda:0
|
||
- match:
|
||
name: "^model\\.layers\\.(0|[1-9]|[12][0-9])\\.mlp\\.experts$"
|
||
replace:
|
||
class: ktransformers.operators.experts.KTransformersExperts # custom MoE Kernel with expert paralleism
|
||
kwargs:
|
||
generate_device: "cpu"
|
||
generate_op: "KExpertsCPU"
|
||
out_device: "cuda:0"
|
||
recursive: False # don't recursively inject submodules of this module
|
||
|
||
# allocate 30-59 layers‘s out_device to cuda:1
|
||
- match:
|
||
name: "^model\\.layers\\.([345][0-9])\\.mlp\\.experts$"
|
||
replace:
|
||
class: ktransformers.operators.experts.KTransformersExperts # custom MoE Kernel with expert paralleism
|
||
kwargs:
|
||
generate_device: "cpu"
|
||
generate_op: "KExpertsCPU"
|
||
out_device: "cuda:1"
|
||
recursive: False # don't recursively inject submodules of this module
|
||
</code></pre>
|
||
<p>For other modules, we can set the device in the same way.</p>
|
||
<h1 id="how-to-fully-utilize-multi-gpus-vram"><a class="header" href="#how-to-fully-utilize-multi-gpus-vram">How to fully utilize multi-GPU's VRAM</a></h1>
|
||
<p>When you have multiple GPUs, you can fully utilize the VRAM of each GPU by moving more weights to the GPU.</p>
|
||
<p>For example, for DeepSeekV2-Chat, we can move the weights of the experts to the GPU.</p>
|
||
<p>For example, the yaml for two GPUs is:</p>
|
||
<pre><code class="language-yaml">- match:
|
||
name: "^model\\.layers\\.(0|[1-9]|[12][0-9])\\.mlp\\.experts$"
|
||
replace:
|
||
class: ktransformers.operators.experts.KTransformersExperts
|
||
kwargs:
|
||
generate_device: "cpu"
|
||
generate_op: "KExpertsCPU"
|
||
out_device: "cuda:0"
|
||
recursive: False
|
||
</code></pre>
|
||
<p>But we got extra 60GB VRAM on cuda:0, we can move experts in layer 4~8 to cuda:0.</p>
|
||
<pre><code class="language-yaml"># Add new rule before old rule.
|
||
- match:
|
||
name: "^model\\.layers\\.([4-8])\\.mlp\\.experts$" # inject experts in layer 4~8 as marlin expert
|
||
replace:
|
||
class: ktransformers.operators.experts.KTransformersExperts
|
||
kwargs:
|
||
generate_device: "cuda:0"
|
||
generate_op: "KExpertsMarlin"
|
||
recursive: False
|
||
|
||
- match:
|
||
name: "^model\\.layers\\.(0|[1-9]|[12][0-9])\\.mlp\\.experts$"
|
||
replace:
|
||
class: ktransformers.operators.experts.KTransformersExperts
|
||
kwargs:
|
||
generate_device: "cpu"
|
||
generate_op: "KExpertsCPU"
|
||
out_device: "cuda:0"
|
||
recursive: False
|
||
</code></pre>
|
||
<p>Adjust the layer range as you want. Note that:</p>
|
||
<ul>
|
||
<li>The loading speed will be significantly slower for each expert moved to the GPU.</li>
|
||
<li>You have to close the cuda graph if you want to move the experts to the GPU.</li>
|
||
<li>For DeepSeek-R1/V3, each expert moved to the GPU will consume approximately 6GB of VRAM.</li>
|
||
<li>The first matched rule in yaml will be applied. For example, if you have two rules that match the same layer, only the first rule's replacement will be valid.</li>
|
||
</ul>
|
||
<div style="break-before: page; page-break-before: always;"></div><h1 id="fp8-linear-kernel-for-deepseek-v3r1"><a class="header" href="#fp8-linear-kernel-for-deepseek-v3r1">FP8 Linear Kernel for DeepSeek-V3/R1</a></h1>
|
||
<h2 id="overview"><a class="header" href="#overview">Overview</a></h2>
|
||
<p>The DeepSeek-AI team provides FP8 safetensors for DeepSeek-R1/V3 models. We achieve performance optimization through the following works:</p>
|
||
<ul>
|
||
<li><strong>FP8 GPU Kernel Integration</strong>: FP8 linear layer acceleration kernels integrated in KTransformers</li>
|
||
<li><strong>Hybrid Quantization Architecture</strong>:
|
||
<ul>
|
||
<li>Attention and Shared-Expert modules use FP8 precision (enhances computational accuracy)</li>
|
||
<li>Experts modules retain GGML quantization (GGUF format, reside in CPU to save GPU memory)</li>
|
||
</ul>
|
||
</li>
|
||
</ul>
|
||
<p>So those who are persuing the best performance can use the FP8 linear kernel for DeepSeek-V3/R1.</p>
|
||
<h2 id="key-features"><a class="header" href="#key-features">Key Features</a></h2>
|
||
<p>✅ Hybrid Precision Architecture (FP8 + GGML)<br>
|
||
✅ Memory Optimization (~19GB VRAM usage)</p>
|
||
<h2 id="quick-start"><a class="header" href="#quick-start">Quick Start</a></h2>
|
||
<h3 id="using-pre-merged-weights"><a class="header" href="#using-pre-merged-weights">Using Pre-Merged Weights</a></h3>
|
||
<p>Pre-merged weights are available on Hugging Face:<br>
|
||
<a href="https://huggingface.co/KVCache-ai/DeepSeek-V3">KVCache-ai/DeepSeek-V3-GGML-FP8-Hybrid</a><br>
|
||
<a href="https://huggingface.co/KVCache-ai/DeepSeek-R1">KVCache-ai/DeepSeek-R1-GGML-FP8-Hybrid</a></p>
|
||
<blockquote>
|
||
<p>Please confirm the weights are fully uploaded before downloading. The large file size may extend Hugging Face upload time.</p>
|
||
</blockquote>
|
||
<p>Download Pre-Merged Weights</p>
|
||
<pre><code class="language-shell">pip install -U huggingface_hub
|
||
|
||
# Optional: Use HF Mirror for faster downloads in special area.
|
||
# export HF_ENDPOINT=https://hf-mirror.com
|
||
|
||
huggingface-cli download --resume-download KVCache-ai/DeepSeek-V3-GGML-FP8-Hybrid --local-dir <local_dir>
|
||
</code></pre>
|
||
<h3 id="using-merge-scripts"><a class="header" href="#using-merge-scripts">Using merge scripts</a></h3>
|
||
<p>If you got local DeepSeek-R1/V3 fp8 safetensors and gguf weights(eg.q4km), you can merge them using the following scripts.</p>
|
||
<pre><code class="language-shell">python merge_tensors/merge_safetensor_gguf.py \
|
||
--safetensor_path <fp8_safetensor_path> \
|
||
--gguf_path <gguf_folder_path> \
|
||
--output_path <merged_output_path>
|
||
</code></pre>
|
||
<ul>
|
||
<li><code>--safetensor_path</code>: input path of safetensor file(<a href="https://huggingface.co/deepseek-ai/DeepSeek-V3/tree/main">Download</a>).</li>
|
||
<li><code>--gguf_path</code>: input path of gguf folder (<a href="https://huggingface.co/unsloth/DeepSeek-V3-GGUF/tree/main/DeepSeek-V3-Q4_K_M">Download</a>).</li>
|
||
<li><code>--output_path</code>: output path of merged file.</li>
|
||
</ul>
|
||
<h3 id="execution-notes"><a class="header" href="#execution-notes">Execution Notes</a></h3>
|
||
<p>Launch local_chat.py with custom quantized experts</p>
|
||
<pre><code class="language-shell">python ktransformers/local_chat.py \
|
||
--model_path deepseek-ai/DeepSeek-V3 \
|
||
--gguf_path <merged_weights_folder> \
|
||
--optimize_config_path ktransformers/optimize/optimize_rules/DeepSeek-V3-Chat-fp8-linear-ggml-experts.yaml \
|
||
--cpu_infer <cpu_cores + 1>
|
||
</code></pre>
|
||
<h2 id="notes"><a class="header" href="#notes">Notes</a></h2>
|
||
<p>⚠️ Hardware Requirements<br></p>
|
||
<ul>
|
||
<li>Recommended minimum 19GB available VRAM for FP8 kernel.</li>
|
||
<li>Requires GPU with FP8 support (e.g., 4090)</li>
|
||
</ul>
|
||
<p>⏳ First-Run Optimization
|
||
JIT compilation causes longer initial execution (subsequent runs retain optimized speed).</p>
|
||
<p>🔄 Temporary Interface<br>
|
||
Current weight loading implementation is provisional - will be refined in future versions</p>
|
||
<p>📁 Path Specification<br>
|
||
Despite hybrid quantization, merged weights are stored as .safetensors - pass the containing folder path to <code>--gguf_path</code></p>
|
||
<div style="break-before: page; page-break-before: always;"></div><h1 id="rocm-support-for-ktransformers-beta"><a class="header" href="#rocm-support-for-ktransformers-beta">ROCm Support for ktransformers (Beta)</a></h1>
|
||
<h2 id="introduction-1"><a class="header" href="#introduction-1">Introduction</a></h2>
|
||
<h3 id="overview-1"><a class="header" href="#overview-1">Overview</a></h3>
|
||
<p>In our effort to expand GPU architecture support beyond NVIDIA, we are excited to introduce <strong>AMD GPU support through ROCm</strong> in ktransformers (Beta release). This implementation has been tested and developed using EPYC 9274F processors and AMD Radeon 7900xtx GPUs.</p>
|
||
<h2 id="installation-guide"><a class="header" href="#installation-guide">Installation Guide</a></h2>
|
||
<h3 id="1-install-rocm-driver"><a class="header" href="#1-install-rocm-driver">1. Install ROCm Driver</a></h3>
|
||
<p>Begin by installing the ROCm drivers for your AMD GPU:</p>
|
||
<ul>
|
||
<li><a href="https://rocm.docs.amd.com/projects/radeon/en/latest/docs/install/native_linux/install-radeon.html">Official ROCm Installation Guide for Radeon GPUs</a></li>
|
||
</ul>
|
||
<h3 id="2-set-up-conda-environment"><a class="header" href="#2-set-up-conda-environment">2. Set Up Conda Environment</a></h3>
|
||
<p>We recommend using Miniconda3/Anaconda3 for environment management:</p>
|
||
<pre><code class="language-bash"># Download Miniconda
|
||
wget https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh
|
||
|
||
# Create environment
|
||
conda create --name ktransformers python=3.11
|
||
conda activate ktransformers
|
||
|
||
# Install required libraries
|
||
conda install -c conda-forge libstdcxx-ng
|
||
|
||
# Verify GLIBCXX version (should include 3.4.32)
|
||
strings ~/anaconda3/envs/ktransformers/lib/libstdc++.so.6 | grep GLIBCXX
|
||
</code></pre>
|
||
<blockquote>
|
||
<p><strong>Note:</strong> Adjust the Anaconda path if your installation directory differs from <code>~/anaconda3</code></p>
|
||
</blockquote>
|
||
<h3 id="3-install-pytorch-for-rocm"><a class="header" href="#3-install-pytorch-for-rocm">3. Install PyTorch for ROCm</a></h3>
|
||
<p>Install PyTorch with ROCm 6.2.4 support:</p>
|
||
<pre><code class="language-bash">pip3 install torch torchvision torchaudio \
|
||
--index-url https://download.pytorch.org/whl/rocm6.2.4
|
||
pip3 install packaging ninja cpufeature numpy
|
||
</code></pre>
|
||
<blockquote>
|
||
<p><strong>Tip:</strong> For other ROCm versions, visit <a href="https://pytorch.org/get-started/previous-versions/">PyTorch Previous Versions</a></p>
|
||
</blockquote>
|
||
<h3 id="4-build-ktransformers"><a class="header" href="#4-build-ktransformers">4. Build ktransformers</a></h3>
|
||
<pre><code class="language-bash"># Clone repository
|
||
git clone https://github.com/kvcache-ai/ktransformers.git
|
||
cd ktransformers
|
||
git submodule update --init
|
||
|
||
# Optional: Compile web interface
|
||
# See: api/server/website.md
|
||
|
||
# Install dependencies
|
||
bash install.sh
|
||
</code></pre>
|
||
<h2 id="running-deepseek-r1-models"><a class="header" href="#running-deepseek-r1-models">Running DeepSeek-R1 Models</a></h2>
|
||
<h3 id="configuration-for-24gb-vram-gpus"><a class="header" href="#configuration-for-24gb-vram-gpus">Configuration for 24GB VRAM GPUs</a></h3>
|
||
<p>Use our optimized configuration for constrained VRAM:</p>
|
||
<pre><code class="language-bash">python ktransformers/local_chat.py \
|
||
--model_path deepseek-ai/DeepSeek-R1 \
|
||
--gguf_path <path_to_gguf_files> \
|
||
--optimize_config_path ktransformers/optimize/optimize_rules/rocm/DeepSeek-V3-Chat.yaml \
|
||
--cpu_infer <cpu_cores + 1>
|
||
</code></pre>
|
||
<blockquote>
|
||
<p><strong>Beta Note:</strong> Current Q8 linear implementation (Marlin alternative) shows suboptimal performance. Expect optimizations in future releases.</p>
|
||
</blockquote>
|
||
<h3 id="configuration-for-40gb-vram-gpus"><a class="header" href="#configuration-for-40gb-vram-gpus">Configuration for 40GB+ VRAM GPUs</a></h3>
|
||
<p>For better performance on high-VRAM GPUs:</p>
|
||
<ol>
|
||
<li>
|
||
<p>Modify <code>DeepSeek-V3-Chat.yaml</code>:</p>
|
||
<pre><code class="language-yaml"># Replace all instances of:
|
||
KLinearMarlin → KLinearTorch
|
||
</code></pre>
|
||
</li>
|
||
<li>
|
||
<p>Execute with:</p>
|
||
<pre><code class="language-bash">python ktransformers/local_chat.py \
|
||
--model_path deepseek-ai/DeepSeek-R1 \
|
||
--gguf_path <path_to_gguf_files> \
|
||
--optimize_config_path <modified_yaml_path> \
|
||
--cpu_infer <cpu_cores + 1>
|
||
</code></pre>
|
||
</li>
|
||
</ol>
|
||
<blockquote>
|
||
<p><strong>Tip:</strong> If you got 2 * 24GB AMD GPUS, you may also do the same modify and run <code>ktransformers/optimize/optimize_rules/DeepSeek-V3-Chat-multi-gpu.yaml</code> instead.</p>
|
||
</blockquote>
|
||
<h2 id="known-limitations"><a class="header" href="#known-limitations">Known Limitations</a></h2>
|
||
<ul>
|
||
<li>Marlin operations not supported on ROCm platform</li>
|
||
<li>Current Q8 linear implementation shows reduced performance (Beta limitation)</li>
|
||
</ul>
|
||
<div style="break-before: page; page-break-before: always;"></div><ul>
|
||
<li>
|
||
<p><a href="en/KTransformers-Fine-Tuning_Developer-Technical-Notes.html#ktransformers-fine-tuning-x-llama-factory-integration-%E2%80%93-developer-technical-notes">KTransformers Fine-Tuning × LLaMA-Factory Integration – Developer Technical Notes</a></p>
|
||
</li>
|
||
<li>
|
||
<p><a href="en/KTransformers-Fine-Tuning_Developer-Technical-Notes.html#introduction">Introduction</a></p>
|
||
</li>
|
||
<li>
|
||
<p><a href="en/KTransformers-Fine-Tuning_Developer-Technical-Notes.html#overall-view-of-the-kt-fine-tuning-framework">Overall View of the KT Fine-Tuning Framework</a></p>
|
||
<ul>
|
||
<li><a href="en/KTransformers-Fine-Tuning_Developer-Technical-Notes.html#attention-lora--kt-coexist">Attention (LoRA + KT coexist)</a></li>
|
||
<li><a href="en/KTransformers-Fine-Tuning_Developer-Technical-Notes.html#moe-operator-encapsulation--backward">MoE (operator encapsulation + backward)</a>
|
||
<ul>
|
||
<li><a href="en/KTransformers-Fine-Tuning_Developer-Technical-Notes.html#encapsulation">Encapsulation</a></li>
|
||
<li><a href="en/KTransformers-Fine-Tuning_Developer-Technical-Notes.html#backward-cpu">Backward (CPU)</a></li>
|
||
</ul>
|
||
</li>
|
||
<li><a href="en/KTransformers-Fine-Tuning_Developer-Technical-Notes.html#multi-gpu-loadingtraining-placement-strategy-instead-of-dataparallel">Multi-GPU Loading/Training: Placement strategy instead of DataParallel</a></li>
|
||
</ul>
|
||
</li>
|
||
<li>
|
||
<p><a href="en/KTransformers-Fine-Tuning_Developer-Technical-Notes.html#kt-lora-fine-tuning-evaluation">KT-LoRA Fine-Tuning Evaluation</a></p>
|
||
<ul>
|
||
<li><a href="en/KTransformers-Fine-Tuning_Developer-Technical-Notes.html#setup">Setup</a></li>
|
||
<li><a href="en/KTransformers-Fine-Tuning_Developer-Technical-Notes.html#results">Results</a>
|
||
<ul>
|
||
<li><a href="en/KTransformers-Fine-Tuning_Developer-Technical-Notes.html#stylized-dialogue-catgirl-tone">Stylized Dialogue (CatGirl tone)</a></li>
|
||
<li><a href="en/KTransformers-Fine-Tuning_Developer-Technical-Notes.html#translational-style-benchmark-generative">Translational-Style benchmark (generative)</a></li>
|
||
<li><a href="en/KTransformers-Fine-Tuning_Developer-Technical-Notes.html#medical-vertical-benchmark-afrimed-saqmcq">Medical Vertical Benchmark (AfriMed-SAQ/MCQ)</a></li>
|
||
<li><a href="en/KTransformers-Fine-Tuning_Developer-Technical-Notes.html#limitations">Limitations</a></li>
|
||
</ul>
|
||
</li>
|
||
</ul>
|
||
</li>
|
||
<li>
|
||
<p><a href="en/KTransformers-Fine-Tuning_Developer-Technical-Notes.html#speed-tests">Speed Tests</a></p>
|
||
<ul>
|
||
<li><a href="en/KTransformers-Fine-Tuning_Developer-Technical-Notes.html#end-to-end-performance">End-to-End Performance</a></li>
|
||
<li><a href="en/KTransformers-Fine-Tuning_Developer-Technical-Notes.html#moe-compute-deepseek-v3-671b">MoE Compute (DeepSeek-V3-671B)</a></li>
|
||
<li><a href="en/KTransformers-Fine-Tuning_Developer-Technical-Notes.html#memory-footprint">Memory Footprint</a></li>
|
||
</ul>
|
||
</li>
|
||
<li>
|
||
<p><a href="en/KTransformers-Fine-Tuning_Developer-Technical-Notes.html#conclusion">Conclusion</a></p>
|
||
</li>
|
||
</ul>
|
||
<h1 id="ktransformers-fine-tuning--llama-factory-integration--developer-technical-notes"><a class="header" href="#ktransformers-fine-tuning--llama-factory-integration--developer-technical-notes">KTransformers Fine-Tuning × LLaMA-Factory Integration – Developer Technical Notes</a></h1>
|
||
<p><strong>MadSys Lab, KVCache-AI Team, Approaching AI, LLaMA-Factory Team</strong></p>
|
||
<h2 id="introduction-2"><a class="header" href="#introduction-2">Introduction</a></h2>
|
||
<p>Recent open-source LLMs—from DeepSeek-V3/R1 to Qwen-MoE and Kimi-K2—have surged in performance and scale. Yet due to <strong>compute and memory constraints</strong>, it is difficult for typical researchers to fine-tune trillion-parameter-class models. We therefore integrate <strong>KTransformers</strong> with <strong>LLaMA-Factory</strong> so that, with <strong>2–4 RTX 4090 GPUs</strong> and sufficient CPU memory, one can fine-tune ultra-large Mixture-of-Experts (MoE) models such as DeepSeek-671B.</p>
|
||
<p>This architecture bridges resource gaps, enabling <strong>local fine-tuning of ultra-large models</strong>, while also supporting <strong>efficient scenario customization</strong> at 14B/30B scales. We validate on stylized dialogue, Westernized translation tone, and medical Q&A, achieving rapid adaptation within hours.</p>
|
||
<p>Architecturally, LLaMA-Factory orchestrates data/config/training, LoRA insertion, and inference; KTransformers is a pluggable, high-performance operator backend that takes over Attention and MoE under the same training code, enabling <strong>GPU+CPU heterogeneity</strong> to accelerate training and reduce GPU memory.</p>
|
||
<p><img src="en/../assets/image-20251011010558909.png" alt="image-20251011010558909" /></p>
|
||
<p>We evaluated LoRA fine-tuning with HuggingFace default, Unsloth, and KTransformers backends (same settings and data). <strong>KTransformers</strong> is currently the only solution feasible on <strong>2–4×24GB 4090s</strong> for <strong>671B-scale MoE</strong>, and also shows higher throughput and lower GPU memory for 14B MoEs.</p>
|
||
<div class="table-wrapper"><table><thead><tr><th>Under LoRA (BF16) + <a href="https://github.com/mindsRiverPonder/LLM-practice">NekoQA-10K stylized dialogue</a></th><th>HuggingFace Backend</th><th>Unsloth Backend</th><th>KTransformers Backend</th></tr></thead><tbody>
|
||
<tr><td>[14B-DeepSeekV2-Lite] LoRA fine-tuning throughput</td><td>303.58 token/s</td><td>455.37 token/s</td><td>530.38 token/s</td></tr>
|
||
<tr><td>[14B-DeepSeekV2-Lite] GPU memory</td><td>32.12 GB</td><td>9.64 GB</td><td>6.08 GB</td></tr>
|
||
<tr><td>[671B-DeepSeekV3] LoRA fine-tuning throughput</td><td><font color='red'>Too Huge to run</font></td><td><font color='red'>NOT SUPPORT</font></td><td>40.35 token/s</td></tr>
|
||
<tr><td>[671B-DeepSeekV3] GPU memory (sum across GPUs)</td><td>theoretical 1400 GB †</td><td><font color='red'>NOT SUPPORT</font></td><td>70 GB †</td></tr>
|
||
</tbody></table>
|
||
</div>
|
||
<p>† The <strong>1400 GB</strong> is the <strong>theoretical</strong> FP16 full-resident footprint (not runnable). <strong>70 GB</strong> is the <strong>measured peak</strong> with KT (Attention on GPU + layered MoE offload).</p>
|
||
<p>From the table above, it can be seen that for the 14B model, the KTransformers backend achieves approximately 75% higher throughput than the default HuggingFace solution, while using only about one-fifth of the GPU memory. For the 671B model, both HuggingFace and Unsloth fail to run on a single 4090 GPU, whereas KTransformers is able to perform LoRA fine-tuning at 40 tokens/s, keeping the GPU memory usage within 70 GB.</p>
|
||
<p><img src="en/../assets/image-compare_model.png" alt="按照模型划分的对比图_02" /></p>
|
||
<h2 id="overall-view-of-the-kt-fine-tuning-framework"><a class="header" href="#overall-view-of-the-kt-fine-tuning-framework">Overall View of the KT Fine-Tuning Framework</a></h2>
|
||
<p>We detail how KTransformers takes over core operators in LLaMA-Factory’s fine-tuning framework to optimize Attention and MoE.</p>
|
||
<p>DeepSeek-V3/V2 MoE models comprise a small-parameter dense Attention part and a large-parameter sparse MoE part. For illustration, consider layer 2 of DeepSeek-V2-Lite-Chat (from which each layer includes both Attention and MoE). Attention compute and KV cache mainly reside on the GPU; the heavyweight MoE part is primarily executed on the CPU. We first cover <strong>Attention replacement and inheritance</strong>, then <strong>MoE encapsulation and backend interfacing</strong>, and finally <strong>multi-GPU placement</strong>.</p>
|
||
<h3 id="attention-lora--kt-coexist"><a class="header" href="#attention-lora--kt-coexist">Attention (LoRA + KT coexist)</a></h3>
|
||
<p>KTransformers provides operator injection (<code>BaseInjectedModule</code>), and PEFT provides LoRA layer insertion. For fine-tuning, we design <code>KTransformersLinearLora</code>, inheriting from both <code>KTransformersLinear</code> and <code>LoraLayer</code>:</p>
|
||
<ul>
|
||
<li><strong>Inheritance:</strong> <code>KTransformersLinearLora</code> retains KT’s high-performance paths (<code>prefill_linear</code>/<code>generate_linear</code>) while accepting LoRA parameters (<code>lora_A/lora_B</code>).</li>
|
||
<li><strong>Replacement:</strong> During preparation, we replace original <code>KTransformersLinear</code> layers (Q/K/V/O) with <code>KTransformersLinearLora</code>, preserving KT optimizations while enabling LoRA trainability.</li>
|
||
</ul>
|
||
<p><img src="en/../assets/image-20251016182810716.png" alt="image-20251016182810716" /></p>
|
||
<p>After replacement, LoRA is inserted at Q/K/V/O linear transforms (left), and <code>KTransformersLinearLora</code> contains both KT fast paths and LoRA matrices (right).</p>
|
||
<p><img src="en/../assets/image-20251016182920722.png" alt="image-20251016182920722" /></p>
|
||
<h3 id="moe-operator-encapsulation--backward"><a class="header" href="#moe-operator-encapsulation--backward">MoE (operator encapsulation + backward)</a></h3>
|
||
<h4 id="encapsulation"><a class="header" href="#encapsulation">Encapsulation</a></h4>
|
||
<p>Given large parameters and sparse compute, we encapsulate the expert computation as a <strong>differentiable black-box operator</strong>—transparent upstream, replaceable downstream.</p>
|
||
<ul>
|
||
<li><strong>Upstream (PyTorch graph):</strong> we register a custom Autograd Function so the MoE layer appears as <strong>a single node</strong>. In the left figure (red box), only <code>KSFTExpertsCPU</code> is visible; on the right, the unencapsulated graph expands routing, dispatch, and FFN experts. Encapsulation makes the MoE layer behave like a standard <code>nn.Module</code> with gradients.</li>
|
||
<li><strong>Downstream (backend):</strong> inside the Autograd Function, pybind11 calls C++ extensions for forward/backward. Multiple <strong>pluggable backends</strong> exist (AMX BF16/INT8; <strong>llamafile</strong>). The backend can be switched via YAML (e.g., <code>"backend": "AMXBF16"</code> vs. <code>"llamafile"</code>).</li>
|
||
</ul>
|
||
<p><img src="en/../assets/image-20250801174623919.png" alt="image-20250801174623919" /></p>
|
||
<h4 id="backward-cpu"><a class="header" href="#backward-cpu">Backward (CPU)</a></h4>
|
||
<p>MoE backward frequently needs the transposed weights $W^\top$. To avoid repeated runtime transposes, we <strong>precompute/cache</strong> $W^\top$ at load time (blue box). We also <strong>cache necessary intermediate activations</strong> (e.g., expert projections, red box) to reuse in backward and reduce recomputation. We provide backward implementations for <strong>llamafile</strong> and <strong>AMX (INT8/BF16)</strong>, with NUMA-aware optimizations.</p>
|
||
<img src="en/../assets/image-20251016182942726.png" alt="image-20251016182942726" style="zoom:33%;" />
|
||
<h3 id="multi-gpu-loadingtraining-placement-strategy-instead-of-dataparallel"><a class="header" href="#multi-gpu-loadingtraining-placement-strategy-instead-of-dataparallel">Multi-GPU Loading/Training: Placement strategy instead of DataParallel</a></h3>
|
||
<p>To lower <strong>per-GPU memory peaks</strong> on 2–4 GPUs, we use <strong>model parallelism + explicit placement</strong>, not DataParallel (which duplicates the whole model on each GPU).</p>
|
||
<p>Key changes:</p>
|
||
<ol>
|
||
<li><strong>KTrainer:</strong> takes over <code>.to(device)</code> to prevent “move whole model to a single GPU”. Using KT’s optimize-rule YAML, each layer declares <code>device: cuda:0/cuda:1/...</code> and is <strong>constructed directly on the target GPU</strong> (no extra copies).</li>
|
||
<li><strong>Disable automatic DataParallel:</strong> when <code>USE_KT=1</code>, we disable automatic DP wrappers from LLaMA-Factory/HF Trainer to avoid duplication and keep full control over sharding.</li>
|
||
<li><strong>Gradient aggregation:</strong> gradients are reduced to <code>cuda:0</code>. Intermediate activations stay local; only necessary tensors are transferred, cutting communication/activation overhead.</li>
|
||
</ol>
|
||
<p>Thus, we keep KT placement strategies under multi-GPU fine-tuning. Users choose a <code>kt_optimize_rule</code> with <code>multi-gpu</code>. For DeepSeek-671B, <code>DeepSeek-V3-Chat-sft-amx-multi-gpu.yaml</code> is a typical 2-GPU plan: KV/attention parts on each GPU; MoE experts sharded on CPU; both GPUs share the workload.</p>
|
||
<h2 id="kt-lora-fine-tuning-evaluation"><a class="header" href="#kt-lora-fine-tuning-evaluation">KT-LoRA Fine-Tuning Evaluation</a></h2>
|
||
<h3 id="setup"><a class="header" href="#setup">Setup</a></h3>
|
||
<p>LLaMA-Factory orchestration, KTransformers backend, LoRA (rank=8, α=32, dropout=0.1, BF16), <code>GAS=16</code>, <code>qlen=512</code>, with the same KT optimize rule as training. We evaluate (a) stylized dialogue transfer and (b) two <strong>small-scale representative</strong> benchmarks: Translational-Style (generative) and AfriMed-QA (medical vertical; <strong>SAQ</strong> and <strong>MCQ</strong>). AMX is enabled; GPUs: 2×48GB RTX 4090; CPU: Intel Xeon Platinum 8488C.</p>
|
||
<h3 id="results"><a class="header" href="#results">Results</a></h3>
|
||
<h4 id="stylized-dialogue-catgirl-tone-1"><a class="header" href="#stylized-dialogue-catgirl-tone-1">Stylized Dialogue (CatGirl tone)</a></h4>
|
||
<p>Dataset: <a href="https://zhuanlan.zhihu.com/p/1934983798233231689">NekoQA-10K</a>. The fine-tuned model consistently exhibits the target style (red boxes) versus neutral/rational base (blue). This shows <strong>KT-LoRA injects style features</strong> into the generation distribution with low GPU cost.</p>
|
||
<p><img src="en/../assets/image-20251016175848143.png" alt="image-20251016175848143" /></p>
|
||
<h4 id="translational-style-benchmark-generative"><a class="header" href="#translational-style-benchmark-generative">Translational-Style benchmark (generative)</a></h4>
|
||
<p>Dataset: <a href="https://github.com/Benson114/Translational-Style-ChatLLM">Translational-Style-ChatLLM</a>. Metrics: BLEU-1/2/3/4, ROUGE-1/2/L.</p>
|
||
<div class="table-wrapper"><table><thead><tr><th>Translational-Style dataset</th><th>BLEU-1</th><th>BLEU-2</th><th>BLEU-3</th><th>BLEU-4</th><th>ROUGE-1</th><th>ROUGE-2</th><th>ROUGE-L</th></tr></thead><tbody>
|
||
<tr><td>V2-Lite (no LoRA)</td><td>20.66</td><td>8.33</td><td>4.54</td><td>2.89</td><td>22.71</td><td>4.52</td><td>19.19</td></tr>
|
||
<tr><td><strong>KT-LoRA fine-tuned V2-Lite</strong></td><td><strong>35.41</strong></td><td><strong>22.44</strong></td><td><strong>15.42</strong></td><td><strong>11.18</strong></td><td><strong>42.03</strong></td><td><strong>18.38</strong></td><td><strong>33.10</strong></td></tr>
|
||
<tr><td>V3 base (no LoRA)</td><td>8.49</td><td>3.34</td><td>1.62</td><td>0.96</td><td>15.91</td><td>2.55</td><td>10.07</td></tr>
|
||
<tr><td><strong>KT-LoRA fine-tuned V3</strong></td><td><strong>37.02</strong></td><td><strong>23.70</strong></td><td><strong>16.21</strong></td><td><strong>11.49</strong></td><td><strong>43.43</strong></td><td><strong>18.96</strong></td><td><strong>34.54</strong></td></tr>
|
||
</tbody></table>
|
||
</div>
|
||
<p>As shown by the test results in the tables above, under a unified workflow and placement strategy, <strong>both model scales exhibit consistent gains after fine-tuning</strong>, supporting the usability and effectiveness of the “KT backend + LoRA fine-tuning” combination for generative style control. At the same time, this indicates that KT’s heterogeneous placement and operator optimizations can stably support small-sample adaptation in the style domain.</p>
|
||
<h4 id="medical-vertical-benchmark-afrimed-saqmcq"><a class="header" href="#medical-vertical-benchmark-afrimed-saqmcq">Medical Vertical Benchmark (AfriMed-SAQ/MCQ)</a></h4>
|
||
<p>The dataset adopts <a href="https://aclanthology.org/2025.acl-long.96/">AfriMed-QA</a> (ACL 2025), a domain-specific dataset for the medical field in Africa with strong scenario customization characteristics, comprising two formats—multiple-choice questions (MCQ) and short-answer questions (SAQ)—which in this case serve as the evaluation for vertical-domain fine-tuning. In terms of evaluation criteria, BLEU/ROUGE are used for SAQ, and Accuracy is used for MCQ.</p>
|
||
<div class="table-wrapper"><table><thead><tr><th>AfriMed-QA (SAQ)</th><th>BLEU-1</th><th>BLEU-2</th><th>BLEU-3</th><th>BLEU-4</th><th>ROUGE-1</th><th>ROUGE-2</th><th>ROUGE-L</th></tr></thead><tbody>
|
||
<tr><td>V2-Lite (no LoRA)</td><td>13.58</td><td>11.12</td><td>9.10</td><td>7.23</td><td>22.48</td><td>7.81</td><td>11.73</td></tr>
|
||
<tr><td><strong>KT-LoRA fine-tuned V2-Lite</strong></td><td><strong>35.90</strong></td><td><strong>27.63</strong></td><td><strong>22.99</strong></td><td><strong>19.15</strong></td><td><strong>35.25</strong></td><td><strong>17.50</strong></td><td><strong>28.44</strong></td></tr>
|
||
<tr><td>V3 base (no LoRA)</td><td>12.75</td><td>10.27</td><td>8.05</td><td>5.99</td><td>20.33</td><td>5.65</td><td>10.11</td></tr>
|
||
<tr><td><strong>KT-LoRA fine-tuned V3</strong></td><td><strong>42.42</strong></td><td><strong>34.12</strong></td><td><strong>28.95</strong></td><td><strong>24.54</strong></td><td><strong>41.97</strong></td><td><strong>22.37</strong></td><td><strong>33.28</strong></td></tr>
|
||
</tbody></table>
|
||
</div><div class="table-wrapper"><table><thead><tr><th>AfriMed-QA (MCQ)</th><th>Accuracy</th></tr></thead><tbody>
|
||
<tr><td>V2-Lite (no LoRA)</td><td>0.0645</td></tr>
|
||
<tr><td><strong>KT-LoRA fine-tuned V2-Lite</strong></td><td><strong>0.4812</strong></td></tr>
|
||
<tr><td>V3 base (no LoRA)</td><td>0.5833</td></tr>
|
||
<tr><td><strong>KT-LoRA fine-tuned V3</strong></td><td><strong>0.7930</strong></td></tr>
|
||
</tbody></table>
|
||
</div>
|
||
<p>As shown in the tables above, (1) DeepSeek-V3 (671B) after KT-LoRA fine-tuning achieves clearly higher performance than the fine-tuned DeepSeek-V2-Lite (14B) on both MCQ and SAQ, and it also surpasses the V3 base model. Within our small-scale setting, this preliminarily indicates that KT-LoRA fine-tuning of ultra-large-parameter models has practical significance in vertical domains.</p>
|
||
<p>(2) Across both SAQ/MCQ sub-tasks, KT-LoRA delivers consistent gains, indicating that—with KT’s heterogeneous placement and backend operator support—LoRA fine-tuning can effectively inject the key knowledge points of vertical domains such as medicine into the model.</p>
|
||
<h4 id="limitations"><a class="header" href="#limitations">Limitations</a></h4>
|
||
<p>At present, most of our testing is conducted on <strong>single datasets</strong> and at <strong>small scale</strong> (≤ 20k examples), with the goal of providing <strong>existence evidence of system effectiveness for KT-LoRA fine-tuning</strong>, rather than drawing generalized conclusions about algorithmic generalization or scaling laws. Our report primarily presents representative figures; to support stronger algorithmic claims, larger sample sizes, multi-lingual/multi-domain datasets, and multi-seed repeated experiments would be required—these are beyond the scope of this work.</p>
|
||
<p><strong>We also warmly welcome everyone to join the open-source LLaMA-Factory KT fine-tuning project. If you have additional test results, we especially welcome you to record them in the shared spreadsheet below, and to include the corresponding <code>kt_optimize_rule</code> files, dataset examples, training/evaluation YAMLs, and detailed GPU-memory and CPU configurations for community reference and reproducibility~!</strong></p>
|
||
<h3 id="speed-tests"><a class="header" href="#speed-tests">Speed Tests</a></h3>
|
||
<h4 id="end-to-end-performance-1"><a class="header" href="#end-to-end-performance-1">End-to-End Performance</a></h4>
|
||
<p><strong>Definitions</strong></p>
|
||
<p><code>step_time</code>:time per optimization step (tensor movement + Attention + MoE + others).</p>
|
||
<p><code>tokens_per_step = GAS × qlen</code>;<code>token/s = tokens_per_step / step_time</code>。 We use <code>GAS=16</code>, <code>qlen=512</code> → <code>tokens_per_step=8192</code>.</p>
|
||
<p><strong>Measured</strong></p>
|
||
<div class="table-wrapper"><table><thead><tr><th>Model</th><th>step_time (s)</th><th>tokens/step</th><th>token/s</th></tr></thead><tbody>
|
||
<tr><td>DeepSeek-V3-671B</td><td>203</td><td>8192</td><td><strong>40.35</strong></td></tr>
|
||
<tr><td>DeepSeek-V2-Lite-14B</td><td>36</td><td>8192</td><td><strong>227.6</strong></td></tr>
|
||
</tbody></table>
|
||
</div>
|
||
<h4 id="moe-compute-deepseek-v3-671b"><a class="header" href="#moe-compute-deepseek-v3-671b">MoE Compute (DeepSeek-V3-671B)</a></h4>
|
||
<p><strong>Theory</strong></p>
|
||
<ul>
|
||
<li>MoE per-layer, per-token FLOPs (forward+backward) approx.:
|
||
$$
|
||
\text{FLOPs}_{\text{per-layer, per-token}} \approx c \cdot k \cdot H \cdot I
|
||
$$</li>
|
||
</ul>
|
||
<p> with $k = 8$(Top-k),$H = 7168$(hidden size),$I = 2048$(intermediate size),$c\approx16$(≈6 forward + ≈10 backward matmuls)。</p>
|
||
<ul>
|
||
<li>Per-step across all MoE layers:
|
||
$$
|
||
\text{FLOPs}<em>{\text{per-step}} \approx c \cdot qlen \cdot k \cdot H \cdot I \cdot L</em>{\text{MoE}}
|
||
$$</li>
|
||
</ul>
|
||
<p> Plugging $c=16, qlen=512, k=8, H=7168, I=2048, L_{MoE}=58$,$\text{FLOPs}_{\text{per-step}} \approx 55.8\ \text{TFLOPs}$.</p>
|
||
<p><strong>Measured (MoE TFLOPS on CPU)</strong></p>
|
||
<p>If the <strong>MoE-only</strong> time per step is <code>t_moe</code> (seconds), $\text{TFLOPS} = \text{FLOPs}_{\text{per-step}} / \text{step_per_second}.$</p>
|
||
<p>Use MoE-phase time, not full <code>step_time</code>, to get MoE throughput.</p>
|
||
<div class="table-wrapper"><table><thead><tr><th>TFLOPS</th><th>Forward</th><th>Backward</th></tr></thead><tbody>
|
||
<tr><td>Average</td><td>17.55</td><td>18.41</td></tr>
|
||
</tbody></table>
|
||
</div>
|
||
<h3 id="memory-footprint"><a class="header" href="#memory-footprint">Memory Footprint</a></h3>
|
||
<ul>
|
||
<li>DeepSeek-V3 (671B; 58 MoE layers out of 61): ~<strong>70 GB</strong> total GPU, ~<strong>1.2–1.3 TB</strong> host memory.</li>
|
||
<li>DeepSeek-V2-Lite (14B; 26 MoE layers out of 27): ~<strong>5 GB</strong> GPU, ~<strong>30 GB</strong> host memory.</li>
|
||
</ul>
|
||
<h2 id="conclusion-1"><a class="header" href="#conclusion-1">Conclusion</a></h2>
|
||
<p>Integrating <strong>KTransformers LoRA</strong> with <strong>LLaMA-Factory</strong> provides a practical path to efficiently train and deploy MoE LLMs. KT contributes placement strategies and operator optimizations (DeepSeek/Qwen/Kimi support with AMX-accelerated kernels), and LoRA enables customization with very low GPU memory; LLaMA-Factory supplies a coherent user-level interface.</p>
|
||
<p>This means even tens-to-hundreds-of-billion-parameter MoE models can be fine-tuned and served with low latency on ordinary hardware. The approach balances <strong>memory savings</strong>, <strong>speed</strong>, and <strong>usability</strong>, turning ultra-large models into tools that developers can actually wield.</p>
|
||
<div style="break-before: page; page-break-before: always;"></div><!-- omit in toc -->
|
||
<h1 id="see-the-issue-faq-page"><a class="header" href="#see-the-issue-faq-page">see the issue <a href="https://github.com/kvcache-ai/ktransformers/issues/1608">FAQ page</a></a></h1>
|
||
|
||
</main>
|
||
|
||
<nav class="nav-wrapper" aria-label="Page navigation">
|
||
<!-- Mobile navigation buttons -->
|
||
|
||
|
||
<div style="clear: both"></div>
|
||
</nav>
|
||
</div>
|
||
</div>
|
||
|
||
<nav class="nav-wide-wrapper" aria-label="Page navigation">
|
||
|
||
</nav>
|
||
|
||
</div>
|
||
|
||
|
||
|
||
|
||
<script>
|
||
window.playground_copyable = true;
|
||
</script>
|
||
|
||
<script src="ace.js"></script>
|
||
<script src="mode-rust.js"></script>
|
||
<script src="editor.js"></script>
|
||
<script src="theme-dawn.js"></script>
|
||
<script src="theme-tomorrow_night.js"></script>
|
||
|
||
<script src="elasticlunr.min.js"></script>
|
||
<script src="mark.min.js"></script>
|
||
<script src="searcher.js"></script>
|
||
|
||
<script src="clipboard.min.js"></script>
|
||
<script src="highlight.js"></script>
|
||
<script src="book.js"></script>
|
||
|
||
<!-- Custom JS scripts -->
|
||
|
||
<script>
|
||
window.addEventListener('load', function() {
|
||
window.setTimeout(window.print, 100);
|
||
});
|
||
</script>
|
||
|
||
|
||
</div>
|
||
</body>
|
||
</html>
|