mirror of
https://github.com/kvcache-ai/ktransformers.git
synced 2026-10-18 14:43:29 +08:00
yyj
aa2cd3072f
* [feat](sft): add K2 MoE SFT backend
* [feat](sft): enable packed KGroup backward for K2 SFT
* [feat](serving): add KT LoRA to SGLang adapter converter
* [test]: cover KT SFT and composite LoRA serving paths
* [perf](bench): add reusable KT E2E benchmark harness
* [test](sft): add K2 MoE AMX and real Kimi load smoke cases
* [perf](bench): add K2 SFT packed backward profile harness
* [test](sglang): cover Qwen3.5 LoRA shape filters
* [perf](sft): add K2 forward profiling logs
* [docs](sft): clarify K2 packed backward flow
* [docs](sft): number K2 backward comment steps
* chore: mirror sap4 loader baseline
* Add K2 SFT profiling ranges
* Add backward NVTX ranges for K2 SFT profiling
* perf(sft): add rank8 LoRA backward vector paths
* perf(sft): reuse down LoRA bprop cache
* [perf]: add K2 RAWINT4 prefill mat-mat dispatch
* [perf] Refactor K2 RAWINT4 prefill accumulation
* [perf]: add blocked RAWINT4 prefill backend
Cherry-picked from be40f8e with local amx.py conflict resolution.
* Optimize blocked RAWINT4 decode vecmul
* perf(sft): checkpoint qj5090 packed backward baseline
* perf(sft): batch expert LoRA backward as matmul
* perf(sft): align K2 base backward with BF16 matmul
* Optimize K2 SFT backward workspace path
* docs(sft): hand off RAWINT4 KGroup build and Kimi configs
* [feat](kt-kernel): add Kimi RAWINT4 LoRA SFT kernels
* [feat](kt-kernel): integrate Kimi RAWINT4 LoRA SFT
* [fix](sft): preserve input gradients through frozen routers
Build the routing graph when either activations or router parameters require gradients, while respecting the ambient grad mode. Keep frozen router parameters frozen and reject detached routing weights whenever gradients are needed.
Correct the existing frozen-router expectation and cover input-gradient equivalence for linear, DeepSeek/Kimi and GLM routing, checkpoint reuse, grad modes and detached-output failures.
Validated 62 routing/shared-expert/checkpoint tests on PyTorch 2.9.1 and a two-step Kimi K2.5 RAWINT4 four-GPU S512 LoRA smoke with per-rank routing-input gradient audits.
* [perf](kt-kernel): accelerate RAWINT4 prefill and backward
Reuse decoded weight columns across prefill rows and prepack BF16 backward coefficients into bounded per-worker panels. Use AMX BF16 tiles when available, retaining the AVX512-BF16 path and frozen base weights.
Balance expert work, parallelize cache copies and router reductions, and correct the signed INT4 -8 dot-product boundary. Add a CPU replay benchmark and forward/backward panel regressions.
Validated on sap4 with Kimi K2.5, TP1/TP2 replay, AMX and AVX-512 multi-GPU training, gradient windows, and saved LoRA artifacts. User-facing clean builds and AMD hardware remain separate validation work.
* [fix](sft): limit default OpenMP pools to the CPU owner
Keep the affinity-visible physical-core budget on rank 0 and default non-owner distributed ranks to one thread. Preserve explicit generic and KT-specific thread overrides, including intentional single-thread settings.
Cover single-process behavior, non-owner ranks, repeated configuration, and explicit overrides. The 109 configuration, activation-policy, and backend-dispatch tests pass.
---------
Co-authored-by: yyj <yyj@qj5090.local>
Co-authored-by: djh <jd4136@columbia.edu>
100 lines
3.9 KiB
Python
100 lines
3.9 KiB
Python
# SPDX-License-Identifier: Apache-2.0
|
|
|
|
import importlib.util
|
|
import os
|
|
import sys
|
|
import types
|
|
from pathlib import Path
|
|
from unittest.mock import patch
|
|
|
|
PACKAGE_PATH = Path(__file__).resolve().parents[2] / "python" / "sft"
|
|
PACKAGE_NAME = "kt_sft_config_under_test"
|
|
package = types.ModuleType(PACKAGE_NAME)
|
|
package.__path__ = [str(PACKAGE_PATH)]
|
|
sys.modules[PACKAGE_NAME] = package
|
|
SPEC = importlib.util.spec_from_file_location(f"{PACKAGE_NAME}.config", PACKAGE_PATH / "config.py")
|
|
assert SPEC is not None and SPEC.loader is not None
|
|
config = importlib.util.module_from_spec(SPEC)
|
|
sys.modules[SPEC.name] = config
|
|
SPEC.loader.exec_module(config)
|
|
|
|
|
|
def test_detect_physical_cpu_count_deduplicates_smt_siblings():
|
|
topology = {
|
|
0: (0, 0),
|
|
1: (0, 1),
|
|
2: (0, 0),
|
|
3: (0, 1),
|
|
4: (1, 0),
|
|
5: (1, 0),
|
|
}
|
|
with (
|
|
patch.object(config, "_available_cpu_ids", return_value=set(topology)),
|
|
patch.object(config, "_read_cpu_topology", side_effect=topology.get),
|
|
):
|
|
assert config.detect_physical_cpu_count() == 3
|
|
|
|
|
|
def test_configure_omp_threads_replaces_accelerate_single_thread_default():
|
|
with (
|
|
patch.dict(os.environ, {"OMP_NUM_THREADS": "1", "RANK": "0", "WORLD_SIZE": "1"}, clear=False),
|
|
patch.object(config, "detect_physical_cpu_count", return_value=96),
|
|
patch.object(config, "_set_torch_num_threads") as set_torch_threads,
|
|
):
|
|
os.environ.pop("ACCELERATE_KT_OMP_NUM_THREADS", None)
|
|
assert config.configure_omp_threads() == 96
|
|
assert os.environ["OMP_NUM_THREADS"] == "96"
|
|
set_torch_threads.assert_called_once_with(96)
|
|
|
|
|
|
def test_configure_omp_threads_preserves_explicit_generic_value():
|
|
with (
|
|
patch.dict(os.environ, {"OMP_NUM_THREADS": "48"}, clear=False),
|
|
patch.object(config, "_set_torch_num_threads") as set_torch_threads,
|
|
):
|
|
os.environ.pop("ACCELERATE_KT_OMP_NUM_THREADS", None)
|
|
assert config.configure_omp_threads() == 48
|
|
set_torch_threads.assert_called_once_with(48)
|
|
|
|
|
|
def test_configure_omp_threads_supports_explicit_single_thread_override():
|
|
with (
|
|
patch.dict(
|
|
os.environ,
|
|
{"OMP_NUM_THREADS": "96", "ACCELERATE_KT_OMP_NUM_THREADS": "1"},
|
|
clear=False,
|
|
),
|
|
patch.object(config, "_set_torch_num_threads") as set_torch_threads,
|
|
):
|
|
assert config.configure_omp_threads() == 1
|
|
assert os.environ["OMP_NUM_THREADS"] == "1"
|
|
set_torch_threads.assert_called_once_with(1)
|
|
|
|
|
|
def test_configure_omp_threads_gives_machine_budget_only_to_cpu_owner():
|
|
for rank, expected in (("0", 96), ("1", 1), ("7", 1)):
|
|
with (
|
|
patch.dict(os.environ, {"OMP_NUM_THREADS": "1", "RANK": rank, "WORLD_SIZE": "8"}, clear=False),
|
|
patch.object(config, "detect_physical_cpu_count", return_value=96),
|
|
patch.object(config, "_set_torch_num_threads") as set_torch_threads,
|
|
):
|
|
os.environ.pop("ACCELERATE_KT_OMP_NUM_THREADS", None)
|
|
assert config.configure_omp_threads() == expected
|
|
assert config.configure_omp_threads() == expected
|
|
assert set_torch_threads.call_count == 2
|
|
set_torch_threads.assert_called_with(expected)
|
|
|
|
|
|
def test_configure_omp_threads_preserves_non_owner_explicit_overrides():
|
|
for explicit, generic, expected in ((None, "24", 24), ("12", "1", 12), ("1", "96", 1)):
|
|
with (
|
|
patch.dict(os.environ, {"OMP_NUM_THREADS": generic, "RANK": "3", "WORLD_SIZE": "8"}, clear=False),
|
|
patch.object(config, "_set_torch_num_threads") as set_torch_threads,
|
|
):
|
|
if explicit is None:
|
|
os.environ.pop("ACCELERATE_KT_OMP_NUM_THREADS", None)
|
|
else:
|
|
os.environ["ACCELERATE_KT_OMP_NUM_THREADS"] = explicit
|
|
assert config.configure_omp_threads() == expected
|
|
set_torch_threads.assert_called_once_with(expected)
|