---
title: '50912'
pageLayout: default
slug: thoughts/inference/vllm-prs/50912
permalink: https://aarnphm.xyz/thoughts/inference/vllm-prs/50912.md
generator:
  quartz: v4.6.0
  hostedProvider: Cloudflare
  baseUrl: aarnphm.xyz
full: https://aarnphm.xyz/llms-full.txt
---
# [#50912](https://github.com/aarnphm/aarnphm.github.io/issues/50912) \[Kimi K3 Perf\] option to shard the shared expert for non mega case, 16.98 GiB memory/GPU saved

[vllm-project/vllm#50912](https://github.com/vllm-project/vllm/pull/50912) · yewentao256 · MERGED · created 2026-08-03T19:49:46Z · merged 2026-08-04T18:00:45Z

## Purpose

A following up PR for [vllm-project/vllm#50656](https://github.com/vllm-project/vllm/pull/50656) @tlrmchlsmth

| config                                  | effectiveness |
| --------------------------------------- | ------------- |
| `VLLM_KIMI_K3_SHARD_SP_SHARED_EXPERT=0` | No            |
| non-mega、TP=4、DP=1                      | No            |
| non-mega、TP=4、DP=2、EP                   | Yes           |
| MegaMoE、TP>1、EP                         | Yes           |
| PP>1                                    | No            |

The update is safe as we make the shared experts overlap off, this might be a future optimization point

Recommend to use with PD disaggregation (decode node)

## Test

Mostly covered by previous PR [#50656](https://github.com/aarnphm/aarnphm.github.io/issues/50656)

For non-mega case can be seen in this AI-generated script



```bash
import gc
import os
import statistics

import torch
import torch.distributed as dist

os.environ["VLLM_KIMI_K3_SHARD_SP_SHARED_EXPERT"] = "1"

from vllm.config import VllmConfig, set_current_vllm_config  # noqa: E402
from vllm.distributed import (  # noqa: E402
    get_tp_group,
    init_distributed_environment,
    initialize_model_parallel,
)
from vllm.models.kimi_k3.nvidia.model import KimiMLP  # noqa: E402
from vllm.utils.torch_utils import set_default_torch_dtype  # noqa: E402

HIDDEN = 7168
INTERMEDIATE = 6144
MOE_LAYERS = 92
TOKENS = (8, 32, 128, 256, 512, 1024, 2048)
WARMUP = 20
REPEATS = 50
SAMPLES = 10


def max_rank(value: float, device: torch.device) -> float:
    result = torch.tensor(value, dtype=torch.float64, device=device)
    dist.all_reduce(result, op=dist.ReduceOp.MAX, group=get_tp_group().device_group)
    return result.item()


def make_layer(sharded: bool, device: torch.device) -> KimiMLP:
    with set_default_torch_dtype(torch.bfloat16):
        layer = KimiMLP(
            hidden_size=HIDDEN,
            intermediate_size=INTERMEDIATE,
            hidden_act="situ",
            reduce_results=False,
            use_sequence_parallel=True,
            can_shard_sequence_parallel=sharded,
            activation_situ_beta=4.0,
            activation_situ_linear_beta=25.0,
        ).to(device)
    with torch.no_grad():
        for parameter in layer.parameters():
            parameter.zero_()
    assert layer.shard_sequence_parallel is sharded
    return layer


def benchmark(
    layer: KimiMLP,
    global_tokens: int,
    device: torch.device,
    cpu_group: dist.ProcessGroup,
) -> tuple[int, float]:
    tp = dist.get_world_size()
    local_tokens = (global_tokens + tp - 1) // tp
    x = torch.ones((local_tokens, HIDDEN), dtype=torch.bfloat16, device=device)

    stream = torch.cuda.Stream()
    stream.wait_stream(torch.cuda.current_stream())
    with torch.cuda.stream(stream):
        for _ in range(3):
            layer(x)
    stream.synchronize()
    graph = torch.cuda.CUDAGraph()
    with torch.cuda.graph(graph, stream=stream):
        layer(x)

    for _ in range(WARMUP):
        graph.replay()
    torch.accelerator.synchronize()

    times = []
    start = torch.cuda.Event(enable_timing=True)
    end = torch.cuda.Event(enable_timing=True)
    for _ in range(SAMPLES):
        dist.barrier(group=cpu_group)
        start.record()
        for _ in range(REPEATS):
            graph.replay()
        end.record()
        end.synchronize()
        times.append(max_rank(start.elapsed_time(end) / REPEATS, device))
    return local_tokens, statistics.median(times)


def run_mode(
    sharded: bool,
    device: torch.device,
    cpu_group: dist.ProcessGroup,
) -> tuple[float, dict[int, tuple[int, float]]]:
    layer = make_layer(sharded, device)
    weight_mib = sum(p.numel() * p.element_size() for p in layer.parameters()) / 2**20
    results = {tokens: benchmark(layer, tokens, device, cpu_group) for tokens in TOKENS}
    del layer
    gc.collect()
    torch.accelerator.empty_cache()
    return weight_mib, results


def main() -> None:
    rank = int(os.environ["RANK"])
    tp = int(os.environ["WORLD_SIZE"])
    local_rank = int(os.environ["LOCAL_RANK"])
    if tp <= 1:
        raise RuntimeError("Run with torchrun and TP > 1")

    device = torch.device("cuda", local_rank)
    torch.accelerator.set_device_index(local_rank)
    init_distributed_environment()
    initialize_model_parallel(tensor_model_parallel_size=tp)
    cpu_group = dist.new_group(backend="gloo")

    rep_weight, rep = run_mode(False, device, cpu_group)
    shard_weight, shard = run_mode(True, device, cpu_group)

    if rank == 0:
        saved = rep_weight - shard_weight
        print(f"Kimi-K3 shared expert BF16, TP={tp}")
        print(
            f"Weight/GPU: {rep_weight:.1f} -> {shard_weight:.1f} MiB; "
            f"save {saved:.1f} MiB/layer ({saved / rep_weight:.1%})"
        )
        print(f"{MOE_LAYERS} MoE layers save {saved * MOE_LAYERS / 1024:.2f} GiB/GPU\n")
        print(f"{'global/local tokens':>19}  {'rep ms':>8}  {'shard ms':>8}  {'change':>8}")
        for tokens in TOKENS:
            local_tokens, rep_ms = rep[tokens]
            _, shard_ms = shard[tokens]
            print(
                f"{tokens:>10}/{local_tokens:<8}  {rep_ms:8.4f}  "
                f"{shard_ms:8.4f}  {shard_ms / rep_ms - 1:+7.1%}"
            )

    dist.destroy_process_group(cpu_group)
    dist.destroy_process_group()


if __name__ == "__main__":
    with set_current_vllm_config(VllmConfig()):
        main()

```

And we will have



```bash
torchrun --standalone --nproc-per-node=4 /home/yewentao256/benchmark_kimi_k3_shared_expert_sp.py

Kimi-K3 shared expert BF16, TP=4
Weight/GPU: 252.0 -> 63.0 MiB; save 189.0 MiB/layer (75.0%)
92 MoE layers save 16.98 GiB/GPU

global/local tokens    rep ms  shard ms    change
         8/2           0.0540    0.0328   -39.3%
        32/8           0.0524    0.0401   -23.5%
       128/32          0.0555    0.0476   -14.1%
       256/64          0.0536    0.0590   +10.0%
       512/128         0.0615    0.0800   +30.2%
      1024/256         0.0692    0.1213   +75.2%
      2048/512         0.1089    0.2562  +135.3%
```

---

## comments

