device: NVIDIA GB300
shape: experts=8, tokens/expert=4096, M=32768, K=8192, N=4096
dtype: torch.bfloat16
torch: 2.12.0a0+5aff3928d8.nv26.05
transformer_engine: 2.15.0+42b84005
nvidia-cudnn-frontend: 1.23.0
nvidia-cutlass-dsl: 4.4.1
timing mode: CUDA graph replay

activation: swiglu
shape: experts=8, tokens/expert=4096, M=32768, K=8192, N=4096
fused GEMM N=8192, fused C input N=None, activation output N=4096
fused cuDNN avg: 1.319 ms
unfused TE avg:  1.695 ms
speedup vs TE:   1.28x
fused samples:   [1.233, 1.243, 1.374, 1.378, 1.369]
TE samples:      [1.82, 1.612, 1.603, 1.703, 1.738]

activation: dswiglu
shape: experts=8, tokens/expert=4096, M=32768, K=8192, N=4096
fused GEMM N=4096, fused C input N=8192, activation output N=8192
fused cuDNN avg: 0.740 ms
unfused TE avg:  1.549 ms
speedup vs TE:   2.09x
fused samples:   [0.71, 0.716, 0.718, 0.763, 0.793]
TE samples:      [1.856, 1.512, 1.464, 1.457, 1.454]

activation: srelu
shape: experts=8, tokens/expert=4096, M=32768, K=8192, N=4096
fused GEMM N=4096, fused C input N=None, activation output N=4096
fused cuDNN avg: 0.675 ms
unfused TE avg:  0.873 ms
speedup vs TE:   1.29x
fused samples:   [0.653, 0.654, 0.686, 0.677, 0.705]
TE samples:      [0.942, 0.921, 0.85, 0.842, 0.811]

activation: dsrelu
shape: experts=8, tokens/expert=4096, M=32768, K=8192, N=4096
fused GEMM N=4096, fused C input N=4096, activation output N=4096
fused cuDNN avg: 0.686 ms
unfused TE avg:  1.163 ms
speedup vs TE:   1.70x
fused samples:   [0.659, 0.666, 0.677, 0.688, 0.743]
TE samples:      [1.343, 1.167, 1.116, 1.096, 1.096]

activation: geglu
shape: experts=8, tokens/expert=4096, M=32768, K=8192, N=4096
fused GEMM N=8192, fused C input N=None, activation output N=4096
fused cuDNN avg: 1.317 ms
unfused TE avg:  1.631 ms
speedup vs TE:   1.24x
fused samples:   [1.231, 1.245, 1.344, 1.376, 1.389]
TE samples:      [1.735, 1.626, 1.58, 1.65, 1.565]

activation: dgeglu
shape: experts=8, tokens/expert=4096, M=32768, K=8192, N=4096
fused GEMM N=4096, fused C input N=8192, activation output N=8192
fused cuDNN avg: 0.751 ms
unfused TE avg:  1.666 ms
speedup vs TE:   2.22x
fused samples:   [0.716, 0.717, 0.75, 0.774, 0.795]
TE samples:      [2.058, 1.595, 1.569, 1.557, 1.553]

summary:
activation   cudnn_ms      te_ms   te/cudnn
swiglu          1.319      1.695      1.28x
dswiglu         0.740      1.549      2.09x
srelu           0.675      0.873      1.29x
dsrelu          0.686      1.163      1.70x
geglu           1.317      1.631      1.24x
dgeglu          0.751      1.666      2.22x
