correct
47_Conv3d_Mish_Tanh.py
started
2026-05-05T00:41:53Z
timingkernelbench.timing
kernel mean / median / std
2.37 / — / 0.11 μs
ref mean / median / std
2.57 / — / 0.01 μs
numerical precisioneval_kernel_against_ref
gpu memoryextended_metrics.measure_memory
kernel launches / fusiontorch.profiler
fusion ratio (ref / kernel)
1.29x
energyNVML (pynvml)
ratio (ref / kernel)
2.50x
SOL (speed-of-light)nsight_hardware_counters
roofline / occupancynsight_hardware_counters
dram bandwidth
7676.52 GB/s
warp stalls (top 5)nsight
reference PyTorch source (the prompt the agent sees)
import torch
import torch.nn as nn
class Model(nn.Module):
"""
Model that performs a 3D convolution, applies Mish activation, and then applies Tanh activation.
"""
def __init__(self, in_channels, out_channels, kernel_size, stride=1, padding=0):
super(Model, self).__init__()
self.conv = nn.Conv3d(in_channels, out_channels, kernel_size, stride=stride, padding=padding)
def forward(self, x):
"""
Args:
x (torch.Tensor): Input tensor of shape (batch_size, in_channels, D, H, W).
Returns:
torch.Tensor: Output tensor of shape (batch_size, out_channels, D', H', W').
"""
x = self.conv(x)
x = torch.nn.functional.mish(x)
x = torch.tanh(x)
return x
batch_size = 16
in_channels = 32
out_channels = 64
D, H, W = 32, 64, 64
kernel_size = 3
def get_inputs():
return [torch.rand(batch_size, in_channels, D, H, W)]
def get_init_inputs():
return [in_channels, out_channels, kernel_size]turn 034.3s turn · 34.3s LLM · 0.0s tools · FINAL
reasoning
**Considering speed optimization**
I’m thinking that the speed might be quicker than the reference due to using fused mish and tanh in place, which avoids allocating two separate kernels. The reference process seems to be convoluted into mish (possibly combining softplus and tanh into one kernel). It looks like the torch functional mish could also work with one kernel, while the tanh might require a second one. So, this fusion might help optimize performance.
call → submit_kernel
callsubmit_kernel
{
"kernel_code": "import torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nfrom torch.utils.cpp_extension import load_inline\n\n\n_cpp_src = r\"\"\"\n#include <torch/extension.h>\n\ntorch::Tensor mish_tanh_inplace(torch::Tensor x);\n\"\"\"\n\n_cuda_src = r\"\"\"\n#include <torch/extension.h>\n#include <ATen/cuda/CUDAContext.h>\n#include <cuda.h>\n#include <cuda_runtime.h>\n\n__device__ __forceinline__ float fast_tanh_f(float x) {\n if (x > 10.0f) return 1.0f;\n if (x < -10.0f) return -1.0f;\n float ax = fabsf(x);\n float e = __expf(-2.0f * ax);\n float y = (1.0f - e) / (1.0f + e);\n return copysignf(y, x);\n}\n\n__device__ __forceinline__ float softplus_f(float x) {\n // Stable softplus. The tails also avoid unnecessary log work.\n if (x > 20.0f) return x;\n if (x < -20.0f) return __expf(x);\n return log1pf(__expf(x));\n}\n\n__global__ void mish_tanh_kernel(float* __restrict__ p, int64_t n) {\n int64_t idx = (int64_t)blockIdx.x * blockDim.x + threadIdx.x;\n int64_t stride = (int64_t)blockDim.x * gridDim.x;\n for (; idx < n; idx += stride) {\n float x = p[idx];\n float sp = softplus_f(x);\n float mish = x * fast_tanh_f(sp);\n p[idx] = fast_tanh_f(mish);\n }\n}\n\ntorch::Tensor mish_tanh_inplace(torch::Tensor x) {\n TORCH_CHECK(x.is_cuda(), \"input must be a CUDA tensor\");\n TORCH_CHECK(x.scalar_type() == at::kFloat, \"only float32 is supported\");\n TORCH_CHECK(x.is_contiguous(), \"input must be contiguous\");\n const int64_t n = x.numel();\n if (n == 0) return x;\n const int threads = 256;\n int blocks = (int)((n + threads - 1) / threads);\n // Enough blocks to cover the device while keeping launch overhead small.\n if (blocks > 65535) blocks = 65535;\n mish_tanh_kernel<<<blocks, threads, 0, at::cuda::getCurrentCUDAStream()>>>(x.data_ptr<float>(), n);\n return x;\n}\n\"\"\"\n\n_ext = load_inline(\n name=\"mish_tanh_fused_ext_v1\",\n cpp_sources=_cpp_src,\n cuda_sources=_cuda_src,\n functions=[\"mish_tanh_inplace\"],\n extra_cuda_cflags=[\"-O3\", \"--use_fast_math\"],\n extra_cflags=[\"-O3\"],\n verbose=False,\n)\n\n\nclass ModelNew(nn.Module):\n \"\"\"\n 3D convolution followed by fused Mish and Tanh activation.\n \"\"\"\n def __init__(self, in_channels, out_channels, kernel_size, stride=1, padding=0):\n super(ModelNew, self).__init__()\n self.conv = nn.Conv3d(in_channels, out_channels, kernel_size, stride=stride, padding=padding)\n\n def forward(self, x):\n y = F.conv3d(x, self.conv.weight, self.conv.bias, self.conv.stride,\n self.conv.padding, self.conv.dilation, self.conv.groups)\n # Conv3d returns a contiguous fresh tensor; apply both pointwise activations in one CUDA pass.\n return _ext.mish_tanh_inplace(y)\n"
}
result ← submit_kernel (ok)
oksubmit_kernel
submit_kernel PASSED: (5 / 5) correctness trials all passed.
Kernel runtime: 2.37 μs
Runtime stats: mean=2.37μs median=?μs std=0.11μs
Numerical precision: max_abs_err=2.38e-07 mean_abs_err=2.18e-08 max_rel_err=1.00e+00
Memory: 996.1 MB peak (ref: 1190.36 MB, ratio: 0.8368x)
Kernel launches: 7 (ref: 9, fusion ratio: 1.2857)
Energy: 483.74 mJ/run (ref: 1209.02 mJ, ratio: 2.4993x)
SOL score: 1.0
final submitted kernel (85 lines, 2,734 chars) — full source, untruncated
import torch
import torch.nn as nn
import torch.nn.functional as F
from torch.utils.cpp_extension import load_inline
_cpp_src = r"""
#include <torch/extension.h>
torch::Tensor mish_tanh_inplace(torch::Tensor x);
"""
_cuda_src = r"""
#include <torch/extension.h>
#include <ATen/cuda/CUDAContext.h>
#include <cuda.h>
#include <cuda_runtime.h>
__device__ __forceinline__ float fast_tanh_f(float x) {
if (x > 10.0f) return 1.0f;
if (x < -10.0f) return -1.0f;
float ax = fabsf(x);
float e = __expf(-2.0f * ax);
float y = (1.0f - e) / (1.0f + e);
return copysignf(y, x);
}
__device__ __forceinline__ float softplus_f(float x) {
// Stable softplus. The tails also avoid unnecessary log work.
if (x > 20.0f) return x;
if (x < -20.0f) return __expf(x);
return log1pf(__expf(x));
}
__global__ void mish_tanh_kernel(float* __restrict__ p, int64_t n) {
int64_t idx = (int64_t)blockIdx.x * blockDim.x + threadIdx.x;
int64_t stride = (int64_t)blockDim.x * gridDim.x;
for (; idx < n; idx += stride) {
float x = p[idx];
float sp = softplus_f(x);
float mish = x * fast_tanh_f(sp);
p[idx] = fast_tanh_f(mish);
}
}
torch::Tensor mish_tanh_inplace(torch::Tensor x) {
TORCH_CHECK(x.is_cuda(), "input must be a CUDA tensor");
TORCH_CHECK(x.scalar_type() == at::kFloat, "only float32 is supported");
TORCH_CHECK(x.is_contiguous(), "input must be contiguous");
const int64_t n = x.numel();
if (n == 0) return x;
const int threads = 256;
int blocks = (int)((n + threads - 1) / threads);
// Enough blocks to cover the device while keeping launch overhead small.
if (blocks > 65535) blocks = 65535;
mish_tanh_kernel<<<blocks, threads, 0, at::cuda::getCurrentCUDAStream()>>>(x.data_ptr<float>(), n);
return x;
}
"""
_ext = load_inline(
name="mish_tanh_fused_ext_v1",
cpp_sources=_cpp_src,
cuda_sources=_cuda_src,
functions=["mish_tanh_inplace"],
extra_cuda_cflags=["-O3", "--use_fast_math"],
extra_cflags=["-O3"],
verbose=False,
)
class ModelNew(nn.Module):
"""
3D convolution followed by fused Mish and Tanh activation.
"""
def __init__(self, in_channels, out_channels, kernel_size, stride=1, padding=0):
super(ModelNew, self).__init__()
self.conv = nn.Conv3d(in_channels, out_channels, kernel_size, stride=stride, padding=padding)
def forward(self, x):
y = F.conv3d(x, self.conv.weight, self.conv.bias, self.conv.stride,
self.conv.padding, self.conv.dilation, self.conv.groups)
# Conv3d returns a contiguous fresh tensor; apply both pointwise activations in one CUDA pass.
return _ext.mish_tanh_inplace(y)