call → submit_kernel
callsubmit_kernel
{
"kernel_code": "import torch\nimport torch.nn as nn\nfrom torch.utils.cpp_extension import load_inline\n\n_cpp_src = r\"\"\"\n#include <torch/extension.h>\n\ntorch::Tensor conv_transpose3d_forward(torch::Tensor x, torch::Tensor weight, torch::Tensor bias,\n int stride, int padding, int output_padding, int groups, bool has_bias);\n\"\"\"\n\n_cuda_src = r\"\"\"\n#include <torch/extension.h>\n#include <ATen/cuda/CUDAContext.h>\n#include <cuda.h>\n#include <cuda_runtime.h>\n\n#define CHECK_CUDA(x) TORCH_CHECK(x.is_cuda(), #x \" must be a CUDA tensor\")\n#define CHECK_CONTIGUOUS(x) TORCH_CHECK(x.is_contiguous(), #x \" must be contiguous\")\n#define CHECK_FLOAT(x) TORCH_CHECK(x.scalar_type() == at::kFloat, #x \" must be float32\")\n\nconstexpr int VEC = 8;\n\n__global__ __launch_bounds__(128, 2)\nvoid deconv3d_48_k3_s1p0_kernel(const float* __restrict__ x,\n const float* __restrict__ w,\n const float* __restrict__ bias,\n float* __restrict__ y,\n int B, int D, int H, int Wd,\n int Do, int Ho, int Wo,\n int has_bias) {\n const int outVol = Do * Ho * Wo;\n const int inHW = H * Wd;\n const int inVol = D * inHW;\n const int64_t total = (int64_t)B * outVol * 6; // 48 / 8\n\n int64_t job = (int64_t)blockIdx.x * blockDim.x + threadIdx.x;\n if (job >= total) return;\n\n const int spatial = (int)(job % outVol);\n int64_t t = job / outVol;\n const int n = (int)(t % B);\n const int oc_base = (int)(t / B) * VEC;\n\n int rem = spatial;\n const int ow = rem % Wo;\n rem /= Wo;\n const int oh = rem % Ho;\n const int od = rem / Ho;\n\n float acc0 = 0.0f, acc1 = 0.0f, acc2 = 0.0f, acc3 = 0.0f;\n float acc4 = 0.0f, acc5 = 0.0f, acc6 = 0.0f, acc7 = 0.0f;\n\n // Transposed convolution, stride=1, padding=0, kernel=3:\n // y[n,oc,od,oh,ow] = sum_ic,k x[n,ic,od-kd,oh-kh,ow-kw] * W[ic,oc,kd,kh,kw]\n #pragma unroll\n for (int kd = 0; kd < 3; ++kd) {\n const int id = od - kd;\n if ((unsigned)id >= (unsigned)D) continue;\n #pragma unroll\n for (int kh = 0; kh < 3; ++kh) {\n const int ih = oh - kh;\n if ((unsigned)ih >= (unsigned)H) continue;\n #pragma unroll\n for (int kw = 0; kw < 3; ++kw) {\n const int iw = ow - kw;\n if ((unsigned)iw >= (unsigned)Wd) continue;\n const int kidx = (kd * 3 + kh) * 3 + kw;\n const int x_sp = id * inHW + ih * Wd + iw;\n #pragma unroll 4\n for (int ic = 0; ic < 48; ++ic) {\n const float xv = __ldg(x + ((n * 48 + ic) * inVol + x_sp));\n const int wb = ((ic * 48 + oc_base) * 27 + kidx);\n acc0 = fmaf(xv, __ldg(w + wb + 0 * 27), acc0);\n acc1 = fmaf(xv, __ldg(w + wb + 1 * 27), acc1);\n acc2 = fmaf(xv, __ldg(w + wb + 2 * 27), acc2);\n acc3 = fmaf(xv, __ldg(w + wb + 3 * 27), acc3);\n acc4 = fmaf(xv, __ldg(w + wb + 4 * 27), acc4);\n acc5 = fmaf(xv, __ldg(w + wb + 5 * 27), acc5);\n acc6 = fmaf(xv, __ldg(w + wb + 6 * 27), acc6);\n acc7 = fmaf(xv, __ldg(w + wb + 7 * 27), acc7);\n }\n }\n }\n }\n\n if (has_bias) {\n acc0 += __ldg(bias + oc_base + 0);\n acc1 += __ldg(bias + oc_base + 1);\n acc2 += __ldg(bias + oc_base + 2);\n acc3 += __ldg(bias + oc_base + 3);\n acc4 += __ldg(bias + oc_base + 4);\n acc5 += __ldg(bias + oc_base + 5);\n acc6 += __ldg(bias + oc_base + 6);\n acc7 += __ldg(bias + oc_base + 7);\n }\n\n float* yb = y + n * 48 * outVol + spatial;\n yb[(oc_base + 0) * outVol] = acc0;\n yb[(oc_base + 1) * outVol] = acc1;\n yb[(oc_base + 2) * outVol] = acc2;\n yb[(oc_base + 3) * outVol] = acc3;\n yb[(oc_base + 4) * outVol] = acc4;\n yb[(oc_base + 5) * outVol] = acc5;\n yb[(oc_base + 6) * outVol] = acc6;\n yb[(oc_base + 7) * outVol] = acc7;\n}\n\n__global__\nvoid deconv3d_generic_kernel(const float* __restrict__ x,\n const float* __restrict__ w,\n const float* __restrict__ bias,\n float* __restrict__ y,\n int B, int IC, int D, int H, int Wd,\n int OC, int kD, int kH, int kW,\n int Do, int Ho, int Wo,\n int stride, int padding, int groups, int has_bias) {\n const int64_t outVol = (int64_t)Do * Ho * Wo;\n const int64_t total = (int64_t)B * OC * outVol;\n int64_t idx = (int64_t)blockIdx.x * blockDim.x + threadIdx.x;\n if (idx >= total) return;\n\n int spatial = (int)(idx % outVol);\n int64_t tmp = idx / outVol;\n int oc = (int)(tmp % OC);\n int n = (int)(tmp / OC);\n\n int rem = spatial;\n int ow = rem % Wo; rem /= Wo;\n int oh = rem % Ho; int od = rem / Ho;\n\n const int ICpg = IC / groups;\n const int OCpg = OC / groups;\n const int g = oc / OCpg;\n const int woc = oc - g * OCpg;\n const int ic0 = g * ICpg;\n const int ic1 = ic0 + ICpg;\n const int inHW = H * Wd;\n const int inVol = D * inHW;\n float acc = 0.0f;\n\n for (int ic = ic0; ic < ic1; ++ic) {\n for (int kd = 0; kd < kD; ++kd) {\n int td = od + padding - kd;\n if (td < 0 || td % stride) continue;\n int id = td / stride;\n if ((unsigned)id >= (unsigned)D) continue;\n for (int kh = 0; kh < kH; ++kh) {\n int th = oh + padding - kh;\n if (th < 0 || th % stride) continue;\n int ih = th / stride;\n if ((unsigned)ih >= (unsigned)H) continue;\n for (int kw = 0; kw < kW; ++kw) {\n int tw = ow + padding - kw;\n if (tw < 0 || tw % stride) continue;\n int iw = tw / stride;\n if ((unsigned)iw >= (unsigned)Wd) continue;\n float xv = x[((n * IC + ic) * D + id) * inHW + ih * Wd + iw];\n float wv = w[(((ic * OCpg + woc) * kD + kd) * kH + kh) * kW + kw];\n acc = fmaf(xv, wv, acc);\n }\n }\n }\n }\n if (has_bias) acc += bias[oc];\n y[idx] = acc;\n}\n\ntorch::Tensor conv_transpose3d_forward(torch::Tensor x, torch::Tensor weight, torch::Tensor bias,\n int stride, int padding, int output_padding, int groups, bool has_bias) {\n CHECK_CUDA(x);\n CHECK_CUDA(weight);\n CHECK_CONTIGUOUS(x);\n CHECK_CONTIGUOUS(weight);\n CHECK_FLOAT(x);\n CHECK_FLOAT(weight);\n if (has_bias) { CHECK_CUDA(bias); CHECK_CONTIGUOUS(bias); CHECK_FLOAT(bias); }\n\n const int B = (int)x.size(0);\n const int IC = (int)x.size(1);\n const int D = (int)x.size(2);\n const int H = (int)x.size(3);\n const int Wd = (int)x.size(4);\n const int kD = (int)weight.size(2);\n const int kH = (int)weight.size(3);\n const int kW = (int)weight.size(4);\n const int OC = (int)weight.size(1) * groups;\n\n const int Do = (D - 1) * stride - 2 * padding + kD + output_padding;\n const int Ho = (H - 1) * stride - 2 * padding + kH + output_padding;\n const int Wo = (Wd - 1) * stride - 2 * padding + kW + output_padding;\n\n auto y = torch::empty({B, OC, Do, Ho, Wo}, x.options());\n cudaStream_t stream = at::cuda::getCurrentCUDAStream();\n\n if (IC == 48 && OC == 48 && kD == 3 && kH == 3 && kW == 3 && stride == 1 && padding == 0 && output_padding == 0 && groups == 1) {\n const int64_t jobs = (int64_t)B * Do * Ho * Wo * 6;\n const int threads = 128;\n const dim3 blocks((unsigned int)((jobs + threads - 1) / threads));\n deconv3d_48_k3_s1p0_kernel<<<blocks, threads, 0, stream>>>(\n x.data_ptr<float>(), weight.data_ptr<float>(), has_bias ? bias.data_ptr<float>() : nullptr,\n y.data_ptr<float>(), B, D, H, Wd, Do, Ho, Wo, has_bias ? 1 : 0);\n } else {\n const int64_t total = (int64_t)B * OC * Do * Ho * Wo;\n const int threads = 256;\n const dim3 blocks((unsigned int)((total + threads - 1) / threads));\n deconv3d_generic_kernel<<<blocks, threads, 0, stream>>>(\n x.data_ptr<float>(), weight.data_ptr<float>(), has_bias ? bias.data_ptr<float>() : nullptr,\n y.data_ptr<float>(), B, IC, D, H, Wd, OC, kD, kH, kW, Do, Ho, Wo, stride, padding, groups, has_bias ? 1 : 0);\n }\n return y;\n}\n\"\"\"\n\n_ext = load_inline(\n name=\"conv_transpose3d_custom_ext_v1\",\n cpp_sources=_cpp_src,\n cuda_sources=_cuda_src,\n functions=[\"conv_transpose3d_forward\"],\n extra_cflags=[\"-O3\"],\n extra_cuda_cflags=[\"-O3\"],\n with_cuda=True,\n verbose=False,\n)\n\nclass ModelNew(nn.Module):\n def __init__(self, in_channels: int, out_channels: int, kernel_size: int, stride: int = 1, padding: int = 0, output_padding: int = 0, groups: int = 1, bias: bool = False):\n super(ModelNew, self).__init__()\n self.conv_transpose3d = nn.ConvTranspose3d(\n in_channels, out_channels,\n kernel_size=(kernel_size, kernel_size, kernel_size),\n stride=stride, padding=padding, output_padding=output_padding,\n groups=groups, bias=bias\n )\n self.stride = stride if isinstance(stride, int) else stride[0]\n self.padding = padding if isinstance(padding, int) else padding[0]\n self.output_padding = output_padding if isinstance(output_padding, int) else output_padding[0]\n self.groups = groups\n\n def forward(self, x: torch.Tensor) -> torch.Tensor:\n # The benchmark uses contiguous CUDA fp32 tensors. Make parameters contiguous if a caller has\n # altered their layout; this does not change numerical behavior.\n xc = x.contiguous()\n w = self.conv_transpose3d.weight.contiguous()\n b = self.conv_transpose3d.bias\n has_b = b is not None\n if has_b:\n bc = b.contiguous()\n else:\n bc = torch.empty(0, device=xc.device, dtype=xc.dtype)\n return _ext.conv_transpose3d_forward(xc, w, bc, int(self.stride), int(self.padding), int(self.output_padding), int(self.groups), bool(has_b))\n"
}