call → submit_kernel
callsubmit_kernel
{
"kernel_code": "import torch\nimport torch.nn as nn\nfrom torch.utils.cpp_extension import load_inline\n\n_cpp_src = r'''\n#include <torch/extension.h>\n\ntorch::Tensor conv_transpose3d_cuda(torch::Tensor x, torch::Tensor weight, torch::Tensor bias,\n int sd, int sh, int sw,\n int pd, int ph, int pw,\n int opd, int oph, int opw,\n int groups);\n\ntorch::Tensor conv_transpose3d_forward(torch::Tensor x, torch::Tensor weight, torch::Tensor bias,\n int sd, int sh, int sw,\n int pd, int ph, int pw,\n int opd, int oph, int opw,\n int groups) {\n return conv_transpose3d_cuda(x, weight, bias, sd, sh, sw, pd, ph, pw, opd, oph, opw, groups);\n}\n'''\n\n_cuda_src = r'''\n#include <torch/extension.h>\n#include <ATen/cuda/CUDAContext.h>\n#include <cuda.h>\n#include <cuda_runtime.h>\n\nstatic __forceinline__ __device__ int ceil_div_pos_maybe(int a, int b) {\n // only used with b > 0; if a <= 0 the clamped lower bound is 0\n return (a <= 0) ? 0 : ((a + b - 1) / b);\n}\n\ntemplate <bool HAS_BIAS>\n__global__ void convt3d_s2_g4_k357_kernel(const float* __restrict__ x,\n const float* __restrict__ w,\n const float* __restrict__ bias,\n float* __restrict__ y,\n int total) {\n // Specialized for the benchmark case:\n // N x 32 x 12 x 24 x 48 -> N x 32 x 24 x 48 x 96\n const int W_in = 48, H_in = 24, D_in = 12;\n const int C_in = 32, C_out = 32;\n const int OW = 96, OH = 48, OD = 24;\n const int Cout_per_g = 8, Cin_per_g = 8;\n const int KD = 3, KH = 5, KW = 7;\n const int padD = 1, padH = 2, padW = 3;\n\n int idx = blockIdx.x * blockDim.x + threadIdx.x;\n if (idx >= total) return;\n\n int t = idx;\n int ow = t % OW; t /= OW;\n int oh = t % OH; t /= OH;\n int od = t % OD; t /= OD;\n int oc = t % C_out;\n int n = t / C_out;\n\n int g = oc >> 3;\n int ocg = oc & 7;\n int ic_base = g << 3;\n\n int td = od + padD;\n int th = oh + padH;\n int tw = ow + padW;\n\n int id0 = ceil_div_pos_maybe(td - (KD - 1), 2);\n int ih0 = ceil_div_pos_maybe(th - (KH - 1), 2);\n int iw0 = ceil_div_pos_maybe(tw - (KW - 1), 2);\n int id1 = td >> 1; if (id1 > D_in - 1) id1 = D_in - 1;\n int ih1 = th >> 1; if (ih1 > H_in - 1) ih1 = H_in - 1;\n int iw1 = tw >> 1; if (iw1 > W_in - 1) iw1 = W_in - 1;\n\n float acc = HAS_BIAS ? __ldg(bias + oc) : 0.0f;\n\n #pragma unroll\n for (int icg = 0; icg < Cin_per_g; ++icg) {\n int ic = ic_base + icg;\n const float* x_ic = x + (((n * C_in + ic) * D_in) * H_in) * W_in;\n const float* w_ic = w + (((ic * Cout_per_g + ocg) * KD) * KH) * KW;\n\n for (int id = id0; id <= id1; ++id) {\n int kd = td - (id << 1);\n const float* x_d = x_ic + (id * H_in) * W_in;\n const float* w_d = w_ic + (kd * KH) * KW;\n for (int ih = ih0; ih <= ih1; ++ih) {\n int kh = th - (ih << 1);\n const float* x_h = x_d + ih * W_in;\n const float* w_h = w_d + kh * KW;\n #pragma unroll 4\n for (int iw = iw0; iw <= iw1; ++iw) {\n int kw = tw - (iw << 1);\n acc = fmaf(__ldg(x_h + iw), __ldg(w_h + kw), acc);\n }\n }\n }\n }\n y[idx] = acc;\n}\n\ntemplate <bool HAS_BIAS>\n__global__ void convt3d_generic_kernel(const float* __restrict__ x,\n const float* __restrict__ w,\n const float* __restrict__ bias,\n float* __restrict__ y,\n int total,\n int N, int C_in, int D_in, int H_in, int W_in,\n int C_out, int OD, int OH, int OW,\n int KD, int KH, int KW,\n int sd, int sh, int sw,\n int pd, int ph, int pw,\n int groups) {\n int idx = blockIdx.x * blockDim.x + threadIdx.x;\n if (idx >= total) return;\n int tmp = idx;\n int ow = tmp % OW; tmp /= OW;\n int oh = tmp % OH; tmp /= OH;\n int od = tmp % OD; tmp /= OD;\n int oc = tmp % C_out;\n int n = tmp / C_out;\n\n int cout_pg = C_out / groups;\n int cin_pg = C_in / groups;\n int g = oc / cout_pg;\n int ocg = oc - g * cout_pg;\n int ic_base = g * cin_pg;\n\n int td = od + pd, th = oh + ph, tw = ow + pw;\n int id0 = ceil_div_pos_maybe(td - (KD - 1), sd);\n int ih0 = ceil_div_pos_maybe(th - (KH - 1), sh);\n int iw0 = ceil_div_pos_maybe(tw - (KW - 1), sw);\n int id1 = td / sd; if (id1 > D_in - 1) id1 = D_in - 1;\n int ih1 = th / sh; if (ih1 > H_in - 1) ih1 = H_in - 1;\n int iw1 = tw / sw; if (iw1 > W_in - 1) iw1 = W_in - 1;\n\n float acc = HAS_BIAS ? __ldg(bias + oc) : 0.0f;\n for (int icg = 0; icg < cin_pg; ++icg) {\n int ic = ic_base + icg;\n const float* x_ic = x + (((n * C_in + ic) * D_in) * H_in) * W_in;\n const float* w_ic = w + (((ic * cout_pg + ocg) * KD) * KH) * KW;\n for (int id = id0; id <= id1; ++id) {\n int kd = td - id * sd;\n const float* x_d = x_ic + (id * H_in) * W_in;\n const float* w_d = w_ic + (kd * KH) * KW;\n for (int ih = ih0; ih <= ih1; ++ih) {\n int kh = th - ih * sh;\n const float* x_h = x_d + ih * W_in;\n const float* w_h = w_d + kh * KW;\n for (int iw = iw0; iw <= iw1; ++iw) {\n int kw = tw - iw * sw;\n acc = fmaf(__ldg(x_h + iw), __ldg(w_h + kw), acc);\n }\n }\n }\n }\n y[idx] = acc;\n}\n\ntorch::Tensor conv_transpose3d_cuda(torch::Tensor x, torch::Tensor weight, torch::Tensor bias,\n int sd, int sh, int sw,\n int pd, int ph, int pw,\n int opd, int oph, int opw,\n int groups) {\n x = x.contiguous();\n weight = weight.contiguous();\n if (bias.numel() != 0) bias = bias.contiguous();\n\n const int N = (int)x.size(0);\n const int C_in = (int)x.size(1);\n const int D_in = (int)x.size(2);\n const int H_in = (int)x.size(3);\n const int W_in = (int)x.size(4);\n const int KD = (int)weight.size(2);\n const int KH = (int)weight.size(3);\n const int KW = (int)weight.size(4);\n const int C_out = (int)weight.size(1) * groups;\n const int OD = (D_in - 1) * sd - 2 * pd + KD + opd;\n const int OH = (H_in - 1) * sh - 2 * ph + KH + oph;\n const int OW = (W_in - 1) * sw - 2 * pw + KW + opw;\n\n auto y = torch::empty({N, C_out, OD, OH, OW}, x.options());\n int total = N * C_out * OD * OH * OW;\n const int threads = 256;\n const dim3 blocks((total + threads - 1) / threads);\n cudaStream_t stream = at::cuda::getCurrentCUDAStream();\n bool has_bias = (bias.numel() != 0);\n\n if (C_in == 32 && C_out == 32 && D_in == 12 && H_in == 24 && W_in == 48 &&\n KD == 3 && KH == 5 && KW == 7 && sd == 2 && sh == 2 && sw == 2 &&\n pd == 1 && ph == 2 && pw == 3 && opd == 1 && oph == 1 && opw == 1 &&\n groups == 4 && OD == 24 && OH == 48 && OW == 96) {\n if (has_bias) convt3d_s2_g4_k357_kernel<true><<<blocks, threads, 0, stream>>>(x.data_ptr<float>(), weight.data_ptr<float>(), bias.data_ptr<float>(), y.data_ptr<float>(), total);\n else convt3d_s2_g4_k357_kernel<false><<<blocks, threads, 0, stream>>>(x.data_ptr<float>(), weight.data_ptr<float>(), nullptr, y.data_ptr<float>(), total);\n } else {\n if (has_bias) convt3d_generic_kernel<true><<<blocks, threads, 0, stream>>>(x.data_ptr<float>(), weight.data_ptr<float>(), bias.data_ptr<float>(), y.data_ptr<float>(), total, N, C_in, D_in, H_in, W_in, C_out, OD, OH, OW, KD, KH, KW, sd, sh, sw, pd, ph, pw, groups);\n else convt3d_generic_kernel<false><<<blocks, threads, 0, stream>>>(x.data_ptr<float>(), weight.data_ptr<float>(), nullptr, y.data_ptr<float>(), total, N, C_in, D_in, H_in, W_in, C_out, OD, OH, OW, KD, KH, KW, sd, sh, sw, pd, ph, pw, groups);\n }\n return y;\n}\n'''\n\n_ext = load_inline(\n name='convt3d_custom_ext_v1',\n cpp_sources=_cpp_src,\n cuda_sources=_cuda_src,\n functions=['conv_transpose3d_forward'],\n extra_cuda_cflags=['-O3'],\n extra_cflags=['-O3'],\n verbose=False,\n)\n\nclass ModelNew(nn.Module):\n def __init__(self, in_channels: int, out_channels: int, kernel_size: tuple, stride: tuple = (1, 1, 1), padding: tuple = (0, 0, 0), output_padding: tuple = (0, 0, 0), groups: int = 1, bias: bool = False):\n super(ModelNew, self).__init__()\n self.conv_transpose3d = nn.ConvTranspose3d(in_channels, out_channels, kernel_size, stride=stride, padding=padding, output_padding=output_padding, groups=groups, bias=bias)\n self.stride = tuple(stride)\n self.padding = tuple(padding)\n self.output_padding = tuple(output_padding)\n self.groups = groups\n\n def forward(self, x: torch.Tensor) -> torch.Tensor:\n b = self.conv_transpose3d.bias\n if b is None:\n b = torch.empty(0, device=x.device, dtype=x.dtype)\n s = self.stride; p = self.padding; op = self.output_padding\n return _ext.conv_transpose3d_forward(x, self.conv_transpose3d.weight, b,\n int(s[0]), int(s[1]), int(s[2]),\n int(p[0]), int(p[1]), int(p[2]),\n int(op[0]), int(op[1]), int(op[2]),\n int(self.groups))\n"
}