call → submit_kernel
callsubmit_kernel
{
"kernel_code": "import torch\nimport torch.nn as nn\nfrom torch.utils.cpp_extension import load_inline\n\n_cpp_src = r'''\n#include <torch/extension.h>\n\ntorch::Tensor conv_transpose3d_forward(torch::Tensor x,\n torch::Tensor weight,\n torch::Tensor bias,\n int64_t stride,\n int64_t padding,\n int64_t output_padding,\n int64_t dilation,\n int64_t groups);\n'''\n\n_cuda_src = r'''\n#include <torch/extension.h>\n#include <ATen/cuda/CUDAContext.h>\n#include <c10/cuda/CUDAGuard.h>\n\n#define CHECK_CUDA(x) TORCH_CHECK(x.is_cuda(), #x \" must be a CUDA tensor\")\n#define CHECK_CONTIGUOUS(x) TORCH_CHECK(x.is_contiguous(), #x \" must be contiguous\")\n\n__global__ __launch_bounds__(256, 2)\nvoid deconv3d_k3s1p0_48_24_oc4_kernel(const float* __restrict__ x,\n const float* __restrict__ w,\n const float* __restrict__ b,\n float* __restrict__ y,\n int N, int D, int H, int W,\n int has_bias) {\n constexpr int IC = 48;\n constexpr int OC = 24;\n constexpr int K = 3;\n const int Do = D + 2;\n const int Ho = H + 2;\n const int Wo = W + 2;\n const int spatial_total = N * Do * Ho * Wo;\n int p = blockIdx.x * blockDim.x + threadIdx.x;\n if (p >= spatial_total) return;\n const int oc0 = blockIdx.y * 4;\n\n int t = p;\n const int ow = t % Wo; t /= Wo;\n const int oh = t % Ho; t /= Ho;\n const int od = t % Do; t /= Do;\n const int n = t;\n\n float a0 = has_bias ? __ldg(b + oc0 + 0) : 0.0f;\n float a1 = has_bias ? __ldg(b + oc0 + 1) : 0.0f;\n float a2 = has_bias ? __ldg(b + oc0 + 2) : 0.0f;\n float a3 = has_bias ? __ldg(b + oc0 + 3) : 0.0f;\n\n const int kd_min = max(0, od - (D - 1));\n const int kd_max = min(2, od);\n const int kh_min = max(0, oh - (H - 1));\n const int kh_max = min(2, oh);\n const int kw_min = max(0, ow - (W - 1));\n const int kw_max = min(2, ow);\n\n const int x_n_base = n * IC * D * H * W;\n\n for (int ic = 0; ic < IC; ++ic) {\n const int x_ic_base = x_n_base + ic * D * H * W;\n const int w_ic_base = ic * OC * K * K * K;\n for (int kd = kd_min; kd <= kd_max; ++kd) {\n const int iz = od - kd;\n const int x_d_base = x_ic_base + iz * H * W;\n const int wk_d = kd * 9;\n for (int kh = kh_min; kh <= kh_max; ++kh) {\n const int iy = oh - kh;\n const int x_h_base = x_d_base + iy * W;\n const int wk_dh = wk_d + kh * 3;\n#pragma unroll\n for (int kw = 0; kw < 3; ++kw) {\n if (kw < kw_min || kw > kw_max) continue;\n const int ix = ow - kw;\n const float xv = __ldg(x + x_h_base + ix);\n const int wk = wk_dh + kw;\n const int wb0 = w_ic_base + (oc0 + 0) * 27 + wk;\n a0 = fmaf(xv, __ldg(w + wb0), a0);\n a1 = fmaf(xv, __ldg(w + wb0 + 27), a1);\n a2 = fmaf(xv, __ldg(w + wb0 + 54), a2);\n a3 = fmaf(xv, __ldg(w + wb0 + 81), a3);\n }\n }\n }\n }\n\n const int out_sp = Do * Ho * Wo;\n const int y_base = (n * OC + oc0) * out_sp + od * Ho * Wo + oh * Wo + ow;\n y[y_base] = a0;\n y[y_base + out_sp] = a1;\n y[y_base + 2 * out_sp] = a2;\n y[y_base + 3 * out_sp] = a3;\n}\n\n__global__ void deconv3d_generic_kernel(const float* __restrict__ x,\n const float* __restrict__ w,\n const float* __restrict__ b,\n float* __restrict__ y,\n int N, int IC, int D, int H, int W,\n int OC, int K, int stride, int padding,\n int dilation, int groups,\n int Do, int Ho, int Wo,\n int has_bias) {\n int64_t idx = (int64_t)blockIdx.x * blockDim.x + threadIdx.x;\n int64_t total = (int64_t)N * OC * Do * Ho * Wo;\n if (idx >= total) return;\n\n int t0 = (int)idx;\n const int ow = t0 % Wo; t0 /= Wo;\n const int oh = t0 % Ho; t0 /= Ho;\n const int od = t0 % Do; t0 /= Do;\n const int oc = t0 % OC; t0 /= OC;\n const int n = t0;\n\n const int OCg = OC / groups;\n const int ICg = IC / groups;\n const int g = oc / OCg;\n const int ocg = oc - g * OCg;\n float acc = has_bias ? __ldg(b + oc) : 0.0f;\n\n for (int ii = 0; ii < ICg; ++ii) {\n const int ic = g * ICg + ii;\n for (int kd = 0; kd < K; ++kd) {\n int zd = od + padding - kd * dilation;\n if (zd < 0 || zd % stride) continue;\n int iz = zd / stride;\n if (iz < 0 || iz >= D) continue;\n for (int kh = 0; kh < K; ++kh) {\n int yh = oh + padding - kh * dilation;\n if (yh < 0 || yh % stride) continue;\n int iy = yh / stride;\n if (iy < 0 || iy >= H) continue;\n for (int kw = 0; kw < K; ++kw) {\n int xw = ow + padding - kw * dilation;\n if (xw < 0 || xw % stride) continue;\n int ix = xw / stride;\n if (ix < 0 || ix >= W) continue;\n int64_t xo = (((int64_t)n * IC + ic) * D + iz) * H * W + iy * W + ix;\n int64_t wo = ((((int64_t)ic * OCg + ocg) * K + kd) * K + kh) * K + kw;\n acc = fmaf(__ldg(x + xo), __ldg(w + wo), acc);\n }\n }\n }\n }\n y[idx] = acc;\n}\n\ntorch::Tensor conv_transpose3d_forward(torch::Tensor x,\n torch::Tensor weight,\n torch::Tensor bias,\n int64_t stride,\n int64_t padding,\n int64_t output_padding,\n int64_t dilation,\n int64_t groups) {\n CHECK_CUDA(x); CHECK_CUDA(weight); CHECK_CUDA(bias);\n CHECK_CONTIGUOUS(x); CHECK_CONTIGUOUS(weight); CHECK_CONTIGUOUS(bias);\n TORCH_CHECK(x.scalar_type() == at::kFloat, \"only float32 is supported\");\n TORCH_CHECK(weight.scalar_type() == at::kFloat, \"only float32 is supported\");\n const c10::cuda::CUDAGuard device_guard(x.device());\n\n const int N = (int)x.size(0);\n const int IC = (int)x.size(1);\n const int D = (int)x.size(2);\n const int H = (int)x.size(3);\n const int W = (int)x.size(4);\n const int OCg = (int)weight.size(1);\n const int K = (int)weight.size(2);\n const int OC = OCg * (int)groups;\n const int Do = (D - 1) * (int)stride - 2 * (int)padding + (int)dilation * (K - 1) + (int)output_padding + 1;\n const int Ho = (H - 1) * (int)stride - 2 * (int)padding + (int)dilation * (K - 1) + (int)output_padding + 1;\n const int Wo = (W - 1) * (int)stride - 2 * (int)padding + (int)dilation * (K - 1) + (int)output_padding + 1;\n\n auto y = torch::empty({N, OC, Do, Ho, Wo}, x.options());\n const int has_bias = (bias.numel() > 0) ? 1 : 0;\n const float* bp = has_bias ? bias.data_ptr<float>() : bias.data_ptr<float>();\n cudaStream_t stream = at::cuda::getCurrentCUDAStream();\n\n if (IC == 48 && OC == 24 && K == 3 && stride == 1 && padding == 0 && output_padding == 0 && dilation == 1 && groups == 1) {\n const int spatial_total = N * Do * Ho * Wo;\n dim3 block(256);\n dim3 grid((spatial_total + block.x - 1) / block.x, 6);\n deconv3d_k3s1p0_48_24_oc4_kernel<<<grid, block, 0, stream>>>(\n x.data_ptr<float>(), weight.data_ptr<float>(), bp, y.data_ptr<float>(), N, D, H, W, has_bias);\n } else {\n int64_t total = (int64_t)N * OC * Do * Ho * Wo;\n dim3 block(256);\n dim3 grid((unsigned int)((total + block.x - 1) / block.x));\n deconv3d_generic_kernel<<<grid, block, 0, stream>>>(\n x.data_ptr<float>(), weight.data_ptr<float>(), bp, y.data_ptr<float>(),\n N, IC, D, H, W, OC, K, (int)stride, (int)padding, (int)dilation, (int)groups,\n Do, Ho, Wo, has_bias);\n }\n C10_CUDA_KERNEL_LAUNCH_CHECK();\n return y;\n}\n'''\n\n_ext = load_inline(\n name='deconv3d_transpose_ext_v1',\n cpp_sources=_cpp_src,\n cuda_sources=_cuda_src,\n functions=['conv_transpose3d_forward'],\n extra_cflags=['-O3'],\n extra_cuda_cflags=['-O3', '--use_fast_math'],\n verbose=False,\n)\n\nclass ModelNew(nn.Module):\n def __init__(self, in_channels: int, out_channels: int, kernel_size: int, stride: int = 1, padding: int = 0, output_padding: int = 0,\n dilation: int = 1, groups: int = 1, bias: bool = False):\n super(ModelNew, self).__init__()\n self.conv_transpose3d = nn.ConvTranspose3d(in_channels, out_channels, (kernel_size, kernel_size, kernel_size),\n stride=stride, padding=padding, output_padding=output_padding,\n dilation=dilation, groups=groups, bias=bias)\n self.stride = stride[0] if isinstance(stride, tuple) else stride\n self.padding = padding[0] if isinstance(padding, tuple) else padding\n self.output_padding = output_padding[0] if isinstance(output_padding, tuple) else output_padding\n self.dilation = dilation[0] if isinstance(dilation, tuple) else dilation\n self.groups = groups\n self.register_buffer('_empty_bias', torch.empty(0), persistent=False)\n\n def forward(self, x: torch.Tensor) -> torch.Tensor:\n if not x.is_contiguous():\n x = x.contiguous()\n w = self.conv_transpose3d.weight.contiguous()\n b = self.conv_transpose3d.bias if self.conv_transpose3d.bias is not None else self._empty_bias\n if b.device != x.device:\n b = torch.empty(0, device=x.device, dtype=x.dtype)\n return _ext.conv_transpose3d_forward(x, w, b.contiguous(), int(self.stride), int(self.padding),\n int(self.output_padding), int(self.dilation), int(self.groups))\n"
}