call → submit_kernel
callsubmit_kernel
{
"kernel_code": "import torch\nimport torch.nn as nn\nfrom torch.utils.cpp_extension import load_inline\n\n_cpp_src = r\"\"\"\n#include <torch/extension.h>\n\ntorch::Tensor conv3d_forward(torch::Tensor x, torch::Tensor weight, torch::Tensor bias,\n int stride, int padding, int dilation, int groups);\n\"\"\"\n\n_cuda_src = r\"\"\"\n#include <torch/extension.h>\n#include <ATen/cuda/CUDAContext.h>\n#include <c10/cuda/CUDAGuard.h>\n#include <cuda.h>\n#include <cuda_runtime.h>\n\n__global__ void conv3d_k3c3_oc8_kernel(const float* __restrict__ x,\n const float* __restrict__ w,\n const float* __restrict__ b,\n float* __restrict__ y,\n int total, int N, int H, int W, int D,\n int O, int Hout, int Wout, int inner,\n bool has_bias) {\n int tid = blockIdx.x * blockDim.x + threadIdx.x;\n int stride_grid = blockDim.x * gridDim.x;\n for (int idx = tid; idx < total; idx += stride_grid) {\n int spatial = idx % inner;\n int t0 = idx / inner;\n int og = t0 & 7; // O/8 == 8 in the target case (O=64)\n int n = t0 >> 3;\n int oc0 = og << 3;\n\n int d = spatial % D;\n int t = spatial / D;\n int ow = t % Wout;\n int oh = t / Wout;\n\n float a0 = has_bias ? b[oc0 + 0] : 0.0f;\n float a1 = has_bias ? b[oc0 + 1] : 0.0f;\n float a2 = has_bias ? b[oc0 + 2] : 0.0f;\n float a3 = has_bias ? b[oc0 + 3] : 0.0f;\n float a4 = has_bias ? b[oc0 + 4] : 0.0f;\n float a5 = has_bias ? b[oc0 + 5] : 0.0f;\n float a6 = has_bias ? b[oc0 + 6] : 0.0f;\n float a7 = has_bias ? b[oc0 + 7] : 0.0f;\n\n #pragma unroll\n for (int c = 0; c < 3; ++c) {\n #pragma unroll\n for (int kh = 0; kh < 3; ++kh) {\n int ih = oh + kh;\n #pragma unroll\n for (int kw = 0; kw < 3; ++kw) {\n int iw = ow + kw;\n float xv = __ldg(x + ((((n * 3 + c) * H + ih) * W + iw) * D + d));\n int wo = ((oc0 * 3 + c) * 3 + kh) * 3 + kw;\n a0 = fmaf(xv, __ldg(w + wo + 0 * 27), a0);\n a1 = fmaf(xv, __ldg(w + wo + 1 * 27), a1);\n a2 = fmaf(xv, __ldg(w + wo + 2 * 27), a2);\n a3 = fmaf(xv, __ldg(w + wo + 3 * 27), a3);\n a4 = fmaf(xv, __ldg(w + wo + 4 * 27), a4);\n a5 = fmaf(xv, __ldg(w + wo + 5 * 27), a5);\n a6 = fmaf(xv, __ldg(w + wo + 6 * 27), a6);\n a7 = fmaf(xv, __ldg(w + wo + 7 * 27), a7);\n }\n }\n }\n\n int out = (n * O + oc0) * inner + spatial;\n y[out + 0 * inner] = a0;\n y[out + 1 * inner] = a1;\n y[out + 2 * inner] = a2;\n y[out + 3 * inner] = a3;\n y[out + 4 * inner] = a4;\n y[out + 5 * inner] = a5;\n y[out + 6 * inner] = a6;\n y[out + 7 * inner] = a7;\n }\n}\n\n__global__ void conv3d_generic_kernel(const float* __restrict__ x,\n const float* __restrict__ w,\n const float* __restrict__ b,\n float* __restrict__ y,\n int total, int N, int C, int H, int W, int D,\n int O, int Cg, int kH, int kW, int kD,\n int Hout, int Wout, int Dout,\n int stride, int pad, int dil, int groups,\n bool has_bias) {\n int tid = blockIdx.x * blockDim.x + threadIdx.x;\n int grid_stride = blockDim.x * gridDim.x;\n for (int idx = tid; idx < total; idx += grid_stride) {\n int od = idx % Dout;\n int q = idx / Dout;\n int ow = q % Wout; q /= Wout;\n int oh = q % Hout; q /= Hout;\n int oc = q % O;\n int n = q / O;\n int g = oc / (O / groups);\n int cbase = g * Cg;\n float acc = has_bias ? b[oc] : 0.0f;\n for (int ci = 0; ci < Cg; ++ci) {\n int ic = cbase + ci;\n for (int kh = 0; kh < kH; ++kh) {\n int ih = oh * stride - pad + kh * dil;\n if ((unsigned)ih >= (unsigned)H) continue;\n for (int kw = 0; kw < kW; ++kw) {\n int iw = ow * stride - pad + kw * dil;\n if ((unsigned)iw >= (unsigned)W) continue;\n for (int kd = 0; kd < kD; ++kd) {\n int id = od * stride - pad + kd * dil;\n if ((unsigned)id >= (unsigned)D) continue;\n float xv = x[((((n * C + ic) * H + ih) * W + iw) * D + id)];\n float wv = w[(((((oc * Cg + ci) * kH + kh) * kW + kw) * kD + kd))];\n acc = fmaf(xv, wv, acc);\n }\n }\n }\n }\n y[idx] = acc;\n }\n}\n\ntorch::Tensor conv3d_forward(torch::Tensor x, torch::Tensor weight, torch::Tensor bias,\n int stride, int padding, int dilation, int groups) {\n const c10::cuda::CUDAGuard device_guard(x.device());\n int N = (int)x.size(0);\n int C = (int)x.size(1);\n int H = (int)x.size(2);\n int W = (int)x.size(3);\n int D = (int)x.size(4);\n int O = (int)weight.size(0);\n int Cg = (int)weight.size(1);\n int kH = (int)weight.size(2);\n int kW = (int)weight.size(3);\n int kD = (int)weight.size(4);\n int Hout = (H + 2 * padding - dilation * (kH - 1) - 1) / stride + 1;\n int Wout = (W + 2 * padding - dilation * (kW - 1) - 1) / stride + 1;\n int Dout = (D + 2 * padding - dilation * (kD - 1) - 1) / stride + 1;\n auto y = torch::empty({N, O, Hout, Wout, Dout}, x.options());\n bool has_bias = bias.numel() != 0;\n const float* bp = has_bias ? bias.data_ptr<float>() : nullptr;\n const int threads = 256;\n cudaStream_t stream = at::cuda::getCurrentCUDAStream();\n\n if (x.scalar_type() == at::kFloat && weight.scalar_type() == at::kFloat &&\n groups == 1 && stride == 1 && padding == 0 && dilation == 1 &&\n C == 3 && Cg == 3 && kH == 3 && kW == 3 && kD == 1 && (O % 8) == 0 && O == 64) {\n int inner = Hout * Wout * Dout;\n int total = N * (O / 8) * inner;\n int blocks = (total + threads - 1) / threads;\n if (blocks > 65535) blocks = 65535;\n conv3d_k3c3_oc8_kernel<<<blocks, threads, 0, stream>>>(\n x.data_ptr<float>(), weight.data_ptr<float>(), bp, y.data_ptr<float>(),\n total, N, H, W, D, O, Hout, Wout, inner, has_bias);\n } else {\n int total = N * O * Hout * Wout * Dout;\n int blocks = (total + threads - 1) / threads;\n if (blocks > 65535) blocks = 65535;\n conv3d_generic_kernel<<<blocks, threads, 0, stream>>>(\n x.data_ptr<float>(), weight.data_ptr<float>(), bp, y.data_ptr<float>(),\n total, N, C, H, W, D, O, Cg, kH, kW, kD, Hout, Wout, Dout,\n stride, padding, dilation, groups, has_bias);\n }\n C10_CUDA_KERNEL_LAUNCH_CHECK();\n return y;\n}\n\"\"\"\n\n_ext = load_inline(\n name=\"conv3d_asym_k3_fast_ext\",\n cpp_sources=_cpp_src,\n cuda_sources=_cuda_src,\n functions=[\"conv3d_forward\"],\n extra_cflags=[\"-O3\"],\n extra_cuda_cflags=[\"-O3\", \"--use_fast_math\"],\n verbose=False,\n)\n\nclass ModelNew(nn.Module):\n def __init__(self, in_channels: int, out_channels: int, kernel_size: int, stride: int = 1, padding: int = 0, dilation: int = 1, groups: int = 1, bias: bool = False):\n super(ModelNew, self).__init__()\n self.conv3d = nn.Conv3d(in_channels, out_channels, (kernel_size, kernel_size, 1), stride=stride, padding=padding, dilation=dilation, groups=groups, bias=bias)\n self._stride = stride if isinstance(stride, int) else stride[0]\n self._padding = padding if isinstance(padding, int) else padding[0]\n self._dilation = dilation if isinstance(dilation, int) else dilation[0]\n self._groups = groups\n\n def forward(self, x: torch.Tensor) -> torch.Tensor:\n if not x.is_contiguous():\n x = x.contiguous()\n w = self.conv3d.weight\n if not w.is_contiguous():\n w = w.contiguous()\n b = self.conv3d.bias\n if b is None:\n b = x.new_empty((0,))\n return _ext.conv3d_forward(x, w, b, int(self._stride), int(self._padding), int(self._dilation), int(self._groups))\n"
}