call → submit_kernel
callsubmit_kernel
{
"kernel_code": "import torch\nimport torch.nn as nn\nfrom torch.utils.cpp_extension import load_inline\n\n_cpp_src = r'''\n#include <torch/extension.h>\n\ntorch::Tensor conv3d_cuda(torch::Tensor x, torch::Tensor weight, torch::Tensor bias,\n int64_t sD, int64_t sH, int64_t sW,\n int64_t pD, int64_t pH, int64_t pW,\n int64_t dD, int64_t dH, int64_t dW);\n\ntorch::Tensor conv3d_forward(torch::Tensor x, torch::Tensor weight, torch::Tensor bias,\n int64_t sD, int64_t sH, int64_t sW,\n int64_t pD, int64_t pH, int64_t pW,\n int64_t dD, int64_t dH, int64_t dW) {\n return conv3d_cuda(x, weight, bias, sD, sH, sW, pD, pH, pW, dD, dH, dW);\n}\n'''\n\n_cuda_src = r'''\n#include <torch/extension.h>\n#include <ATen/cuda/CUDAContext.h>\n#include <cuda.h>\n#include <cuda_runtime.h>\n\n__global__ __launch_bounds__(256)\nvoid conv3d_3_64_3x5x7_d2_kernel(const float* __restrict__ x,\n const float* __restrict__ w,\n const float* __restrict__ b,\n float* __restrict__ y,\n int N, int D, int H, int W,\n int Do, int Ho, int Wo,\n int has_bias) {\n extern __shared__ float sw[];\n const int tx = threadIdx.x; // 0..63, output W\n const int ty = threadIdx.y; // 0..3, output H\n const int tid = ty * 64 + tx;\n\n int gz = blockIdx.z;\n const int od_pair = gz % ((Do + 1) >> 1);\n gz /= ((Do + 1) >> 1);\n const int oc = gz & 63; // OC == 64\n const int n = gz >> 6;\n\n for (int i = tid; i < 315; i += 256) {\n sw[i] = w[oc * 315 + i];\n }\n __syncthreads();\n\n const int ow = blockIdx.x * 64 + tx;\n const int oh = blockIdx.y * 4 + ty;\n const int od0 = od_pair << 1;\n\n if (ow >= Wo || oh >= Ho || n >= N || od0 >= Do) return;\n\n float sum0 = has_bias ? b[oc] : 0.0f;\n float sum1 = has_bias ? b[oc] : 0.0f;\n\n const int HW = H * W;\n const int in_n_base = n * 3 * D * HW;\n\n #pragma unroll\n for (int c = 0; c < 3; ++c) {\n const int cbase = in_n_base + c * D * HW;\n #pragma unroll\n for (int kh = 0; kh < 5; ++kh) {\n const int row_base = cbase + od0 * HW + (oh + kh) * W + ow;\n #pragma unroll\n for (int kw = 0; kw < 7; ++kw) {\n const float v0 = x[row_base + kw];\n const float v1 = x[row_base + HW + kw];\n const float v2 = x[row_base + 2 * HW + kw];\n const float v3 = x[row_base + 3 * HW + kw];\n const int wbase = c * 105 + kh * 7 + kw;\n const float w0 = sw[wbase];\n const float w1 = sw[wbase + 35];\n const float w2 = sw[wbase + 70];\n sum0 = fmaf(v0, w0, sum0);\n sum0 = fmaf(v1, w1, sum0);\n sum0 = fmaf(v2, w2, sum0);\n sum1 = fmaf(v1, w0, sum1);\n sum1 = fmaf(v2, w1, sum1);\n sum1 = fmaf(v3, w2, sum1);\n }\n }\n }\n\n const int out_base = ((n * 64 + oc) * Do + od0) * Ho * Wo + oh * Wo + ow;\n y[out_base] = sum0;\n if (od0 + 1 < Do) {\n y[out_base + Ho * Wo] = sum1;\n }\n}\n\n__global__ __launch_bounds__(256)\nvoid conv3d_generic_kernel(const float* __restrict__ x,\n const float* __restrict__ w,\n const float* __restrict__ b,\n float* __restrict__ y,\n long total,\n int N, int C, int D, int H, int Wd,\n int OC, int kD, int kH, int kW,\n int Do, int Ho, int Wo,\n int sD, int sH, int sW,\n int pD, int pH, int pW,\n int dilD, int dilH, int dilW,\n int groups, int has_bias) {\n long idx = (long)blockIdx.x * blockDim.x + threadIdx.x;\n if (idx >= total) return;\n\n int ow = idx % Wo; idx /= Wo;\n int oh = idx % Ho; idx /= Ho;\n int od = idx % Do; idx /= Do;\n int oc = idx % OC; idx /= OC;\n int n = (int)idx;\n\n int ocpg = OC / groups;\n int cpg = C / groups;\n int g = oc / ocpg;\n int c_start = g * cpg;\n\n float sum = has_bias ? b[oc] : 0.0f;\n for (int ci = 0; ci < cpg; ++ci) {\n int c = c_start + ci;\n for (int kd = 0; kd < kD; ++kd) {\n int id = od * sD - pD + kd * dilD;\n if ((unsigned)id >= (unsigned)D) continue;\n for (int kh = 0; kh < kH; ++kh) {\n int ih = oh * sH - pH + kh * dilH;\n if ((unsigned)ih >= (unsigned)H) continue;\n for (int kw = 0; kw < kW; ++kw) {\n int iw = ow * sW - pW + kw * dilW;\n if ((unsigned)iw >= (unsigned)Wd) continue;\n float xv = x[((n * C + c) * D + id) * H * Wd + ih * Wd + iw];\n float wv = w[(((oc * cpg + ci) * kD + kd) * kH + kh) * kW + kw];\n sum = fmaf(xv, wv, sum);\n }\n }\n }\n }\n y[(((n * OC + oc) * Do + od) * Ho + oh) * Wo + ow] = sum;\n}\n\ntorch::Tensor conv3d_cuda(torch::Tensor x, torch::Tensor weight, torch::Tensor bias,\n int64_t sD64, int64_t sH64, int64_t sW64,\n int64_t pD64, int64_t pH64, int64_t pW64,\n int64_t dD64, int64_t dH64, int64_t dW64) {\n const int N = (int)x.size(0);\n const int C = (int)x.size(1);\n const int D = (int)x.size(2);\n const int H = (int)x.size(3);\n const int Wd = (int)x.size(4);\n const int OC = (int)weight.size(0);\n const int Cw = (int)weight.size(1);\n const int kD = (int)weight.size(2);\n const int kH = (int)weight.size(3);\n const int kW = (int)weight.size(4);\n const int sD = (int)sD64, sH = (int)sH64, sW = (int)sW64;\n const int pD = (int)pD64, pH = (int)pH64, pW = (int)pW64;\n const int dilD = (int)dD64, dilH = (int)dH64, dilW = (int)dW64;\n const int Do = (D + 2 * pD - dilD * (kD - 1) - 1) / sD + 1;\n const int Ho = (H + 2 * pH - dilH * (kH - 1) - 1) / sH + 1;\n const int Wo = (Wd + 2 * pW - dilW * (kW - 1) - 1) / sW + 1;\n auto y = torch::empty({N, OC, Do, Ho, Wo}, x.options());\n if (y.numel() == 0) return y;\n\n const int has_bias = (bias.numel() != 0);\n cudaStream_t stream = at::cuda::getCurrentCUDAStream();\n\n if (x.scalar_type() == at::kFloat && weight.scalar_type() == at::kFloat &&\n C == 3 && OC == 64 && Cw == 3 && kD == 3 && kH == 5 && kW == 7 &&\n sD == 1 && sH == 1 && sW == 1 && pD == 0 && pH == 0 && pW == 0 &&\n dilD == 1 && dilH == 1 && dilW == 1) {\n dim3 block(64, 4, 1);\n dim3 grid((Wo + 63) / 64, (Ho + 3) / 4, N * OC * ((Do + 1) / 2));\n conv3d_3_64_3x5x7_d2_kernel<<<grid, block, 315 * sizeof(float), stream>>>(\n x.data_ptr<float>(), weight.data_ptr<float>(),\n has_bias ? bias.data_ptr<float>() : nullptr, y.data_ptr<float>(),\n N, D, H, Wd, Do, Ho, Wo, has_bias);\n } else {\n int groups = C / Cw;\n long total = (long)N * OC * Do * Ho * Wo;\n dim3 block(256);\n dim3 grid((total + 255) / 256);\n conv3d_generic_kernel<<<grid, block, 0, stream>>>(\n x.data_ptr<float>(), weight.data_ptr<float>(),\n has_bias ? bias.data_ptr<float>() : nullptr, y.data_ptr<float>(), total,\n N, C, D, H, Wd, OC, kD, kH, kW, Do, Ho, Wo,\n sD, sH, sW, pD, pH, pW, dilD, dilH, dilW, groups, has_bias);\n }\n return y;\n}\n'''\n\n_ext = load_inline(\n name='conv3d_asym_kernel_ext_v1',\n cpp_sources=_cpp_src,\n cuda_sources=_cuda_src,\n functions=['conv3d_forward'],\n extra_cflags=['-O3'],\n extra_cuda_cflags=['-O3', '--use_fast_math'],\n verbose=False,\n)\n\n\ndef _triple(v):\n if isinstance(v, tuple):\n return v\n return (v, v, v)\n\nclass ModelNew(nn.Module):\n def __init__(self, in_channels: int, out_channels: int, kernel_size: tuple, stride: int = 1, padding: int = 0, dilation: int = 1, groups: int = 1, bias: bool = False):\n super(ModelNew, self).__init__()\n self.conv3d = nn.Conv3d(in_channels, out_channels, kernel_size, stride=stride, padding=padding, dilation=dilation, groups=groups, bias=bias)\n\n def forward(self, x: torch.Tensor) -> torch.Tensor:\n st = self.conv3d.stride\n pa = self.conv3d.padding\n di = self.conv3d.dilation\n b = self.conv3d.bias\n if b is None:\n b = torch.empty(0, device=x.device, dtype=x.dtype)\n return _ext.conv3d_forward(x.contiguous(), self.conv3d.weight.contiguous(), b,\n int(st[0]), int(st[1]), int(st[2]),\n int(pa[0]), int(pa[1]), int(pa[2]),\n int(di[0]), int(di[1]), int(di[2]))\n"
}