call → submit_kernel
callsubmit_kernel
{
"kernel_code": "import torch\nimport torch.nn as nn\nfrom torch.utils.cpp_extension import load_inline\n\n_cpp_src = r'''\n#include <torch/extension.h>\n\ntorch::Tensor avgpool3d_forward_cuda(torch::Tensor x, int64_t k, int64_t s, int64_t p);\n\ntorch::Tensor avgpool3d_forward(torch::Tensor x, int64_t k, int64_t s, int64_t p) {\n return avgpool3d_forward_cuda(x, k, s, p);\n}\n'''\n\n_cuda_src = r'''\n#include <torch/extension.h>\n#include <ATen/cuda/CUDAContext.h>\n#include <c10/cuda/CUDAException.h>\n\n__device__ __forceinline__ float ldg_f(const float* p) {\n#if __CUDA_ARCH__ >= 350\n return __ldg(p);\n#else\n return *p;\n#endif\n}\n\n__global__ __launch_bounds__(128)\nvoid avgpool3d_k3s2p1_tiled_kernel(const float* __restrict__ x,\n float* __restrict__ y,\n int N, int C, int D, int H, int W,\n int Do, int Ho, int Wo,\n int Dp, int Hp, int Wp,\n long long total_tiles) {\n long long tid = (long long)blockIdx.x * blockDim.x + threadIdx.x;\n if (tid >= total_tiles) return;\n\n int wp = (int)(tid % Wp);\n long long q = tid / Wp;\n int hp = (int)(q % Hp);\n q /= Hp;\n int dp = (int)(q % Dp);\n long long nc = q / Dp;\n\n int od0 = dp << 1;\n int oh0 = hp << 1;\n int ow0 = wp << 1;\n int iz0 = (od0 << 1) - 1;\n int iy0 = (oh0 << 1) - 1;\n int ix0 = (ow0 << 1) - 1;\n\n const long long DHW = (long long)D * H * W;\n const long long HoWo = (long long)Ho * Wo;\n const long long base_in = nc * DHW;\n const long long base_out = nc * (long long)Do * HoWo;\n const float inv27 = 0.037037037037037035f;\n\n // Fast path for the overwhelmingly common, fully-valid 2x2x2 output tile.\n if (od0 + 1 < Do && oh0 + 1 < Ho && ow0 + 1 < Wo &&\n iz0 >= 0 && iy0 >= 0 && ix0 >= 0 &&\n iz0 + 4 < D && iy0 + 4 < H && ix0 + 4 < W) {\n\n float s000 = 0.f, s001 = 0.f, s010 = 0.f, s011 = 0.f;\n float s100 = 0.f, s101 = 0.f, s110 = 0.f, s111 = 0.f;\n\n #pragma unroll\n for (int zz = 0; zz < 5; ++zz) {\n float q00 = 0.f, q01 = 0.f, q10 = 0.f, q11 = 0.f;\n const long long zbase = base_in + (long long)(iz0 + zz) * H * W + (long long)iy0 * W + ix0;\n\n #pragma unroll\n for (int yy = 0; yy < 5; ++yy) {\n const float* row = x + zbase + (long long)yy * W;\n float v0 = ldg_f(row + 0);\n float v1 = ldg_f(row + 1);\n float v2 = ldg_f(row + 2);\n float v3 = ldg_f(row + 3);\n float v4 = ldg_f(row + 4);\n float a = v0 + v1 + v2;\n float b = v2 + v3 + v4;\n if (yy < 3) { q00 += a; q01 += b; }\n if (yy >= 2) { q10 += a; q11 += b; }\n }\n if (zz < 3) { s000 += q00; s001 += q01; s010 += q10; s011 += q11; }\n if (zz >= 2) { s100 += q00; s101 += q01; s110 += q10; s111 += q11; }\n }\n\n long long o_d0 = base_out + (long long)od0 * HoWo;\n long long o_d1 = o_d0 + HoWo;\n long long o_h0 = (long long)oh0 * Wo + ow0;\n long long o_h1 = o_h0 + Wo;\n y[o_d0 + o_h0] = s000 * inv27;\n y[o_d0 + o_h0 + 1] = s001 * inv27;\n y[o_d0 + o_h1] = s010 * inv27;\n y[o_d0 + o_h1 + 1] = s011 * inv27;\n y[o_d1 + o_h0] = s100 * inv27;\n y[o_d1 + o_h0 + 1] = s101 * inv27;\n y[o_d1 + o_h1] = s110 * inv27;\n y[o_d1 + o_h1 + 1] = s111 * inv27;\n return;\n }\n\n // Boundary/odd-size path, still count_include_pad=True (fixed divisor 27).\n #pragma unroll\n for (int dz = 0; dz < 2; ++dz) {\n int od = od0 + dz;\n if (od >= Do) continue;\n int izs = (od << 1) - 1;\n #pragma unroll\n for (int dy = 0; dy < 2; ++dy) {\n int oh = oh0 + dy;\n if (oh >= Ho) continue;\n int iys = (oh << 1) - 1;\n #pragma unroll\n for (int dx = 0; dx < 2; ++dx) {\n int ow = ow0 + dx;\n if (ow >= Wo) continue;\n int ixs = (ow << 1) - 1;\n float sum = 0.f;\n #pragma unroll\n for (int kz = 0; kz < 3; ++kz) {\n int iz = izs + kz;\n if ((unsigned)iz >= (unsigned)D) continue;\n #pragma unroll\n for (int ky = 0; ky < 3; ++ky) {\n int iy = iys + ky;\n if ((unsigned)iy >= (unsigned)H) continue;\n const float* row = x + base_in + (long long)iz * H * W + (long long)iy * W;\n #pragma unroll\n for (int kx = 0; kx < 3; ++kx) {\n int ix = ixs + kx;\n if ((unsigned)ix < (unsigned)W) sum += ldg_f(row + ix);\n }\n }\n }\n y[base_out + (long long)od * HoWo + (long long)oh * Wo + ow] = sum * inv27;\n }\n }\n }\n}\n\n__global__ __launch_bounds__(256)\nvoid avgpool3d_generic_kernel(const float* __restrict__ x,\n float* __restrict__ y,\n int D, int H, int W, int Do, int Ho, int Wo,\n int k, int s, int p, long long total) {\n long long idx = (long long)blockIdx.x * blockDim.x + threadIdx.x;\n if (idx >= total) return;\n int ow = (int)(idx % Wo);\n long long q = idx / Wo;\n int oh = (int)(q % Ho); q /= Ho;\n int od = (int)(q % Do); q /= Do;\n long long nc = q;\n long long base = nc * (long long)D * H * W;\n int izs = od * s - p;\n int iys = oh * s - p;\n int ixs = ow * s - p;\n float sum = 0.f;\n for (int kz = 0; kz < k; ++kz) {\n int iz = izs + kz;\n if ((unsigned)iz >= (unsigned)D) continue;\n for (int ky = 0; ky < k; ++ky) {\n int iy = iys + ky;\n if ((unsigned)iy >= (unsigned)H) continue;\n const float* row = x + base + (long long)iz * H * W + (long long)iy * W;\n for (int kx = 0; kx < k; ++kx) {\n int ix = ixs + kx;\n if ((unsigned)ix < (unsigned)W) sum += ldg_f(row + ix);\n }\n }\n }\n y[idx] = sum / (float)(k * k * k);\n}\n\ntorch::Tensor avgpool3d_forward_cuda(torch::Tensor x, int64_t k64, int64_t s64, int64_t p64) {\n TORCH_CHECK(x.is_cuda(), \"input must be CUDA\");\n TORCH_CHECK(x.scalar_type() == at::kFloat, \"only float32 is supported\");\n TORCH_CHECK(x.dim() == 5, \"expected 5D NCDHW input\");\n\n int N = (int)x.size(0), C = (int)x.size(1), D = (int)x.size(2), H = (int)x.size(3), W = (int)x.size(4);\n int k = (int)k64, s = (int)s64, p = (int)p64;\n int Do = (D + 2 * p - k) / s + 1;\n int Ho = (H + 2 * p - k) / s + 1;\n int Wo = (W + 2 * p - k) / s + 1;\n auto y = torch::empty({N, C, Do, Ho, Wo}, x.options());\n\n const float* xp = x.data_ptr<float>();\n float* yp = y.data_ptr<float>();\n cudaStream_t stream = at::cuda::getCurrentCUDAStream();\n\n if (k == 3 && s == 2 && p == 1) {\n int Dp = (Do + 1) >> 1;\n int Hp = (Ho + 1) >> 1;\n int Wp = (Wo + 1) >> 1;\n long long total_tiles = (long long)N * C * Dp * Hp * Wp;\n dim3 block(128);\n dim3 grid((unsigned int)((total_tiles + block.x - 1) / block.x));\n avgpool3d_k3s2p1_tiled_kernel<<<grid, block, 0, stream>>>(xp, yp, N, C, D, H, W, Do, Ho, Wo, Dp, Hp, Wp, total_tiles);\n } else {\n long long total = (long long)N * C * Do * Ho * Wo;\n dim3 block(256);\n dim3 grid((unsigned int)((total + block.x - 1) / block.x));\n avgpool3d_generic_kernel<<<grid, block, 0, stream>>>(xp, yp, D, H, W, Do, Ho, Wo, k, s, p, total);\n }\n C10_CUDA_KERNEL_LAUNCH_CHECK();\n return y;\n}\n'''\n\n_ext = load_inline(\n name='avgpool3d_custom_ext_v1',\n cpp_sources=_cpp_src,\n cuda_sources=_cuda_src,\n functions=['avgpool3d_forward'],\n extra_cflags=['-O3'],\n extra_cuda_cflags=['-O3'],\n verbose=False,\n)\n\nclass ModelNew(nn.Module):\n def __init__(self, kernel_size: int, stride: int = None, padding: int = 0):\n super(ModelNew, self).__init__()\n self.kernel_size = int(kernel_size)\n self.stride = int(kernel_size if stride is None else stride)\n self.padding = int(padding)\n\n def forward(self, x: torch.Tensor) -> torch.Tensor:\n return _ext.avgpool3d_forward(x, self.kernel_size, self.stride, self.padding)\n"
}