call → submit_kernel
callsubmit_kernel
{
"kernel_code": "import torch\nimport torch.nn as nn\nfrom torch.utils.cpp_extension import load_inline\n\n_cuda_src = r'''\n#include <torch/extension.h>\n#include <ATen/cuda/CUDAContext.h>\n#include <cuda.h>\n#include <cuda_runtime.h>\n#include <float.h>\n\n// Convert PyTorch conv weight layout [OC,C,3,3] to [C*3*3,OC].\n// The transformed matrix is tiny (128*64*9 floats) and stays resident in cache\n// during the fused convolution/min kernel.\n__global__ void transform_w_3x3_128_kernel(const float* __restrict__ w,\n float* __restrict__ wt) {\n int idx = blockIdx.x * blockDim.x + threadIdx.x;\n const int total = 128 * 64 * 9;\n if (idx >= total) return;\n int rem = idx;\n int kw = rem % 3; rem /= 3;\n int kh = rem % 3; rem /= 3;\n int c = rem % 64; rem /= 64;\n int oc = rem;\n int r = (c * 9 + kh * 3 + kw) * 128 + oc;\n wt[r] = w[idx];\n}\n\n__device__ __forceinline__ float warp_min(float v) {\n unsigned mask = 0xffffffffu;\n v = fminf(v, __shfl_down_sync(mask, v, 16));\n v = fminf(v, __shfl_down_sync(mask, v, 8));\n v = fminf(v, __shfl_down_sync(mask, v, 4));\n v = fminf(v, __shfl_down_sync(mask, v, 2));\n v = fminf(v, __shfl_down_sync(mask, v, 1));\n return v;\n}\n\n// One warp computes one output pixel. Each lane computes four output channels\n// (lane, lane+32, lane+64, lane+96); the warp then reduces the 128 channel\n// values to the required channel-minimum. The convolution, scale and reduction\n// are fully fused, avoiding the 64*128*254*254 intermediate tensor and the\n// separate multiply/min kernels in the reference.\n__global__ __launch_bounds__(256, 2)\nvoid conv3x3_min_scale_64_128_kernel(const float* __restrict__ x,\n const float* __restrict__ wt,\n const float* __restrict__ bias,\n float* __restrict__ out,\n int N, int H, int W,\n float scale) {\n const int lane = threadIdx.x & 31;\n const int warp_in_block = threadIdx.x >> 5;\n const int warps_per_block = blockDim.x >> 5;\n const int OH = H - 2;\n const int OW = W - 2;\n const int64_t num_pix = (int64_t)N * OH * OW;\n int64_t pix = (int64_t)blockIdx.x * warps_per_block + warp_in_block;\n if (pix >= num_pix) return;\n\n int ow = (int)(pix % OW);\n int t = (int)(pix / OW);\n int oh = t % OH;\n int n = t / OH;\n\n float a0 = __ldg(bias + lane);\n float a1 = __ldg(bias + lane + 32);\n float a2 = __ldg(bias + lane + 64);\n float a3 = __ldg(bias + lane + 96);\n\n // Base pointer for this image. Input is NCHW contiguous.\n const int img_stride = 64 * H * W;\n const float* xN = x + (int64_t)n * img_stride;\n\n #pragma unroll 1\n for (int c = 0; c < 64; ++c) {\n const float* xc = xN + (int64_t)c * H * W + oh * W + ow;\n int wr = c * 9 * 128 + lane;\n\n float xv;\n xv = __ldg(xc + 0);\n a0 = fmaf(xv, __ldg(wt + wr + 0), a0);\n a1 = fmaf(xv, __ldg(wt + wr + 32), a1);\n a2 = fmaf(xv, __ldg(wt + wr + 64), a2);\n a3 = fmaf(xv, __ldg(wt + wr + 96), a3);\n\n xv = __ldg(xc + 1); wr += 128;\n a0 = fmaf(xv, __ldg(wt + wr + 0), a0);\n a1 = fmaf(xv, __ldg(wt + wr + 32), a1);\n a2 = fmaf(xv, __ldg(wt + wr + 64), a2);\n a3 = fmaf(xv, __ldg(wt + wr + 96), a3);\n\n xv = __ldg(xc + 2); wr += 128;\n a0 = fmaf(xv, __ldg(wt + wr + 0), a0);\n a1 = fmaf(xv, __ldg(wt + wr + 32), a1);\n a2 = fmaf(xv, __ldg(wt + wr + 64), a2);\n a3 = fmaf(xv, __ldg(wt + wr + 96), a3);\n\n xc += W; wr += 128;\n xv = __ldg(xc + 0);\n a0 = fmaf(xv, __ldg(wt + wr + 0), a0);\n a1 = fmaf(xv, __ldg(wt + wr + 32), a1);\n a2 = fmaf(xv, __ldg(wt + wr + 64), a2);\n a3 = fmaf(xv, __ldg(wt + wr + 96), a3);\n\n xv = __ldg(xc + 1); wr += 128;\n a0 = fmaf(xv, __ldg(wt + wr + 0), a0);\n a1 = fmaf(xv, __ldg(wt + wr + 32), a1);\n a2 = fmaf(xv, __ldg(wt + wr + 64), a2);\n a3 = fmaf(xv, __ldg(wt + wr + 96), a3);\n\n xv = __ldg(xc + 2); wr += 128;\n a0 = fmaf(xv, __ldg(wt + wr + 0), a0);\n a1 = fmaf(xv, __ldg(wt + wr + 32), a1);\n a2 = fmaf(xv, __ldg(wt + wr + 64), a2);\n a3 = fmaf(xv, __ldg(wt + wr + 96), a3);\n\n xc += W; wr += 128;\n xv = __ldg(xc + 0);\n a0 = fmaf(xv, __ldg(wt + wr + 0), a0);\n a1 = fmaf(xv, __ldg(wt + wr + 32), a1);\n a2 = fmaf(xv, __ldg(wt + wr + 64), a2);\n a3 = fmaf(xv, __ldg(wt + wr + 96), a3);\n\n xv = __ldg(xc + 1); wr += 128;\n a0 = fmaf(xv, __ldg(wt + wr + 0), a0);\n a1 = fmaf(xv, __ldg(wt + wr + 32), a1);\n a2 = fmaf(xv, __ldg(wt + wr + 64), a2);\n a3 = fmaf(xv, __ldg(wt + wr + 96), a3);\n\n xv = __ldg(xc + 2); wr += 128;\n a0 = fmaf(xv, __ldg(wt + wr + 0), a0);\n a1 = fmaf(xv, __ldg(wt + wr + 32), a1);\n a2 = fmaf(xv, __ldg(wt + wr + 64), a2);\n a3 = fmaf(xv, __ldg(wt + wr + 96), a3);\n }\n\n a0 *= scale; a1 *= scale; a2 *= scale; a3 *= scale;\n float m = fminf(fminf(a0, a1), fminf(a2, a3));\n m = warp_min(m);\n if (lane == 0) out[pix] = m;\n}\n\ntorch::Tensor fused_conv_min_forward(torch::Tensor x, torch::Tensor weight,\n torch::Tensor bias, double scale) {\n TORCH_CHECK(x.is_cuda() && weight.is_cuda() && bias.is_cuda(), \"all tensors must be CUDA tensors\");\n TORCH_CHECK(x.scalar_type() == torch::kFloat32, \"only float32 is supported\");\n TORCH_CHECK(weight.scalar_type() == torch::kFloat32 && bias.scalar_type() == torch::kFloat32, \"only float32 weights are supported\");\n TORCH_CHECK(x.is_contiguous() && weight.is_contiguous() && bias.is_contiguous(), \"tensors must be contiguous\");\n TORCH_CHECK(x.dim() == 4 && weight.dim() == 4, \"bad tensor rank\");\n TORCH_CHECK(x.size(1) == 64 && weight.size(0) == 128 && weight.size(1) == 64 && weight.size(2) == 3 && weight.size(3) == 3,\n \"this optimized kernel expects C=64, OC=128, K=3\");\n\n const int N = (int)x.size(0);\n const int H = (int)x.size(2);\n const int W = (int)x.size(3);\n auto wt = torch::empty({64 * 9, 128}, x.options());\n auto out = torch::empty({N, 1, H - 2, W - 2}, x.options());\n\n cudaStream_t stream = at::cuda::getCurrentCUDAStream();\n transform_w_3x3_128_kernel<<<(128*64*9 + 255) / 256, 256, 0, stream>>>(\n weight.data_ptr<float>(), wt.data_ptr<float>());\n\n const int threads = 256;\n const int warps = threads / 32;\n int64_t num_pix = (int64_t)N * (H - 2) * (W - 2);\n int blocks = (int)((num_pix + warps - 1) / warps);\n conv3x3_min_scale_64_128_kernel<<<blocks, threads, 0, stream>>>(\n x.data_ptr<float>(), wt.data_ptr<float>(), bias.data_ptr<float>(), out.data_ptr<float>(),\n N, H, W, (float)scale);\n return out;\n}\n'''\n\n_ext = load_inline(\n name='fused_conv_min_ext_v1',\n cpp_sources='',\n cuda_sources=_cuda_src,\n functions=['fused_conv_min_forward'],\n extra_cuda_cflags=['-O3', '--use_fast_math'],\n extra_cflags=['-O3'],\n verbose=False,\n)\n\nclass ModelNew(nn.Module):\n def __init__(self, in_channels, out_channels, kernel_size, scale_factor):\n super(ModelNew, self).__init__()\n self.conv = nn.Conv2d(in_channels, out_channels, kernel_size)\n self.scale_factor = float(scale_factor)\n\n def forward(self, x):\n return _ext.fused_conv_min_forward(x.contiguous(), self.conv.weight.contiguous(), self.conv.bias.contiguous(), self.scale_factor)\n"
}