call → submit_kernel
callsubmit_kernel
{
"kernel_code": "import torch\nimport torch.nn as nn\nfrom torch.utils.cpp_extension import load_inline\n\n_cpp_src = r'''\n#include <torch/extension.h>\ntorch::Tensor convt3d_cublas_forward(torch::Tensor x, torch::Tensor weight);\n'''\n\n_cuda_src = r'''\n#include <torch/extension.h>\n#include <ATen/cuda/CUDAContext.h>\n#include <cuda_runtime.h>\n#include <cublas_v2.h>\n#include <stdint.h>\n\n#define CHECK_CUDA(x) TORCH_CHECK(x.is_cuda(), #x \" must be CUDA\")\n#define CHECK_CONTIGUOUS(x) TORCH_CHECK(x.is_contiguous(), #x \" must be contiguous\")\n\nstatic inline int grid_for(long long n) {\n long long g = (n + 255LL) / 256LL;\n return (int)(g < 65535LL ? g : 65535LL);\n}\n\n__global__ void pack_w_kernel(const float* __restrict__ w, float* __restrict__ wp, int IC, int OC) {\n int idx = blockIdx.x * blockDim.x + threadIdx.x;\n int total = 27 * OC * IC;\n if (idx < total) {\n int ic = idx % IC;\n int t = idx / IC;\n int oc = t % OC;\n int sh = t / OC;\n // wp[sh, oc, ic] = weight[ic, oc, kd, kh, kw]\n wp[((sh * OC + oc) * IC) + ic] = w[((long long)ic * OC + oc) * 27 + sh];\n }\n}\n\n__global__ void fill_ptrs_kernel(const float* x, const float* wp, float* temp,\n unsigned long long* Ap, unsigned long long* Bp, unsigned long long* Cp,\n int Bsz, int IC, int OC, long long S) {\n int idx = blockIdx.x * blockDim.x + threadIdx.x;\n int total = 27 * Bsz;\n if (idx < total) {\n int n = idx % Bsz;\n int sh = idx / Bsz;\n Ap[idx] = (unsigned long long)(x + (long long)n * IC * S);\n Bp[idx] = (unsigned long long)(wp + (long long)sh * OC * IC);\n Cp[idx] = (unsigned long long)(temp + (long long)idx * OC * S);\n }\n}\n\n__global__ void reduce_kernel(const float* __restrict__ temp, float* __restrict__ y,\n int Bsz, int OC, int D, int H, int W, int OD, int OH, int OW,\n long long S, long long total) {\n long long idx = blockIdx.x * blockDim.x + threadIdx.x;\n long long st = (long long)blockDim.x * gridDim.x;\n for (; idx < total; idx += st) {\n int ow = idx % OW;\n long long q = idx / OW;\n int oh = q % OH; q /= OH;\n int od = q % OD; q /= OD;\n int oc = q % OC;\n int n = q / OC;\n\n float sum = 0.0f;\n int kd0 = od - (D - 1); if (kd0 < 0) kd0 = 0;\n int kd1 = od < 2 ? od : 2;\n int kh0 = oh - (H - 1); if (kh0 < 0) kh0 = 0;\n int kh1 = oh < 2 ? oh : 2;\n int kw0 = ow - (W - 1); if (kw0 < 0) kw0 = 0;\n int kw1 = ow < 2 ? ow : 2;\n\n for (int kd = kd0; kd <= kd1; ++kd) {\n int id = od - kd;\n for (int kh = kh0; kh <= kh1; ++kh) {\n int ih = oh - kh;\n for (int kw = kw0; kw <= kw1; ++kw) {\n int iw = ow - kw;\n int sh = kd * 9 + kh * 3 + kw;\n long long ti = (((long long)sh * Bsz + n) * OC + oc) * S + ((long long)id * H + ih) * W + iw;\n sum += temp[ti];\n }\n }\n }\n y[idx] = sum;\n }\n}\n\ntorch::Tensor convt3d_cublas_forward(torch::Tensor x, torch::Tensor weight) {\n CHECK_CUDA(x); CHECK_CUDA(weight);\n CHECK_CONTIGUOUS(x); CHECK_CONTIGUOUS(weight);\n TORCH_CHECK(x.scalar_type() == torch::kFloat32, \"x fp32\");\n TORCH_CHECK(weight.scalar_type() == torch::kFloat32, \"w fp32\");\n\n int Bsz = (int)x.size(0);\n int IC = (int)x.size(1);\n int D = (int)x.size(2);\n int H = (int)x.size(3);\n int W = (int)x.size(4);\n int OC = (int)weight.size(1);\n TORCH_CHECK(weight.size(2) == 3 && weight.size(3) == 3 && weight.size(4) == 3, \"only k3\");\n\n int OD = D + 2, OH = H + 2, OW = W + 2;\n long long S = (long long)D * H * W;\n int batch = 27 * Bsz;\n\n auto y = torch::empty({Bsz, OC, OD, OH, OW}, x.options());\n auto temp = torch::empty({batch, OC, D, H, W}, x.options());\n auto wp = torch::empty({27, OC, IC}, x.options());\n auto ptr_opts = torch::TensorOptions().dtype(torch::kInt64).device(x.device());\n auto Ap = torch::empty({batch}, ptr_opts);\n auto Bp = torch::empty({batch}, ptr_opts);\n auto Cp = torch::empty({batch}, ptr_opts);\n\n cudaStream_t stream = at::cuda::getCurrentCUDAStream();\n pack_w_kernel<<<grid_for(27LL * OC * IC), 256, 0, stream>>>(weight.data_ptr<float>(), wp.data_ptr<float>(), IC, OC);\n fill_ptrs_kernel<<<grid_for(batch), 256, 0, stream>>>(\n x.data_ptr<float>(), wp.data_ptr<float>(), temp.data_ptr<float>(),\n (unsigned long long*)Ap.data_ptr<int64_t>(), (unsigned long long*)Bp.data_ptr<int64_t>(), (unsigned long long*)Cp.data_ptr<int64_t>(),\n Bsz, IC, OC, S);\n\n cublasHandle_t handle = at::cuda::getCurrentCUDABlasHandle();\n cublasSetStream(handle, stream);\n cublasSetMathMode(handle, CUBLAS_TF32_TENSOR_OP_MATH);\n float alpha = 1.0f, beta = 0.0f;\n cublasStatus_t st = cublasSgemmBatched(handle, CUBLAS_OP_N, CUBLAS_OP_N,\n (int)S, OC, IC,\n &alpha,\n (const float**)Ap.data_ptr<int64_t>(), (int)S,\n (const float**)Bp.data_ptr<int64_t>(), IC,\n &beta,\n (float**)Cp.data_ptr<int64_t>(), (int)S,\n batch);\n TORCH_CHECK(st == CUBLAS_STATUS_SUCCESS, \"cublas failed\");\n\n reduce_kernel<<<grid_for(y.numel()), 256, 0, stream>>>(temp.data_ptr<float>(), y.data_ptr<float>(),\n Bsz, OC, D, H, W, OD, OH, OW, S, y.numel());\n return y;\n}\n'''\n\n_ext = load_inline(\n name='convt3d_final_ext',\n cpp_sources=_cpp_src,\n cuda_sources=_cuda_src,\n functions=['convt3d_cublas_forward'],\n extra_cuda_cflags=['-O3'],\n extra_cflags=['-O3'],\n extra_ldflags=['-lcublas'],\n verbose=False,\n)\n\nclass ModelNew(nn.Module):\n def __init__(self, in_channels: int, out_channels: int, kernel_size: int, stride: int = 1,\n padding: int = 0, output_padding: int = 0, dilation: int = 1,\n groups: int = 1, bias: bool = False):\n super().__init__()\n self.conv_transpose3d = nn.ConvTranspose3d(in_channels, out_channels,\n (kernel_size, kernel_size, kernel_size),\n stride=stride, padding=padding,\n output_padding=output_padding,\n dilation=dilation, groups=groups, bias=bias)\n\n def forward(self, x: torch.Tensor) -> torch.Tensor:\n return _ext.convt3d_cublas_forward(x.contiguous(), self.conv_transpose3d.weight.contiguous())\n"
}