correct
40_SmithWatermanDPScore.py
started
2026-05-11T14:32:38Z
timingkernelbench.timing
kernel mean / median / std
0.21 / — / 7.70e-04 μs
ref mean / median / std
128.00 / — / 7.25 μs
numerical precisioneval_kernel_against_ref
gpu memoryextended_metrics.measure_memory
kernel launches / fusiontorch.profiler
fusion ratio (ref / kernel)
12291.20x
energyNVML (pynvml)
ratio (ref / kernel)
69.01x
SOL (speed-of-light)nsight_hardware_counters
roofline / occupancynsight_hardware_counters
warp stalls (top 5)nsight
reference PyTorch source (the prompt the agent sees)
import torch
import torch.nn as nn
import torch.nn.functional as F
class Model(nn.Module):
"""
Differentiable Smith-Waterman-style local sequence alignment scoring.
Uses a soft-max relaxation of the DP recurrence so gradients can flow
through the alignment matrix.
"""
def __init__(self, gap_open=-1.0, temperature=1.0):
super().__init__()
self.gap_open = gap_open
self.temperature = temperature
def forward(self, score_matrix: torch.Tensor) -> torch.Tensor:
"""
Args:
score_matrix: (B, M, N) – per-position similarity scores
Returns:
alignment_score: (B,) – soft local-alignment score (log-sum-exp over DP cells)
"""
B, M, N = score_matrix.shape
neg_inf = torch.tensor(float("-inf"), device=score_matrix.device, dtype=score_matrix.dtype)
# H[i][j] = max(0, H[i-1][j-1]+S[i][j], H[i-1][j]+gap, H[i][j-1]+gap)
H = score_matrix.new_full((B, M + 1, N + 1), 0.0)
for i in range(1, M + 1):
for j in range(1, N + 1):
match = H[:, i - 1, j - 1] + score_matrix[:, i - 1, j - 1]
delete = H[:, i - 1, j] + self.gap_open
insert = H[:, i, j - 1] + self.gap_open
zero = torch.zeros(B, device=score_matrix.device)
stack = torch.stack([zero, match, delete, insert], dim=-1)
H[:, i, j] = self.temperature * torch.logsumexp(stack / self.temperature, dim=-1)
return H.view(B, -1).max(dim=-1).values
seq_m = 32
seq_n = 32
batch_size = 16
def get_inputs():
return [torch.randn(popcorn_pri.jitter_int(batch_size), seq_m, seq_n)]
def get_init_inputs():
return []
turn 0550.5s turn · 3.0s LLM · 547.5s tools · tokens in=1,709 · out=192 · total=1,901
reasoning
Let me understand the task:
1. I need to replace a Smith-Waterman-style local sequence alignment scoring with a custom CUDA kernel
2. The forward pass takes a score_matrix of shape (B, M, N) and computes a soft local-alignment score
3. The algorithm uses a logsumexp relaxation of the max operation over the DP recurrence
Key observations:
- The algorithm is a dynamic programming approach with dependencies: H[i][j] depends on H[i-1][j-1], H[i-1][j], and H[i][j-1]
- This creates a data dependency that's tricky to parallelize naively
- The final output is the max over all cells of H
Let me first get GPU specs to understand the hardware constraints.
assistantI'll start by understanding the GPU specifications and then develop a custom CUDA kernel for this Smith-Waterman-style local alignment scoring.
call → get_gpu_specs
callget_gpu_specs
{}
result ← get_gpu_specs (ok)
okget_gpu_specs
GPU specs for NVIDIA H100 80GB HBM3:
total memory (runtime): 79.1 GB
spec entry: H100
GPU Architecture: Hopper
GPU Memory: 80GB
Memory Bandwidth: 3.35 TB/s
FP64 TFLOPS: 34
FP64 Tensor Core TFLOPS: 67
FP32 TFLOPS: 67
TF32 Tensor Core TFLOPS: 989 with sparsity
BFLOAT16 Tensore Core TFLOPS: 1979 with sparsity
FP16 Tensor Core TFLOPS: 1979 with sparsity
FP8 Tensor Core TFLOPS: 3958 with sparsity
INT8 Tensor Core TOPS: 3958 with sparsity
Register File Size: 64K 32-bit registers per SM
Maximum number of registers per thread: 255
Maximum number of thread blocks per SM: 32
Shared memory capacity per SM: 228 KB
Maximum shared memory per thread block: 227 KB
turn 1 · 1 tool call(s) · 916.3s turn · 13.5s LLM · tokens in=1,983 · out=1,376 · total=3,359 · 6,153 chars (click to expand)
turn 1916.3s turn · 13.5s LLM · 902.8s tools · tokens in=1,983 · out=1,376 · total=3,359
reasoning
Now let me understand the problem better. This is a Smith-Waterman-style alignment where:
- We have a batch of score matrices (B, M, N)
- We compute a DP matrix H where each cell depends on three previous cells (diagonal, top, left)
- We use soft-max relaxation with logsumexp
- The final output is the maximum value in each batch
The key insight is that each row of the DP matrix can be computed in parallel within the row, but rows depend on previous rows. So the natural parallelization is:
- Each thread handles one batch element
- Within each thread, we iterate through rows, and for each row we iterate through columns
Let me write a first kernel that does this.
assistantLet me analyze the algorithm and write an optimized CUDA kernel. This is a Smith-Waterman DP with soft-max relaxation where each cell depends on the previous row and column values.
call → compile_kernel
callcompile_kernel
{
"kernel_code": "import torch\nimport torch.nn as nn\nfrom torch.utils.cpp_extension import load_inline\n\ncuda_source = \"\"\"\n#include <torch/extension.h>\n#include <cuda.h>\n#include <cuda_runtime.h>\n#include <c10/cuda/CUDAException.h>\n\ntemplate <typename scalar_t>\n__global__ void sw_forward_kernel(\n const scalar_t* __restrict__ score_matrix,\n scalar_t* __restrict__ H,\n scalar_t* __restrict__ output,\n int B, int M, int N, \n scalar_t gap_open, \n scalar_t temperature) {\n \n int batch_idx = blockIdx.x * blockDim.x + threadIdx.x;\n if (batch_idx >= B) return;\n \n // Each thread handles one batch element\n // H has shape (B, M+1, N+1), first row and column are 0\n \n // Initialize first row and column to 0\n for (int j = 0; j <= N; j++) {\n H[batch_idx * (M+1) * (N+1) + j] = 0.0f;\n }\n for (int i = 1; i <= M; i++) {\n H[batch_idx * (M+1) * (N+1) + i * (N+1)] = 0.0f;\n }\n \n scalar_t max_val = 0.0f;\n \n // Iterate through the DP matrix\n for (int i = 1; i <= M; i++) {\n for (int j = 1; j <= N; j++) {\n int idx_prev_row = batch_idx * (M+1) * (N+1) + (i-1) * (N+1) + (j-1);\n int idx_curr = batch_idx * (M+1) * (N+1) + i * (N+1) + j;\n \n scalar_t h_diag = H[idx_prev_row]; // H[i-1][j-1]\n scalar_t h_up = H[idx_prev_row + 1]; // H[i-1][j]\n scalar_t h_left = H[idx_curr - 1]; // H[i][j-1]\n \n scalar_t s = score_matrix[batch_idx * M * N + (i-1) * N + (j-1)];\n \n scalar_t match = h_diag + s;\n scalar_t del = h_up + gap_open;\n scalar_t ins = h_left + gap_open;\n \n // logsumexp over [0, match, del, ins]\n scalar_t inv_temp = 1.0f / temperature;\n scalar_t a0 = 0.0f * inv_temp;\n scalar_t a1 = match * inv_temp;\n scalar_t a2 = del * inv_temp;\n scalar_t a3 = ins * inv_temp;\n \n scalar_t max_a = a0;\n max_a = fmaxf(max_a, a1);\n max_a = fmaxf(max_a, a2);\n max_a = fmaxf(max_a, a3);\n \n scalar_t sum_exp = expf(a0 - max_a) + expf(a1 - max_a) + expf(a2 - max_a) + expf(a3 - max_a);\n scalar_t logsumexp = max_a + logf(sum_exp);\n \n scalar_t h_val = temperature * logsumexp;\n H[idx_curr] = h_val;\n \n if (h_val > max_val) max_val = h_val;\n }\n }\n \n output[batch_idx] = max_val;\n}\n\ntorch::Tensor sw_forward_cuda(torch::Tensor score_matrix, double gap_open, double temperature) {\n TORCH_CHECK(score_matrix.is_cuda(), \"score_matrix must be a CUDA tensor\");\n TORCH_CHECK(score_matrix.dim() == 3, \"score_matrix must be 3D\");\n \n int B = score_matrix.size(0);\n int M = score_matrix.size(1);\n int N = score_matrix.size(2);\n \n auto options = score_matrix.options();\n auto H = torch::zeros({B, M + 1, N + 1}, options);\n auto output = torch::empty({B}, options);\n \n int threads = 256;\n int blocks = (B + threads - 1) / threads;\n \n AT_DISPATCH_FLOATING_TYPES(score_matrix.scalar_type(), \"sw_forward_kernel\", ([&] {\n sw_forward_kernel<scalar_t><<<blocks, threads>>>(\n score_matrix.data_ptr<scalar_t>(),\n H.data_ptr<scalar_t>(),\n output.data_ptr<scalar_t>(),\n B, M, N,\n static_cast<scalar_t>(gap_open),\n static_cast<scalar_t>(temperature)\n );\n }));\n \n C10_CUDA_CHECK(cudaGetLastError());\n return output;\n}\n\"\"\"\n\ncpp_source = \"\"\"\ntorch::Tensor sw_forward_cuda(torch::Tensor score_matrix, double gap_open, double temperature);\n\"\"\"\n\nsw_module = load_inline(\n name='sw_module',\n cpp_sources=cpp_source,\n cuda_sources=cuda_source,\n functions=['sw_forward_cuda'],\n verbose=True,\n)\n\nclass ModelNew(nn.Module):\n def __init__(self, gap_open=-1.0, temperature=1.0):\n super().__init__()\n self.gap_open = gap_open\n self.temperature = temperature\n\n def forward(self, score_matrix: torch.Tensor) -> torch.Tensor:\n return sw_module.sw_forward_cuda(score_matrix, self.gap_open, self.temperature)"
}
result ← compile_kernel (ok)
okcompile_kernel
compile_kernel PASSED: kernel compiled without errors.
turn 2678.5s turn · 10.5s LLM · 667.9s tools · tokens in=3,220 · out=1,202 · total=4,422
reasoning
Good, the kernel compiles. Now let me check correctness.
call → run_correctness
callrun_correctness
{
"kernel_code": "import torch\nimport torch.nn as nn\nfrom torch.utils.cpp_extension import load_inline\n\ncuda_source = \"\"\"\n#include <torch/extension.h>\n#include <cuda.h>\n#include <cuda_runtime.h>\n#include <c10/cuda/CUDAException.h>\n\ntemplate <typename scalar_t>\n__global__ void sw_forward_kernel(\n const scalar_t* __restrict__ score_matrix,\n scalar_t* __restrict__ H,\n scalar_t* __restrict__ output,\n int B, int M, int N, \n scalar_t gap_open, \n scalar_t temperature) {\n \n int batch_idx = blockIdx.x * blockDim.x + threadIdx.x;\n if (batch_idx >= B) return;\n \n // Each thread handles one batch element\n // H has shape (B, M+1, N+1), first row and column are 0\n \n // Initialize first row and column to 0\n for (int j = 0; j <= N; j++) {\n H[batch_idx * (M+1) * (N+1) + j] = 0.0f;\n }\n for (int i = 1; i <= M; i++) {\n H[batch_idx * (M+1) * (N+1) + i * (N+1)] = 0.0f;\n }\n \n scalar_t max_val = 0.0f;\n \n // Iterate through the DP matrix\n for (int i = 1; i <= M; i++) {\n for (int j = 1; j <= N; j++) {\n int idx_prev_row = batch_idx * (M+1) * (N+1) + (i-1) * (N+1) + (j-1);\n int idx_curr = batch_idx * (M+1) * (N+1) + i * (N+1) + j;\n \n scalar_t h_diag = H[idx_prev_row]; // H[i-1][j-1]\n scalar_t h_up = H[idx_prev_row + 1]; // H[i-1][j]\n scalar_t h_left = H[idx_curr - 1]; // H[i][j-1]\n \n scalar_t s = score_matrix[batch_idx * M * N + (i-1) * N + (j-1)];\n \n scalar_t match = h_diag + s;\n scalar_t del = h_up + gap_open;\n scalar_t ins = h_left + gap_open;\n \n // logsumexp over [0, match, del, ins]\n scalar_t inv_temp = 1.0f / temperature;\n scalar_t a0 = 0.0f * inv_temp;\n scalar_t a1 = match * inv_temp;\n scalar_t a2 = del * inv_temp;\n scalar_t a3 = ins * inv_temp;\n \n scalar_t max_a = a0;\n max_a = fmaxf(max_a, a1);\n max_a = fmaxf(max_a, a2);\n max_a = fmaxf(max_a, a3);\n \n scalar_t sum_exp = expf(a0 - max_a) + expf(a1 - max_a) + expf(a2 - max_a) + expf(a3 - max_a);\n scalar_t logsumexp = max_a + logf(sum_exp);\n \n scalar_t h_val = temperature * logsumexp;\n H[idx_curr] = h_val;\n \n if (h_val > max_val) max_val = h_val;\n }\n }\n \n output[batch_idx] = max_val;\n}\n\ntorch::Tensor sw_forward_cuda(torch::Tensor score_matrix, double gap_open, double temperature) {\n TORCH_CHECK(score_matrix.is_cuda(), \"score_matrix must be a CUDA tensor\");\n TORCH_CHECK(score_matrix.dim() == 3, \"score_matrix must be 3D\");\n \n int B = score_matrix.size(0);\n int M = score_matrix.size(1);\n int N = score_matrix.size(2);\n \n auto options = score_matrix.options();\n auto H = torch::zeros({B, M + 1, N + 1}, options);\n auto output = torch::empty({B}, options);\n \n int threads = 256;\n int blocks = (B + threads - 1) / threads;\n \n AT_DISPATCH_FLOATING_TYPES(score_matrix.scalar_type(), \"sw_forward_kernel\", ([&] {\n sw_forward_kernel<scalar_t><<<blocks, threads>>>(\n score_matrix.data_ptr<scalar_t>(),\n H.data_ptr<scalar_t>(),\n output.data_ptr<scalar_t>(),\n B, M, N,\n static_cast<scalar_t>(gap_open),\n static_cast<scalar_t>(temperature)\n );\n }));\n \n C10_CUDA_CHECK(cudaGetLastError());\n return output;\n}\n\"\"\"\n\ncpp_source = \"\"\"\ntorch::Tensor sw_forward_cuda(torch::Tensor score_matrix, double gap_open, double temperature);\n\"\"\"\n\nsw_module = load_inline(\n name='sw_module',\n cpp_sources=cpp_source,\n cuda_sources=cuda_source,\n functions=['sw_forward_cuda'],\n verbose=True,\n)\n\nclass ModelNew(nn.Module):\n def __init__(self, gap_open=-1.0, temperature=1.0):\n super().__init__()\n self.gap_open = gap_open\n self.temperature = temperature\n\n def forward(self, score_matrix: torch.Tensor) -> torch.Tensor:\n return sw_module.sw_forward_cuda(score_matrix, self.gap_open, self.temperature)"
}
result ← run_correctness (ok)
okrun_correctness
run_correctness PASSED: (5 / 5) trials all matched the reference.
Numerical precision: max_abs_err=3.81e-06 mean_abs_err=2.86e-07 max_rel_err=1.32e-07
turn 3 · 1 tool call(s) · 748.2s turn · 16.9s LLM · tokens in=4,466 · out=1,808 · total=6,274 · 8,176 chars (click to expand)
turn 3748.2s turn · 16.9s LLM · 731.3s tools · tokens in=4,466 · out=1,808 · total=6,274
reasoning
The kernel is correct. Now let me optimize it further. The current implementation works but there are several improvements I can make:
1. Use shared memory to cache the DP matrix - each row of the DP matrix depends on the previous row, so we could use double buffering with shared memory
2. Improve memory access patterns
3. Use more efficient math functions
Let me try using shared memory to cache two rows of H at a time, which will reduce global memory access:
Actually, thinking more carefully - the issue is that the algorithm has inherent sequential dependencies:
- H[i][j] depends on H[i-1][j-1], H[i-1][j], and H[i][j-1]
For each batch element, we process one element at a time. The benefit of shared memory would be:
- We load H[i-1][:] into shared memory once per row i
- We compute all H[i][:] using that shared memory row plus the current row being computed
But the current implementation already does this reasonably - it accesses global memory but with relatively good locality since we're iterating through the matrix in row-major order.
Let me try a few optimizations:
1. Pre-compute inv_temp and use it
2. Use __ldg() for read-only access to score_matrix
3. Better math functions for logsumexp
Actually, let me think about this differently. For small M and N (32x32), we might not need shared memory since the entire H matrix for one batch element is small: (33*33) * 4 bytes = ~4.4KB.
The bottleneck is likely:
1. Global memory access for H - each element written once and read 3 times
2. The sequential nature of the computation
Let me try an optimization where I use a register-backed approach for the current row of H, and keep the previous row in registers as well. But with M=32 and N=32, we could potentially fit everything in registers.
Actually, let me try using shared memory for the DP rows of all batches in a block. Each batch in a block could share their rows...
No, that doesn't work since each batch is independent.
Let me try a different approach - keep two rows in registers if possible, or use a shared memory buffer for the current and previous rows:
For each batch element:
- prev_row[N+1]: stores H[i-1][:]
- curr_row[N+1]: stores H[i][:]
When we finish row i, we swap prev_row and curr_row.
With N=32, N+1=33, so each row is 33 floats = 132 bytes. Two rows = 264 bytes.
This fits easily in shared memory.
Let me rewrite with this optimization.
assistantThe kernel is correct. Now let me optimize it. I'll use shared memory to cache the DP matrix rows, reducing global memory access.
call → compile_kernel
callcompile_kernel
{
"kernel_code": "import torch\nimport torch.nn as nn\nfrom torch.utils.cpp_extension import load_inline\n\ncuda_source = \"\"\"\n#include <torch/extension.h>\n#include <cuda.h>\n#include <cuda_runtime.h>\n#include <c10/cuda/CUDAException.h>\n#include <cmath>\n\ntemplate <typename scalar_t>\n__global__ void sw_forward_kernel(\n const scalar_t* __restrict__ score_matrix,\n scalar_t* __restrict__ output,\n int B, int M, int N, \n scalar_t gap_open, \n scalar_t temperature) {\n \n int batch_idx = blockIdx.x * blockDim.x + threadIdx.x;\n if (batch_idx >= B) return;\n \n // Use shared memory for DP rows - each thread gets its own section\n // prev_row[N+1] and curr_row[N+1] for each thread\n extern __shared__ char shared_mem[];\n scalar_t* prev_row = (scalar_t*)shared_mem; // Will be offset by threadIdx.x\n scalar_t* curr_row = prev_row + (blockDim.x * (N + 1));\n \n // Each thread works on its own slice\n int offset = threadIdx.x * (N + 1);\n scalar_t* my_prev = prev_row + offset;\n scalar_t* my_curr = curr_row + offset;\n \n // Initialize first row to 0\n for (int j = 0; j <= N; j++) {\n my_prev[j] = 0.0f;\n }\n \n scalar_t max_val = 0.0f;\n scalar_t inv_temp = 1.0f / temperature;\n \n const scalar_t* score_ptr = score_matrix + batch_idx * M * N;\n \n // Iterate through the DP matrix row by row\n for (int i = 1; i <= M; i++) {\n my_curr[0] = 0.0f; // First column is 0\n \n for (int j = 1; j <= N; j++) {\n scalar_t h_diag = my_prev[j - 1]; // H[i-1][j-1]\n scalar_t h_up = my_prev[j]; // H[i-1][j]\n scalar_t h_left = my_curr[j - 1]; // H[i][j-1]\n \n scalar_t s = score_ptr[(i - 1) * N + (j - 1)];\n \n scalar_t match = h_diag + s;\n scalar_t del = h_up + gap_open;\n scalar_t ins = h_left + gap_open;\n \n // logsumexp over [0, match, del, ins]\n scalar_t a0 = 0.0f;\n scalar_t a1 = match * inv_temp;\n scalar_t a2 = del * inv_temp;\n scalar_t a3 = ins * inv_temp;\n \n scalar_t max_a = fmaxf(fmaxf(a0, a1), fmaxf(a2, a3));\n \n scalar_t sum_exp = expf(a0 - max_a) + expf(a1 - max_a) + \n expf(a2 - max_a) + expf(a3 - max_a);\n scalar_t logsumexp = max_a + logf(sum_exp);\n \n scalar_t h_val = temperature * logsumexp;\n my_curr[j] = h_val;\n \n if (h_val > max_val) max_val = h_val;\n }\n \n // Swap rows: curr becomes prev for next iteration\n scalar_t* tmp = my_prev;\n my_prev = my_curr;\n my_curr = tmp;\n }\n \n output[batch_idx] = max_val;\n}\n\ntorch::Tensor sw_forward_cuda(torch::Tensor score_matrix, double gap_open, double temperature) {\n TORCH_CHECK(score_matrix.is_cuda(), \"score_matrix must be a CUDA tensor\");\n TORCH_CHECK(score_matrix.dim() == 3, \"score_matrix must be 3D\");\n \n int B = score_matrix.size(0);\n int M = score_matrix.size(1);\n int N = score_matrix.size(2);\n \n auto options = score_matrix.options();\n auto output = torch::empty({B}, options);\n \n int threads = 256;\n int blocks = (B + threads - 1) / threads;\n \n // Shared memory: 2 rows of (N+1) floats per thread\n size_t shared_mem_size = threads * 2 * (N + 1) * sizeof(float);\n \n AT_DISPATCH_FLOATING_TYPES(score_matrix.scalar_type(), \"sw_forward_kernel\", ([&] {\n sw_forward_kernel<scalar_t><<<blocks, threads, shared_mem_size>>>(\n score_matrix.data_ptr<scalar_t>(),\n output.data_ptr<scalar_t>(),\n B, M, N,\n static_cast<scalar_t>(gap_open),\n static_cast<scalar_t>(temperature)\n );\n }));\n \n C10_CUDA_CHECK(cudaGetLastError());\n return output;\n}\n\"\"\"\n\ncpp_source = \"\"\"\ntorch::Tensor sw_forward_cuda(torch::Tensor score_matrix, double gap_open, double temperature);\n\"\"\"\n\nsw_module = load_inline(\n name='sw_module',\n cpp_sources=cpp_source,\n cuda_sources=cuda_source,\n functions=['sw_forward_cuda'],\n verbose=True,\n)\n\nclass ModelNew(nn.Module):\n def __init__(self, gap_open=-1.0, temperature=1.0):\n super().__init__()\n self.gap_open = gap_open\n self.temperature = temperature\n\n def forward(self, score_matrix: torch.Tensor) -> torch.Tensor:\n return sw_module.sw_forward_cuda(score_matrix, self.gap_open, self.temperature)"
}
result ← compile_kernel (ok)
okcompile_kernel
compile_kernel PASSED: kernel compiled without errors.
turn 4677.0s turn · 10.6s LLM · 666.4s tools · tokens in=5,721 · out=1,224 · total=6,945
reasoning
Let me run correctness to verify this version.
call → run_correctness
callrun_correctness
{
"kernel_code": "import torch\nimport torch.nn as nn\nfrom torch.utils.cpp_extension import load_inline\n\ncuda_source = \"\"\"\n#include <torch/extension.h>\n#include <cuda.h>\n#include <cuda_runtime.h>\n#include <c10/cuda/CUDAException.h>\n#include <cmath>\n\ntemplate <typename scalar_t>\n__global__ void sw_forward_kernel(\n const scalar_t* __restrict__ score_matrix,\n scalar_t* __restrict__ output,\n int B, int M, int N, \n scalar_t gap_open, \n scalar_t temperature) {\n \n int batch_idx = blockIdx.x * blockDim.x + threadIdx.x;\n if (batch_idx >= B) return;\n \n // Use shared memory for DP rows - each thread gets its own section\n // prev_row[N+1] and curr_row[N+1] for each thread\n extern __shared__ char shared_mem[];\n scalar_t* prev_row = (scalar_t*)shared_mem; // Will be offset by threadIdx.x\n scalar_t* curr_row = prev_row + (blockDim.x * (N + 1));\n \n // Each thread works on its own slice\n int offset = threadIdx.x * (N + 1);\n scalar_t* my_prev = prev_row + offset;\n scalar_t* my_curr = curr_row + offset;\n \n // Initialize first row to 0\n for (int j = 0; j <= N; j++) {\n my_prev[j] = 0.0f;\n }\n \n scalar_t max_val = 0.0f;\n scalar_t inv_temp = 1.0f / temperature;\n \n const scalar_t* score_ptr = score_matrix + batch_idx * M * N;\n \n // Iterate through the DP matrix row by row\n for (int i = 1; i <= M; i++) {\n my_curr[0] = 0.0f; // First column is 0\n \n for (int j = 1; j <= N; j++) {\n scalar_t h_diag = my_prev[j - 1]; // H[i-1][j-1]\n scalar_t h_up = my_prev[j]; // H[i-1][j]\n scalar_t h_left = my_curr[j - 1]; // H[i][j-1]\n \n scalar_t s = score_ptr[(i - 1) * N + (j - 1)];\n \n scalar_t match = h_diag + s;\n scalar_t del = h_up + gap_open;\n scalar_t ins = h_left + gap_open;\n \n // logsumexp over [0, match, del, ins]\n scalar_t a0 = 0.0f;\n scalar_t a1 = match * inv_temp;\n scalar_t a2 = del * inv_temp;\n scalar_t a3 = ins * inv_temp;\n \n scalar_t max_a = fmaxf(fmaxf(a0, a1), fmaxf(a2, a3));\n \n scalar_t sum_exp = expf(a0 - max_a) + expf(a1 - max_a) + \n expf(a2 - max_a) + expf(a3 - max_a);\n scalar_t logsumexp = max_a + logf(sum_exp);\n \n scalar_t h_val = temperature * logsumexp;\n my_curr[j] = h_val;\n \n if (h_val > max_val) max_val = h_val;\n }\n \n // Swap rows: curr becomes prev for next iteration\n scalar_t* tmp = my_prev;\n my_prev = my_curr;\n my_curr = tmp;\n }\n \n output[batch_idx] = max_val;\n}\n\ntorch::Tensor sw_forward_cuda(torch::Tensor score_matrix, double gap_open, double temperature) {\n TORCH_CHECK(score_matrix.is_cuda(), \"score_matrix must be a CUDA tensor\");\n TORCH_CHECK(score_matrix.dim() == 3, \"score_matrix must be 3D\");\n \n int B = score_matrix.size(0);\n int M = score_matrix.size(1);\n int N = score_matrix.size(2);\n \n auto options = score_matrix.options();\n auto output = torch::empty({B}, options);\n \n int threads = 256;\n int blocks = (B + threads - 1) / threads;\n \n // Shared memory: 2 rows of (N+1) floats per thread\n size_t shared_mem_size = threads * 2 * (N + 1) * sizeof(float);\n \n AT_DISPATCH_FLOATING_TYPES(score_matrix.scalar_type(), \"sw_forward_kernel\", ([&] {\n sw_forward_kernel<scalar_t><<<blocks, threads, shared_mem_size>>>(\n score_matrix.data_ptr<scalar_t>(),\n output.data_ptr<scalar_t>(),\n B, M, N,\n static_cast<scalar_t>(gap_open),\n static_cast<scalar_t>(temperature)\n );\n }));\n \n C10_CUDA_CHECK(cudaGetLastError());\n return output;\n}\n\"\"\"\n\ncpp_source = \"\"\"\ntorch::Tensor sw_forward_cuda(torch::Tensor score_matrix, double gap_open, double temperature);\n\"\"\"\n\nsw_module = load_inline(\n name='sw_module',\n cpp_sources=cpp_source,\n cuda_sources=cuda_source,\n functions=['sw_forward_cuda'],\n verbose=True,\n)\n\nclass ModelNew(nn.Module):\n def __init__(self, gap_open=-1.0, temperature=1.0):\n super().__init__()\n self.gap_open = gap_open\n self.temperature = temperature\n\n def forward(self, score_matrix: torch.Tensor) -> torch.Tensor:\n return sw_module.sw_forward_cuda(score_matrix, self.gap_open, self.temperature)"
}
result ← run_correctness (fail)
failrun_correctness
run_correctness FAILED: ? trials did not all match.
runtime_error: CUDA error: invalid argument
Search for `cudaErrorInvalidValue' in https://docs.nvidia.com/cuda/cuda-runtime-api/group__CUDART__TYPES.html for more information.
CUDA kernel errors might be asynchro...
turn 5 · 1 tool call(s) · 235.9s turn · 11.7s LLM · tokens in=7,002 · out=1,354 · total=8,356 · 6,320 chars (click to expand)
turn 5235.9s turn · 11.7s LLM · 224.3s tools · tokens in=7,002 · out=1,354 · total=8,356
reasoning
The shared memory size might be too large. Let me check: with N=32, threads=256, that's 256 * 2 * 33 * 4 bytes = 67584 bytes ≈ 66KB, which should fit. But let me verify the shared memory allocation and also make sure the kernel is correct. Maybe I need to use the scalar_t type correctly for shared memory.
Actually, the issue might be that I'm using `sizeof(float)` instead of `sizeof(scalar_t)` in the shared memory calculation. Let me fix that.
assistantThe shared memory size calculation might be incorrect. Let me fix it to use the correct type size:
call → compile_kernel
callcompile_kernel
{
"kernel_code": "import torch\nimport torch.nn as nn\nfrom torch.utils.cpp_extension import load_inline\n\ncuda_source = \"\"\"\n#include <torch/extension.h>\n#include <cuda.h>\n#include <cuda_runtime.h>\n#include <c10/cuda/CUDAException.h>\n#include <cmath>\n\ntemplate <typename scalar_t>\n__global__ void sw_forward_kernel(\n const scalar_t* __restrict__ score_matrix,\n scalar_t* __restrict__ output,\n int B, int M, int N, \n scalar_t gap_open, \n scalar_t temperature,\n size_t row_stride) { // N+1 for stride\n \n int batch_idx = blockIdx.x * blockDim.x + threadIdx.x;\n if (batch_idx >= B) return;\n \n // Use shared memory for DP rows - each thread gets its own section\n extern __shared__ __align__(sizeof(scalar_t)) char shared_mem[];\n scalar_t* rows = reinterpret_cast<scalar_t*>(shared_mem);\n \n // Each thread works on its own slice: 2 rows of (N+1) elements\n size_t offset = threadIdx.x * 2 * row_stride;\n scalar_t* my_prev = rows + offset;\n scalar_t* my_curr = rows + offset + row_stride;\n \n // Initialize first row to 0\n #pragma unroll 4\n for (int j = 0; j <= N; j++) {\n my_prev[j] = scalar_t(0);\n }\n \n scalar_t max_val = scalar_t(0);\n scalar_t inv_temp = scalar_t(1) / temperature;\n \n const scalar_t* __restrict__ score_ptr = score_matrix + batch_idx * M * N;\n \n // Iterate through the DP matrix row by row\n for (int i = 1; i <= M; i++) {\n my_curr[0] = scalar_t(0); // First column is 0\n \n for (int j = 1; j <= N; j++) {\n scalar_t h_diag = my_prev[j - 1]; // H[i-1][j-1]\n scalar_t h_up = my_prev[j]; // H[i-1][j]\n scalar_t h_left = my_curr[j - 1]; // H[i][j-1]\n \n scalar_t s = score_ptr[(i - 1) * N + (j - 1)];\n \n scalar_t match = h_diag + s;\n scalar_t del = h_up + gap_open;\n scalar_t ins = h_left + gap_open;\n \n // logsumexp over [0, match, del, ins]\n scalar_t a0 = scalar_t(0);\n scalar_t a1 = match * inv_temp;\n scalar_t a2 = del * inv_temp;\n scalar_t a3 = ins * inv_temp;\n \n scalar_t max_a = fmax(fmax(a0, a1), fmax(a2, a3));\n \n scalar_t sum_exp = exp(a0 - max_a) + exp(a1 - max_a) + \n exp(a2 - max_a) + exp(a3 - max_a);\n scalar_t logsumexp = max_a + log(sum_exp);\n \n scalar_t h_val = temperature * logsumexp;\n my_curr[j] = h_val;\n \n if (h_val > max_val) max_val = h_val;\n }\n \n // Swap rows: curr becomes prev for next iteration\n scalar_t* tmp = my_prev;\n my_prev = my_curr;\n my_curr = tmp;\n }\n \n output[batch_idx] = max_val;\n}\n\ntorch::Tensor sw_forward_cuda(torch::Tensor score_matrix, double gap_open, double temperature) {\n TORCH_CHECK(score_matrix.is_cuda(), \"score_matrix must be a CUDA tensor\");\n TORCH_CHECK(score_matrix.dim() == 3, \"score_matrix must be 3D\");\n \n int64_t B = score_matrix.size(0);\n int64_t M = score_matrix.size(1);\n int64_t N = score_matrix.size(2);\n \n auto options = score_matrix.options();\n auto output = torch::empty({B}, options);\n \n int threads = 256;\n int blocks = (B + threads - 1) / threads;\n \n // Shared memory: 2 rows of (N+1) elements per thread\n size_t row_stride = N + 1;\n size_t shared_mem_size = threads * 2 * row_stride * score_matrix.element_size();\n \n AT_DISPATCH_FLOATING_TYPES(score_matrix.scalar_type(), \"sw_forward_kernel\", ([&] {\n sw_forward_kernel<scalar_t><<<blocks, threads, shared_mem_size>>>(\n score_matrix.data_ptr<scalar_t>(),\n output.data_ptr<scalar_t>(),\n B, M, N,\n static_cast<scalar_t>(gap_open),\n static_cast<scalar_t>(temperature),\n row_stride\n );\n }));\n \n C10_CUDA_CHECK(cudaGetLastError());\n return output;\n}\n\"\"\"\n\ncpp_source = \"\"\"\ntorch::Tensor sw_forward_cuda(torch::Tensor score_matrix, double gap_open, double temperature);\n\"\"\"\n\nsw_module = load_inline(\n name='sw_module',\n cpp_sources=cpp_source,\n cuda_sources=cuda_source,\n functions=['sw_forward_cuda'],\n verbose=True,\n)\n\nclass ModelNew(nn.Module):\n def __init__(self, gap_open=-1.0, temperature=1.0):\n super().__init__()\n self.gap_open = gap_open\n self.temperature = temperature\n\n def forward(self, score_matrix: torch.Tensor) -> torch.Tensor:\n return sw_module.sw_forward_cuda(score_matrix, self.gap_open, self.temperature)"
}
result ← compile_kernel (ok)
okcompile_kernel
compile_kernel PASSED: kernel compiled without errors.
turn 683.7s turn · 10.1s LLM · 73.6s tools · tokens in=8,253 · out=1,225 · total=9,478
reasoning
Let me check correctness now.
call → run_correctness
callrun_correctness
{
"kernel_code": "import torch\nimport torch.nn as nn\nfrom torch.utils.cpp_extension import load_inline\n\ncuda_source = \"\"\"\n#include <torch/extension.h>\n#include <cuda.h>\n#include <cuda_runtime.h>\n#include <c10/cuda/CUDAException.h>\n#include <cmath>\n\ntemplate <typename scalar_t>\n__global__ void sw_forward_kernel(\n const scalar_t* __restrict__ score_matrix,\n scalar_t* __restrict__ output,\n int B, int M, int N, \n scalar_t gap_open, \n scalar_t temperature,\n size_t row_stride) { // N+1 for stride\n \n int batch_idx = blockIdx.x * blockDim.x + threadIdx.x;\n if (batch_idx >= B) return;\n \n // Use shared memory for DP rows - each thread gets its own section\n extern __shared__ __align__(sizeof(scalar_t)) char shared_mem[];\n scalar_t* rows = reinterpret_cast<scalar_t*>(shared_mem);\n \n // Each thread works on its own slice: 2 rows of (N+1) elements\n size_t offset = threadIdx.x * 2 * row_stride;\n scalar_t* my_prev = rows + offset;\n scalar_t* my_curr = rows + offset + row_stride;\n \n // Initialize first row to 0\n #pragma unroll 4\n for (int j = 0; j <= N; j++) {\n my_prev[j] = scalar_t(0);\n }\n \n scalar_t max_val = scalar_t(0);\n scalar_t inv_temp = scalar_t(1) / temperature;\n \n const scalar_t* __restrict__ score_ptr = score_matrix + batch_idx * M * N;\n \n // Iterate through the DP matrix row by row\n for (int i = 1; i <= M; i++) {\n my_curr[0] = scalar_t(0); // First column is 0\n \n for (int j = 1; j <= N; j++) {\n scalar_t h_diag = my_prev[j - 1]; // H[i-1][j-1]\n scalar_t h_up = my_prev[j]; // H[i-1][j]\n scalar_t h_left = my_curr[j - 1]; // H[i][j-1]\n \n scalar_t s = score_ptr[(i - 1) * N + (j - 1)];\n \n scalar_t match = h_diag + s;\n scalar_t del = h_up + gap_open;\n scalar_t ins = h_left + gap_open;\n \n // logsumexp over [0, match, del, ins]\n scalar_t a0 = scalar_t(0);\n scalar_t a1 = match * inv_temp;\n scalar_t a2 = del * inv_temp;\n scalar_t a3 = ins * inv_temp;\n \n scalar_t max_a = fmax(fmax(a0, a1), fmax(a2, a3));\n \n scalar_t sum_exp = exp(a0 - max_a) + exp(a1 - max_a) + \n exp(a2 - max_a) + exp(a3 - max_a);\n scalar_t logsumexp = max_a + log(sum_exp);\n \n scalar_t h_val = temperature * logsumexp;\n my_curr[j] = h_val;\n \n if (h_val > max_val) max_val = h_val;\n }\n \n // Swap rows: curr becomes prev for next iteration\n scalar_t* tmp = my_prev;\n my_prev = my_curr;\n my_curr = tmp;\n }\n \n output[batch_idx] = max_val;\n}\n\ntorch::Tensor sw_forward_cuda(torch::Tensor score_matrix, double gap_open, double temperature) {\n TORCH_CHECK(score_matrix.is_cuda(), \"score_matrix must be a CUDA tensor\");\n TORCH_CHECK(score_matrix.dim() == 3, \"score_matrix must be 3D\");\n \n int64_t B = score_matrix.size(0);\n int64_t M = score_matrix.size(1);\n int64_t N = score_matrix.size(2);\n \n auto options = score_matrix.options();\n auto output = torch::empty({B}, options);\n \n int threads = 256;\n int blocks = (B + threads - 1) / threads;\n \n // Shared memory: 2 rows of (N+1) elements per thread\n size_t row_stride = N + 1;\n size_t shared_mem_size = threads * 2 * row_stride * score_matrix.element_size();\n \n AT_DISPATCH_FLOATING_TYPES(score_matrix.scalar_type(), \"sw_forward_kernel\", ([&] {\n sw_forward_kernel<scalar_t><<<blocks, threads, shared_mem_size>>>(\n score_matrix.data_ptr<scalar_t>(),\n output.data_ptr<scalar_t>(),\n B, M, N,\n static_cast<scalar_t>(gap_open),\n static_cast<scalar_t>(temperature),\n row_stride\n );\n }));\n \n C10_CUDA_CHECK(cudaGetLastError());\n return output;\n}\n\"\"\"\n\ncpp_source = \"\"\"\ntorch::Tensor sw_forward_cuda(torch::Tensor score_matrix, double gap_open, double temperature);\n\"\"\"\n\nsw_module = load_inline(\n name='sw_module',\n cpp_sources=cpp_source,\n cuda_sources=cuda_source,\n functions=['sw_forward_cuda'],\n verbose=True,\n)\n\nclass ModelNew(nn.Module):\n def __init__(self, gap_open=-1.0, temperature=1.0):\n super().__init__()\n self.gap_open = gap_open\n self.temperature = temperature\n\n def forward(self, score_matrix: torch.Tensor) -> torch.Tensor:\n return sw_module.sw_forward_cuda(score_matrix, self.gap_open, self.temperature)"
}
result ← run_correctness (fail)
failrun_correctness
run_correctness FAILED: ? trials did not all match.
runtime_error: CUDA error: invalid argument
Search for `cudaErrorInvalidValue' in https://docs.nvidia.com/cuda/cuda-runtime-api/group__CUDART__TYPES.html for more information.
CUDA kernel errors might be asynchro...
turn 7 · 1 tool call(s) · 93.4s turn · 12.1s LLM · tokens in=9,538 · out=1,355 · total=10,893 · 6,056 chars (click to expand)
turn 793.4s turn · 12.1s LLM · 81.4s tools · tokens in=9,538 · out=1,355 · total=10,893
reasoning
There's still a shared memory issue. For N=32, N+1=33, with 256 threads and 2 rows per thread:
- 256 * 2 * 33 * 4 bytes = 67,584 bytes ~ 66 KB
This should fit in shared memory (227 KB per block), so maybe it's something else. Let me check if the issue is with the parameter types.
Actually, looking more carefully, the issue might be that I'm using `int` for M and N but the values come from `int64_t`. Let me use int consistently or be more careful about the type conversion.
Also, I should simplify and avoid shared memory for now to ensure correctness, then add optimization incrementally.
assistantLet me go back to a simpler approach that doesn't use shared memory for debugging, then optimize:
call → compile_kernel
callcompile_kernel
{
"kernel_code": "import torch\nimport torch.nn as nn\nfrom torch.utils.cpp_extension import load_inline\n\ncuda_source = \"\"\"\n#include <torch/extension.h>\n#include <cuda.h>\n#include <cuda_runtime.h>\n#include <c10/cuda/CUDAException.h>\n#include <cmath>\n\ntemplate <typename scalar_t>\n__global__ void sw_forward_kernel(\n const scalar_t* __restrict__ score_matrix,\n scalar_t* __restrict__ H,\n scalar_t* __restrict__ output,\n int B, int M, int N, \n scalar_t gap_open, \n scalar_t temperature) {\n \n int batch_idx = blockIdx.x * blockDim.x + threadIdx.x;\n if (batch_idx >= B) return;\n \n // Each thread handles one batch element\n // H has shape (B, M+1, N+1), first row and column are 0\n \n // Initialize first row to 0\n for (int j = 0; j <= N; j++) {\n H[batch_idx * (M+1) * (N+1) + j] = scalar_t(0);\n }\n // Initialize first column to 0\n for (int i = 1; i <= M; i++) {\n H[batch_idx * (M+1) * (N+1) + i * (N+1)] = scalar_t(0);\n }\n \n scalar_t max_val = scalar_t(0);\n scalar_t inv_temp = scalar_t(1) / temperature;\n \n // Iterate through the DP matrix\n for (int i = 1; i <= M; i++) {\n for (int j = 1; j <= N; j++) {\n int64_t idx_prev_row = batch_idx * (int64_t)(M+1) * (N+1) + (i-1) * (N+1) + (j-1);\n int64_t idx_curr = batch_idx * (int64_t)(M+1) * (N+1) + i * (N+1) + j;\n \n scalar_t h_diag = H[idx_prev_row]; // H[i-1][j-1]\n scalar_t h_up = H[idx_prev_row + 1]; // H[i-1][j]\n scalar_t h_left = H[idx_curr - 1]; // H[i][j-1]\n \n scalar_t s = score_matrix[batch_idx * (int64_t)M * N + (i-1) * N + (j-1)];\n \n scalar_t match = h_diag + s;\n scalar_t del = h_up + gap_open;\n scalar_t ins = h_left + gap_open;\n \n // logsumexp over [0, match, del, ins]\n scalar_t a0 = scalar_t(0);\n scalar_t a1 = match * inv_temp;\n scalar_t a2 = del * inv_temp;\n scalar_t a3 = ins * inv_temp;\n \n scalar_t max_a = fmax(fmax(a0, a1), fmax(a2, a3));\n \n scalar_t sum_exp = exp(a0 - max_a) + exp(a1 - max_a) + \n exp(a2 - max_a) + exp(a3 - max_a);\n scalar_t logsumexp = max_a + log(sum_exp);\n \n scalar_t h_val = temperature * logsumexp;\n H[idx_curr] = h_val;\n \n if (h_val > max_val) max_val = h_val;\n }\n }\n \n output[batch_idx] = max_val;\n}\n\ntorch::Tensor sw_forward_cuda(torch::Tensor score_matrix, double gap_open, double temperature) {\n TORCH_CHECK(score_matrix.is_cuda(), \"score_matrix must be a CUDA tensor\");\n TORCH_CHECK(score_matrix.dim() == 3, \"score_matrix must be 3D\");\n \n int B = score_matrix.size(0);\n int M = score_matrix.size(1);\n int N = score_matrix.size(2);\n \n auto options = score_matrix.options();\n auto H = torch::zeros({B, M + 1, N + 1}, options);\n auto output = torch::empty({B}, options);\n \n int threads = 256;\n int blocks = (B + threads - 1) / threads;\n \n AT_DISPATCH_FLOATING_TYPES(score_matrix.scalar_type(), \"sw_forward_kernel\", ([&] {\n sw_forward_kernel<scalar_t><<<blocks, threads>>>(\n score_matrix.data_ptr<scalar_t>(),\n H.data_ptr<scalar_t>(),\n output.data_ptr<scalar_t>(),\n B, M, N,\n static_cast<scalar_t>(gap_open),\n static_cast<scalar_t>(temperature)\n );\n }));\n \n C10_CUDA_CHECK(cudaGetLastError());\n return output;\n}\n\"\"\"\n\ncpp_source = \"\"\"\ntorch::Tensor sw_forward_cuda(torch::Tensor score_matrix, double gap_open, double temperature);\n\"\"\"\n\nsw_module = load_inline(\n name='sw_module',\n cpp_sources=cpp_source,\n cuda_sources=cuda_source,\n functions=['sw_forward_cuda'],\n verbose=True,\n)\n\nclass ModelNew(nn.Module):\n def __init__(self, gap_open=-1.0, temperature=1.0):\n super().__init__()\n self.gap_open = gap_open\n self.temperature = temperature\n\n def forward(self, score_matrix: torch.Tensor) -> torch.Tensor:\n return sw_module.sw_forward_cuda(score_matrix, self.gap_open, self.temperature)"
}
result ← compile_kernel (ok)
okcompile_kernel
compile_kernel PASSED: kernel compiled without errors.
turn 8395.9s turn · 10.0s LLM · 386.0s tools · tokens in=10,753 · out=1,195 · total=11,948
reasoning
Let me run correctness to see if this simpler version works.
call → run_correctness
callrun_correctness
{
"kernel_code": "import torch\nimport torch.nn as nn\nfrom torch.utils.cpp_extension import load_inline\n\ncuda_source = \"\"\"\n#include <torch/extension.h>\n#include <cuda.h>\n#include <cuda_runtime.h>\n#include <c10/cuda/CUDAException.h>\n#include <cmath>\n\ntemplate <typename scalar_t>\n__global__ void sw_forward_kernel(\n const scalar_t* __restrict__ score_matrix,\n scalar_t* __restrict__ H,\n scalar_t* __restrict__ output,\n int B, int M, int N, \n scalar_t gap_open, \n scalar_t temperature) {\n \n int batch_idx = blockIdx.x * blockDim.x + threadIdx.x;\n if (batch_idx >= B) return;\n \n // Each thread handles one batch element\n // H has shape (B, M+1, N+1), first row and column are 0\n \n // Initialize first row to 0\n for (int j = 0; j <= N; j++) {\n H[batch_idx * (M+1) * (N+1) + j] = scalar_t(0);\n }\n // Initialize first column to 0\n for (int i = 1; i <= M; i++) {\n H[batch_idx * (M+1) * (N+1) + i * (N+1)] = scalar_t(0);\n }\n \n scalar_t max_val = scalar_t(0);\n scalar_t inv_temp = scalar_t(1) / temperature;\n \n // Iterate through the DP matrix\n for (int i = 1; i <= M; i++) {\n for (int j = 1; j <= N; j++) {\n int64_t idx_prev_row = batch_idx * (int64_t)(M+1) * (N+1) + (i-1) * (N+1) + (j-1);\n int64_t idx_curr = batch_idx * (int64_t)(M+1) * (N+1) + i * (N+1) + j;\n \n scalar_t h_diag = H[idx_prev_row]; // H[i-1][j-1]\n scalar_t h_up = H[idx_prev_row + 1]; // H[i-1][j]\n scalar_t h_left = H[idx_curr - 1]; // H[i][j-1]\n \n scalar_t s = score_matrix[batch_idx * (int64_t)M * N + (i-1) * N + (j-1)];\n \n scalar_t match = h_diag + s;\n scalar_t del = h_up + gap_open;\n scalar_t ins = h_left + gap_open;\n \n // logsumexp over [0, match, del, ins]\n scalar_t a0 = scalar_t(0);\n scalar_t a1 = match * inv_temp;\n scalar_t a2 = del * inv_temp;\n scalar_t a3 = ins * inv_temp;\n \n scalar_t max_a = fmax(fmax(a0, a1), fmax(a2, a3));\n \n scalar_t sum_exp = exp(a0 - max_a) + exp(a1 - max_a) + \n exp(a2 - max_a) + exp(a3 - max_a);\n scalar_t logsumexp = max_a + log(sum_exp);\n \n scalar_t h_val = temperature * logsumexp;\n H[idx_curr] = h_val;\n \n if (h_val > max_val) max_val = h_val;\n }\n }\n \n output[batch_idx] = max_val;\n}\n\ntorch::Tensor sw_forward_cuda(torch::Tensor score_matrix, double gap_open, double temperature) {\n TORCH_CHECK(score_matrix.is_cuda(), \"score_matrix must be a CUDA tensor\");\n TORCH_CHECK(score_matrix.dim() == 3, \"score_matrix must be 3D\");\n \n int B = score_matrix.size(0);\n int M = score_matrix.size(1);\n int N = score_matrix.size(2);\n \n auto options = score_matrix.options();\n auto H = torch::zeros({B, M + 1, N + 1}, options);\n auto output = torch::empty({B}, options);\n \n int threads = 256;\n int blocks = (B + threads - 1) / threads;\n \n AT_DISPATCH_FLOATING_TYPES(score_matrix.scalar_type(), \"sw_forward_kernel\", ([&] {\n sw_forward_kernel<scalar_t><<<blocks, threads>>>(\n score_matrix.data_ptr<scalar_t>(),\n H.data_ptr<scalar_t>(),\n output.data_ptr<scalar_t>(),\n B, M, N,\n static_cast<scalar_t>(gap_open),\n static_cast<scalar_t>(temperature)\n );\n }));\n \n C10_CUDA_CHECK(cudaGetLastError());\n return output;\n}\n\"\"\"\n\ncpp_source = \"\"\"\ntorch::Tensor sw_forward_cuda(torch::Tensor score_matrix, double gap_open, double temperature);\n\"\"\"\n\nsw_module = load_inline(\n name='sw_module',\n cpp_sources=cpp_source,\n cuda_sources=cuda_source,\n functions=['sw_forward_cuda'],\n verbose=True,\n)\n\nclass ModelNew(nn.Module):\n def __init__(self, gap_open=-1.0, temperature=1.0):\n super().__init__()\n self.gap_open = gap_open\n self.temperature = temperature\n\n def forward(self, score_matrix: torch.Tensor) -> torch.Tensor:\n return sw_module.sw_forward_cuda(score_matrix, self.gap_open, self.temperature)"
}
result ← run_correctness (ok)
okrun_correctness
run_correctness PASSED: (5 / 5) trials all matched the reference.
Numerical precision: max_abs_err=3.81e-06 mean_abs_err=2.86e-07 max_rel_err=1.32e-07
turn 910.2s turn · 10.2s LLM · 0.0s tools · tokens in=12,025 · out=1,207 · total=13,232
reasoning
The kernel is correct and working. Let me check the static analysis to make sure there are no issues, then submit it.
call → static_check
callstatic_check
{
"kernel_code": "import torch\nimport torch.nn as nn\nfrom torch.utils.cpp_extension import load_inline\n\ncuda_source = \"\"\"\n#include <torch/extension.h>\n#include <cuda.h>\n#include <cuda_runtime.h>\n#include <c10/cuda/CUDAException.h>\n#include <cmath>\n\ntemplate <typename scalar_t>\n__global__ void sw_forward_kernel(\n const scalar_t* __restrict__ score_matrix,\n scalar_t* __restrict__ H,\n scalar_t* __restrict__ output,\n int B, int M, int N, \n scalar_t gap_open, \n scalar_t temperature) {\n \n int batch_idx = blockIdx.x * blockDim.x + threadIdx.x;\n if (batch_idx >= B) return;\n \n // Each thread handles one batch element\n // H has shape (B, M+1, N+1), first row and column are 0\n \n // Initialize first row to 0\n for (int j = 0; j <= N; j++) {\n H[batch_idx * (M+1) * (N+1) + j] = scalar_t(0);\n }\n // Initialize first column to 0\n for (int i = 1; i <= M; i++) {\n H[batch_idx * (M+1) * (N+1) + i * (N+1)] = scalar_t(0);\n }\n \n scalar_t max_val = scalar_t(0);\n scalar_t inv_temp = scalar_t(1) / temperature;\n \n // Iterate through the DP matrix\n for (int i = 1; i <= M; i++) {\n for (int j = 1; j <= N; j++) {\n int64_t idx_prev_row = batch_idx * (int64_t)(M+1) * (N+1) + (i-1) * (N+1) + (j-1);\n int64_t idx_curr = batch_idx * (int64_t)(M+1) * (N+1) + i * (N+1) + j;\n \n scalar_t h_diag = H[idx_prev_row]; // H[i-1][j-1]\n scalar_t h_up = H[idx_prev_row + 1]; // H[i-1][j]\n scalar_t h_left = H[idx_curr - 1]; // H[i][j-1]\n \n scalar_t s = score_matrix[batch_idx * (int64_t)M * N + (i-1) * N + (j-1)];\n \n scalar_t match = h_diag + s;\n scalar_t del = h_up + gap_open;\n scalar_t ins = h_left + gap_open;\n \n // logsumexp over [0, match, del, ins]\n scalar_t a0 = scalar_t(0);\n scalar_t a1 = match * inv_temp;\n scalar_t a2 = del * inv_temp;\n scalar_t a3 = ins * inv_temp;\n \n scalar_t max_a = fmax(fmax(a0, a1), fmax(a2, a3));\n \n scalar_t sum_exp = exp(a0 - max_a) + exp(a1 - max_a) + \n exp(a2 - max_a) + exp(a3 - max_a);\n scalar_t logsumexp = max_a + log(sum_exp);\n \n scalar_t h_val = temperature * logsumexp;\n H[idx_curr] = h_val;\n \n if (h_val > max_val) max_val = h_val;\n }\n }\n \n output[batch_idx] = max_val;\n}\n\ntorch::Tensor sw_forward_cuda(torch::Tensor score_matrix, double gap_open, double temperature) {\n TORCH_CHECK(score_matrix.is_cuda(), \"score_matrix must be a CUDA tensor\");\n TORCH_CHECK(score_matrix.dim() == 3, \"score_matrix must be 3D\");\n \n int B = score_matrix.size(0);\n int M = score_matrix.size(1);\n int N = score_matrix.size(2);\n \n auto options = score_matrix.options();\n auto H = torch::zeros({B, M + 1, N + 1}, options);\n auto output = torch::empty({B}, options);\n \n int threads = 256;\n int blocks = (B + threads - 1) / threads;\n \n AT_DISPATCH_FLOATING_TYPES(score_matrix.scalar_type(), \"sw_forward_kernel\", ([&] {\n sw_forward_kernel<scalar_t><<<blocks, threads>>>(\n score_matrix.data_ptr<scalar_t>(),\n H.data_ptr<scalar_t>(),\n output.data_ptr<scalar_t>(),\n B, M, N,\n static_cast<scalar_t>(gap_open),\n static_cast<scalar_t>(temperature)\n );\n }));\n \n C10_CUDA_CHECK(cudaGetLastError());\n return output;\n}\n\"\"\"\n\ncpp_source = \"\"\"\ntorch::Tensor sw_forward_cuda(torch::Tensor score_matrix, double gap_open, double temperature);\n\"\"\"\n\nsw_module = load_inline(\n name='sw_module',\n cpp_sources=cpp_source,\n cuda_sources=cuda_source,\n functions=['sw_forward_cuda'],\n verbose=True,\n)\n\nclass ModelNew(nn.Module):\n def __init__(self, gap_open=-1.0, temperature=1.0):\n super().__init__()\n self.gap_open = gap_open\n self.temperature = temperature\n\n def forward(self, score_matrix: torch.Tensor) -> torch.Tensor:\n return sw_module.sw_forward_cuda(score_matrix, self.gap_open, self.temperature)"
}
result ← static_check (ok)
okstatic_check
static_check PASSED: no violations or warnings detected.
turn 10175.1s turn · 9.9s LLM · 165.2s tools · tokens in=13,253 · out=1,198 · total=14,451 · FINAL
reasoning
The kernel has passed both correctness and static checks. I should submit it now.
call → submit_kernel
callsubmit_kernel
{
"kernel_code": "import torch\nimport torch.nn as nn\nfrom torch.utils.cpp_extension import load_inline\n\ncuda_source = \"\"\"\n#include <torch/extension.h>\n#include <cuda.h>\n#include <cuda_runtime.h>\n#include <c10/cuda/CUDAException.h>\n#include <cmath>\n\ntemplate <typename scalar_t>\n__global__ void sw_forward_kernel(\n const scalar_t* __restrict__ score_matrix,\n scalar_t* __restrict__ H,\n scalar_t* __restrict__ output,\n int B, int M, int N, \n scalar_t gap_open, \n scalar_t temperature) {\n \n int batch_idx = blockIdx.x * blockDim.x + threadIdx.x;\n if (batch_idx >= B) return;\n \n // Each thread handles one batch element\n // H has shape (B, M+1, N+1), first row and column are 0\n \n // Initialize first row to 0\n for (int j = 0; j <= N; j++) {\n H[batch_idx * (M+1) * (N+1) + j] = scalar_t(0);\n }\n // Initialize first column to 0\n for (int i = 1; i <= M; i++) {\n H[batch_idx * (M+1) * (N+1) + i * (N+1)] = scalar_t(0);\n }\n \n scalar_t max_val = scalar_t(0);\n scalar_t inv_temp = scalar_t(1) / temperature;\n \n // Iterate through the DP matrix\n for (int i = 1; i <= M; i++) {\n for (int j = 1; j <= N; j++) {\n int64_t idx_prev_row = batch_idx * (int64_t)(M+1) * (N+1) + (i-1) * (N+1) + (j-1);\n int64_t idx_curr = batch_idx * (int64_t)(M+1) * (N+1) + i * (N+1) + j;\n \n scalar_t h_diag = H[idx_prev_row]; // H[i-1][j-1]\n scalar_t h_up = H[idx_prev_row + 1]; // H[i-1][j]\n scalar_t h_left = H[idx_curr - 1]; // H[i][j-1]\n \n scalar_t s = score_matrix[batch_idx * (int64_t)M * N + (i-1) * N + (j-1)];\n \n scalar_t match = h_diag + s;\n scalar_t del = h_up + gap_open;\n scalar_t ins = h_left + gap_open;\n \n // logsumexp over [0, match, del, ins]\n scalar_t a0 = scalar_t(0);\n scalar_t a1 = match * inv_temp;\n scalar_t a2 = del * inv_temp;\n scalar_t a3 = ins * inv_temp;\n \n scalar_t max_a = fmax(fmax(a0, a1), fmax(a2, a3));\n \n scalar_t sum_exp = exp(a0 - max_a) + exp(a1 - max_a) + \n exp(a2 - max_a) + exp(a3 - max_a);\n scalar_t logsumexp = max_a + log(sum_exp);\n \n scalar_t h_val = temperature * logsumexp;\n H[idx_curr] = h_val;\n \n if (h_val > max_val) max_val = h_val;\n }\n }\n \n output[batch_idx] = max_val;\n}\n\ntorch::Tensor sw_forward_cuda(torch::Tensor score_matrix, double gap_open, double temperature) {\n TORCH_CHECK(score_matrix.is_cuda(), \"score_matrix must be a CUDA tensor\");\n TORCH_CHECK(score_matrix.dim() == 3, \"score_matrix must be 3D\");\n \n int B = score_matrix.size(0);\n int M = score_matrix.size(1);\n int N = score_matrix.size(2);\n \n auto options = score_matrix.options();\n auto H = torch::zeros({B, M + 1, N + 1}, options);\n auto output = torch::empty({B}, options);\n \n int threads = 256;\n int blocks = (B + threads - 1) / threads;\n \n AT_DISPATCH_FLOATING_TYPES(score_matrix.scalar_type(), \"sw_forward_kernel\", ([&] {\n sw_forward_kernel<scalar_t><<<blocks, threads>>>(\n score_matrix.data_ptr<scalar_t>(),\n H.data_ptr<scalar_t>(),\n output.data_ptr<scalar_t>(),\n B, M, N,\n static_cast<scalar_t>(gap_open),\n static_cast<scalar_t>(temperature)\n );\n }));\n \n C10_CUDA_CHECK(cudaGetLastError());\n return output;\n}\n\"\"\"\n\ncpp_source = \"\"\"\ntorch::Tensor sw_forward_cuda(torch::Tensor score_matrix, double gap_open, double temperature);\n\"\"\"\n\nsw_module = load_inline(\n name='sw_module',\n cpp_sources=cpp_source,\n cuda_sources=cuda_source,\n functions=['sw_forward_cuda'],\n verbose=True,\n)\n\nclass ModelNew(nn.Module):\n def __init__(self, gap_open=-1.0, temperature=1.0):\n super().__init__()\n self.gap_open = gap_open\n self.temperature = temperature\n\n def forward(self, score_matrix: torch.Tensor) -> torch.Tensor:\n return sw_module.sw_forward_cuda(score_matrix, self.gap_open, self.temperature)"
}
result ← submit_kernel (ok)
oksubmit_kernel
submit_kernel PASSED: (5 / 5) correctness trials all passed.
Kernel runtime: 0.21 μs
Runtime stats: mean=0.21μs median=?μs std=0.00μs
Numerical precision: max_abs_err=3.81e-06 mean_abs_err=2.86e-07 max_rel_err=1.32e-07
Memory: 33.13 MB peak (ref: 33.13 MB, ratio: 0.9999x)
Kernel launches: 5 (ref: 61456, fusion ratio: 12291.2)
Energy: 241.46 mJ/run (ref: 16663.44 mJ, ratio: 69.0112x)
SOL score: 0.0058
Flagged for excessive speedup — this submission may have been rejected by automated review.
final submitted kernel (127 lines, 4,196 chars) — full source, untruncated
import torch
import torch.nn as nn
from torch.utils.cpp_extension import load_inline
cuda_source = """
#include <torch/extension.h>
#include <cuda.h>
#include <cuda_runtime.h>
#include <c10/cuda/CUDAException.h>
#include <cmath>
template <typename scalar_t>
__global__ void sw_forward_kernel(
const scalar_t* __restrict__ score_matrix,
scalar_t* __restrict__ H,
scalar_t* __restrict__ output,
int B, int M, int N,
scalar_t gap_open,
scalar_t temperature) {
int batch_idx = blockIdx.x * blockDim.x + threadIdx.x;
if (batch_idx >= B) return;
// Each thread handles one batch element
// H has shape (B, M+1, N+1), first row and column are 0
// Initialize first row to 0
for (int j = 0; j <= N; j++) {
H[batch_idx * (M+1) * (N+1) + j] = scalar_t(0);
}
// Initialize first column to 0
for (int i = 1; i <= M; i++) {
H[batch_idx * (M+1) * (N+1) + i * (N+1)] = scalar_t(0);
}
scalar_t max_val = scalar_t(0);
scalar_t inv_temp = scalar_t(1) / temperature;
// Iterate through the DP matrix
for (int i = 1; i <= M; i++) {
for (int j = 1; j <= N; j++) {
int64_t idx_prev_row = batch_idx * (int64_t)(M+1) * (N+1) + (i-1) * (N+1) + (j-1);
int64_t idx_curr = batch_idx * (int64_t)(M+1) * (N+1) + i * (N+1) + j;
scalar_t h_diag = H[idx_prev_row]; // H[i-1][j-1]
scalar_t h_up = H[idx_prev_row + 1]; // H[i-1][j]
scalar_t h_left = H[idx_curr - 1]; // H[i][j-1]
scalar_t s = score_matrix[batch_idx * (int64_t)M * N + (i-1) * N + (j-1)];
scalar_t match = h_diag + s;
scalar_t del = h_up + gap_open;
scalar_t ins = h_left + gap_open;
// logsumexp over [0, match, del, ins]
scalar_t a0 = scalar_t(0);
scalar_t a1 = match * inv_temp;
scalar_t a2 = del * inv_temp;
scalar_t a3 = ins * inv_temp;
scalar_t max_a = fmax(fmax(a0, a1), fmax(a2, a3));
scalar_t sum_exp = exp(a0 - max_a) + exp(a1 - max_a) +
exp(a2 - max_a) + exp(a3 - max_a);
scalar_t logsumexp = max_a + log(sum_exp);
scalar_t h_val = temperature * logsumexp;
H[idx_curr] = h_val;
if (h_val > max_val) max_val = h_val;
}
}
output[batch_idx] = max_val;
}
torch::Tensor sw_forward_cuda(torch::Tensor score_matrix, double gap_open, double temperature) {
TORCH_CHECK(score_matrix.is_cuda(), "score_matrix must be a CUDA tensor");
TORCH_CHECK(score_matrix.dim() == 3, "score_matrix must be 3D");
int B = score_matrix.size(0);
int M = score_matrix.size(1);
int N = score_matrix.size(2);
auto options = score_matrix.options();
auto H = torch::zeros({B, M + 1, N + 1}, options);
auto output = torch::empty({B}, options);
int threads = 256;
int blocks = (B + threads - 1) / threads;
AT_DISPATCH_FLOATING_TYPES(score_matrix.scalar_type(), "sw_forward_kernel", ([&] {
sw_forward_kernel<scalar_t><<<blocks, threads>>>(
score_matrix.data_ptr<scalar_t>(),
H.data_ptr<scalar_t>(),
output.data_ptr<scalar_t>(),
B, M, N,
static_cast<scalar_t>(gap_open),
static_cast<scalar_t>(temperature)
);
}));
C10_CUDA_CHECK(cudaGetLastError());
return output;
}
"""
cpp_source = """
torch::Tensor sw_forward_cuda(torch::Tensor score_matrix, double gap_open, double temperature);
"""
sw_module = load_inline(
name='sw_module',
cpp_sources=cpp_source,
cuda_sources=cuda_source,
functions=['sw_forward_cuda'],
verbose=True,
)
class ModelNew(nn.Module):
def __init__(self, gap_open=-1.0, temperature=1.0):
super().__init__()
self.gap_open = gap_open
self.temperature = temperature
def forward(self, score_matrix: torch.Tensor) -> torch.Tensor:
return sw_module.sw_forward_cuda(score_matrix, self.gap_open, self.temperature)