From f9e90f7a9e99462264a2445610642f4efb024c8c Mon Sep 17 00:00:00 2001 From: vsergeev Date: Thu, 19 Nov 2026 13:05:55 +0300 Subject: [PATCH 1/2] Reduce peak attention memory in AutoInt and FT-Transformer --- lightautoml/ml_algo/torch_based/_attention.py | 36 ++++++ .../torch_based/autoint/autoint_utils.py | 16 +-- .../fttransformer/fttransformer_utils.py | 47 ++++---- .../test_ml_algo/test_attention_memory.py | 103 ++++++++++++++++++ 4 files changed, 174 insertions(+), 28 deletions(-) create mode 100644 lightautoml/ml_algo/torch_based/_attention.py create mode 100644 tests/unit/test_ml_algo/test_attention_memory.py diff --git a/lightautoml/ml_algo/torch_based/_attention.py b/lightautoml/ml_algo/torch_based/_attention.py new file mode 100644 index 00000000..76ec65e9 --- /dev/null +++ b/lightautoml/ml_algo/torch_based/_attention.py @@ -0,0 +1,36 @@ +"""Limit the size and lifetime of intermediate self-attention scores.""" + +from inspect import signature + +import torch + +from torch.utils.checkpoint import checkpoint + + +_SCORE_BLOCK_BYTES = 32 * 1024 ** 2 +try: + _HAS_NON_REENTRANT_CHECKPOINT = "use_reentrant" in signature(checkpoint).parameters +except (TypeError, ValueError): + # The documentation build imports this module with a mocked torch. + _HAS_NON_REENTRANT_CHECKPOINT = False + + +def attention_by_query(attend, query, key, *args, num_heads): + """Split independent query fields while keeping the softmax over all keys.""" + batch, fields = query.shape[:2] + # A score block has shape (batch, heads, query_fields, key_fields). + # The math backend may compute the scores in float32 for half inputs. + bytes_per_query = batch * num_heads * key.size(1) * max(query.element_size(), 4) + block_size = max(1, _SCORE_BLOCK_BYTES // max(bytes_per_query, 1)) + if fields <= block_size: + return attend(query, key, *args) + + blocks = [] + for block in query.split(block_size, dim=1): + if torch.is_grad_enabled() and _HAS_NON_REENTRANT_CHECKPOINT: + # Recompute each block's scores in backward, with the same dropout mask. + out = checkpoint(attend, block, key, *args, use_reentrant=False) + else: + out = attend(block, key, *args) + blocks.append(out) + return torch.cat(blocks, dim=1) diff --git a/lightautoml/ml_algo/torch_based/autoint/autoint_utils.py b/lightautoml/ml_algo/torch_based/autoint/autoint_utils.py index 0cce7652..a80d4b27 100644 --- a/lightautoml/ml_algo/torch_based/autoint/autoint_utils.py +++ b/lightautoml/ml_algo/torch_based/autoint/autoint_utils.py @@ -8,6 +8,8 @@ from torch import nn, Tensor from torch.nn import functional as F +from .._attention import attention_by_query + EmbeddingInfo = namedtuple("EmbeddingInfo", ["num_fields", "output_size"]) UniformEmbeddingInfo = namedtuple("EmbeddingInfo", ["num_fields", "embedding_size", "output_size"]) @@ -138,6 +140,10 @@ def __init__( self.to(device) + def _attention_mass(self, queries, keys): + scores = torch.einsum("rdoh,rfoh->rdfh", queries, keys) + return self.dropout(F.softmax(scores, dim=2)).sum(dim=2) + def forward(self, x: Tensor) -> Tensor: """Transform the input tensor with attention interaction. @@ -164,13 +170,9 @@ def forward(self, x: Tensor) -> Tensor: if self.use_residual: rsdl = torch.tensordot(x, self.w_act(self.W_r), dims=([-1], [0])) - product = torch.einsum("rdoh,rfoh->rdfh", qrys, keys) # (R, F, F, H) - - alpha = F.softmax(product, dim=2) # (R, F, F, H) - alpha = self.dropout(alpha) - - # (R, F, F, H) * (R, F, O, H) -> (R, F, O, H) - out = torch.einsum("rfdh,rfoh->rfoh", alpha, vals) + # Each query scales its own value by the sum of its post-dropout weights. + mass = attention_by_query(self._attention_mass, qrys, keys, num_heads=keys.size(-1)) + out = vals * mass.unsqueeze(2) out = out.reshape((num_rows, num_fields, -1)) # (R, F, O * H) if self.use_residual: out = out + rsdl # (R, F, O * H) diff --git a/lightautoml/ml_algo/torch_based/fttransformer/fttransformer_utils.py b/lightautoml/ml_algo/torch_based/fttransformer/fttransformer_utils.py index a88be717..af0d2e7a 100644 --- a/lightautoml/ml_algo/torch_based/fttransformer/fttransformer_utils.py +++ b/lightautoml/ml_algo/torch_based/fttransformer/fttransformer_utils.py @@ -4,6 +4,8 @@ import torch.nn as nn import torch.nn.functional as F +from .._attention import attention_by_query + class GEGLU(nn.Module): """GEGLU activation for Attention block.""" @@ -41,16 +43,28 @@ def __init__(self, dim, heads=8, dim_head=64, dropout=0.0): self.to_out = nn.Linear(inner_dim, dim, bias=False) self.dropout = nn.Dropout(dropout) - def forward(self, x): + def _weights(self, q, k): + return torch.softmax(torch.matmul(q * self.scale, k.transpose(-2, -1)), dim=-1) + + def _attend(self, q, k, v): + q, k, v = q.transpose(1, 2), k.transpose(1, 2), v.transpose(1, 2) + if hasattr(F, "scaled_dot_product_attention"): + out = F.scaled_dot_product_attention(q, k, v, dropout_p=self.dropout.p if self.dropout.training else 0.0) + else: + out = torch.matmul(self.dropout(self._weights(q, k)), v) + return out.transpose(1, 2) + + def forward(self, x, return_attn: bool = True): """Transform the input tensor with attention. Args: x : torch.Tensor 3-d tensor; for example, embedded numeric and/or categorical values, or the output of a previous attention layer. + return_attn: Return the pre-dropout attention weights alongside the output. Returns: - torch.Tensor + Projected output and the attention weights, or an empty tensor if omitted. """ batch_size, seq_len, dim = x.shape @@ -60,20 +74,15 @@ def forward(self, x): qkv = self.to_qkv(x) q, k, v = torch.split(qkv, qkv.size(-1) // 3, dim=-1) - q = q.view(batch_size, seq_len, h, -1).permute(0, 2, 1, 3) - k = k.view(batch_size, seq_len, h, -1).permute(0, 2, 1, 3) - v = v.view(batch_size, seq_len, h, -1).permute(0, 2, 1, 3) - - q = q * self.scale - sim = torch.matmul(q, k.transpose(-2, -1)) - attn = torch.softmax(sim, dim=-1) - dropped_attn = self.dropout(attn) - - out = torch.matmul(dropped_attn, v) - out = out.permute(0, 2, 1, 3).contiguous().view(batch_size, seq_len, -1) - out = self.to_out(out) - - return out, attn + q = q.view(batch_size, seq_len, h, -1) + k = k.view(batch_size, seq_len, h, -1) + v = v.view(batch_size, seq_len, h, -1) + if torch.jit.is_scripting() or return_attn: + weights = self._weights(q.transpose(1, 2), k.transpose(1, 2)) + out = torch.matmul(self.dropout(weights), v.transpose(1, 2)).transpose(1, 2) + else: + out = attention_by_query(self._attend, q, k, v, num_heads=h) + return self.to_out(out.reshape(batch_size, seq_len, -1)), weights if return_attn else x.new_empty((0,)) # transformer @@ -87,7 +96,6 @@ class Transformer(nn.Module): dim_head: Attention head dimension. attn_dropout: Post-Attention dropout. ff_dropout: Feed-Forward Dropout. - return_attn: Return attention scores or not. """ def __init__(self, dim, depth, heads, dim_head, attn_dropout, ff_dropout): @@ -116,11 +124,8 @@ def forward(self, x): torch.Tensor """ - post_softmax_attns = [] - for attn, ff in self.layers: - attn_out, post_softmax_attn = attn(x) - post_softmax_attns.append(post_softmax_attn) + attn_out, _ = attn(x, return_attn=False) x = attn_out + x x = ff(x) + x diff --git a/tests/unit/test_ml_algo/test_attention_memory.py b/tests/unit/test_ml_algo/test_attention_memory.py new file mode 100644 index 00000000..0c9ff888 --- /dev/null +++ b/tests/unit/test_ml_algo/test_attention_memory.py @@ -0,0 +1,103 @@ +import pytest +import torch + +from torch.nn import functional as F + +from lightautoml.ml_algo.torch_based import _attention +from lightautoml.ml_algo.torch_based.autoint.autoint_utils import AttnInteractionLayer +from lightautoml.ml_algo.torch_based.fttransformer.fttransformer_utils import Attention + + +class AttentionWeightsSum(torch.nn.Module): + def __init__(self, layer): + super().__init__() + self.layer = layer + + def forward(self, x): + return self.layer(x)[1].sum(-1) + + +def assert_outputs_and_gradients(layer, x, actual, expected): + torch.testing.assert_close(actual, expected) + grad = torch.randn_like(actual) + inputs = (x, *layer.parameters()) + actual_grads = torch.autograd.grad(actual, inputs, grad) + expected_grads = torch.autograd.grad(expected, inputs, grad) + for actual_grad, expected_grad in zip(actual_grads, expected_grads): + torch.testing.assert_close(actual_grad, expected_grad) + + +@pytest.mark.filterwarnings("ignore:`torch.jit.script` is deprecated") +def test_scripted_ft_attention_still_returns_weights(): + layer = Attention(dim=8, heads=2, dim_head=4).eval() + x = torch.randn(3, 5, 8) + consumer = AttentionWeightsSum(layer) + torch.testing.assert_close(torch.jit.script(consumer)(x), consumer(x)) + + +@pytest.mark.parametrize("block_fields", [2, 5]) +@pytest.mark.parametrize("legacy", [True, False]) +def test_ft_attention_weights_and_output_only(monkeypatch, block_fields, legacy): + torch.manual_seed(42) + layer = Attention(dim=8, heads=2, dim_head=4, dropout=0.4).double().eval() + x = torch.randn(3, 5, 8, dtype=torch.float64, requires_grad=True) + monkeypatch.setattr(_attention, "_SCORE_BLOCK_BYTES", block_fields * 3 * 2 * 5 * 8) + if legacy: + monkeypatch.delattr(F, "scaled_dot_product_attention", raising=False) + monkeypatch.setattr(_attention, "_HAS_NON_REENTRANT_CHECKPOINT", False) + + expected, weights = layer(x) + actual, omitted = layer(x, return_attn=False) + assert omitted.numel() == 0 + assert weights.shape == (3, 2, 5, 5) + torch.testing.assert_close(weights.sum(-1), torch.ones_like(weights[..., 0])) + assert_outputs_and_gradients(layer, x, actual, expected) + + +def test_query_block_limit_in_backward(monkeypatch): + query = torch.randn(3, 5, 2, 4, requires_grad=True) + key = torch.randn_like(query, requires_grad=True) + monkeypatch.setattr(_attention, "_SCORE_BLOCK_BYTES", 3 * 2 * 5 * 4 * 2) + block_sizes = [] + + def attend(q, k): + block_sizes.append(q.size(1)) + return torch.einsum("bqhd,bkhd->bqhk", q, k).softmax(-1) + + out = _attention.attention_by_query(attend, query, key, num_heads=2) + assert block_sizes == [2, 2, 1] + out.backward(torch.randn_like(out)) + if _attention._HAS_NON_REENTRANT_CHECKPOINT: + assert sorted(block_sizes[3:]) == [1, 2, 2] + + +def test_ft_attention_training_dropout(monkeypatch): + layer = Attention(dim=8, heads=2, dim_head=4, dropout=1.0).train() + x = torch.randn(3, 5, 8, requires_grad=True) + monkeypatch.setattr(_attention, "_SCORE_BLOCK_BYTES", 2 * 3 * 2 * 5 * 4) + expected, _ = layer(x) + actual, _ = layer(x, return_attn=False) + assert torch.count_nonzero(actual) == 0 + assert_outputs_and_gradients(layer, x, actual, expected) + + +@pytest.mark.parametrize("block_fields,dropout", [(5, 0.35), (2, 0.35), (2, 0.0)]) +def test_autoint_preserves_query_value_weighting(monkeypatch, block_fields, dropout): + torch.manual_seed(42) + layer = AttnInteractionLayer(3, field_output_size=4, num_heads=2, dropout=dropout).double() + x = torch.randn(3, 5, 3, dtype=torch.float64, requires_grad=True) + monkeypatch.setattr(_attention, "_SCORE_BLOCK_BYTES", block_fields * 3 * 2 * 5 * 8) + + # Original contraction, with the same query blocks so dropout draws the same masks. + query, key, value = (torch.tensordot(x, w, dims=([-1], [0])) for w in (layer.W_q, layer.W_k, layer.W_v)) + torch.manual_seed(7) + blocks = [] + for q, v in zip(query.split(block_fields, dim=1), value.split(block_fields, dim=1)): + scores = torch.einsum("rdoh,rfoh->rdfh", q, key) + weights = layer.dropout(scores.softmax(dim=2)) + blocks.append(torch.einsum("rfdh,rfoh->rfoh", weights, v)) + expected = torch.cat(blocks, dim=1).flatten(2) + x @ layer.W_r + expected = layer.layer_norm(F.leaky_relu(expected)) + torch.manual_seed(7) + actual = layer(x) + assert_outputs_and_gradients(layer, x, actual, expected) From 3a9fdcc33ecee2c982f0a51cf46158b0124d3fa8 Mon Sep 17 00:00:00 2001 From: vsergeev Date: Thu, 19 Nov 2026 17:38:23 +0300 Subject: [PATCH 2/2] Handle PyTorch warnings in the TorchScript compatibility test --- tests/unit/test_ml_algo/test_attention_memory.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tests/unit/test_ml_algo/test_attention_memory.py b/tests/unit/test_ml_algo/test_attention_memory.py index 0c9ff888..b53d2840 100644 --- a/tests/unit/test_ml_algo/test_attention_memory.py +++ b/tests/unit/test_ml_algo/test_attention_memory.py @@ -27,7 +27,10 @@ def assert_outputs_and_gradients(layer, x, actual, expected): torch.testing.assert_close(actual_grad, expected_grad) -@pytest.mark.filterwarnings("ignore:`torch.jit.script` is deprecated") +# PyTorch 2.2 reads string literals via ast.Str and its s attribute on Python 3.12. +@pytest.mark.filterwarnings(r"ignore:(ast\.Str|Attribute s) is deprecated:DeprecationWarning:torch\.jit\.frontend$") +@pytest.mark.filterwarnings(r"ignore:`torch\.jit\.script` is not supported in Python 3\.14\+:FutureWarning") +@pytest.mark.filterwarnings(r"ignore:`torch\.jit\.script` is deprecated") def test_scripted_ft_attention_still_returns_weights(): layer = Attention(dim=8, heads=2, dim_head=4).eval() x = torch.randn(3, 5, 8)