Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
36 changes: 36 additions & 0 deletions lightautoml/ml_algo/torch_based/_attention.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,36 @@
"""Limit the size and lifetime of intermediate self-attention scores."""

from inspect import signature

import torch

from torch.utils.checkpoint import checkpoint


_SCORE_BLOCK_BYTES = 32 * 1024 ** 2
try:
_HAS_NON_REENTRANT_CHECKPOINT = "use_reentrant" in signature(checkpoint).parameters
except (TypeError, ValueError):
# The documentation build imports this module with a mocked torch.
_HAS_NON_REENTRANT_CHECKPOINT = False


def attention_by_query(attend, query, key, *args, num_heads):
"""Split independent query fields while keeping the softmax over all keys."""
batch, fields = query.shape[:2]
# A score block has shape (batch, heads, query_fields, key_fields).
# The math backend may compute the scores in float32 for half inputs.
bytes_per_query = batch * num_heads * key.size(1) * max(query.element_size(), 4)
block_size = max(1, _SCORE_BLOCK_BYTES // max(bytes_per_query, 1))
if fields <= block_size:
return attend(query, key, *args)

blocks = []
for block in query.split(block_size, dim=1):
if torch.is_grad_enabled() and _HAS_NON_REENTRANT_CHECKPOINT:
# Recompute each block's scores in backward, with the same dropout mask.
out = checkpoint(attend, block, key, *args, use_reentrant=False)
else:
out = attend(block, key, *args)
blocks.append(out)
return torch.cat(blocks, dim=1)
16 changes: 9 additions & 7 deletions lightautoml/ml_algo/torch_based/autoint/autoint_utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,8 @@
from torch import nn, Tensor
from torch.nn import functional as F

from .._attention import attention_by_query


EmbeddingInfo = namedtuple("EmbeddingInfo", ["num_fields", "output_size"])
UniformEmbeddingInfo = namedtuple("EmbeddingInfo", ["num_fields", "embedding_size", "output_size"])
Expand Down Expand Up @@ -138,6 +140,10 @@ def __init__(

self.to(device)

def _attention_mass(self, queries, keys):
scores = torch.einsum("rdoh,rfoh->rdfh", queries, keys)
return self.dropout(F.softmax(scores, dim=2)).sum(dim=2)

def forward(self, x: Tensor) -> Tensor:
"""Transform the input tensor with attention interaction.

Expand All @@ -164,13 +170,9 @@ def forward(self, x: Tensor) -> Tensor:
if self.use_residual:
rsdl = torch.tensordot(x, self.w_act(self.W_r), dims=([-1], [0]))

product = torch.einsum("rdoh,rfoh->rdfh", qrys, keys) # (R, F, F, H)

alpha = F.softmax(product, dim=2) # (R, F, F, H)
alpha = self.dropout(alpha)

# (R, F, F, H) * (R, F, O, H) -> (R, F, O, H)
out = torch.einsum("rfdh,rfoh->rfoh", alpha, vals)
# Each query scales its own value by the sum of its post-dropout weights.
mass = attention_by_query(self._attention_mass, qrys, keys, num_heads=keys.size(-1))
out = vals * mass.unsqueeze(2)
out = out.reshape((num_rows, num_fields, -1)) # (R, F, O * H)
if self.use_residual:
out = out + rsdl # (R, F, O * H)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,8 @@
import torch.nn as nn
import torch.nn.functional as F

from .._attention import attention_by_query


class GEGLU(nn.Module):
"""GEGLU activation for Attention block."""
Expand Down Expand Up @@ -41,16 +43,28 @@ def __init__(self, dim, heads=8, dim_head=64, dropout=0.0):
self.to_out = nn.Linear(inner_dim, dim, bias=False)
self.dropout = nn.Dropout(dropout)

def forward(self, x):
def _weights(self, q, k):
return torch.softmax(torch.matmul(q * self.scale, k.transpose(-2, -1)), dim=-1)

def _attend(self, q, k, v):
q, k, v = q.transpose(1, 2), k.transpose(1, 2), v.transpose(1, 2)
if hasattr(F, "scaled_dot_product_attention"):
out = F.scaled_dot_product_attention(q, k, v, dropout_p=self.dropout.p if self.dropout.training else 0.0)
else:
out = torch.matmul(self.dropout(self._weights(q, k)), v)
return out.transpose(1, 2)

def forward(self, x, return_attn: bool = True):
"""Transform the input tensor with attention.

Args:
x : torch.Tensor
3-d tensor; for example, embedded numeric and/or categorical values,
or the output of a previous attention layer.
return_attn: Return the pre-dropout attention weights alongside the output.

Returns:
torch.Tensor
Projected output and the attention weights, or an empty tensor if omitted.

"""
batch_size, seq_len, dim = x.shape
Expand All @@ -60,20 +74,15 @@ def forward(self, x):
qkv = self.to_qkv(x)
q, k, v = torch.split(qkv, qkv.size(-1) // 3, dim=-1)

q = q.view(batch_size, seq_len, h, -1).permute(0, 2, 1, 3)
k = k.view(batch_size, seq_len, h, -1).permute(0, 2, 1, 3)
v = v.view(batch_size, seq_len, h, -1).permute(0, 2, 1, 3)

q = q * self.scale
sim = torch.matmul(q, k.transpose(-2, -1))
attn = torch.softmax(sim, dim=-1)
dropped_attn = self.dropout(attn)

out = torch.matmul(dropped_attn, v)
out = out.permute(0, 2, 1, 3).contiguous().view(batch_size, seq_len, -1)
out = self.to_out(out)

return out, attn
q = q.view(batch_size, seq_len, h, -1)
k = k.view(batch_size, seq_len, h, -1)
v = v.view(batch_size, seq_len, h, -1)
if torch.jit.is_scripting() or return_attn:
weights = self._weights(q.transpose(1, 2), k.transpose(1, 2))
out = torch.matmul(self.dropout(weights), v.transpose(1, 2)).transpose(1, 2)
else:
out = attention_by_query(self._attend, q, k, v, num_heads=h)
return self.to_out(out.reshape(batch_size, seq_len, -1)), weights if return_attn else x.new_empty((0,))


# transformer
Expand All @@ -87,7 +96,6 @@ class Transformer(nn.Module):
dim_head: Attention head dimension.
attn_dropout: Post-Attention dropout.
ff_dropout: Feed-Forward Dropout.
return_attn: Return attention scores or not.
"""

def __init__(self, dim, depth, heads, dim_head, attn_dropout, ff_dropout):
Expand Down Expand Up @@ -116,11 +124,8 @@ def forward(self, x):
torch.Tensor

"""
post_softmax_attns = []

for attn, ff in self.layers:
attn_out, post_softmax_attn = attn(x)
post_softmax_attns.append(post_softmax_attn)
attn_out, _ = attn(x, return_attn=False)
x = attn_out + x
x = ff(x) + x

Expand Down
106 changes: 106 additions & 0 deletions tests/unit/test_ml_algo/test_attention_memory.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,106 @@
import pytest
import torch

from torch.nn import functional as F

from lightautoml.ml_algo.torch_based import _attention
from lightautoml.ml_algo.torch_based.autoint.autoint_utils import AttnInteractionLayer
from lightautoml.ml_algo.torch_based.fttransformer.fttransformer_utils import Attention


class AttentionWeightsSum(torch.nn.Module):
def __init__(self, layer):
super().__init__()
self.layer = layer

def forward(self, x):
return self.layer(x)[1].sum(-1)


def assert_outputs_and_gradients(layer, x, actual, expected):
torch.testing.assert_close(actual, expected)
grad = torch.randn_like(actual)
inputs = (x, *layer.parameters())
actual_grads = torch.autograd.grad(actual, inputs, grad)
expected_grads = torch.autograd.grad(expected, inputs, grad)
for actual_grad, expected_grad in zip(actual_grads, expected_grads):
torch.testing.assert_close(actual_grad, expected_grad)


# PyTorch 2.2 reads string literals via ast.Str and its s attribute on Python 3.12.
@pytest.mark.filterwarnings(r"ignore:(ast\.Str|Attribute s) is deprecated:DeprecationWarning:torch\.jit\.frontend$")
@pytest.mark.filterwarnings(r"ignore:`torch\.jit\.script` is not supported in Python 3\.14\+:FutureWarning")
@pytest.mark.filterwarnings(r"ignore:`torch\.jit\.script` is deprecated")
def test_scripted_ft_attention_still_returns_weights():
layer = Attention(dim=8, heads=2, dim_head=4).eval()
x = torch.randn(3, 5, 8)
consumer = AttentionWeightsSum(layer)
torch.testing.assert_close(torch.jit.script(consumer)(x), consumer(x))


@pytest.mark.parametrize("block_fields", [2, 5])
@pytest.mark.parametrize("legacy", [True, False])
def test_ft_attention_weights_and_output_only(monkeypatch, block_fields, legacy):
torch.manual_seed(42)
layer = Attention(dim=8, heads=2, dim_head=4, dropout=0.4).double().eval()
x = torch.randn(3, 5, 8, dtype=torch.float64, requires_grad=True)
monkeypatch.setattr(_attention, "_SCORE_BLOCK_BYTES", block_fields * 3 * 2 * 5 * 8)
if legacy:
monkeypatch.delattr(F, "scaled_dot_product_attention", raising=False)
monkeypatch.setattr(_attention, "_HAS_NON_REENTRANT_CHECKPOINT", False)

expected, weights = layer(x)
actual, omitted = layer(x, return_attn=False)
assert omitted.numel() == 0
assert weights.shape == (3, 2, 5, 5)
torch.testing.assert_close(weights.sum(-1), torch.ones_like(weights[..., 0]))
assert_outputs_and_gradients(layer, x, actual, expected)


def test_query_block_limit_in_backward(monkeypatch):
query = torch.randn(3, 5, 2, 4, requires_grad=True)
key = torch.randn_like(query, requires_grad=True)
monkeypatch.setattr(_attention, "_SCORE_BLOCK_BYTES", 3 * 2 * 5 * 4 * 2)
block_sizes = []

def attend(q, k):
block_sizes.append(q.size(1))
return torch.einsum("bqhd,bkhd->bqhk", q, k).softmax(-1)

out = _attention.attention_by_query(attend, query, key, num_heads=2)
assert block_sizes == [2, 2, 1]
out.backward(torch.randn_like(out))
if _attention._HAS_NON_REENTRANT_CHECKPOINT:
assert sorted(block_sizes[3:]) == [1, 2, 2]


def test_ft_attention_training_dropout(monkeypatch):
layer = Attention(dim=8, heads=2, dim_head=4, dropout=1.0).train()
x = torch.randn(3, 5, 8, requires_grad=True)
monkeypatch.setattr(_attention, "_SCORE_BLOCK_BYTES", 2 * 3 * 2 * 5 * 4)
expected, _ = layer(x)
actual, _ = layer(x, return_attn=False)
assert torch.count_nonzero(actual) == 0
assert_outputs_and_gradients(layer, x, actual, expected)


@pytest.mark.parametrize("block_fields,dropout", [(5, 0.35), (2, 0.35), (2, 0.0)])
def test_autoint_preserves_query_value_weighting(monkeypatch, block_fields, dropout):
torch.manual_seed(42)
layer = AttnInteractionLayer(3, field_output_size=4, num_heads=2, dropout=dropout).double()
x = torch.randn(3, 5, 3, dtype=torch.float64, requires_grad=True)
monkeypatch.setattr(_attention, "_SCORE_BLOCK_BYTES", block_fields * 3 * 2 * 5 * 8)

# Original contraction, with the same query blocks so dropout draws the same masks.
query, key, value = (torch.tensordot(x, w, dims=([-1], [0])) for w in (layer.W_q, layer.W_k, layer.W_v))
torch.manual_seed(7)
blocks = []
for q, v in zip(query.split(block_fields, dim=1), value.split(block_fields, dim=1)):
scores = torch.einsum("rdoh,rfoh->rdfh", q, key)
weights = layer.dropout(scores.softmax(dim=2))
blocks.append(torch.einsum("rfdh,rfoh->rfoh", weights, v))
expected = torch.cat(blocks, dim=1).flatten(2) + x @ layer.W_r
expected = layer.layer_norm(F.leaky_relu(expected))
torch.manual_seed(7)
actual = layer(x)
assert_outputs_and_gradients(layer, x, actual, expected)
Loading