From f4c022112c27a6a3afb588d3d295fc3a7dedd6b5 Mon Sep 17 00:00:00 2001 From: sohailshk <122166523+sohailshk@users.noreply.github.com> Date: Wed, 11 Nov 2026 09:35:48 +0530 Subject: [PATCH 1/3] [RunInference] Do not mutate inference_args in the Anthropic request functions --- .../ml/inference/anthropic_inference.py | 2 ++ .../ml/inference/anthropic_inference_test.py | 22 +++++++++++++++++++ 2 files changed, 24 insertions(+) diff --git a/sdks/python/apache_beam/ml/inference/anthropic_inference.py b/sdks/python/apache_beam/ml/inference/anthropic_inference.py index e35ae52b9a22..040cb20ef4ad 100644 --- a/sdks/python/apache_beam/ml/inference/anthropic_inference.py +++ b/sdks/python/apache_beam/ml/inference/anthropic_inference.py @@ -116,6 +116,7 @@ def message_from_string( inference_args: additional arguments passed to the messages.create call. Common args include 'max_tokens', 'system', 'temperature', 'top_p'. """ + inference_args = dict(inference_args) max_tokens = inference_args.pop('max_tokens', 1024) responses = [] for prompt in batch: @@ -146,6 +147,7 @@ def message_from_conversation( client: the Anthropic client instance. inference_args: additional arguments passed to the messages.create call. """ + inference_args = dict(inference_args) max_tokens = inference_args.pop('max_tokens', 1024) responses = [] for messages in batch: diff --git a/sdks/python/apache_beam/ml/inference/anthropic_inference_test.py b/sdks/python/apache_beam/ml/inference/anthropic_inference_test.py index eb5102edad3f..e97bc43f98ac 100644 --- a/sdks/python/apache_beam/ml/inference/anthropic_inference_test.py +++ b/sdks/python/apache_beam/ml/inference/anthropic_inference_test.py @@ -135,6 +135,17 @@ def test_default_max_tokens(self): call_args = client.messages.create.call_args self.assertEqual(call_args.kwargs['max_tokens'], 1024) + def test_does_not_mutate_inference_args(self): + # RunInference reuses the same inference_args dict for every batch. + client = mock.MagicMock() + client.messages.create.return_value = _make_fake_response("ok") + inference_args = {'max_tokens': 2048, 'temperature': 0.5} + message_from_string(_TEST_MODEL, ['a'], client, inference_args) + message_from_string(_TEST_MODEL, ['b'], client, inference_args) + self.assertEqual(inference_args, {'max_tokens': 2048, 'temperature': 0.5}) + for call_args in client.messages.create.call_args_list: + self.assertEqual(call_args.kwargs['max_tokens'], 2048) + class MessageFromConversationTest(unittest.TestCase): def test_sends_conversation(self): @@ -150,6 +161,17 @@ def test_sends_conversation(self): call_args = client.messages.create.call_args self.assertEqual(call_args.kwargs['messages'], convo) + def test_does_not_mutate_inference_args(self): + client = mock.MagicMock() + client.messages.create.return_value = _make_fake_response("ok") + convo = [{"role": "user", "content": "hi"}] + inference_args = {'max_tokens': 2048} + message_from_conversation(_TEST_MODEL, [convo], client, inference_args) + message_from_conversation(_TEST_MODEL, [convo], client, inference_args) + self.assertEqual(inference_args, {'max_tokens': 2048}) + for call_args in client.messages.create.call_args_list: + self.assertEqual(call_args.kwargs['max_tokens'], 2048) + class AnthropicModelHandlerTest(unittest.TestCase): def test_max_batch_weight_requires_element_size_fn(self): From 3408f45bad110f09700324941eb1c4c64f5d0503 Mon Sep 17 00:00:00 2001 From: sohailshk <122166523+sohailshk@users.noreply.github.com> Date: Thu, 12 Nov 2026 09:05:21 +0530 Subject: [PATCH 2/3] Pass the max_tokens default without copying inference_args --- .../ml/inference/anthropic_inference.py | 15 ++++++--------- 1 file changed, 6 insertions(+), 9 deletions(-) diff --git a/sdks/python/apache_beam/ml/inference/anthropic_inference.py b/sdks/python/apache_beam/ml/inference/anthropic_inference.py index 040cb20ef4ad..4fef42ef2514 100644 --- a/sdks/python/apache_beam/ml/inference/anthropic_inference.py +++ b/sdks/python/apache_beam/ml/inference/anthropic_inference.py @@ -116,16 +116,17 @@ def message_from_string( inference_args: additional arguments passed to the messages.create call. Common args include 'max_tokens', 'system', 'temperature', 'top_p'. """ - inference_args = dict(inference_args) - max_tokens = inference_args.pop('max_tokens', 1024) + # Pass the default only when the caller gave no max_tokens, without + # changing inference_args: RunInference reuses that dict for every batch. + default_args = {} if 'max_tokens' in inference_args else {'max_tokens': 1024} responses = [] for prompt in batch: response = client.messages.create( model=model_name, - max_tokens=max_tokens, messages=[{ "role": "user", "content": prompt }], + **default_args, **inference_args) responses.append(response) return responses @@ -147,15 +148,11 @@ def message_from_conversation( client: the Anthropic client instance. inference_args: additional arguments passed to the messages.create call. """ - inference_args = dict(inference_args) - max_tokens = inference_args.pop('max_tokens', 1024) + default_args = {} if 'max_tokens' in inference_args else {'max_tokens': 1024} responses = [] for messages in batch: response = client.messages.create( - model=model_name, - max_tokens=max_tokens, - messages=messages, - **inference_args) + model=model_name, messages=messages, **default_args, **inference_args) responses.append(response) return responses From 8820eeb4dc8adb6026606788bc67a0f278ece40d Mon Sep 17 00:00:00 2001 From: sohailshk <122166523+sohailshk@users.noreply.github.com> Date: Mon, 16 Nov 2026 22:16:32 +0530 Subject: [PATCH 3/3] Set the Anthropic max_tokens default once in the model handler --- .../ml/inference/anthropic_inference.py | 25 +++++----- .../ml/inference/anthropic_inference_test.py | 49 ++++++++++++++++--- 2 files changed, 53 insertions(+), 21 deletions(-) diff --git a/sdks/python/apache_beam/ml/inference/anthropic_inference.py b/sdks/python/apache_beam/ml/inference/anthropic_inference.py index 4fef42ef2514..13e374b8491a 100644 --- a/sdks/python/apache_beam/ml/inference/anthropic_inference.py +++ b/sdks/python/apache_beam/ml/inference/anthropic_inference.py @@ -115,10 +115,8 @@ def message_from_string( client: the Anthropic client instance. inference_args: additional arguments passed to the messages.create call. Common args include 'max_tokens', 'system', 'temperature', 'top_p'. + AnthropicModelHandler sets max_tokens to 1024 unless it is given. """ - # Pass the default only when the caller gave no max_tokens, without - # changing inference_args: RunInference reuses that dict for every batch. - default_args = {} if 'max_tokens' in inference_args else {'max_tokens': 1024} responses = [] for prompt in batch: response = client.messages.create( @@ -126,7 +124,6 @@ def message_from_string( messages=[{ "role": "user", "content": prompt }], - **default_args, **inference_args) responses.append(response) return responses @@ -148,11 +145,10 @@ def message_from_conversation( client: the Anthropic client instance. inference_args: additional arguments passed to the messages.create call. """ - default_args = {} if 'max_tokens' in inference_args else {'max_tokens': 1024} responses = [] for messages in batch: response = client.messages.create( - model=model_name, messages=messages, **default_args, **inference_args) + model=model_name, messages=messages, **inference_args) responses.append(response) return responses @@ -248,6 +244,12 @@ def __init__( self.system = system self.output_config = output_config + self._default_inference_args = {'max_tokens': 1024} + if system is not None: + self._default_inference_args['system'] = system + if output_config is not None: + self._default_inference_args['output_config'] = output_config + super().__init__( namespace='AnthropicModelHandler', retry_filter=_retry_on_appropriate_error, @@ -275,8 +277,8 @@ def request( ) -> Iterable[PredictionResult]: """Sends a prediction request to the Anthropic API. - Handler-level system and output_config are injected into - inference_args before calling the request function. Per-request + Handler-level defaults (max_tokens, system and output_config) are merged + with inference_args before calling the request function. Per-request values in inference_args take precedence over handler-level values. Args: @@ -288,11 +290,6 @@ def request( Returns: An iterable of PredictionResults. """ - if inference_args is None: - inference_args = {} - if self.system is not None and 'system' not in inference_args: - inference_args['system'] = self.system - if self.output_config is not None and 'output_config' not in inference_args: - inference_args['output_config'] = self.output_config + inference_args = {**self._default_inference_args, **(inference_args or {})} responses = self.request_fn(self.model_name, batch, model, inference_args) return utils._convert_to_result(batch, responses, self.model_name) diff --git a/sdks/python/apache_beam/ml/inference/anthropic_inference_test.py b/sdks/python/apache_beam/ml/inference/anthropic_inference_test.py index e97bc43f98ac..9a3ca6f9fd07 100644 --- a/sdks/python/apache_beam/ml/inference/anthropic_inference_test.py +++ b/sdks/python/apache_beam/ml/inference/anthropic_inference_test.py @@ -128,13 +128,6 @@ def test_passes_inference_args(self): self.assertEqual(call_args.kwargs['max_tokens'], 2048) self.assertEqual(call_args.kwargs['temperature'], 0.5) - def test_default_max_tokens(self): - client = mock.MagicMock() - client.messages.create.return_value = _make_fake_response("ok") - message_from_string(_TEST_MODEL, ['test'], client, {}) - call_args = client.messages.create.call_args - self.assertEqual(call_args.kwargs['max_tokens'], 1024) - def test_does_not_mutate_inference_args(self): # RunInference reuses the same inference_args dict for every batch. client = mock.MagicMock() @@ -216,6 +209,48 @@ def test_request_returns_prediction_results(self): self.assertEqual(results[1].example, 'q2') self.assertEqual(results[1].inference, resp2) + def test_request_applies_default_max_tokens(self): + handler = AnthropicModelHandler( + model_name=_TEST_MODEL, request_fn=message_from_string, api_key='fake') + mock_client = mock.MagicMock() + mock_client.messages.create.return_value = _make_fake_response("ok") + + handler.request(['test'], mock_client, {'temperature': 0.5}) + handler.request(['test'], mock_client, None) + + for call_args in mock_client.messages.create.call_args_list: + self.assertEqual(call_args.kwargs['max_tokens'], 1024) + + def test_request_max_tokens_override(self): + handler = AnthropicModelHandler( + model_name=_TEST_MODEL, request_fn=message_from_string, api_key='fake') + mock_client = mock.MagicMock() + mock_client.messages.create.return_value = _make_fake_response("ok") + + handler.request(['test'], mock_client, {'max_tokens': 2048}) + + call_args = mock_client.messages.create.call_args + self.assertEqual(call_args.kwargs['max_tokens'], 2048) + + def test_request_does_not_mutate_inference_args(self): + # RunInference reuses the same inference_args dict for every batch. + handler = AnthropicModelHandler( + model_name=_TEST_MODEL, + request_fn=message_from_string, + api_key='fake', + system='Be concise.', + output_config={'format': { + 'type': 'text' + }}) + mock_client = mock.MagicMock() + mock_client.messages.create.return_value = _make_fake_response("ok") + inference_args = {'temperature': 0.5} + + handler.request(['a'], mock_client, inference_args) + handler.request(['b'], mock_client, inference_args) + + self.assertEqual(inference_args, {'temperature': 0.5}) + def test_batch_elements_kwargs(self): handler = AnthropicModelHandler( model_name=_TEST_MODEL,