diff --git a/sdks/python/apache_beam/ml/inference/anthropic_inference.py b/sdks/python/apache_beam/ml/inference/anthropic_inference.py index e35ae52b9a22..13e374b8491a 100644 --- a/sdks/python/apache_beam/ml/inference/anthropic_inference.py +++ b/sdks/python/apache_beam/ml/inference/anthropic_inference.py @@ -115,13 +115,12 @@ def message_from_string( client: the Anthropic client instance. inference_args: additional arguments passed to the messages.create call. Common args include 'max_tokens', 'system', 'temperature', 'top_p'. + AnthropicModelHandler sets max_tokens to 1024 unless it is given. """ - max_tokens = inference_args.pop('max_tokens', 1024) responses = [] for prompt in batch: response = client.messages.create( model=model_name, - max_tokens=max_tokens, messages=[{ "role": "user", "content": prompt }], @@ -146,14 +145,10 @@ def message_from_conversation( client: the Anthropic client instance. inference_args: additional arguments passed to the messages.create call. """ - max_tokens = inference_args.pop('max_tokens', 1024) responses = [] for messages in batch: response = client.messages.create( - model=model_name, - max_tokens=max_tokens, - messages=messages, - **inference_args) + model=model_name, messages=messages, **inference_args) responses.append(response) return responses @@ -249,6 +244,12 @@ def __init__( self.system = system self.output_config = output_config + self._default_inference_args = {'max_tokens': 1024} + if system is not None: + self._default_inference_args['system'] = system + if output_config is not None: + self._default_inference_args['output_config'] = output_config + super().__init__( namespace='AnthropicModelHandler', retry_filter=_retry_on_appropriate_error, @@ -276,8 +277,8 @@ def request( ) -> Iterable[PredictionResult]: """Sends a prediction request to the Anthropic API. - Handler-level system and output_config are injected into - inference_args before calling the request function. Per-request + Handler-level defaults (max_tokens, system and output_config) are merged + with inference_args before calling the request function. Per-request values in inference_args take precedence over handler-level values. Args: @@ -289,11 +290,6 @@ def request( Returns: An iterable of PredictionResults. """ - if inference_args is None: - inference_args = {} - if self.system is not None and 'system' not in inference_args: - inference_args['system'] = self.system - if self.output_config is not None and 'output_config' not in inference_args: - inference_args['output_config'] = self.output_config + inference_args = {**self._default_inference_args, **(inference_args or {})} responses = self.request_fn(self.model_name, batch, model, inference_args) return utils._convert_to_result(batch, responses, self.model_name) diff --git a/sdks/python/apache_beam/ml/inference/anthropic_inference_test.py b/sdks/python/apache_beam/ml/inference/anthropic_inference_test.py index eb5102edad3f..9a3ca6f9fd07 100644 --- a/sdks/python/apache_beam/ml/inference/anthropic_inference_test.py +++ b/sdks/python/apache_beam/ml/inference/anthropic_inference_test.py @@ -128,12 +128,16 @@ def test_passes_inference_args(self): self.assertEqual(call_args.kwargs['max_tokens'], 2048) self.assertEqual(call_args.kwargs['temperature'], 0.5) - def test_default_max_tokens(self): + def test_does_not_mutate_inference_args(self): + # RunInference reuses the same inference_args dict for every batch. client = mock.MagicMock() client.messages.create.return_value = _make_fake_response("ok") - message_from_string(_TEST_MODEL, ['test'], client, {}) - call_args = client.messages.create.call_args - self.assertEqual(call_args.kwargs['max_tokens'], 1024) + inference_args = {'max_tokens': 2048, 'temperature': 0.5} + message_from_string(_TEST_MODEL, ['a'], client, inference_args) + message_from_string(_TEST_MODEL, ['b'], client, inference_args) + self.assertEqual(inference_args, {'max_tokens': 2048, 'temperature': 0.5}) + for call_args in client.messages.create.call_args_list: + self.assertEqual(call_args.kwargs['max_tokens'], 2048) class MessageFromConversationTest(unittest.TestCase): @@ -150,6 +154,17 @@ def test_sends_conversation(self): call_args = client.messages.create.call_args self.assertEqual(call_args.kwargs['messages'], convo) + def test_does_not_mutate_inference_args(self): + client = mock.MagicMock() + client.messages.create.return_value = _make_fake_response("ok") + convo = [{"role": "user", "content": "hi"}] + inference_args = {'max_tokens': 2048} + message_from_conversation(_TEST_MODEL, [convo], client, inference_args) + message_from_conversation(_TEST_MODEL, [convo], client, inference_args) + self.assertEqual(inference_args, {'max_tokens': 2048}) + for call_args in client.messages.create.call_args_list: + self.assertEqual(call_args.kwargs['max_tokens'], 2048) + class AnthropicModelHandlerTest(unittest.TestCase): def test_max_batch_weight_requires_element_size_fn(self): @@ -194,6 +209,48 @@ def test_request_returns_prediction_results(self): self.assertEqual(results[1].example, 'q2') self.assertEqual(results[1].inference, resp2) + def test_request_applies_default_max_tokens(self): + handler = AnthropicModelHandler( + model_name=_TEST_MODEL, request_fn=message_from_string, api_key='fake') + mock_client = mock.MagicMock() + mock_client.messages.create.return_value = _make_fake_response("ok") + + handler.request(['test'], mock_client, {'temperature': 0.5}) + handler.request(['test'], mock_client, None) + + for call_args in mock_client.messages.create.call_args_list: + self.assertEqual(call_args.kwargs['max_tokens'], 1024) + + def test_request_max_tokens_override(self): + handler = AnthropicModelHandler( + model_name=_TEST_MODEL, request_fn=message_from_string, api_key='fake') + mock_client = mock.MagicMock() + mock_client.messages.create.return_value = _make_fake_response("ok") + + handler.request(['test'], mock_client, {'max_tokens': 2048}) + + call_args = mock_client.messages.create.call_args + self.assertEqual(call_args.kwargs['max_tokens'], 2048) + + def test_request_does_not_mutate_inference_args(self): + # RunInference reuses the same inference_args dict for every batch. + handler = AnthropicModelHandler( + model_name=_TEST_MODEL, + request_fn=message_from_string, + api_key='fake', + system='Be concise.', + output_config={'format': { + 'type': 'text' + }}) + mock_client = mock.MagicMock() + mock_client.messages.create.return_value = _make_fake_response("ok") + inference_args = {'temperature': 0.5} + + handler.request(['a'], mock_client, inference_args) + handler.request(['b'], mock_client, inference_args) + + self.assertEqual(inference_args, {'temperature': 0.5}) + def test_batch_elements_kwargs(self): handler = AnthropicModelHandler( model_name=_TEST_MODEL,