Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
26 changes: 11 additions & 15 deletions sdks/python/apache_beam/ml/inference/anthropic_inference.py
Original file line number Diff line number Diff line change
Expand Up @@ -115,13 +115,12 @@ def message_from_string(
client: the Anthropic client instance.
inference_args: additional arguments passed to the messages.create call.
Common args include 'max_tokens', 'system', 'temperature', 'top_p'.
AnthropicModelHandler sets max_tokens to 1024 unless it is given.
"""
max_tokens = inference_args.pop('max_tokens', 1024)
responses = []
for prompt in batch:
response = client.messages.create(
model=model_name,
max_tokens=max_tokens,
messages=[{
"role": "user", "content": prompt
}],
Expand All @@ -146,14 +145,10 @@ def message_from_conversation(
client: the Anthropic client instance.
inference_args: additional arguments passed to the messages.create call.
"""
max_tokens = inference_args.pop('max_tokens', 1024)
responses = []
for messages in batch:
response = client.messages.create(
model=model_name,
max_tokens=max_tokens,
messages=messages,
**inference_args)
model=model_name, messages=messages, **inference_args)
responses.append(response)
return responses

Expand Down Expand Up @@ -249,6 +244,12 @@ def __init__(
self.system = system
self.output_config = output_config

self._default_inference_args = {'max_tokens': 1024}
if system is not None:
self._default_inference_args['system'] = system
if output_config is not None:
self._default_inference_args['output_config'] = output_config

super().__init__(
namespace='AnthropicModelHandler',
retry_filter=_retry_on_appropriate_error,
Expand Down Expand Up @@ -276,8 +277,8 @@ def request(
) -> Iterable[PredictionResult]:
"""Sends a prediction request to the Anthropic API.

Handler-level system and output_config are injected into
inference_args before calling the request function. Per-request
Handler-level defaults (max_tokens, system and output_config) are merged
with inference_args before calling the request function. Per-request
values in inference_args take precedence over handler-level values.

Args:
Expand All @@ -289,11 +290,6 @@ def request(
Returns:
An iterable of PredictionResults.
"""
if inference_args is None:
inference_args = {}
if self.system is not None and 'system' not in inference_args:
inference_args['system'] = self.system
if self.output_config is not None and 'output_config' not in inference_args:
inference_args['output_config'] = self.output_config
inference_args = {**self._default_inference_args, **(inference_args or {})}
responses = self.request_fn(self.model_name, batch, model, inference_args)
return utils._convert_to_result(batch, responses, self.model_name)
65 changes: 61 additions & 4 deletions sdks/python/apache_beam/ml/inference/anthropic_inference_test.py
Original file line number Diff line number Diff line change
Expand Up @@ -128,12 +128,16 @@ def test_passes_inference_args(self):
self.assertEqual(call_args.kwargs['max_tokens'], 2048)
self.assertEqual(call_args.kwargs['temperature'], 0.5)

def test_default_max_tokens(self):
def test_does_not_mutate_inference_args(self):
# RunInference reuses the same inference_args dict for every batch.
client = mock.MagicMock()
client.messages.create.return_value = _make_fake_response("ok")
message_from_string(_TEST_MODEL, ['test'], client, {})
call_args = client.messages.create.call_args
self.assertEqual(call_args.kwargs['max_tokens'], 1024)
inference_args = {'max_tokens': 2048, 'temperature': 0.5}
message_from_string(_TEST_MODEL, ['a'], client, inference_args)
message_from_string(_TEST_MODEL, ['b'], client, inference_args)
self.assertEqual(inference_args, {'max_tokens': 2048, 'temperature': 0.5})
for call_args in client.messages.create.call_args_list:
self.assertEqual(call_args.kwargs['max_tokens'], 2048)


class MessageFromConversationTest(unittest.TestCase):
Expand All @@ -150,6 +154,17 @@ def test_sends_conversation(self):
call_args = client.messages.create.call_args
self.assertEqual(call_args.kwargs['messages'], convo)

def test_does_not_mutate_inference_args(self):
client = mock.MagicMock()
client.messages.create.return_value = _make_fake_response("ok")
convo = [{"role": "user", "content": "hi"}]
inference_args = {'max_tokens': 2048}
message_from_conversation(_TEST_MODEL, [convo], client, inference_args)
message_from_conversation(_TEST_MODEL, [convo], client, inference_args)
self.assertEqual(inference_args, {'max_tokens': 2048})
for call_args in client.messages.create.call_args_list:
self.assertEqual(call_args.kwargs['max_tokens'], 2048)


class AnthropicModelHandlerTest(unittest.TestCase):
def test_max_batch_weight_requires_element_size_fn(self):
Expand Down Expand Up @@ -194,6 +209,48 @@ def test_request_returns_prediction_results(self):
self.assertEqual(results[1].example, 'q2')
self.assertEqual(results[1].inference, resp2)

def test_request_applies_default_max_tokens(self):
handler = AnthropicModelHandler(
model_name=_TEST_MODEL, request_fn=message_from_string, api_key='fake')
mock_client = mock.MagicMock()
mock_client.messages.create.return_value = _make_fake_response("ok")

handler.request(['test'], mock_client, {'temperature': 0.5})
handler.request(['test'], mock_client, None)

for call_args in mock_client.messages.create.call_args_list:
self.assertEqual(call_args.kwargs['max_tokens'], 1024)

def test_request_max_tokens_override(self):
handler = AnthropicModelHandler(
model_name=_TEST_MODEL, request_fn=message_from_string, api_key='fake')
mock_client = mock.MagicMock()
mock_client.messages.create.return_value = _make_fake_response("ok")

handler.request(['test'], mock_client, {'max_tokens': 2048})

call_args = mock_client.messages.create.call_args
self.assertEqual(call_args.kwargs['max_tokens'], 2048)

def test_request_does_not_mutate_inference_args(self):
# RunInference reuses the same inference_args dict for every batch.
handler = AnthropicModelHandler(
model_name=_TEST_MODEL,
request_fn=message_from_string,
api_key='fake',
system='Be concise.',
output_config={'format': {
'type': 'text'
}})
mock_client = mock.MagicMock()
mock_client.messages.create.return_value = _make_fake_response("ok")
inference_args = {'temperature': 0.5}

handler.request(['a'], mock_client, inference_args)
handler.request(['b'], mock_client, inference_args)

self.assertEqual(inference_args, {'temperature': 0.5})

def test_batch_elements_kwargs(self):
handler = AnthropicModelHandler(
model_name=_TEST_MODEL,
Expand Down