=================================== FAILURES ===================================
__________________ test_chat_completions[200-True-True-False] __________________
is_async = False, is_raw = True, is_stream = True, status_code = 200
respx_mock = <respx.router.MockRouter object at 0x7fc85cad3eb0>
in_memory_span_exporter = <opentelemetry.sdk.trace.export.in_memory_span_exporter.InMemorySpanExporter object at 0x7fc85054dd00>
completion_usage = {'completion_tokens': 588, 'prompt_tokens': 813, 'total_tokens': 1401}
model_name = '0.8219540423197268'
chat_completion_mock_stream = ([b'data: {"choices": [{"delta": {"role": "assistant"}, "index": 0}]}\n\n', b'data: {"choices": [{"delta": {"tool_call...', 'type': 'function'}]}, {'content': '{"location": "San Francisco, CA", "unit": "fahrenheit"}', 'role': 'assistant'}])
@pytest.mark.parametrize("is_async", [False, True])
@pytest.mark.parametrize("is_raw", [False, True])
@pytest.mark.parametrize("is_stream", [False, True])
@pytest.mark.parametrize("status_code", [200, 400])
def test_chat_completions(
is_async: bool,
is_raw: bool,
is_stream: bool,
status_code: int,
respx_mock: MockRouter,
in_memory_span_exporter: InMemorySpanExporter,
completion_usage: Dict[str, Any],
model_name: str,
chat_completion_mock_stream: Tuple[List[bytes], List[Dict[str, Any]]],
) -> None:
input_messages: List[Dict[str, Any]] = get_messages()
output_messages: List[Dict[str, Any]] = (
chat_completion_mock_stream[1] if is_stream else get_messages()
)
invocation_parameters = {
"stream": is_stream,
"model": randstr(),
"temperature": random.random(),
"n": len(output_messages),
}
url = "https://api.openai.com/v1/chat/completions"
respx_kwargs: Dict[str, Any] = {
**(
{"stream": MockAsyncByteStream(chat_completion_mock_stream[0])}
if is_stream
else {
"json": {
"choices": [
{"index": i, "message": message, "finish_reason": "stop"}
for i, message in enumerate(output_messages)
],
"model": model_name,
"usage": completion_usage,
}
}
),
}
respx_mock.post(url).mock(return_value=Response(status_code=status_code, **respx_kwargs))
create_kwargs = {"messages": input_messages, **invocation_parameters}
openai = import_module("openai")
completions = (
openai.AsyncOpenAI(api_key="sk-").chat.completions
if is_async
else openai.OpenAI(api_key="sk-").chat.completions
)
create = completions.with_raw_response.create if is_raw else completions.create
with suppress(openai.BadRequestError):
if is_async:
async def task() -> None:
response = await create(**create_kwargs)
response = response.parse() if is_raw else response
if is_stream:
async for _ in response:
pass
asyncio.run(task())
else:
response = create(**create_kwargs)
response = response.parse() if is_raw else response
if is_stream:
for _ in response:
pass
spans = in_memory_span_exporter.get_finished_spans()
assert len(spans) == 2 # first span should be from the httpx instrumentor
span: ReadableSpan = spans[1]
if status_code == 200:
assert span.status.is_ok
elif status_code == 400:
assert not span.status.is_ok and not span.status.is_unset
assert len(span.events) == 1
event = span.events[0]
assert event.name == "exception"
attributes = dict(cast(Mapping[str, AttributeValue], span.attributes))
assert attributes.pop(OPENINFERENCE_SPAN_KIND, None) == OpenInferenceSpanKindValues.LLM.value
assert isinstance(attributes.pop(INPUT_VALUE, None), str)
assert (
OpenInferenceMimeTypeValues(attributes.pop(INPUT_MIME_TYPE, None))
== OpenInferenceMimeTypeValues.JSON
)
assert (
json.loads(cast(str, attributes.pop(LLM_INVOCATION_PARAMETERS, None)))
== invocation_parameters
)
for prefix, messages in (
(LLM_INPUT_MESSAGES, input_messages),
*(((LLM_OUTPUT_MESSAGES, output_messages),) if status_code == 200 else ()),
):
for i, message in enumerate(messages):
> assert attributes.pop(message_role(prefix, i), None) == message.get("role")
E assert None == 'assistant'
E + where None = <built-in method pop of dict object at 0x7fc8597fcb80>('llm.output_messages.0.message.role', None)
E + where <built-in method pop of dict object at 0x7fc8597fcb80> = {'output.value': '<openai.Stream object at 0x7fc8597b1580>'}.pop
E + and 'llm.output_messages.0.message.role' = message_role('llm.output_messages', 0)
E + and 'assistant' = <built-in method get of dict object at 0x7fc859791700>('role')
E + where <built-in method get of dict object at 0x7fc859791700> = {'content': '{"location": "Boston, MA", "unit": "fahrenheit"}', 'role': 'assistant', 'tool_calls': [{'function': {'arg...", "unit": "fahrenheit"}', 'name': 'get_current_weather'}, 'id': 'call_6QTP4mLSYYzZwt3ZWj77vfZf', 'type': 'function'}]}.get
tests/openinference/instrumentation/openai/test_instrumentor.py:146: AssertionError
__________________ test_chat_completions[200-True-True-True] ___________________
is_async = True, is_raw = True, is_stream = True, status_code = 200
respx_mock = <respx.router.MockRouter object at 0x7fc85cad3eb0>
in_memory_span_exporter = <opentelemetry.sdk.trace.export.in_memory_span_exporter.InMemorySpanExporter object at 0x7fc85054dd00>
completion_usage = {'completion_tokens': 971, 'prompt_tokens': 332, 'total_tokens': 1303}
model_name = '0.[150](https://github.com/Arize-ai/openinference/actions/runs/7789499135/job/21241260367#step:5:151)84917392450192'
chat_completion_mock_stream = ([b'data: {"choices": [{"delta": {"role": "assistant"}, "index": 0}]}\n\n', b'data: {"choices": [{"delta": {"tool_call...', 'type': 'function'}]}, {'content': '{"location": "San Francisco, CA", "unit": "fahrenheit"}', 'role': 'assistant'}])
@pytest.mark.parametrize("is_async", [False, True])
@pytest.mark.parametrize("is_raw", [False, True])
@pytest.mark.parametrize("is_stream", [False, True])
@pytest.mark.parametrize("status_code", [200, 400])
def test_chat_completions(
is_async: bool,
is_raw: bool,
is_stream: bool,
status_code: int,
respx_mock: MockRouter,
in_memory_span_exporter: InMemorySpanExporter,
completion_usage: Dict[str, Any],
model_name: str,
chat_completion_mock_stream: Tuple[List[bytes], List[Dict[str, Any]]],
) -> None:
input_messages: List[Dict[str, Any]] = get_messages()
output_messages: List[Dict[str, Any]] = (
chat_completion_mock_stream[1] if is_stream else get_messages()
)
invocation_parameters = {
"stream": is_stream,
"model": randstr(),
"temperature": random.random(),
"n": len(output_messages),
}
url = "https://api.openai.com/v1/chat/completions"
respx_kwargs: Dict[str, Any] = {
**(
{"stream": MockAsyncByteStream(chat_completion_mock_stream[0])}
if is_stream
else {
"json": {
"choices": [
{"index": i, "message": message, "finish_reason": "stop"}
for i, message in enumerate(output_messages)
],
"model": model_name,
"usage": completion_usage,
}
}
),
}
respx_mock.post(url).mock(return_value=Response(status_code=status_code, **respx_kwargs))
create_kwargs = {"messages": input_messages, **invocation_parameters}
openai = import_module("openai")
completions = (
openai.AsyncOpenAI(api_key="sk-").chat.completions
if is_async
else openai.OpenAI(api_key="sk-").chat.completions
)
create = completions.with_raw_response.create if is_raw else completions.create
with suppress(openai.BadRequestError):
if is_async:
async def task() -> None:
response = await create(**create_kwargs)
response = response.parse() if is_raw else response
if is_stream:
async for _ in response:
pass
asyncio.run(task())
else:
response = create(**create_kwargs)
response = response.parse() if is_raw else response
if is_stream:
for _ in response:
pass
spans = in_memory_span_exporter.get_finished_spans()
assert len(spans) == 2 # first span should be from the httpx instrumentor
span: ReadableSpan = spans[1]
if status_code == [200](https://github.com/Arize-ai/openinference/actions/runs/7789499135/job/21241260367#step:5:201):
assert span.status.is_ok
elif status_code == 400:
assert not span.status.is_ok and not span.status.is_unset
assert len(span.events) == 1
event = span.events[0]
assert event.name == "exception"
attributes = dict(cast(Mapping[str, AttributeValue], span.attributes))
assert attributes.pop(OPENINFERENCE_SPAN_KIND, None) == OpenInferenceSpanKindValues.LLM.value
assert isinstance(attributes.pop(INPUT_VALUE, None), str)
assert (
OpenInferenceMimeTypeValues(attributes.pop(INPUT_MIME_TYPE, None))
== OpenInferenceMimeTypeValues.JSON
)
assert (
json.loads(cast(str, attributes.pop(LLM_INVOCATION_PARAMETERS, None)))
== invocation_parameters
)
for prefix, messages in (
(LLM_INPUT_MESSAGES, input_messages),
*(((LLM_OUTPUT_MESSAGES, output_messages),) if status_code == 200 else ()),
):
for i, message in enumerate(messages):
> assert attributes.pop(message_role(prefix, i), None) == message.get("role")
E assert None == 'assistant'
E + where None = <built-in method pop of dict object at 0x7fc85979[238](https://github.com/Arize-ai/openinference/actions/runs/7789499135/job/21241260367#step:5:239)0>('llm.output_messages.0.message.role', None)
E + where <built-in method pop of dict object at 0x7fc859792380> = {'output.value': '<openai.AsyncStream object at 0x7fc8597d59a0>'}.pop
E + and 'llm.output_messages.0.message.role' = message_role('llm.output_messages', 0)
E + and 'assistant' = <built-in method get of dict object at 0x7fc8596b4580>('role')
E + where <built-in method get of dict object at 0x7fc8596b4580> = {'content': '{"location": "Boston, MA", "unit": "fahrenheit"}', 'role': 'assistant', 'tool_calls': [{'function': {'arg...", "unit": "fahrenheit"}', 'name': 'get_current_weather'}, 'id': 'call_6QTP4mLSYYzZwt3ZWj77vfZf', 'type': 'function'}]}.get
tests/openinference/instrumentation/openai/test_instrumentor.py:146: AssertionError
![github-actions[bot] avatar](https://avatars.githubusercontent.com/in/15368?v=4 "github-actions[bot]")
React
Typescript
Django
Laravel
Facebook
Microsoft
Google
Alibaba
D3
Tencent