@@ -247,6 +247,23 @@ def _is_lc_content_block(part: dict) -> bool:
247
247
return "type" in part
248
248
249
249
250
+ def _is_openai_image_block (block : dict ) -> bool :
251
+ """Check if the block contains image data in OpenAI Chat Completions format."""
252
+ if block .get ("type" ) == "image_url" :
253
+ if (
254
+ (set (block .keys ()) <= {"type" , "image_url" , "detail" })
255
+ and (image_url := block .get ("image_url" ))
256
+ and isinstance (image_url , dict )
257
+ ):
258
+ url = image_url .get ("url" )
259
+ if isinstance (url , str ):
260
+ return True
261
+ else :
262
+ return False
263
+
264
+ return False
265
+
266
+
250
267
def _convert_to_parts (
251
268
raw_content : Union [str , Sequence [Union [str , dict ]]],
252
269
) -> List [Part ]:
@@ -334,14 +351,28 @@ def _convert_to_parts(
334
351
return parts
335
352
336
353
337
- def _convert_tool_message_to_part (
354
+ def _convert_tool_message_to_parts (
338
355
message : ToolMessage | FunctionMessage , name : Optional [str ] = None
339
- ) -> Part :
356
+ ) -> list [ Part ] :
340
357
"""Converts a tool or function message to a google part."""
341
358
# Legacy agent stores tool name in message.additional_kwargs instead of message.name
342
359
name = message .name or name or message .additional_kwargs .get ("name" )
343
360
response : Any
344
- if not isinstance (message .content , str ):
361
+ parts : list [Part ] = []
362
+ if isinstance (message .content , list ):
363
+ media_blocks = []
364
+ other_blocks = []
365
+ for block in message .content :
366
+ if isinstance (block , dict ) and (
367
+ is_data_content_block (block ) or _is_openai_image_block (block )
368
+ ):
369
+ media_blocks .append (block )
370
+ else :
371
+ other_blocks .append (block )
372
+ parts .extend (_convert_to_parts (media_blocks ))
373
+ response = other_blocks
374
+
375
+ elif not isinstance (message .content , str ):
345
376
response = message .content
346
377
else :
347
378
try :
@@ -356,7 +387,8 @@ def _convert_tool_message_to_part(
356
387
),
357
388
)
358
389
)
359
- return part
390
+ parts .append (part )
391
+ return parts
360
392
361
393
362
394
def _get_ai_message_tool_messages_parts (
@@ -374,8 +406,10 @@ def _get_ai_message_tool_messages_parts(
374
406
break
375
407
if message .tool_call_id in tool_calls_ids :
376
408
tool_call = tool_calls_ids [message .tool_call_id ]
377
- part = _convert_tool_message_to_part (message , name = tool_call .get ("name" ))
378
- parts .append (part )
409
+ message_parts = _convert_tool_message_to_parts (
410
+ message , name = tool_call .get ("name" )
411
+ )
412
+ parts .extend (message_parts )
379
413
# remove the id from the dict, so that we do not iterate over it again
380
414
tool_calls_ids .pop (message .tool_call_id )
381
415
return parts
@@ -442,7 +476,7 @@ def _parse_chat_history(
442
476
system_instruction = None
443
477
elif isinstance (message , FunctionMessage ):
444
478
role = "user"
445
- parts = [ _convert_tool_message_to_part (message )]
479
+ parts = _convert_tool_message_to_parts (message )
446
480
else :
447
481
raise ValueError (
448
482
f"Unexpected message with type { type (message )} at the position { i } ."
0 commit comments