plugins/agent/model/src/native/chat_completions.rs

/*
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing,
 * software distributed under the License is distributed on an
 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
 * KIND, either express or implied.  See the License for the
 * specific language governing permissions and limitations
 * under the License.
 */

use crate::prompt::{ContentPart, Message, ToolOutput};
use maka_agent_facts::language::Language;

/// Chat has text-only tool messages. Keep the entire result group together and
/// carry media in the following user message, never between parallel results.
pub(super) fn project(messages: Vec<Message>, language: Language) -> Vec<Message> {
    let mut projected = Vec::with_capacity(messages.len());
    let mut images = Vec::new();
    for mut message in messages {
        if let Message::Tool { content, .. } = &mut message {
            for result in content {
                let ToolOutput::Content(parts) = &mut result.output else {
                    continue;
                };
                let mut labelled = false;
                for part in parts {
                    if matches!(part, ContentPart::File { media_type, .. } if media_type.starts_with("image/") || media_type.starts_with("audio/"))
                    {
                        let audio = matches!(part, ContentPart::File { media_type, .. } if media_type.starts_with("audio/"));
                        let kind = match (language, audio) {
                            (Language::En, true) => "Audio",
                            (Language::En, false) => "Image",
                            (Language::ZhCn, true) => "音频",
                            (Language::ZhCn, false) => "图像",
                            (Language::ZhTw, true) => "音訊",
                            (Language::ZhTw, false) => "影像",
                        };
                        if !labelled {
                            let name = &result.tool_name;
                            let id = &result.tool_call_id;
                            images.push(ContentPart::text(match language {
                                Language::En => format!("{kind} from tool {name} ({id}):"),
                                Language::ZhCn => format!("来自工具 {name}（{id}）的{kind}："),
                                Language::ZhTw => format!("來自工具 {name}（{id}）的{kind}："),
                            }));
                            labelled = true;
                        }
                        images.push(std::mem::replace(
                            part,
                            ContentPart::text(match language {
                                Language::En => format!("{kind} supplied below."),
                                Language::ZhCn => format!("{kind}见下方。"),
                                Language::ZhTw => format!("{kind}見下方。"),
                            }),
                        ));
                    }
                }
            }
        } else if !images.is_empty() {
            projected.push(Message::User {
                content: std::mem::take(&mut images),
                provider_options: None,
            });
        }
        projected.push(message);
    }
    if !images.is_empty() {
        projected.push(Message::User {
            content: images,
            provider_options: None,
        });
    }
    projected
}

#[cfg(test)]
mod tests {
    use super::*;
    use crate::prompt::FileData;

    #[test]
    fn tool_media_projection_uses_captured_language_and_keeps_payload() {
        let media = ContentPart::File {
            data: FileData::Data("aW1hZ2U=".into()),
            media_type: "image/png".into(),
            provider_options: None,
        };
        for (language, expected) in [
            (Language::ZhCn, "来自工具 read（call-原文）的图像："),
            (Language::ZhTw, "來自工具 read（call-原文）的影像："),
        ] {
            let messages = project(
                vec![Message::tool(
                    "call-原文",
                    "read",
                    ToolOutput::Content(vec![ContentPart::text("Tool result 原文"), media.clone()]),
                )],
                language,
            );
            let Message::User { content, .. } = &messages[1] else {
                panic!("media user message")
            };
            assert_eq!(content, &[ContentPart::text(expected), media.clone()]);
            let Message::Tool { content, .. } = &messages[0] else {
                panic!("settled tool result")
            };
            let ToolOutput::Content(parts) = &content[0].output else {
                panic!("tool content")
            };
            assert_eq!(parts[0], ContentPart::text("Tool result 原文"));
            assert_eq!(content[0].tool_call_id, "call-原文");
        }
    }
}