platform/computer-use/src/protocol.rs

/*
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing,
 * software distributed under the License is distributed on an
 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
 * KIND, either express or implied.  See the License for the
 * specific language governing permissions and limitations
 * under the License.
 */

use serde::{Deserialize, Serialize};

#[derive(Debug, Serialize, Deserialize)]
#[serde(
    tag = "kind",
    content = "id",
    rename_all = "snake_case",
    deny_unknown_fields
)]
pub enum Handle {
    App(String),
    Tab(String),
}

#[derive(Debug, Serialize, Deserialize)]
#[serde(tag = "method", rename_all = "camelCase", deny_unknown_fields)]
pub enum Command {
    Documentation,
    ConfigureCursor {
        options: crate::cursor::Options,
    },
    CursorState {
        options: InventoryOptions,
    },
    GetState {
        options: InventoryOptions,
    },
    ListApps {
        options: InventoryOptions,
    },
    ListWindows {
        options: InventoryOptions,
    },
    GetApp {
        target: AppReference,
    },
    LaunchApp {
        app: String,
    },
    Observe {
        handle: Handle,
        kind: ObservationKind,
        options: ObservationOptions,
    },
    Action {
        handle: Handle,
        action: Action,
    },
    ListBrowsers {
        options: InventoryOptions,
    },
    ListTabs {
        options: InventoryOptions,
    },
    GetBrowser {
        options: BrowserOptions,
    },
    GetTab {
        reference: TabReference,
        options: InventoryOptions,
    },
    CreateBrowserTab {
        #[serde(rename = "browserId")]
        browser_id: String,
        url: String,
        options: CreateTabOptions,
    },
}
#[derive(Debug, Default, Serialize, Deserialize)]
#[serde(deny_unknown_fields)]
pub struct InventoryOptions {
    pub emit: Option<bool>,
    pub browser: Option<String>,
}
#[derive(Debug, Default, Serialize, Deserialize)]
#[serde(deny_unknown_fields, rename_all = "camelCase")]
pub struct ObservationOptions {
    pub emit: Option<bool>,
    #[serde(default)]
    pub disable_diffing: bool,
}
#[derive(Debug, Clone, Copy, Serialize, Deserialize)]
#[serde(rename_all = "snake_case")]
pub enum ObservationKind {
    Ax,
    Screenshot,
    Both,
}
#[derive(Debug, Serialize, Deserialize)]
#[serde(untagged)]
pub enum AppReference {
    Name(String),
    Window {
        #[serde(rename = "windowId")]
        window_id: u64,
    },
}
#[derive(Debug, Serialize, Deserialize)]
#[serde(untagged)]
pub enum TabReference {
    Id(String),
    Url { url: String },
    Mention { mention: String },
}
#[derive(Debug, Serialize, Deserialize)]
#[serde(deny_unknown_fields, rename_all = "camelCase")]
pub struct BrowserOptions {
    pub id: Option<String>,
    pub url: Option<String>,
    pub extension_instance_id: Option<String>,
}
#[derive(Debug, Serialize, Deserialize)]
#[serde(deny_unknown_fields, rename_all = "camelCase")]
pub struct CreateTabOptions {
    pub visible: Option<bool>,
    pub session_name: Option<String>,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
#[serde(untagged)]
pub enum Position {
    Element(u64),
    Point([f64; 2]),
}
#[derive(Debug, Default, Serialize, Deserialize)]
#[serde(deny_unknown_fields, rename_all = "camelCase")]
pub struct ClickOptions {
    pub mouse_button: Option<MouseButton>,
    pub click_count: Option<u32>,
}
#[derive(Debug, Clone, Copy, Serialize, Deserialize)]
#[serde(rename_all = "lowercase")]
pub enum MouseButton {
    #[serde(alias = "l")]
    Left,
    #[serde(alias = "r")]
    Right,
    #[serde(alias = "m")]
    Middle,
}
impl MouseButton {
    pub fn name(self) -> &'static str {
        match self {
            Self::Left => "left",
            Self::Right => "right",
            Self::Middle => "middle",
        }
    }
}
#[derive(Debug, Serialize, Deserialize)]
#[serde(rename_all = "lowercase")]
pub enum Direction {
    #[serde(alias = "u")]
    Up,
    #[serde(alias = "d")]
    Down,
    #[serde(alias = "l")]
    Left,
    #[serde(alias = "r")]
    Right,
}
impl Direction {
    pub fn name(&self) -> &'static str {
        match self {
            Self::Up => "up",
            Self::Down => "down",
            Self::Left => "left",
            Self::Right => "right",
        }
    }
}
#[derive(Debug, Serialize, Deserialize)]
#[serde(untagged)]
pub enum Distance {
    Pages(u64),
    Pixels { pixels: u64 },
}
#[derive(Debug, Default, Serialize, Deserialize)]
#[serde(deny_unknown_fields)]
pub struct PasteOptions {
    pub format: Option<PasteFormat>,
}
#[derive(Debug, Serialize, Deserialize)]
#[serde(rename_all = "lowercase")]
pub enum PasteFormat {
    Text,
    Md,
    Html,
}
#[derive(Debug, Default, Serialize, Deserialize)]
#[serde(deny_unknown_fields, rename_all = "camelCase")]
pub struct SelectOptions {
    pub prefix: Option<String>,
    pub suffix: Option<String>,
    pub selection_type: Option<SelectionType>,
}
#[derive(Debug, Serialize, Deserialize)]
#[serde(rename_all = "snake_case")]
pub enum SelectionType {
    Text,
    CursorBefore,
    CursorAfter,
}
#[derive(Debug, Serialize, Deserialize)]
#[serde(tag = "kind", rename_all = "camelCase", deny_unknown_fields)]
pub enum Action {
    MoveCursor {
        target: Position,
    },
    Click {
        target: Position,
        options: ClickOptions,
    },
    Drag {
        from: [f64; 2],
        to: [f64; 2],
    },
    Scroll {
        target: Position,
        direction: Direction,
        distance: Option<Distance>,
    },
    SetValue {
        index: u64,
        value: String,
    },
    SelectText {
        index: u64,
        text: String,
        options: SelectOptions,
    },
    Secondary {
        index: u64,
        action: String,
    },
    TypeText {
        index: Option<u64>,
        text: String,
    },
    Paste {
        index: Option<u64>,
        text: String,
        options: PasteOptions,
    },
    PressKey {
        index: Option<u64>,
        key: String,
    },
    Navigate {
        url: String,
    },
    Back,
    Forward,
    Reload,
    Close,
}