Automating Real Desktop Tasks with Anthropic Claude Computer Use API

Claude Model
    ▲   │
    │   ▼ (Sends Action: mouse_move, left_click, type, key, screenshot)
Container OS / Virtual Display (X11 / Xvfb)
    │   ▲
    └───┴ (Takes 1024x768 PNG Screenshot & Returns Coordinates)

# Dockerfile
FROM ubuntu:22.04

ENV DEBIAN_FRONTEND=noninteractive
ENV DISPLAY=:1

# Install X11 virtual frame buffer, window manager, and automation tools
RUN apt-get update && apt-get install -y \
    xvfb \
    xdotool \
    scrot \
    fluxbox \
    python3 \
    python3-pip \
    chromium-browser \
    curl \
    && rm -rf /var/lib/apt/lists/*

WORKDIR /app
COPY requirements.txt .
RUN pip3 install --no-cache-dir -r requirements.txt

COPY . .

# Start Xvfb virtual screen at 1024x768 with 24-bit color
CMD ["bash", "-c", "Xvfb :1 -screen 0 1024x768x24 & fluxbox & python3 agent.py"]

# computer_agent.py
import os
import base64
import subprocess
import time
from anthropic import Anthropic

client = Anthropic(api_key=os.environ.get("ANTHROPIC_API_KEY"))

DISPLAY_WIDTH = 1024
DISPLAY_HEIGHT = 768

def take_screenshot() -> str:
    """Takes a screenshot using scrot and returns base64 string."""
    screenshot_path = "/tmp/screenshot.png"
    subprocess.run(["scrot", "-z", screenshot_path], check=True)
    with open(screenshot_path, "rb") as f:
        return base64.b64encode(f.read()).decode("utf-8")

def execute_computer_action(action: str, coordinate=None, text=None):
    """Translates Claude's tool action into OS commands using xdotool."""
    if action == "mouse_move" and coordinate:
        subprocess.run(["xdotool", "mousemove", str(coordinate[0]), str(coordinate[1])], check=True)
    elif action == "left_click":
        subprocess.run(["xdotool", "click", "1"], check=True)
    elif action == "type" and text:
        subprocess.run(["xdotool", "type", "--delay", "12", text], check=True)
    elif action == "key" and text:
        subprocess.run(["xdotool", "key", text], check=True)
    elif action == "screenshot":
        pass  # Just capture on next loop

    # Allow UI animation to settle
    time.sleep(0.5)

def run_agent_loop(instruction: str, max_steps: int = 25):
    messages = [
        {
            "role": "user",
            "content": [
                {"type": "text", "text": instruction},
                {
                    "type": "image",
                    "source": {
                        "type": "base64",
                        "media_type": "image/png",
                        "data": take_screenshot()
                    }
                }
            ]
        }
    ]

    tools = [
        {
            "type": "computer_20241022",
            "name": "computer",
            "display_width_px": DISPLAY_WIDTH,
            "display_height_px": DISPLAY_HEIGHT,
            "display_number": 1
        }
    ]

    for step in range(max_steps):
        response = client.beta.messages.create(
            model="claude-sonnet-5",
            max_tokens=4096,
            tools=tools,
            messages=messages,
            betas=["computer-use-2024-10-22"]
        )

        tool_calls = [c for c in response.content if c.type == "tool_use"]
        
        if not tool_calls:
            print("Task completed successfully!")
            break

        for tool_call in tool_calls:
            action = tool_call.input.get("action")
            coord = tool_call.input.get("coordinate")
            text = tool_call.input.get("text")
            
            execute_computer_action(action, coordinate=coord, text=text)

            # Append tool result with fresh screenshot
            messages.append({"role": "assistant", "content": response.content})
            messages.append({
                "role": "user",
                "content": [
                    {
                        "type": "tool_result",
                        "tool_use_id": tool_call.id,
                        "content": [
                            {
                                "type": "image",
                                "source": {
                                    "type": "base64",
                                    "media_type": "image/png",
                                    "data": take_screenshot()
                                }
                            }
                        ]
                    }
                ]
            })