AI 电脑与浏览器操作完全指南
Codex × Claude 终极实战手册

import { Stagehand } from "@browserbasehq/stagehand";
import { z } from "zod";

async function runCheckoutAudit() {
  const stagehand = new Stagehand({
    env: "LOCAL", // 本地无头模式运行，也可无缝切至 "BROWSERBASE" 云端集群
    verbose: 1,
    debugDom: true,
    modelName: "claude-3-7-sonnet-latest",
    modelClientOptions: { apiKey: process.env.ANTHROPIC_API_KEY }
  });

  await stagehand.init();
  const page = stagehand.page;

  try {
    // 1. 打开目标商城测试页面
    await page.goto("https://store.example.com/products/nexus-pro");

    // 2. 意图驱动操作：添加购物车
    await stagehand.act({ action: "点击'加入购物车'按钮" });

    // 3. 页面跳转与操作
    await stagehand.act({ action: "点击右上角购物车图标并前往结算页" });

    // 4. 利用 Zod 提取结算页全部账单明细
    const orderSummary = await stagehand.extract({
      instruction: "提取订单金额小计、运费、税额、折扣以及最终应付总额",
      schema: z.object({
        subtotal: z.number().describe("商品小计金额"),
        shipping: z.number().describe("运费"),
        tax: z.number().describe("计算所得税额"),
        finalTotal: z.number().describe("最终应付总额"),
        appliedCoupon: z.string().nullable().describe("当前生效的优惠券代码"),
      }),
    });

    console.log("✅ 账单明细提取成功:", orderSummary);

    // 5. 校验业务数学逻辑
    const expected = orderSummary.subtotal + orderSummary.shipping + orderSummary.tax;
    if (Math.abs(orderSummary.finalTotal - expected) > 0.01) {
      throw new Error(`结算金额逻辑异常！预期: ${expected}, 实际: ${orderSummary.finalTotal}`);
    }
    console.log("🎉 结算链路 E2E 数值校验 100% 通过！");
  } finally {
    await stagehand.close();
  }
}

runCheckoutAudit();

import asyncio
import json
from browser_use import Agent, Tools, ActionResult
from langchain_anthropic import ChatAnthropic

# 1. 实例化工具箱
tools = Tools()

# 2. 注册自定义外部动作 (大模型会根据 description 自主决定何时调用)
@tools.action(description="将调研得出的结构化商品比对数据写入本地 json 文件")
async def save_research_data(data_json_str: str, filename: str = "competitor_intel.json"):
    try:
        parsed = json.loads(data_json_str)
        with open(filename, "w", encoding="utf-8") as f:
            json.dump(parsed, f, ensure_ascii=False, indent=2)
        return ActionResult(extracted_content=f"已成功将 {len(parsed)} 条数据写入 {filename}")
    except Exception as e:
        return ActionResult(error=f"写入文件失败: {str(e)}")

async def run_autonomous_research():
    # 使用 Claude 3.7 Sonnet 作为高推理决策大脑
    llm = ChatAnthropic(model_name="claude-3-7-sonnet-20250219", timeout=90)

    task_prompt = (
        "打开 https://github.com/trending，找到今天榜单排名前 3 的开源项目，"
        "分别点击进入每个项目仓库的主页，提取项目名称、Star 数量与核心架构简述，"
        "最后调用 save_research_data 工具将整理好的数据保存为 trending_repos.json。"
    )

    agent = Agent(
        task=task_prompt,
        llm=llm,
        tools=tools, # 注入自定义工具
        use_vision=True, # 开启多模态视觉比对辅助
        max_actions_per_step=3,
        max_steps=25 # 防御性步数熔断
    )

    history = await agent.run()
    print("🎯 自动化任务执行完成！")
    print(history.final_result())

if __name__ == "__main__":
    asyncio.run(run_autonomous_research())

import agentql
from playwright.sync_api import sync_playwright

with sync_playwright() as p:
    browser = p.chromium.launch(headless=False)
    page = agentql.wrap(browser.new_page())
    page.goto("https://news.ycombinator.com")

    # 声明式语义提取：哪怕目标网站重构 DOM，AgentQL 依然通过语义精准捕获
    QUERY = """
    {
        top_stories[] {
            rank
            title
            score
            comments_count
            story_link
        }
    }
    """
    response = page.query_data(QUERY)
    for story in response["top_stories"][:5]:
        print(f"[{story['rank']}] {story['title']} ({story['score']} 点赞)")
    browser.close()

import asyncio
from crawl4ai import AsyncWebCrawler

async def extract_clean_content():
    async with AsyncWebCrawler(verbose=True) as crawler:
        result = await crawler.arun(
            url="https://docs.anthropic.com/en/docs/build-with-claude/computer-use",
            word_count_threshold=10, # 过滤碎片杂讯
            bypass_cache=True
        )
        # 直接输出去除了所有无用 CSS、广告、内联样式的纯粹 Markdown！
        clean_markdown = result.markdown
        print("提纯后 Markdown 字符数:", len(clean_markdown))
        with open("claude_computer_use_doc.md", "w", encoding="utf-8") as f:
            f.write(clean_markdown)

if __name__ == "__main__":
    asyncio.run(extract_clean_content())

import { defineConfig, devices } from '@playwright/test';

export default defineConfig({
  testDir: './tests/e2e',
  timeout: 30000,
  retries: 1,
  // 关键：在 Codex / 终端中运行严禁使用默认冗长模式，采用 line 极简模式防止爆 Token
  reporter: [['line'], ['html', { open: 'never' }]],
  use: {
    baseURL: 'http://localhost:3000',
    trace: 'on-first-retry',
    screenshot: 'only-on-failure',
    storageState: 'tests/auth_state.json', // 自动复用已登录会话
  },
  projects: [
    { name: 'chromium', use: { ...devices['Desktop Chrome'] } },
  ],
});

{
  "mcpServers": {
    "playwright": {
      "command": "npx",
      "args": ["-y", "@executeautomation/playwright-mcp-server"],
      "env": {
        "HEADLESS": "false",
        "BROWSER": "chromium",
        "DEFAULT_TIMEOUT": "20000"
      }
    }
  }
}

// 错误写法：直接 fill 导致 React 状态无法同步更新
// await page.locator('#title').fill('新标题');

// 生产级绝技：模拟人类键盘组合键 Meta+A ➔ Backspace ➔ 拟人化键入
const titleInput = page.locator('#title');
await titleInput.focus();
// Mac 模拟 Command+A，Linux/Windows 模拟 Control+A
await page.keyboard.press(process.platform === 'darwin' ? 'Meta+A' : 'Control+A');
await page.keyboard.press('Backspace');
// 以 30ms 随机延迟模拟真实人类按键，确保触发每一次 keydown 与 input 事件！
await page.keyboard.type('2026 旗舰级生产环境架构方案', { delay: 35 });

// 监听等待 context 的 page 创建事件与点击动作并发执行
const [popupPage] = await Promise.all([
  page.context().waitForEvent('page'),
  page.getByRole('button', { name: '使用 Google 账号登录' }).click(),
]);

await popupPage.waitForLoadState('domcontentloaded');
console.log("新弹窗 URL:", popupPage.url());
// 在新标签页中完成输入后关闭，焦点无缝切回原主页
await popupPage.getByRole('textbox', { name: 'Email' }).fill('admin@corp.com');
await popupPage.close();

// 1. 静默过滤无用外部打点，极大提升页面加载速度并节省 Token
await page.route('**/{google-analytics,analytics,sentry,mixpanel}/**', route => route.abort());

// 2. 模拟后端返回 500 异常，测试前端错误兜底 UI 是否正确展示
await page.route('**/api/v1/payment/checkout', route => {
  route.fulfill({
    status: 500,
    contentType: 'application/json',
    body: JSON.stringify({ error: 'SYSTEM_MAINTENANCE', message: '支付系统维护中' })
  });
});

import path from 'path';

// 触发文件选择框并注入本地文件
const fileChooserPromise = page.waitForEvent('filechooser');
await page.getByRole('button', { name: '上传营业执照' }).click();
const fileChooser = await fileChooserPromise;
await fileChooser.setFiles(path.resolve('./fixtures/test_license.pdf'));

const downloadPromise = page.waitForEvent('download');
await page.getByRole('button', { name: '导出 CSV 报表' }).click();
const download = await downloadPromise;

const savePath = './downloads/' + download.suggestedFilename();
await download.saveAs(savePath);
console.log(`✅ 报表已成功下载落盘: ${savePath}`);

# 1. 弹出受控窗口，人工完成一次登录
npx playwright codegen --save-storage=auth_state.json https://admin.yourcompany.com

# 2. 生成包含 Session Cookie 与 Token 的 auth_state.json

// 原始 HTML: 3.5MB (包含成千上万行无用 CSS 与 SVG 路径)
// const rawHtml = await page.content(); // 坚决禁止！

// 2026 极致脱水：仅保留具备交互含义的角色、名称与状态 (仅 12KB!)
const ariaYaml = await page.locator('body').ariaSnapshot();

// 输出示例 (直接发给 Claude 或 Codex，模型秒懂且绝不幻觉):
// - banner:
//   - link "首页"
//   - button "切换暗黑模式"
// - main:
//   - heading "结算收银台" [level=1]
//   - textbox "优惠码"
//   - button "应用优惠" [disabled]
//   - button "确认支付 ¥299.00"

import { test, expect } from '@playwright/test';

test.describe('全量发布前核心链路巡检', () => {
  test('管理员仪表盘与结算漏斗回归', async ({ page }) => {
    // 1. 访问主站控制台 (复用 auth_state.json)
    await page.goto('/dashboard');

    // 2. 验证关键数据图表是否加载完毕
    await expect(page.getByTestId('revenue-chart')).toBeVisible({ timeout: 10000 });

    // 3. 拦截任何未捕获的前端 JS 异常
    page.on('pageerror', error => {
      throw new Error(`[CRITICAL FRONTEND ERROR] ${error.message}`);
    });

    // 4. 模拟创建新租户
    await page.getByRole('button', { name: '新建项目' }).click();
    await page.getByLabel('项目名称').fill('Release-Audit-2026');
    await page.getByRole('button', { name: '确认创建' }).click();

    // 5. 校验创建成功状态
    await expect(page.getByText('项目创建成功')).toBeVisible();

    // 6. 保存视觉证据
    await page.screenshot({ path: 'artifacts/release_proof.png', fullPage: true });
  });
});

import { Stagehand } from "@browserbasehq/stagehand";
import { z } from "zod";
import * as fs from "fs";

async function monitorStockAndPricing() {
  const stagehand = new Stagehand({ env: "LOCAL" });
  await stagehand.init();
  const page = stagehand.page;

  await page.goto("https://marketplace.example.com/category/gpu");

  // 触发无限滚动以加载动态内容
  await page.evaluate(() => window.scrollTo(0, document.body.scrollHeight));
  await page.waitForTimeout(2000);

  const catalog = await stagehand.extract({
    instruction: "提取当前展示的所有显卡型号名称、实时标价以及是否有现货库存",
    schema: z.object({
      items: z.array(z.object({
        model: z.string(),
        priceRMB: z.number(),
        inStock: z.boolean(),
      }))
    })
  });

  fs.writeFileSync("gpu_inventory.json", JSON.stringify(catalog, null, 2));
  console.log(`成功监控并归档 ${catalog.items.length} 件商品情报！`);
  await stagehand.close();
}
monitorStockAndPricing();

import { chromium } from 'playwright';

const records = [
  { id: '1001', name: '研发部张工', amount: 4500, category: '云服务器扩容' },
  { id: '1002', name: '运营部李经', amount: 1200, category: '技术沙龙场地' }
];

async function batchFillLegacyERP() {
  // 直接吸附至工程师已登录 ERP 的 Chrome
  const browser = await chromium.connectOverCDP('http://localhost:9222');
  const page = browser.contexts()[0].pages()[0];

  for (const record of records) {
    console.log(`正在填报工单: ${record.id} - ${record.name}`);
    await page.getByRole('button', { name: '新增报销申请' }).click();

    // 动态级联下拉框处理
    await page.getByLabel('报销人姓名').fill(record.name);
    await page.getByLabel('报销金额').fill(record.amount.toString());
    await page.getByLabel('开支类别').selectOption({ label: record.category });

    await page.getByRole('button', { name: '保存并暂存' }).click();
    await page.waitForTimeout(1000); // 规避高频提交防刷机制
  }
  console.log("🎉 全部历史凭证批量录入完成！");
}
batchFillLegacyERP();

FROM node:22-slim

# 安装无头 X11、VNC 服务与 Chrome 依赖
RUN apt-get update && apt-get install -y     xvfb     x11vnc     fluxbox     chromium     chromium-sandbox     novnc     websockify     && rm -rf /var/lib/apt/lists/*

ENV DISPLAY=:99
ENV CHROME_PATH=/usr/bin/chromium

WORKDIR /app
COPY package*.json ./
RUN npm install

COPY . .

# 同时暴露 6080 端口，可通过浏览器随时远程实时监视 AI 点击动作！
EXPOSE 6080

CMD Xvfb :99 -screen 0 1280x800x24 &     fluxbox &     x11vnc -display :99 -nopw -listen localhost -xkb &     websockify --web /usr/share/novnc 6080 localhost:5900 &     npm run start:agent