Skip to content

Commit 2cc12da

Browse files
authored
feat(mcp): provide midscene mcp server (#562)
1 parent 7596da2 commit 2cc12da

23 files changed

+2864
-133
lines changed

biome.json

+1
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@
88
"static/**",
99
"node_modules/**",
1010
"**/midscene_run",
11+
"**/extension_output",
1112
".nx",
1213
"**/dist",
1314
"test-data/**",

package.json

+3-3
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,7 @@
1313
"test:ai:all": "npm run e2e && npm run e2e:cache && npm run e2e:report && npm run test:ai && npm run e2e:visualizer",
1414
"prepare": "pnpm run build && simple-git-hooks",
1515
"check-dependency-version": "check-dependency-version-consistency .",
16-
"lint": "npx biome check . --diagnostic-level=warn --no-errors-on-unmatched --fix",
16+
"lint": "npx biome check . --diagnostic-level=info --no-errors-on-unmatched --fix",
1717
"format:ci": "pretty-quick --since HEAD~1",
1818
"format": "pretty-quick --staged",
1919
"commit": "cz",
@@ -23,9 +23,9 @@
2323
"pre-commit": "npx nano-staged"
2424
},
2525
"nano-staged": {
26-
"*.{md,mdx,json,css,less,scss}": "npx biome check . --diagnostic-level=warn --no-errors-on-unmatched --fix",
26+
"*.{md,mdx,json,css,less,scss}": "npx biome check . --diagnostic-level=info --no-errors-on-unmatched --fix --verbose",
2727
"*.{js,jsx,ts,tsx,mjs,cjs,json}": [
28-
"npx biome check . --diagnostic-level=warn --no-errors-on-unmatched --fix"
28+
"npx biome check . --diagnostic-level=info --no-errors-on-unmatched --fix --verbose"
2929
],
3030
"package.json": "pnpm run check-dependency-version"
3131
},

packages/android-playground/.gitignore

+1
Original file line numberDiff line numberDiff line change
@@ -2,3 +2,4 @@
22
# Midscene.js dump files
33
midscene_run/report
44
midscene_run/tmp
5+
static/

packages/core/src/env.ts

+1-1
Original file line numberDiff line numberDiff line change
@@ -41,7 +41,7 @@ export const ANTHROPIC_API_KEY = 'ANTHROPIC_API_KEY';
4141
// @deprecated
4242
export const OPENAI_USE_AZURE = 'OPENAI_USE_AZURE';
4343

44-
const allConfigFromEnv = () => {
44+
export const allConfigFromEnv = () => {
4545
return {
4646
[MIDSCENE_OPENAI_INIT_CONFIG_JSON]:
4747
process.env[MIDSCENE_OPENAI_INIT_CONFIG_JSON] || undefined,

packages/mcp/README.md

+23
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,23 @@
1+
# Midscene MCP
2+
3+
## Inspect the MCP server
4+
5+
```bash
6+
# before run this command, you need to build the library first
7+
pnpm run inspect
8+
```
9+
10+
> [!NOTE]
11+
12+
Starting multiple inspect pages may cause the /message sse link error to occur in MTP
13+
14+
## TODO
15+
16+
- [ ] Support launching in Puppeteer mode
17+
- [ ] Provide comprehensive usage documentation
18+
- [ ] Provide examples
19+
- [ ] Optimize automated tests
20+
- [ ] Test usability/effectiveness
21+
- Tools
22+
- [ ] Support getting tab list, allowing LLM to decide which tab to use
23+
- [ ] Test effectiveness of controlling Android

packages/mcp/package.json

+39
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,39 @@
1+
{
2+
"name": "@midscene/mcp",
3+
"version": "0.14.3",
4+
"type": "module",
5+
"exports": {
6+
".": {
7+
"types": "./dist/index.d.ts",
8+
"import": "./dist/index.js",
9+
"require": "./dist/index.cjs"
10+
}
11+
},
12+
"bin": {
13+
"mcp-server-midscene": "dist/index.cjs"
14+
},
15+
"main": "./dist/index.cjs",
16+
"module": "./dist/index.js",
17+
"types": "./dist/index.d.ts",
18+
"files": ["dist"],
19+
"scripts": {
20+
"build": "rslib build",
21+
"dev": "rslib build --watch",
22+
"test": "vitest run",
23+
"inspect": "node scripts/inspect.mjs",
24+
"inspect2": "mcp-inspector node ./dist/test2.cjs"
25+
},
26+
"devDependencies": {
27+
"@modelcontextprotocol/inspector": "0.9.0",
28+
"@rslib/core": "^0.6.2",
29+
"@types/node": "^18.0.0",
30+
"typescript": "^5.8.2",
31+
"vitest": "3.0.5",
32+
"dotenv": "16.4.5"
33+
},
34+
"dependencies": {
35+
"@midscene/web": "workspace:*",
36+
"@modelcontextprotocol/sdk": "1.9.0",
37+
"puppeteer": "24.2.0"
38+
}
39+
}

packages/mcp/rslib.config.ts

+24
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,24 @@
1+
import { defineConfig } from '@rslib/core';
2+
import { version } from './package.json';
3+
4+
export default defineConfig({
5+
source: {
6+
define: {
7+
__VERSION__: `'${version}'`,
8+
},
9+
entry: {
10+
index: './src/index.ts',
11+
},
12+
},
13+
lib: [
14+
{
15+
format: 'esm',
16+
syntax: 'es2021',
17+
dts: true,
18+
},
19+
{
20+
format: 'cjs',
21+
syntax: 'es2021',
22+
},
23+
],
24+
});

packages/mcp/scripts/inspect.mjs

+78
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,78 @@
1+
import { spawn } from 'node:child_process';
2+
import path from 'node:path';
3+
import { fileURLToPath } from 'node:url';
4+
import {
5+
AgentOverChromeBridge,
6+
allConfigFromEnv,
7+
overrideAIConfig,
8+
} from '@midscene/web/bridge-mode';
9+
import dotenv from 'dotenv';
10+
11+
// Get the directory name in ES module scope
12+
const __filename = fileURLToPath(import.meta.url);
13+
const __dirname = path.dirname(__filename);
14+
15+
// Construct the path to the .env file (two levels up from scripts directory)
16+
const envPath = path.resolve(__dirname, '..', '..', '..', '.env');
17+
18+
console.log(`Attempting to load environment variables from: ${envPath}`);
19+
20+
// Load environment variables from the specified path
21+
const configResult = dotenv.config({
22+
path: envPath,
23+
});
24+
25+
if (configResult.error) {
26+
console.warn(
27+
`Warning: Could not load .env file from ${envPath}. Proceeding without it.`,
28+
configResult.error,
29+
);
30+
} else {
31+
console.log(`.env file loaded successfully from ${envPath}`);
32+
}
33+
34+
// Prepare the command and arguments
35+
const command = 'npx';
36+
const keys = Object.keys(allConfigFromEnv());
37+
const envOverrides = {};
38+
for (const key of keys) {
39+
const value = process.env[key];
40+
if (value !== undefined) {
41+
envOverrides[key] = value;
42+
}
43+
}
44+
console.log(envOverrides);
45+
const args = [
46+
'mcp-inspector',
47+
'node',
48+
path.resolve(__dirname, '..', 'dist', 'index.cjs'), // Use resolved path for robustness
49+
...Object.entries(envOverrides).map(([key, value]) => `-e ${key}=${value}`),
50+
];
51+
52+
console.log(`Executing command: ${command} ${args.join(' ')}`);
53+
54+
// Spawn the child process
55+
const child = spawn(command, args, {
56+
stdio: 'inherit', // Inherit stdin, stdout, stderr from the parent process
57+
shell: process.platform === 'win32', // Use shell on Windows for npx compatibility
58+
});
59+
60+
// Handle errors during spawning (e.g., command not found)
61+
child.on('error', (error) => {
62+
console.error(`Failed to start subprocess: ${error.message}`);
63+
});
64+
65+
// Handle process exit
66+
child.on('close', (code) => {
67+
console.log(`Subprocess exited with code ${code}`);
68+
process.exit(code !== null && code !== undefined ? code : 1);
69+
});
70+
71+
// Handle signals to gracefully shut down the child process
72+
const handleSignal = (signal) => {
73+
console.log(`Received ${signal}. Forwarding to subprocess.`);
74+
child.kill(signal);
75+
};
76+
77+
process.on('SIGINT', handleSignal);
78+
process.on('SIGTERM', handleSignal);

packages/mcp/src/index.ts

+104
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,104 @@
1+
#!/usr/bin/env node
2+
3+
import { Server } from '@modelcontextprotocol/sdk/server/index.js';
4+
import { StdioServerTransport } from '@modelcontextprotocol/sdk/server/stdio.js';
5+
import {
6+
CallToolRequestSchema,
7+
ListResourcesRequestSchema,
8+
ListToolsRequestSchema,
9+
ReadResourceRequestSchema,
10+
} from '@modelcontextprotocol/sdk/types.js';
11+
import { MidsceneManager } from './midscene.js';
12+
import { TOOLS } from './tools.js';
13+
14+
declare const __VERSION__: string;
15+
16+
const server = new Server(
17+
{
18+
name: '@midscene/mcp',
19+
version: __VERSION__,
20+
description:
21+
'Midscene MCP Server: Control the browser using natural language commands for navigation, clicking, input, hovering, and achieving goals. Also supports screenshots and JavaScript execution.',
22+
},
23+
{
24+
capabilities: {
25+
resources: {},
26+
tools: {},
27+
},
28+
},
29+
);
30+
31+
const midsceneManager = new MidsceneManager(server);
32+
33+
server.setRequestHandler(ListResourcesRequestSchema, async () => ({
34+
resources: [
35+
{
36+
uri: 'console://logs',
37+
mimeType: 'text/plain',
38+
name: 'Browser console logs',
39+
},
40+
...midsceneManager.listScreenshotNames().map((name) => ({
41+
uri: `screenshot://${name}`,
42+
mimeType: 'image/png',
43+
name: `Screenshot: ${name}`,
44+
})),
45+
],
46+
}));
47+
48+
server.setRequestHandler(ReadResourceRequestSchema, async (request) => {
49+
const uri = request.params.uri.toString();
50+
51+
if (uri === 'console://logs') {
52+
return {
53+
contents: [
54+
{
55+
uri,
56+
mimeType: 'text/plain',
57+
text: midsceneManager.getConsoleLogs(),
58+
},
59+
],
60+
};
61+
}
62+
63+
if (uri.startsWith('screenshot://')) {
64+
const name = uri.split('://')[1];
65+
const screenshot = midsceneManager.getScreenshot(name);
66+
if (screenshot) {
67+
return {
68+
contents: [
69+
{
70+
uri,
71+
mimeType: 'image/png',
72+
blob: screenshot,
73+
},
74+
],
75+
};
76+
}
77+
}
78+
79+
throw new Error(`Resource not found: ${uri}`);
80+
});
81+
82+
server.setRequestHandler(ListToolsRequestSchema, async () => ({
83+
tools: TOOLS,
84+
}));
85+
86+
server.setRequestHandler(CallToolRequestSchema, async (request) =>
87+
midsceneManager.handleToolCall(
88+
request.params.name,
89+
request.params.arguments ?? {},
90+
),
91+
);
92+
93+
async function runServer() {
94+
const transport = new StdioServerTransport();
95+
await server.connect(transport);
96+
}
97+
98+
runServer().catch(console.error);
99+
100+
process.stdin.on('close', () => {
101+
console.error('Midscene MCP Server closing, cleaning up browser...');
102+
midsceneManager.closeBrowser().catch(console.error);
103+
server.close();
104+
});

0 commit comments

Comments
 (0)