chore: improve prompt to use code frame and inline error (#35032)

pavelfeldman · web-flow · commit 78fa91175614 · 2025-03-04T19:28:03.000-08:00
diff --git a/examples/todomvc/playwright.config.ts b/examples/todomvc/playwright.config.ts
@@ -12,6 +12,8 @@ export default defineConfig({
   /* Maximum time one test can run for. */
   timeout: 15_000,
 
+  captureGitInfo: { commit: true, diff: true },
+
   expect: {
 
     /**
diff --git a/packages/playwright-core/src/utils/isomorphic/stackTrace.ts b/packages/playwright-core/src/utils/isomorphic/stackTrace.ts
@@ -134,6 +134,34 @@ export function splitErrorMessage(message: string): { name: string, message: str
   };
 }
 
+export function parseErrorStack(stack: string, pathSeparator: string, showInternalStackFrames: boolean = false): {
+  message: string;
+  stackLines: string[];
+  location?: StackFrame;
+} {
+  const lines = stack.split('\n');
+  let firstStackLine = lines.findIndex(line => line.startsWith('    at '));
+  if (firstStackLine === -1)
+    firstStackLine = lines.length;
+  const message = lines.slice(0, firstStackLine).join('\n');
+  const stackLines = lines.slice(firstStackLine);
+  let location: StackFrame | undefined;
+  for (const line of stackLines) {
+    const frame = parseStackFrame(line, pathSeparator, showInternalStackFrames);
+    if (!frame || !frame.file)
+      continue;
+    if (belongsToNodeModules(frame.file, pathSeparator))
+      continue;
+    location = { file: frame.file, column: frame.column || 0, line: frame.line || 0 };
+    break;
+  }
+  return { message, stackLines, location };
+}
+
+function belongsToNodeModules(file: string, pathSeparator: string) {
+  return file.includes(`${pathSeparator}node_modules${pathSeparator}`);
+}
+
 const re = new RegExp('^' +
   // Sometimes we strip out the '    at' because it's noisy
   '(?:\\s*at )?' +
diff --git a/packages/playwright/src/DEPS.list b/packages/playwright/src/DEPS.list
@@ -8,7 +8,11 @@ common/
 
 [index.ts]
 @testIsomorphic/**
+./prompt.ts
 ./worker/testTracing.ts
 
 [internalsForTest.ts]
 **
+
+[prompt.ts]
+./transform/babelBundle.ts
diff --git a/packages/playwright/src/index.ts b/packages/playwright/src/index.ts
@@ -22,15 +22,15 @@ import { setBoxedStackPrefixes, asLocator, createGuid, currentZone, debugMode, i
 
 import { currentTestInfo } from './common/globals';
 import { rootTestType } from './common/testType';
-import { stripAnsiEscapes } from './util';
+import { attachErrorPrompts } from './prompt';
 
-import type { MetadataWithCommitInfo } from './isomorphic/types';
 import type { Fixtures, PlaywrightTestArgs, PlaywrightTestOptions, PlaywrightWorkerArgs, PlaywrightWorkerOptions, ScreenshotMode, TestInfo, TestType, VideoMode } from '../types/test';
 import type { ContextReuseMode } from './common/config';
 import type { TestInfoImpl, TestStepInternal } from './worker/testInfo';
 import type { ApiCallData, ClientInstrumentation, ClientInstrumentationListener } from '../../playwright-core/src/client/clientInstrumentation';
 import type { Playwright as PlaywrightImpl } from '../../playwright-core/src/client/playwright';
 import type { APIRequestContext, Browser, BrowserContext, BrowserContextOptions, LaunchOptions, Page, Tracing, Video } from 'playwright-core';
+
 export { expect } from './matchers/expect';
 export const _baseTest: TestType<{}, {}> = rootTestType.test;
 
@@ -619,6 +619,7 @@ class ArtifactsRecorder {
 
   private _screenshotRecorder: SnapshotRecorder;
   private _pageSnapshot: string | undefined;
+  private _sourceCache: Map<string, string> = new Map();
 
   constructor(playwright: PlaywrightImpl, artifactsDir: string, screenshot: ScreenshotOption) {
     this._playwright = playwright;
@@ -701,71 +702,7 @@ class ArtifactsRecorder {
     })));
 
     await this._screenshotRecorder.persistTemporary();
-    await this._attachErrorPrompts();
-  }
-
-  private async _attachErrorPrompts() {
-    if (process.env.PLAYWRIGHT_NO_COPY_PROMPT)
-      return;
-
-    if (this._testInfo.errors.length === 0)
-      return;
-
-    const testSources = await fs.promises.readFile(this._testInfo.file, 'utf-8');
-    for (const [index, error] of this._testInfo.errors.entries()) {
-      if (this._testInfo.attachments.find(a => a.name === `_prompt-${index}`))
-        continue;
-
-      const metadata = this._testInfo.config.metadata as MetadataWithCommitInfo;
-
-      const promptParts = [
-        `My Playwright test failed.`,
-        `Explain why, be concise, respect Playwright best practices.`,
-        '',
-        `Failed test: ${this._testInfo.titlePath.join(' >> ')}`,
-        '',
-        'Error:',
-        '',
-        '```',
-        stripAnsiEscapes(error.stack || error.message || ''),
-        '```',
-      ];
-
-      if (this._pageSnapshot) {
-        promptParts.push(
-            '',
-            'Page snapshot:',
-            '```yaml',
-            this._pageSnapshot,
-            '```',
-        );
-      }
-
-      if (metadata.gitDiff) {
-        promptParts.push(
-            '',
-            'Local changes:',
-            '```diff',
-            metadata.gitDiff,
-            '```',
-        );
-      }
-
-      promptParts.push(
-          '',
-          'Test file:',
-          '```ts',
-          `// ${this._testInfo.file}`,
-          testSources,
-          '```',
-      );
-
-      this._testInfo._attach({
-        name: `_prompt-${index}`,
-        contentType: 'text/markdown',
-        body: Buffer.from(promptParts.join('\n')),
-      }, undefined);
-    }
+    await attachErrorPrompts(this._testInfo, this._sourceCache, this._pageSnapshot);
   }
 
   private async _startTraceChunkOnContextCreation(tracing: Tracing) {
diff --git a/packages/playwright/src/prompt.ts b/packages/playwright/src/prompt.ts
@@ -0,0 +1,124 @@
+/**
+ * Copyright (c) Microsoft Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the 'License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import * as fs from 'fs';
+import * as path from 'path';
+
+import { parseErrorStack } from 'playwright-core/lib/utils';
+
+import { stripAnsiEscapes } from './util';
+import { codeFrameColumns } from './transform/babelBundle';
+
+import type { TestInfo } from '../types/test';
+import type { MetadataWithCommitInfo } from './isomorphic/types';
+import type { TestInfoImpl } from './worker/testInfo';
+
+export async function attachErrorPrompts(testInfo: TestInfo, sourceCache: Map<string, string>, ariaSnapshot: string | undefined) {
+  if (process.env.PLAYWRIGHT_NO_COPY_PROMPT)
+    return;
+
+  for (const [index, error] of testInfo.errors.entries()) {
+    if (testInfo.attachments.find(a => a.name === `_prompt-${index}`))
+      continue;
+
+    const metadata = testInfo.config.metadata as MetadataWithCommitInfo;
+
+    const promptParts = [
+      `# Instructions`,
+      '',
+      `- Following Playwright test failed.`,
+      `- Explain why, be concise, respect Playwright best practices.`,
+      `- Provide a snippet of code with the fix is possible.`,
+      '',
+      `# Test info`,
+      '',
+      `- Name: ${testInfo.titlePath.slice(1).join(' >> ')}`,
+      `- Location: ${testInfo.file}:${testInfo.line}:${testInfo.column}`,
+      '',
+      '# Error details',
+      '',
+      '```',
+      stripAnsiEscapes(error.stack || error.message || ''),
+      '```',
+    ];
+
+    if (ariaSnapshot) {
+      promptParts.push(
+          '',
+          '# Page snapshot',
+          '',
+          '```yaml',
+          ariaSnapshot,
+          '```',
+      );
+    }
+
+    const parsedError = error.stack ? parseErrorStack(error.stack, path.sep) : undefined;
+    const inlineMessage = stripAnsiEscapes(parsedError?.message || error.message || '').split('\n')[0];
+    const location = parsedError?.location || { file: testInfo.file, line: testInfo.line, column: testInfo.column };
+    const source = await loadSource(location.file, sourceCache);
+    const codeFrame = codeFrameColumns(
+        source,
+        {
+          start: {
+            line: location.line,
+            column: location.column
+          },
+        },
+        {
+          highlightCode: false,
+          linesAbove: 100,
+          linesBelow: 100,
+          message: inlineMessage || undefined,
+        }
+    );
+    promptParts.push(
+        '',
+        '# Test source',
+        '',
+        '```ts',
+        codeFrame,
+        '```',
+    );
+
+    if (metadata.gitDiff) {
+      promptParts.push(
+          '',
+          '# Local changes',
+          '',
+          '```diff',
+          metadata.gitDiff,
+          '```',
+      );
+    }
+
+    (testInfo as TestInfoImpl)._attach({
+      name: `_prompt-${index}`,
+      contentType: 'text/markdown',
+      body: Buffer.from(promptParts.join('\n')),
+    }, undefined);
+  }
+}
+
+async function loadSource(file: string, sourceCache: Map<string, string>) {
+  let source = sourceCache.get(file);
+  if (!source) {
+    // A mild race is Ok here.
+    source = await fs.promises.readFile(file, 'utf8');
+    sourceCache.set(file, source);
+  }
+  return source;
+}
diff --git a/packages/playwright/src/reporters/base.ts b/packages/playwright/src/reporters/base.ts
@@ -16,8 +16,7 @@
 
 import path from 'path';
 
-import { getPackageManagerExecCommand } from 'playwright-core/lib/utils';
-import { parseStackFrame } from 'playwright-core/lib/utils';
+import { getPackageManagerExecCommand, parseErrorStack } from 'playwright-core/lib/utils';
 import { ms as milliseconds } from 'playwright-core/lib/utilsBundle';
 import { colors as realColors, noColors } from 'playwright-core/lib/utils';
 
@@ -540,23 +539,7 @@ export function prepareErrorStack(stack: string): {
   stackLines: string[];
   location?: Location;
 } {
-  const lines = stack.split('\n');
-  let firstStackLine = lines.findIndex(line => line.startsWith('    at '));
-  if (firstStackLine === -1)
-    firstStackLine = lines.length;
-  const message = lines.slice(0, firstStackLine).join('\n');
-  const stackLines = lines.slice(firstStackLine);
-  let location: Location | undefined;
-  for (const line of stackLines) {
-    const frame = parseStackFrame(line, path.sep, !!process.env.PWDEBUGIMPL);
-    if (!frame || !frame.file)
-      continue;
-    if (belongsToNodeModules(frame.file))
-      continue;
-    location = { file: frame.file, column: frame.column || 0, line: frame.line || 0 };
-    break;
-  }
-  return { message, stackLines, location };
+  return parseErrorStack(stack, path.sep, !!process.env.PWDEBUGIMPL);
 }
 
 function characterWidth(c: string) {
@@ -611,10 +594,6 @@ export function fitToWidth(line: string, width: number, prefix?: string): string
   return taken.reverse().join('');
 }
 
-function belongsToNodeModules(file: string) {
-  return file.includes(`${path.sep}node_modules${path.sep}`);
-}
-
 function resolveFromEnv(name: string): string | undefined {
   const value = process.env[name];
   if (value)
diff --git a/tests/playwright-test/ui-mode-llm.spec.ts b/tests/playwright-test/ui-mode-llm.spec.ts
@@ -18,7 +18,7 @@ import { test, expect, retries } from './ui-mode-fixtures';
 
 test.describe.configure({ mode: 'parallel', retries });
 
-test('openai', async ({ runUITest, server }) => {
+test.fixme('openai', async ({ runUITest, server }) => {
   server.setRoute('/v1/chat/completions', async (req, res) => {
     res.setHeader('Access-Control-Allow-Origin', '*');
     res.setHeader('Access-Control-Allow-Headers', '*');
@@ -59,7 +59,7 @@ test('openai', async ({ runUITest, server }) => {
   `);
 });
 
-test('anthropic', async ({ runUITest, server }) => {
+test.fixme('anthropic', async ({ runUITest, server }) => {
   server.setRoute('/v1/messages', async (req, res) => {
     res.setHeader('Access-Control-Allow-Origin', '*');
     res.setHeader('Access-Control-Allow-Headers', '*');
diff --git a/tests/playwright-test/ui-mode-trace.spec.ts b/tests/playwright-test/ui-mode-trace.spec.ts
@@ -519,9 +519,11 @@ test('fails', async ({ page }) => {
   const prompt = await page.evaluate(() => navigator.clipboard.readText());
   expect(prompt, 'contains error').toContain('expect(received).toBe(expected)');
   expect(prompt.replaceAll('\r\n', '\n'), 'contains test sources').toContain(`
-test('fails', async ({ page }) => {
-  await page.setContent('<button>Submit</button>');
-  expect(1).toBe(2);
-});
+  1 | import { test, expect } from '@playwright/test';
+  2 | test('fails', async ({ page }) => {
+  3 |   await page.setContent('<button>Submit</button>');
+> 4 |   expect(1).toBe(2);
+    |             ^ Error: expect(received).toBe(expected) // Object.is equality
+  5 | });
     `.trim());
 });
