feat(reporter): report TestStep#attachments (#34037)

docs/src/test-reporter-api/class-teststep.md

@@ -50,6 +50,16 @@ Start time of this particular test step.
 
 List of steps inside this step.
 
+## property: TestStep.attachments
+* since: v1.50
+- type: <[Array]<[Object]>>
+  - `name` <[string]> Attachment name.
+  - `contentType` <[string]> Content type of this attachment to properly present in the report, for example `'application/json'` or `'image/png'`.
+  - `path` ?<[string]> Optional path on the filesystem to the attached file.
+  - `body` ?<[Buffer]> Optional attachment body used instead of a file.
+
+The list of files or buffers attached in the step execution through [`method: TestInfo.attach`].
+
 ## property: TestStep.title
 * since: v1.10
 - type: <[string]>

packages/html-reporter/src/links.tsx

@@ -68,11 +68,12 @@ export const ProjectLink: React.FunctionComponent<{
 
 export const AttachmentLink: React.FunctionComponent<{
   attachment: TestAttachment,
+  result: TestResult,
   href?: string,
   linkName?: string,
   openInNewTab?: boolean,
-}> = ({ attachment, href, linkName, openInNewTab }) => {
-  const isAnchored = useIsAnchored('attachment-' + attachment.name);
+}> = ({ attachment, result, href, linkName, openInNewTab }) => {
+  const isAnchored = useIsAnchored('attachment-' + result.attachments.indexOf(attachment));
   return <TreeItem title={<span>
     {attachment.contentType === kMissingContentType ? icons.warning() : icons.attachment()}
     {attachment.path && <a href={href || attachment.path} download={downloadFileNameForAttachment(attachment)}>{linkName || attachment.name}</a>}

packages/html-reporter/src/testCaseView.spec.tsx

@@ -37,8 +37,10 @@ const result: TestResult = {
       duration: 10,
       location: { file: 'test.spec.ts', line: 82, column: 0 },
       steps: [],
+      attachments: [],
       count: 1,
     }],
+    attachments: [],
   }],
   attachments: [],
   status: 'passed',
@@ -139,6 +141,7 @@ const resultWithAttachment: TestResult = {
     location: { file: 'test.spec.ts', line: 62, column: 0 },
     count: 1,
     steps: [],
+    attachments: [1],
   }],
   attachments: [{
     name: 'first attachment',

packages/html-reporter/src/testFileView.tsx

@@ -75,7 +75,7 @@ function imageDiffBadge(test: TestCaseSummary): JSX.Element | undefined {
   for (const result of test.results) {
     for (const attachment of result.attachments) {
       if (attachment.contentType.startsWith('image/') && !!attachment.name.match(/-(expected|actual|diff)/))
-        return <Link href={testResultHref({ test, result, anchor: `attachment-${attachment.name}` })} title='View images' className='test-file-badge'>{image()}</Link>;
+        return <Link href={testResultHref({ test, result, anchor: `attachment-${result.attachments.indexOf(attachment)}` })} title='View images' className='test-file-badge'>{image()}</Link>;
     }
   }
 }

packages/html-reporter/src/testResultView.tsx

@@ -32,7 +32,7 @@ interface ImageDiffWithAnchors extends ImageDiff {
   anchors: string[];
 }
 
-function groupImageDiffs(screenshots: Set<TestAttachment>): ImageDiffWithAnchors[] {
+function groupImageDiffs(screenshots: Set<TestAttachment>, result: TestResult): ImageDiffWithAnchors[] {
   const snapshotNameToImageDiff = new Map<string, ImageDiffWithAnchors>();
   for (const attachment of screenshots) {
     const match = attachment.name.match(/^(.*)-(expected|actual|diff|previous)(\.[^.]+)?$/);
@@ -45,7 +45,7 @@ function groupImageDiffs(screenshots: Set<TestAttachment>): ImageDiffWithAnchors
       imageDiff = { name: snapshotName, anchors: [`attachment-${name}`] };
       snapshotNameToImageDiff.set(snapshotName, imageDiff);
     }
-    imageDiff.anchors.push(`attachment-${attachment.name}`);
+    imageDiff.anchors.push(`attachment-${result.attachments.indexOf(attachment)}`);
     if (category === 'actual')
       imageDiff.actual = { attachment };
     if (category === 'expected')
@@ -72,15 +72,15 @@ export const TestResultView: React.FC<{
   result: TestResult,
 }> = ({ test, result }) => {
   const { screenshots, videos, traces, otherAttachments, diffs, errors, otherAttachmentAnchors, screenshotAnchors } = React.useMemo(() => {
-    const attachments = result?.attachments || [];
+    const attachments = result.attachments;
     const screenshots = new Set(attachments.filter(a => a.contentType.startsWith('image/')));
-    const screenshotAnchors = [...screenshots].map(a => `attachment-${a.name}`);
+    const screenshotAnchors = [...screenshots].map(a => `attachment-${attachments.indexOf(a)}`);
     const videos = attachments.filter(a => a.contentType.startsWith('video/'));
     const traces = attachments.filter(a => a.name === 'trace');
     const otherAttachments = new Set<TestAttachment>(attachments);
     [...screenshots, ...videos, ...traces].forEach(a => otherAttachments.delete(a));
-    const otherAttachmentAnchors = [...otherAttachments].map(a => `attachment-${a.name}`);
-    const diffs = groupImageDiffs(screenshots);
+    const otherAttachmentAnchors = [...otherAttachments].map(a => `attachment-${attachments.indexOf(a)}`);
+    const diffs = groupImageDiffs(screenshots, result);
     const errors = classifyErrors(result.errors, diffs);
     return { screenshots: [...screenshots], videos, traces, otherAttachments, diffs, errors, otherAttachmentAnchors, screenshotAnchors };
   }, [result]);
@@ -107,11 +107,11 @@ export const TestResultView: React.FC<{
 
     {!!screenshots.length && <AutoChip header='Screenshots' revealOnAnchorId={screenshotAnchors}>
       {screenshots.map((a, i) => {
-        return <Anchor key={`screenshot-${i}`} id={`attachment-${a.name}`}>
+        return <Anchor key={`screenshot-${i}`} id={`attachment-${result.attachments.indexOf(a)}`}>
           <a href={a.path}>
             <img className='screenshot' src={a.path} />
           </a>
-          <AttachmentLink attachment={a}></AttachmentLink>
+          <AttachmentLink attachment={a} result={result}></AttachmentLink>
         </Anchor>;
       })}
     </AutoChip>}
@@ -121,7 +121,7 @@ export const TestResultView: React.FC<{
         <a href={generateTraceUrl(traces)}>
           <img className='screenshot' src={traceImage} style={{ width: 192, height: 117, marginLeft: 20 }} />
         </a>
-        {traces.map((a, i) => <AttachmentLink key={`trace-${i}`} attachment={a} linkName={traces.length === 1 ? 'trace' : `trace-${i + 1}`}></AttachmentLink>)}
+        {traces.map((a, i) => <AttachmentLink key={`trace-${i}`} attachment={a} result={result} linkName={traces.length === 1 ? 'trace' : `trace-${i + 1}`}></AttachmentLink>)}
       </div>}
     </AutoChip></Anchor>}
 
@@ -130,14 +130,14 @@ export const TestResultView: React.FC<{
         <video controls>
           <source src={a.path} type={a.contentType}/>
         </video>
-        <AttachmentLink attachment={a}></AttachmentLink>
+        <AttachmentLink attachment={a} result={result}></AttachmentLink>
       </div>)}
     </AutoChip></Anchor>}
 
     {!!otherAttachments.size && <AutoChip header='Attachments' revealOnAnchorId={otherAttachmentAnchors}>
       {[...otherAttachments].map((a, i) =>
-        <Anchor key={`attachment-link-${i}`} id={`attachment-${a.name}`}>
-          <AttachmentLink attachment={a} openInNewTab={a.contentType.startsWith('text/html')} />
+        <Anchor key={`attachment-link-${i}`} id={`attachment-${result.attachments.indexOf(a)}`}>
+          <AttachmentLink attachment={a} result={result} openInNewTab={a.contentType.startsWith('text/html')} />
         </Anchor>
       )}
     </AutoChip>}
@@ -174,10 +174,9 @@ const StepTreeItem: React.FC<{
   step: TestStep;
   depth: number,
 }> = ({ test, step, result, depth }) => {
-  const attachmentName = step.title.match(/^attach "(.*)"$/)?.[1];
   return <TreeItem title={<span aria-label={step.title}>
     <span style={{ float: 'right' }}>{msToString(step.duration)}</span>
-    {attachmentName && <a style={{ float: 'right' }} title='link to attachment' href={testResultHref({ test, result, anchor: `attachment-${attachmentName}` })} onClick={evt => { evt.stopPropagation(); }}>{icons.attachment()}</a>}
+    {step.attachments.length > 0 && <a style={{ float: 'right' }} title='link to attachment' href={testResultHref({ test, result, anchor: `attachment-${step.attachments[0]}` })} onClick={evt => { evt.stopPropagation(); }}>{icons.attachment()}</a>}
     {statusIcon(step.error || step.duration === -1 ? 'failed' : 'passed')}
     <span>{step.title}</span>
     {step.count > 1 && <> ✕ <span className='test-result-counter'>{step.count}</span></>}

packages/html-reporter/src/types.d.ts

@@ -108,5 +108,6 @@ export type TestStep = {
   snippet?: string;
   error?: string;
   steps: TestStep[];
+  attachments: number[];
   count: number;
 };

packages/playwright/src/common/ipc.ts

@@ -75,6 +75,7 @@ export type AttachmentPayload = {
   path?: string;
   body?: string;
   contentType: string;
+  stepId?: string;
 };
 
 export type TestInfoErrorImpl = TestInfoError & {

packages/playwright/src/isomorphic/teleReceiver.ts

@@ -108,6 +108,7 @@ export type JsonTestStepEnd = {
   id: string;
   duration: number;
   error?: reporterTypes.TestError;
+  attachments?: number[]; // index of JsonTestResultEnd.attachments
 };
 
 export type JsonFullResult = {
@@ -249,7 +250,7 @@ export class TeleReporterReceiver {
     const parentStep = payload.parentStepId ? result._stepMap.get(payload.parentStepId) : undefined;
 
     const location = this._absoluteLocation(payload.location);
-    const step = new TeleTestStep(payload, parentStep, location);
+    const step = new TeleTestStep(payload, parentStep, location, result);
     if (parentStep)
       parentStep.steps.push(step);
     else
@@ -262,6 +263,7 @@ export class TeleReporterReceiver {
     const test = this._tests.get(testId)!;
     const result = test.results.find(r => r._id === resultId)!;
     const step = result._stepMap.get(payload.id)!;
+    step._endPayload = payload;
     step.duration = payload.duration;
     step.error = payload.error;
     this._reporter.onStepEnd?.(test, result, step);
@@ -512,15 +514,20 @@ class TeleTestStep implements reporterTypes.TestStep {
   parent: reporterTypes.TestStep | undefined;
   duration: number = -1;
   steps: reporterTypes.TestStep[] = [];
+  error: reporterTypes.TestError | undefined;
+
+  private _result: TeleTestResult;
+  _endPayload?: JsonTestStepEnd;
 
   private _startTime: number = 0;
 
-  constructor(payload: JsonTestStepStart, parentStep: reporterTypes.TestStep | undefined, location: reporterTypes.Location | undefined) {
+  constructor(payload: JsonTestStepStart, parentStep: reporterTypes.TestStep | undefined, location: reporterTypes.Location | undefined, result: TeleTestResult) {
     this.title = payload.title;
     this.category = payload.category;
     this.location = location;
     this.parent = parentStep;
     this._startTime = payload.startTime;
+    this._result = result;
   }
 
   titlePath() {
@@ -535,6 +542,10 @@ class TeleTestStep implements reporterTypes.TestStep {
   set startTime(value: Date) {
     this._startTime = +value;
   }
+
+  get attachments() {
+    return this._endPayload?.attachments?.map(index => this._result.attachments[index]) ?? [];
+  }
 }
 
 export class TeleTestResult implements reporterTypes.TestResult {
@@ -550,7 +561,7 @@ export class TeleTestResult implements reporterTypes.TestResult {
   errors: reporterTypes.TestResult['errors'] = [];
   error: reporterTypes.TestResult['error'];
 
-  _stepMap: Map<string, reporterTypes.TestStep> = new Map();
+  _stepMap = new Map<string, TeleTestStep>();
   _id: string;
 
   private _startTime: number = 0;

packages/playwright/src/reporters/html.ts

@@ -505,7 +505,7 @@ class HtmlBuilder {
       duration: result.duration,
       startTime: result.startTime.toISOString(),
       retry: result.retry,
-      steps: dedupeSteps(result.steps).map(s => this._createTestStep(s)),
+      steps: dedupeSteps(result.steps).map(s => this._createTestStep(s, result)),
       errors: formatResultFailure(test, result, '', true).map(error => error.message),
       status: result.status,
       attachments: this._serializeAttachments([
@@ -515,20 +515,26 @@ class HtmlBuilder {
     };
   }
 
-  private _createTestStep(dedupedStep: DedupedStep): TestStep {
+  private _createTestStep(dedupedStep: DedupedStep, result: api.TestResult): TestStep {
     const { step, duration, count } = dedupedStep;
-    const result: TestStep = {
+    const testStep: TestStep = {
       title: step.title,
       startTime: step.startTime.toISOString(),
       duration,
-      steps: dedupeSteps(step.steps).map(s => this._createTestStep(s)),
+      steps: dedupeSteps(step.steps).map(s => this._createTestStep(s, result)),
+      attachments: step.attachments.map(s => {
+        const index = result.attachments.indexOf(s);
+        if (index === -1)
+          throw new Error('Unexpected, attachment not found');
+        return index;
+      }),
       location: this._relativeLocation(step.location),
       error: step.error?.message,
       count
     };
     if (step.location)
-      this._stepsInFile.set(step.location.file, result);
-    return result;
+      this._stepsInFile.set(step.location.file, testStep);
+    return testStep;
   }
 
   private _relativeLocation(location: api.Location | undefined): api.Location | undefined {

packages/playwright/src/reporters/teleEmitter.ts

@@ -100,7 +100,7 @@ export class TeleReporterEmitter implements ReporterV2 {
       params: {
         testId: test.id,
         resultId: (result as any)[this._idSymbol],
-        step: this._serializeStepEnd(step)
+        step: this._serializeStepEnd(step, result)
       }
     });
   }
@@ -251,11 +251,12 @@ export class TeleReporterEmitter implements ReporterV2 {
     };
   }
 
-  private _serializeStepEnd(step: reporterTypes.TestStep): teleReceiver.JsonTestStepEnd {
+  private _serializeStepEnd(step: reporterTypes.TestStep, result: reporterTypes.TestResult): teleReceiver.JsonTestStepEnd {
     return {
       id: (step as any)[this._idSymbol],
       duration: step.duration,
       error: step.error,
+      attachments: step.attachments.map(a => result.attachments.indexOf(a)),
     };
   }
 

packages/playwright/src/runner/dispatcher.ts

@@ -320,6 +320,7 @@ class JobDispatcher {
       startTime: new Date(params.wallTime),
       duration: -1,
       steps: [],
+      attachments: [],
       location: params.location,
     };
     steps.set(params.stepId, step);
@@ -361,6 +362,13 @@ class JobDispatcher {
       body: params.body !== undefined ? Buffer.from(params.body, 'base64') : undefined
     };
     data.result.attachments.push(attachment);
+    if (params.stepId) {
+      const step = data.steps.get(params.stepId);
+      if (step)
+        step.attachments.push(attachment);
+      else
+        this._reporter.onStdErr?.('Internal error: step id not found: ' + params.stepId);
+    }
   }
 
   private _failTestWithErrors(test: TestCase, errors: TestError[]) {