feat: introduce the new reporter API

docs/.vitepress/config.ts

@@ -320,6 +320,10 @@ export default ({ mode }: { mode: string }) => {
                 text: 'Running Tests',
                 link: '/advanced/guide/tests',
               },
+              {
+                text: 'Test Lifecycle',
+                link: '/advanced/guide/lifecycle',
+              },
               {
                 text: 'Extending Reporters',
                 link: '/advanced/reporters',

docs/advanced/guide/lifecycle.md

@@ -0,0 +1,3 @@
+# Test Lifecycle
+
+<!-- TODO: lifecyle diagram and reporter API -->

packages/vitest/src/node/reporters/reported-tasks.ts

@@ -5,7 +5,7 @@ import type {
   Suite as RunnerTestSuite,
   TaskMeta,
 } from '@vitest/runner'
-import type { TestError } from '@vitest/utils'
+import type { SerializedError, TestError } from '@vitest/utils'
 import type { TestProject } from '../project'
 
 class ReportedTaskImplementation {
@@ -122,12 +122,19 @@ export class TestCase extends ReportedTaskImplementation {
   }
 
   /**
-   * Test results. Will be `undefined` if test is skipped, not finished yet or was just collected.
+   * Test results.
+   * - **pending**: Test was collected, but didn't finish running yet.
+   * - **passed**: Test passed successfully
+   * - **failed**: Test failed to execute
+   * - **skipped**: Test was skipped during collection or dynamically with `ctx.skip()`.
    */
-  public result(): TestResult | undefined {
+  public result(): TestResult {
     const result = this.task.result
     if (!result || result.state === 'run') {
-      return undefined
+      return {
+        state: 'pending',
+        errors: undefined,
+      }
     }
     const state = result.state === 'fail'
       ? 'failed' as const
@@ -298,10 +305,33 @@ class TestCollection {
 
 export type { TestCollection }
 
+export interface TestSuiteStatistics {
+  total: number
+  completed: number
+  passed: number
+  failed: number
+  skipped: number
+  todo: number
+}
+
+function createStatistics() {
+  return {
+    total: 0,
+    completed: 0,
+    passed: 0,
+    failed: 0,
+    skipped: 0,
+    todo: 0,
+  }
+}
+
 abstract class SuiteImplementation extends ReportedTaskImplementation {
   /** @internal */
   declare public readonly task: RunnerTestSuite | RunnerTestFile
 
+  /** @internal */
+  public _statistic: TestSuiteStatistics = createStatistics()
+
   /**
    * Collection of suites and tests that are part of this suite.
    */
@@ -314,18 +344,38 @@ abstract class SuiteImplementation extends ReportedTaskImplementation {
   }
 
   /**
-   * Checks if the suite was skipped during collection.
+   * The number of tests in this suite with a specific state.
    */
-  public skipped(): boolean {
+  public statistics(): TestSuiteStatistics {
+    return { ...this._statistic }
+  }
+
+  /**
+   * Checks the running state of the suite.
+   */
+  public state(): TestSuiteState {
     const mode = this.task.mode
-    return mode === 'skip' || mode === 'todo'
+    const state = this.task.result?.state
+    if (mode === 'skip' || mode === 'todo' || state === 'skip' || state === 'todo') {
+      return 'skipped'
+    }
+    if (state == null || state === 'run' || state === 'only') {
+      return 'pending'
+    }
+    if (state === 'fail') {
+      return 'failed'
+    }
+    if (state === 'pass') {
+      return 'passed'
+    }
+    throw new Error(`Unknown suite state: ${state}`)
   }
 
   /**
    * Errors that happened outside of the test run during collection, like syntax errors.
    */
-  public errors(): TestError[] {
-    return (this.task.result?.errors as TestError[] | undefined) || []
+  public errors(): SerializedError[] {
+    return (this.task.result?.errors as SerializedError[] | undefined) || []
   }
 }
 
@@ -402,8 +452,8 @@ export class TestModule extends SuiteImplementation {
 
   /**
    * This is usually an absolute UNIX file path.
-   * It can be a virtual id if the file is not on the disk.
-   * This value corresponds to Vite's `ModuleGraph` id.
+   * It can be a virtual ID if the file is not on the disk.
+   * This value corresponds to the ID in the Vite's module graph.
    */
   public readonly moduleId: string
 
@@ -420,9 +470,9 @@ export class TestModule extends SuiteImplementation {
   declare public ok: () => boolean
 
   /**
-   * Checks if the module was skipped and didn't run.
+   * Checks the running state of the test file.
    */
-  declare public skipped: () => boolean
+  declare public state: () => TestSuiteState
 
   /**
    * Useful information about the module like duration, memory usage, etc.
@@ -446,6 +496,7 @@ export class TestModule extends SuiteImplementation {
 
 export interface TaskOptions {
   each: boolean | undefined
+  fails: boolean | undefined
   concurrent: boolean | undefined
   shuffle: boolean | undefined
   retry: number | undefined
@@ -454,10 +505,11 @@ export interface TaskOptions {
 }
 
 function buildOptions(
-  task: RunnerTestCase | RunnerTestFile | RunnerTestSuite,
+  task: RunnerTestCase | RunnerTestSuite,
 ): TaskOptions {
   return {
     each: task.each,
+    fails: task.type === 'test' && task.fails,
     concurrent: task.concurrent,
     shuffle: task.shuffle,
     retry: task.retry,
@@ -466,7 +518,24 @@ function buildOptions(
   }
 }
 
-export type TestResult = TestResultPassed | TestResultFailed | TestResultSkipped
+export type TestSuiteState = 'skipped' | 'pending' | 'failed' | 'passed'
+
+export type TestResult =
+  | TestResultPassed
+  | TestResultFailed
+  | TestResultSkipped
+  | TestResultPending
+
+export interface TestResultPending {
+  /**
+   * The test was collected, but didn't finish running yet.
+   */
+  state: 'pending'
+  /**
+   * Pending tests have no errors.
+   */
+  errors: undefined
+}
 
 export interface TestResultPassed {
   /**

packages/vitest/src/node/test-run.ts

@@ -0,0 +1,48 @@
+import type { TaskResultPack } from '@vitest/runner'
+import type { Vitest } from './core'
+import type { TestModule } from './reporters/reported-tasks'
+import type { TestSpecification } from './spec'
+
+export class TestRun {
+  private tests = emptyCounters()
+  private suites = emptyCounters()
+
+  constructor(private vitest: Vitest) {}
+
+  async start(specifications: TestSpecification[]) {
+    this.tests = emptyCounters()
+    this.suites = emptyCounters()
+    this.suites.total = specifications.length
+    await this.vitest.report('onTestRunStart', specifications)
+  }
+
+  enqueued(module: TestModule) {
+    // TODO
+  }
+
+  collected(modules: TestModule[]) {
+    // TODO
+  }
+
+  updated(update: TaskResultPack[]) {
+    // TODO
+  }
+
+  async end() {
+    // TODO
+    await this.vitest.report('onTestRunEnd', [], [], 'passed')
+  }
+}
+
+interface Counter {
+  total: number
+  completed: number
+  passed: number
+  failed: number
+  skipped: number
+  todo: number
+}
+
+function emptyCounters(): Counter {
+  return { completed: 0, passed: 0, failed: 0, skipped: 0, todo: 0, total: 0 }
+}

packages/vitest/src/node/types/reporter.ts

@@ -1,7 +1,10 @@
 import type { File, TaskResultPack } from '@vitest/runner'
+import type { SerializedError } from '@vitest/utils'
 import type { SerializedTestSpecification } from '../../runtime/types/utils'
 import type { Awaitable, UserConsoleLog } from '../../types/general'
 import type { Vitest } from '../core'
+import type { TestModule } from '../reporters/reported-tasks'
+import type { TestSpecification } from '../spec'
 
 export interface Reporter {
   onInit?: (ctx: Vitest) => void
@@ -20,4 +23,12 @@ export interface Reporter {
   onServerRestart?: (reason?: string) => Awaitable<void>
   onUserConsoleLog?: (log: UserConsoleLog) => Awaitable<void>
   onProcessTimeout?: () => Awaitable<void>
+
+  // new API
+  onTestRunStart?: (specifications: TestSpecification[]) => Awaitable<void>
+  onTestRunEnd?: (
+    testModules: TestModule[],
+    errors: SerializedError[],
+    reason: 'passed' | 'interrupted' | 'failed'
+  ) => Awaitable<void>
 }

packages/vitest/src/public/node.ts

@@ -33,8 +33,8 @@ export type { JUnitOptions } from '../node/reporters/junit'
 
 export type {
   ModuleDiagnostic,
-
   TaskOptions,
+
   TestCase,
   TestCollection,
   TestDiagnostic,
@@ -44,6 +44,7 @@ export type {
   TestResultPassed,
   TestResultSkipped,
   TestSuite,
+  TestSuiteStatistics,
 } from '../node/reporters/reported-tasks'
 export { BaseSequencer } from '../node/sequencers/BaseSequencer'