Files
video.js-mirror/.github/workflows/e2e.yml

470 lines
18 KiB
YAML

name: E2E Tests
on:
push:
branches:
- main
pull_request:
types: [opened, labeled, synchronize]
concurrency:
group: e2e-${{ github.ref }}
cancel-in-progress: true
jobs:
# Pull requests run the browser suites when they touch code the suites render; the `test:e2e` label forces them. The
# release pull request only bumps versions and changelogs on top of `main`, whose push already ran every suite.
changes:
name: Changed areas
if: github.event_name == 'pull_request'
runs-on: blacksmith-2vcpu-ubuntu-2404
outputs:
player: ${{ steps.filter.outputs.player }}
skins: ${{ steps.filter.outputs.skins }}
skin_projects: ${{ steps.filter.outputs.skin_projects }}
steps:
- name: Detect e2e-relevant changes
id: filter
env:
GH_TOKEN: ${{ github.token }}
GH_REPO: ${{ github.repository }}
NUMBER: ${{ github.event.pull_request.number }}
HEAD_REF: ${{ github.head_ref }}
run: |
if [ "$HEAD_REF" = 'release-please--branches--main' ]; then
printf 'player=false\nskins=false\nskin_projects=false\n' >> "$GITHUB_OUTPUT"
exit 0
fi
files=$(gh api "repos/$GH_REPO/pulls/$NUMBER/files" --paginate --jq '.[].filename')
flag() {
if grep -Eq "$2" <<<"$files"; then echo "$1=true" >> "$GITHUB_OUTPUT"; else echo "$1=false" >> "$GITHUB_OUTPUT"; fi
}
# The player suite renders the built packages; skin parity compiles the authored skins in the sandbox.
flag player '^(apps/e2e/|apps/sandbox/|packages/|\.github/workflows/e2e\.yml$|pnpm-lock\.yaml$)'
flag skins '^(apps/e2e/(shared|suites/skin-parity)/|apps/sandbox/|packages/(skins|vjsc|html|react)/|\.github/workflows/e2e\.yml$|pnpm-lock\.yaml$)'
# `skin create` projects meet the CLI and its templates, the installation schema, and the registry build.
flag skin_projects '^(packages/(cli|installation|skins|vjsc)/|\.github/workflows/e2e\.yml$|pnpm-lock\.yaml$)'
# Every browser job replays this run's Vite Task cache, so the packages, CDN bundles, generated skins, and sandbox
# setup compile once per run instead of once per job. The build needs no browser, so it runs on the runner itself:
# cold builds inside the Playwright container stall on Blacksmith, and the runner can start from the `packages-`
# cache the CI build saves.
build:
name: Build for browser suites
needs: changes
if: ${{ !cancelled() && (github.event_name == 'push' || needs.changes.outputs.player == 'true' || needs.changes.outputs.skins == 'true' || contains(github.event.pull_request.labels.*.name, 'test:e2e') || contains(github.event.pull_request.labels.*.name, 'record:snapshots')) }}
runs-on: blacksmith-4vcpu-ubuntu-2404
timeout-minutes: 15
steps:
- name: Checkout code
uses: actions/checkout@v5
- name: Setup pnpm
uses: pnpm/action-setup@v6.1.0
- name: Setup Node.js
uses: actions/setup-node@v5
with:
node-version-file: '.nvmrc'
cache: pnpm
- name: Install dependencies
run: pnpm install --frozen-lockfile
- name: Restore Vite Task cache
id: vite-task-cache
uses: actions/cache/restore@v6
with:
path: node_modules/.vite/task-cache
key: vite-task-${{ runner.os }}-${{ runner.arch }}-e2e-${{ github.sha }}-${{ github.run_attempt }}
restore-keys: |
vite-task-${{ runner.os }}-${{ runner.arch }}-e2e-
vite-task-${{ runner.os }}-${{ runner.arch }}-packages-
- name: Build packages and CDN bundles
run: pnpm build:packages && pnpm build:cdn
- name: Prepare the e2e pages, generated skins, and sandbox
run: pnpm exec vp run '@videojs/e2e#prepare:player' && pnpm exec vp run '@videojs/skins#generate' && pnpm exec vp run '@videojs/sandbox#setup'
- name: Save Vite Task cache
uses: actions/cache/save@v6
with:
path: node_modules/.vite/task-cache
key: ${{ steps.vite-task-cache.outputs.cache-primary-key }}
e2e:
name: E2E (${{ matrix.browser }}${{ matrix.shards > 1 && format(' {0}/{1}', matrix.shard, matrix.shards) || '' }})
needs: [changes, build]
if: ${{ !cancelled() && needs.build.result == 'success' && (github.event_name == 'push' || needs.changes.outputs.player == 'true' || contains(github.event.pull_request.labels.*.name, 'test:e2e')) }}
runs-on: blacksmith-4vcpu-ubuntu-2404
container:
image: mcr.microsoft.com/playwright:v1.59.1-noble
# The container's default PID 1 never reaps orphaned helpers such as esbuild services, and `vp run` waits on
# their zombies until the job times out. An init process reaps them.
options: --init
timeout-minutes: 15
strategy:
fail-fast: false
# WebKit takes more than twice Chromium's wall time, so pull requests run Chromium only and main covers both,
# splitting WebKit in two so it finishes with the other jobs.
matrix: ${{ fromJSON(github.event_name == 'push' && '{"include":[{"browser":"chromium","shard":1,"shards":1},{"browser":"webkit","shard":1,"shards":2},{"browser":"webkit","shard":2,"shards":2}]}' || '{"include":[{"browser":"chromium","shard":1,"shards":1}]}') }}
steps:
- name: Checkout code
uses: actions/checkout@v5
- name: Setup pnpm
uses: pnpm/action-setup@v6.1.0
- name: Setup Node.js
uses: actions/setup-node@v5
with:
node-version-file: '.nvmrc'
cache: pnpm
- name: Install dependencies
run: pnpm install --frozen-lockfile
- name: Install zstd
# actions/cache falls back to gzip without zstd, which keys a separate cache from the runner jobs that build it.
run: apt-get update -qq && apt-get install -y -qq --no-install-recommends zstd
- name: Restore Vite Task cache
uses: actions/cache/restore@v6
with:
path: node_modules/.vite/task-cache
key: vite-task-${{ runner.os }}-${{ runner.arch }}-e2e-${{ github.sha }}-${{ github.run_attempt }}
restore-keys: |
vite-task-${{ runner.os }}-${{ runner.arch }}-e2e-
vite-task-${{ runner.os }}-${{ runner.arch }}-packages-
- name: Replay the build
run: pnpm build:packages && pnpm build:cdn && pnpm exec vp run '@videojs/e2e#prepare:player'
- name: Run E2E tests
run: pnpm --dir apps/e2e exec playwright test --config suites/player/playwright.config.ts --project=vite-${{ matrix.browser }} --shard=${{ matrix.shard }}/${{ matrix.shards }}
- name: Upload test report
if: always()
uses: actions/upload-artifact@v4
with:
name: playwright-report-${{ matrix.browser }}${{ matrix.shards > 1 && format('-{0}', matrix.shard) || '' }}
path: apps/e2e/playwright-report/
retention-days: 14
- name: Upload test traces
if: failure()
uses: actions/upload-artifact@v4
with:
name: playwright-traces-${{ matrix.browser }}${{ matrix.shards > 1 && format('-{0}', matrix.shard) || '' }}
path: apps/e2e/test-results/
retention-days: 7
sandbox:
name: Sandbox
needs: [changes, build]
if: ${{ !cancelled() && needs.build.result == 'success' && (github.event_name == 'push' || needs.changes.outputs.player == 'true' || contains(github.event.pull_request.labels.*.name, 'test:e2e')) }}
runs-on: blacksmith-4vcpu-ubuntu-2404
container:
image: mcr.microsoft.com/playwright:v1.59.1-noble
options: --init
timeout-minutes: 15
steps:
- name: Checkout code
uses: actions/checkout@v5
- name: Setup pnpm
uses: pnpm/action-setup@v6.1.0
- name: Setup Node.js
uses: actions/setup-node@v5
with:
node-version-file: '.nvmrc'
cache: pnpm
- name: Install dependencies
run: pnpm install --frozen-lockfile
- name: Install zstd
# actions/cache falls back to gzip without zstd, which keys a separate cache from the runner jobs that build it.
run: apt-get update -qq && apt-get install -y -qq --no-install-recommends zstd
- name: Restore Vite Task cache
uses: actions/cache/restore@v6
with:
path: node_modules/.vite/task-cache
key: vite-task-${{ runner.os }}-${{ runner.arch }}-e2e-${{ github.sha }}-${{ github.run_attempt }}
restore-keys: |
vite-task-${{ runner.os }}-${{ runner.arch }}-e2e-
vite-task-${{ runner.os }}-${{ runner.arch }}-packages-
- name: Run sandbox E2E tests
run: pnpm exec vp run '@videojs/e2e#test:sandbox'
- name: Upload test report
if: always()
uses: actions/upload-artifact@v4
with:
name: playwright-report-sandbox
path: apps/e2e/playwright-report/sandbox/
retention-days: 14
- name: Upload test traces
if: failure()
uses: actions/upload-artifact@v4
with:
name: playwright-traces-sandbox
path: apps/e2e/test-results/sandbox/
retention-days: 7
skin-parity:
name: Skin parity (${{ matrix.name }}${{ matrix.shards > 1 && format(' {0}/{1}', matrix.shard, matrix.shards) || '' }})
needs: [changes, build]
if: ${{ !cancelled() && needs.build.result == 'success' && (github.event_name == 'push' || needs.changes.outputs.skins == 'true' || contains(github.event.pull_request.labels.*.name, 'test:e2e')) }}
runs-on: blacksmith-4vcpu-ubuntu-2404
container:
image: mcr.microsoft.com/playwright:v1.59.1-noble
options: --init
timeout-minutes: 15
strategy:
fail-fast: false
matrix:
# Slices take about the same time: the video preset carries twice the cases of live-video, and the two audio
# presets together about as many as one video slice. Each job costs a runner slot and a minute of setup.
include:
- { name: video, presets: video, shard: 1, shards: 2 }
- { name: video, presets: video, shard: 2, shards: 2 }
- { name: live-video, presets: live-video, shard: 1, shards: 1 }
- { name: audio, presets: 'audio,live-audio', shard: 1, shards: 1 }
steps:
- name: Checkout code
uses: actions/checkout@v5
- name: Setup pnpm
uses: pnpm/action-setup@v6.1.0
- name: Setup Node.js
uses: actions/setup-node@v5
with:
node-version-file: '.nvmrc'
cache: pnpm
- name: Install dependencies
run: pnpm install --frozen-lockfile
- name: Install zstd
# actions/cache falls back to gzip without zstd, which keys a separate cache from the runner jobs that build it.
run: apt-get update -qq && apt-get install -y -qq --no-install-recommends zstd
- name: Restore Vite Task cache
uses: actions/cache/restore@v6
with:
path: node_modules/.vite/task-cache
key: vite-task-${{ runner.os }}-${{ runner.arch }}-e2e-${{ github.sha }}-${{ github.run_attempt }}
restore-keys: |
vite-task-${{ runner.os }}-${{ runner.arch }}-e2e-
vite-task-${{ runner.os }}-${{ runner.arch }}-packages-
- name: Run skin parity E2E tests
run: pnpm exec vp run '@videojs/e2e#test:skin-parity'
env:
VJSC_SKIN_PRESET: ${{ matrix.presets }}
VJSC_SKIN_SHARD: ${{ matrix.shard }}/${{ matrix.shards }}
- name: Upload test report
if: always()
uses: actions/upload-artifact@v4
with:
name: playwright-report-skin-parity-${{ matrix.name }}${{ matrix.shards > 1 && format('-{0}', matrix.shard) || '' }}
path: apps/e2e/playwright-report/skin-parity/
retention-days: 14
- name: Upload test traces
if: failure()
uses: actions/upload-artifact@v4
with:
name: playwright-traces-skin-parity-${{ matrix.name }}${{ matrix.shards > 1 && format('-{0}', matrix.shard) || '' }}
path: apps/e2e/test-results/skin-parity/
retention-days: 7
# Baselines only match when recorded in the image the suites run in. Label a pull request `record:snapshots` to
# record the player visual specs here; download the artifact, copy the `*-snapshots` directories into place, and
# commit them.
snapshots:
name: Record snapshots
needs: [changes, build]
if: ${{ !cancelled() && needs.build.result == 'success' && contains(github.event.pull_request.labels.*.name, 'record:snapshots') }}
runs-on: blacksmith-4vcpu-ubuntu-2404
container:
image: mcr.microsoft.com/playwright:v1.59.1-noble
options: --init
timeout-minutes: 20
steps:
- name: Checkout code
uses: actions/checkout@v5
- name: Setup pnpm
uses: pnpm/action-setup@v6.1.0
- name: Setup Node.js
uses: actions/setup-node@v5
with:
node-version-file: '.nvmrc'
cache: pnpm
- name: Install dependencies
run: pnpm install --frozen-lockfile
- name: Install zstd
# actions/cache falls back to gzip without zstd, which keys a separate cache from the runner jobs that build it.
run: apt-get update -qq && apt-get install -y -qq --no-install-recommends zstd
- name: Restore Vite Task cache
uses: actions/cache/restore@v6
with:
path: node_modules/.vite/task-cache
key: vite-task-${{ runner.os }}-${{ runner.arch }}-e2e-${{ github.sha }}-${{ github.run_attempt }}
restore-keys: |
vite-task-${{ runner.os }}-${{ runner.arch }}-e2e-
vite-task-${{ runner.os }}-${{ runner.arch }}-packages-
- name: Replay the build
run: pnpm build:packages && pnpm build:cdn && pnpm exec vp run '@videojs/e2e#prepare:player'
- name: Record player visual snapshots
working-directory: apps/e2e
run: pnpm exec playwright test --config suites/player/playwright.config.ts visual --update-snapshots --project=vite-chromium --project=vite-webkit
- name: Upload snapshots
if: always()
uses: actions/upload-artifact@v4
with:
name: e2e-snapshots
path: apps/e2e/suites/player/tests/**/*-snapshots/
retention-days: 7
# `skin create` copies the registry, the workbench templates, and a generated player into one project, and each of
# them changes on its own. Create every framework and styling pair with each preset, and each base skin, against this
# checkout, then install, typecheck, and build them. No browser runs, so the job needs no Playwright image.
skin-projects:
name: Skin projects
needs: changes
if: ${{ !cancelled() && (github.event_name == 'push' || needs.changes.outputs.skin_projects == 'true' || contains(github.event.pull_request.labels.*.name, 'test:e2e')) }}
runs-on: blacksmith-4vcpu-ubuntu-2404
timeout-minutes: 15
steps:
- name: Checkout code
uses: actions/checkout@v5
- name: Setup pnpm
uses: pnpm/action-setup@v6.1.0
- name: Setup Node.js
uses: actions/setup-node@v5
with:
node-version-file: '.nvmrc'
cache: pnpm
- name: Install dependencies
run: pnpm install --frozen-lockfile
- name: Restore Vite Task cache
id: vite-task-cache
uses: actions/cache/restore@v6
with:
path: node_modules/.vite/task-cache
key: vite-task-${{ runner.os }}-${{ runner.arch }}-skin-projects-${{ github.sha }}-${{ github.run_attempt }}
restore-keys: |
vite-task-${{ runner.os }}-${{ runner.arch }}-skin-projects-
vite-task-${{ runner.os }}-${{ runner.arch }}-packages-
- name: Build packages
run: pnpm build:packages
- name: Create, install, typecheck, and build skin projects
run: pnpm --silent -F @videojs/cli check:projects
- name: Save Vite Task cache
if: success()
uses: actions/cache/save@v6
with:
path: node_modules/.vite/task-cache
key: ${{ steps.vite-task-cache.outputs.cache-primary-key }}
registry:
name: Registry consumers
runs-on: blacksmith-4vcpu-ubuntu-2404
container:
image: mcr.microsoft.com/playwright:v1.59.1-noble
options: --init
timeout-minutes: 15
env:
PLAYWRIGHT_BROWSERS_PATH: /ms-playwright
steps:
- name: Checkout code
uses: actions/checkout@v5
- name: Setup pnpm
uses: pnpm/action-setup@v6.1.0
- name: Setup Node.js
uses: actions/setup-node@v5
with:
node-version-file: '.nvmrc'
cache: pnpm
- name: Install dependencies
run: pnpm install --frozen-lockfile
- name: Install zstd
# actions/cache falls back to gzip without zstd, which keys a separate cache from the runner jobs that build it.
run: apt-get update -qq && apt-get install -y -qq --no-install-recommends zstd
- name: Restore Vite Task cache
id: vite-task-cache
uses: actions/cache/restore@v6
with:
path: node_modules/.vite/task-cache
key: vite-task-${{ runner.os }}-${{ runner.arch }}-registry-${{ github.sha }}
restore-keys: |
vite-task-${{ runner.os }}-${{ runner.arch }}-registry-
vite-task-${{ runner.os }}-${{ runner.arch }}-packages-
- name: Validate registry consumers
if: github.event_name == 'pull_request'
run: pnpm exec vp run '@videojs/e2e#test:registry'
- name: Validate registry consumers without task cache
if: github.event_name == 'push'
run: pnpm exec vp run '@videojs/e2e#test:registry:full'
- name: Save Vite Task cache
if: success()
uses: actions/cache/save@v6
with:
path: node_modules/.vite/task-cache
key: ${{ steps.vite-task-cache.outputs.cache-primary-key }}
- name: Upload test report
if: always()
uses: actions/upload-artifact@v4
with:
name: playwright-report-registry
path: apps/e2e/playwright-report/registry/
retention-days: 14
- name: Upload test traces
if: failure()
uses: actions/upload-artifact@v4
with:
name: playwright-traces-registry
path: apps/e2e/test-results/registry/
retention-days: 7