diff --git a/docs/tests/e2e-playwright-suite.md b/docs/tests/e2e-playwright-suite.md index 0c62dc5a612..b5b628ce5b3 100644 --- a/docs/tests/e2e-playwright-suite.md +++ b/docs/tests/e2e-playwright-suite.md @@ -49,9 +49,11 @@ Steps: #### Troubleshooting -1. **To run just one test** you can do: `yarn workspace @trezor/suite-desktop-core test:e2e:web general/wallet-discovery.test.ts` +1. **To run just one test file** you can do: `yarn workspace @trezor/suite-desktop-core test:e2e:web general/wallet-discovery.test.ts` -1. **To run one group** you can do: `yarn workspace @trezor/suite-desktop-core test:e2e:web --grep=@group=wallet` +1. **To run just one test** you can do: `yarn workspace @trezor/suite-desktop-core test:e2e:web -g "Basic cardano walkthrough"` + +1. **To run one group** you can do: `yarn workspace @trezor/suite-desktop-core test:e2e:web --grep @group=wallet` 1. **To check for flakiness** you can specify test/suite and how many time it should run: `yarn workspace @trezor/suite-desktop-core test:e2e:web general/wallet-discovery.test.ts --repeat-each=10` @@ -73,6 +75,8 @@ At the moment, there are the following tags: - @group=[string] - @desktopOnly +- @webOnly +- @snapshot #### @group @@ -81,9 +85,45 @@ Assigning a @group allows test runner to run the groups in parallel on CI. At th - `@group=wallet` - `@group=settings` -#### @desktopOnly +#### @desktopOnly or @webOnly + +Some tests are only applicable for Desktop app or Web and you can use this tag to notify the runner, that the test should be ignored when running against opposite Suite. + +#### @snapshot + +Some tests are using visual regression comparison by comparing specific element with a prerecorded snapshot or by comparing Aria snapshot. This tag serve for easier updating of snapshots and monitoring. + +### Updating snapshots + +Changes in implementation or environment may demand updating our Aria snapshots or prerecorded image snapshots that serve for visual regression comparison. To do so: + +1. Run `yarn test:e2e:update-snapshots`, this will run all tests with @snapshot tag and record new snapshots if there is difference. + - Alternatively, you can run `yarn test:e2e --update-snapshots` to run even smaller amount of tests and to do a update just for that test or test file. +1. Once these tests finish, you will be presented with results which may contain diff patch files of Aria snapshots like in this example: + +``` +$ yarn test:e2e ios --update-snapshots + +Running 1 test using 1 worker + + ✓ 1 …wser › Suite does not support iOS @group=other @webOnly @snapshot (5.7s) + +New baselines created for: + + e2e/tests/browser/ios.test.ts + + git apply e2e/test-results/rebaselines.patch + + 1 passed (6.4s) + +To open last HTML report run: + + yarn playwright show-report +``` -Some tests are only applicable for Desktop app and you can use this tag to notify the runner, that the test should be ignored when running against web Suite. +1. Apply all patches and review that aria snapshot changes. In above example it would be `git apply e2e/test-results/rebaselines.patch` +1. Review all new image snapshots that were generated by the first step. They are automatically stored in `packages/suite-desktop-core/e2e/snapshots` folder and should be visible in your `git status`. +1. Add, commit, and create PR ## Results @@ -91,7 +131,6 @@ Some tests are only applicable for Desktop app and you can use this tag to notif Test reports are uploaded to [currents.dev](https://app.currents.dev/) -### Track suite +### Artifacts on CI -There is a tool to track tests runs and their results, temporarily hosted here https://track-suite.herokuapp.com/ -Repo here: https://github.com/mroz22/track-suite +Every Playwright test run contains attached artifact of both Playwright report and docker logs. Download, unpack and show with `yarn playwright show-report ./path/to/report` diff --git a/packages/suite-desktop-core/e2e/tests/browser/ios.test.ts b/packages/suite-desktop-core/e2e/tests/browser/ios.test.ts index 52a4b26f1b9..ff25ef1fedb 100644 --- a/packages/suite-desktop-core/e2e/tests/browser/ios.test.ts +++ b/packages/suite-desktop-core/e2e/tests/browser/ios.test.ts @@ -12,12 +12,16 @@ const iosAria = ` `; test.use({ startEmulator: false, browserName: 'chromium', ...devices['iPhone 15 Pro'] }); -test.describe('iPhone with Chrome browser', { tag: ['@group=other', '@webOnly'] }, () => { - test('Suite does not support iOS', async ({ page }) => { - await expect( - page.getByRole('heading', { name: 'Suite doesn’t work on iOS yet' }), - ).toBeVisible(); - await expect(page.locator('body')).toMatchAriaSnapshot(iosAria); - await expect(page.getByText('Continue at my own risk')).not.toBeVisible(); - }); -}); +test.describe( + 'iPhone with Chrome browser', + { tag: ['@group=other', '@webOnly', '@snapshot'] }, + () => { + test('Suite does not support iOS', async ({ page }) => { + await expect( + page.getByRole('heading', { name: 'Suite doesn’t work on iOS yet' }), + ).toBeVisible(); + await expect(page.locator('body')).toMatchAriaSnapshot(iosAria); + await expect(page.getByText('Continue at my own risk')).not.toBeVisible(); + }); + }, +); diff --git a/packages/suite-desktop-core/e2e/tests/browser/safari.test.ts b/packages/suite-desktop-core/e2e/tests/browser/safari.test.ts index eaf70105e22..68f72d51727 100644 --- a/packages/suite-desktop-core/e2e/tests/browser/safari.test.ts +++ b/packages/suite-desktop-core/e2e/tests/browser/safari.test.ts @@ -14,7 +14,7 @@ const safariAria = ` `; test.use({ startEmulator: false, browserName: 'webkit' }); -test.describe('Safari', { tag: ['@group=other', '@webOnly'] }, () => { +test.describe('Safari', { tag: ['@group=other', '@webOnly', '@snapshot'] }, () => { test('Suite does not support Safari', async ({ page, onboardingPage }) => { await expect(page.locator('body')).toMatchAriaSnapshot(safariAria); await expect(page.getByTestId('@continue-to-suite')).toHaveText('Continue at my own risk'); diff --git a/packages/suite-desktop-core/e2e/tests/coin-market/buy-coins.test.ts b/packages/suite-desktop-core/e2e/tests/coin-market/buy-coins.test.ts index 2c300223f95..098694f579c 100644 --- a/packages/suite-desktop-core/e2e/tests/coin-market/buy-coins.test.ts +++ b/packages/suite-desktop-core/e2e/tests/coin-market/buy-coins.test.ts @@ -2,7 +2,7 @@ import { test, expect } from '../../support/fixtures'; const regexpBtcValue = /^\d+(\.\d+)? BTC$/; -test.describe('Coin market buy', { tag: ['@group=other'] }, () => { +test.describe('Coin market buy', { tag: ['@group=other', '@snapshot'] }, () => { test.beforeEach(async ({ onboardingPage, dashboardPage, walletPage }) => { await onboardingPage.completeOnboarding(); await dashboardPage.discoveryShouldFinish(); diff --git a/packages/suite-desktop-core/e2e/tests/coin-market/exchange-coins.test.ts b/packages/suite-desktop-core/e2e/tests/coin-market/exchange-coins.test.ts index b70ebf9f97e..65243f9998c 100644 --- a/packages/suite-desktop-core/e2e/tests/coin-market/exchange-coins.test.ts +++ b/packages/suite-desktop-core/e2e/tests/coin-market/exchange-coins.test.ts @@ -1,6 +1,6 @@ import { test, expect } from '../../support/fixtures'; -test.describe('Coinmarket Exchange', { tag: ['@group=other'] }, () => { +test.describe('Coinmarket Exchange', { tag: ['@group=other', '@snapshot'] }, () => { test.use({ emulatorSetupConf: { mnemonic: diff --git a/packages/suite-desktop-core/e2e/tests/dashboard/assets.test.ts b/packages/suite-desktop-core/e2e/tests/dashboard/assets.test.ts index 1d1ff35f6b7..3b26786d27d 100644 --- a/packages/suite-desktop-core/e2e/tests/dashboard/assets.test.ts +++ b/packages/suite-desktop-core/e2e/tests/dashboard/assets.test.ts @@ -17,26 +17,26 @@ test.describe('Assets', { tag: ['@group=suite'] }, () => { await expect(marketPage.section).toBeVisible(); }); - test('New asset is shown in both grid and row', async ({ - assetsPage, - dashboardPage, - settingsPage, - }) => { - await dashboardPage.discoveryShouldFinish(); - await assetsPage.enableMoreCoins.click(); - await settingsPage.coins.enableNetwork('eth'); + test( + 'New asset is shown in both grid and row', + { tag: ['@snapshot'] }, + async ({ assetsPage, dashboardPage, settingsPage }) => { + await dashboardPage.discoveryShouldFinish(); + await assetsPage.enableMoreCoins.click(); + await settingsPage.coins.enableNetwork('eth'); - await dashboardPage.navigateTo(); - await dashboardPage.discoveryShouldFinish(); - await expect(assetsPage.section).toHaveScreenshot('new-asset-grid.png', { - mask: [assetsPage.bottomInfo], - }); + await dashboardPage.navigateTo(); + await dashboardPage.discoveryShouldFinish(); + await expect(assetsPage.section).toHaveScreenshot('new-asset-grid.png', { + mask: [assetsPage.bottomInfo], + }); - await assetsPage.tableIcon.click(); - await expect(assetsPage.section).toHaveScreenshot('new-asset-table.png', { - mask: [assetsPage.assetExchangeRate, assetsPage.assetWeekChange], - //The width of the table is not fixed -> higher maxDiffPixelRatio - maxDiffPixelRatio: 0.025, - }); - }); + await assetsPage.tableIcon.click(); + await expect(assetsPage.section).toHaveScreenshot('new-asset-table.png', { + mask: [assetsPage.assetExchangeRate, assetsPage.assetWeekChange], + //The width of the table is not fixed -> higher maxDiffPixelRatio + maxDiffPixelRatio: 0.025, + }); + }, + ); }); diff --git a/packages/suite-desktop-core/e2e/tests/wallet/cardano.test.ts b/packages/suite-desktop-core/e2e/tests/wallet/cardano.test.ts index d439dea56ac..206009c3070 100644 --- a/packages/suite-desktop-core/e2e/tests/wallet/cardano.test.ts +++ b/packages/suite-desktop-core/e2e/tests/wallet/cardano.test.ts @@ -7,7 +7,7 @@ import { test, expect } from '../../support/fixtures'; // todo: setup emu with 24 words mnemonic so that we can test different cardano derivation and its 'auto-discovery; feature //mnemonic: 'clot trim improve bag pigeon party wave mechanic beyond clean cake maze protect left assist carry guitar bridge nest faith critic excuse tooth dutch', -test.describe('Cardano', { tag: ['@group=wallet'] }, () => { +test.describe('Cardano', { tag: ['@group=wallet', '@snapshot'] }, () => { test.beforeEach(async ({ onboardingPage, dashboardPage, settingsPage }) => { await onboardingPage.completeOnboarding(); await dashboardPage.discoveryShouldFinish(); diff --git a/packages/suite-desktop-core/package.json b/packages/suite-desktop-core/package.json index 3132c693bd6..634edaaa19f 100644 --- a/packages/suite-desktop-core/package.json +++ b/packages/suite-desktop-core/package.json @@ -12,7 +12,8 @@ "test:unit": "yarn g:jest", "test:e2e": "yarn xvfb-maybe -- playwright test --config=./e2e/playwright.config.ts", "test:e2e:web": "yarn xvfb-maybe -- playwright test --config=./e2e/playwright.config.ts --project=web", - "test:e2e:desktop": "yarn xvfb-maybe -- playwright test --config=./e2e/playwright.config.ts --project=desktop" + "test:e2e:desktop": "yarn xvfb-maybe -- playwright test --config=./e2e/playwright.config.ts --project=desktop", + "test:e2e:update-snapshots": "yarn xvfb-maybe -- playwright test --config=./e2e/playwright.config.ts --project=web --grep @snapshot --update-snapshots" }, "dependencies": { "@sentry/electron": "^5.9.0",