feat(e2e): adds script for updating snapshots + docs (#16315)

docs/tests/e2e-playwright-suite.md

@@ -49,9 +49,11 @@ Steps:
 
 #### Troubleshooting
 
-1. **To run just one test** you can do: `yarn workspace @trezor/suite-desktop-core test:e2e:web general/wallet-discovery.test.ts`
+1. **To run just one test file** you can do: `yarn workspace @trezor/suite-desktop-core test:e2e:web general/wallet-discovery.test.ts`
 
-1. **To run one group** you can do: `yarn workspace @trezor/suite-desktop-core test:e2e:web --grep=@group=wallet`
+1. **To run just one test** you can do: `yarn workspace @trezor/suite-desktop-core test:e2e:web -g "Basic cardano walkthrough"`
+
+1. **To run one group** you can do: `yarn workspace @trezor/suite-desktop-core test:e2e:web --grep @group=wallet`
 
 1. **To check for flakiness** you can specify test/suite and how many time it should run: `yarn workspace @trezor/suite-desktop-core test:e2e:web general/wallet-discovery.test.ts --repeat-each=10`
 
@@ -73,6 +75,8 @@ At the moment, there are the following tags:
 
 -   @group=[string]
 -   @desktopOnly
+-   @webOnly
+-   @snapshot
 
 #### @group
 
@@ -81,17 +85,52 @@ Assigning a @group allows test runner to run the groups in parallel on CI. At th
 -   `@group=wallet`
 -   `@group=settings`
 
-#### @desktopOnly
+#### @desktopOnly or @webOnly
+
+Some tests are only applicable for Desktop app or Web and you can use this tag to notify the runner, that the test should be ignored when running against opposite Suite.
+
+#### @snapshot
+
+Some tests are using visual regression comparison by comparing specific element with a prerecorded snapshot or by comparing Aria snapshot. This tag serve for easier updating of snapshots and monitoring.
+
+### Updating snapshots
+
+Changes in implementation or environment may demand updating our Aria snapshots or prerecorded image snapshots that serve for visual regression comparison. To do so:
+
+1. Run `yarn test:e2e:update-snapshots`, this will run all tests with @snapshot tag and record new snapshots if there is difference.
+    - Alternatively, you can run `yarn test:e2e <filter one specific test or test file> --update-snapshots` to run even smaller amount of tests and to do a update just for that test or test file.
+1. Once these tests finish, you will be presented with results which may contain diff patch files of Aria snapshots like in this example:
+
+```
+$ yarn test:e2e ios --update-snapshots
+
+Running 1 test using 1 worker
+
+  ✓  1 …wser › Suite does not support iOS @group=other @webOnly @snapshot (5.7s)
+
+New baselines created for:
+
+  e2e/tests/browser/ios.test.ts
+
+  git apply e2e/test-results/rebaselines.patch
+
+  1 passed (6.4s)
+
+To open last HTML report run:
+
+  yarn playwright show-report
+```
 
-Some tests are only applicable for Desktop app and you can use this tag to notify the runner, that the test should be ignored when running against web Suite.
+1. Apply all patches and review that aria snapshot changes. In above example it would be `git apply e2e/test-results/rebaselines.patch`
+1. Review all new image snapshots that were generated by the first step. They are automatically stored in `packages/suite-desktop-core/e2e/snapshots` folder and should be visible in your `git status`.
+1. Add, commit, and create PR
 
 ## Results
 
 ### Currents.dev
 
 Test reports are uploaded to [currents.dev](https://app.currents.dev/)
 
-### Track suite
+### Artifacts on CI
 
-There is a tool to track tests runs and their results, temporarily hosted here https://track-suite.herokuapp.com/
-Repo here: https://github.com/mroz22/track-suite
+Every Playwright test run contains attached artifact of both Playwright report and docker logs. Download, unpack and show with `yarn playwright show-report ./path/to/report`

packages/suite-desktop-core/e2e/tests/browser/ios.test.ts

@@ -12,12 +12,16 @@ const iosAria = `
 `;
 
 test.use({ startEmulator: false, browserName: 'chromium', ...devices['iPhone 15 Pro'] });
-test.describe('iPhone with Chrome browser', { tag: ['@group=other', '@webOnly'] }, () => {
-    test('Suite does not support iOS', async ({ page }) => {
-        await expect(
-            page.getByRole('heading', { name: 'Suite doesn’t work on iOS yet' }),
-        ).toBeVisible();
-        await expect(page.locator('body')).toMatchAriaSnapshot(iosAria);
-        await expect(page.getByText('Continue at my own risk')).not.toBeVisible();
-    });
-});
+test.describe(
+    'iPhone with Chrome browser',
+    { tag: ['@group=other', '@webOnly', '@snapshot'] },
+    () => {
+        test('Suite does not support iOS', async ({ page }) => {
+            await expect(
+                page.getByRole('heading', { name: 'Suite doesn’t work on iOS yet' }),
+            ).toBeVisible();
+            await expect(page.locator('body')).toMatchAriaSnapshot(iosAria);
+            await expect(page.getByText('Continue at my own risk')).not.toBeVisible();
+        });
+    },
+);

packages/suite-desktop-core/e2e/tests/browser/safari.test.ts

@@ -14,7 +14,7 @@ const safariAria = `
 `;
 
 test.use({ startEmulator: false, browserName: 'webkit' });
-test.describe('Safari', { tag: ['@group=other', '@webOnly'] }, () => {
+test.describe('Safari', { tag: ['@group=other', '@webOnly', '@snapshot'] }, () => {
     test('Suite does not support Safari', async ({ page, onboardingPage }) => {
         await expect(page.locator('body')).toMatchAriaSnapshot(safariAria);
         await expect(page.getByTestId('@continue-to-suite')).toHaveText('Continue at my own risk');

packages/suite-desktop-core/e2e/tests/coin-market/buy-coins.test.ts

@@ -2,7 +2,7 @@ import { test, expect } from '../../support/fixtures';
 
 const regexpBtcValue = /^\d+(\.\d+)? BTC$/;
 
-test.describe('Coin market buy', { tag: ['@group=other'] }, () => {
+test.describe('Coin market buy', { tag: ['@group=other', '@snapshot'] }, () => {
     test.beforeEach(async ({ onboardingPage, dashboardPage, walletPage }) => {
         await onboardingPage.completeOnboarding();
         await dashboardPage.discoveryShouldFinish();

packages/suite-desktop-core/e2e/tests/coin-market/exchange-coins.test.ts

@@ -1,6 +1,6 @@
 import { test, expect } from '../../support/fixtures';
 
-test.describe('Coinmarket Exchange', { tag: ['@group=other'] }, () => {
+test.describe('Coinmarket Exchange', { tag: ['@group=other', '@snapshot'] }, () => {
     test.use({
         emulatorSetupConf: {
             mnemonic:

packages/suite-desktop-core/e2e/tests/dashboard/assets.test.ts

@@ -17,26 +17,26 @@ test.describe('Assets', { tag: ['@group=suite'] }, () => {
         await expect(marketPage.section).toBeVisible();
     });
 
-    test('New asset is shown in both grid and row', async ({
-        assetsPage,
-        dashboardPage,
-        settingsPage,
-    }) => {
-        await dashboardPage.discoveryShouldFinish();
-        await assetsPage.enableMoreCoins.click();
-        await settingsPage.coins.enableNetwork('eth');
+    test(
+        'New asset is shown in both grid and row',
+        { tag: ['@snapshot'] },
+        async ({ assetsPage, dashboardPage, settingsPage }) => {
+            await dashboardPage.discoveryShouldFinish();
+            await assetsPage.enableMoreCoins.click();
+            await settingsPage.coins.enableNetwork('eth');
 
-        await dashboardPage.navigateTo();
-        await dashboardPage.discoveryShouldFinish();
-        await expect(assetsPage.section).toHaveScreenshot('new-asset-grid.png', {
-            mask: [assetsPage.bottomInfo],
-        });
+            await dashboardPage.navigateTo();
+            await dashboardPage.discoveryShouldFinish();
+            await expect(assetsPage.section).toHaveScreenshot('new-asset-grid.png', {
+                mask: [assetsPage.bottomInfo],
+            });
 
-        await assetsPage.tableIcon.click();
-        await expect(assetsPage.section).toHaveScreenshot('new-asset-table.png', {
-            mask: [assetsPage.assetExchangeRate, assetsPage.assetWeekChange],
-            //The width of the table is not fixed -> higher maxDiffPixelRatio
-            maxDiffPixelRatio: 0.025,
-        });
-    });
+            await assetsPage.tableIcon.click();
+            await expect(assetsPage.section).toHaveScreenshot('new-asset-table.png', {
+                mask: [assetsPage.assetExchangeRate, assetsPage.assetWeekChange],
+                //The width of the table is not fixed -> higher maxDiffPixelRatio
+                maxDiffPixelRatio: 0.025,
+            });
+        },
+    );
 });

packages/suite-desktop-core/e2e/tests/wallet/cardano.test.ts

@@ -7,7 +7,7 @@ import { test, expect } from '../../support/fixtures';
 // todo: setup emu with 24 words mnemonic so that we can test different cardano derivation and its 'auto-discovery; feature
 //mnemonic: 'clot trim improve bag pigeon party wave mechanic beyond clean cake maze protect left assist carry guitar bridge nest faith critic excuse tooth dutch',
 
-test.describe('Cardano', { tag: ['@group=wallet'] }, () => {
+test.describe('Cardano', { tag: ['@group=wallet', '@snapshot'] }, () => {
     test.beforeEach(async ({ onboardingPage, dashboardPage, settingsPage }) => {
         await onboardingPage.completeOnboarding();
         await dashboardPage.discoveryShouldFinish();

packages/suite-desktop-core/package.json

@@ -12,7 +12,8 @@
         "test:unit": "yarn g:jest",
         "test:e2e": "yarn xvfb-maybe -- playwright test --config=./e2e/playwright.config.ts",
         "test:e2e:web": "yarn xvfb-maybe -- playwright test --config=./e2e/playwright.config.ts --project=web",
-        "test:e2e:desktop": "yarn xvfb-maybe -- playwright test --config=./e2e/playwright.config.ts --project=desktop"
+        "test:e2e:desktop": "yarn xvfb-maybe -- playwright test --config=./e2e/playwright.config.ts --project=desktop",
+        "test:e2e:update-snapshots": "yarn xvfb-maybe -- playwright test --config=./e2e/playwright.config.ts --project=web --grep @snapshot --update-snapshots"
     },
     "dependencies": {
         "@sentry/electron": "^5.9.0",