Skip to content

test(e2e): Run live delivery specs in parallel #1331

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
utkarshcloudinary wants to merge 1 commit into from
Closed

test(e2e): Run live delivery specs in parallel #1331

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 3 additions & 2 deletions package.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -40,8 +40,9 @@
"postinstall": "patch-package && composer install",
"readme": "composer readme",
"prepare": "husky",
"test:e2e": "npm-run-all --silent test:e2e:parallel test:e2e:serial",
"test:e2e:parallel": "./.wp-env/scripts/run-e2e.sh --grep-invert @serial",
"test:e2e": "npm-run-all --silent test:e2e:parallel test:e2e:live test:e2e:serial",
"test:e2e:parallel": "./.wp-env/scripts/run-e2e.sh --grep-invert \"@serial|@live\"",
"test:e2e:live": "./.wp-env/scripts/run-e2e.sh --grep @live",
"test:e2e:serial": "./.wp-env/scripts/run-e2e.sh --grep @serial --workers=1",
"test:e2e:debug": "./.wp-env/scripts/run-e2e.sh --ui",
"test:unit": "wp-env run tests-cli --env-cwd=\"wp-content/plugins/$(basename \"$PWD\")\" vendor/bin/phpunit"
Expand Down
25 changes: 10 additions & 15 deletions tests/e2e/cloudinary-asset-edit.spec.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,9 +8,12 @@
/**
* Internal dependencies
*/
const { ensureCloudinaryConnected } = require( './utils/connection' );
const { expectCloudinaryUrl } = require( './utils/delivery' );
const { wpCli } = require( './utils/wizard' );
const {
connectLive,
deletePostsLive,
syncLive,
} = require( './utils/live-site' );

const FIXTURE_PATH = path.join( __dirname, 'fixtures', 'test-image.jpg' );

Expand Down Expand Up @@ -124,13 +127,13 @@
};
}

// @serial: needs real credentials in `cloudinary_connect` to sync assets,
// @live: needs real credentials in `cloudinary_connect` to sync assets,
// while every analytics spec overwrites that option with fake ones.
test.describe( 'Cloudinary per-asset editing', { tag: '@serial' }, () => {
test.describe( 'Cloudinary per-asset editing', { tag: '@live' }, () => {
test.describe.configure( { mode: 'serial' } );

test.beforeAll( async ( { requestUtils } ) => {
( { cloudName } = ensureCloudinaryConnected() );
( { cloudName } = connectLive() );

const media = await requestUtils.rest( {
method: 'POST',
Expand Down Expand Up @@ -165,7 +168,7 @@

// The Edit Asset page and the front-end rewrite both need the
// asset to be synced to Cloudinary.
wpCli( [ 'cloudinary', 'sync' ] );
syncLive();
} );

test.afterAll( () => {
Expand All @@ -175,15 +178,7 @@
const { postId, attachmentId } = created;
created = null;

// WP-CLI rather than REST for the same reason as the image delivery
// spec: `force=true` on a `?rest_route=` URL is fragile.
for ( const id of [ postId, attachmentId ] ) {
try {
wpCli( [ 'post', 'delete', String( id ), '--force' ] );
} catch ( e ) {
console.warn( `Cleanup of post ${ id } failed:`, e.message );
}
}
deletePostsLive( [ postId, attachmentId ] );
} );

test( 'text overlay updates the delivery URL and reverts when removed', async ( {
Expand Down Expand Up @@ -255,6 +250,6 @@
);
expect( cleared.transformations ).toBe( '' );

expect( await readDelivery( page ) ).toEqual( baseline );

Check failure on line 253 in tests/e2e/cloudinary-asset-edit.spec.js

View workflow job for this annotation

GitHub Actions / E2E (Playwright)

[chromium] › tests/e2e/cloudinary-asset-edit.spec.js:225:2 › Cloudinary per-asset editing › crop updates the delivery URL and reverts when removed @live 

  1) [chromium] › tests/e2e/cloudinary-asset-edit.spec.js:225:2 › Cloudinary per-asset editing › crop updates the delivery URL and reverts when removed @LiVe 

    Retry #1 ───────────────────────────────────────────────────────────────────────────────────────
    Error: expect(received).toEqual(expected) // deep equality

    - Expected  - 2
    + Received  + 2

      Object {
    -   "src": "https://res.cloudinary.com/small-credit/images/f_auto,q_auto/v1792496558/asset-edit-e2e-1792496556110/asset-edit-e2e-1792496556110.jpg?_i=AA",
    -   "transformations": "f_auto,q_auto",
    +   "src": "https://res.cloudinary.com/small-credit/images/f_webp,q_80/v1792496558/asset-edit-e2e-1792496556110/asset-edit-e2e-1792496556110.jpg?_i=AA",
    +   "transformations": "f_webp,q_80",
      }

      251 | 		expect( cleared.transformations ).toBe( '' );
      252 |
    > 253 | 		expect( await readDelivery( page ) ).toEqual( baseline );
          | 		                                     ^
      254 | 	} );
      255 | } );
      256 |
        at /home/runner/work/cloudinary_wordpress/cloudinary_wordpress/tests/e2e/cloudinary-asset-edit.spec.js:253:40

Check failure on line 253 in tests/e2e/cloudinary-asset-edit.spec.js

View workflow job for this annotation

GitHub Actions / E2E (Playwright)

[chromium] › tests/e2e/cloudinary-asset-edit.spec.js:225:2 › Cloudinary per-asset editing › crop updates the delivery URL and reverts when removed @live 

  1) [chromium] › tests/e2e/cloudinary-asset-edit.spec.js:225:2 › Cloudinary per-asset editing › crop updates the delivery URL and reverts when removed @LiVe 
    Error: expect(received).toEqual(expected) // deep equality

    - Expected  - 2
    + Received  + 2

      Object {
    -   "src": "https://res.cloudinary.com/small-credit/images/f_webp,q_80/v1792496516/asset-edit-e2e-1792496511124/asset-edit-e2e-1792496511124.jpg?_i=AA",
    -   "transformations": "f_webp,q_80",
    +   "src": "https://res.cloudinary.com/small-credit/images/f_auto,q_auto/v1792496516/asset-edit-e2e-1792496511124/asset-edit-e2e-1792496511124.jpg?_i=AA",
    +   "transformations": "f_auto,q_auto",
      }

      251 | 		expect( cleared.transformations ).toBe( '' );
      252 |
    > 253 | 		expect( await readDelivery( page ) ).toEqual( baseline );
          | 		                                     ^
      254 | 	} );
      255 | } );
      256 |
        at /home/runner/work/cloudinary_wordpress/cloudinary_wordpress/tests/e2e/cloudinary-asset-edit.spec.js:253:40
} );
} );
36 changes: 17 additions & 19 deletions tests/e2e/cloudinary-gallery-delivery.spec.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,9 +8,12 @@ const { test, expect } = require( './fixtures' );
/**
* Internal dependencies
*/
const { ensureCloudinaryConnected } = require( './utils/connection' );
const { expectCloudinaryUrl, srcsetUrls } = require( './utils/delivery' );
const { wpCli } = require( './utils/wizard' );
const {
connectLive,
deletePostsLive,
withSiteLock,
} = require( './utils/live-site' );

const FIXTURE_PATH = path.join( __dirname, 'fixtures', 'test-image.jpg' );

Expand Down Expand Up @@ -100,13 +103,13 @@ async function collectImageUrls( images ) {
);
}

// @serial: needs real credentials in `cloudinary_connect` to sync assets,
// @live: needs real credentials in `cloudinary_connect` to sync assets,
// while every analytics spec overwrites that option with fake ones.
test.describe( 'Cloudinary Gallery block delivery', { tag: '@serial' }, () => {
test.describe( 'Cloudinary Gallery block delivery', { tag: '@live' }, () => {
test.describe.configure( { mode: 'serial' } );

test.beforeAll( async ( { requestUtils } ) => {
( { cloudName } = ensureCloudinaryConnected() );
( { cloudName } = connectLive() );

const file = fs.readFileSync( FIXTURE_PATH );
const attachmentIds = [];
Expand All @@ -126,11 +129,14 @@ test.describe( 'Cloudinary Gallery block delivery', { tag: '@serial' }, () => {
// Same endpoint the block editor calls when images are picked in
// the media modal. It uploads each attachment to Cloudinary if it
// is not synced yet and returns the `selectedImages` attribute.
const selectedImages = await requestUtils.rest( {
method: 'POST',
path: '/cloudinary/v1/image_data',
data: { images: attachmentIds.map( ( id ) => ( { id } ) ) },
} );
// Under the site lock so it cannot race another worker's sync.
const selectedImages = await withSiteLock( () =>
requestUtils.rest( {
method: 'POST',
path: '/cloudinary/v1/image_data',
data: { images: attachmentIds.map( ( id ) => ( { id } ) ) },
} )
);

expect(
selectedImages,
Expand Down Expand Up @@ -165,15 +171,7 @@ test.describe( 'Cloudinary Gallery block delivery', { tag: '@serial' }, () => {
const { postId, attachmentIds } = created;
created = null;

// WP-CLI rather than REST for the same reason as the image delivery
// spec: `force=true` on a `?rest_route=` URL is fragile.
for ( const id of [ postId, ...attachmentIds ] ) {
try {
wpCli( [ 'post', 'delete', String( id ), '--force' ] );
} catch ( e ) {
console.warn( `Cleanup of post ${ id } failed:`, e.message );
}
}
deletePostsLive( [ postId, ...attachmentIds ] );
} );

test( 'renders exactly N images, all served from Cloudinary', async ( {
Expand Down
31 changes: 10 additions & 21 deletions tests/e2e/cloudinary-image-delivery.spec.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,9 +8,12 @@ const { test, expect } = require( './fixtures' );
/**
* Internal dependencies
*/
const { ensureCloudinaryConnected } = require( './utils/connection' );
const { expectCloudinaryUrl } = require( './utils/delivery' );
const { wpCli } = require( './utils/wizard' );
const {
connectLive,
deletePostsLive,
syncLive,
} = require( './utils/live-site' );

const FIXTURE_PATH = path.join( __dirname, 'fixtures', 'test-image.jpg' );

Expand All @@ -23,11 +26,11 @@ let cloudName;
*/
let created = null;

// @serial: needs real credentials in `cloudinary_connect` for `wp cloudinary
// @live: needs real credentials in `cloudinary_connect` for `wp cloudinary
// sync`, while every analytics spec overwrites that option with fake ones.
test.describe( 'Cloudinary image delivery', { tag: '@serial' }, () => {
test.describe( 'Cloudinary image delivery', { tag: '@live' }, () => {
test.beforeAll( () => {
( { cloudName } = ensureCloudinaryConnected() );
( { cloudName } = connectLive() );
} );

test.beforeEach( async ( { requestUtils } ) => {
Expand Down Expand Up @@ -75,7 +78,7 @@ test.describe( 'Cloudinary image delivery', { tag: '@serial' }, () => {
// first front-end render queues the sync but renders local
// URLs. Driving `wp cloudinary sync` here makes the test
// deterministic without relying on the cron-driven queue.
wpCli( [ 'cloudinary', 'sync' ] );
syncLive();
} );

test.afterEach( async () => {
Expand All @@ -85,21 +88,7 @@ test.describe( 'Cloudinary image delivery', { tag: '@serial' }, () => {
const { postId, attachmentId } = created;
created = null;

// Best-effort cleanup via WP-CLI. We do not use the REST API
// here because the test wp-env runs without pretty permalinks,
// which makes appending `force=true` to a `?rest_route=`-style
// URL fragile. WP-CLI is unambiguous and we already use it in
// other helpers (see utils/wizard.js).
try {
wpCli( [ 'post', 'delete', String( postId ), '--force' ] );
} catch ( e ) {
console.warn( 'Post cleanup failed:', e.message );
}
try {
wpCli( [ 'post', 'delete', String( attachmentId ), '--force' ] );
} catch ( e ) {
console.warn( 'Media cleanup failed:', e.message );
}
deletePostsLive( [ postId, attachmentId ] );
} );

test( 'serves featured image and inline image via Cloudinary', async ( {
Expand Down
35 changes: 18 additions & 17 deletions tests/e2e/cloudinary-svg-delivery.spec.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,9 +8,14 @@ const { test, expect } = require( './fixtures' );
/**
* Internal dependencies
*/
const { ensureCloudinaryConnected } = require( './utils/connection' );
const { expectCloudinaryUrl } = require( './utils/delivery' );
const { wpCli, wpEvalFile } = require( './utils/wizard' );
const { wpEvalFile } = require( './utils/wizard' );
const {
connectLive,
deletePostsLive,
syncLive,
withSiteLock,
} = require( './utils/live-site' );

const VALID_FIXTURE = path.join( __dirname, 'fixtures', 'svg-valid.svg' );
const UNSAFE_FIXTURE = path.join( __dirname, 'fixtures', 'svg-unsafe.svg' );
Expand Down Expand Up @@ -54,8 +59,11 @@ let createdIds = [];
* @return {Record<string, string>} Setting slug to the value before the write.
*/
function saveSettings( values ) {
return JSON.parse(
wpEvalFile( `
// Under the site lock: the settings share one option with the video
// spec's `video_player`, and each write rewrites the whole array.
return withSiteLock( () =>
JSON.parse(
wpEvalFile( `
$settings = get_plugin_instance()->settings;
$previous = array();
foreach ( json_decode( '${ JSON.stringify(
Expand All @@ -67,6 +75,7 @@ function saveSettings( values ) {
}
echo wp_json_encode( $previous );
` )
)
);
}

Expand Down Expand Up @@ -117,7 +126,7 @@ async function uploadSvgPost( requestUtils, fixturePath, name ) {

// Same reason as the image delivery spec: sync synchronously instead
// of waiting for the auto-sync queue.
wpCli( [ 'cloudinary', 'sync' ] );
syncLive();

return { attachmentId: media.id, postLink: post.link, media };
}
Expand Down Expand Up @@ -188,11 +197,11 @@ async function fetchAsset( request, url ) {
};
}

// @serial: needs real credentials in `cloudinary_connect` for `wp cloudinary
// @live: needs real credentials in `cloudinary_connect` for `wp cloudinary
// sync`, while every analytics spec overwrites that option with fake ones.
test.describe( 'Cloudinary SVG upload and delivery', { tag: '@serial' }, () => {
test.describe( 'Cloudinary SVG upload and delivery', { tag: '@live' }, () => {
test.beforeAll( () => {
( { cloudName } = ensureCloudinaryConnected() );
( { cloudName } = connectLive() );

// SVG support is off by default. Without it WordPress rejects the
// upload and the plugin does not sync or sanitize SVGs.
Expand All @@ -210,15 +219,7 @@ test.describe( 'Cloudinary SVG upload and delivery', { tag: '@serial' }, () => {
const ids = createdIds;
createdIds = [];

// WP-CLI rather than REST for the same reason as the image delivery
// spec: `force=true` on a `?rest_route=` URL is fragile.
for ( const id of ids ) {
try {
wpCli( [ 'post', 'delete', String( id ), '--force' ] );
} catch ( e ) {
console.warn( `Cleanup of post ${ id } failed:`, e.message );
}
}
deletePostsLive( ids );
} );

test( 'a valid SVG uploads, syncs and is delivered as an SVG', async ( {
Expand Down
43 changes: 19 additions & 24 deletions tests/e2e/cloudinary-video-delivery.spec.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,8 +9,13 @@ const { test, expect } = require( './fixtures' );
/**
* Internal dependencies
*/
const { ensureCloudinaryConnected } = require( './utils/connection' );
const { wpCli, getCliContainer } = require( './utils/wizard' );
const { getCliContainer } = require( './utils/wizard' );
const {
connectLive,
deletePostsLive,
syncLive,
withSiteLock,
} = require( './utils/live-site' );

const FIXTURE_PATH = path.join( __dirname, 'fixtures', 'test-video.mp4' );

Expand Down Expand Up @@ -78,18 +83,20 @@ function setVideoPlayer( value ) {
// `timeout` matters: execSync blocks the worker's event loop, so
// Playwright's own test timeout cannot interrupt a hung docker/wp
// call. Without it a wedged container hangs the whole suite.
execSync( cmd, {
encoding: 'utf8',
stdio: [ 'ignore', 'pipe', 'pipe' ],
timeout: 30_000,
} );
withSiteLock( () =>
execSync( cmd, {
encoding: 'utf8',
stdio: [ 'ignore', 'pipe', 'pipe' ],
timeout: 30_000,
} )
);
}

// @serial: needs real credentials in `cloudinary_connect` for `wp cloudinary
// @live: needs real credentials in `cloudinary_connect` for `wp cloudinary
// sync`, while every analytics spec overwrites that option with fake ones.
test.describe( 'Cloudinary video delivery', { tag: '@serial' }, () => {
test.describe( 'Cloudinary video delivery', { tag: '@live' }, () => {
test.beforeAll( () => {
( { cloudName } = ensureCloudinaryConnected() );
( { cloudName } = connectLive() );
} );

test.beforeEach( async ( { requestUtils } ) => {
Expand Down Expand Up @@ -134,7 +141,7 @@ test.describe( 'Cloudinary video delivery', { tag: '@serial' }, () => {
// The plugin's URL rewriting depends on the asset being synced
// to Cloudinary. Drive the sync synchronously here so the
// front-end visit sees rewritten URLs.
wpCli( [ 'cloudinary', 'sync' ] );
syncLive();
} );

test.afterEach( async () => {
Expand All @@ -144,19 +151,7 @@ test.describe( 'Cloudinary video delivery', { tag: '@serial' }, () => {
const { postId, attachmentId } = created;
created = null;

// Best-effort cleanup via WP-CLI. Matches the image spec; we
// do not use the REST API because this wp-env runs without
// pretty permalinks, which makes `?force=true` fragile.
try {
wpCli( [ 'post', 'delete', String( postId ), '--force' ] );
} catch ( e ) {
console.warn( 'Post cleanup failed:', e.message );
}
try {
wpCli( [ 'post', 'delete', String( attachmentId ), '--force' ] );
} catch ( e ) {
console.warn( 'Media cleanup failed:', e.message );
}
deletePostsLive( [ postId, attachmentId ] );
} );

test.describe( 'with default WP player', () => {
Expand Down
10 changes: 7 additions & 3 deletions tests/e2e/playwright.config.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -60,9 +60,13 @@ module.exports = defineConfig( {
retries: process.env.CI ? 2 : 0,
// Spec files are spread across workers; tests within one file still run
// in order (fullyParallel is off), which the delivery specs' shared
// beforeAll/afterAll state relies on. Specs tagged @serial mutate
// site-wide state (connection, plugin activation) and are run in a
// second, single-worker pass by `npm run test:e2e`; see package.json.
// beforeAll/afterAll state relies on. `npm run test:e2e` runs three
// passes; see package.json. Specs tagged @live need the real Cloudinary
// connection, which the analytics specs overwrite with fake credentials,
// so they get a pass of their own; they take turns on site-wide steps
// through tests/e2e/utils/live-site.js. Specs tagged @serial mutate
// site-wide state (connection, plugin activation) and run last, in a
// single-worker pass.
// Analytics specs are safe to run concurrently because tests/e2e/fixtures.js
// gives each worker its own analytics capture log.
workers: 3,
Expand Down
Loading
Loading