<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.1//EN"
"http://www.w3.org/TR/xhtml11/DTD/xhtml11.dtd">
<html xmlns="http://www.w3.org/1999/xhtml">
<head><meta http-equiv="content-type" content="text/html; charset=utf-8" />
<title>[56506] trunk: Build/Test Tools: Compare results in performance measurement workflow.</title>
</head>
<body>
<style type="text/css"><!--
#msg dl.meta { border: 1px #006 solid; background: #369; padding: 6px; color: #fff; }
#msg dl.meta dt { float: left; width: 6em; font-weight: bold; }
#msg dt:after { content:':';}
#msg dl, #msg dt, #msg ul, #msg li, #header, #footer, #logmsg { font-family: verdana,arial,helvetica,sans-serif; font-size: 10pt; }
#msg dl a { font-weight: bold}
#msg dl a:link { color:#fc3; }
#msg dl a:active { color:#ff0; }
#msg dl a:visited { color:#cc6; }
h3 { font-family: verdana,arial,helvetica,sans-serif; font-size: 10pt; font-weight: bold; }
#msg pre { white-space: pre-line; overflow: auto; background: #ffc; border: 1px #fa0 solid; padding: 6px; }
#logmsg { background: #ffc; border: 1px #fa0 solid; padding: 1em 1em 0 1em; }
#logmsg p, #logmsg pre, #logmsg blockquote { margin: 0 0 1em 0; }
#logmsg p, #logmsg li, #logmsg dt, #logmsg dd { line-height: 14pt; }
#logmsg h1, #logmsg h2, #logmsg h3, #logmsg h4, #logmsg h5, #logmsg h6 { margin: .5em 0; }
#logmsg h1:first-child, #logmsg h2:first-child, #logmsg h3:first-child, #logmsg h4:first-child, #logmsg h5:first-child, #logmsg h6:first-child { margin-top: 0; }
#logmsg ul, #logmsg ol { padding: 0; list-style-position: inside; margin: 0 0 0 1em; }
#logmsg ul { text-indent: -1em; padding-left: 1em; }#logmsg ol { text-indent: -1.5em; padding-left: 1.5em; }
#logmsg > ul, #logmsg > ol { margin: 0 0 1em 0; }
#logmsg pre { background: #eee; padding: 1em; }
#logmsg blockquote { border: 1px solid #fa0; border-left-width: 10px; padding: 1em 1em 0 1em; background: white;}
#logmsg dl { margin: 0; }
#logmsg dt { font-weight: bold; }
#logmsg dd { margin: 0; padding: 0 0 0.5em 0; }
#logmsg dd:before { content:'\00bb';}
#logmsg table { border-spacing: 0px; border-collapse: collapse; border-top: 4px solid #fa0; border-bottom: 1px solid #fa0; background: #fff; }
#logmsg table th { text-align: left; font-weight: normal; padding: 0.2em 0.5em; border-top: 1px dotted #fa0; }
#logmsg table td { text-align: right; border-top: 1px dotted #fa0; padding: 0.2em 0.5em; }
#logmsg table thead th { text-align: center; border-bottom: 1px solid #fa0; }
#logmsg table th.Corner { text-align: left; }
#logmsg hr { border: none 0; border-top: 2px dashed #fa0; height: 1px; }
#header, #footer { color: #fff; background: #636; border: 1px #300 solid; padding: 6px; }
#patch { width: 100%; }
#patch h4 {font-family: verdana,arial,helvetica,sans-serif;font-size:10pt;padding:8px;background:#369;color:#fff;margin:0;}
#patch .propset h4, #patch .binary h4 {margin:0;}
#patch pre {padding:0;line-height:1.2em;margin:0;}
#patch .diff {width:100%;background:#eee;padding: 0 0 10px 0;overflow:auto;}
#patch .propset .diff, #patch .binary .diff {padding:10px 0;}
#patch span {display:block;padding:0 10px;}
#patch .modfile, #patch .addfile, #patch .delfile, #patch .propset, #patch .binary, #patch .copfile {border:1px solid #ccc;margin:10px 0;}
#patch ins {background:#dfd;text-decoration:none;display:block;padding:0 10px;}
#patch del {background:#fdd;text-decoration:none;display:block;padding:0 10px;}
#patch .lines, .info {color:#888;background:#fff;}
--></style>
<div id="msg">
<dl class="meta" style="font-size: 105%">
<dt style="float: left; width: 6em; font-weight: bold">Revision</dt> <dd><a style="font-weight: bold" href="https://core.trac.wordpress.org/changeset/56506">56506</a><script type="application/ld+json">{"@context":"http://schema.org","@type":"EmailMessage","description":"Review this Commit","action":{"@type":"ViewAction","url":"https://core.trac.wordpress.org/changeset/56506","name":"Review Commit"}}</script></dd>
<dt style="float: left; width: 6em; font-weight: bold">Author</dt> <dd>swissspidy</dd>
<dt style="float: left; width: 6em; font-weight: bold">Date</dt> <dd>2023-09-01 17:24:56 +0000 (Fri, 01 Sep 2023)</dd>
</dl>
<pre style='padding-left: 1em; margin: 2em 0; border-left: 2px solid #ccc; line-height: 1.25; font-size: 105%; font-family: sans-serif'>Build/Test Tools: Compare results in performance measurement workflow.
This extends the performance test workflow added in <a href="https://core.trac.wordpress.org/changeset/55459">[55459]</a> to also run tests against the target branch (if running on a pull request) or the previous commit (if running on trunk).
Those results are then compared with the ones from the current commit, and the difference is displayed as a GitHub Actions workflow summary for convenience.
Props mukesh27, flixos90, desrosj, joemcgill, swissspidy.
Fixes <a href="https://core.trac.wordpress.org/ticket/58358">#58358</a>, <a href="https://core.trac.wordpress.org/ticket/58359">#58359</a>.
See <a href="https://core.trac.wordpress.org/ticket/56150">#56150</a>.</pre>
<h3>Modified Paths</h3>
<ul>
<li><a href="#trunkgithubworkflowsperformanceyml">trunk/.github/workflows/performance.yml</a></li>
</ul>
<h3>Added Paths</h3>
<ul>
<li><a href="#trunktestsperformancecompareresultsjs">trunk/tests/performance/compare-results.js</a></li>
</ul>
</div>
<div id="patch">
<h3>Diff</h3>
<a id="trunkgithubworkflowsperformanceyml"></a>
<div class="modfile"><h4 style="background-color: #eee; color: inherit; margin: 1em 0; padding: 1.3em; font-size: 115%">Modified: trunk/.github/workflows/performance.yml</h4>
<pre class="diff"><span>
<span class="info" style="display: block; padding: 0 10px; color: #888">--- trunk/.github/workflows/performance.yml 2023-09-01 11:05:32 UTC (rev 56505)
+++ trunk/.github/workflows/performance.yml 2023-09-01 17:24:56 UTC (rev 56506)
</span><span class="lines" style="display: block; padding: 0 10px; color: #888">@@ -44,6 +44,8 @@
</span><span class="cx" style="display: block; padding: 0 10px"> # changed if we want to normalize results against a different baseline.
</span><span class="cx" style="display: block; padding: 0 10px"> BASE_TAG: '6.1.1'
</span><span class="cx" style="display: block; padding: 0 10px"> LOCAL_DIR: build
</span><ins style="background-color: #dfd; text-decoration:none; display:block; padding: 0 10px">+ TARGET_REF: ${{ github.event_name == 'pull_request' && github.event.pull_request.base.ref || '' }}
+ TARGET_SHA: ${{ github.event_name == 'pull_request' && github.event.pull_request.base.sha || github.event.before }}
</ins><span class="cx" style="display: block; padding: 0 10px">
</span><span class="cx" style="display: block; padding: 0 10px"> jobs:
</span><span class="cx" style="display: block; padding: 0 10px"> # Runs the performance test suite.
</span><span class="lines" style="display: block; padding: 0 10px; color: #888">@@ -65,9 +67,17 @@
</span><span class="cx" style="display: block; padding: 0 10px"> # - Install MU plugin.
</span><span class="cx" style="display: block; padding: 0 10px"> # - Run performance tests (current commit).
</span><span class="cx" style="display: block; padding: 0 10px"> # - Print performance tests results.
</span><ins style="background-color: #dfd; text-decoration:none; display:block; padding: 0 10px">+ # - Check out target commit (target branch or previous commit).
+ # - Install npm dependencies.
+ # - Build WordPress.
+ # - Run performance tests (previous/target commit).
+ # - Print target performance tests results.
+ # - Reset to original commit.
</ins><span class="cx" style="display: block; padding: 0 10px"> # - Set the environment to the baseline version.
</span><span class="cx" style="display: block; padding: 0 10px"> # - Run baseline performance tests.
</span><del style="background-color: #fdd; text-decoration:none; display:block; padding: 0 10px">- # - Print base line performance tests results.
</del><ins style="background-color: #dfd; text-decoration:none; display:block; padding: 0 10px">+ # - Print baseline performance tests results.
+ # - Compare results with base.
+ # - Add workflow summary.
</ins><span class="cx" style="display: block; padding: 0 10px"> # - Set the base sha.
</span><span class="cx" style="display: block; padding: 0 10px"> # - Set commit details.
</span><span class="cx" style="display: block; padding: 0 10px"> # - Publish performance results.
</span><span class="lines" style="display: block; padding: 0 10px; color: #888">@@ -152,8 +162,32 @@
</span><span class="cx" style="display: block; padding: 0 10px"> run: npm run test:performance
</span><span class="cx" style="display: block; padding: 0 10px">
</span><span class="cx" style="display: block; padding: 0 10px"> - name: Print performance tests results
</span><del style="background-color: #fdd; text-decoration:none; display:block; padding: 0 10px">- run: "node ./tests/performance/results.js"
</del><ins style="background-color: #dfd; text-decoration:none; display:block; padding: 0 10px">+ run: node ./tests/performance/results.js
</ins><span class="cx" style="display: block; padding: 0 10px">
</span><ins style="background-color: #dfd; text-decoration:none; display:block; padding: 0 10px">+ - name: Check out target commit (target branch or previous commit)
+ run: |
+ if [[ -z "$TARGET_REF" ]]; then
+ git fetch -n origin $TARGET_SHA
+ else
+ git fetch -n origin $TARGET_REF
+ fi
+ git reset --hard $TARGET_SHA
+
+ - name: Install npm dependencies
+ run: npm ci
+
+ - name: Build WordPress
+ run: npm run build
+
+ - name: Run target performance tests (base/previous commit)
+ run: npm run test:performance -- --prefix=before
+
+ - name: Print target performance tests results
+ run: node ./tests/performance/results.js --prefix=before
+
+ - name: Reset to original commit
+ run: git reset --hard $GITHUB_SHA
+
</ins><span class="cx" style="display: block; padding: 0 10px"> - name: Set the environment to the baseline version
</span><span class="cx" style="display: block; padding: 0 10px"> run: |
</span><span class="cx" style="display: block; padding: 0 10px"> npm run env:cli -- core update --version=${{ env.BASE_TAG }} --force --path=/var/www/${{ env.LOCAL_DIR }}
</span><span class="lines" style="display: block; padding: 0 10px; color: #888">@@ -162,9 +196,15 @@
</span><span class="cx" style="display: block; padding: 0 10px"> - name: Run baseline performance tests
</span><span class="cx" style="display: block; padding: 0 10px"> run: npm run test:performance -- --prefix=base
</span><span class="cx" style="display: block; padding: 0 10px">
</span><del style="background-color: #fdd; text-decoration:none; display:block; padding: 0 10px">- - name: Print base line performance tests results
- run: "node ./tests/performance/results.js --prefix=base"
</del><ins style="background-color: #dfd; text-decoration:none; display:block; padding: 0 10px">+ - name: Print baseline performance tests results
+ run: node ./tests/performance/results.js --prefix=base
</ins><span class="cx" style="display: block; padding: 0 10px">
</span><ins style="background-color: #dfd; text-decoration:none; display:block; padding: 0 10px">+ - name: Compare results with base
+ run: node ./tests/performance/compare-results.js ${{ runner.temp }}/summary.md
+
+ - name: Add workflow summary
+ run: cat ${{ runner.temp }}/summary.md >> $GITHUB_STEP_SUMMARY
+
</ins><span class="cx" style="display: block; padding: 0 10px"> - name: Set the base sha
</span><span class="cx" style="display: block; padding: 0 10px"> # Only needed when publishing results.
</span><span class="cx" style="display: block; padding: 0 10px"> if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/trunk' }}
</span></span></pre></div>
<a id="trunktestsperformancecompareresultsjs"></a>
<div class="addfile"><h4 style="background-color: #eee; color: inherit; margin: 1em 0; padding: 1.3em; font-size: 115%">Added: trunk/tests/performance/compare-results.js</h4>
<pre class="diff"><span>
<span class="info" style="display: block; padding: 0 10px; color: #888">--- trunk/tests/performance/compare-results.js (rev 0)
+++ trunk/tests/performance/compare-results.js 2023-09-01 17:24:56 UTC (rev 56506)
</span><span class="lines" style="display: block; padding: 0 10px; color: #888">@@ -0,0 +1,165 @@
</span><ins style="background-color: #dfd; text-decoration:none; display:block; padding: 0 10px">+#!/usr/bin/env node
+
+/**
+ * External dependencies.
+ */
+const fs = require( 'fs' );
+const path = require( 'path' );
+const { median } = require( './utils' );
+
+/**
+ * Parse test files into JSON objects.
+ *
+ * @param {string} fileName The name of the file.
+ * @returns An array of parsed objects from each file.
+ */
+const parseFile = ( fileName ) =>
+ JSON.parse(
+ fs.readFileSync( path.join( __dirname, '/specs/', fileName ), 'utf8' )
+ );
+
+// The list of test suites to log.
+const testSuites = [ 'home-block-theme', 'home-classic-theme' ];
+
+// The current commit's results.
+const testResults = Object.fromEntries(
+ testSuites.map( ( key ) => [
+ key,
+ parseFile( `${ key }.test.results.json` ),
+ ] )
+);
+
+// The previous commit's results.
+const prevResults = Object.fromEntries(
+ testSuites.map( ( key ) => [
+ key,
+ parseFile( `before-${ key }.test.results.json` ),
+ ] )
+);
+
+const args = process.argv.slice( 2 );
+
+const summaryFile = args[ 0 ];
+
+/**
+ * Formats an array of objects as a Markdown table.
+ *
+ * For example, this array:
+ *
+ * [
+ * {
+ * foo: 123,
+ * bar: 456,
+ * baz: 'Yes',
+ * },
+ * {
+ * foo: 777,
+ * bar: 999,
+ * baz: 'No',
+ * }
+ * ]
+ *
+ * Will result in the following table:
+ *
+ * | foo | bar | baz |
+ * |-----|-----|-----|
+ * | 123 | 456 | Yes |
+ * | 777 | 999 | No |
+ *
+ * @param {Array<Object>} rows Table rows.
+ * @returns {string} Markdown table content.
+ */
+function formatAsMarkdownTable( rows ) {
+ let result = '';
+ const headers = Object.keys( rows[ 0 ] );
+ for ( const header of headers ) {
+ result += `| ${ header } `;
+ }
+ result += '|\n';
+ for ( const header of headers ) {
+ result += '| ------ ';
+ }
+ result += '|\n';
+
+ for ( const row of rows ) {
+ for ( const value of Object.values( row ) ) {
+ result += `| ${ value } `;
+ }
+ result += '|\n';
+ }
+
+ return result;
+}
+
+/**
+ * Returns a Markdown link to a Git commit on the current GitHub repository.
+ *
+ * For example, turns `a5c3785ed8d6a35868bc169f07e40e889087fd2e`
+ * into (https://github.com/wordpress/wordpress-develop/commit/36fe58a8c64dcc83fc21bddd5fcf054aef4efb27)[36fe58a].
+ *
+ * @param {string} sha Commit SHA.
+ * @return string Link
+ */
+function linkToSha(sha) {
+ const repoName = process.env.GITHUB_REPOSITORY || 'wordpress/wordpress-develop';
+
+ return `[${sha.slice(0, 7)}](https://github.com/${repoName}/commit/${sha})`;
+}
+
+let summaryMarkdown = `# Performance Test Results\n\n`;
+
+if ( process.env.GITHUB_SHA ) {
+ summaryMarkdown += `🛎️ Performance test results for ${ linkToSha( process.env.GITHUB_SHA ) } are in!\n\n`;
+} else {
+ summaryMarkdown += `🛎️ Performance test results are in!\n\n`;
+}
+
+if ( process.env.TARGET_SHA ) {
+ summaryMarkdown += `This compares the results from this commit with the ones from ${ linkToSha( process.env.TARGET_SHA ) }.\n\n`;
+}
+
+if ( process.env.GITHUB_SHA ) {
+ summaryMarkdown += `**Note:** Due to the nature of how GitHub Actions work, some variance in the results is expected.\n\n`;
+}
+
+console.log( 'Performance Test Results\n' );
+
+console.log( 'Note: Due to the nature of how GitHub Actions work, some variance in the results is expected.\n' );
+
+for ( const key of testSuites ) {
+ const current = testResults[ key ];
+ const prev = prevResults[ key ];
+
+ const title = ( key.charAt( 0 ).toUpperCase() + key.slice( 1 ) ).replace(
+ /-+/g,
+ ' '
+ );
+
+ const rows = [];
+
+ for ( const [ metric, values ] of Object.entries( current ) ) {
+ const value = median( values );
+ const prevValue = median( prev[ metric ] );
+
+ const delta = value - prevValue;
+ const percentage = Math.round( ( delta / value ) * 100 );
+ rows.push( {
+ Metric: metric,
+ Before: `${ prevValue.toFixed( 2 ) } ms`,
+ After: `${ value.toFixed( 2 ) } ms`,
+ 'Diff abs.': `${ delta.toFixed( 2 ) } ms`,
+ 'Diff %': `${ percentage.toFixed( 2 ) } %`,
+ } );
+ }
+
+ summaryMarkdown += `## ${ title }\n\n`;
+ summaryMarkdown += `${ formatAsMarkdownTable( rows ) }\n`;
+
+ console.log( title );
+ console.table( rows );
+}
+
+fs.writeFileSync(
+ summaryFile,
+ summaryMarkdown
+);
</ins><span class="cx" style="display: block; padding: 0 10px">Property changes on: trunk/tests/performance/compare-results.js
</span><span class="cx" style="display: block; padding: 0 10px">___________________________________________________________________
</span></span></pre></div>
<a id="svneolstyle"></a>
<div class="addfile"><h4 style="background-color: #eee; color: inherit; margin: 1em 0; padding: 1.3em; font-size: 115%">Added: svn:eol-style</h4></div>
<ins style="background-color: #dfd; text-decoration:none; display:block; padding: 0 10px">+native
</ins><span class="cx" style="display: block; padding: 0 10px">\ No newline at end of property
</span></div>
</body>
</html>