From 11d757a59005d40f7ebfb27e3728eb9d9fb8ffe3 Mon Sep 17 00:00:00 2001 From: Johnnie Birch Date: Fri, 8 Jul 2022 20:02:43 -0700 Subject: [PATCH] Adds github action to support x64 performance testing using a sightglass This github action allows performance testing using sightglass. The action is triggered either via a workflow dispatch or with the comment '/bench_x64', in a pull request. Once triggered the action will send a request to a private repository that supports using a self-hosted runner to do comparisons of "refs/feature/commit" vs "refs/heads/main" for wasmtime. If the action is triggered via a comment in a pull request (with '/bench_x64') then the commit referenced by the pull request is used for the comparison against refs/head/main. If triggered via a workflow dispatch the interface will request the commit to compare against refs/head/main. The results of the performance tests, run via sightglass, will be a table showing a percentage change in clock ticks in various stages requried for executing the benchmark, namely instantiate, compiliation, and execution. This patch is intended to be just a starting patch with much to tweak and improve. One of the TODOs will be adding support for aarch64 .. currently this patch supports only x64. Note also that the logic for actually doing the comparison and parsing the results occurs with the action associated with the private repo and so this patch itself (though the trigger) is fairly straight forward. --- .github/workflows/performance.yml | 98 +++++++++++++++++++++++++++++++ 1 file changed, 98 insertions(+) create mode 100644 .github/workflows/performance.yml diff --git a/.github/workflows/performance.yml b/.github/workflows/performance.yml new file mode 100644 index 000000000000..65024e9907e5 --- /dev/null +++ b/.github/workflows/performance.yml @@ -0,0 +1,98 @@ +# This is a workflow triggered by PR or triggered manually +# Runs quick performance tests and reports the comparison against HEAD +# Test should take less than 10 minutes to run on current self-hosted devices +name: "Performance Testing" + +# Controls when the action will run. +# This workflow runs when manually triggered using the UI or API. +on: + issue_comment: + types: [created, edited] + pull_request_review_comment: + types: [created, edited] + workflow_dispatch: + inputs: + actor: + description: 'User that requested the workflow' + type: string + required: false + pr_number: + description: 'Pull request number for where to post the results' + type: string + required: false + default: '' + post_results: + description: 'Check to post results to the given PR' + type: boolean + required: false + default: false + repository: + description: 'Repository for the commit' + type: string + default: 'bytecodealliance/wasmtime' + required: true + refs: + description: 'Ref for the commit' + type: string + default: 'refs/heads/main' + required: true + sg_commit: + description: 'Sightglass ref' + type: string + default: '81c425a' + required: false + message: + description: 'Message to print with posted PR comment' + type: string + default: 'Triggered by wasmtime workflow dispatch' + required: false + +# Env variables +env: + SG_COMMIT: 649509c + TOKEN: ${{ secrets.SIGHTGLASS_BENCHMARKING_TOKEN }} + GITHUB_CONTEXT: ${{ toJson(github) }} + +jobs: + Sightglass_x86-64_from_pr_comment: + name: Sightglass x86-64_from_pr_comment + runs-on: ubuntu-latest + if: | + startsWith(github.event.comment.body, '/bench_x64') && + (github.event.issue.pull_request) && + (('abrown' == github.event.comment.user.login) + || ('afonso360' == github.event.comment.user.login) + || ('akirilov-arm' == github.event.comment.user.login) + || ('alexcrichton' == github.event.comment.user.login) + || ('bbouvier' == github.event.comment.user.login) + || ('bjorn3' == github.event.comment.user.login) + || ('cfallin' == github.event.comment.user.login) + || ('fitzgen' == github.event.comment.user.login) + || ('jlb6740' == github.event.comment.user.login) + || ('sparker-arm' == github.event.comment.user.login) + || ('uweigand' == github.event.comment.user.login)) + steps: + - run: echo "MESSAGE=Requested from pull request comment." >> $GITHUB_ENV + - run: echo "ACTOR=${{ github.event.comment.user.login }}" >> $GITHUB_ENV + - run: echo "REPOSITORY=${{ github.repository }}" >> $GITHUB_ENV + - run: echo "REFS=${{ github.ref }}" >> $GITHUB_ENV + - run: echo "PR_NUMBER=${{ github.event.issue.number }}" >> $GITHUB_ENV + - run: echo "PUBLISH=true" >> $GITHUB_ENV + - run: echo "$GITHUB_CONTEXT" + - run: | + curl -X POST -H "Accept: application/vnd.github.v3+json" -H "Authorization: token ${{ env.TOKEN }}" https://api.github.com/repos/bytecodealliance/wasmtime-sightglass-benchmarking/dispatches -d '{"event_type":"bench-sightglass", "client_payload":{"message":"${{ env.MESSAGE }}", "actor":"${{ env.ACTOR }}", "repository":"${{ env.REPOSITORY }}", "ref":"${{ env.REFS }}", "pr_number":"${{ env.PR_NUMBER }}", "publish":"${{ env.PUBLISH }}" }' + + Sightglass_x86-64_from_workflow_dispatch: + name: Performance x86-64_from_workflow_dispatch + runs-on: ubuntu-latest + if: (github.event_name == 'workflow_dispatch') + steps: + - run: echo "MESSAGE=${{ github.event.inputs.message }}" >> $GITHUB_ENV + - run: echo "ACTOR=${{ github.event.inputs.actor }}" >> $GITHUB_ENV + - run: echo "REPOSITORY=${{ github.event.inputs.repository }}" >> $GITHUB_ENV + - run: echo "REFS=${{ github.event.inputs.refs }}" >> $GITHUB_ENV + - run: echo "PR_NUMBER=${{ github.event.inputs.pr_number }}" >> $GITHUB_ENV + - run: echo "PUBLISH=${{ github.event.inputs.post_results }}" >> $GITHUB_ENV + - run: echo "$GITHUB_CONTEXT" + - run: | + curl -X POST -H "Accept: application/vnd.github.v3+json" -H "Authorization: token ${{ env.TOKEN }}" https://api.github.com/repos/bytecodealliance/wasmtime-sightglass-benchmarking/dispatches -d '{"event_type":"bench-sightglass", "client_payload":{"message":"${{ env.MESSAGE }}", "actor":"${{ env.ACTOR }}", "repository":"${{ env.REPOSITORY }}", "ref":"${{ env.REFS }}", "pr_number":"${{ env.PR_NUMBER }}", "publish":"${{ env.PUBLISH }}" }' \ No newline at end of file