Temp and minor change for testing workflow

.github/workflows/performance.yml

@@ -4,106 +4,159 @@
 name: "Performance Testing"
 
 # Controls when the action will run.
-# This workflow runs when manually triggered using the UI or API.
+# This workflow runs when manually triggered by keywords used in the start of a review comment
+# Currently that phrase is /bench_x64. /bench_aarch64 and /bench_all are TODOs.
 on:
-  pull_request_review_comment:
-    types: [created, edited]
-#  push:
+  pull_request_review:
+    types: [submitted, edited]
+  push:
 
 # Env variables
 env:
-  SG_COMMIT: HEAD
+  SG_COMMIT: b4971ae
   TOKEN: ${{ secrets.SIGHTGLASS_BENCHMARKING_TOKEN }}
   GITHUB_CONTEXT: ${{ toJson(github) }}
-  GITHUB_CONTEXT_FROM_PATCH: ""
 
 jobs:
   Wasmtime_Repo_On_PR_Comment:
     name: Benchmark x64 on PR comment Wasmtime repo
     runs-on: ubuntu-latest
-#    if: (github.event_name == 'push') && (github.repository != 'bytecodealliance/wasmtime-sightglass-benchmarking')
     if: |
-      startsWith(github.event.comment.body, '/bench_x64') &&
-      (github.event.issue.pull_request_review_comment) &&
-      (('abrown' == github.event.comment.user.login)
-        || ('afonso360' == github.event.comment.user.login)
-        || ('akirilov-arm' == github.event.comment.user.login)
-        || ('alexcrichton' == github.event.comment.user.login)
-        || ('bbouvier' == github.event.comment.user.login)
-        || ('bjorn3' == github.event.comment.user.login)
-        || ('cfallin' == github.event.comment.user.login)
-        || ('fitzgen' == github.event.comment.user.login)
-        || ('jlb6740' == github.event.comment.user.login)
-        || ('sparker-arm' == github.event.comment.user.login)
-        || ('uweigand' == github.event.comment.user.login))
+      (github.event_name == 'pull_request_review') &&
+      (contains(github.event.review.body, '/bench_x64')) &&
+      (('abrown' == github.event.review.user.login)
+        || ('afonso360' == github.event.review.user.login)
+        || ('akirilov-arm' == github.event.review.user.login)
+        || ('alexcrichton' == github.event.review.user.login)
+        || ('bbouvier' == github.event.review.user.login)
+        || ('bjorn3' == github.event.review.user.login)
+        || ('cfallin' == github.event.review.user.login)
+        || ('fitzgen' == github.event.review.user.login)
+        || ('jlb6740' == github.event.review.user.login)
+        || ('sparker-arm' == github.event.review.user.login)
+        || ('uweigand' == github.event.review.user.login))
     steps:
-      - run: echo "MESSAGE=Requested from pull request comment." >> $GITHUB_ENV
-      - run: echo "ACTOR=${{ github.event.comment.user.login }}" >> $GITHUB_ENV
-      - run: echo "REPOSITORY=${{ github.repository }}" >> $GITHUB_ENV
-      - run: echo "REFS=${{ github.ref }}" >> $GITHUB_ENV
-      - run: echo "PR_NUMBER=${{ github.event.issue.number }}" >> $GITHUB_ENV
-      - run: echo "PUBLISH=true" >> $GITHUB_ENV
       - run: echo "$GITHUB_CONTEXT"
-      - run: echo "$GITHUB_ENV"
       - run: |
           # Create and Push Branch
           git clone https://jlb6740:${{env.TOKEN}}@github.com/bytecodealliance/wasmtime-sightglass-benchmarking.git
           cd wasmtime-sightglass-benchmarking
           git remote add wasmtime ${{ github.event.repository.clone_url }}
-          git fetch wasmtime
-          git checkout wasmtime/${{ github.ref_name }} -b ${{ github.ref_name }}
+          git fetch wasmtime refs/pull/*/merge:refs/remotes/wasmtime/pull/*/merge
+          git checkout wasmtime/pull/${{ github.ref_name }} -b ${{ github.ref_name }}
           git submodule update --init --recursive
-          git checkout -b wasmtime-performance-testing/FB/${{ github.ref_name }}/${{ github.sha }}
-          echo ${{ github.event.commits[0].committer.name  }}
-          echo "email - ${{ github.event.commits[0].committer.email }}"
-          git config user.name '${{ github.event.commits[0].committer.name }}'
-          git config user.email '${{ github.event.commits[0].committer.email }}'
-          git commit --allow-empty -m "${GITHUB_CONTEXT}"
-          git push origin wasmtime-performance-testing/FB/${{ github.ref_name }}/${{ github.sha }}
-          #curl -X POST -H "Accept: application/vnd.github.v3+json" -H "Authorization: token ${{ env.TOKEN }}" https://api.github.com/repos/bytecodealliance/wasmtime-sightglass-benchmarking/dispatches -d '{"event_type":"Performance Testing", "client_payload":{"message":"${{ env.MESSAGE }}", "actor":"${{ env.ACTOR }}", "repository":"${{ env.REPOSITORY }}", "ref":"${{ env.REFS }}", "pr_number":"${{ env.PR_NUMBER }}", "publish":"${{ env.PUBLISH }}" }'
+          git checkout -b wasmtime/${{ github.ref }}/${{ github.sha }}
+          export commit_url=${{ github.event.pull_request._links.commits.href }}
+          git config user.name $(curl -sSL $commit_url | jq -r '.[].commit.committer.name' | tail -n 1)
+          git config user.email $(curl -sSL $commit_url | jq -r '.[].commit.committer.email' | tail -n 1)
+          git commit --allow-empty -m "${{ github.event.pull_request._links.comments.href }}"
+          git push origin wasmtime/${{ github.ref }}/${{ github.sha }}
 
   Performance_Repo_On_Push:
     name: Benchmark x64 on push Performance repo
     runs-on: [self-hosted, linux, x64]
     if: (github.event_name == 'push') && (github.repository == 'bytecodealliance/wasmtime-sightglass-benchmarking')
     steps:
-#      - run: echo "$GITHUB_CONTEXT"
+      - run: echo "$GITHUB_CONTEXT"
       - run: echo "${{ github.event.head_commit.message }}"
-#      - name: "Build sightglass commit '${{ env.SG_COMMIT }}'"
-#        run: |
-#          cd ../ && ls -l && rm -rf ./sightglass
-#          git clone https://github.com/bytecodealliance/sightglass.git && cd ./sightglass
-#          git checkout ${{env.SG_COMMIT}}
-#          cargo build --release
+      - name: "Build sightglass commit '${{ env.SG_COMMIT }}'"
+        run: |
+          cd ../ && ls -l && rm -rf ./sightglass
+          git clone https://github.com/bytecodealliance/sightglass.git && cd ./sightglass
+          git checkout ${{env.SG_COMMIT}}
+          cargo build --release
+
+      - name: Checkout patch from bytecodealliance/wasmtime (pushed and triggering on this perf repo)
+        uses: actions/checkout@v3
+        with:
+          submodules: true
+          path: wasmtime_commit
+
+      - name: Build patch from bytecodealliance/wasmtime (pushed and triggering on this perf repo)
+        working-directory: ./wasmtime_commit
+        run: |
+          cargo build --release -p wasmtime-bench-api
+          cp target/release/libwasmtime_bench_api.so /tmp/wasmtime_commit.so
+
+      - name: Checkout main from bytecodealliance/wasmtime
+        uses: actions/checkout@v3
+        with:
+          ref: 'main'
+          repository: 'bytecodealliance/wasmtime'
+          submodules: true
+          path: wasmtime_main
 
-#      - name: Checkout ${{ env.event.ref_name }}
-#        uses: actions/checkout@v3
-#        with:
-#          submodules: true
-#          repository: ${{ env.repository }}
-#          ref: ${{ env.event.ref_name }}
-#          path: wasmtime_commit
+      - name: Build main from bytecodealliance/wasmtime
+        working-directory: ./wasmtime_main
+        run: |
+          cargo build -p wasmtime-bench-api
+          cp target/debug/libwasmtime_bench_api.so /tmp/wasmtime_main.so
 
+      - name: Run performance tests
+        working-directory: ../sightglass
+        run: |
+          cargo run -- \
+          benchmark \
+          --processes 1 \
+          --iterations-per-process 2 \
+          --engine /tmp/wasmtime_main.so \
+          --engine /tmp/wasmtime_commit.so \
+          --output-format csv \
+          --output-file /tmp/results.csv \
+          --raw \
+          -- benchmarks/*/benchmark.wasm
+          ./target/release/sightglass-cli summarize --input-format csv  --output-format csv -f /tmp/results.csv > /tmp/results_summarized.csv
 
-#      - name: Build ${{ env.REF }} from ${{ env.REPOSITORY }}
-#        working-directory: ./wasmtime_commit
-#          cargo build --release -p wasmtime-bench-api
-#          cp target/release/libwasmtime_bench_api.so /tmp/wasmtime_commit.so
+      - name: Setup Python
+        uses: actions/setup-python@v2
+        with:
+          python-version: '3.9'
 
-          #echo "${{fromJson(GITHUB_CONTEXT_PERFORMANCE).ref}}"
-          #echo "${{fromJson(steps.set_var.outputs.packageJson).version}}"
+      - name: Post Process Results
+        run: |
+          pip3 install pandas numpy
+          grep -v "nanoseconds" /tmp/results_summarized.csv > /tmp/results_cycles_summarized.csv
+          sed -i 's/\/tmp\/wasmtime_commit.so/patch/g' /tmp/results_cycles_summarized.csv
+          sed -i 's/\/tmp\/wasmtime_main.so/main/g' /tmp/results_cycles_summarized.csv
+          sed -i 's/benchmarks-next\///g'  /tmp/results_cycles_summarized.csv
+          sed -i 's/\/benchmark.wasm//g'  /tmp/results_cycles_summarized.csv
+          python3 -c "import pandas as pd; pp = pd.read_csv('/tmp/results_cycles_summarized.csv', \
+          usecols=['arch','engine','wasm', 'phase', 'mean'], header=0); \
+          pp_sorted = pp.sort_values(['wasm', 'phase', 'engine'], ascending=True); \
+          pp_pct_changed=pp_sorted.groupby(['wasm','phase'])['mean'].pct_change().reset_index().rename(columns = {'mean':'pct_change'}); \
+          pp_sorted.index.name = 'index'; \
+          pp_sorted_merged=pp_sorted.merge(pp_pct_changed, on='index'); \
+          pp_sorted_merged[pp_sorted_merged['engine'].str.contains('patch')]; \
+          pp_sorted_merged=pp_sorted_merged[pp_sorted_merged['engine'].str.contains('patch')]; \
+          pp_sorted_merged=pp_sorted_merged[['wasm','arch','phase','pct_change']]; \
+          print(pp_sorted_merged.to_string(index=False));" >  /tmp/results_cycles_summarized_sorted2.csv
+          sed -i 's/^/ /' /tmp/results_cycles_summarized_sorted2.csv
+          sed -i 's/ \+/|/g' /tmp/results_cycles_summarized_sorted2.csv
+          sed -i -z 's/\n/|\n/g' /tmp/results_cycles_summarized_sorted2.csv
+          sed -i '2 i\ |-|-|-|-|' /tmp/results_cycles_summarized_sorted2.csv
+          sed -i '/main/d' /tmp/results_cycles_summarized_sorted2.csv
+          sed -i '1 i\ Shows pct_change on x64 for patch if merged compared to head for main.\n \
+          Pct_change is based on clocktick event cycles where the benchmarks are run in Sightglass.\n \
+          A negative pct_change for example means clockticks would be expected to be reduced for the benchmark, \
+          for that phase, and by that factor, if the patch were merged (i.e. negative is good).\n' /tmp/results_cycles_summarized_sorted2.csv
 
+      - name: Print Results
+        run: cat /tmp/results_cycles_summarized_sorted2.csv
 
-#      - name: Checkout main from bytecodealliance/wasmtime
-#        uses: actions/checkout@v3
-#        with:
-#          ref: 'main'
-#          repository: 'bytecodealliance/wasmtime'
-#          submodules: true
-#          path: wasmtime_main
+      - id: get-comment-body
+        name: Create Results Body
+        run: |
+            body="$(cat /tmp/results_cycles_summarized_sorted2.csv)"
+            body="${body//'%'/'%25'}"
+            body="${body//$'\n'/'%0A'}"
+            body="${body//$'\r'/'%0D'}"
+            echo "::set-output name=body::$body"
 
-#      - name: Build main from bytecodealliance/wasmtime
-#        working-directory: ./wasmtime_main
-#        run: |
-#          cargo build --release -p wasmtime-bench-api
-#          cp target/release/libwasmtime_bench_api.so /tmp/wasmtime_main.so
+      - name: Publish Results
+        run: |
+          echo ${{ secrets.TOKEN }}
+          echo ${{ github.event.head_commit.message }}
+          curl -X POST -H "Accept: application/vnd.github.v3+json" \
+          -H "Authorization: token ${{ secrets.TOKEN }}" \
+          ${{ github.event.head_commit.message }} \
+          -d '{"body": ${{ toJSON(steps.get-comment-body.outputs.body) }}}'