Add shimmer benchmarks (#5597)

Author: bengl

.gitlab/benchmarks.yml

@@ -65,20 +65,26 @@ benchmark:
         GROUP: 2
       - MAJOR_VERSION: 18
         GROUP: 3
+      - MAJOR_VERSION: 18
+        GROUP: 4
       - MAJOR_VERSION: 20
         GROUP: 1
       - MAJOR_VERSION: 20
         GROUP: 2
       - MAJOR_VERSION: 20
         GROUP: 3
+      - MAJOR_VERSION: 20
+        GROUP: 4
       - MAJOR_VERSION: 22
         GROUP: 1
       - MAJOR_VERSION: 22
         GROUP: 2
       - MAJOR_VERSION: 22
         GROUP: 3
+      - MAJOR_VERSION: 22
+        GROUP: 4
   variables:
-    SPLITS: 3
+    SPLITS: 4
 
 benchmark-serverless:
   stage: benchmarks

benchmark/sirun/README.md

@@ -2,7 +2,29 @@
 
 This directory contains several different types of benchmarks.
 
-Some of these benchmarks rely on [sirun](https://github.com/DataDog/sirun/) for execution.
+These benchmarks rely on [sirun](https://github.com/DataDog/sirun/) for execution.
+
+## Running Benchmarks Locally
+
+First, install sirun:
+
+```sh  
+cargo install --git https://github.com/DataDog/sirun.git --branch main
+```
+
+Then, get into one of the directories alongside this file, and run the following:
+
+```sh  
+node ../run-all-variants.js
+```
+
+You can pipe this to `sirun --summarize` to get a summary of the resulting data. This can also be piped to the `means.js` script in this directory to view it in tabular form.
+
+Putting that all together, the following will run benchmarks, summarize them, and give you nice tabular output.
+
+```sh
+node ../run-all-variants.js | sirun --summarize | node ../means.js
+```
 
 ## Running Benchmarks via Docker
 

benchmark/sirun/means.js

@@ -0,0 +1,29 @@
+/* eslint-disable no-console */
+
+const chunks = []
+process.stdin.on('data', (data) => {
+  chunks.push(data)
+})
+process.stdin.on('end', () => {
+  const data = Buffer.concat(chunks).toString()
+  const json = JSON.parse(data)
+  for (const testSuite in json) {
+    console.log('Test Suite: ', testSuite)
+    const variants = Object.keys(json[testSuite])
+    const results = {}
+    for (const variant of variants) {
+      const { summary } = json[testSuite][variant]
+      results[variant] = {
+        wallTime: summary['wall.time'].mean,
+        userTime: summary['user.time'].mean,
+        sysTime: summary['system.time'].mean,
+        maxResSize: summary['max.res.size'].mean
+      }
+    }
+    const newResults = []
+    Object.keys(results).sort().forEach((name) => {
+      newResults.push({ name, ...results[name] })
+    })
+    console.table(newResults)
+  }
+})

benchmark/sirun/shimmer-runtime/README.md

@@ -0,0 +1,3 @@
+This is a test of shimmer performance at startup, meaning the actual shimming operations of shimmer.wrap and shimmer.wrapFunction.
+
+There are variants for different kinds of functions, and for both types of shimming.

benchmark/sirun/shimmer-runtime/index.js

@@ -0,0 +1,68 @@
+/* eslint-disable require-await */
+
+const shimmer = require('../../../packages/datadog-shimmer')
+
+const {
+  ENABLED,
+  WRAP_FUNCTION,
+  FUNCTION_NAME
+} = process.env
+
+const ITERATIONS = 2e6
+
+let counter = 0
+function declared () {
+  return ++counter // Do very little
+}
+
+const arrow = () => {
+  return ++counter // Do very little
+}
+
+async function asyncDeclared () {
+  return ++counter // Do very little
+}
+
+const asyncArrow = async () => {
+  return ++counter // Do very little
+}
+
+const testedFn = {
+  declared,
+  arrow,
+  asyncDeclared,
+  asyncArrow
+}[FUNCTION_NAME]
+if (!testedFn) {
+  throw new Error(`Function ${FUNCTION_NAME} not found`)
+}
+
+if (ENABLED !== 'true') {
+  for (let i = 0; i < ITERATIONS; i++) {
+    testedFn()
+  }
+} else {
+  if (WRAP_FUNCTION === 'true') {
+    const wrapped = shimmer.wrapFunction(testedFn, (original) => {
+      return function () {
+        return original.apply(this, arguments)
+      }
+    })
+    for (let i = 0; i < ITERATIONS; i++) {
+      wrapped()
+    }
+  } else {
+    const obj = {
+      testedFn
+    }
+    shimmer.wrap(obj, 'testedFn', (original) => {
+      return function () {
+        return original.apply(this, arguments)
+      }
+    })
+    const wrapped = obj.testedFn
+    for (let i = 0; i < ITERATIONS; i++) {
+      wrapped()
+    }
+  }
+}

benchmark/sirun/shimmer-runtime/meta.json

@@ -0,0 +1,90 @@
+{
+  "name": "shimmer-runtime",
+  "run": "node index.js",
+  "run_with_affinity": "bash -c \"taskset -c $CPU_AFFINITY node index.js\"",
+  "cachegrind": false,
+  "iterations": 10,
+  "instructions": true,
+  "variants": {
+    "declared-baseline": {
+      "env": {
+        "ENABLED": "false",
+        "FUNCTION_NAME": "declared"
+      }
+    },
+    "asyncdeclared-baseline": {
+      "env": {
+        "ENABLED": "false",
+        "FUNCTION_NAME": "asyncDeclared"
+      }
+    },
+    "arrow-baseline": {
+      "env": {
+        "ENABLED": "false",
+        "FUNCTION_NAME": "arrow"
+      }
+    },
+    "asyncarrow-baseline": {
+      "env": {
+        "ENABLED": "false",
+        "FUNCTION_NAME": "asyncArrow"
+      }
+    },
+    "declared-wrapfn": {
+      "env": {
+        "ENABLED": "true",
+        "WRAP_FUNCTION": "true",
+        "FUNCTION_NAME": "declared"
+      }
+    },
+    "asyncdeclared-wrapfn": {
+      "env": {
+        "ENABLED": "true",
+        "WRAP_FUNCTION": "true",
+        "FUNCTION_NAME": "asyncDeclared"
+      }
+    },
+    "arrow-wrapfn": {
+      "env": {
+        "ENABLED": "true",
+        "WRAP_FUNCTION": "true",
+        "FUNCTION_NAME": "arrow"
+      }
+    },
+    "asyncarrow-wrapfn": {
+      "env": {
+        "ENABLED": "true",
+        "WRAP_FUNCTION": "true",
+        "FUNCTION_NAME": "asyncArrow"
+      }
+    },
+    "declared-wrap": {
+      "env": {
+        "ENABLED": "true",
+        "WRAP_FUNCTION": "false",
+        "FUNCTION_NAME": "declared"
+      }
+    },
+    "asyncdeclared-wrap": {
+      "env": {
+        "ENABLED": "true",
+        "WRAP_FUNCTION": "false",
+        "FUNCTION_NAME": "asyncDeclared"
+      }
+    },
+    "arrow-wrap": {
+      "env": {
+        "ENABLED": "true",
+        "WRAP_FUNCTION": "false",
+        "FUNCTION_NAME": "arrow"
+      }
+    },
+    "asyncarrow-wrap": {
+      "env": {
+        "ENABLED": "true",
+        "WRAP_FUNCTION": "false",
+        "FUNCTION_NAME": "asyncArrow"
+      }
+    }
+  }
+}

benchmark/sirun/shimmer-startup/README.md

@@ -0,0 +1,3 @@
+This is a test of shimmer performance at runtime, meaning the performance of functions shimmed by shimmer.wrap and shimmer.wrapFunction.
+
+There are variants for different kinds of functions, and for both types of shimming.

benchmark/sirun/shimmer-startup/index.js

@@ -0,0 +1,61 @@
+/* eslint-disable require-await */
+
+const shimmer = require('../../../packages/datadog-shimmer')
+
+const {
+  ENABLED,
+  WRAP_FUNCTION,
+  FUNCTION_NAME
+} = process.env
+
+const ITERATIONS = 2e6
+
+let counter = 0
+function declared () {
+  return ++counter // Do very little
+}
+
+const arrow = () => {
+  return ++counter // Do very little
+}
+
+async function asyncDeclared () {
+  return ++counter // Do very little
+}
+
+const asyncArrow = async () => {
+  return ++counter // Do very little
+}
+
+const testedFn = {
+  declared,
+  arrow,
+  asyncDeclared,
+  asyncArrow
+}[FUNCTION_NAME]
+if (!testedFn) {
+  throw new Error(`Function ${FUNCTION_NAME} not found`)
+}
+
+if (ENABLED === 'true') {
+  if (WRAP_FUNCTION === 'true') {
+    for (let i = 0; i < ITERATIONS; i++) {
+      shimmer.wrapFunction(testedFn, (original) => {
+        return function () {
+          return original.apply(this, arguments)
+        }
+      })
+    }
+  } else {
+    const obj = {
+      testedFn
+    }
+    for (let i = 0; i < ITERATIONS; i++) {
+      shimmer.wrap(obj, 'testedFn', (original) => {
+        return function () {
+          return original.apply(this, arguments)
+        }
+      })
+    }
+  }
+}