Merge pull request #21 from conbench/aus/paginate3

paginate over compare runs endpoint

Author: boshek

DESCRIPTION

@@ -1,6 +1,6 @@
 Package: conbenchcoms
 Title: An API wrapper for conbench communications
-Version: 0.0.8
+Version: 0.0.9
 Authors@R: c(
     person("Jonathan", "Keane", , "[email protected]", role = c("aut", "ctb"),
            comment = c(ORCID = "0000-0001-7087-9776")),

NAMESPACE

@@ -2,7 +2,8 @@
 
 export(benchmark_results)
 export(benchmarks)
-export(compare)
+export(compare_results)
+export(compare_runs)
 export(conbench_perform)
 export(conbench_request)
 export(hardware)

NEWS.md

@@ -1,3 +1,8 @@
+# conbenchcoms 0.0.9
+* Remove `compare` function
+* Add `compare_results` function (similar to old `compare` but only works for comparing benchmark results)
+* Add `compare_runs` function, which returns a tibble, paginating until all matching data is returned, which works only with Conbench servers of at least version [ad06af9](https://github.com/conbench/conbench/commit/ad06af9)
+
 # conbenchcoms 0.0.8
 * Remove `simplifyVector` and `flatten` arguments from `history`
 * `history` now always returns a `tibble`

R/compare.R

@@ -1,30 +1,27 @@
-#' Compare runs, benchmarks, batches, or commits
+#' Compare benchmark results
 #'
-#' @param type the type of comparison to make (one of: runs, benchmarks, batches, commits)
-#' @param baseline the baseline sha of the entity to compare
-#' @param contender the contender sha of the entity to compare
-#' @param zscore_threshold the zscore threshold to mark regressions and improvements. 
+#' @param baseline the ID of the baseline result to compare
+#' @param contender the ID of the contender result to compare
+#' @param zscore_threshold the zscore threshold to mark regressions and improvements.
 #'   Default is defined at the Conbench api level.
 #' @param pairwise_percent_threshold the pairwise_percent_threshold to mark regressions and improvements.
 #'   Default is defined at the Conbench api level.
 #' @inheritDotParams httr2::resp_body_json
 #'
 #' @return the JSON response
 #' @export
-compare <- function(type = c("runs", "benchmarks", "batches", "commits"),
-                    baseline,
-                    contender,
-                    zscore_threshold = NULL,
-                    pairwise_percent_threshold = NULL,
-                    ...) {
-  type <- match.arg(type)
+compare_results <- function(baseline,
+                            contender,
+                            zscore_threshold = NULL,
+                            pairwise_percent_threshold = NULL,
+                            ...) {
   stopifnot("zscore_threshold must be numeric" = is.numeric(zscore_threshold) || is.null(zscore_threshold))
   stopifnot("pairwise_percent_threshold must be numeric" = is.numeric(pairwise_percent_threshold) || is.null(pairwise_percent_threshold))
 
   req <- req_url_path_append(
     conbench_request(),
     "compare",
-    type,
+    "benchmark-results",
     paste0(baseline, "...", contender)
   )
 
@@ -40,3 +37,66 @@ compare <- function(type = c("runs", "benchmarks", "batches", "commits"),
 
   resp_body_json(resp, ...)
 }
+
+#' Compare runs
+#'
+#' @param baseline the ID of the baseline run to compare
+#' @param contender the ID of the contender run to compare
+#' @param zscore_threshold the zscore threshold to mark regressions and improvements.
+#'   Default is defined at the Conbench api level.
+#' @param pairwise_percent_threshold the pairwise_percent_threshold to mark regressions and improvements.
+#'   Default is defined at the Conbench api level.
+#'
+#' @return a tibble of run comparisons
+#' @export
+compare_runs <- function(baseline,
+                         contender,
+                         zscore_threshold = NULL,
+                         pairwise_percent_threshold = NULL) {
+  stopifnot("zscore_threshold must be numeric" = is.numeric(zscore_threshold) || is.null(zscore_threshold))
+  stopifnot("pairwise_percent_threshold must be numeric" = is.numeric(pairwise_percent_threshold) || is.null(pairwise_percent_threshold))
+
+  req <- req_url_path_append(
+    conbench_request(),
+    "compare",
+    "runs",
+    paste0(baseline, "...", contender)
+  )
+  req <- req_url_query(req, page_size = 500)
+  if (!is.null(zscore_threshold)) {
+    req <- req_url_query(req, threshold_z = zscore_threshold)
+  }
+  if (!is.null(pairwise_percent_threshold)) {
+    req <- req_url_query(req, threshold = pairwise_percent_threshold)
+  }
+
+  resp <- conbench_perform(req)
+  json <- resp_body_json(resp, simplifyVector = TRUE, flatten = TRUE)
+  data <- dplyr::as_tibble(json[["data"]])
+
+  while (!is.null(json[["metadata"]][["next_page_cursor"]])) {
+    req <- req_url_path_append(
+      conbench_request(),
+      "compare",
+      "runs",
+      paste0(baseline, "...", contender)
+    )
+    req <- req_url_query(
+      req,
+      page_size = 500,
+      cursor = json[["metadata"]][["next_page_cursor"]]
+    )
+    if (!is.null(zscore_threshold)) {
+      req <- req_url_query(req, threshold_z = zscore_threshold)
+    }
+    if (!is.null(pairwise_percent_threshold)) {
+      req <- req_url_query(req, threshold = pairwise_percent_threshold)
+    }
+
+    resp <- conbench_perform(req)
+    json <- resp_body_json(resp, simplifyVector = TRUE, flatten = TRUE)
+    data <- dplyr::bind_rows(data, dplyr::as_tibble(json[["data"]]))
+  }
+
+  data
+}

man/compare.Rd man/compare_results.Rd

@@ -0,0 +1,16 @@
+{
+    "analysis": {
+        "lookback_z_score": {
+            "improvement_indicated": false,
+            "regression_indicated": false,
+            "z_score": -0.5,
+            "z_threshold": 11.1
+        },
+        "pairwise": {
+            "improvement_indicated": false,
+            "percent_change": -26,
+            "percent_threshold": 20.2,
+            "regression_indicated": true
+        }
+    }
+}

man/compare_runs.Rd

@@ -0,0 +1,23 @@
+{
+    "data": [
+        {
+            "analysis": {
+                "lookback_z_score": {
+                    "improvement_indicated": false,
+                    "regression_indicated": false,
+                    "z_score": -0.5,
+                    "z_threshold": 11.1
+                },
+                "pairwise": {
+                    "improvement_indicated": false,
+                    "percent_change": -26,
+                    "percent_threshold": 20.2,
+                    "regression_indicated": true
+                }
+            }
+        }
+    ],
+    "metadata": {
+        "next_page_cursor": "curse"
+    }
+}

tests/testthat/resp-class/localhost/api/compare/runs/10aded...0af-81f373.json

@@ -0,0 +1,23 @@
+{
+    "data": [
+        {
+            "analysis": {
+                "lookback_z_score": {
+                    "improvement_indicated": false,
+                    "regression_indicated": false,
+                    "z_score": -0.5,
+                    "z_threshold": 11.1
+                },
+                "pairwise": {
+                    "improvement_indicated": false,
+                    "percent_change": -26,
+                    "percent_threshold": 20.2,
+                    "regression_indicated": true
+                }
+            }
+        }
+    ],
+    "metadata": {
+        "next_page_cursor": null
+    }
+}

tests/testthat/resp-class/localhost/api/compare/runs/ee1...f00d-dd5b56.json

@@ -12,7 +12,7 @@ with_mock_dir(test_path("logged-in"), {
   })
 })
 
-with_mock_dir(test_path("resp-class"), {
+with_mock_dir(test_path("resp"), {
   run_id_1 <- "5a1ad"
   run_id_2 <- "5eaf00d"
   batch_id_1 <- "abba0123"