add test case for fread N=cols rnorm

.ci/atime/bisect-7907.R

@@ -0,0 +1,27 @@
+## This script is for running the git bisect analysis in PR7907, which
+## revealed two slowdowns in fread().
+pkg.path <- "~/R/data.table"
+Test <- "fread N=cols regression"
+library(data.table)
+if(FALSE){
+  ## atime functions for git bisect were added in this branch:
+  remotes::install_github("tdhock/atime@bisect")
+  ## if you want to start the bisect over from scratch, first remove
+  ## all historical versions of data.table installed by atime:
+  atime::atime_versions_remove("data.table")
+}
+## Commits other than Fast and Slow have been commented in tests.R so
+## to re-run the bisect, you would first have to uncomment them in
+## tests.R. These are the commits visualized on
+## https://tdhock.github.io/2026-10-15-bisect-performance-pr7907
+(bisect.dt <- rowwiseDT(
+  old=,     new=, label=, expected=, stage=,
+  "Fast", "Slow", "PR7406", "?", "discovery",
+  "PR7406", "Slow", "PR7370", "?", "discovery",
+  "Fast", "PR7383", "PR4164", "?", "discovery",
+  "PR7375", "Slow", "PR7370", "PR7370", "final",
+  "Fast", "Parent7370", "PR7375", "PR7375", "final"))
+for(bisect.i in 1:nrow(bisect.dt)){
+  print(bisect.row <- bisect.dt[bisect.i])
+  bisect.row[, atime::bisect(pkg.path, Test, old, new)]
+}

.ci/atime/bisect-animint.R

@@ -0,0 +1,148 @@
+## This script is for visualizing results computed in bisect-7907.R
+library(animint2)
+library(data.table)
+
+Test.dir <- "bisect/fread_N_cols_regression/Fast-Slow"
+(results.wide <- data.table(
+  csv=Sys.glob(file.path(Test.dir, "csv", "*"))
+)[
+, fread(csv), by=csv
+][
+, rank := rank(time)
+][order(rank)][, let(
+  Rank = factor(rank, labels=format(time)),
+  Status = factor(status)
+)][])
+(results.long <- melt(
+  results.wide,
+  measure.vars=measure(variable, expr.name, sep=".")))
+
+details.dt.list <- list()
+dot.dt.list <- list()
+common.dt.list <- list()
+for(result.RData in Sys.glob(file.path(Test.dir, "RData", "*"))){
+  (objs <- load(result.RData))
+  commit <- sub(".RData", "", basename(result.RData))
+  details <- tres.or.status$measurements[, .(
+    commit, expr.name, N, median, min, max
+  )]
+  common.dt.list[[commit]] <- details[, .(
+    versions=.N
+  ), by=N][
+    versions==max(versions)
+  ][.N, data.table(commit, N)]
+  details.dt.list[[commit]] <- details
+  dot.dt.list[[commit]] <- data.table(
+    commit,
+    tpred$prediction)
+}
+(details.dt <- rbindlist(details.dt.list))
+(dot.dt <- rbindlist(dot.dt.list))
+(common.dt <- rbindlist(common.dt.list))
+
+limit.dt <- data.table(seconds=tpred$seconds.limit)
+animint(
+  overview=ggplot()+
+    ggtitle("Overview, select commit")+
+    theme_bw()+
+    theme_animint(height=600)+
+    geom_tallrect(aes(
+      xmin=rank-0.5, xmax=rank+0.5,
+      key=rank,
+      fill=Status),
+      color=NA,
+      data=results.wide)+
+    geom_line(aes(
+      rank, value, color=expr.name, group=expr.name),
+      data=results.long)+
+    facet_grid(variable ~ ., scales="free")+
+    theme(axis.text.x=element_text(angle=30, hjust=1))+
+    geom_label_aligned(aes(
+      rank, value, label=expr.name, color=expr.name),
+      data=results.long[rank==max(rank)],
+      hjust=0,
+      alignment="vertical")+
+    scale_fill_manual(values=c(
+      "0"="grey",
+      "1"="white"))+
+    scale_y_continuous("")+
+    scale_x_continuous("commit rank over time")+
+    geom_tallrect(aes(
+      xmin=rank-0.5, xmax=rank+0.5),
+      clickSelects="commit",
+      alpha=1,
+      alpha_off=0,
+      color="black",
+      color_off=NA,
+      fill=NA,
+      data=results.wide),
+  details=ggplot()+
+    ggtitle("Details for selected commit")+
+    theme_bw()+
+    theme(legend.position="none")+
+    theme_animint(width=600, height=600, last_in_row=TRUE)+
+    geom_hline(aes(
+      yintercept=seconds),
+      color="grey",
+      data=limit.dt)+
+    geom_text(aes(
+      0, seconds*1.1,
+      label=sprintf(
+        "time limit = %s seconds",
+        paste(seconds)
+      )),
+      hjust=0,
+      color="grey50",
+      data=limit.dt)+
+    geom_vline(aes(
+      xintercept=N,
+      key=1),
+      data=common.dt,
+      color="grey",
+      showSelected="commit")+
+    geom_text(aes(
+      N, 0, label=sprintf("largest common N=%s", paste(N)),
+      key=1),
+      data=common.dt,
+      hjust=1,
+      color="grey50",
+      showSelected="commit")+
+    geom_text(aes(
+      0, 0, label=commit,
+      key=1),
+      hjust=0,
+      vjust=0,
+      data=common.dt,
+      showSelected="commit")+
+    geom_ribbon(aes(
+      N, ymin=min, ymax=max,
+      key=expr.name, fill=expr.name, group=expr.name),
+      showSelected=c("expr.name","commit"),
+      alpha=0.5,
+      color=NA,
+      data=details.dt)+
+    geom_line(aes(
+      N, median,
+      key=expr.name, color=expr.name, group=expr.name),
+      showSelected=c("expr.name","commit"),
+      data=details.dt)+
+    geom_point(aes(
+      N, unit.value,
+      key=expr.name, color=expr.name),
+      showSelected=c("expr.name","commit"),
+      fill="white",
+      data=dot.dt)+
+    geom_label_aligned(aes(
+      N, unit.value, label=expr.name,
+      key=expr.name, color=expr.name),
+      showSelected=c("expr.name","commit"),
+      alignment="horizontal",
+      vjust=-0.1,
+      alpha=0.5,
+      data=dot.dt)+
+    scale_x_log10()+
+    scale_y_log10("seconds"),
+  duration=list(
+    commit=1000)
+)
+

.ci/atime/tests.R

@@ -1,3 +1,9 @@
+bisect.restore.fun <- function(tinfo){
+  ## necessary to avoid failed checkout when bisecting some old commits
+  ## https://github.com/Rdatatable/data.table/issues/7909
+  cat("", file=file.path(tinfo$checkout.path, ".gitattributes"))
+  gert::git_restore(".", repo=tinfo$checkout.path)
+}
 pval.thresh <- 0.001 # to reduce false positives.
 
 # Test case adapted from https://github.com/Rdatatable/data.table/issues/6105#issue-2268691745 which is where the issue was reported.
@@ -260,6 +266,27 @@ test.list <- atime::atime_test_list(
     }
   },
 
+  "fread N=cols regression" = atime::atime_test(
+    N = unique(as.integer(2^seq(0, 20, by=0.25))), # finer N grid for git bisect.
+    setup={
+      set.seed(1)
+      n.rows <- 100
+      input.vec <- rnorm(n.rows*N)
+      input.mat <- matrix(input.vec, n.rows, N)
+      input.dt <- data.table(input.mat)
+      input.csv <- tempfile()
+      data.table::fwrite(input.dt, input.csv)
+    },
+    seconds.limit=0.1,
+    Fast="1.14.8",
+    ## PR7375="59f966cf6839a0494e42ed1ab98616eea03dcb36", #fread slowdown added comment.char arg
+    ## PR7383="67db7f7fb33b99da8cc5dc575714924b96d0f0fe", #not as slow as Slow, but using this as new in bisect with old=1.14.8 yields PR#7375.
+    ## PR7406="1685a3b47d48f323afeae589545f7cbb7717ec28", #not as fast as Fast, but using this as old in bisect with new=1.18.6 yields PR#7370.
+    ## Parent7370="1685a3b47d48f323afeae589545f7cbb7717ec28",
+    ## PR7370="df7fa8071c6b818d24e47cc1a84a6dc0491550d6", #fread slowdown sep detection single column quoted
+    Slow="1.18.6",
+    expr=data.table::fread(input.csv, showProgress = FALSE, nThread=1)),
+
   # Constant overhead improvement https://github.com/Rdatatable/data.table/pull/6925
   # Test case adapted from https://github.com/Rdatatable/data.table/pull/7022#discussion_r2107900643
   "fread disk overhead improved in #6925" = atime::atime_test(