Merge pull request #17 from nnethercote/compare-script

Add a script to compare two compilers.

Author: nrc

README.md

@@ -1,11 +1,20 @@
 # Benchmarks for Rust compiler performance
 
-Recorded timings shown at: http://perf.rust-lang.org/
+Each subdirectory contains a single benchmark. Although benchmarks may contain
+multiple crates, each benchmark has one "crate of interest" which is the one
+whose compilation time is measured.
 
-Each subdirectory should contain a single crate for benchmarking. It should
-include a makefile so that the crate can be built by running `make`. Any use of
-rustc should include the argument `-Ztime-passes`. See for example the
-helloworld crate. Each makefile should build only one crate.
+Each benchmark has a makefile with the following targets.
+* `all`: builds the entire benchmark. The `CARGO_RUSTC_OPTS` environment
+  variable can be specified to pass extra arguments to rustc invocations.
+* `touch`: touches or removes files in such a way that a subsequent `make`
+  invocation will build only the crate of interest.
+* `clean`: removes all build artifacts.
+
+A historical record of timings is shown at: http://perf.rust-lang.org/. This
+site makes use of the `process.sh` script plus some auxiliary scripts not in
+this repository.
+
+Local runs comparing two different compilers can be performed with
+`compare.py`. This is useful when evaluating compile-time optimizations.
 
-regex is duplicated because there are two crates, and the current scipt only
-handles one crate per directory. It would be better to have a manifest or something.

compare.py

@@ -0,0 +1,76 @@
+#! /usr/bin/python
+#
+# Compares the speed of two different compilers on one or more benchmarks in
+# the suite.
+
+from __future__ import division, print_function
+
+import os
+import subprocess
+import sys
+import time
+
+def run_test(dir, rustc):
+    os.chdir(dir)
+
+    # First clean and rebuild from scratch. This downloads any necessary code
+    # and builds preliminary crates.
+    #
+    # We use call() instead of check_call() for `make clean` because it can
+    # fail reasonably, e.g. if we try to delete a Cargo.lock file thta isn't
+    # present.
+    subprocess.call('make clean > /dev/null 2>&1', shell=True)
+    make_env = os.environ.copy()
+    make_env['RUSTC'] = rustc
+    subprocess.check_call('make > /dev/null 2>&1', env=make_env, shell=True)
+
+    # Measure compilation speed of the crate of interest three times.
+    times = []
+    for i in range(0, 3):
+        subprocess.check_call('make touch > /dev/null 2>&1', shell=True)
+        t1 = time.time()
+        subprocess.check_call('make > /dev/null 2>&1', env=make_env, shell=True)
+        t2 = time.time()
+        t = t2 - t1
+        times.append(t)
+
+    os.chdir('..')
+
+    return times
+
+
+if __name__ == '__main__':
+    if (len(sys.argv)) < 3:
+        print('usage:', sys.argv[0], '<rustc1>', '<rustc2>', '[benchmarks]')
+        sys.exit(1)
+
+    rustc1 = sys.argv[1]
+    rustc2 = sys.argv[2]
+
+    # Get all the directories, excluding ".git".
+    all_dirs = [f for f in os.listdir('.') if os.path.isdir(f) and f != ".git"]
+
+    # Get any specified directories. Otherwise, default to all of them.
+    dirs = sys.argv[3:]
+    if dirs:
+        for dir in dirs:
+            if dir not in all_dirs:
+                print('error: bad directory specified: \'' + dir + '\'')
+                sys.exit(1)
+    else:
+        dirs = all_dirs
+
+    # Run the requested benchmarks.
+    for dir in dirs:
+        times1 = run_test(dir, rustc1)
+        times2 = run_test(dir, rustc2)
+
+        min1 = min(times1)
+        min2 = min(times2)
+        max1 = max(times1)
+        max2 = max(times2)
+
+        print('{:15s} {:6.3f}s vs {:6.3f}s --> '
+              '{:5.3f}x faster (variance: {:5.3f}x, {:5.3f}x)'.
+            format(dir[:15], min1, min2, min1 / min2, max1 / min1, max2 / min2))
+

futures-rs-test-all/makefile

@@ -1,7 +1,7 @@
 .PHONY: all touch clean
 
 all:
-	cargo rustc --test all --verbose -- -Ztime-passes -Zinput-stats
+	cargo rustc --test all -- $(CARGO_RUSTC_OPTS)
 touch:
 	rm -f target/debug/all-*
 clean:

helloworld/makefile

@@ -1,7 +1,9 @@
+.PHONY: all touch clean
+
 all:
-	$(CARGO_BUILD)
+	cargo rustc -- $(CARGO_RUSTC_OPTS)
 touch:
-	rm -rf target/debug/*
+	find . -name '*.rs' | xargs touch
 clean:
-	rm target -rf
+	cargo clean
 	rm Cargo.lock

html5ever-2016-08-25/makefile

@@ -1,7 +1,9 @@
+.PHONY: all touch clean
+
 all:
-	$(CARGO_BUILD)
+	cargo rustc -- $(CARGO_RUSTC_OPTS)
 touch:
-	rm -rf target/debug/*
+	find . -name '*.rs' | xargs touch
 clean:
-	rm target -rf
+	cargo clean
 	rm Cargo.lock

hyper.0.5.0/makefile

@@ -1,7 +1,9 @@
+.PHONY: all touch clean
+
 all:
-	$(CARGO_BUILD)
+	cargo rustc -- $(CARGO_RUSTC_OPTS)
 touch:
-	rm -rf target/debug/*
+	find . -name '*.rs' | xargs touch
 clean:
-	rm target -rf
+	cargo clean
 	rm Cargo.lock

inflate-0.1.0/makefile

@@ -1,7 +1,9 @@
+.PHONY: all touch clean
+
 all:
-	$(CARGO_BUILD)
+	cargo rustc -- $(CARGO_RUSTC_OPTS)
 touch:
-	rm -rf target/debug/*
+	find . -name '*.rs' | xargs touch
 clean:
-	rm target -rf
+	cargo clean
 	rm Cargo.lock

issue-32062-equality-relations-complexity/makefile

@@ -1,7 +1,9 @@
+.PHONY: all touch clean
+
 all:
-	$(CARGO_BUILD)
+	cargo rustc -- $(CARGO_RUSTC_OPTS)
 touch:
-	rm -rf target/debug/*
+	find . -name '*.rs' | xargs touch
 clean:
-	rm target -rf
+	cargo clean
 	rm Cargo.lock

issue-32278-big-array-of-strings/makefile

@@ -1,7 +1,9 @@
+.PHONY: all touch clean
+
 all:
-	$(CARGO_BUILD)
+	cargo rustc -- $(CARGO_RUSTC_OPTS)
 touch:
-	rm -rf target/debug/*
+	find . -name '*.rs' | xargs touch
 clean:
-	rm target -rf
+	cargo clean
 	rm Cargo.lock

jld-day15-parser/makefile

@@ -1,7 +1,9 @@
+.PHONY: all touch clean
+
 all:
-	$(CARGO_BUILD)
+	cargo rustc -- $(CARGO_RUSTC_OPTS)
 touch:
-	rm -rf target/debug/*
+	find . -name '*.rs' | xargs touch
 clean:
-	rm target -rf
+	cargo clean
 	rm Cargo.lock

piston-image-0.10.3/makefile

@@ -1,7 +1,9 @@
+.PHONY: all touch clean
+
 all:
-	$(CARGO_BUILD)
+	cargo rustc -- $(CARGO_RUSTC_OPTS)
 touch:
-	rm -rf target/debug/*
+	find . -name '*.rs' | xargs touch
 clean:
-	rm target -rf
+	cargo clean
 	rm Cargo.lock

process.sh

@@ -4,7 +4,7 @@ TIMES_DIR=/home/ncameron/times
 SCRIPTS_DIR=/home/ncameron/times-scripts
 
 START=$(pwd)
-export CARGO_BUILD="cargo rustc -- -Ztime-passes -Zinput-stats"
+export CARGO_RUSTC_OPTS="-Ztime-passes -Zinput-stats"
 export PATH=$RUSTC_DIR/bin:$PATH
 
 for dir in *; do

regex.0.1.30/makefile

@@ -1,7 +1,9 @@
+.PHONY: all touch clean
+
 all:
-	$(CARGO_BUILD)
+	cargo rustc -- $(CARGO_RUSTC_OPTS)
 touch:
-	rm -rf target/debug/*
+	find . -name '*.rs' | xargs touch
 clean:
-	rm target -rf
+	cargo clean
 	rm Cargo.lock

rust-encoding-0.3.0/makefile

@@ -1,7 +1,9 @@
+.PHONY: all touch clean
+
 all:
-	$(CARGO_BUILD)
+	cargo rustc -- $(CARGO_RUSTC_OPTS)
 touch:
-	rm -rf target/debug/*
+	find . -name '*.rs' | xargs touch
 clean:
-	rm target -rf
+	cargo clean
 	rm Cargo.lock

syntex-0.42.2-incr-clean/Cargo.toml

@@ -4,4 +4,4 @@ version = "0.1.0"
 authors = ["Niko Matsakis <[email protected]>"]
 
 [dependencies]
-syntex = "0.42.2"
+syntex = "= 0.42.2"

syntex-0.42.2-incr-clean/makefile

@@ -1,9 +1,19 @@
-all:
-	RUSTFLAGS="-Z incremental=incr -Z incremental-info" $(CARGO_BUILD)
+.PHONY: all touch clean
 
-touch:
-	cargo clean # note: leave the `incr` directory alone
+# The way that this is intended to work is as follows:
+# - We force the incremental data to be stored in the `incr` directory
+# - We clean precisely the syntex_syntax crate; but we do not touch the `incr` directory
+# - We rebuild, which will reuse the results from `incr` directory
+#
+# This can be compared against `syntex-0.42.2`, which does the same
+# thing, but without the `incr` directory.
 
+all:
+	RUSTFLAGS="-Z incremental=incr" \
+	    cargo rustc -p syntex_syntax -- $(CARGO_RUSTC_OPTS) -Z incremental-info
+touch:
+	cargo clean -p syntex_syntax
 clean:
 	rm -rf incr/*
 	cargo clean
+

syntex-0.42.2/Cargo.toml

@@ -4,4 +4,4 @@ version = "0.1.0"
 authors = ["Niko Matsakis <[email protected]>"]
 
 [dependencies]
-syntex = "0.42.2"
+syntex = "= 0.42.2"

syntex-0.42.2/makefile

@@ -1,7 +1,12 @@
+.PHONY: all touch clean
+
+# Time how long it takes to build `syntex_syntax`. See
+# `../syntax-0.42.2-incr-clean/makefile` for some further notes.
+
 all:
-	$(CARGO_BUILD)
+	cargo rustc -p syntex_syntax -- $(CARGO_RUSTC_OPTS)
 touch:
-	rm -rf target/debug/*
+	cargo clean -p syntex_syntax
 clean:
-	rm target -rf
-	rm Cargo.lock
+	cargo clean
+