Add in changes to fix csv printing for python

Author: weiya711

Makefile

@@ -10,6 +10,7 @@ benches_name := $(patsubst %.py,%,$(BENCHES))
 benches_name := $(subst /,_,$(benches_name))
 benches_name := $(subst *,_,$(benches_name))
 NUMPY_JSON ?= results/numpy/$(benches_name)benches_$(shell date +%Y_%m_%d_%H%M%S).json
+NUMPY_JSON := $(NUMPY_JSON)
 
 # Taco Specific Flags
 TACO_OUT = results/taco/$(benches_name)benches_$(shell date +%Y_%m_%d_%H%M%S).csv
@@ -24,12 +25,16 @@ export TACO_TENSOR_PATH = data/
 # command above.
 python-bench: results numpy/*.py
 	echo $(benches_name)
-	pytest $(IGNORE_FLAGS) --benchmark-json=$(NUMPY_JSON) $(BENCHFLAGS) $(BENCHES)@
+	-pytest $(IGNORE_FLAGS) --benchmark-json=$(NUMPY_JSON) $(BENCHFLAGS) $(BENCHES) 
 	make convert-csv
 
 .PHONY: convert-csv
 convert-csv:
-	py.test-benchmark compare --csv=$(patsubst %.json,%.csv,$(NUMPY_JSON)) $(NUMPY_JSON)
+	python numpy/converter.py --json_name $(NUMPY_JSON)
+
+.PHONY: convert-csv-all
+convert-csv-all:
+	python numpy/converter.py --all
 
 taco-bench: taco/build/taco-bench
 ifeq ($(BENCHES),"")

data/bench_ufunc_sparse_xor_10_010000_A.txt

@@ -1,8 +1,12 @@
 import pytest
 @pytest.fixture
 def tacoBench(benchmark):
-    def f(func):
+    def f(func, extra_info = None):
         # Take statistics based on 10 rounds.
+        if extra_info is not None:
+            for k, v in extra_info.items():
+                benchmark.extra_info[k] = v
+            print(benchmark)
         benchmark.pedantic(func, rounds=10, iterations=1, warmup_rounds=1)
     return f
 

data/bench_ufunc_sparse_xor_10_010000_B.txt

@@ -0,0 +1,74 @@
+import json
+import csv
+import argparse
+import os
+
+UNWANTED_KEYS = ['group', 'params', 'options']
+FLATTEN_KEYS = ['extra_info', 'stats']
+RESULTS_DIR = 'results/numpy/'
+
+def convert(json_names, csv_names):
+    for json_name, csv_name in zip(json_names, csv_names):
+        print("Converting", json_name)
+        
+        if os.stat(json_name).st_size == 0:
+            print(json_name + " was empty...continuing")
+            continue
+
+        with open(json_name) as json_file:
+            data = json.load(json_file)
+        benchmark_data = data['benchmarks']
+
+        filtered_benchmark_data = [{k:v for k, v in d.items() if k not in UNWANTED_KEYS} for d in benchmark_data]
+
+        flattened_benchmark_data = []
+        for d in filtered_benchmark_data:
+            flattened_data = dict()
+            for k, v in d.items():
+                if k in FLATTEN_KEYS:
+                    for vk, vv in v.items():
+                        flattened_data[vk] = vv
+                else:
+                    flattened_data[k] = v
+            flattened_benchmark_data.append(flattened_data)
+
+        csv_file = open(csv_name, 'w')
+        csv_writer = csv.writer(csv_file)
+
+        count = 0
+        for benchmark in flattened_benchmark_data:
+            if count == 0:
+                header = benchmark.keys() 
+                csv_writer.writerow(header)
+                count += 1
+            csv_writer.writerow(benchmark.values())
+
+        csv_file.close()
+
+parser = argparse.ArgumentParser()
+parser.add_argument('--json_name', type=str, default=None, help="Input JSON file name. --all overrides this flag")
+parser.add_argument('--csv_name', type=str, default=None, help="Output CSV file name")
+parser.add_argument('--all', action='store_true', default=False, help='Convert all files in results/numpy from json to csv')
+
+args = parser.parse_args()
+
+json_name = args.json_name
+if args.all:
+    json_files = [RESULTS_DIR + pos_json for pos_json in os.listdir(RESULTS_DIR) if pos_json.endswith('.json')]
+    print("JSON Files being converted")
+    print(json_files)
+    csv_files = [os.path.splitext(json_file)[0] + '.csv' for json_file in json_files] 
+    
+else:
+    if args.json_name == None:
+        raise ValueError('Set --json_name or pass --all.')
+    else:
+        json_files = [args.json_name]
+        if args.csv_name is None:
+            csv_name = os.path.splitext(json_name)[0] + '.csv'
+        else:
+            csv_name = args.csv_name 
+        csv_files = [csv_name]
+
+convert(json_files, csv_files)
+

data/bench_ufunc_sparse_xor_10_010000_C.txt

@@ -93,15 +93,17 @@ def bench():
 @pytest.mark.parametrize("tensor", FROSTTTensors.getTensors(), ids=FROSTTTensors.getTensorNames())
 @pytest.mark.parametrize("ufunc", [numpy.logical_xor, numpy.ldexp, numpy.right_shift])
 def bench_pydata_frostt_ufunc_sparse(tacoBench, tensor, ufunc):
-    benchmark.extra_info['tensor_str'] = str(tensor)
-    benchmark.extra_info['ufunc_str'] = ufunc.__name__
+
     frTensor = tensor.load().astype('int64')
     shifter = PydataTensorShifter()
     other = shifter.shiftLastMode(frTensor).astype('int64')
     def bench():
         c = ufunc(frTensor, other)
         return c
-    tacoBench(bench)
+    extra_info = dict()
+    extra_info['tensor_str'] = str(tensor)
+    extra_info['ufunc_str'] = ufunc.__name__
+    tacoBench(bench, extra_info)
 
 # Run benchmarks against the SuiteSparse collection.
 SuiteSparseTensors = TensorCollectionSuiteSparse()
@@ -114,7 +116,10 @@ def bench_pydata_suitesparse_ufunc_sparse(tacoBench, tensor, ufunc):
     def bench():
         c = ufunc(ssTensor, other)
         return c
-    tacoBench(bench)
+    extra_info = dict()
+    extra_info['tensor_str'] = str(tensor)
+    extra_info['ufunc_str'] = ufunc.__name__
+    tacoBench(bench, extra_info)
 
 # TODO (rohany): scipy doesn't support these, I forgot. If that's the case,
 #  do we really need to compare against suitesparse?

numpy/conftest.py

@@ -0,0 +1,10 @@
+name,fullname,param,tensor_str,ufunc_str,min,max,mean,stddev,rounds,median,iqr,q1,q3,iqr_outliers,stddev_outliers,outliers,ld15iqr,hd15iqr,ops,total,data,iterations
+bench_pydata_frostt_ufunc_sparse[logical_xor-nips],numpy/ufuncs.py::bench_pydata_frostt_ufunc_sparse[logical_xor-nips],logical_xor-nips,nips,logical_xor,1.822955725,2.4525968160000033,2.0602508429999995,0.19386880478446047,10,2.0453308819999982,0.16968363700000566,1.9467769429999962,2.116460580000002,1,4,4;1,1.822955725,2.4525968160000033,0.4853777895044406,20.602508429999993,"[2.4525968160000033, 2.0615151009999977, 2.029146662999999, 1.9467769429999962, 1.9849041019999945, 2.286069008999995, 2.116460580000002, 2.074517507000003, 1.822955725, 1.8275659840000031]",1
+bench_pydata_frostt_ufunc_sparse[logical_xor-uber-pickups],numpy/ufuncs.py::bench_pydata_frostt_ufunc_sparse[logical_xor-uber-pickups],logical_xor-uber-pickups,uber-pickups,logical_xor,2.317601517,2.731015868,2.4882607672999995,0.147473936691085,10,2.4504938194999966,0.1928200049999873,2.3691089550000015,2.561928959999989,0,4,4;0,2.317601517,2.731015868,0.40188713865592773,24.882607672999995,"[2.317601517, 2.542051298000004, 2.4698561860000012, 2.431131452999992, 2.3691089550000015, 2.4024516449999993, 2.340417983000009, 2.561928959999989, 2.731015868, 2.7170438079999997]",1
+bench_pydata_frostt_ufunc_sparse[logical_xor-chicago-crime],numpy/ufuncs.py::bench_pydata_frostt_ufunc_sparse[logical_xor-chicago-crime],logical_xor-chicago-crime,chicago-crime,logical_xor,5.445818615000007,12.92657782400002,7.553489786099999,2.911133829439281,10,6.185826714000001,2.8649710269999957,5.583289406999995,8.448260433999991,1,2,2;1,5.445818615000007,12.92657782400002,0.13238913777843575,75.53489786099999,"[5.583289406999995, 6.041946718999981, 5.445818615000007, 6.050176551000021, 6.514814454999993, 5.471875188000013, 6.321476876999981, 12.730661790999989, 12.92657782400002, 8.448260433999991]",1
+bench_pydata_frostt_ufunc_sparse[ldexp-nips],numpy/ufuncs.py::bench_pydata_frostt_ufunc_sparse[ldexp-nips],ldexp-nips,nips,ldexp,1.0387709159999758,1.2385625539999978,1.1480890831999944,0.06697271461753504,10,1.1329468484999836,0.08622687099995119,1.1197424960000149,1.205969366999966,0,4,4;0,1.0387709159999758,1.2385625539999978,0.8710125500129003,11.480890831999943,"[1.1302454510000075, 1.2295803659999933, 1.1893897670000229, 1.205969366999966, 1.2385625539999978, 1.1356482459999597, 1.1197424960000149, 1.1275876980000135, 1.0387709159999758, 1.0653939709999918]",1
+bench_pydata_frostt_ufunc_sparse[ldexp-uber-pickups],numpy/ufuncs.py::bench_pydata_frostt_ufunc_sparse[ldexp-uber-pickups],ldexp-uber-pickups,uber-pickups,ldexp,1.5599719740000069,2.5879317320000155,1.7007887874999994,0.31251550398791017,10,1.6063927390000003,0.04187554499998214,1.5909746959999893,1.6328502409999714,1,1,1;1,1.5599719740000069,2.5879317320000155,0.5879624838483951,17.007887874999994,"[1.6074822269999913, 1.5599719740000069, 1.609637848000034, 1.5909746959999893, 1.634472125000002, 1.6053032510000094, 2.5879317320000155, 1.5790253959999632, 1.6002383850000115, 1.6328502409999714]",1
+bench_pydata_frostt_ufunc_sparse[ldexp-chicago-crime],numpy/ufuncs.py::bench_pydata_frostt_ufunc_sparse[ldexp-chicago-crime],ldexp-chicago-crime,chicago-crime,ldexp,2.7888812660000326,3.8013393520000136,2.9617262340000026,0.3089325679343463,10,2.8162597015000017,0.1920973789999607,2.804531273000009,2.99662865199997,1,1,1;1,2.7888812660000326,3.8013393520000136,0.33764092998205153,29.617262340000025,"[2.9710761829999797, 3.0215025900000114, 2.804531273000009, 2.809344662000001, 3.8013393520000136, 2.8158012270000086, 2.791438959000004, 2.816718175999995, 2.99662865199997, 2.7888812660000326]",1
+bench_pydata_frostt_ufunc_sparse[right_shift-nips],numpy/ufuncs.py::bench_pydata_frostt_ufunc_sparse[right_shift-nips],right_shift-nips,nips,right_shift,0.9811653080000156,1.0736368420000417,1.0053329320000046,0.02772759025184652,10,0.9937236615000131,0.025227635000021564,0.9877143429999933,1.0129419780000148,1,1,1;1,0.9811653080000156,1.0736368420000417,0.9946953572988052,10.053329320000046,"[0.9951138800000194, 1.0129419780000148, 0.9850496770000063, 0.9877143429999933, 0.9811653080000156, 1.0241106329999639, 1.0736368420000417, 0.9923334430000068, 0.9901451929999894, 1.0111180229999945]",1
+bench_pydata_frostt_ufunc_sparse[right_shift-uber-pickups],numpy/ufuncs.py::bench_pydata_frostt_ufunc_sparse[right_shift-uber-pickups],right_shift-uber-pickups,uber-pickups,right_shift,1.409366609000017,1.5303829069999892,1.464751917299992,0.03494558979210242,10,1.4620323535000068,0.03385861999993267,1.4548961830000167,1.4887548029999493,0,3,3;0,1.409366609000017,1.5303829069999892,0.6827094664899439,14.647519172999921,"[1.4696743939999806, 1.4559511119999797, 1.5303829069999892, 1.4938150350000114, 1.4206134229999634, 1.4661046080000233, 1.4579600989999904, 1.409366609000017, 1.4887548029999493, 1.4548961830000167]",1
+bench_pydata_frostt_ufunc_sparse[right_shift-chicago-crime],numpy/ufuncs.py::bench_pydata_frostt_ufunc_sparse[right_shift-chicago-crime],right_shift-chicago-crime,chicago-crime,right_shift,2.716730036999934,3.0463827789999414,2.8108573993999926,0.10493311128204733,10,2.7685638090000566,0.09015235400011079,2.737604155999975,2.8277565100000857,1,2,2;1,2.716730036999934,3.0463827789999414,0.35576333406791133,28.108573993999926,"[2.716730036999934, 2.7537595070000407, 3.0463827789999414, 2.8245740309999974, 2.9350721799999064, 2.7295671759999323, 2.7634836920000225, 2.8277565100000857, 2.737604155999975, 2.7736439260000907]",1