Use NUMERIC for --convert-numbers implementation in SQLiteWriter (#75)

* Use NUMERIC for --convert-numbers implementation in SQLiteWriter

* Add fritz test

* Fix 74 test

Author: eatonphil

main.go

@@ -582,8 +582,10 @@ func _main() error {
 		connector.DatabaseConnectorInfo.Database.Database = cachedPath
 	}
 
+	justDumpResults := lastNonFlagArg == "" && !args.isInteractive
+
 	// Check if we can use direct SQLite writer
-	useSQLiteWriter := !args.noSQLiteWriter && !args.convertNumbers && !args.schema
+	useSQLiteWriter := !args.noSQLiteWriter && !args.schema && !justDumpResults
 	if useSQLiteWriter && !args.cacheSettings.Enabled {
 		tmp, err := ioutil.TempFile("", "dsq-sqlite-shared")
 		if err != nil {
@@ -624,14 +626,22 @@ func _main() error {
 		for i, file := range files {
 			panelId := uuid.New().String()
 
+			convertNumbers := args.convertNumbers
+
 			var w *runner.ResultWriter
 			if useSQLiteWriter {
 				tableName := fmt.Sprintf("t_%d", i)
-				sw, err := openSQLiteResultItemWriter(connector.DatabaseConnectorInfo.Database.Database, tableName)
+				sw, err := openSQLiteResultItemWriter(connector.DatabaseConnectorInfo.Database.Database, tableName, convertNumbers)
 				if err != nil {
 					return err
 				}
 
+				// Prevent DataStation/runner from
+				// doing conversion in Go. SQLite will
+				// handle this as part of the NUMERIC
+				// field types.
+				convertNumbers = false
+
 				w = runner.NewResultWriter(sw)
 			} else {
 				// Use JSONWriter
@@ -641,7 +651,7 @@ func _main() error {
 				}
 			}
 
-			panel, err := importFile(project.Id, panelId, file, mimetypeOverride[file], args.convertNumbers, w, !useSQLiteWriter)
+			panel, err := importFile(project.Id, panelId, file, mimetypeOverride[file], convertNumbers, w, !useSQLiteWriter)
 			if err != nil {
 				return err
 			}
@@ -655,7 +665,7 @@ func _main() error {
 	}
 
 	// No query, just dump transformed file directly out
-	if lastNonFlagArg == "" && !args.isInteractive {
+	if justDumpResults {
 		resultFile := ec.GetPanelResultsFile(project.Id, project.Pages[0].Panels[0].Id)
 		return dumpJSONFile(resultFile, args.pretty, args.schema)
 	}

scripts/test.py

@@ -325,6 +325,52 @@ def test(name, to_run, want, fail=False, sort=False, winSkip=False, within_secon
 want = '[{"host":"com","count":2}]'
 test("URL functions", to_run, want=want, sort=True)
 
+# Number conversion
+to_run = """./dsq testdata/convert.csv 'SELECT * FROM {}'"""
+want = """[{"test":"1"},
+{"test":"1.1"},
+{"test":"+1"},
+{"test":"01"},
+{"test":"001"},
+{"test":"0001.1"}]"""
+test("No number conversion, with query", to_run, want=want, sort=True)
+
+to_run = """./dsq --convert-numbers testdata/convert.csv 'SELECT * FROM {}'"""
+want = """[{"test":1},
+{"test":1.1},
+{"test":1},
+{"test":1},
+{"test":1},
+{"test":1.1}]"""
+test("Number conversion, with query", to_run, want=want, sort=True)
+
+to_run = """./dsq testdata/convert.csv"""
+want = """[{"test":"1"},
+{"test":"1.1"},
+{"test":"+1"},
+{"test":"01"},
+{"test":"001"},
+{"test":"0001.1"}]"""
+test("No number conversion, no query", to_run, want=want, sort=True)
+
+to_run = """./dsq --convert-numbers testdata/convert.csv"""
+want = """[{"test":1},
+{"test":1.1},
+{"test":1},
+{"test":1},
+{"test":1},
+{"test":1.1}]"""
+test("Number conversion, no query", to_run, want=want, sort=True)
+
+to_run = """./dsq testdata/csv/numberconvert.csv 'select * from {} where score > "90"'"""
+want = """[{"Score": "95", "Name": "Rainer"}]"""
+test("No number conversion, does alphabet ordering", to_run, want=want, sort=True)
+
+to_run = """./dsq --convert-numbers testdata/csv/numberconvert.csv 'select * from {} where score > "90"'"""
+want = """[{"Name":"Rainer","Score":95},
+{"Name":"Fountainer","Score":100}]"""
+test("Number conversion, number ordering", to_run, want=want, sort=True)
+
 # END OF TESTS
 
 # START OF REGRESSION TESTS
@@ -341,6 +387,11 @@ def test(name, to_run, want, fail=False, sort=False, winSkip=False, within_secon
 want = '[{"count": 1}]'
 test("https://github.com/multiprocessio/dsq/issues/67", to_run, want, sort=True)
 
+to_run = """./dsq ./testdata/regr/74.csv 'SELECT * FROM {}'"""
+want = '[{"a": "1", "a b": "2"}]'
+test("https://github.com/multiprocessio/dsq/issues/74", to_run, want, sort=True)
+
+
 # END OF REGRESSION TESTS
 
 print(f"{tests - failures} of {tests} succeeded.")

sqlite.go

@@ -10,15 +10,17 @@ import (
 )
 
 type SQLiteResultItemWriter struct {
-	db        *sql.DB
-	fields    []string
-	panelId   string
-	rowBuffer runner.Vector[any]
+	db             *sql.DB
+	fields         []string
+	panelId        string
+	rowBuffer      runner.Vector[any]
+	convertNumbers bool
 }
 
-func openSQLiteResultItemWriter(f string, panelId string) (runner.ResultItemWriter, error) {
+func openSQLiteResultItemWriter(f string, panelId string, convertNumbers bool) (runner.ResultItemWriter, error) {
 	var sw SQLiteResultItemWriter
 	sw.panelId = panelId
+	sw.convertNumbers = convertNumbers
 
 	sw.rowBuffer = runner.Vector[any]{}
 
@@ -32,11 +34,17 @@ func openSQLiteResultItemWriter(f string, panelId string) (runner.ResultItemWrit
 }
 
 func (sw *SQLiteResultItemWriter) createTable() error {
+	fieldType := "TEXT"
+	if sw.convertNumbers {
+		fieldType = "NUMERIC"
+	}
+
 	var columns []string
 	for _, field := range sw.fields {
-		columns = append(columns, field+" TEXT")
+		columns = append(columns, `"`+field+`" `+fieldType)
 	}
-	_, err := sw.db.Exec("CREATE TABLE \"" + sw.panelId + "\"(" + strings.Join(columns, ", ") + ");")
+	create := "CREATE TABLE \"" + sw.panelId + "\"(" + strings.Join(columns, ", ") + ");"
+	_, err := sw.db.Exec(create)
 	return err
 }
 
@@ -166,5 +174,6 @@ func (sw *SQLiteResultItemWriter) Close() error {
 			return err
 		}
 	}
+
 	return sw.db.Close()
 }

testdata/convert.csv

@@ -0,0 +1,7 @@
+test
+1
+1.1
++1
+01
+001
+0001.1

testdata/csv/numberconvert.csv

@@ -0,0 +1,4 @@
+Name,Score
+Fritz,90
+Rainer,95
+Fountainer,100

testdata/regr/74.csv

@@ -0,0 +1,2 @@
+"a","a b"
+1,2