Optionally convert numbers in CSV files (#65)

* Bump datastation/runner for new `ContentTypeInfo.ConvertNumbers` flag

* Expose `convertNumbers` flag as command line argument

* Add documentation for the `--convert-numbers` flag

Author: fritzgrabo

README.md

@@ -481,6 +481,47 @@ dsq> SELECT * FROM {} WHERE NAME = 'Kevin';
 (0 rows)
 ```
 
+### Converting numbers in CSV and TSV files
+
+CSV and TSV files do not allow to specify the type of the individual
+values contained in them. All values are treated as strings by default.
+
+This can lead to unexpected results in queries. Consider the following
+example:
+
+```
+$ cat scores.csv
+name,score
+Fritz,90
+Rainer,95.2
+Fountainer,100
+
+$ dsq scores.csv "SELECT * FROM {} ORDER BY score"
+[{"name":"Fountainer","score":"100"},
+{"name":"Fritz","score":"90"},
+{"name":"Rainer","score":"95.2"}]
+```
+
+Note how the `score` column contains numerical values only. Still,
+sorting by that column yields unexpected results because the values are
+treated as strings, and sorted lexically. (You can tell that the
+individual scores were imported as strings because they're quoted in the
+JSON result.)
+
+Use the `-n` or `--convert-numbers` flag to auto-detect and convert
+numerical values (integers and floats) in imported files:
+
+```
+$ dsq ~/scores.csv --convert-numbers "SELECT * FROM {} ORDER BY score"
+[{"name":"Fritz","score":90},
+{"name":"Rainer","score":95.2},
+{"name":"Fountainer","score":100}]
+```
+
+Note how the scores are imported as numbers now and how the records in
+the result set are sorted by their numerical value. Also note that the
+individual scores are no longer quoted in the JSON result.
+
 ## Supported Data Types
 
 | Name | File Extension(s) | Mime Type | Notes |

go.mod

@@ -8,7 +8,7 @@ go 1.18
 require (
 	github.com/chzyer/readline v1.5.0
 	github.com/google/uuid v1.3.0
-	github.com/multiprocessio/datastation/runner v0.0.0-20220601032709-9bda16b723bb
+	github.com/multiprocessio/datastation/runner v0.0.0-20220609232347-405d8c1a88b2
 	github.com/olekukonko/tablewriter v0.0.5
 )
 

go.sum

@@ -447,6 +447,8 @@ github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826 h1:RWengNIwukTxcDr9
 github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826/go.mod h1:TaXosZuwdSHYgviHp1DAtfrULt5eUgsSMsZf+YrPgl8=
 github.com/multiprocessio/datastation/runner v0.0.0-20220601032709-9bda16b723bb h1:sG23Q6XOfcOtK9bM4QhcmGiqsFVzoXwkZRvb8OJ3EiU=
 github.com/multiprocessio/datastation/runner v0.0.0-20220601032709-9bda16b723bb/go.mod h1:UCms/xK08DspNqDDZ5XsaIlc39AuREmeELspFYghMGI=
+github.com/multiprocessio/datastation/runner v0.0.0-20220609232347-405d8c1a88b2 h1:WWCPwJPWfBVUhuAfFZJGs6vxemeeqW8ahDRtTtbGyxw=
+github.com/multiprocessio/datastation/runner v0.0.0-20220609232347-405d8c1a88b2/go.mod h1:UCms/xK08DspNqDDZ5XsaIlc39AuREmeELspFYghMGI=
 github.com/multiprocessio/go-json v0.0.0-20220308002443-61d497dd7b9e h1:NlPl7amllnQyVAkZgjBvFEkKxJSba/R8ZpaTodc7SIQ=
 github.com/multiprocessio/go-json v0.0.0-20220308002443-61d497dd7b9e/go.mod h1:huI4M/MrI5px/SgmXYi0a2byKikSLgDrnMQuXOqKtw4=
 github.com/multiprocessio/go-openoffice v0.0.0-20220110232726-064f5dda1956 h1:WVofL03Eq+z3LbDOfH5eKzu2U85LFZZngOMBlNaO/H0=

main.go

@@ -40,7 +40,7 @@ func resolveContentType(fileExtensionOrContentType string) runner.MimeType {
 	return runner.GetMimeType("x."+fileExtensionOrContentType, runner.ContentTypeInfo{})
 }
 
-func evalFileInto(file, mimetype string, out *os.File) error {
+func evalFileInto(file, mimetype string, convertNumbers bool, out *os.File) error {
 	if mimetype == "" {
 		mimetype = string(runner.GetMimeType(file, runner.ContentTypeInfo{}))
 	} else {
@@ -55,7 +55,8 @@ func evalFileInto(file, mimetype string, out *os.File) error {
 	defer w.Flush()
 
 	return runner.TransformFile(file, runner.ContentTypeInfo{
-		Type: mimetype,
+		Type:           mimetype,
+		ConvertNumbers: convertNumbers,
 	}, w)
 }
 
@@ -213,7 +214,7 @@ func getFilesContentHash(files []string) (string, error) {
 	return hex.EncodeToString(sha1.Sum(nil)), nil
 }
 
-func importFile(projectId string, file, mimetype string, ec runner.EvalContext) (*runner.PanelInfo, error) {
+func importFile(projectId string, file, mimetype string, convertNumbers bool, ec runner.EvalContext) (*runner.PanelInfo, error) {
 	panelId := uuid.New().String()
 	resultFile := ec.GetPanelResultsFile(projectId, panelId)
 	out, err := openTruncate(resultFile)
@@ -222,7 +223,7 @@ func importFile(projectId string, file, mimetype string, ec runner.EvalContext)
 	}
 	defer out.Close()
 
-	if err := evalFileInto(file, mimetype, out); err != nil {
+	if err := evalFileInto(file, mimetype, convertNumbers, out); err != nil {
 		return nil, err
 	}
 
@@ -333,14 +334,15 @@ func repl(project *runner.ProjectState, ec *runner.EvalContext, args *args, file
 }
 
 type args struct {
-	pipedMimetype string
-	pretty        bool
-	schema        bool
-	sqlFile       string
-	cacheSettings runner.CacheSettings
-	nonFlagArgs   []string
-	dumpCacheFile bool
-	isInteractive bool
+	pipedMimetype  string
+	pretty         bool
+	schema         bool
+	sqlFile        string
+	cacheSettings  runner.CacheSettings
+	nonFlagArgs    []string
+	dumpCacheFile  bool
+	isInteractive  bool
+	convertNumbers bool
 }
 
 func getArgs() (*args, error) {
@@ -415,6 +417,11 @@ func getArgs() (*args, error) {
 			continue
 		}
 
+		if arg == "-n" || arg == "--convert-numbers" {
+			args.convertNumbers = true
+			continue
+		}
+
 		args.nonFlagArgs = append(args.nonFlagArgs, arg)
 	}
 
@@ -558,7 +565,7 @@ func _main() error {
 	// When dumping schema, need to injest even if cache is on.
 	if !args.cacheSettings.CachePresent || !args.cacheSettings.Enabled || lastNonFlagArg == "" {
 		for _, file := range files {
-			panel, err := importFile(project.Id, file, mimetypeOverride[file], ec)
+			panel, err := importFile(project.Id, file, mimetypeOverride[file], args.convertNumbers, ec)
 			if err != nil {
 				return err
 			}