Feat/yaml support (#82)

* update readme

* adds test and updates runner

* include parquet fix

Author: Sajuno

README.md

@@ -574,6 +574,7 @@ your environment. Turning this on disables some optimizations.
 | ORC | `orc` | `orc` ||
 | Parquet | `parquet` | `parquet` ||
 | Avro | `avro` || `application/avro` || 
+| YAML | `yaml`, `yml` | `application/yaml` || 
 | Excel | `xlsx`, `xls` | `application/vnd.ms-excel` | If you have multiple sheets, you must [specify a sheet path](https://github.com/multiprocessio/dsq#multiple-excel-sheets). |
 | ODS | `ods` |`application/vnd.oasis.opendocument.spreadsheet` | If you have multiple sheets, you must [specify a sheet path](https://github.com/multiprocessio/dsq#multiple-excel-sheets). |
 | Apache Error Logs | NA | `text/apache2error` | Currently only works if being piped in. |
@@ -591,7 +592,7 @@ kinds of SQL queries on arbitrary (structured) data.
 
 | Name | Link | Caching | Engine | Supported File Types | Binary Size |
 |-|---|-|-|------------------------------------------------------------------------|-|
-| dsq | Here | Yes | SQLite | CSV, TSV, a few variations of JSON, Parquet, Excel, ODS (OpenOffice Calc), ORC, Avro, Logs | 49M |
+| dsq | Here | Yes | SQLite | CSV, TSV, a few variations of JSON, Parquet, Excel, ODS (OpenOffice Calc), ORC, Avro, YAML, Logs | 49M |
 | q | http://harelba.github.io/q/ | Yes | SQLite | CSV, TSV | 82M |
 | textql | https://github.com/dinedal/textql | No | SQLite | CSV, TSV | 7.3M |
 | octoql | https://github.com/cube2222/octosql | No | Custom engine | JSON, CSV, Excel, Parquet | 18M |

go.mod

@@ -8,7 +8,7 @@ go 1.18
 require (
 	github.com/chzyer/readline v1.5.0
 	github.com/google/uuid v1.3.0
-	github.com/multiprocessio/datastation/runner v0.0.0-20220616090857-d42388371862
+	github.com/multiprocessio/datastation/runner v0.0.0-20220629131342-6165f9d14a67
 	github.com/olekukonko/tablewriter v0.0.5
 )
 
@@ -20,6 +20,7 @@ require (
 	github.com/Azure/azure-pipeline-go v0.2.3 // indirect
 	github.com/Azure/azure-storage-blob-go v0.14.0 // indirect
 	github.com/ClickHouse/clickhouse-go/v2 v2.0.14 // indirect
+	github.com/alexbrainman/odbc v0.0.0-20211220213544-9c9a2e61c5e2 // indirect
 	github.com/apache/arrow/go/arrow v0.0.0-20211112161151-bc219186db40 // indirect
 	github.com/apache/thrift v0.15.0 // indirect
 	github.com/araddon/dateparse v0.0.0-20210429162001-6b43995a97de // indirect
@@ -114,4 +115,5 @@ require (
 	google.golang.org/protobuf v1.28.0 // indirect
 	gopkg.in/inf.v0 v0.9.1 // indirect
 	gopkg.in/yaml.v2 v2.4.0 // indirect
+	gopkg.in/yaml.v3 v3.0.1 // indirect
 )

go.sum

@@ -90,6 +90,8 @@ github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuy
 github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
 github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
 github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho=
+github.com/alexbrainman/odbc v0.0.0-20211220213544-9c9a2e61c5e2 h1:090cWAt7zsbdvRegKCBVwcCTghjxhUh1PK2KNSq82vw=
+github.com/alexbrainman/odbc v0.0.0-20211220213544-9c9a2e61c5e2/go.mod h1:c5eyz5amZqTKvY3ipqerFO/74a/8CYmXOahSr40c+Ww=
 github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY=
 github.com/apache/arrow/go/arrow v0.0.0-20200730104253-651201b0f516/go.mod h1:QNYViu/X0HXDHw7m3KXzWSVXIbfUvJqBFe6Gj8/pYA0=
 github.com/apache/arrow/go/arrow v0.0.0-20211112161151-bc219186db40 h1:q4dksr6ICHXqG5hm0ZW5IHyeEJXoIJSOZeBLmWPNeIQ=
@@ -237,6 +239,8 @@ github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbV
 github.com/go-logr/logr v1.2.3/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
 github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE=
 github.com/go-ole/go-ole v1.2.4/go.mod h1:XCwSNxSkXRo4vlyPy93sltvi/qJq0jqQhjqQNIwKuxM=
+github.com/go-ole/go-ole v1.2.5 h1:t4MGB5xEDZvXI+0rMjjsfBsD7yAgp/s9ZDkL1JndXwY=
+github.com/go-ole/go-ole v1.2.5/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0=
 github.com/go-openapi/jsonpointer v0.19.5/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg=
 github.com/go-openapi/swag v0.19.5/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk=
 github.com/go-sql-driver/mysql v1.4.0/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w=
@@ -445,8 +449,8 @@ github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjY
 github.com/modocache/gover v0.0.0-20171022184752-b58185e213c5/go.mod h1:caMODM3PzxT8aQXRPkAt8xlV/e7d7w8GM5g0fa5F0D8=
 github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826 h1:RWengNIwukTxcDr9M+97sNutRR1RKhG96O6jWumTTnw=
 github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826/go.mod h1:TaXosZuwdSHYgviHp1DAtfrULt5eUgsSMsZf+YrPgl8=
-github.com/multiprocessio/datastation/runner v0.0.0-20220616090857-d42388371862 h1:bgzazkNqaktd7lQqOcG6TXG1iz8S1GeGbwEcy0/Zh9k=
-github.com/multiprocessio/datastation/runner v0.0.0-20220616090857-d42388371862/go.mod h1:UCms/xK08DspNqDDZ5XsaIlc39AuREmeELspFYghMGI=
+github.com/multiprocessio/datastation/runner v0.0.0-20220629131342-6165f9d14a67 h1:MXPCwo4cL9tzW/gvUxH1eDS1ycr2frlQJmY6ssPGnOU=
+github.com/multiprocessio/datastation/runner v0.0.0-20220629131342-6165f9d14a67/go.mod h1:rc+sDVOazxslD0J+vqCLim1dnAcK+G6Yel/OkubUmkw=
 github.com/multiprocessio/go-json v0.0.0-20220308002443-61d497dd7b9e h1:NlPl7amllnQyVAkZgjBvFEkKxJSba/R8ZpaTodc7SIQ=
 github.com/multiprocessio/go-json v0.0.0-20220308002443-61d497dd7b9e/go.mod h1:huI4M/MrI5px/SgmXYi0a2byKikSLgDrnMQuXOqKtw4=
 github.com/multiprocessio/go-openoffice v0.0.0-20220110232726-064f5dda1956 h1:WVofL03Eq+z3LbDOfH5eKzu2U85LFZZngOMBlNaO/H0=
@@ -750,6 +754,7 @@ golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
@@ -1092,8 +1097,9 @@ gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
 gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
 gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
-gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b h1:h8qDotaEPuJATrMmW04NCwg7v22aHH28wwpauUhK9Oo=
 gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
+gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
 honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
 honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=

scripts/test.py

@@ -184,6 +184,11 @@ def test(name, to_run, want, fail=False, sort=False, winSkip=False, within_secon
 want = '[{"COUNT(*)":25}]'
 test("Supports Avro files", to_run, want, sort=True)
 
+# YAML support
+to_run = """./dsq ./testdata/yaml/userdata.yaml 'SELECT COUNT(*) FROM {} WHERE activated=false'"""
+want = '[{"COUNT(*)":3}]'
+test("Supports YAML files", to_run, want, sort=True)
+
 # Version test
 to_run = """./dsq -v"""
 want_stderr = "dsq latest\n"

testdata/yaml/userdata.yaml

@@ -0,0 +1,86 @@
+- " Name ": Michelle Yost
+  'Phone Number ': "(804) 896-6245 x269"
+  Email: [email protected]
+  Street: '081 Senger Streets'
+  "    City ": North Jerel
+  State: Nebraska
+  'Zip Code ': 67681-6032
+  'Routing Number   ': '058655724'
+  Department: Garden
+  "Company\t": Stiedemann Inc
+  'Created At ': '2021-10-18T04:23:27.810Z'
+  Profile Photo: http://placeimg.com/640/480
+  "  Description": Adipisci animi nulla consectetur quia qui amet at. Hic earum nisi
+    quam deleniti cupiditate impedit iusto. Autem et quasi amet et ut molestiae. Porro
+    ipsam ducimus perspiciatis cumque et voluptatem sit corrupti voluptas. Ad facere
+    unde animi numquam dolorum delectus neque consequatur. Sit cupiditate aperiam
+    vel commodi quo aut libero.
+  Activated: true
+- " Name ": Guadalupe Schimmel II
+  'Phone Number ': 203.466.9566 x3281
+  Email: [email protected]
+  Street: 514 Evangeline Divide
+  "    City ": East Brenden
+  State: Ohio
+  'Zip Code ': 77642-5209
+  'Routing Number   ': '452226335'
+  Department: Garden
+  "Company\t": Hammes, Medhurst and Hilpert
+  'Created At ': '2021-10-21T07:36:16.072Z'
+  Profile Photo: http://placeimg.com/640/480
+  "  Description": Natus illo non ratione possimus est quidem ea eius. Et accusamus
+    recusandae cum quas eum perferendis quo modi dolorem. Doloremque quos fuga omnis
+    placeat assumenda. Omnis quis consequatur.
+  Activated: false
+- " Name ": Corey Beier
+  'Phone Number ': 1-604-307-7955 x87013
+  Email: [email protected]
+  Street: 77080 Kale Plaza
+  "    City ": Pico Rivera
+  State: West Virginia
+  'Zip Code ': '93612'
+  'Routing Number   ': '506297760'
+  Department: Industrial
+  "Company\t": Leffler - Prosacco
+  'Created At ': '2021-05-15T23:53:08.950Z'
+  Profile Photo: http://placeimg.com/640/480
+  "  Description": Et laborum voluptas officia recusandae et. Quibusdam voluptatem
+    aspernatur accusamus in sed vero. Est est tempore sit amet cupiditate quasi. Modi
+    id quae deleniti. Praesentium rerum corrupti architecto harum ut et ipsa velit.
+    Delectus quis nobis officia et at veniam assumenda.
+  Activated: false
+- " Name ": Margie Greenholt
+  'Phone Number ': 962.314.6602
+  Email: [email protected]
+  Street: 7666 Larue Manor
+  "    City ": Martinaton
+  State: Iowa
+  'Zip Code ': '29526'
+  'Routing Number   ': '671746379'
+  Department: Electronics
+  "Company\t": Jones Inc
+  'Created At ': '2021-11-13T16:43:12.607Z'
+  Profile Photo: http://placeimg.com/640/480
+  "  Description": Sit quia in et harum architecto aut qui consectetur. Tempora dolor
+    earum omnis mollitia magnam quis. Dolor qui dignissimos pariatur minima veniam
+    illo eligendi. Et expedita consectetur facilis iure. Nostrum sit natus autem tempore.
+  Activated: true
+- " Name ": Delores Bahringer
+  'Phone Number ': 567.433.0968
+  Email: [email protected]
+  Street: 293 Deja Hills
+  "    City ": Lake Justus
+  State: Connecticut
+  'Zip Code ': 50214-4237
+  'Routing Number   ': '009476813'
+  Department: Grocery
+  "Company\t": Gaylord, Hegmann and Stehr
+  'Created At ': '2021-04-18T04:34:02.883Z'
+  Profile Photo: http://placeimg.com/640/480
+  "  Description": Aperiam repellat ut deleniti minus sint rerum. Quaerat eveniet
+    tempora eius et quis libero. Quis omnis et aut suscipit ducimus corrupti. Dolorum
+    dolore eos unde reprehenderit dicta magni blanditiis voluptate. Velit neque iste
+    ipsam nobis dolore vel corrupti. Non suscipit et nisi rem pariatur id aut illo
+    facilis.
+  Activated: false
+