Merge pull request #54 from target/sql-var-double

extend JsonUtils for doubles
target · Jan 14, 2021 · 532ab96 · 532ab96
2 parents 01d7f7d + 44c9863
commit 532ab96
Show file tree

Hide file tree

Showing 2 changed files with 12 additions and 3 deletions.
diff --git a/src/main/scala/com/target/data_validator/JsonUtils.scala b/src/main/scala/com/target/data_validator/JsonUtils.scala
@@ -46,6 +46,7 @@ object JsonUtils extends LazyLogging {
       case IntegerType => Json.fromInt(row.getInt(col))
       case NullType => Json.Null
       case BooleanType => Json.fromBoolean(row.getBoolean(col))
+      case DoubleType => Json.fromDoubleOrNull(row.getDouble(col))
       case _: StructType => row2Json(row.getStruct(col))
       case _ =>
         logger.error(s"Unimplemented dataType '${dataType.typeName}' in column: ${row.schema(col).name} " +

diff --git a/src/test/scala/com/target/data_validator/JsonUtilsSpec.scala b/src/test/scala/com/target/data_validator/JsonUtilsSpec.scala
@@ -73,18 +73,20 @@ class JsonUtilsSpec extends FunSpec with Matchers with TestingSparkSession {
       val TEST_LONG = Random.nextLong
       val TEST_INT = Random.nextInt
       val TEST_BOOLEAN = Random.nextBoolean
+      val TEST_DOUBLE = Random.nextDouble
 
       val schema = StructType(
         List(
           StructField("string", StringType),
           StructField("long", LongType),
           StructField("int", IntegerType),
           StructField("null", NullType),
-          StructField("bool", BooleanType)
+          StructField("bool", BooleanType),
+          StructField("double", DoubleType)
         )
       )
 
-      val sampleData = List(Row(TEST_STRING, TEST_LONG, TEST_INT, null, TEST_BOOLEAN)) // scalastyle:ignore
+      val sampleData = List(Row(TEST_STRING, TEST_LONG, TEST_INT, null, TEST_BOOLEAN, TEST_DOUBLE)) // scalastyle:ignore
 
       def mkRow: Row = spark.createDataFrame(sc.parallelize(sampleData), schema).head()
 
@@ -113,14 +115,20 @@ class JsonUtilsSpec extends FunSpec with Matchers with TestingSparkSession {
         assert(row2Json(sut, 4) == Json.fromBoolean(TEST_BOOLEAN)) // scalastyle:ignore
       }
 
+      it("Row with double") {
+        val sut = mkRow
+        assert(row2Json(sut, 5) == Json.fromDoubleOrNull(TEST_DOUBLE)) // scalastyle:ignore
+      }
+
       it("Full Row") {
         val sut = mkRow
         assert(row2Json(sut) == Json.obj(
           ("string", Json.fromString(TEST_STRING)),
           ("long", Json.fromLong(TEST_LONG)),
           ("int", Json.fromInt(TEST_INT)),
           ("null", Json.Null),
-          ("bool", Json.fromBoolean(TEST_BOOLEAN))
+          ("bool", Json.fromBoolean(TEST_BOOLEAN)),
+          ("double", Json.fromDoubleOrNull(TEST_DOUBLE))
         ))
       }