introduces FastDoubleParser and the useFastDoubleParser parameter in ParserOptions, plus tests

Author: Jolanrensen

core/build.gradle.kts

@@ -71,6 +71,7 @@ dependencies {
     implementation(libs.commonsIo)
     implementation(libs.serialization.core)
     implementation(libs.serialization.json)
+    implementation(libs.fastDoubleParser)
 
     implementation(libs.fuel)
 

core/src/main/kotlin/org/jetbrains/kotlinx/dataframe/api/convert.kt

@@ -185,21 +185,32 @@ public fun <T : Any> DataColumn<T>.convertToDouble(): DataColumn<Double> = conve
 public fun <T : Any> DataColumn<T?>.convertToDouble(): DataColumn<Double?> = convertTo()
 
 /**
- * Parse String column to Double considering locale (number format).
+ * Parses a String column to Double considering locale (number format).
  * If [locale] parameter is defined, it's number format is used for parsing.
- * If [locale] parameter is null, the current system locale is used. If column can not be parsed, then POSIX format is used.
+ * If [locale] parameter is null, the current system locale is used.
+ * If the column cannot be parsed, then the POSIX format is used.
+ *
+ * @param useFastDoubleParser whether to use the new _experimental_ FastDoubleParser, defaults to `false` for now.
  */
 @JvmName("convertToDoubleFromString")
-public fun DataColumn<String>.convertToDouble(locale: Locale? = null): DataColumn<Double> =
-    this.castToNullable().convertToDouble(locale).castToNotNullable()
+public fun DataColumn<String>.convertToDouble(
+    locale: Locale? = null,
+    useFastDoubleParser: Boolean = false,
+): DataColumn<Double> = this.castToNullable().convertToDouble(locale, useFastDoubleParser).castToNotNullable()
 
 /**
- * Parse String column to Double considering locale (number format).
+ * Parses a String column to Double considering locale (number format).
  * If [locale] parameter is defined, it's number format is used for parsing.
- * If [locale] parameter is null, the current system locale is used. If column can not be parsed, then POSIX format is used.
+ * If [locale] parameter is null, the current system locale is used.
+ * If the column cannot be parsed, then the POSIX format is used.
+ *
+ * @param useFastDoubleParser whether to use the new _experimental_ FastDoubleParser, defaults to `false` for now.
  */
 @JvmName("convertToDoubleFromStringNullable")
-public fun DataColumn<String?>.convertToDouble(locale: Locale? = null): DataColumn<Double?> {
+public fun DataColumn<String?>.convertToDouble(
+    locale: Locale? = null,
+    useFastDoubleParser: Boolean = false,
+): DataColumn<Double?> {
     fun applyParser(parser: (String) -> Double?): DataColumn<Double?> {
         var currentRow = 0
         try {
@@ -220,14 +231,14 @@ public fun DataColumn<String?>.convertToDouble(locale: Locale? = null): DataColu
     }
 
     return if (locale != null) {
-        val explicitParser = Parsers.getDoubleParser(locale)
+        val explicitParser = Parsers.getDoubleParser(locale, useFastDoubleParser)
         applyParser(explicitParser)
     } else {
         try {
-            val defaultParser = Parsers.getDoubleParser()
+            val defaultParser = Parsers.getDoubleParser(useFastDoubleParser = useFastDoubleParser)
             applyParser(defaultParser)
         } catch (e: TypeConversionException) {
-            val posixParser = Parsers.getDoubleParser(Locale.forLanguageTag("C.UTF-8"))
+            val posixParser = Parsers.getDoubleParser(Locale.forLanguageTag("C.UTF-8"), useFastDoubleParser)
             applyParser(posixParser)
         }
     }

core/src/main/kotlin/org/jetbrains/kotlinx/dataframe/api/parse.kt

@@ -40,12 +40,29 @@ public interface GlobalParserOptions {
     public var locale: Locale
 }
 
+/**
+ * ### Options for parsing [String]`?` columns
+ *
+ * @param locale locale to use for parsing dates and numbers, defaults to the System default locale.
+ *   If specified instead of [dateTimeFormatter], it will be used in combination with [dateTimePattern]
+ *   to create a [DateTimeFormatter]. Just providing [locale] will not allow you to parse
+ *   locale-specific dates!
+ * @param dateTimeFormatter a [DateTimeFormatter] to use for parsing dates, if not specified, it will be created
+ *   from [dateTimePattern] and [locale]. If neither [dateTimeFormatter] nor [dateTimePattern] are specified,
+ *   [DateTimeFormatter.ISO_LOCAL_DATE_TIME] will be used.
+ * @param dateTimePattern a pattern to use for parsing dates. If specified instead of [dateTimeFormatter],
+ *   it will be used to create a [DateTimeFormatter].
+ * @param nullStrings a set of strings that should be treated as `null` values. By default, it's
+ *   ["null", "NULL", "NA", "N/A"].
+ * @param useFastDoubleParser whether to use the new _experimental_ FastDoubleParser, defaults to `false` for now.
+ */
 public data class ParserOptions(
     val locale: Locale? = null,
     // TODO, migrate to kotlinx.datetime.format.DateTimeFormat? https://github.com/Kotlin/dataframe/issues/876
     val dateTimeFormatter: DateTimeFormatter? = null,
     val dateTimePattern: String? = null,
     val nullStrings: Set<String>? = null,
+    val useFastDoubleParser: Boolean = false,
 ) {
     internal fun getDateTimeFormatter(): DateTimeFormatter? =
         when {

core/src/main/kotlin/org/jetbrains/kotlinx/dataframe/documentation/utils.kt

@@ -19,24 +19,31 @@ import kotlin.annotation.AnnotationTarget.VALUE_PARAMETER
  * {@include [Indent]}
  *
  */
+@ExcludeFromSources
 internal interface LineBreak
 
 /**   */
+@ExcludeFromSources
 internal interface QuarterIndent
 
 /**    */
+@ExcludeFromSources
 internal interface HalfIndent
 
 /**      */
+@ExcludeFromSources
 internal interface Indent
 
 /**          */
+@ExcludeFromSources
 internal interface DoubleIndent
 
 /**              */
+@ExcludeFromSources
 internal interface TripleIndent
 
 /**                  */
+@ExcludeFromSources
 internal interface QuadrupleIndent
 
 /**

core/src/main/kotlin/org/jetbrains/kotlinx/dataframe/impl/api/parse.kt

@@ -33,13 +33,13 @@ import org.jetbrains.kotlinx.dataframe.hasNulls
 import org.jetbrains.kotlinx.dataframe.impl.canParse
 import org.jetbrains.kotlinx.dataframe.impl.catchSilent
 import org.jetbrains.kotlinx.dataframe.impl.createStarProjectedType
+import org.jetbrains.kotlinx.dataframe.impl.io.FastDoubleParser
 import org.jetbrains.kotlinx.dataframe.impl.javaDurationCanParse
 import org.jetbrains.kotlinx.dataframe.io.isURL
 import org.jetbrains.kotlinx.dataframe.io.readJsonStr
 import org.jetbrains.kotlinx.dataframe.values
 import java.math.BigDecimal
 import java.net.URL
-import java.text.NumberFormat
 import java.text.ParsePosition
 import java.time.format.DateTimeFormatter
 import java.time.format.DateTimeFormatterBuilder
@@ -274,29 +274,6 @@ internal object Parsers : GlobalParserOptions {
             null
         }
 
-    private fun String.parseDouble(format: NumberFormat) =
-        when (uppercase(Locale.getDefault())) {
-            "NAN" -> Double.NaN
-
-            "INF" -> Double.POSITIVE_INFINITY
-
-            "-INF" -> Double.NEGATIVE_INFINITY
-
-            "INFINITY" -> Double.POSITIVE_INFINITY
-
-            "-INFINITY" -> Double.NEGATIVE_INFINITY
-
-            else -> {
-                val parsePosition = ParsePosition(0)
-                val result: Double? = format.parse(this, parsePosition)?.toDouble()
-                if (parsePosition.index != this.length) {
-                    null
-                } else {
-                    result
-                }
-            }
-        }
-
     inline fun <reified T : Any> stringParser(
         catch: Boolean = false,
         coveredBy: Set<KType> = emptySet(),
@@ -316,11 +293,15 @@ internal object Parsers : GlobalParserOptions {
     ): StringParserWithFormat<T> = StringParserWithFormat(typeOf<T>(), coveredBy, body)
 
     private val parserToDoubleWithOptions = stringParserWithOptions { options ->
-        val numberFormat = NumberFormat.getInstance(options?.locale ?: Locale.getDefault())
-        val parser = { it: String -> it.parseDouble(numberFormat) }
+        val fastDoubleParser = FastDoubleParser(options ?: ParserOptions())
+        val parser = { it: String -> fastDoubleParser.parseOrNull(it) }
         parser
     }
 
+    private val posixDoubleParser = FastDoubleParser(
+        ParserOptions(locale = Locale.forLanguageTag("C.UTF-8")),
+    )
+
     internal val parsersOrder = listOf(
         // Int
         stringParser<Int> { it.toIntOrNull() },
@@ -383,7 +364,7 @@ internal object Parsers : GlobalParserOptions {
         // Double, with explicit number format or taken from current locale
         parserToDoubleWithOptions,
         // Double, with POSIX format
-        stringParser<Double> { it.parseDouble(NumberFormat.getInstance(Locale.forLanguageTag("C.UTF-8"))) },
+        stringParser<Double> { posixDoubleParser.parseOrNull(it) },
         // Boolean
         stringParser<Boolean> { it.toBooleanOrNull() },
         // BigDecimal
@@ -448,9 +429,9 @@ internal object Parsers : GlobalParserOptions {
         return parser.applyOptions(options)
     }
 
-    internal fun getDoubleParser(locale: Locale? = null): (String) -> Double? {
+    internal fun getDoubleParser(locale: Locale? = null, useFastDoubleParser: Boolean): (String) -> Double? {
         val options = if (locale != null) {
-            ParserOptions(locale = locale)
+            ParserOptions(locale = locale, useFastDoubleParser = useFastDoubleParser)
         } else {
             null
         }