Merge 2a90396 into b3b1f64

Author: Jolanrensen

core/generated-sources/src/main/kotlin/org/jetbrains/kotlinx/dataframe/api/parse.kt

@@ -41,6 +41,7 @@ public interface GlobalParserOptions {
 
 public data class ParserOptions(
  val locale: Locale? = null,
+ // TODO, migrate to kotlinx.datetime.format.DateTimeFormat? https://github.com/Kotlin/dataframe/issues/876
  val dateTimeFormatter: DateTimeFormatter? = null,
  val dateTimePattern: String? = null,
  val nullStrings: Set<String>? = null,

core/generated-sources/src/main/kotlin/org/jetbrains/kotlinx/dataframe/impl/api/parse.kt

@@ -1,20 +1,35 @@
 package org.jetbrains.kotlinx.dataframe.api
 
+import io.kotest.matchers.should
 import io.kotest.matchers.shouldBe
+import kotlinx.datetime.DateTimeUnit
 import kotlinx.datetime.Instant
 import kotlinx.datetime.LocalDate
 import kotlinx.datetime.LocalDateTime
 import kotlinx.datetime.LocalTime
 import kotlinx.datetime.Month
+import kotlinx.datetime.format.DateTimeComponents
+import kotlinx.datetime.plus
+import kotlinx.datetime.toJavaInstant
+import kotlinx.datetime.toKotlinInstant
 import org.jetbrains.kotlinx.dataframe.DataFrame
+import org.jetbrains.kotlinx.dataframe.impl.api.Parsers
+import org.jetbrains.kotlinx.dataframe.impl.catchSilent
 import org.jetbrains.kotlinx.dataframe.type
 import org.junit.Test
 import java.util.Locale
+import kotlin.random.Random
 import kotlin.reflect.typeOf
+import kotlin.time.Duration
 import kotlin.time.Duration.Companion.days
 import kotlin.time.Duration.Companion.hours
+import kotlin.time.Duration.Companion.microseconds
+import kotlin.time.Duration.Companion.milliseconds
 import kotlin.time.Duration.Companion.minutes
+import kotlin.time.Duration.Companion.nanoseconds
 import kotlin.time.Duration.Companion.seconds
+import java.time.Duration as JavaDuration
+import java.time.Instant as JavaInstant
 
 class ParseTests {
  @Test
@@ -142,9 +157,340 @@ class ParseTests {
  columnOf("2022-01-23T04:29:40").parse().type shouldBe typeOf<LocalDateTime>()
  }
 
+ @Test
+ fun `can parse instants`() {
+ val instantParser = Parsers[typeOf<Instant>()]!!.applyOptions(null)
+ val javaInstantParser = Parsers[typeOf<JavaInstant>()]!!.applyOptions(null)
+
+ // from the kotlinx-datetime tests, java instants treat leap seconds etc. like this
+ fun parseInstantLikeJavaDoesOrNull(input: String): Instant? =
+ catchSilent {
+ DateTimeComponents.Formats.ISO_DATE_TIME_OFFSET.parseOrNull(input)?.apply {
+ when {
+ hour == 24 && minute == 0 && second == 0 && nanosecond == 0 -> {
+ setDate(toLocalDate().plus(1, DateTimeUnit.DAY))
+ hour = 0
+ }
+
+ hour == 23 && minute == 59 && second == 60 -> second = 59
+ }
+ }?.toInstantUsingOffset()
+ }
+
+ fun formatTwoDigits(i: Int) = if (i < 10) "0$i" else "$i"
+
+ for (hour in 23..25) {
+ for (minute in listOf(0..5, 58..62).flatten()) {
+ for (second in listOf(0..5, 58..62).flatten()) {
+ val input = "2020-03-16T$hour:${formatTwoDigits(minute)}:${formatTwoDigits(second)}Z"
+
+ val myParserRes = instantParser(input) as Instant?
+ val myJavaParserRes = javaInstantParser(input) as JavaInstant?
+ val instantRes = catchSilent { Instant.parse(input) }
+ val instantLikeJava = parseInstantLikeJavaDoesOrNull(input)
+ val javaInstantRes = catchSilent { JavaInstant.parse(input) }
+
+ // our parser has a fallback mechanism built in, like this
+ myParserRes shouldBe (instantRes ?: javaInstantRes?.toKotlinInstant())
+ myParserRes shouldBe instantLikeJava
+
+ myJavaParserRes shouldBe javaInstantRes
+
+ myParserRes?.toJavaInstant() shouldBe instantLikeJava?.toJavaInstant()
+ instantLikeJava?.toJavaInstant() shouldBe myJavaParserRes
+ myJavaParserRes shouldBe javaInstantRes
+ }
+ }
+ }
+ }
+
  @Test
  fun `parse duration`() {
  columnOf("1d 15m", "20h 35m 11s").parse() shouldBe
  columnOf(1.days + 15.minutes, 20.hours + 35.minutes + 11.seconds)
  }
+
+ @Test
+ fun `can parse duration isoStrings`() {
+ val durationParser = Parsers[typeOf<Duration>()]!!.applyOptions(null) as (String) -> Duration?
+ val javaDurationParser = Parsers[typeOf<JavaDuration>()]!!.applyOptions(null) as (String) -> JavaDuration?
+
+ fun testSuccess(duration: Duration, vararg isoStrings: String) {
+ isoStrings.first() shouldBe duration.toIsoString()
+ for (isoString in isoStrings) {
+ Duration.parse(isoString) shouldBe duration
+ durationParser(isoString) shouldBe duration
+
+ javaDurationParser(isoString) shouldBe catchSilent { JavaDuration.parse(isoString) }
+ }
+ }
+
+ // zero
+ testSuccess(Duration.ZERO, "PT0S", "P0D", "PT0H", "PT0M", "P0DT0H", "PT0H0M", "PT0H0S")
+
+ // single unit
+ testSuccess(1.days, "PT24H", "P1D", "PT1440M", "PT86400S")
+ testSuccess(1.hours, "PT1H")
+ testSuccess(1.minutes, "PT1M")
+ testSuccess(1.seconds, "PT1S")
+ testSuccess(1.milliseconds, "PT0.001S")
+ testSuccess(1.microseconds, "PT0.000001S")
+ testSuccess(1.nanoseconds, "PT0.000000001S", "PT0.0000000009S")
+ testSuccess(0.9.nanoseconds, "PT0.000000001S")
+
+ // rounded to zero
+ testSuccess(0.1.nanoseconds, "PT0S")
+ testSuccess(Duration.ZERO, "PT0S", "PT0.0000000004S")
+
+ // several units combined
+ testSuccess(1.days + 1.minutes, "PT24H1M")
+ testSuccess(1.days + 1.seconds, "PT24H0M1S")
+ testSuccess(1.days + 1.milliseconds, "PT24H0M0.001S")
+ testSuccess(1.hours + 30.minutes, "PT1H30M")
+ testSuccess(1.hours + 500.milliseconds, "PT1H0M0.500S")
+ testSuccess(2.minutes + 500.milliseconds, "PT2M0.500S")
+ testSuccess(90_500.milliseconds, "PT1M30.500S")
+
+ // with sign
+ testSuccess(-1.days + 15.minutes, "-PT23H45M", "PT-23H-45M", "+PT-24H+15M")
+ testSuccess(-1.days - 15.minutes, "-PT24H15M", "PT-24H-15M", "-PT25H-45M")
+ testSuccess(Duration.ZERO, "PT0S", "P1DT-24H", "+PT-1H+60M", "-PT1M-60S")
+
+ // infinite
+ testSuccess(
+ Duration.INFINITE,
+ "PT9999999999999H",
+ "PT+10000000000000H",
+ "-PT-9999999999999H",
+ "-PT-1234567890123456789012S",
+ )
+ testSuccess(-Duration.INFINITE, "-PT9999999999999H", "-PT10000000000000H", "PT-1234567890123456789012S")
+
+ fun testFailure(isoString: String) {
+ catchSilent { Duration.parse(isoString) } shouldBe durationParser(isoString)
+ catchSilent { JavaDuration.parse(isoString) } shouldBe javaDurationParser(isoString)
+ }
+
+ listOf(
+ "",
+ " ",
+ "P",
+ "PT",
+ "P1DT",
+ "P1",
+ "PT1",
+ "0",
+ "+P",
+ "+",
+ "-",
+ "h",
+ "H",
+ "something",
+ "1m",
+ "1d",
+ "2d 11s",
+ "Infinity",
+ "-Infinity", // successful in kotlin, not in java
+ "P+12+34D",
+ "P12-34D",
+ "PT1234567890-1234567890S",
+ " P1D",
+ "PT1S ",
+ "P3W",
+ "P1Y",
+ "P1M",
+ "P1S",
+ "PT1D",
+ "PT1Y",
+ "PT1S2S",
+ "PT1S2H",
+ "P9999999999999DT-9999999999999H",
+ "PT1.5H",
+ "PT0.5D",
+ "PT.5S",
+ "PT0.25.25S",
+ ).forEach(::testFailure)
+ }
+
+ @Test
+ fun `can parse duration default kotlin strings`() {
+ val durationParser = Parsers[typeOf<Duration>()]!!.applyOptions(null) as (String) -> Duration?
+
+ fun testParsing(string: String, expectedDuration: Duration) {
+ Duration.parse(string) shouldBe expectedDuration
+ durationParser(string) shouldBe expectedDuration
+ }
+
+ fun testSuccess(duration: Duration, vararg expected: String) {
+ val actual = duration.toString()
+ actual shouldBe expected.first()
+
+ if (duration.isPositive()) {
+ if (' ' in actual) {
+ (-duration).toString() shouldBe "-($actual)"
+ } else {
+ (-duration).toString() shouldBe "-$actual"
+ }
+ }
+
+ for (string in expected) {
+ testParsing(string, duration)
+ if (duration.isPositive() && duration.isFinite()) {
+ testParsing("+($string)", duration)
+ testParsing("-($string)", -duration)
+ if (' ' !in string) {
+ testParsing("+$string", duration)
+ testParsing("-$string", -duration)
+ }
+ }
+ }
+ }
+
+ testSuccess(101.days, "101d", "2424h")
+ testSuccess(45.3.days, "45d 7h 12m", "45.3d", "45d 7.2h") // 0.3d == 7.2h
+ testSuccess(45.days, "45d")
+
+ testSuccess(40.5.days, "40d 12h", "40.5d", "40d 720m")
+ testSuccess(40.days + 20.minutes, "40d 0h 20m", "40d 20m", "40d 1200s")
+ testSuccess(40.days + 20.seconds, "40d 0h 0m 20s", "40d 20s")
+ testSuccess(40.days + 100.nanoseconds, "40d 0h 0m 0.000000100s", "40d 100ns")
+
+ testSuccess(40.hours + 15.minutes, "1d 16h 15m", "40h 15m")
+ testSuccess(40.hours, "1d 16h", "40h")
+
+ testSuccess(12.5.hours, "12h 30m")
+ testSuccess(12.hours + 15.seconds, "12h 0m 15s")
+ testSuccess(12.hours + 1.nanoseconds, "12h 0m 0.000000001s")
+ testSuccess(30.minutes, "30m")
+ testSuccess(17.5.minutes, "17m 30s")
+
+ testSuccess(16.5.minutes, "16m 30s")
+ testSuccess(1097.1.seconds, "18m 17.1s")
+ testSuccess(90.36.seconds, "1m 30.36s")
+ testSuccess(50.seconds, "50s")
+ testSuccess(1.3.seconds, "1.3s")
+ testSuccess(1.seconds, "1s")
+
+ testSuccess(0.5.seconds, "500ms")
+ testSuccess(40.2.milliseconds, "40.2ms")
+ testSuccess(4.225.milliseconds, "4.225ms")
+ testSuccess(4.24501.milliseconds, "4.245010ms", "4ms 245us 10ns")
+ testSuccess(1.milliseconds, "1ms")
+
+ testSuccess(0.75.milliseconds, "750us")
+ testSuccess(75.35.microseconds, "75.35us")
+ testSuccess(7.25.microseconds, "7.25us")
+ testSuccess(1.035.microseconds, "1.035us")
+ testSuccess(1.005.microseconds, "1.005us")
+ testSuccess(1800.nanoseconds, "1.8us", "1800ns", "0.0000000005h")
+
+ testSuccess(950.5.nanoseconds, "951ns")
+ testSuccess(85.23.nanoseconds, "85ns")
+ testSuccess(8.235.nanoseconds, "8ns")
+ testSuccess(1.nanoseconds, "1ns", "0.9ns", "0.001us", "0.0009us")
+ testSuccess(1.3.nanoseconds, "1ns")
+ testSuccess(0.75.nanoseconds, "1ns")
+ testSuccess(0.7512.nanoseconds, "1ns")
+
+ // equal to zero
+// testSuccess(0.023.nanoseconds, "0.023ns")
+// testSuccess(0.0034.nanoseconds, "0.0034ns")
+// testSuccess(0.0000035.nanoseconds, "0.0000035ns")
+
+ testSuccess(Duration.ZERO, "0s", "0.4ns", "0000.0000ns")
+ testSuccess(365.days * 10000, "3650000d")
+ testSuccess(300.days * 100000, "30000000d")
+ testSuccess(365.days * 100000, "36500000d")
+ testSuccess(((Long.MAX_VALUE / 2) - 1).milliseconds, "53375995583d 15h 36m 27.902s") // max finite value
+
+ // all infinite
+// val universeAge = Duration.days(365.25) * 13.799e9
+// val planckTime = Duration.seconds(5.4e-44)
+
+// testSuccess(universeAge, "5.04e+12d")
+// testSuccess(planckTime, "5.40e-44s")
+// testSuccess(Duration.nanoseconds(Double.MAX_VALUE), "2.08e+294d")
+ testSuccess(Duration.INFINITE, "Infinity", "53375995583d 20h", "+Infinity")
+ testSuccess(-Duration.INFINITE, "-Infinity", "-(53375995583d 20h)")
+
+ fun testFailure(isoString: String) {
+ catchSilent { Duration.parse(isoString) } shouldBe durationParser(isoString)
+ }
+
+ listOf(
+ "",
+ " ",
+ "P",
+ "PT",
+ "P1DT",
+ "P1",
+ "PT1",
+ "0",
+ "+P",
+ "+",
+ "-",
+ "h",
+ "H",
+ "something",
+ "1234567890123456789012ns",
+ "Inf",
+ "-Infinity value",
+ "1s ",
+ " 1s",
+ "1d 1m 1h",
+ "1s 2s",
+ "-12m 15s",
+ "-12m -15s",
+ "-()",
+ "-(12m 30s",
+ "+12m 15s",
+ "+12m +15s",
+ "+()",
+ "+(12m 30s",
+ "()",
+ "(12m 30s)",
+ "12.5m 11.5s",
+ ".2s",
+ "0.1553.39m",
+ "P+12+34D",
+ "P12-34D",
+ "PT1234567890-1234567890S",
+ " P1D",
+ "PT1S ",
+ "P1Y",
+ "P1M",
+ "P1S",
+ "PT1D",
+ "PT1Y",
+ "PT1S2S",
+ "PT1S2H",
+ "P9999999999999DT-9999999999999H",
+ "PT1.5H",
+ "PT0.5D",
+ "PT.5S",
+ "PT0.25.25S",
+ ).forEach(::testFailure)
+ }
+
+ @Test
+ fun `Parse normal string column`() {
+ val df = dataFrameOf(List(5_000) { "_$it" }).fill(100) {
+ Random.nextInt().toChar().toString() + Random.nextInt().toChar()
+ }
+
+ df.parse()
+ }
+
+ /**
+ * Asserts that all elements of the iterable are equal to each other
+ */
+ private fun <T> Iterable<T>.shouldAllBeEqual(): Iterable<T> {
+ this should {
+ it.reduce { a, b ->
+ a shouldBe b
+ b
+ }
+ }
+ return this
+ }
 }