Merge branch 'main' into java8

# Conflicts: #	buildSrc/src/main/kotlin/Versions.kt

Author: Jolanrensen

.github/workflows/build.yml

@@ -10,24 +10,24 @@ jobs:
  build-all-versions:
  strategy:
  matrix:
- scala: [ "2.12.16", "2.13.8" ]
- spark: [ "3.3.0", "3.2.1", "3.2.0", "3.1.3", "3.1.2", "3.1.1", "3.1.0", "3.0.3", "3.0.2", "3.0.1", "3.0.0" ]
+ scala: [ "2.12.17", "2.13.10" ]
+ spark: [ "3.3.1", "3.3.0", "3.2.3", "3.2.2", "3.2.1", "3.2.0", "3.1.3", "3.1.2", "3.1.1", "3.1.0", "3.0.3", "3.0.2", "3.0.1", "3.0.0" ]
  exclude:
- - scala: "2.13.8"
+ - scala: "2.13.10"
  spark: "3.1.3"
- - scala: "2.13.8"
+ - scala: "2.13.10"
  spark: "3.1.2"
- - scala: "2.13.8"
+ - scala: "2.13.10"
  spark: "3.1.1"
- - scala: "2.13.8"
+ - scala: "2.13.10"
  spark: "3.1.0"
- - scala: "2.13.8"
+ - scala: "2.13.10"
  spark: "3.0.3"
- - scala: "2.13.8"
+ - scala: "2.13.10"
  spark: "3.0.2"
- - scala: "2.13.8"
+ - scala: "2.13.10"
  spark: "3.0.1"
- - scala: "2.13.8"
+ - scala: "2.13.10"
  spark: "3.0.0"
  runs-on: ubuntu-latest
 

.github/workflows/publish_dev_version.yml

@@ -9,24 +9,24 @@ jobs:
  build-and-deploy:
  strategy:
  matrix:
- scala: [ "2.12.16", "2.13.8" ]
- spark: [ "3.3.0", "3.2.1", "3.2.0", "3.1.3", "3.1.2", "3.1.1", "3.1.0", "3.0.3", "3.0.2", "3.0.1", "3.0.0" ]
+ scala: [ "2.12.17", "2.13.10" ]
+ spark: [ "3.3.1", "3.3.0", "3.2.3", "3.2.2", "3.2.1", "3.2.0", "3.1.3", "3.1.2", "3.1.1", "3.1.0", "3.0.3", "3.0.2", "3.0.1", "3.0.0" ]
  exclude:
- - scala: "2.13.8"
+ - scala: "2.13.10"
  spark: "3.1.3"
- - scala: "2.13.8"
+ - scala: "2.13.10"
  spark: "3.1.2"
- - scala: "2.13.8"
+ - scala: "2.13.10"
  spark: "3.1.1"
- - scala: "2.13.8"
+ - scala: "2.13.10"
  spark: "3.1.0"
- - scala: "2.13.8"
+ - scala: "2.13.10"
  spark: "3.0.3"
- - scala: "2.13.8"
+ - scala: "2.13.10"
  spark: "3.0.2"
- - scala: "2.13.8"
+ - scala: "2.13.10"
  spark: "3.0.1"
- - scala: "2.13.8"
+ - scala: "2.13.10"
  spark: "3.0.0"
  runs-on: ubuntu-latest
  permissions:
@@ -72,7 +72,7 @@ jobs:
  ./gradlew 
  -Pspark=${{ matrix.spark }} 
  -Pscala=${{ matrix.scala }} 
- -PskipScalaTuplesInKotlin=${{ !(matrix.spark == '3.0.0' || matrix.scala == '2.13.8' && matrix.spark == '3.2.0') }} 
+ -PskipScalaTuplesInKotlin=${{ !(matrix.spark == '3.0.0' || matrix.scala == '2.13.10' && matrix.spark == '3.2.0') }} 
  clean 
  publishMavenPublicationToGitHubPackagesRepository 
  --scan

.github/workflows/publish_release_version.yml

@@ -1,34 +1,31 @@
 name: Deploy release version to Maven Central
 
 on:
-# release: TODO temporary, revert to this after 1.2.1 release
-# types: [created]
- push:
- branches:
- - "release"
+ release:
+ types: [created]
 
 jobs:
  build-and-deploy-mvn-central:
  strategy:
  matrix:
- scala: [ "2.12.16", "2.13.8" ]
- spark: [ "3.3.0", "3.2.1", "3.2.0", "3.1.3", "3.1.2", "3.1.1", "3.1.0", "3.0.3", "3.0.2", "3.0.1", "3.0.0" ]
+ scala: [ "2.12.17", "2.13.10" ]
+ spark: [ "3.3.1", "3.3.0", "3.2.3", "3.2.2", "3.2.1", "3.2.0", "3.1.3", "3.1.2", "3.1.1", "3.1.0", "3.0.3", "3.0.2", "3.0.1", "3.0.0" ]
  exclude:
- - scala: "2.13.8"
+ - scala: "2.13.10"
  spark: "3.1.3"
- - scala: "2.13.8"
+ - scala: "2.13.10"
  spark: "3.1.2"
- - scala: "2.13.8"
+ - scala: "2.13.10"
  spark: "3.1.1"
- - scala: "2.13.8"
+ - scala: "2.13.10"
  spark: "3.1.0"
- - scala: "2.13.8"
+ - scala: "2.13.10"
  spark: "3.0.3"
- - scala: "2.13.8"
+ - scala: "2.13.10"
  spark: "3.0.2"
- - scala: "2.13.8"
+ - scala: "2.13.10"
  spark: "3.0.1"
- - scala: "2.13.8"
+ - scala: "2.13.10"
  spark: "3.0.0"
  runs-on: ubuntu-latest
  permissions:
@@ -77,7 +74,7 @@ jobs:
  ./gradlew 
  -Pspark=${{ matrix.spark }} 
  -Pscala=${{ matrix.scala }} 
- -PskipScalaTuplesInKotlin=${{ !(matrix.spark == '3.0.0' || matrix.scala == '2.13.8' && matrix.spark == '3.2.0') }} 
+ -PskipScalaTuplesInKotlin=${{ !(matrix.spark == '3.0.0' || matrix.scala == '2.13.10' && matrix.spark == '3.2.0') }} 
  clean 
  publishMavenPublicationToMavenCentralRepository
  --scan

README.md

@@ -2,7 +2,7 @@
 
 [![Kotlin Stable](https://kotl.in/badges/stable.svg)](https://kotlinlang.org/docs/components-stability.html)
 [![JetBrains official project](http://jb.gg/badges/official.svg)](https://confluence.jetbrains.com/display/ALL/JetBrains+on+GitHub)
-[![Maven Central](https://img.shields.io/maven-central/v/org.jetbrains.kotlinx.spark/kotlin-spark-api_3.3.0_2.13.svg?label=Maven%20Central)](https://search.maven.org/search?q=g:"org.jetbrains.kotlinx.spark"%20AND%20a:"kotlin-spark-api_3.3.0_2.13")
+[![Maven Central](https://img.shields.io/maven-central/v/org.jetbrains.kotlinx.spark/kotlin-spark-api_3.3.1_2.13.svg?label=Maven%20Central)](https://search.maven.org/search?q=g:"org.jetbrains.kotlinx.spark"%20AND%20a:"kotlin-spark-api_3.3.1_2.13")
 [![Join the chat at https://gitter.im/JetBrains/kotlin-spark-api](https://badges.gitter.im/JetBrains/kotlin-spark-api.svg)](https://gitter.im/JetBrains/kotlin-spark-api?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge)
 
 Your next API to work with [Apache Spark](https://spark.apache.org/). 
@@ -38,8 +38,14 @@ We have opened a Spark Project Improvement Proposal: [Kotlin support for Apache
 
 | Apache Spark | Scala | Kotlin for Apache Spark |
 |:------------:|:-----:|:-----------------------------------:|
+| 3.3.1 | 2.13 | kotlin-spark-api_3.3.1_2.13:VERSION |
+| | 2.12 | kotlin-spark-api_3.3.1_2.12:VERSION |
 | 3.3.0 | 2.13 | kotlin-spark-api_3.3.0_2.13:VERSION |
 | | 2.12 | kotlin-spark-api_3.3.0_2.12:VERSION |
+| 3.2.3 | 2.13 | kotlin-spark-api_3.2.3_2.13:VERSION |
+| | 2.12 | kotlin-spark-api_3.2.3_2.12:VERSION |
+| 3.2.2 | 2.13 | kotlin-spark-api_3.2.2_2.13:VERSION |
+| | 2.12 | kotlin-spark-api_3.2.2_2.12:VERSION |
 | 3.2.1 | 2.13 | kotlin-spark-api_3.2.1_2.13:VERSION |
 | | 2.12 | kotlin-spark-api_3.2.1_2.12:VERSION |
 | 3.2.0 | 2.13 | kotlin-spark-api_3.2.0_2.13:VERSION |
@@ -67,7 +73,7 @@ The Kotlin for Spark artifacts adhere to the following convention:
 The only exception to this is `scala-tuples-in-kotlin_[Scala core version]:[Kotlin for Apache Spark API version]`, which is 
 independent of Spark.
 
-[![Maven Central](https://img.shields.io/maven-central/v/org.jetbrains.kotlinx.spark/kotlin-spark-api_3.3.0_2.13.svg?label=Maven%20Central)](https://search.maven.org/search?q=g:"org.jetbrains.kotlinx.spark"%20AND%20a:"kotlin-spark-api_3.3.0_2.13")
+[![Maven Central](https://img.shields.io/maven-central/v/org.jetbrains.kotlinx.spark/kotlin-spark-api_3.3.1_2.13.svg?label=Maven%20Central)](https://search.maven.org/search?q=g:"org.jetbrains.kotlinx.spark"%20AND%20a:"kotlin-spark-api_3.3.1_2.13")
 
 ## How to configure Kotlin for Apache Spark in your project
 
@@ -78,7 +84,7 @@ Here's an example `pom.xml`:
 ```xml
 <dependency>
  <groupId>org.jetbrains.kotlinx.spark</groupId>
- <artifactId>kotlin-spark-api_3.3.0_2.13</artifactId>
+ <artifactId>kotlin-spark-api_3.3.1_2.13</artifactId>
  <version>${kotlin-spark-api.version}</version>
 </dependency>
 <dependency>
@@ -111,7 +117,7 @@ To it, simply add
 to the top of your notebook. This will get the latest version of the API, together with the latest version of Spark.
 To define a certain version of Spark or the API itself, simply add it like this:
 ```jupyterpython
-%use spark(spark=3.3.0, scala=2.13, v=1.2.0)
+%use spark(spark=3.3.1, scala=2.13, v=1.2.2)
 ```
 
 Inside the notebook a Spark session will be initiated automatically. This can be accessed via the `spark` value.

buildSrc/src/main/kotlin/Dependencies.kt

@@ -20,6 +20,7 @@ object Dependencies {
  inline val kotlinTest get() = "org.jetbrains.kotlin:kotlin-test:${Versions.kotlin}"
  inline val kotlinScriptingCommon get() = "org.jetbrains.kotlin:kotlin-scripting-common"
  inline val kotlinScriptingJvm get() = "org.jetbrains.kotlin:kotlin-scripting-jvm"
+ inline val jacksonDatabind get() = "com.fasterxml.jackson.core:jackson-databind:${Versions.jacksonDatabind}"
 }
 
 

buildSrc/src/main/kotlin/Versions.kt

@@ -1,5 +1,5 @@
 object Versions {
- const val project = "1.2.2-SNAPSHOT"
+ const val project = "1.2.3-SNAPSHOT"
  const val groupID = "org.jetbrains.kotlinx.spark"
  const val kotlin = "1.7.20"
  const val jvmTarget = "8"
@@ -10,10 +10,10 @@ object Versions {
  inline val sparkMinor get() = spark.substringBeforeLast('.')
  inline val scalaCompat get() = scala.substringBeforeLast('.')
 
- const val jupyter = "0.11.0-171-1"
+ const val jupyter = "0.11.0-134"
  const val kotest = "5.3.2"
  const val kotestTestContainers = "1.3.3"
- const val dokka = "1.7.20"
+ const val dokka = "1.7.10"
  const val jcp = "7.0.5"
  const val mavenPublish = "0.20.0"
  const val atrium = "0.17.0"
@@ -22,6 +22,7 @@ object Versions {
  const val hadoop = "3.3.1"
  const val kotlinxHtml = "0.7.5"
  const val klaxon = "5.5"
+ const val jacksonDatabind = "2.13.4.2"
 
  inline val versionMap
  get() = mapOf(

core/build.gradle.kts

@@ -26,6 +26,9 @@ dependencies {
  reflect,
  )
 
+ // https://github.com/FasterXML/jackson-bom/issues/52
+ if (Versions.spark == "3.3.1") implementation(jacksonDatabind)
+
  implementation(
  sparkSql,
  )
@@ -35,14 +38,18 @@ dependencies {
 
 java {
  toolchain {
- languageVersion.set(
- JavaLanguageVersion.of(Versions.jvmTarget)
- )
+ if (Versions.scalaCompat.toDouble() > 2.12) { // scala 2.12 will always target java 8
+ languageVersion.set(
+ JavaLanguageVersion.of(Versions.jvmTarget)
+ )
+ }
  }
 }
 
 tasks.withType<ScalaCompile> {
- targetCompatibility = Versions.jvmTarget
+ if (Versions.scalaCompat.toDouble() > 2.12) { // scala 2.12 will always target java 8
+ targetCompatibility = Versions.jvmTarget
+ }
 }
 
 val scalaMainSources = sourceSets.main.get().scala.sourceDirectories

docs/quick-start-guide.md

@@ -72,13 +72,13 @@ Here's what the `pom.xml` looks like for this example:
  </dependency>
  <dependency> <!-- Kotlin Spark API dependency -->
  <groupId>org.jetbrains.kotlinx.spark</groupId>
- <artifactId>kotlin-spark-api-3.0.0_2.12</artifactId>
- <version>1.0.1</version>
+ <artifactId>kotlin-spark-api_3.3.1_2.13</artifactId>
+ <version>1.2.2</version>
  </dependency>
  <dependency> <!-- Spark dependency -->
  <groupId>org.apache.spark</groupId>
  <artifactId>spark-sql_2.12</artifactId>
- <version>3.0.0</version>
+ <version>3.3.1</version>
  </dependency>
  </dependencies>
 
@@ -163,8 +163,8 @@ dependencies {
  // Kotlin stdlib
  implementation 'org.jetbrains.kotlin:kotlin-stdlib:1.4.0'
  // Kotlin Spark API
- implementation 'org.jetbrains.kotlinx.spark:kotlin-spark-api-3.0.0_2.12:1.0.1' // Apache Spark
- compileOnly 'org.apache.spark:spark-sql_2.12:3.0.0'
+ implementation 'org.jetbrains.kotlinx.spark:kotlin-spark-api_3.3.1_2.13:1.2.2' // Apache Spark
+ compileOnly 'org.apache.spark:spark-sql_2.12:3.3.1'
 }
 
 compileKotlin {
@@ -198,9 +198,9 @@ dependencies {
  // Kotlin stdlib
  implementation ("org.jetbrains.kotlin:kotlin-stdlib:1.4.0")
  // Kotlin Spark API
- implementation ("org.jetbrains.kotlinx.spark:kotlin-spark-api-3.0.0_2.12:1.0.1")
+ implementation ("org.jetbrains.kotlinx.spark:kotlin-spark-api_3.3.1_2.13:1.2.2")
  // Apache Spark
- compileOnly ("org.apache.spark:spark-sql_2.12:3.0.0")
+ compileOnly ("org.apache.spark:spark-sql_2.12:3.3.1")
 }
 
 compileKotlin.kotlinOptions.jvmTarget = "1.8"

examples/build.gradle.kts

@@ -21,6 +21,10 @@ dependencies {
  }
 
  with(Dependencies) {
+
+ // https://github.com/FasterXML/jackson-bom/issues/52
+ if (Versions.spark == "3.3.1") implementation(jacksonDatabind)
+
  implementation(
  sparkSql,
  sparkMl,

gradle.properties

@@ -7,9 +7,9 @@ GROUP=org.jetbrains.kotlinx.spark
 
 # Controls the spark and scala version for the entire project
 # can also be defined like ./gradlew -Pspark=X.X.X -Pscala=X.X.X build
-spark=3.3.0
-scala=2.13.8
-#scala=2.12.16
+spark=3.3.1
+scala=2.13.10
+# scala=2.12.17
 skipScalaTuplesInKotlin=false
 
 org.gradle.caching=true