Merge branch 'main' into release

Author: asm0dey

.github/workflows/build.yml

@@ -59,7 +59,7 @@ jobs:
  -Pspark=${{ matrix.spark }}
  -Pscala=${{ matrix.scala }}
  clean
- build
+ test
  --scan
 
  # qodana:

.github/workflows/publish_dev_version.yml

@@ -57,17 +57,25 @@ jobs:
  - name: Validate Gradle wrapper
  uses: gradle/wrapper-validation-action@v1
 
- - name: Deploy to GH Packages with Gradle
+ - name: Setup Gradle
  uses: gradle/gradle-build-action@v2
+
+ - name: Set Swap Space
+ uses: pierotofy/set-swap-space@master
  with:
- arguments: |
- -Pspark=${{ matrix.spark }}
- -Pscala=${{ matrix.scala }}
- -PskipScalaTuplesInKotlin=${{ !(matrix.spark == '3.0.0' || matrix.scala == '2.13.8' && matrix.spark == '3.2.0') }}
- clean
- publishMavenPublicationToGitHubPackagesRepository
- --scan
+ swap-size-gb: 12
+
+ - name: Deploy to GH Packages with Gradle
  env:
  GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+ run: >
+ ./gradlew 
+ -Pspark=${{ matrix.spark }} 
+ -Pscala=${{ matrix.scala }} 
+ -PskipScalaTuplesInKotlin=${{ !(matrix.spark == '3.0.0' || matrix.scala == '2.13.8' && matrix.spark == '3.2.0') }} 
+ clean 
+ publishMavenPublicationToGitHubPackagesRepository 
+ --scan
+ 
 
 

.github/workflows/publish_release_version.yml

@@ -0,0 +1,88 @@
+name: Deploy release version to Maven Central
+
+on:
+ release:
+ types: [created]
+
+jobs:
+ build-and-deploy-mvn-central:
+ strategy:
+ matrix:
+ scala: [ "2.12.15", "2.13.8" ]
+ spark: [ "3.3.0", "3.2.1", "3.2.0", "3.1.3", "3.1.2", "3.1.1", "3.1.0", "3.0.3", "3.0.2", "3.0.1", "3.0.0" ]
+ exclude:
+ - scala: "2.13.8"
+ spark: "3.1.3"
+ - scala: "2.13.8"
+ spark: "3.1.2"
+ - scala: "2.13.8"
+ spark: "3.1.1"
+ - scala: "2.13.8"
+ spark: "3.1.0"
+ - scala: "2.13.8"
+ spark: "3.0.3"
+ - scala: "2.13.8"
+ spark: "3.0.2"
+ - scala: "2.13.8"
+ spark: "3.0.1"
+ - scala: "2.13.8"
+ spark: "3.0.0"
+ runs-on: ubuntu-latest
+ permissions:
+ contents: read
+ packages: write
+
+ steps:
+ - uses: actions/checkout@v3
+
+ - name: Set up JDK 11
+ uses: actions/setup-java@v3
+ with:
+ distribution: adopt
+ java-version: 11
+ check-latest: true
+ server-id: ossrh
+ server-username: ORG_GRADLE_PROJECT_mavenCentralUsername
+ server-password: ORG_GRADLE_PROJECT_mavenCentralPassword
+ gpg-private-key: ${{ secrets.MAVEN_GPG_PRIVATE_KEY }}
+ gpg-passphrase: ORG_GRADLE_PROJECT_signingInMemoryKeyPassword
+
+ - name: Cache Gradle packages
+ uses: actions/cache@v3
+ with:
+ path: |
+ ~/.gradle/caches
+ ~/.gradle/wrapper
+ ~/.gradle/jdks
+ key: ${{ runner.os }}-gradle-spark-${{ matrix.spark }}-${{ matrix.scala }}
+ restore-keys: |
+ ${{ runner.os }}-gradle-
+
+ - name: Validate Gradle wrapper
+ uses: gradle/wrapper-validation-action@v1
+
+ - name: Setup Gradle
+ uses: gradle/gradle-build-action@v2
+
+ - name: Set Swap Space
+ uses: pierotofy/set-swap-space@master
+ with:
+ swap-size-gb: 12
+
+
+ - name: Deploy to Maven Central with Gradle
+ env:
+ ORG_GRADLE_PROJECT_mavenCentralUsername: ${{ secrets.OSSRH_USERNAME }}
+ ORG_GRADLE_PROJECT_mavenCentralPassword: ${{ secrets.OSSRH_TOKEN }}
+ ORG_GRADLE_PROJECT_signingInMemoryKeyPassword: ${{ secrets.MAVEN_GPG_PASSPHRASE }}
+ run: >
+ ./gradlew 
+ -Pspark=${{ matrix.spark }} 
+ -Pscala=${{ matrix.scala }} 
+ -PskipScalaTuplesInKotlin=${{ !(matrix.spark == '3.0.0' || matrix.scala == '2.13.8' && matrix.spark == '3.2.0') }} 
+ clean 
+ publishMavenPublicationToMavenCentralRepository
+ --scan
+ 
+
+

README.md

@@ -2,7 +2,7 @@
 
 [![Kotlin Stable](https://kotl.in/badges/stable.svg)](https://kotlinlang.org/docs/components-stability.html)
 [![JetBrains official project](http://jb.gg/badges/official.svg)](https://confluence.jetbrains.com/display/ALL/JetBrains+on+GitHub)
-[![Maven Central](https://img.shields.io/maven-central/v/org.jetbrains.kotlinx.spark/kotlin-spark-api-parent-3.2.svg?label=Maven%20Central)](https://search.maven.org/search?q=g:org.jetbrains.kotlinx.spark%20AND%20v:1.1.0)
+[![Maven Central](https://img.shields.io/maven-central/v/org.jetbrains.kotlinx.spark/kotlin-spark-api_3.3.0_2.13.svg?label=Maven%20Central)](https://search.maven.org/search?q=g:"org.jetbrains.kotlinx.spark"%20AND%20a:"kotlin-spark-api_3.3.0_2.13")
 [![Join the chat at https://gitter.im/JetBrains/kotlin-spark-api](https://badges.gitter.im/JetBrains/kotlin-spark-api.svg)](https://gitter.im/JetBrains/kotlin-spark-api?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge)
 
 Your next API to work with [Apache Spark](https://spark.apache.org/). 
@@ -62,9 +62,12 @@ We have opened a Spark Project Improvement Proposal: [Kotlin support for Apache
 
 The list of Kotlin for Apache Spark releases is available [here](https://github.com/JetBrains/kotlin-spark-api/releases/).
 The Kotlin for Spark artifacts adhere to the following convention:
-`[Apache Spark version]_[Scala core version]:[Kotlin for Apache Spark API version]` 
+`[name]_[Apache Spark version]_[Scala core version]:[Kotlin for Apache Spark API version]` 
 
-[![Maven Central](https://img.shields.io/maven-central/v/org.jetbrains.kotlinx.spark/kotlin-spark-api-parent-3.2.svg?label=Maven%20Central)](https://search.maven.org/search?q=g:"org.jetbrains.kotlinx.spark"%20AND%20a:"kotlin-spark-api-3.2")
+The only exception to this is `scala-tuples-in-kotlin_[Scala core version]:[Kotlin for Apache Spark API version]`, which is 
+independent of Spark.
+
+[![Maven Central](https://img.shields.io/maven-central/v/org.jetbrains.kotlinx.spark/kotlin-spark-api_3.3.0_2.13.svg?label=Maven%20Central)](https://search.maven.org/search?q=g:"org.jetbrains.kotlinx.spark"%20AND%20a:"kotlin-spark-api_3.3.0_2.13")
 
 ## How to configure Kotlin for Apache Spark in your project
 
@@ -360,7 +363,7 @@ Of the [main](https://github.com/Kotlin/kotlin-spark-api/tree/main) branch, deve
 [GitHub Packages](https://github.com/orgs/Kotlin/packages?tab=packages&q=kotlin-spark-api). This way, new features can be
 tested quickly without having to wait for a full release. 
 
-For full releases, the `release` branch is updated.
+For full releases, the [release](https://github.com/Kotlin/kotlin-spark-api/tree/release) branch is updated.
 
 ## Code of Conduct
 This project and the corresponding community is governed by the [JetBrains Open Source and Community Code of Conduct](https://confluence.jetbrains.com/display/ALL/JetBrains+Open+Source+and+Community+Code+of+Conduct). Please make sure you read it. 

build.gradle.kts

@@ -15,6 +15,7 @@ plugins {
  mavenPublish version Versions.mavenPublish
  dokka version Versions.dokka
  idea
+ kotlin version Versions.kotlin apply false
 }
 
 group = Versions.groupID
@@ -58,6 +59,12 @@ allprojects {
  // ORG_GRADLE_PROJECT_mavenCentralUsername and
  // ORG_GRADLE_PROJECT_mavenCentralPassword.
 
+ // also ORG_GRADLE_PROJECT_signingInMemoryKey=exported_ascii_armored_key
+ // # optional
+ // ORG_GRADLE_PROJECT_signingInMemoryKeyId=24875D73
+ // # if key was created with a password
+ // ORG_GRADLE_PROJECT_signingInMemoryKeyPassword=secret
+
  signAllPublications()
  pom {
  name.set("Kotlin Spark API")
@@ -101,12 +108,11 @@ allprojects {
  }
 
  scm {
- connection.set("scm:git:https://github.com/JetBrains/kotlin-spark-api.git")
- url.set("https://github.com/JetBrains/kotlin-spark-api")
+ connection.set("scm:git:https://github.com/Kotlin/kotlin-spark-api.git")
+ url.set("https://github.com/Kotlin/kotlin-spark-api")
  tag.set("HEAD")
  }
  }
-
  }
  }
 }

buildSrc/src/main/kotlin/Plugins.kt

@@ -4,7 +4,7 @@ import org.gradle.plugin.use.PluginDependenciesSpec
 
 
 inline val PluginDependenciesSpec.kotlin
- get() = kotlin("jvm") version Versions.kotlin
+ get() = kotlin("jvm")
 
 inline val PluginDependenciesSpec.dokka
  get() = id("org.jetbrains.dokka")
@@ -15,7 +15,6 @@ inline val PluginDependenciesSpec.license
 inline val PluginDependenciesSpec.jcp
  get() = id("com.igormaznitsa.jcp")
 
-
 inline val DependencyHandlerScope.jcp
  get() = "com.igormaznitsa:jcp:${Versions.jcp}"
 

buildSrc/src/main/kotlin/Versions.kt

@@ -2,20 +2,20 @@ object Versions {
  const val project = "1.1.1"
  const val groupID = "org.jetbrains.kotlinx.spark"
  const val kotlin = "1.7.10"
+ const val jvmTarget = "11"
 
  inline val spark get() = System.getProperty("spark") as String
  inline val scala get() = System.getProperty("scala") as String
  inline val sparkMinor get() = spark.substringBeforeLast('.')
  inline val scalaCompat get() = scala.substringBeforeLast('.')
 
- const val jupyter = "0.11.0-95"
+ const val jupyter = "0.11.0-134"
  const val kotest = "5.3.2"
  const val kotestTestContainers = "1.3.3"
- const val dokka = "1.7.0"
+ const val dokka = "1.7.10"
  const val jcp = "7.0.5"
  const val mavenPublish = "0.20.0"
  const val atrium = "0.17.0"
- const val kotestExtensionAllure = "1.1.0"
  const val licenseGradlePluginVersion = "0.15.0"
  const val kafkaStreamsTestUtils = "3.1.0"
  const val hadoop = "3.3.1"
@@ -29,6 +29,7 @@ object Versions {
  "scalaCompat" to scalaCompat,
  "spark" to spark,
  "sparkMinor" to sparkMinor,
+ "version" to project,
  )
 
 }

core/build.gradle.kts

@@ -1,8 +1,7 @@
-@file:Suppress("UnstableApiUsage")
+@file:Suppress("UnstableApiUsage", "NOTHING_TO_INLINE")
 
 import com.igormaznitsa.jcp.gradle.JcpTask
 import com.vanniktech.maven.publish.JavaLibrary
-import com.vanniktech.maven.publish.JavadocJar
 import com.vanniktech.maven.publish.JavadocJar.Javadoc
 
 plugins {
@@ -48,6 +47,7 @@ tasks.compileScala {
  outputs.upToDateWhen {
  preprocessMain.outcomingFiles.files.isEmpty()
  }
+
  doFirst {
  scala {
  sourceSets {

core/src/main/scala/org/jetbrains/kotlinx/spark/extensions/KSparkExtensions.scala

@@ -21,9 +21,8 @@ package org.jetbrains.kotlinx.spark.extensions
 
 import org.apache.spark.SparkContext
 import org.apache.spark.sql._
-
 import java.util
-import scala.collection.JavaConverters
+import scala.reflect.ClassTag
 
 object KSparkExtensions {
 
@@ -39,7 +38,13 @@ object KSparkExtensions {
 
  def lit(literal: Any): Column = functions.lit(literal)
 
- def collectAsList[T](ds: Dataset[T]): util.List[T] = JavaConverters.seqAsJavaList(ds.collect())
+ def collectAsList[T](ds: Dataset[T]): util.List[T] = {
+ //#if scalaCompat >= 2.13
+ scala.jdk.javaapi.CollectionConverters.asJava(ds.collect())
+ //#else
+ //$scala.collection.JavaConverters.seqAsJavaList(ds.collect())
+ //#endif
+ }
 
 
  def debugCodegen(df: Dataset[_]): Unit = {
@@ -53,4 +58,17 @@ object KSparkExtensions {
  }
 
  def sparkContext(s: SparkSession): SparkContext = s.sparkContext
+
+ /**
+ * Produces a ClassTag[T], which is actually just a casted ClassTag[AnyRef].
+ *
+ * This method is used to keep ClassTags out of the external Java API, as the Java compiler
+ * cannot produce them automatically. While this ClassTag-faking does please the compiler,
+ * it can cause problems at runtime if the Scala API relies on ClassTags for correctness.
+ *
+ * Often, though, a ClassTag[AnyRef] will not lead to incorrect behavior, just worse performance
+ * or security issues. For instance, an Array[AnyRef] can hold any type T, but may lose primitive
+ * specialization.
+ */
+ def fakeClassTag[T]: ClassTag[T] = ClassTag.AnyRef.asInstanceOf[ClassTag[T]]
 }

examples/build.gradle.kts

@@ -1,3 +1,5 @@
+import org.jetbrains.kotlin.gradle.tasks.KotlinCompile
+
 plugins {
  kotlin
  idea
@@ -28,3 +30,9 @@ dependencies {
 
  }
 }
+
+tasks.withType<KotlinCompile> {
+ kotlinOptions {
+ jvmTarget = Versions.jvmTarget
+ }
+}