💾 spark-gradle-pluginα
Gradle plugin for launching Spark applications.
This plugin is available from the Gradle plugin portal.
Add this in your build.gradle.kts or build.gradle file:
Kotlin
plugins {
id("dev.reimer.spark") version "<version>"
}Groovy
plugins {
id "dev.reimer.spark" version "<version>"
}Alternatively, snapshot builds are available on JitPack.
Add this in your build.gradle.kts or build.gradle file:
Kotlin
buildscript {
repositories {
maven("https://jitpack.io")
}
dependencies {
implementation("dev.reimer:spark-gradle-plugin:<version>")
}
}Groovy
buildscript {
repositories {
maven { url 'https://jitpack.io' }
}
dependencies {
implementation 'dev.reimer:spark-gradle-plugin:<version>'
}
}Run Spark applications by registering tasks:
Kotlin
tasks.register<SparkSubmit>("launchSparkApp") {
applicationResource.set(file("app.jar"))
mainClass.set("com.example.Example")
}Groovy
task launchSparkApp(type: SparkSubmit) {
applicationResource = file("app.jar")
mainClass = "com.example.Example"
}(You may need to import SparkSubmit)
Options can be applied to each task individually.
Kotlin
tasks.register<SparkSubmit>("launchSparkApp") {
applicationResource.set(file("…"))
mainClass.set("…")
applicationArguments.add("…")
applicationName.set("…")
deployMode.set(SparkSubmit.DeployMode.Cluster)
master.set("yarn")
configuration.put("…", 3)
propertiesFile.set(file("…"))
jars.from("…")
files.from("…")
pythonFiles.from("…")
verbose.set(true)
javaHome.set("…")
sparkHome.set("…")
awaitCompletion.set(false)
}Groovy
task launchSparkApp(type: SparkSubmit) {
applicationResource = file("…")
mainClass = "…"
applicationArguments.add("…")
applicationName = "…"
deployMode = SparkSubmit.DeployMode.Cluster
master = "yarn"
configuration["…"] = 3
propertiesFile = file("…")
jars.from("…")
files.from("…")
pythonFiles.from("…")
verbose = true
javaHome = "…"
sparkHome = "…"
awaitCompletion = false
}- The API may be changed at any time without further notice.
- Development still happens on
master. - Pull Requests are highly appreciated!
![dependabot[bot] avatar](https://avatars.githubusercontent.com/in/29110?v=4 "dependabot[bot]")
React
Typescript
Django
Laravel
Facebook
Microsoft
Google
Alibaba
D3
Tencent