Skip to content

Commit 8f454d0

Browse files
SparkConfig refactor (#90)
1 parent 656d01c commit 8f454d0

File tree

1 file changed

+4
-8
lines changed

1 file changed

+4
-8
lines changed

nebula-algorithm/src/main/scala/com/vesoft/nebula/algorithm/config/SparkConfig.scala

+4-8
Original file line numberDiff line numberDiff line change
@@ -11,26 +11,22 @@ case class SparkConfig(spark: SparkSession, partitionNum: Int)
1111

1212
object SparkConfig {
1313

14-
var spark: SparkSession = _
15-
16-
var partitionNum: String = _
17-
1814
def getSpark(configs: Configs, defaultAppName: String = "algorithm"): SparkConfig = {
1915
val sparkConfigs = configs.sparkConfig.map
2016
val session = SparkSession.builder
2117
.appName(defaultAppName)
2218
.config("spark.serializer", "org.apache.spark.serializer.KryoSerializer")
2319

24-
for (key <- sparkConfigs.keySet) {
25-
session.config(key, sparkConfigs(key))
20+
sparkConfigs.foreach { case (key, value) =>
21+
session.config(key, value)
2622
}
27-
partitionNum = sparkConfigs.getOrElse("spark.app.partitionNum", "0")
23+
val partitionNum = sparkConfigs.getOrElse("spark.app.partitionNum", "0")
2824
val spark = session.getOrCreate()
2925
validate(spark.version, "2.4.*")
3026
SparkConfig(spark, partitionNum.toInt)
3127
}
3228

33-
def validate(sparkVersion: String, supportedVersions: String*): Unit = {
29+
private def validate(sparkVersion: String, supportedVersions: String*): Unit = {
3430
if (sparkVersion != "UNKNOWN" && !supportedVersions.exists(sparkVersion.matches)) {
3531
throw new RuntimeException(
3632
s"""Your current spark version ${sparkVersion} is not supported by the current NebulaGraph Algorithm.

0 commit comments

Comments
 (0)