Skip to content

Instantly share code, notes, and snippets.

@dyno

dyno/SharedSparkContext.scala

Created March 10, 2022 22:20
Show Gist options
  • Save dyno/c81922d5e35a3d6c98a874bbaeee90c0 to your computer and use it in GitHub Desktop.
Save dyno/c81922d5e35a3d6c98a874bbaeee90c0 to your computer and use it in GitHub Desktop.
Download ZIP

Revisions

  1. dyno created this gist Mar 10, 2022.
    24 changes: 24 additions & 0 deletions SharedSparkContext.scala
    View file
    Original file line number Diff line number Diff line change
    @@ -0,0 +1,24 @@
    import org.apache.spark.sql.SparkSession
    import org.apache.spark.{SparkConf, SparkContext}
    import org.scalatest.{BeforeAndAfterAll, Suite}

    // https://www.slideshare.net/SparkSummit/spark-summit-eu-talk-by-ted-malaska
    trait SharedSparkContext extends BeforeAndAfterAll { self: Suite =>
    @transient private var _sc: SparkContext = _

    def sc: SparkContext = _sc
    def spark: SparkSession = SparkSession.builder().config(_sc.getConf).getOrCreate()

    override def beforeAll() {
    super.beforeAll()

    val conf = new SparkConf().setAppName("applogs-test").setMaster("local[2]")
    _sc = new SparkContext(conf)
    }

    override def afterAll() {
    _sc.stop()

    super.afterAll()
    }
    }