tests

Author: mtumilowicz

build.sbt

@@ -5,3 +5,5 @@ version := "0.1"
 scalaVersion := "2.12.10"
 
 libraryDependencies += "org.apache.spark" %% "spark-sql" % "3.0.1"
+libraryDependencies += "org.scalatest" %% "scalatest" % "3.1.2" % "test"
+libraryDependencies += "org.scalacheck" %% "scalacheck" % "1.14.1" % "test"

src/main/scala/app/SimpleApp.scala

@@ -7,7 +7,7 @@ object SimpleApp {
     val spark = bootstrapSpark()
     val logFile = "Task1.txt"
     val logData: Dataset[String] = spark.read.textFile(logFile).cache()
-    val fileAnalysis = new FileAnalysis(logData)
+    val fileAnalysis = new TextFileAnalysis(logData)
     val numAs = fileAnalysis.lineContaining("a").count()
     val numBs = fileAnalysis.lineContaining("b").count()
     val numCs = fileAnalysis.lineContaining("c").count()
@@ -18,6 +18,6 @@ object SimpleApp {
   def bootstrapSpark(): SparkSession =
     SparkSession.builder
       .appName("Simple Application")
-      .config("spark.master", "local")
+      .master("local")
       .getOrCreate()
 }

src/main/scala/app/FileAnalysis.scala renamed to src/main/scala/app/TextFileAnalysis.scala

@@ -2,6 +2,6 @@ package app
 
 import org.apache.spark.sql.Dataset
 
-class FileAnalysis(file: Dataset[String]) {
+class TextFileAnalysis(file: Dataset[String]) {
   def lineContaining(str: String): Dataset[String] = file.filter(_.contains(str))
 }

src/test/scala/app/SparkSessionTestWrapper.scala

@@ -0,0 +1,15 @@
+package app
+
+import org.apache.spark.sql.SparkSession
+
+trait SparkSessionTestWrapper {
+
+  lazy val spark: SparkSession = {
+    SparkSession
+      .builder()
+      .master("local")
+      .appName("spark test example")
+      .getOrCreate()
+  }
+
+}

src/test/scala/app/TextFileAnalysisITTest.scala

@@ -0,0 +1,23 @@
+package app
+
+import org.scalatest.matchers.should.Matchers.convertToAnyShouldWrapper
+
+
+class TextFileAnalysisITTest extends org.scalatest.FunSuite with SparkSessionTestWrapper {
+
+  test("testLineContaining") {
+    import spark.implicits._
+
+    val dataset = Seq("a", "a", "a", "b").toDS()
+    val fileAnalysis2 = new TextFileAnalysis(dataset)
+    val numAs2 = fileAnalysis2.lineContaining("a").count()
+    val numBs2 = fileAnalysis2.lineContaining("b").count()
+    val numCs2 = fileAnalysis2.lineContaining("c").count()
+
+    numAs2 shouldBe 3
+    numBs2 shouldBe 1
+    numCs2 shouldBe 0
+
+  }
+
+}