Change to per suite for Scala

scalatest-listener/build.sbt

@@ -19,8 +19,6 @@ libraryDependencies ++= Seq(
   "org.mockito" %% "mockito-scala-scalatest" % "1.17.12" % Test
 )
 
-Test / testOptions += Tests.Argument(TestFrameworks.ScalaTest, "-C", "io.agodadev.testmetricsscala.TestMetricsReporter")
-
 // Maven Central publishing settings
 publishMavenStyle := true
 publishTo := sonatypePublishToBundle.value

scalatest-listener/src/main/scala/io/agodadev/testmetricsscala/TestMetricsReporter.scala

@@ -25,78 +25,87 @@ class TestMetricsReporter extends Reporter {
   private val endpointUrl = sys.env.getOrElse("BUILD_METRICS_ES_ENDPOINT", "http://compilation-metrics/scala/scalatest")
 
   case class TestCaseInfo(
-    id: String,
-    name: String,
-    suiteName: String,
-    status: String,
-    startTime: Instant,
-    endTime: Instant = Instant.now(),
-    duration: Long = 0
-  )
+                           id: String,
+                           name: String,
+                           suiteName: String,
+                           status: String,
+                           startTime: Instant,
+                           endTime: Instant = Instant.now(),
+                           duration: Long = 0
+                         )
+
+  case class SuiteInfo(
+                        name: String,
+                        startTime: Instant,
+                        testCases: mutable.Map[String, TestCaseInfo] = mutable.Map()
+                      )
+
+  private val suites = mutable.Map[String, SuiteInfo]()
 
   protected def createHttpRequest(url: String): HttpRequest = Http(url)
 
   override def apply(event: Event): Unit = event match {
     case RunStarting(ordinal, testCount, configMap, formatter, location, payload, threadName, timeStamp) =>
-      suiteStartTime = Instant.now()
       totalTests = testCount
 
+    case SuiteStarting(ordinal, suiteName, suiteId, suiteClassName, formatter, location, rerunner, payload, threadName, timeStamp) =>
+      suites += (suiteName -> SuiteInfo(suiteName, Instant.now()))
+
     case TestStarting(ordinal, suiteName, suiteId, suiteClassName, testName, testText, formatter, location, rerunner, payload, threadName, timeStamp) =>
       val testId = UUID.randomUUID().toString
-      testCases += (testId -> TestCaseInfo(testId, testName, suiteName, "Started", Instant.now()))
+      val testInfo = TestCaseInfo(testId, testName, suiteName, "Started", Instant.now())
+      testCases += (testId -> testInfo)
+      suites(suiteName).testCases += (testId -> testInfo)
 
     case TestSucceeded(ordinal, suiteName, suiteId, suiteClassName, testName, testText, recordedEvents, duration, formatter, location, rerunner, payload, threadName, timeStamp) =>
-      updateTestCase(testName, "Passed", duration)
+      updateTestCase(suiteName, testName, "Passed", duration)
       succeededTests += 1
 
     case TestFailed(ordinal, message, suiteName, suiteId, suiteClassName, testName, testText, recordedEvents, analysis, throwable, duration, formatter, location, rerunner, payload, threadName, timeStamp) =>
-      updateTestCase(testName, "Failed", duration)
+      updateTestCase(suiteName, testName, "Failed", duration)
       failedTests += 1
 
     case TestIgnored(ordinal, suiteName, suiteId, suiteClassName, testName, testText, formatter, location, payload, threadName, timeStamp) =>
       val testId = UUID.randomUUID().toString
-      testCases += (testId -> TestCaseInfo(testId, testName, suiteName, "Ignored", Instant.now()))
+      val testInfo = TestCaseInfo(testId, testName, suiteName, "Ignored", Instant.now())
+      testCases += (testId -> testInfo)
+      suites(suiteName).testCases += (testId -> testInfo)
       ignoredTests += 1
 
-    case RunCompleted(ordinal, duration, summary, formatter, location, payload, threadName, timeStamp) =>
-      val jsonReport = generateJsonReport(summary.getOrElse(Summary(0, 0, 0, 0, 0, 0, 0, 0)), duration)
+    case SuiteCompleted(ordinal, suiteName, suiteId, suiteClassName, duration, formatter, location, rerunner, payload, threadName, timeStamp) =>
+      val jsonReport = generateJsonReport(suiteName, duration)
       sendReportToEndpoint(jsonReport)
+      // Clear the suite data after sending the report
+      suites.remove(suiteName)
 
     case _ => // Ignore other events
   }
 
-  private def updateTestCase(testName: String, status: String, duration: Option[Long]): Unit = {
-    testCases.find(_._2.name == testName).foreach { case (id, testCase) =>
-      testCases(id) = testCase.copy(
+  private def updateTestCase(suiteName: String, testName: String, status: String, duration: Option[Long]): Unit = {
+    suites(suiteName).testCases.find(_._2.name == testName).foreach { case (id, testCase) =>
+      val updatedTestCase = testCase.copy(
         status = status,
         endTime = Instant.now(),
         duration = duration.getOrElse(0)
       )
+      suites(suiteName).testCases(id) = updatedTestCase
+      testCases(id) = updatedTestCase
     }
   }
 
-  private def generateJsonReport(summary: Summary, duration: Option[Long]): ObjectNode = {
+  private def generateJsonReport(suiteName: String, duration: Option[Long]): ObjectNode = {
     val rootNode = objectMapper.createObjectNode()
-    // Update runId logic to check for CI_JOB_ID
-    val runId = sys.env.get("CI_JOB_ID") match {
-      case Some(ciJobId) if ciJobId.nonEmpty => ciJobId
-      case _ => UUID.randomUUID().toString
-    }
+    val runId = sys.env.getOrElse("CI_JOB_ID", UUID.randomUUID().toString)
     rootNode.put("id", runId)
 
-    // Update username logic to check for GITLAB_USER_LOGIN
-    val userName = sys.env.get("GITLAB_USER_LOGIN") match {
-      case Some(gitlabUser) if gitlabUser.nonEmpty => gitlabUser
-      case _ => System.getProperty("user.name")
-    }
+    val userName = sys.env.getOrElse("GITLAB_USER_LOGIN", System.getProperty("user.name"))
     rootNode.put("userName", userName)
     rootNode.put("cpuCount", Runtime.getRuntime().availableProcessors())
     rootNode.put("hostname", java.net.InetAddress.getLocalHost.getHostName)
     rootNode.put("os", s"${System.getProperty("os.name")} ${System.getProperty("os.version")}")
     rootNode.put("platform", determinePlatform())
     rootNode.put("isDebuggerAttached", java.lang.management.ManagementFactory.getRuntimeMXBean.getInputArguments.toString.contains("-agentlib:jdwp"))
 
-    // Add Git context information
     Try(GitContextReader.getGitContext()) match {
       case Success(gitContext) =>
         rootNode.put("repositoryUrl", gitContext.repositoryUrl)
@@ -107,7 +116,8 @@ class TestMetricsReporter extends Reporter {
     }
 
     val testCasesNode = rootNode.putArray("scalaTestCases")
-    testCases.values.foreach { testCase =>
+    val suiteInfo = suites(suiteName)
+    suiteInfo.testCases.values.foreach { testCase =>
       val testCaseNode = testCasesNode.addObject()
       testCaseNode.put("id", testCase.id)
       testCaseNode.put("name", testCase.name)
@@ -118,15 +128,11 @@ class TestMetricsReporter extends Reporter {
       testCaseNode.put("duration", testCase.duration)
     }
 
-    rootNode.put("totalTests", totalTests)
-    rootNode.put("succeededTests", succeededTests)
-    rootNode.put("failedTests", failedTests)
-    rootNode.put("ignoredTests", ignoredTests)
-    rootNode.put("pendingTests", summary.testsPendingCount)
-    rootNode.put("canceledTests", summary.testsCanceledCount)
-    rootNode.put("completedSuites", summary.suitesCompletedCount)
-    rootNode.put("abortedSuites", summary.suitesAbortedCount)
-    rootNode.put("pendingScopes", summary.scopesPendingCount)
+    rootNode.put("suiteName", suiteName)
+    rootNode.put("totalTests", suiteInfo.testCases.size)
+    rootNode.put("succeededTests", suiteInfo.testCases.count(_._2.status == "Passed"))
+    rootNode.put("failedTests", suiteInfo.testCases.count(_._2.status == "Failed"))
+    rootNode.put("ignoredTests", suiteInfo.testCases.count(_._2.status == "Ignored"))
     rootNode.put("runTime", duration.getOrElse(0L))
     rootNode.put("runId", UUID.randomUUID().toString)
 
@@ -159,4 +165,4 @@ class TestMetricsReporter extends Reporter {
     else if (System.getenv("AWS_EXECUTION_ENV") != null) "AWS"
     else "JVM"
   }
-}
+}

scalatest-listener/src/test/scala/io/agodadev/testmetricsscala/TestMetricsReporterSpec.scala

@@ -1,23 +1,22 @@
 package io.agodadev.testmetricsscala
-
+import com.fasterxml.jackson.databind.{JsonNode, ObjectMapper}
+import org.mockito.ArgumentMatchers.{any, argThat}
 import org.scalatest.funspec.AnyFunSpec
 import org.scalatest.matchers.should.Matchers
-import org.scalatest.events._
-import org.scalatest.ConfigMap
-import com.fasterxml.jackson.databind.{JsonNode, ObjectMapper}
-import com.fasterxml.jackson.module.scala.DefaultScalaModule
 import org.mockito.MockitoSugar
-import org.mockito.ArgumentMatchers.{any, argThat}
+import org.scalatest.events._
 import scalaj.http.{HttpRequest, HttpResponse}
 import scalaj.http.HttpOptions.HttpOption
 
+// Use scala.collection.JavaConverters for Scala 2.12 compatibility
+import scala.collection.JavaConverters._
+
 class TestMetricsReporterSpec extends AnyFunSpec with Matchers with MockitoSugar {
-  
-  val objectMapper = new ObjectMapper().registerModule(DefaultScalaModule)
+
+  val objectMapper = new ObjectMapper().registerModule(new com.fasterxml.jackson.module.scala.DefaultScalaModule)
 
   describe("TestMetricsReporter") {
-    it("should handle test events and send correct JSON report to endpoint") {
-      // Mock HTTP request and response
+    it("should handle multiple test cases within a suite") {
       val mockRequest = mock[HttpRequest]
       val mockResponse = mock[HttpResponse[String]]
 
@@ -27,36 +26,62 @@ class TestMetricsReporterSpec extends AnyFunSpec with Matchers with MockitoSugar
       when(mockRequest.asString).thenReturn(mockResponse)
       when(mockResponse.isSuccess).thenReturn(true)
 
-      // Create a TestMetricsReporter with our mocked HTTP client
       val reporter = new TestMetricsReporter {
         override protected def createHttpRequest(url: String): HttpRequest = mockRequest
       }
-
-      val runStartingEvent = RunStarting(ordinal = new Ordinal(1), testCount = 3, configMap = ConfigMap.empty, formatter = None, location = None, payload = None, threadName = "main", timeStamp = 123)
-      val testStartingEvent = TestStarting(ordinal = new Ordinal(2), suiteName = "TestSuite", suiteId = "suiteId", suiteClassName = Some("TestSuite"), testName = "test1", testText = "should do something", formatter = None, location = None, rerunner = None, payload = None, threadName = "main", timeStamp = 124)
-      val testSucceededEvent = TestSucceeded(ordinal = new Ordinal(3), suiteName = "TestSuite", suiteId = "suiteId", suiteClassName = Some("TestSuite"), testName = "test1", testText = "should do something", recordedEvents = Vector.empty, duration = Some(100), formatter = None, location = None, rerunner = None, payload = None, threadName = "main", timeStamp = 224)
-      val runCompletedEvent = RunCompleted(ordinal = new Ordinal(4), duration = Some(200), summary = Some(Summary(testsSucceededCount = 1, testsFailedCount = 0, testsIgnoredCount = 0, testsPendingCount = 0, testsCanceledCount = 0, suitesCompletedCount = 1, suitesAbortedCount = 0, scopesPendingCount = 0)), formatter = None, location = None, payload = None, threadName = "main", timeStamp = 324)
-
-      reporter(runStartingEvent)
-      reporter(testStartingEvent)
-      reporter(testSucceededEvent)
-      reporter(runCompletedEvent)
-
-      // Verify that HTTP request was made with correct data
+
+      val suiteStartingEvent = SuiteStarting(ordinal = new Ordinal(1), suiteName = "TestSuite", suiteId = "suiteId", suiteClassName = Some("TestSuite"), formatter = None, location = None, rerunner = None, payload = None, threadName = "main", timeStamp = 100)
+      val test1StartingEvent = TestStarting(ordinal = new Ordinal(2), suiteName = "TestSuite", suiteId = "suiteId", suiteClassName = Some("TestSuite"), testName = "test1", testText = "should do something", formatter = None, location = None, rerunner = None, payload = None, threadName = "main", timeStamp = 101)
+      val test1SucceededEvent = TestSucceeded(ordinal = new Ordinal(3), suiteName = "TestSuite", suiteId = "suiteId", suiteClassName = Some("TestSuite"), testName = "test1", testText = "should do something", recordedEvents = Vector.empty[RecordableEvent], duration = Some(50), formatter = None, location = None, rerunner = None, payload = None, threadName = "main", timeStamp = 151)
+      val test2StartingEvent = TestStarting(ordinal = new Ordinal(4), suiteName = "TestSuite", suiteId = "suiteId", suiteClassName = Some("TestSuite"), testName = "test2", testText = "should do something else", formatter = None, location = None, rerunner = None, payload = None, threadName = "main", timeStamp = 152)
+      val test2FailedEvent = TestFailed(ordinal = new Ordinal(5), message = "assertion failed", suiteName = "TestSuite", suiteId = "suiteId", suiteClassName = Some("TestSuite"), testName = "test2", testText = "should do something else", recordedEvents = Vector.empty[RecordableEvent], analysis = Vector.empty[String], throwable = None, duration = Some(75), formatter = None, location = None, rerunner = None, payload = None, threadName = "main", timeStamp = 227)
+      val suiteCompletedEvent = SuiteCompleted(ordinal = new Ordinal(6), suiteName = "TestSuite", suiteId = "suiteId", suiteClassName = Some("TestSuite"), duration = Some(150), formatter = None, location = None, rerunner = None, payload = None, threadName = "main", timeStamp = 250)
+
+      reporter(suiteStartingEvent)
+      reporter(test1StartingEvent)
+      reporter(test1SucceededEvent)
+      reporter(test2StartingEvent)
+      reporter(test2FailedEvent)
+      reporter(suiteCompletedEvent)
+
       verify(mockRequest).postData(argThat { json: String =>
         val jsonNode = objectMapper.readTree(json)
-        jsonNode.get("totalTests").asInt() == 3 &&
-        jsonNode.get("succeededTests").asInt() == 1 &&
-        jsonNode.get("failedTests").asInt() == 0 &&
-        jsonNode.get("ignoredTests").asInt() == 0 &&
-        jsonNode.get("runTime").asLong() == 200 &&
-        jsonNode.get("scalaTestCases").isArray &&
-        jsonNode.get("scalaTestCases").size() == 1 &&
-        jsonNode.get("scalaTestCases").get(0).get("name").asText() == "test1" &&
-        jsonNode.get("scalaTestCases").get(0).get("status").asText() == "Passed"
+        println(s"Received JSON: ${objectMapper.writeValueAsString(jsonNode)}")
+        val result = validateJsonContent(jsonNode)
+        if (!result) {
+          println("JSON validation failed. Details:")
+          println(s"suiteName: ${jsonNode.get("suiteName").asText()}")
+          println(s"totalTests: ${jsonNode.get("totalTests").asInt()}")
+          println(s"succeededTests: ${jsonNode.get("succeededTests").asInt()}")
+          println(s"failedTests: ${jsonNode.get("failedTests").asInt()}")
+          println(s"ignoredTests: ${jsonNode.get("ignoredTests").asInt()}")
+          println(s"runTime: ${jsonNode.get("runTime").asLong()}")
+          println(s"scalaTestCases: ${jsonNode.get("scalaTestCases").toString}")
+        }
+        result
       })
       verify(mockRequest).asString
       verify(mockResponse).isSuccess
     }
   }
+
+  def validateJsonContent(jsonNode: JsonNode): Boolean = {
+    val testCases = jsonNode.get("scalaTestCases")
+
+    jsonNode.get("suiteName").asText() == "TestSuite" &&
+      jsonNode.get("totalTests").asInt() == 2 &&
+      jsonNode.get("succeededTests").asInt() == 1 &&
+      jsonNode.get("failedTests").asInt() == 1 &&
+      jsonNode.get("ignoredTests").asInt() == 0 &&
+      jsonNode.get("runTime").asLong() == 150 &&
+      testCases.isArray &&
+      testCases.size() == 2 &&
+      testCasesContain(testCases, "test1", "Passed") &&
+      testCasesContain(testCases, "test2", "Failed")
+  }
+
+  def testCasesContain(testCases: JsonNode, name: String, status: String): Boolean = {
+    // Use JavaConverters for Scala 2.12 compatibility
+    testCases.asScala.exists(tc => tc.get("name").asText() == name && tc.get("status").asText() == status)
+  }
 }