feat(tests): add lang heuristics testing (#278)

* feat(tests) - add lang heuristics testing

* chore: style change

* Add .gitattributes to exclude sampels from github stats

Author: asurkov

.gitattributes

@@ -0,0 +1 @@
+src/test/resources/samples/* linguist-vendored

build.gradle

@@ -107,6 +107,7 @@ dependencies {
     compile 'org.slf4j:slf4j-nop:1.7.2'
     compile 'io.sentry:sentry:1.7.3'
 
+    testCompile 'commons-io:commons-io:2.6'
     testCompile 'org.jetbrains.kotlin:kotlin-test'
     testCompile 'org.jetbrains.spek:spek-api:1.1.5'
     testCompile 'org.junit.platform:junit-platform-runner:1.0.1'

src/main/kotlin/app/extractors/Heuristics.kt

@@ -326,7 +326,7 @@ val Heuristics = mapOf<String, (List<String>) -> ExtractorInterface?>(
         else if (Regex("^\\s*;").matches(buf)) CommonExtractor("m")
         else if (Regex("\\*\\)$").matches(buf)) CommonExtractor("mathematica")
         else if (Regex("^\\s*%").matches(buf)) CommonExtractor("matlab")
-        else if (Regex("^\\w+\\s*:\\s*module\\s*{").matches(buf)) CommonExtractor("limbo")
+        else if (Regex("^\\w+\\s*:\\s*module\\s*\\{").matches(buf)) CommonExtractor("limbo")
         else CommonExtractor("wolframlanguage")
     },
     "makefile" to { _ ->

src/test/kotlin/test/tests/extractors/DirToLangMap.kt

@@ -0,0 +1,12 @@
+// Copyright 2018 Sourcerer Inc. All Rights Reserved.
+// Author: Alexander Surkov ([email protected])
+
+package test.tests.extractors
+
+// TODO(anatoly): Map all supported languages.
+val dirToLangMap = mapOf(
+    "C" to "c",
+    "C#" to "csharp",
+    "C++" to "cpp",
+    "CSS" to "css"
+)

src/test/kotlin/test/tests/extractors/HeuristicsTest.kt

@@ -0,0 +1,54 @@
+// Copyright 2018 Sourcerer Inc. All Rights Reserved.
+// Author: Alexander Surkov ([email protected])
+
+package test.tests.extractors
+
+import app.Logger
+import app.model.DiffFile
+import app.model.DiffContent
+import app.extractors.*
+import org.apache.commons.io.FilenameUtils
+import org.eclipse.jgit.diff.DiffEntry.ChangeType
+import org.jetbrains.spek.api.Spek
+import org.jetbrains.spek.api.dsl.given
+import org.jetbrains.spek.api.dsl.it
+import java.io.File
+import kotlin.test.assertEquals
+
+const val LANG_SAMPLES_PATH = "src/test/resources/samples/"
+
+fun assertLang(file: File, expectedLang: String) {
+    val diffFile = DiffFile(
+        file.path,
+        changeType = ChangeType.ADD,
+        new = DiffContent(content = file.readLines())
+    )
+    Extractor().extract(listOf(diffFile))
+
+    val actualLang = diffFile.language
+
+    // TODO(anatoly): Verify all sample files from ignored list.
+    // TODO(anatoly): Add support for all languages of samples.
+    for (wc in ignoredSamplesWildcards) {
+        if (FilenameUtils.wildcardMatchOnSystem(file.path, wc)) {
+            Logger.debug { "-> File: ${file.absolutePath}. " +
+                "Expected: <$expectedLang>, actual: <$actualLang>" }
+            return
+        }
+    }
+
+    assertEquals(expectedLang, actualLang, "Unexpected lang for ${file.path}")
+}
+
+class HeuristicsTest : Spek({
+    given("heuristics test") {
+        it("all language samples") {
+            for (dir in File(LANG_SAMPLES_PATH).listFiles()) {
+                val expectedLang = dirToLangMap.getOrDefault(dir.name, dir.name)
+                for (file in dir.walkTopDown()) {
+                    if (file.isFile) assertLang(file, expectedLang)
+                }
+            }
+        }
+    }
+})