benchmarkResult.ipynb

{
 "cells": [
  {
   "metadata": {},
   "cell_type": "code",
   "source": "val reportFolder = \"reports\"",
   "outputs": [],
   "execution_count": null
  },
  {
   "metadata": {},
   "cell_type": "code",
   "source": [
    "import java.io.File\n",
    "import java.util.Properties\n",
    "\n",
    "val properties = Properties()\n",
    "File(\"gradle.properties\").bufferedReader().use {\n",
    "    properties.load(it)\n",
    "}\n",
    "\n",
    "val projectPath = properties.getProperty(\"project.path\") ?: properties.getProperty(\"project.git.url\")\n",
    "val projectName: String = projectPath?.substringAfterLast(\"\\\\\")?.substringAfterLast(\"/\")?.removeSuffix(\".git\") ?: \"User\""
   ],
   "outputs": [],
   "execution_count": null
  },
  {
   "metadata": {},
   "cell_type": "code",
   "source": [
    "enum class CompilerMetrics(val readableName: String) {\n",
    "    GRADLE_TASK(\"Total Gradle task time\"),\n",
    "\n",
    "    COMPILATION_ROUND(\"Sources compilation round\"),\n",
    "    \n",
    "    //compiler metrics\n",
    "    COMPILER_INITIALIZATION(\"Compiler initialization time\"),\n",
    "    CODE_ANALYSIS(\"Compiler code analysis\"),\n",
    "    IR_TRANSLATION(\"Compiler IR translation\"),\n",
    "    IR_LOWERING(\"Compiler IR lowering\"),\n",
    "    IR_GENERATION(\"Compiler IR generation\"),\n",
    "    ;\n",
    "}"
   ],
   "outputs": [],
   "execution_count": null
  },
  {
   "metadata": {},
   "cell_type": "code",
   "source": "%use dataframe",
   "outputs": [],
   "execution_count": null
  },
  {
   "metadata": {},
   "cell_type": "code",
   "source": [
    "import java.io.File\n",
    "import org.jetbrains.kotlinx.dataframe.DataFrame\n",
    "\n",
    "//Json report print time in milliseconds \n",
    "fun readJsonReportFile(file: File): Map<CompilerMetrics, Long?> {\n",
    "    val report = DataFrame.read(file)\n",
    "    val timeMetrics = report.get(\"aggregatedMetrics\").get(\"buildTimes\").get(\"buildTimesNs\")[0] as DataRow<*>\n",
    "\n",
    "    return CompilerMetrics.values().associate {\n",
    "        val value = timeMetrics.getOrNull(it.name)?.let { \n",
    "            when(it) {\n",
    "                is Int -> it.toLong()\n",
    "                is Long -> it\n",
    "                is String -> it.toLong()\n",
    "                else -> throw IllegalStateException(\"Unknown type ${it.javaClass} to convert to Long\")\n",
    "            }\n",
    "        }\n",
    "        Pair(it, value)\n",
    "    }\n",
    "}"
   ],
   "outputs": [],
   "execution_count": null
  },
  {
   "metadata": {},
   "cell_type": "code",
   "source": "",
   "outputs": [],
   "execution_count": null
  },
  {
   "metadata": {},
   "cell_type": "code",
   "source": [
    "import kotlin.streams.toList\n",
    "\n",
    "//Text report print time in seconds with 2 decemal digits, so there is an error during conversion to nanoSeconds\n",
    "fun readTextReportFile(file: File): Map<CompilerMetrics, Long?> {\n",
    "    val aggregatedTimeMetrics = file.bufferedReader().use {\n",
    "        it.lines().dropWhile { !it.startsWith(\"Time metrics:\") }.takeWhile { it.isNotBlank() }.toList()\n",
    "    }\n",
    "\n",
    "    return CompilerMetrics.values().associate { metric ->\n",
    "        val metricLine = aggregatedTimeMetrics.firstOrNull { it.trim().startsWith(metric.readableName) }\n",
    "        val value = metricLine?.substringAfter(\": \")?.substringBefore(\" \")?.replace(\",\", \"\")?.toDouble()\n",
    "        Pair(metric, value?.times(1_000_000_000)?.toLong())\n",
    "    }\n",
    "}"
   ],
   "outputs": [],
   "execution_count": null
  },
  {
   "metadata": {},
   "cell_type": "markdown",
   "source": [
    "# Read compiler metrics\n"
   ]
  },
  {
   "metadata": {},
   "cell_type": "code",
   "source": "%use kandy",
   "outputs": [],
   "execution_count": null
  },
  {
   "metadata": {},
   "cell_type": "code",
   "source": [
    "import org.jetbrains.kotlinx.dataframe.api.toDataFrame\n",
    "import java.nio.file.Files\n",
    "import kotlin.io.path.Path\n",
    "\n",
    "fun readMetrics(\n",
    "    directory: File,\n",
    "    readReportFile: (File) -> Map<CompilerMetrics, Long?>, predicate: (File) -> kotlin.Boolean\n",
    ") = directory.listFiles().filter { !it.name.startsWith(\"buildSrc\") }\n",
    "    .filter(predicate)\n",
    "    .sortedByDescending(File::lastModified)\n",
    "    .toList().take(10).map {\n",
    "         readReportFile(it)\n",
    "    }.toList()\n",
    "\n",
    "fun collectMeanValue(\n",
    "    metrics: List<Map<CompilerMetrics, Long?>>,\n",
    "): Map<String, Any?> {\n",
    "    val rows = CompilerMetrics.values().map { compilerMetric ->\n",
    "        val values = metrics.map { it[compilerMetric] }.filterNotNull()\n",
    "        Pair(compilerMetric.name, values)\n",
    "    }.filter { \n",
    "        val allValuesPresent = (it.second.size == 10)\n",
    "        if (!allValuesPresent) {\n",
    "            println(\"Unable to calculate mean value for ${it.first}, only ${it.second.size} runs are found\")\n",
    "        }\n",
    "        allValuesPresent\n",
    "    }.toTypedArray()\n",
    "\n",
    "    val meanValue = dataFrameOf(*rows).mean()\n",
    "\n",
    "    return CompilerMetrics.values().associate {\n",
    "        Pair(it.name, meanValue.getOrNull(it.name))\n",
    "    }\n",
    "} \n"
   ],
   "outputs": [],
   "execution_count": null
  },
  {
   "metadata": {},
   "cell_type": "markdown",
   "source": ""
  },
  {
   "metadata": {},
   "cell_type": "markdown",
   "source": []
  },
  {
   "metadata": {},
   "cell_type": "markdown",
   "source": "# Collect data into dataframe"
  },
  {
   "metadata": {},
   "cell_type": "code",
   "source": "",
   "outputs": [],
   "execution_count": null
  },
  {
   "metadata": {},
   "cell_type": "code",
   "source": [
    "val kotlinVersionColumn = \"kotlinVersion\"\n",
    "val buildScenarioColumn = \"buildScenario\"\n",
    "\n",
    "val columnNames = CompilerMetrics.values().map { it.name }.toMutableList()\n",
    "columnNames.add(kotlinVersionColumn)\n",
    "columnNames.add(buildScenarioColumn)\n",
    "\n",
    "fun collectResultsIntoDataFrame(\n",
    "    directory: File,\n",
    "    readReportFile: (File) -> Map<CompilerMetrics, Long?>,\n",
    "    predicate: (File) -> kotlin.Boolean,\n",
    "): AnyFrame {\n",
    "    val rows = directory.listFiles().flatMap { versionDir ->\n",
    "        versionDir.listFiles().map { scenarioDir ->\n",
    "            val buildMetrics = readMetrics(scenarioDir, readReportFile, predicate)\n",
    "            //rename scenarios for graphics only\n",
    "            val scenario = when (scenarioDir.name) {\n",
    "                \"clean_build\" -> \"Clean build\"\n",
    "                \"incremental_abi_build\" -> \"Incremental build for ABI changes\"\n",
    "                \"incremental_non_abi_build\" -> \"Incremental build for non ABI changes\"\n",
    "                else -> scenarioDir.name\n",
    "            }\n",
    "\n",
    "            println(\"\\n Validate \\'$scenario\\' scenario for ${versionDir.name} kotlin version\")\n",
    "            val row = collectMeanValue(buildMetrics).toMutableMap()\n",
    "            row.put(kotlinVersionColumn, versionDir.name)\n",
    "            row.put(buildScenarioColumn, scenario)\n",
    "            row\n",
    "        }\n",
    "    }\n",
    "\n",
    "    val db = columnNames.map { compilerMetric -> Pair(compilerMetric, rows.map { it[compilerMetric] })}.toTypedArray()\n",
    "    return dataFrameOf(*db)\n",
    "}"
   ],
   "outputs": [],
   "execution_count": null
  },
  {
   "metadata": {},
   "cell_type": "code",
   "source": [
    "\n",
    "val dataFrame = collectResultsIntoDataFrame(File(reportFolder), ::readTextReportFile){\n",
    "    it.extension == \"txt\" && it.readText().isNotEmpty()\n",
    "}.sortBy(kotlinVersionColumn)\n",
    "val scenarios = dataFrame[buildScenarioColumn]\n"
   ],
   "outputs": [],
   "execution_count": null
  },
  {
   "metadata": {},
   "cell_type": "code",
   "source": [
    "val plots = scenarios.distinct().toList().map { scenario ->\n",
    "    dataFrame.filter { buildScenario == scenario }.plot {\n",
    "        layout.title = \"$projectName project - $scenario\"\n",
    "        bars {\n",
    "            x(kotlinVersionColumn) {\n",
    "                axis.name = \"Kotlin version\"\n",
    "            }\n",
    "            y(CompilerMetrics.COMPILATION_ROUND.name) {\n",
    "                axis.name = \"Compiler build time\"\n",
    "            }\n",
    "        }\n",
    "        \n",
    "    }\n",
    "}\n",
    "plotGrid(plots, 1)"
   ],
   "outputs": [],
   "execution_count": null
  },
  {
   "metadata": {},
   "cell_type": "code",
   "source": "",
   "outputs": [],
   "execution_count": null
  },
  {
   "metadata": {},
   "cell_type": "code",
   "source": [
    "\n",
    "fun gradleBuildTime(scenarios: List<String>, name: String) =\n",
    "    dataFrame.filter { buildScenario in scenarios }.plot {\n",
    "        layout.title = \"$projectName project - $name\"\n",
    "        bars {\n",
    "            x(buildScenarioColumn) {\n",
    "                axis.name = \"Gradle build time\"\n",
    "            }\n",
    "            y(GRADLE_TASK.name) {\n",
    "                axis.name = \"Time in nanoseconds\"\n",
    "            }\n",
    "            fillColor(kotlinVersionColumn) {\n",
    "                legend.name = \"Kotlin version\"\n",
    "            }\n",
    "        }\n",
    "    }\n"
   ],
   "outputs": [],
   "execution_count": null
  },
  {
   "metadata": {},
   "cell_type": "code",
   "source": "",
   "outputs": [],
   "execution_count": null
  },
  {
   "metadata": {},
   "cell_type": "code",
   "source": "gradleBuildTime(listOf(\"Clean build\"), \"Clean build\")",
   "outputs": [],
   "execution_count": null
  },
  {
   "metadata": {},
   "cell_type": "code",
   "source": "gradleBuildTime(listOf(\"Incremental build for ABI changes\", \"Incremental build for non ABI changes\"), \"Incremental build\")",
   "outputs": [],
   "execution_count": null
  },
  {
   "metadata": {},
   "cell_type": "code",
   "source": [
    "fun kotlinCompilationTime(scenarios: List<String>, name: String) =\n",
    "dataFrame.filter { buildScenario in scenarios }.plot {\n",
    "    layout.title = \"$projectName project - $name\"\n",
    "    bars {\n",
    "        x(buildScenarioColumn) {\n",
    "            axis.name = \"Kotlin compilation time\"\n",
    "        }\n",
    "        y(GRADLE_TASK.name) {\n",
    "            axis.name = \"Time in nanoseconds\"\n",
    "        }\n",
    "        fillColor(kotlinVersionColumn) {\n",
    "            legend.name = \"Kotlin version\"\n",
    "        }\n",
    "    }\n",
    "}\n"
   ],
   "outputs": [],
   "execution_count": null
  },
  {
   "metadata": {},
   "cell_type": "code",
   "source": "kotlinCompilationTime(listOf(\"Clean build\"), \"Clean build\")\n",
   "outputs": [],
   "execution_count": null
  },
  {
   "metadata": {},
   "cell_type": "code",
   "source": "kotlinCompilationTime(listOf(\"Incremental build for ABI changes\", \"Incremental build for non ABI changes\"), \"Incremental build\")",
   "outputs": [],
   "execution_count": null
  },
  {
   "metadata": {},
   "cell_type": "markdown",
   "source": "# Compare Gradle and Compiler time"
  },
  {
   "metadata": {},
   "cell_type": "code",
   "source": [
    "\n",
    " fun getPlotForGradleCompilerTime(scenario: String) = dataFrame.filter { it[\"buildScenario\"] == scenario }\n",
    "        .gather { GRADLE_TASK.name and COMPILATION_ROUND.name }.into(\"metric\", \"time_ns\")\n",
    "        .update(\"metric\").with {\n",
    "            when (it) {\n",
    "                \"GRADLE_TASK\"  -> \"Gradle time\"\n",
    "                \"COMPILATION_ROUND\" -> \"Kotlin Compiler time\"\n",
    "                else -> throw IllegalStateException(\"Unknown metrics: $it\")\n",
    "            }\n",
    "        }\n",
    "        .sortBy(\"time_ns\")\n",
    "        .plot {\n",
    "            layout {\n",
    "                title = \"$projectName project\"\n",
    "                subtitle = scenario\n",
    "                xAxisLabel = \"Gradle compilation breakdown\"\n",
    "                theme() {\n",
    "                    legend.position = LegendPosition.Top\n",
    "                }\n",
    "            }\n",
    "            barsH {\n",
    "                y(kotlinVersion) {\n",
    "                    axis.name = \"Kotin version\"\n",
    "                }\n",
    "                x(\"time_ns\"<Double>()) {\n",
    "                    axis.name = \"Time in nanoseconds\"\n",
    "                }\n",
    "                fillColor(\"metric\") {\n",
    "                    legend.name = \"Build time\"\n",
    "                }\n",
    "                position = Position.identity()\n",
    "            }\n",
    "        }\n",
    "\n",
    "\n"
   ],
   "outputs": [],
   "execution_count": null
  },
  {
   "metadata": {},
   "cell_type": "code",
   "source": [
    "val versionByScenario =\n",
    "    scenarios.distinct().toList().associate { name -> Pair(name as String, getPlotForGradleCompilerTime(name)) }"
   ],
   "outputs": [],
   "execution_count": null
  },
  {
   "metadata": {},
   "cell_type": "code",
   "source": "versionByScenario.get(\"Clean build\")",
   "outputs": [],
   "execution_count": null
  },
  {
   "metadata": {},
   "cell_type": "code",
   "source": "versionByScenario.get(\"Incremental build for non ABI changes\")",
   "outputs": [],
   "execution_count": null
  },
  {
   "metadata": {},
   "cell_type": "code",
   "source": "versionByScenario.get(\"Incremental build for ABI changes\")",
   "outputs": [],
   "execution_count": null
  },
  {
   "metadata": {},
   "cell_type": "markdown",
   "source": "# Collect Compiler metrics"
  },
  {
   "metadata": {},
   "cell_type": "code",
   "source": [
    "import org.jetbrains.kotlinx.kandy.ir.scale.PositionalContinuousScale\n",
    "import org.jetbrains.kotlinx.kandy.ir.scale.PositionalScale\n",
    "\n",
    "val metricsList = dataFrame\n",
    "    .gather { COMPILER_INITIALIZATION.name and CODE_ANALYSIS.name and IR_TRANSLATION.name and IR_LOWERING.name and IR_GENERATION.name }.into(\"metric\", \"time_ns\")\n",
    "\n",
    "\n",
    "fun getPlotForCompilerMetrics(scenario: String) = metricsList.filter { it[\"buildScenario\"] == scenario }\n",
    "    .plot {\n",
    "    layout {\n",
    "        title = \"$projectName project\"\n",
    "        subtitle = scenario\n",
    "        xAxisLabel = \"Kotlin compilation breakdown\"\n",
    "        theme() {\n",
    "            legend.position = LegendPosition.Top\n",
    "        }\n",
    "        size = Pair(1000, 300)\n",
    "    }\n",
    "    barsH {\n",
    "        y(kotlinVersion) {\n",
    "            axis.name = \"Kotlin version\"\n",
    "        }\n",
    "        x(\"time_ns\"<Double>()) {\n",
    "            axis.name = \"Time in nanoseconds\"\n",
    "        }\n",
    "        fillColor(\"metric\")\n",
    "        position = Position.stack()\n",
    "        \n",
    "    }\n",
    "}\n"
   ],
   "outputs": [],
   "execution_count": null
  },
  {
   "metadata": {},
   "cell_type": "code",
   "source": "val compilerMetrics = scenarios.distinct().toList().associate { Pair(it as String, getPlotForCompilerMetrics(it)) }",
   "outputs": [],
   "execution_count": null
  },
  {
   "metadata": {},
   "cell_type": "code",
   "source": "compilerMetrics.get(\"Clean build\") \n",
   "outputs": [],
   "execution_count": null
  },
  {
   "metadata": {},
   "cell_type": "code",
   "source": "compilerMetrics.get(\"Incremental build for non ABI changes\") ",
   "outputs": [],
   "execution_count": null
  },
  {
   "metadata": {},
   "cell_type": "code",
   "source": "compilerMetrics.get(\"Incremental build for ABI changes\") ",
   "outputs": [],
   "execution_count": null
  },
  {
   "metadata": {},
   "cell_type": "code",
   "source": "",
   "outputs": [],
   "execution_count": null
  },
  {
   "metadata": {},
   "cell_type": "code",
   "source": "",
   "outputs": [],
   "execution_count": null
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Kotlin",
   "language": "kotlin",
   "name": "kotlin"
  },
  "language_info": {
   "name": "kotlin",
   "version": "1.9.0",
   "mimetype": "text/x-kotlin",
   "file_extension": ".kt",
   "pygments_lexer": "kotlin",
   "codemirror_mode": "text/x-kotlin",
   "nbconvert_exporter": ""
  },
  "ktnbPluginMetadata": {
   "projectDependencies": [
    "buildSrc"
   ]
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}