Fix build metrics for JPS
diff --git a/compiler/build-tools/kotlin-build-statistics/src/org/jetbrains/kotlin/build/report/metrics/BuildTime.kt b/compiler/build-tools/kotlin-build-statistics/src/org/jetbrains/kotlin/build/report/metrics/BuildTime.kt
index abdb7e0..686032c 100644
--- a/compiler/build-tools/kotlin-build-statistics/src/org/jetbrains/kotlin/build/report/metrics/BuildTime.kt
+++ b/compiler/build-tools/kotlin-build-statistics/src/org/jetbrains/kotlin/build/report/metrics/BuildTime.kt
@@ -21,9 +21,11 @@
enum class JpsBuildTime(private val parent: JpsBuildTime? = null, private val readableString: String) : BuildTime {
-
- JPS_ITERATION(readableString = "Jps iteration")
+ // @formatter:off
+ JPS_ITERATION(readableString = "Jps iteration"),
+ COMPILATION_ROUND(JPS_ITERATION, "Sources compilation round"),
;
+ // @formatter:on
override fun getReadableString(): String = readableString
override fun getParent(): BuildTime? = parent
diff --git a/jps/jps-plugin/src/org/jetbrains/kotlin/jps/build/KotlinBuilder.kt b/jps/jps-plugin/src/org/jetbrains/kotlin/jps/build/KotlinBuilder.kt
index c4aa99c..339e8c3 100644
--- a/jps/jps-plugin/src/org/jetbrains/kotlin/jps/build/KotlinBuilder.kt
+++ b/jps/jps-plugin/src/org/jetbrains/kotlin/jps/build/KotlinBuilder.kt
@@ -23,7 +23,9 @@
import org.jetbrains.kotlin.build.report.ICReporter.ReportSeverity
import org.jetbrains.kotlin.build.report.ICReporterBase
import org.jetbrains.kotlin.build.report.debug
+import org.jetbrains.kotlin.build.report.metrics.BuildMetricsReporter
import org.jetbrains.kotlin.build.report.metrics.JpsBuildTime
+import org.jetbrains.kotlin.build.report.metrics.measure
import org.jetbrains.kotlin.cli.common.ExitCode
import org.jetbrains.kotlin.cli.common.arguments.CommonCompilerArguments
import org.jetbrains.kotlin.cli.common.messages.CompilerMessageSeverity.ERROR
@@ -43,6 +45,7 @@
import org.jetbrains.kotlin.jps.incremental.JpsIncrementalCache
import org.jetbrains.kotlin.jps.incremental.JpsLookupStorageManager
import org.jetbrains.kotlin.jps.model.kotlinKind
+import org.jetbrains.kotlin.jps.statistic.JpsBuilderMetricReporter
import org.jetbrains.kotlin.jps.statistic.JpsStatisticsReportService
import org.jetbrains.kotlin.jps.targets.KotlinJvmModuleBuildTarget
import org.jetbrains.kotlin.jps.targets.KotlinModuleBuildTarget
@@ -217,7 +220,7 @@
private fun markAdditionalFilesForInitialRound(
kotlinChunk: KotlinChunk,
chunk: ModuleChunk,
- kotlinContext: KotlinCompileContext
+ kotlinContext: KotlinCompileContext,
) {
val context = kotlinContext.jpsContext
val dirtyFilesHolder = KotlinDirtySourceFilesHolder(
@@ -295,9 +298,25 @@
context: CompileContext,
chunk: ModuleChunk,
dirtyFilesHolder: DirtyFilesHolder<JavaSourceRootDescriptor, ModuleBuildTarget>,
- outputConsumer: OutputConsumer
+ outputConsumer: OutputConsumer,
): ExitCode {
- reportService.moduleBuildStarted(chunk)
+ val reporter = reportService.moduleBuildStarted(chunk)
+ try {
+ return build(context, chunk, dirtyFilesHolder, outputConsumer, reporter).also { reporter.setResult(it) }
+ } finally {
+ reportService.moduleBuildFinished(chunk, context)
+ }
+ }
+
+
+ private fun build(
+ context: CompileContext,
+ chunk: ModuleChunk,
+ dirtyFilesHolder: DirtyFilesHolder<JavaSourceRootDescriptor, ModuleBuildTarget>,
+ outputConsumer: OutputConsumer,
+ metricsReporter: JpsBuilderMetricReporter,
+ ): ExitCode {
+
if (chunk.isDummy(context))
return NOTHING_DONE
@@ -322,15 +341,23 @@
val fsOperations = FSOperationsHelper(context, chunk, kotlinDirtyFilesHolder, LOG)
try {
- return reportService.reportMetrics(chunk, JpsBuildTime.JPS_ITERATION) {
+ return metricsReporter.measure(JpsBuildTime.JPS_ITERATION) {
val proposedExitCode =
- doBuild(chunk, kotlinTarget, context, kotlinDirtyFilesHolder, messageCollector, outputConsumer, fsOperations)
+ doBuild(
+ chunk,
+ kotlinTarget,
+ context,
+ kotlinDirtyFilesHolder,
+ messageCollector,
+ outputConsumer,
+ fsOperations,
+ metricsReporter
+ )
val actualExitCode =
if (proposedExitCode == OK && fsOperations.hasMarkedDirty) ADDITIONAL_PASS_REQUIRED else proposedExitCode
LOG.debug("Build result: $actualExitCode")
-
context.testingContext?.buildLogger?.buildFinished(actualExitCode)
actualExitCode
}
@@ -344,8 +371,6 @@
LOG.info("Caught exception: $e")
MessageCollectorUtil.reportException(messageCollector, e)
return ABORT
- } finally {
- reportService.moduleBuildFinished(chunk, context)
}
}
@@ -356,7 +381,8 @@
kotlinDirtyFilesHolder: KotlinDirtySourceFilesHolder,
messageCollector: MessageCollectorAdapter,
outputConsumer: OutputConsumer,
- fsOperations: FSOperationsHelper
+ fsOperations: FSOperationsHelper,
+ metricsReporter: JpsBuilderMetricReporter,
): ExitCode {
// Workaround for Android Studio
if (representativeTarget is KotlinJvmModuleBuildTarget && !JavaBuilder.IS_ENABLED[context, true]) {
@@ -440,6 +466,7 @@
kotlinDirtyFilesHolder.allDirtyFiles,
kotlinDirtyFilesHolder.allRemovedFilesFiles
)
+ metricsReporter.addSourcesInformation(kotlinDirtyFilesHolder)
cleanJsOutputs(context, kotlinChunk, incrementalCaches, kotlinDirtyFilesHolder)
@@ -448,16 +475,18 @@
}
val start = System.nanoTime()
- val outputItemCollector = doCompileModuleChunk(
- kotlinChunk,
- representativeTarget,
- kotlinChunk.compilerArguments,
- context,
- kotlinDirtyFilesHolder,
- fsOperations,
- environment,
- incrementalCaches
- )
+ val outputItemCollector = metricsReporter.measure(JpsBuildTime.COMPILATION_ROUND) {
+ doCompileModuleChunk(
+ kotlinChunk,
+ representativeTarget,
+ kotlinChunk.compilerArguments,
+ context,
+ kotlinDirtyFilesHolder,
+ fsOperations,
+ environment,
+ incrementalCaches
+ )
+ }
statisticsLogger.registerStatistic(chunk, System.nanoTime() - start)
@@ -547,7 +576,7 @@
context: CompileContext,
kotlinChunk: KotlinChunk,
incrementalCaches: Map<KotlinModuleBuildTarget<*>, JpsIncrementalCache>,
- kotlinDirtyFilesHolder: KotlinDirtySourceFilesHolder
+ kotlinDirtyFilesHolder: KotlinDirtySourceFilesHolder,
) {
for (target in kotlinChunk.targets) {
val cache = incrementalCaches[target] ?: continue
@@ -590,7 +619,7 @@
dirtyFilesHolder: KotlinDirtySourceFilesHolder,
fsOperations: FSOperationsHelper,
environment: JpsCompilerEnvironment,
- incrementalCaches: Map<KotlinModuleBuildTarget<*>, JpsIncrementalCache>
+ incrementalCaches: Map<KotlinModuleBuildTarget<*>, JpsIncrementalCache>,
): OutputItemsCollector? {
kotlinChunk.targets.forEach {
it.nextRound(context)
@@ -626,7 +655,7 @@
inlineConstTracker: InlineConstTracker,
enumWhenTracker: EnumWhenTracker,
chunk: ModuleChunk,
- messageCollector: MessageCollectorAdapter
+ messageCollector: MessageCollectorAdapter,
): JpsCompilerEnvironment? {
val compilerServices = with(Services.Builder()) {
kotlinModuleBuilderTarget.makeServices(
@@ -674,7 +703,7 @@
private fun getGeneratedFiles(
context: CompileContext,
chunk: ModuleChunk,
- outputItemCollector: OutputItemsCollectorImpl
+ outputItemCollector: OutputItemsCollectorImpl,
): Map<ModuleBuildTarget, List<GeneratedFile>> {
// If there's only one target, this map is empty: get() always returns null, and the representativeTarget will be used below
val sourceToTarget = HashMap<File, ModuleBuildTarget>()
@@ -704,7 +733,7 @@
private fun updateLookupStorage(
lookupTracker: LookupTracker,
lookupStorageManager: JpsLookupStorageManager,
- dirtyFilesHolder: KotlinDirtySourceFilesHolder
+ dirtyFilesHolder: KotlinDirtySourceFilesHolder,
) {
if (lookupTracker !is LookupTrackerImpl)
throw AssertionError("Lookup tracker is expected to be LookupTrackerImpl, got ${lookupTracker::class.java}")
@@ -719,7 +748,7 @@
generatedFiles: Map<ModuleBuildTarget, List<GeneratedFile>>,
kotlinContext: KotlinCompileContext,
incrementalCaches: Map<KotlinModuleBuildTarget<*>, JpsIncrementalCache>,
- fsOperations: FSOperationsHelper
+ fsOperations: FSOperationsHelper,
) {
for ((target, files) in generatedFiles) {
val kotlinModuleBuilderTarget = kotlinContext.targetsBinding[target] ?: continue
@@ -754,7 +783,7 @@
compiledFiles: Set<File>,
lookupStorageManager: JpsLookupStorageManager,
fsOperations: FSOperationsHelper,
- caches: Iterable<JpsIncrementalCache>
+ caches: Iterable<JpsIncrementalCache>,
) {
val allCaches = caches.flatMap { it.thisWithDependentCaches }
val reporter = JpsICReporter()
@@ -780,7 +809,7 @@
private fun ChangesCollector.getDirtyFiles(
caches: Iterable<IncrementalCacheCommon>,
- lookupStorageManager: JpsLookupStorageManager
+ lookupStorageManager: JpsLookupStorageManager,
): FilesToRecompile {
val reporter = JpsICReporter()
val (dirtyLookupSymbols, dirtyClassFqNames, forceRecompile) = getChangedAndImpactedSymbols(caches, reporter)
diff --git a/jps/jps-plugin/src/org/jetbrains/kotlin/jps/statistic/JpsStatisticsReportService.kt b/jps/jps-plugin/src/org/jetbrains/kotlin/jps/statistic/JpsStatisticsReportService.kt
index aa80b11..a65783d 100644
--- a/jps/jps-plugin/src/org/jetbrains/kotlin/jps/statistic/JpsStatisticsReportService.kt
+++ b/jps/jps-plugin/src/org/jetbrains/kotlin/jps/statistic/JpsStatisticsReportService.kt
@@ -8,6 +8,7 @@
import com.intellij.openapi.diagnostic.Logger
import org.jetbrains.jps.ModuleChunk
import org.jetbrains.jps.incremental.CompileContext
+import org.jetbrains.jps.incremental.ModuleLevelBuilder
import org.jetbrains.kotlin.build.report.FileReportSettings
import org.jetbrains.kotlin.build.report.HttpReportSettings
import org.jetbrains.kotlin.build.report.metrics.*
@@ -17,15 +18,25 @@
import org.jetbrains.kotlin.build.report.statistics.StatTag
import org.jetbrains.kotlin.build.report.statistics.file.FileReportService
import org.jetbrains.kotlin.compilerRunner.JpsKotlinLogger
+import org.jetbrains.kotlin.jps.build.KotlinDirtySourceFilesHolder
import java.io.File
import java.net.InetAddress
import java.util.*
import kotlin.collections.ArrayList
+import kotlin.collections.HashMap
+import kotlin.collections.HashSet
interface JpsBuilderMetricReporter : BuildMetricsReporter<JpsBuildTime, JpsBuildPerformanceMetric> {
fun flush(context: CompileContext): JpsCompileStatisticsData
-
fun buildFinish(moduleChunk: ModuleChunk, context: CompileContext)
+ fun setResult(exitCode: ModuleLevelBuilder.ExitCode)
+ fun addCompiledSources(files: Collection<String>)
+ fun addChangedFiles(files: Collection<String>)
+ fun addSourcesInformation(fileHolder: KotlinDirtySourceFilesHolder)
+
+ fun join(reporter: JpsBuilderMetricReporter)
+
+ fun getModuleName(): String
}
private const val jpsBuildTaskName = "JPS build"
@@ -34,7 +45,7 @@
chunk: ModuleChunk,
private val reporter: BuildMetricsReporterImpl<JpsBuildTime, JpsBuildPerformanceMetric>,
private val label: String? = null,
- private val kotlinVersion: String = "kotlin_version"
+ private val kotlinVersion: String = "kotlin_version",
) :
JpsBuilderMetricReporter, BuildMetricsReporter<JpsBuildTime, JpsBuildPerformanceMetric> by reporter {
@@ -52,24 +63,52 @@
private val startTime = System.currentTimeMillis()
private var finishTime: Long = 0L
private val tags = HashSet<StatTag>()
- private val moduleString = chunk.name
+ private val module = chunk.name
+ private var exitCode: ModuleLevelBuilder.ExitCode? = null
+ private val compiledSources = HashSet<String>()
+ private val changedFiles = HashSet<String>()
override fun buildFinish(moduleChunk: ModuleChunk, context: CompileContext) {
finishTime = System.currentTimeMillis()
}
+ override fun setResult(exitCode: ModuleLevelBuilder.ExitCode) {
+ this.exitCode = exitCode
+ }
+
+ override fun addCompiledSources(files: Collection<String>) {
+ compiledSources.addAll(files)
+ }
+
+ override fun addChangedFiles(files: Collection<String>) {
+ changedFiles.addAll(files)
+ }
+
+ override fun addSourcesInformation(fileHolder: KotlinDirtySourceFilesHolder) {
+ addCompiledSources(fileHolder.allDirtyFiles.map { it.path })
+ addChangedFiles(fileHolder.allDirtyFiles.map { it.path })
+ addChangedFiles(fileHolder.allRemovedFilesFiles.map { it.path })
+ }
+
+ override fun join(reporter: JpsBuilderMetricReporter) {
+ addMetrics(reporter.getMetrics())
+ addCompiledSources(compiledSources)
+ }
+
+ override fun getModuleName(): String = module
+
override fun flush(context: CompileContext): JpsCompileStatisticsData {
val buildMetrics = reporter.getMetrics()
return JpsCompileStatisticsData(
projectName = context.projectDescriptor.project.name,
label = label,
- taskName = moduleString,
- taskResult = "Unknown",//TODO will be updated in KT-58026
+ taskName = module,
+ taskResult = exitCode?.name,
startTimeMs = startTime,
durationMs = finishTime - startTime,
tags = tags,
buildUuid = uuid.toString(),
- changes = emptyList(), //TODO will be updated in KT-58026
+ changes = changedFiles.toList(),
kotlinVersion = kotlinVersion,
hostName = hostName,
finishTime = finishTime,
@@ -79,7 +118,7 @@
nonIncrementalAttributes = emptySet(),
type = BuildDataType.JPS_DATA.name,
fromKotlinPlugin = true,
- compiledSources = emptyList(),
+ compiledSources = compiledSources.toList(),
skipMessage = null,
icLogLines = emptyList(),
gcTimeMetrics = buildMetrics.gcMetrics.asGcTimeMap(),
@@ -117,17 +156,18 @@
private val loggerAdapter = JpsKotlinLogger(log)
private val httpService = httpReportSettings?.let { HttpReportService(it.url, it.user, it.password) }
- fun moduleBuildStarted(chunk: ModuleChunk) {
+ fun moduleBuildStarted(chunk: ModuleChunk): JpsBuilderMetricReporter {
val moduleName = chunk.name
- if (buildMetrics[moduleName] != null) {
+ buildMetrics[moduleName]?.also {
log.warn("Service already initialized for context")
- return
+ return it
}
log.info("JpsStatisticsReportService: Service started")
- buildMetrics[moduleName] = JpsBuilderMetricReporterImpl(chunk, BuildMetricsReporterImpl())
+ val reporter = JpsBuilderMetricReporterImpl(chunk, BuildMetricsReporterImpl())
+ buildMetrics[moduleName] = reporter
+ return reporter
}
-
fun moduleBuildFinished(chunk: ModuleChunk, context: CompileContext) {
val moduleName = chunk.name
val metrics = buildMetrics.remove(moduleName)
@@ -142,6 +182,14 @@
fun buildFinish(context: CompileContext) {
val compileStatisticsData = finishedModuleBuildMetrics.map { it.flush(context) }
+// finishedModuleBuildMetrics.groupBy { it.getModuleName() }
+// .values.map {
+// it.reduce { first, second ->
+// first.join(second)
+// first
+// }.flush(context)
+// }
+
httpService?.sendData(compileStatisticsData, loggerAdapter)
fileReportSettings?.also {
FileReportService.reportBuildStatInFile(
@@ -151,7 +199,6 @@
}
}
-
fun <T> reportMetrics(chunk: ModuleChunk, metric: JpsBuildTime, action: () -> T): T {
val moduleName = chunk.name
val metrics = buildMetrics[moduleName]
@@ -167,6 +214,7 @@
loggerAdapter.info("Build started for $context")
}
+
}
