[Commonizer] Logging: Implement ProgressLogger.fork

^KT-36679
This commit is contained in:
sebastian.sellmair
2021-04-29 10:45:44 +02:00
committed by TeamCityServer
parent ec440aecf6
commit 7349ec8f2f
12 changed files with 79 additions and 72 deletions

View File

@@ -9,15 +9,16 @@ import org.jetbrains.kotlin.commonizer.konan.NativeManifestDataProvider
import org.jetbrains.kotlin.commonizer.mergedtree.CirFictitiousFunctionClassifiers
import org.jetbrains.kotlin.commonizer.mergedtree.CirProvidedClassifiers
import org.jetbrains.kotlin.commonizer.stats.StatsCollector
import org.jetbrains.kotlin.commonizer.utils.ProgressLogger
class CommonizerParameters(
data class CommonizerParameters(
val outputTarget: SharedCommonizerTarget,
val manifestProvider: TargetDependent<NativeManifestDataProvider>,
val dependenciesProvider: TargetDependent<ModulesProvider?>,
val targetProviders: TargetDependent<TargetProvider?>,
val resultsConsumer: ResultsConsumer,
val statsCollector: StatsCollector? = null,
val progressLogger: ((String) -> Unit)? = null,
val logger: ProgressLogger? = null,
)
internal fun CommonizerParameters.dependencyClassifiers(target: CommonizerTarget): CirProvidedClassifiers {
@@ -30,3 +31,5 @@ internal fun CommonizerParameters.dependencyClassifiers(target: CommonizerTarget
CirProvidedClassifiers.of(classifiers, CirProvidedClassifiers.by(module))
}
}
internal fun CommonizerParameters.with(logger: ProgressLogger?) = copy(logger = logger)

View File

@@ -31,19 +31,19 @@ interface ResultsConsumer {
/**
* Consume a single [ModuleResult] for the specified [CommonizerTarget].
*/
fun consume(target: CommonizerTarget, moduleResult: ModuleResult) = Unit
fun consume(parameters: CommonizerParameters, target: CommonizerTarget, moduleResult: ModuleResult) = Unit
/**
* Mark the specified [CommonizerTarget] as fully consumed.
* It's forbidden to make subsequent [consume] calls for fully consumed targets.
*/
fun targetConsumed(target: CommonizerTarget) = Unit
fun targetConsumed(parameters: CommonizerParameters, target: CommonizerTarget) = Unit
/**
* Notify that all results have been consumed.
* It's forbidden to make any subsequent [consume] and [targetConsumed] calls after this call.
*/
fun allConsumed(status: Status) = Unit
fun allConsumed(parameters: CommonizerParameters, status: Status) = Unit
}
internal class ResultsConsumerBuilder {
@@ -72,21 +72,21 @@ operator fun ResultsConsumer.plus(other: ResultsConsumer?): ResultsConsumer {
}
private class CompositeResultsConsumer(val consumers: List<ResultsConsumer>) : ResultsConsumer {
override fun consume(target: CommonizerTarget, moduleResult: ResultsConsumer.ModuleResult) {
override fun consume(parameters: CommonizerParameters, target: CommonizerTarget, moduleResult: ResultsConsumer.ModuleResult) {
consumers.forEach { consumer ->
consumer.consume(target, moduleResult)
consumer.consume(parameters, target, moduleResult)
}
}
override fun targetConsumed(target: CommonizerTarget) {
override fun targetConsumed(parameters: CommonizerParameters, target: CommonizerTarget) {
consumers.forEach { consumer ->
consumer.targetConsumed(target)
consumer.targetConsumed(parameters, target)
}
}
override fun allConsumed(status: ResultsConsumer.Status) {
override fun allConsumed(parameters: CommonizerParameters, status: ResultsConsumer.Status) {
consumers.forEach { consumer ->
consumer.allConsumed(status)
consumer.allConsumed(parameters, status)
}
}
}

View File

@@ -8,13 +8,15 @@ package org.jetbrains.kotlin.commonizer.cli
import org.jetbrains.kotlin.util.Logger
import kotlin.system.exitProcess
internal class CliLoggerAdapter(indentSize: Int) : Logger {
internal class CliLoggerAdapter(indentSize: Int = 0) : Logger {
private val indent = " ".repeat(indentSize)
override fun log(message: String) = printlnIndented(message)
override fun warning(message: String) = printlnIndented("Warning: $message")
override fun error(message: String) = fatal(message)
override fun fatal(message: String): Nothing {
printlnIndented("Error: $message\n")
exitProcess(1)

View File

@@ -54,7 +54,7 @@ internal class NativeKlibCommonize(options: Collection<Option<*>>) : Task(option
val konanTargets = outputCommonizerTarget.konanTargets
val commonizerTargets = konanTargets.map(::CommonizerTarget)
val progressLogger = ProgressLogger(CliLoggerAdapter(2), startImmediately = true)
val progressLogger = ProgressLogger(CliLoggerAdapter(2))
val libraryLoader = DefaultNativeLibraryLoader(progressLogger)
val statsCollector = StatsCollector(statsType, commonizerTargets)
val repository = FilesRepository(targetLibraries.toSet(), libraryLoader)
@@ -62,9 +62,9 @@ internal class NativeKlibCommonize(options: Collection<Option<*>>) : Task(option
val resultsConsumer = buildResultsConsumer {
this add ModuleSerializer(destination, HierarchicalCommonizerOutputLayout)
this add CopyUnconsumedModulesAsIsConsumer(
repository, destination, commonizerTargets.toSet(), NativeDistributionCommonizerOutputLayout, progressLogger
repository, destination, commonizerTargets.toSet(), NativeDistributionCommonizerOutputLayout
)
this add LoggingResultsConsumer(outputCommonizerTarget, progressLogger)
this add LoggingResultsConsumer(outputCommonizerTarget)
}
LibraryCommonizer(
@@ -98,17 +98,17 @@ internal class NativeDistributionCommonize(options: Collection<Option<*>>) : Tas
val copyEndorsedLibs = getOptional<Boolean, BooleanOptionType> { it == "copy-endorsed-libs" } ?: false
val statsType = getOptional<StatsType, StatsTypeOptionType> { it == "log-stats" } ?: StatsType.NONE
val progressLogger = ProgressLogger(CliLoggerAdapter(2), startImmediately = true)
val progressLogger = ProgressLogger(CliLoggerAdapter(2))
val libraryLoader = DefaultNativeLibraryLoader(progressLogger)
val repository = KonanDistributionRepository(distribution, outputTarget.konanTargets, libraryLoader)
val statsCollector = StatsCollector(statsType, outputTarget.withAllAncestors().toList())
val resultsConsumer = buildResultsConsumer {
this add ModuleSerializer(destination, outputLayout)
this add CopyUnconsumedModulesAsIsConsumer(repository, destination, outputTarget.allLeaves(), outputLayout, progressLogger)
if (copyStdlib) this add CopyStdlibResultsConsumer(distribution, destination, progressLogger)
if (copyEndorsedLibs) this add CopyEndorsedLibrairesResultsConsumer(distribution, destination, progressLogger)
this add LoggingResultsConsumer(outputTarget, progressLogger)
this add CopyUnconsumedModulesAsIsConsumer(repository, destination, outputTarget.allLeaves(), outputLayout)
if (copyStdlib) this add CopyStdlibResultsConsumer(distribution, destination)
if (copyEndorsedLibs) this add CopyEndorsedLibrairesResultsConsumer(distribution, destination)
this add LoggingResultsConsumer(outputTarget)
}
val descriptionSuffix = estimateLibrariesCount(repository, outputTarget.allLeaves()).let { " ($it items)" }

View File

@@ -26,13 +26,13 @@ import org.jetbrains.kotlin.storage.StorageManager
fun runCommonization(parameters: CommonizerParameters) {
if (!parameters.containsCommonModuleNames()) {
parameters.resultsConsumer.allConsumed(Status.NOTHING_TO_DO)
parameters.resultsConsumer.allConsumed(parameters, Status.NOTHING_TO_DO)
return
}
val storageManager = LockBasedStorageManager("Declarations commonization")
commonize(parameters, storageManager, parameters.outputTarget)
parameters.resultsConsumer.allConsumed(Status.DONE)
parameters.resultsConsumer.allConsumed(parameters, Status.DONE)
}
private fun commonize(
@@ -41,7 +41,7 @@ private fun commonize(
target: SharedCommonizerTarget,
): CirRootNode? {
val cirTrees = getCirTree(parameters, storageManager, target)
parameters.progressLogger?.invoke("Build cir tree for $target")
parameters.logProgress("Built declaration tree for $target")
// build merged tree:
val classifiers = CirKnownClassifiers(
@@ -51,10 +51,9 @@ private fun commonize(
val mergedTree = merge(storageManager, classifiers, cirTrees) ?: return null
mergedTree.accept(CommonizationVisitor(classifiers, mergedTree), Unit)
parameters.progressLogger?.invoke("Commonized declarations for $target")
parameters.logProgress("Commonized declarations for $target")
serialize(parameters, mergedTree, target)
parameters.progressLogger?.invoke("Commonized target $target")
return mergedTree
}
@@ -65,7 +64,9 @@ private fun getCirTree(
return EagerTargetDependent(target.targets) { childTarget ->
when (childTarget) {
is LeafCommonizerTarget -> deserialize(parameters, childTarget)
is SharedCommonizerTarget -> commonize(parameters, storageManager, childTarget)?.assembleCirTree()
is SharedCommonizerTarget -> commonize(parameters.fork(), storageManager, childTarget)?.assembleCirTree().also {
parameters.logProgress("Commonized target $childTarget")
}
}
}
}
@@ -103,9 +104,9 @@ private fun serialize(parameters: CommonizerParameters, mergedTree: CirRootNode,
SerializedMetadata(header, fragments, fragmentNames)
}
val manifestData = parameters.manifestProvider[target].buildManifest(libraryName)
parameters.resultsConsumer.consume(target, ModuleResult.Commonized(libraryName, serializedMetadata, manifestData))
parameters.resultsConsumer.consume(parameters, target, ModuleResult.Commonized(libraryName, serializedMetadata, manifestData))
}
parameters.resultsConsumer.targetConsumed(target)
parameters.resultsConsumer.targetConsumed(parameters, target)
}
private fun serializeMissingModules(parameters: CommonizerParameters, requestedTarget: CommonizerTarget) {
@@ -114,8 +115,14 @@ private fun serializeMissingModules(parameters: CommonizerParameters, requestedT
targetProvider.modulesProvider.loadModuleInfos()
.filter { it.name !in commonModuleNames }
.forEach { missingModule -> parameters.resultsConsumer.consume(requestedTarget, Missing(missingModule.originalLocation)) }
.forEach { missingModule ->
parameters.resultsConsumer.consume(parameters, requestedTarget, Missing(missingModule.originalLocation))
}
}
private fun CommonizerParameters.fork(): CommonizerParameters = with(logger?.fork())
private fun CommonizerParameters.logProgress(message: String) = logger?.progress(message)
private val KLIB_FRAGMENT_WRITE_STRATEGY = ChunkedKlibModuleFragmentWriteStrategy()

View File

@@ -7,23 +7,22 @@
package org.jetbrains.kotlin.commonizer.konan
import org.jetbrains.kotlin.commonizer.CommonizerParameters
import org.jetbrains.kotlin.commonizer.KonanDistribution
import org.jetbrains.kotlin.commonizer.ResultsConsumer
import org.jetbrains.kotlin.commonizer.klibDir
import org.jetbrains.kotlin.konan.library.KONAN_DISTRIBUTION_COMMON_LIBS_DIR
import org.jetbrains.kotlin.konan.library.KONAN_STDLIB_NAME
import org.jetbrains.kotlin.util.Logger
import java.io.File
internal fun CopyStdlibResultsConsumer(
konanDistribution: KonanDistribution,
destination: File,
logger: Logger
destination: File
): ResultsConsumer {
return CopyLibrariesFromKonanDistributionResultsConsumer(
konanDistribution,
destination,
invokeWhenCopied = { logger.log("Copied standard library") },
invokeWhenCopied = { logger?.progress("Copied standard library") },
copyFileIf = { file -> file.endsWith(KONAN_STDLIB_NAME) }
)
}
@@ -31,12 +30,11 @@ internal fun CopyStdlibResultsConsumer(
internal fun CopyEndorsedLibrairesResultsConsumer(
konanDistribution: KonanDistribution,
destination: File,
logger: Logger
): ResultsConsumer {
return CopyLibrariesFromKonanDistributionResultsConsumer(
konanDistribution,
destination,
invokeWhenCopied = { logger.log("Copied endorsed libraries") },
invokeWhenCopied = { logger?.progress("Copied endorsed libraries") },
copyFileIf = { file -> !file.endsWith(KONAN_STDLIB_NAME) }
)
}
@@ -44,10 +42,10 @@ internal fun CopyEndorsedLibrairesResultsConsumer(
private class CopyLibrariesFromKonanDistributionResultsConsumer(
private val konanDistribution: KonanDistribution,
private val destination: File,
private val invokeWhenCopied: () -> Unit = {},
private val invokeWhenCopied: CommonizerParameters.() -> Unit = {},
private val copyFileIf: (File) -> Boolean = { true }
) : ResultsConsumer {
override fun allConsumed(status: ResultsConsumer.Status) {
override fun allConsumed(parameters: CommonizerParameters, status: ResultsConsumer.Status) {
konanDistribution.klibDir
.resolve(KONAN_DISTRIBUTION_COMMON_LIBS_DIR)
.listFiles().orEmpty()
@@ -57,6 +55,6 @@ private class CopyLibrariesFromKonanDistributionResultsConsumer(
val libraryDestination = destination.resolve(KONAN_DISTRIBUTION_COMMON_LIBS_DIR).resolve(libraryOrigin.name)
libraryOrigin.copyRecursively(libraryDestination)
}
invokeWhenCopied()
parameters.invokeWhenCopied()
}
}

View File

@@ -7,7 +7,6 @@ package org.jetbrains.kotlin.commonizer.konan
import org.jetbrains.kotlin.commonizer.*
import org.jetbrains.kotlin.commonizer.repository.Repository
import org.jetbrains.kotlin.util.Logger
import java.io.File
internal class CopyUnconsumedModulesAsIsConsumer(
@@ -15,25 +14,24 @@ internal class CopyUnconsumedModulesAsIsConsumer(
private val destination: File,
private val targets: Set<LeafCommonizerTarget>,
private val outputLayout: CommonizerOutputLayout,
private val logger: Logger? = null
) : ResultsConsumer {
private val consumedTargets = mutableSetOf<LeafCommonizerTarget>()
override fun targetConsumed(target: CommonizerTarget) {
override fun targetConsumed(parameters: CommonizerParameters, target: CommonizerTarget) {
if (target is LeafCommonizerTarget) {
consumedTargets += target
}
}
override fun allConsumed(status: ResultsConsumer.Status) {
override fun allConsumed(parameters: CommonizerParameters, status: ResultsConsumer.Status) {
when (status) {
ResultsConsumer.Status.NOTHING_TO_DO -> targets.forEach(::copyTargetAsIs)
ResultsConsumer.Status.DONE -> targets.minus(consumedTargets).forEach(::copyTargetAsIs)
ResultsConsumer.Status.NOTHING_TO_DO -> targets.forEach { target -> copyTargetAsIs(parameters, target) }
ResultsConsumer.Status.DONE -> targets.minus(consumedTargets).forEach { target -> copyTargetAsIs(parameters, target) }
}
}
private fun copyTargetAsIs(target: LeafCommonizerTarget) {
private fun copyTargetAsIs(parameters: CommonizerParameters, target: LeafCommonizerTarget) {
val libraries = repository.getLibraries(target)
val librariesDestination = outputLayout.getTargetDirectory(destination, target)
librariesDestination.mkdirs() // always create an empty directory even if there is nothing to copy
@@ -41,6 +39,6 @@ internal class CopyUnconsumedModulesAsIsConsumer(
libraryFile.copyRecursively(destination.resolve(libraryFile.name))
}
logger?.log("Copied ${libraries.size} libraries for ${target.prettyName}")
parameters.logger?.progress("Copied ${libraries.size} libraries for ${target.prettyName}")
}
}

View File

@@ -39,7 +39,7 @@ internal class LibraryCommonizer internal constructor(
)
}
progressLogger.log("Resolved libraries to be commonized")
progressLogger.progress("Resolved libraries to be commonized")
return libraries
}
@@ -51,7 +51,7 @@ internal class LibraryCommonizer internal constructor(
dependenciesProvider = createDependenciesProvider(),
resultsConsumer = resultsConsumer,
statsCollector = statsCollector,
progressLogger = progressLogger::log
logger = progressLogger
)
runCommonization(parameters)
}

View File

@@ -5,16 +5,12 @@
package org.jetbrains.kotlin.commonizer.konan
import org.jetbrains.kotlin.commonizer.CommonizerTarget
import org.jetbrains.kotlin.commonizer.ResultsConsumer
import org.jetbrains.kotlin.commonizer.SharedCommonizerTarget
import org.jetbrains.kotlin.commonizer.prettyName
import org.jetbrains.kotlin.util.Logger
import org.jetbrains.kotlin.commonizer.*
internal class LoggingResultsConsumer(
private val outputCommonizerTarget: SharedCommonizerTarget, private val logger: Logger
private val outputCommonizerTarget: SharedCommonizerTarget
) : ResultsConsumer {
override fun targetConsumed(target: CommonizerTarget) {
logger.log("Written libraries for ${outputCommonizerTarget.prettyName(target)}")
override fun targetConsumed(parameters: CommonizerParameters, target: CommonizerTarget) {
parameters.logger?.progress("Written libraries for ${outputCommonizerTarget.prettyName(target)}")
}
}

View File

@@ -6,6 +6,7 @@
package org.jetbrains.kotlin.commonizer.konan
import org.jetbrains.kotlin.commonizer.CommonizerOutputLayout
import org.jetbrains.kotlin.commonizer.CommonizerParameters
import org.jetbrains.kotlin.commonizer.CommonizerTarget
import org.jetbrains.kotlin.commonizer.ResultsConsumer
import org.jetbrains.kotlin.library.SerializedMetadata
@@ -19,7 +20,7 @@ internal class ModuleSerializer(
private val destination: File,
private val outputLayout: CommonizerOutputLayout,
) : ResultsConsumer {
override fun consume(target: CommonizerTarget, moduleResult: ResultsConsumer.ModuleResult) {
override fun consume(parameters: CommonizerParameters, target: CommonizerTarget, moduleResult: ResultsConsumer.ModuleResult) {
val librariesDestination = outputLayout.getTargetDirectory(destination, target)
when (moduleResult) {
is ResultsConsumer.ModuleResult.Commonized -> {

View File

@@ -5,28 +5,26 @@
package org.jetbrains.kotlin.commonizer.utils
import org.jetbrains.kotlin.commonizer.cli.CliLoggerAdapter
import org.jetbrains.kotlin.util.Logger
class ProgressLogger(
private val wrapped: Logger,
startImmediately: Boolean = false
private val wrapped: Logger = CliLoggerAdapter(0),
private val indent: Int = 0,
) : Logger by wrapped {
private val clockMark = ResettableClockMark()
private var finished = true
private var finished = false
private val prefix = " ".repeat(indent) + " * "
init {
if (startImmediately)
reset()
}
fun reset() {
clockMark.reset()
finished = false
require(indent >= 0) { "Required indent >= 1" }
}
override fun log(message: String) {
fun progress(message: String) {
check(!finished)
wrapped.log("* $message in ${clockMark.elapsedSinceLast()}")
wrapped.log("$prefix$message in ${clockMark.elapsedSinceLast()}")
}
fun logTotal() {
@@ -34,4 +32,8 @@ class ProgressLogger(
wrapped.log("TOTAL: ${clockMark.elapsedSinceStart()}")
finished = true
}
fun fork(): ProgressLogger {
return ProgressLogger(this, indent + 1)
}
}

View File

@@ -146,7 +146,7 @@ internal class MockResultsConsumer : ResultsConsumer {
lateinit var status: ResultsConsumer.Status
override fun consume(target: CommonizerTarget, moduleResult: ModuleResult) {
override fun consume(parameters: CommonizerParameters, target: CommonizerTarget, moduleResult: ModuleResult) {
check(!this::status.isInitialized)
check(target !in finishedTargets) { "$target already finished" }
val moduleResults: ModuleResults = _modulesByTargets.getOrPut(target) { ModuleResults() }
@@ -154,13 +154,13 @@ internal class MockResultsConsumer : ResultsConsumer {
check(oldResult == null) // to avoid accidental overwriting
}
override fun targetConsumed(target: CommonizerTarget) {
override fun targetConsumed(parameters: CommonizerParameters, target: CommonizerTarget) {
check(!this::status.isInitialized)
check(target !in finishedTargets)
finishedTargets += target
}
override fun allConsumed(status: ResultsConsumer.Status) {
override fun allConsumed(parameters: CommonizerParameters, status: ResultsConsumer.Status) {
check(!this::status.isInitialized)
check(finishedTargets.containsAll(_modulesByTargets.keys))
this.status = status