[Commonizer] Log commonization stats into a single file

This commit is contained in:
Dmitriy Dolovov
2019-12-25 17:45:04 +07:00
parent f18aa3e0be
commit e0d90ccf4b
16 changed files with 304 additions and 99 deletions

View File

@@ -0,0 +1,42 @@
/*
* Copyright 2010-2019 JetBrains s.r.o. and Kotlin Programming Language contributors.
* Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
*/
package org.jetbrains.kotlin.descriptors.commonizer
import org.jetbrains.kotlin.descriptors.ModuleDescriptor
class CommonizationParameters(
val statsCollector: StatsCollector? = null
) {
// use linked hash map to preserve order
private val modulesByTargets = LinkedHashMap<InputTarget, Collection<ModuleDescriptor>>()
fun addTarget(target: InputTarget, modules: Collection<ModuleDescriptor>): CommonizationParameters {
require(target !in modulesByTargets) { "Target $target is already added" }
val modulesWithUniqueNames = modules.groupingBy { it.name }.eachCount()
require(modulesWithUniqueNames.size == modules.size) {
"Modules with duplicated names found: ${modulesWithUniqueNames.filter { it.value > 1 }}"
}
modulesByTargets[target] = modules
return this
}
// get them as ordered immutable collection (List) for further processing
fun getModulesByTargets(): List<Pair<InputTarget, Collection<ModuleDescriptor>>> =
modulesByTargets.map { it.key to it.value }
fun hasIntersection(): Boolean {
if (modulesByTargets.size < 2)
return false
return modulesByTargets.flatMap { it.value }
.groupingBy { it.name }
.eachCount()
.any { it.value == modulesByTargets.size }
}
}

View File

@@ -0,0 +1,24 @@
/*
* Copyright 2010-2019 JetBrains s.r.o. and Kotlin Programming Language contributors.
* Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
*/
package org.jetbrains.kotlin.descriptors.commonizer
import org.jetbrains.kotlin.descriptors.ModuleDescriptor
sealed class CommonizationResult
object NothingToCommonize : CommonizationResult()
class CommonizationPerformed(
val modulesByTargets: Map<Target, Collection<ModuleDescriptor>>
) : CommonizationResult() {
val commonTarget: OutputTarget by lazy {
modulesByTargets.keys.filterIsInstance<OutputTarget>().single()
}
val concreteTargets: Set<InputTarget> by lazy {
modulesByTargets.keys.filterIsInstance<InputTarget>().toSet()
}
}

View File

@@ -0,0 +1,13 @@
/*
* Copyright 2010-2019 JetBrains s.r.o. and Kotlin Programming Language contributors.
* Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
*/
package org.jetbrains.kotlin.descriptors.commonizer
import org.jetbrains.kotlin.descriptors.DeclarationDescriptor
import java.io.Closeable
interface StatsCollector : Closeable {
fun logStats(output: List<DeclarationDescriptor?>)
}

View File

@@ -52,8 +52,7 @@ internal class DeclarationsBuilderVisitor1(
override fun visitModuleNode(node: CirModuleNode, data: List<DeclarationDescriptor?>): List<ModuleDescriptorImpl?> {
// build module descriptors:
val moduleDescriptorsGroup =
CommonizedGroup<ModuleDescriptorImpl>(node.dimension)
val moduleDescriptorsGroup = CommonizedGroup<ModuleDescriptorImpl>(node.dimension)
node.buildDescriptors(components, moduleDescriptorsGroup)
val moduleDescriptors = moduleDescriptorsGroup.toList()
@@ -76,8 +75,7 @@ internal class DeclarationsBuilderVisitor1(
val containingDeclarations = data.asListContaining<ModuleDescriptorImpl>()
// build package fragments:
val packageFragmentsGroup =
CommonizedGroup<CommonizedPackageFragmentDescriptor>(node.dimension)
val packageFragmentsGroup = CommonizedGroup<CommonizedPackageFragmentDescriptor>(node.dimension)
node.buildDescriptors(components, packageFragmentsGroup, containingDeclarations)
val packageFragments = packageFragmentsGroup.toList()
@@ -105,10 +103,7 @@ internal class DeclarationsBuilderVisitor1(
error("This method should not be called in ${this::class.java}")
override fun visitClassNode(node: CirClassNode, data: List<DeclarationDescriptor?>): List<DeclarationDescriptor?> {
val classesGroup =
CommonizedGroup<ClassifierDescriptorWithTypeParameters>(
node.dimension
)
val classesGroup = CommonizedGroup<ClassifierDescriptorWithTypeParameters>(node.dimension)
node.buildDescriptors(components, classesGroup, data)
val classes = classesGroup.toList().asListContaining<CommonizedClassDescriptor>()
@@ -130,10 +125,7 @@ internal class DeclarationsBuilderVisitor1(
error("This method should not be called in ${this::class.java}")
override fun visitTypeAliasNode(node: CirTypeAliasNode, data: List<DeclarationDescriptor?>): List<DeclarationDescriptor?> {
val typeAliasesGroup =
CommonizedGroup<ClassifierDescriptorWithTypeParameters>(
node.dimension
)
val typeAliasesGroup = CommonizedGroup<ClassifierDescriptorWithTypeParameters>(node.dimension)
node.buildDescriptors(components, typeAliasesGroup, data)
val typeAliases = typeAliasesGroup.toList()

View File

@@ -65,16 +65,14 @@ internal class DeclarationsBuilderVisitor2(
}
override fun visitPropertyNode(node: CirPropertyNode, data: List<DeclarationDescriptor?>): List<PropertyDescriptor?> {
val propertyDescriptorsGroup =
CommonizedGroup<PropertyDescriptor>(node.dimension)
val propertyDescriptorsGroup = CommonizedGroup<PropertyDescriptor>(node.dimension)
node.buildDescriptors(components, propertyDescriptorsGroup, data)
return propertyDescriptorsGroup.toList()
}
override fun visitFunctionNode(node: CirFunctionNode, data: List<DeclarationDescriptor?>): List<DeclarationDescriptor?> {
val functionDescriptorsGroup =
CommonizedGroup<SimpleFunctionDescriptor>(node.dimension)
val functionDescriptorsGroup = CommonizedGroup<SimpleFunctionDescriptor>(node.dimension)
node.buildDescriptors(components, functionDescriptorsGroup, data)
return functionDescriptorsGroup.toList()
@@ -116,8 +114,7 @@ internal class DeclarationsBuilderVisitor2(
override fun visitClassConstructorNode(node: CirClassConstructorNode, data: List<DeclarationDescriptor?>): List<DeclarationDescriptor?> {
val containingDeclarations = data.asListContaining<CommonizedClassDescriptor>()
val constructorsGroup =
CommonizedGroup<ClassConstructorDescriptor>(node.dimension)
val constructorsGroup = CommonizedGroup<ClassConstructorDescriptor>(node.dimension)
node.buildDescriptors(components, constructorsGroup, containingDeclarations)
return constructorsGroup.toList()

View File

@@ -26,6 +26,9 @@ internal fun CirClassNode.buildDescriptors(
}
commonClass?.buildDescriptor(components, output, indexOfCommon, containingDeclarations, fqName, isExpect = true)
// log stats
components.statsCollector?.logStats(output.toList())
}
internal fun CirClass.buildDescriptor(
@@ -79,6 +82,9 @@ internal fun CirClassConstructorNode.buildDescriptors(
}
commonConstructor?.buildDescriptor(components, output, indexOfCommon, containingDeclarations, isExpect = true)
// log stats
components.statsCollector?.logStats(output.toList())
}
private fun CirClassConstructor.buildDescriptor(

View File

@@ -7,6 +7,7 @@ package org.jetbrains.kotlin.descriptors.commonizer.builder
import org.jetbrains.kotlin.builtins.KotlinBuiltIns
import org.jetbrains.kotlin.descriptors.*
import org.jetbrains.kotlin.descriptors.commonizer.StatsCollector
import org.jetbrains.kotlin.descriptors.commonizer.Target
import org.jetbrains.kotlin.descriptors.commonizer.mergedtree.ir.CirRootNode
import org.jetbrains.kotlin.descriptors.commonizer.mergedtree.ir.dimension
@@ -108,7 +109,8 @@ class DeclarationsBuilderCache(dimension: Int) {
class GlobalDeclarationsBuilderComponents(
val storageManager: StorageManager,
val targetComponents: List<TargetDeclarationsBuilderComponents>,
val cache: DeclarationsBuilderCache
val cache: DeclarationsBuilderCache,
val statsCollector: StatsCollector?
) {
init {
check(targetComponents.size > 1)
@@ -154,7 +156,10 @@ class TargetDeclarationsBuilderComponents(
}
}
fun CirRootNode.createGlobalBuilderComponents(storageManager: StorageManager): GlobalDeclarationsBuilderComponents {
fun CirRootNode.createGlobalBuilderComponents(
storageManager: StorageManager,
statsCollector: StatsCollector?
): GlobalDeclarationsBuilderComponents {
val cache = DeclarationsBuilderCache(dimension)
val targetContexts = (0 until dimension).map { index ->
@@ -176,7 +181,7 @@ fun CirRootNode.createGlobalBuilderComponents(storageManager: StorageManager): G
)
}
return GlobalDeclarationsBuilderComponents(storageManager, targetContexts, cache)
return GlobalDeclarationsBuilderComponents(storageManager, targetContexts, cache, statsCollector)
}
interface TypeParameterResolver {

View File

@@ -28,6 +28,9 @@ internal fun CirFunctionNode.buildDescriptors(
}
commonFunction?.buildDescriptor(components, output, indexOfCommon, containingDeclarations, isExpect = markAsExpectAndActual)
// log stats
components.statsCollector?.logStats(output.toList())
}
private fun CirFunction.buildDescriptor(

View File

@@ -20,6 +20,9 @@ internal fun CirModuleNode.buildDescriptors(
}
common()?.buildDescriptor(components, output, indexOfCommon)
// log stats
components.statsCollector?.logStats(output.toList())
}
private fun CirModule.buildDescriptor(

View File

@@ -31,6 +31,9 @@ internal fun CirPropertyNode.buildDescriptors(
}
commonProperty?.buildDescriptor(components, output, indexOfCommon, containingDeclarations, isExpect = markAsExpectAndActual)
// log stats
components.statsCollector?.logStats(output.toList())
}
private fun CirProperty.buildDescriptor(

View File

@@ -27,6 +27,9 @@ internal fun CirTypeAliasNode.buildDescriptors(
}
commonClass?.buildDescriptor(components, output, indexOfCommon, containingDeclarations, fqName, isExpect = true)
// log stats
components.statsCollector?.logStats(output.toList())
}
private fun CirTypeAlias.buildDescriptor(

View File

@@ -28,10 +28,13 @@ fun main(args: Array<String>) {
val destination = parsedArgs["-output"]?.firstOrNull()?.let(::File) ?: printUsageAndExit("output not specified")
val withStats = parsedArgs["-stats"]?.firstOrNull()?.toLowerCase() in setOf("1", "on", "yes", "true")
NativeDistributionCommonizer(
repository = repository,
targets = targets,
destination = destination,
withStats = withStats,
handleError = ::printErrorAndExit,
log = ::println
).run()

View File

@@ -6,61 +6,13 @@
package org.jetbrains.kotlin.descriptors.commonizer
import org.jetbrains.kotlin.descriptors.ModuleDescriptor
import org.jetbrains.kotlin.descriptors.commonizer.builder.*
import org.jetbrains.kotlin.descriptors.commonizer.builder.DeclarationsBuilderVisitor1
import org.jetbrains.kotlin.descriptors.commonizer.builder.DeclarationsBuilderVisitor2
import org.jetbrains.kotlin.descriptors.commonizer.builder.createGlobalBuilderComponents
import org.jetbrains.kotlin.descriptors.commonizer.core.CommonizationVisitor
import org.jetbrains.kotlin.descriptors.commonizer.mergedtree.mergeRoots
import org.jetbrains.kotlin.storage.LockBasedStorageManager
class CommonizationParameters {
// use linked hash map to preserve order
private val modulesByTargets = LinkedHashMap<InputTarget, Collection<ModuleDescriptor>>()
fun addTarget(target: InputTarget, modules: Collection<ModuleDescriptor>): CommonizationParameters {
require(target !in modulesByTargets) { "Target $target is already added" }
val modulesWithUniqueNames = modules.groupingBy { it.name }.eachCount()
require(modulesWithUniqueNames.size == modules.size) {
"Modules with duplicated names found: ${modulesWithUniqueNames.filter { it.value > 1 }}"
}
modulesByTargets[target] = modules
return this
}
// get them as ordered immutable collection (List) for further processing
fun getModulesByTargets(): List<Pair<InputTarget, Collection<ModuleDescriptor>>> =
modulesByTargets.map { it.key to it.value }
fun hasIntersection(): Boolean {
if (modulesByTargets.size < 2)
return false
return modulesByTargets.flatMap { it.value }
.groupingBy { it.name }
.eachCount()
.any { it.value == modulesByTargets.size }
}
}
sealed class CommonizationResult
object NothingToCommonize : CommonizationResult()
class CommonizationPerformed(
val modulesByTargets: Map<Target, Collection<ModuleDescriptor>>
) : CommonizationResult() {
val commonTarget: OutputTarget by lazy {
modulesByTargets.keys.filterIsInstance<OutputTarget>().single()
}
val concreteTargets: Set<InputTarget> by lazy {
modulesByTargets.keys.filterIsInstance<InputTarget>().toSet()
}
}
fun runCommonization(parameters: CommonizationParameters): CommonizationResult {
if (!parameters.hasIntersection())
return NothingToCommonize
@@ -74,7 +26,7 @@ fun runCommonization(parameters: CommonizationParameters): CommonizationResult {
mergedTree.accept(CommonizationVisitor(mergedTree), Unit)
// build resulting descriptors:
val components = mergedTree.createGlobalBuilderComponents(storageManager)
val components = mergedTree.createGlobalBuilderComponents(storageManager, parameters.statsCollector)
mergedTree.accept(DeclarationsBuilderVisitor1(components), emptyList())
mergedTree.accept(DeclarationsBuilderVisitor2(components), emptyList())

View File

@@ -14,6 +14,7 @@ import org.jetbrains.kotlin.descriptors.commonizer.Target
import org.jetbrains.kotlin.descriptors.commonizer.konan.NativeModuleForCommonization.DeserializedModule
import org.jetbrains.kotlin.descriptors.commonizer.konan.NativeModuleForCommonization.SyntheticModule
import org.jetbrains.kotlin.descriptors.commonizer.utils.EmptyDescriptorTable
import org.jetbrains.kotlin.descriptors.commonizer.utils.ResettableClockMark
import org.jetbrains.kotlin.descriptors.commonizer.utils.createKotlinNativeForwardDeclarationsModule
import org.jetbrains.kotlin.konan.library.*
import org.jetbrains.kotlin.konan.target.KonanTarget
@@ -27,13 +28,13 @@ import org.jetbrains.kotlin.serialization.konan.impl.KlibResolvedModuleDescripto
import org.jetbrains.kotlin.storage.LockBasedStorageManager
import java.io.File
import kotlin.time.ExperimentalTime
import kotlin.time.MonoClock
import org.jetbrains.kotlin.konan.file.File as KFile
class NativeDistributionCommonizer(
private val repository: File,
private val targets: List<KonanTarget>,
private val destination: File,
private val withStats: Boolean,
private val handleError: (String) -> Nothing,
private val log: (String) -> Unit
) {
@@ -41,23 +42,21 @@ class NativeDistributionCommonizer(
fun run() {
checkPreconditions()
val startMark = MonoClock.markNow()
val modulesByTargets = loadModules(repository, targets)
val loadDuration = startMark.elapsedNow()
with(ResettableClockMark()) {
// 1. load modules
val modulesByTargets = loadModules()
log("Loaded lazy (uninitialized) libraries in ${elapsedSinceLast()}")
log("Loaded lazy (uninitialized) libraries in $loadDuration")
// 2. run commonization
val result = commonize(modulesByTargets)
log("Commonization performed in ${elapsedSinceLast()}")
val result = commonize(modulesByTargets)
val commonizationDuration = startMark.elapsedNow() - loadDuration
// 3. write new libraries
saveModules(modulesByTargets, result)
log("Written libraries in ${elapsedSinceLast()}")
log("Commonization performed in $commonizationDuration")
saveModules(modulesByTargets, destination, result)
val totalDuration = startMark.elapsedNow()
val writeDuration = totalDuration - commonizationDuration - loadDuration
log("Written libraries in $writeDuration")
log("TOTAL: $totalDuration")
log("TOTAL: ${elapsedSinceStart()}")
}
log("Done.")
log("")
@@ -79,7 +78,7 @@ class NativeDistributionCommonizer(
}
}
private fun loadModules(repository: File, targets: List<KonanTarget>): Map<InputTarget, List<NativeModuleForCommonization>> {
private fun loadModules(): Map<InputTarget, List<NativeModuleForCommonization>> {
val stdlibPath = repository.resolve(konanCommonLibraryPath(KONAN_STDLIB_NAME))
val stdlib = loadLibrary(stdlibPath)
@@ -162,22 +161,24 @@ class NativeDistributionCommonizer(
}
private fun commonize(modulesByTargets: Map<InputTarget, List<NativeModuleForCommonization>>): CommonizationPerformed {
val parameters = CommonizationParameters().apply {
modulesByTargets.forEach { (target, modules) ->
addTarget(target, modules.map { it.module })
val statsCollector = if (withStats) NativeStatsCollector(targets, destination) else null
statsCollector.use {
val parameters = CommonizationParameters(statsCollector).apply {
modulesByTargets.forEach { (target, modules) ->
addTarget(target, modules.map { it.module })
}
}
}
val result = runCommonization(parameters)
return when (result) {
is NothingToCommonize -> handleError("too few targets specified: ${modulesByTargets.keys}")
is CommonizationPerformed -> result
val result = runCommonization(parameters)
return when (result) {
is NothingToCommonize -> handleError("too few targets specified: ${modulesByTargets.keys}")
is CommonizationPerformed -> result
}
}
}
private fun saveModules(
originalModulesByTargets: Map<InputTarget, List<NativeModuleForCommonization>>,
destination: File,
result: CommonizationPerformed
) {
// optimization: stdlib effectively remains the same across all Kotlin/Native targets,
@@ -205,8 +206,6 @@ class NativeDistributionCommonizer(
skipExpects = false
)
val stdlibName = Name.special("<$KONAN_STDLIB_NAME>")
fun serializeTarget(target: Target) {
val libsDestination: File
val newModulesManifestData: Map<Name, NativeSensitiveManifestData>
@@ -226,7 +225,9 @@ class NativeDistributionCommonizer(
for (newModule in newModules) {
val libraryName = newModule.name
if (libraryName == stdlibName || libraryName == KlibResolvedModuleDescriptorsFactoryImpl.FORWARD_DECLARATIONS_MODULE_NAME) continue
if (!shouldBeSerialized(libraryName))
continue
val metadata = serializer.serializeModule(newModule)
val manifestData = newModulesManifestData.getValue(newModule.name)
@@ -250,4 +251,11 @@ class NativeDistributionCommonizer(
manifestData.applyTo(library.base as BaseWriterImpl)
library.commit()
}
private companion object {
val stdlibName = Name.special("<$KONAN_STDLIB_NAME>")
fun shouldBeSerialized(libraryName: Name) =
libraryName != stdlibName && libraryName != KlibResolvedModuleDescriptorsFactoryImpl.FORWARD_DECLARATIONS_MODULE_NAME
}
}

View File

@@ -0,0 +1,132 @@
/*
* Copyright 2010-2019 JetBrains s.r.o. and Kotlin Programming Language contributors.
* Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
*/
package org.jetbrains.kotlin.descriptors.commonizer.konan
import org.jetbrains.kotlin.descriptors.*
import org.jetbrains.kotlin.descriptors.commonizer.StatsCollector
import org.jetbrains.kotlin.descriptors.commonizer.utils.firstNonNull
import org.jetbrains.kotlin.konan.target.KonanTarget
import org.jetbrains.kotlin.resolve.descriptorUtil.fqNameSafe
import java.io.File
/**
* Allows printing commonization statistics to the file system.
*
* File format: text, "|"-separated columns.
*
* Header row: "FQ Name|Declaration Type|common|<platform1>|<platform2>[|<platformN>...]"
*
* Possible values for "Declaration Type":
* - MODULE
* - CLASS
* - INTERFACE
* - OBJECT
* - COMPANION_OBJECT
* - ENUM_CLASS
* - ENUM_ENTRY
* - TYPE_ALIAS
* - CLASS_CONSTRUCTOR
* - FUN
* - VAL
*
* Possible values for "common" column:
* - E = successfully commonized, expect declaration generated
* - "-" = no common declaration
*
* Possible values for each target platform column:
* - A = successfully commonized, actual declaration generated
* - O = not commonized, the declaration is as in the original library
* - "-" = no such declaration in the original library
*
* Example of output:
FQ Name|Declaration Type|common|macos_x64|ios_x64
<SystemConfiguration>|MODULE|E|A|A
platform.SystemConfiguration.SCPreferencesContext|CLASS|E|A|A
platform.SystemConfiguration.SCPreferencesContext.Companion|COMPANION_OBJECT|E|A|A
platform.SystemConfiguration.SCNetworkConnectionContext|CLASS|E|A|A
platform.SystemConfiguration.SCNetworkConnectionContext.Companion|COMPANION_OBJECT|E|A|A
platform.SystemConfiguration.SCDynamicStoreRefVar|TYPE_ALIAS|-|O|O
platform.SystemConfiguration.SCVLANInterfaceRef|TYPE_ALIAS|-|O|O
*/
class NativeStatsCollector(
private val targets: List<KonanTarget>,
destination: File
) : StatsCollector {
init {
destination.mkdirs()
}
private val writer = destination.resolve("plain_stats.csv").printWriter()
private var headerWritten = false
override fun logStats(output: List<DeclarationDescriptor?>) {
if (!headerWritten) {
headerWritten = true
writeHeader()
}
val firstNotNull = output.firstNonNull()
val lastIsNull = output.last() == null
val row = buildString {
append((firstNotNull as? ModuleDescriptor)?.name ?: firstNotNull.fqNameSafe) // FQN (or name for module descriptor)
append(SEPARATOR)
append(firstNotNull.declarationType) // readable declaration type
append(SEPARATOR)
append(if (lastIsNull) '-' else 'E') // common
for (index in 0 until output.size - 1) {
append(SEPARATOR)
append(
when {
output[index] == null -> '-' // absent
lastIsNull -> 'O' // original (not commonized)
else -> 'A' // actual (commonized)
}
)
}
}
writer.println(row)
}
override fun close() = writer.close()
private fun writeHeader() {
val row = buildString {
append("FQ Name")
append(SEPARATOR)
append("Declaration Type")
append(SEPARATOR)
append("common")
targets.forEach { target ->
append(SEPARATOR)
append(target.name)
}
}
writer.println(row)
}
companion object {
private const val SEPARATOR = '|'
private inline val DeclarationDescriptor.declarationType: String
get() = when (this) {
is ClassDescriptor -> if (isCompanionObject) "COMPANION_OBJECT" else kind.toString()
is TypeAliasDescriptor -> "TYPE_ALIAS"
is ClassConstructorDescriptor -> "CLASS_CONSTRUCTOR"
is FunctionDescriptor -> "FUN"
is PropertyDescriptor -> "VAL"
is ModuleDescriptor -> "MODULE"
else -> "UNKNOWN: ${this::class.java}"
}
}
}

View File

@@ -0,0 +1,19 @@
/*
* Copyright 2010-2019 JetBrains s.r.o. and Kotlin Programming Language contributors.
* Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
*/
package org.jetbrains.kotlin.descriptors.commonizer.utils
import kotlin.time.Duration
import kotlin.time.ExperimentalTime
import kotlin.time.MonoClock
@ExperimentalTime
internal class ResettableClockMark {
private val startMark = MonoClock.markNow()
private var lastMark = startMark
fun elapsedSinceLast(): Duration = lastMark.elapsedNow().also { lastMark = lastMark.plus(it) }
fun elapsedSinceStart(): Duration = startMark.elapsedNow()
}