jlengrand/kotlin
1
0
Fork 0
mirror of https://github.com/jlengrand/kotlin.git synced 2026-03-10 08:31:29 +00:00
Code Issues Packages Projects Releases Wiki Activity

[Commonizer] Add 'commonizer_target' and 'commonizer_native_targets' to commonized klib's manifest

Browse Source 
^KT-46077 Fixed
This commit is contained in:
sebastian.sellmair
2021-04-14 13:58:10 +02:00
committed by Space
parent 6f96c225d7
commit bbcfde7231
13 changed files with 147 additions and 99 deletions
Download Patch File Download Diff File Expand all files Collapse all files

24
compiler/util-klib/src/org/jetbrains/kotlin/library/KotlinLibrary.kt
View File

@@ -22,6 +22,19 @@ const val KLIB_PROPERTY_INTEROP = "interop"
const val KLIB_PROPERTY_EXPORT_FORWARD_DECLARATIONS = "exportForwardDeclarations"
const val KLIB_PROPERTY_NATIVE_TARGETS = "native_targets"
// Commonizer-specific:
/**
* Identity String of the commonizer target representing this artifact.
* This will also include native targets that were absent during commonization
*/
const val KLIB_PROPERTY_COMMONIZER_TARGET = "commonizer_target"
/**
* Similar to [KLIB_PROPERTY_NATIVE_TARGETS] but this will also preserve targets
* that were unsupported on the host creating this artifact
*/
const val KLIB_PROPERTY_COMMONIZER_NATIVE_TARGETS = "commonizer_native_targets"
/**
* Abstractions for getting access to the information stored inside of Kotlin/Native library.
*/
@@ -31,6 +44,7 @@ interface BaseKotlinLibrary {
val libraryFile: File
val componentList: List<String>
val versions: KotlinLibraryVersioning
// Whether this library is default (provided by distribution)?
val isDefault: Boolean
val manifestProperties: Properties
@@ -80,4 +94,12 @@ val BaseKotlinLibrary.nativeTargets: List<String>
get() = manifestProperties.propertyList(KLIB_PROPERTY_NATIVE_TARGETS)
val KotlinLibrary.containsErrorCode: Boolean
get() = manifestProperties.getProperty(KLIB_PROPERTY_CONTAINS_ERROR_CODE) == "true"
get() = manifestProperties.getProperty(KLIB_PROPERTY_CONTAINS_ERROR_CODE) == "true"
val KotlinLibrary.commonizerTarget: String?
get() = manifestProperties.getProperty(KLIB_PROPERTY_COMMONIZER_TARGET)
val KotlinLibrary.commonizerNativeTargets: List<String>?
get() = if (manifestProperties.containsKey(KLIB_PROPERTY_COMMONIZER_NATIVE_TARGETS))
manifestProperties.propertyList(KLIB_PROPERTY_COMMONIZER_NATIVE_TARGETS, escapeInQuotes = true)
else null

10
native/commonizer-api/test/org/jetbrains/kotlin/commonizer/CommonizeLibcurlTest.kt
View File

@@ -81,6 +81,14 @@ class CommonizeLibcurlTest {
assertContainsManifestWithContent(x64OutputDirectory, "native_targets=linux_x64")
assertContainsManifestWithContent(arm64OutputDirectory, "native_targets=linux_arm64")
assertContainsManifestWithContent(commonOutputDirectory, "native_targets=linux_x64 linux_arm64")
assertContainsManifestWithContent(commonOutputDirectory, "native_targets=linux_arm64 linux_x64")
assertContainsManifestWithContent(x64OutputDirectory, "commonizer_target=linux_x64")
assertContainsManifestWithContent(arm64OutputDirectory, "commonizer_target=linux_arm64")
assertContainsManifestWithContent(commonOutputDirectory, "commonizer_native_targets=linux_arm64 linux_x64")
assertContainsManifestWithContent(
commonOutputDirectory, "commonizer_target=${CommonizerTarget(LINUX_X64, LINUX_ARM64).identityString}"
)
}
}

8
native/commonizer/src/org/jetbrains/kotlin/commonizer/UniqueLibraryName.kt Normal file
View File

@@ -0,0 +1,8 @@
/*
* Copyright 2010-2021 JetBrains s.r.o. and Kotlin Programming Language contributors.
* Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
*/
package org.jetbrains.kotlin.commonizer
internal typealias UniqueLibraryName = String

2
native/commonizer/src/org/jetbrains/kotlin/commonizer/facade.kt
View File

@@ -105,7 +105,7 @@ private fun serialize(parameters: CommonizerParameters, mergedTree: CirRootNode,
val serializedMetadata = with(metadataModule.write(KLIB_FRAGMENT_WRITE_STRATEGY)) {
SerializedMetadata(header, fragments, fragmentNames)
}
val manifestData = parameters.manifestProvider[target].getManifest(libraryName)
val manifestData = parameters.manifestProvider[target].buildManifest(libraryName)
parameters.resultsConsumer.consume(target, ModuleResult.Commonized(libraryName, serializedMetadata, manifestData))
}
parameters.resultsConsumer.targetConsumed(target)

6
native/commonizer/src/org/jetbrains/kotlin/commonizer/konan/LibraryCommonizer.kt
View File

@@ -29,7 +29,7 @@ internal class LibraryCommonizer internal constructor(
private fun loadLibraries(): TargetDependent<NativeLibrariesToCommonize?> {
val libraries = EagerTargetDependent(outputTarget.allLeaves()) { target ->
repository.getLibraries(target).toList().ifNotEmpty(::NativeLibrariesToCommonize)
repository.getLibraries(target).toList().ifNotEmpty { NativeLibrariesToCommonize(target, this) }
}
libraries.forEachWithTarget { target, librariesOrNull ->
@@ -76,8 +76,8 @@ internal class LibraryCommonizer internal constructor(
return TargetDependent(outputTarget.withAllAncestors()) { target ->
when (target) {
is LeafCommonizerTarget -> libraries[target] ?: error("Can't provide manifest for missing target $target")
is SharedCommonizerTarget -> CommonNativeManifestDataProvider(
target.allLeaves().mapNotNull { leafTarget -> libraries.getOrNull(leafTarget) }
is SharedCommonizerTarget -> NativeManifestDataProvider(
target, target.allLeaves().mapNotNull { leafTarget -> libraries.getOrNull(leafTarget) }
)
}
}

4
native/commonizer/src/org/jetbrains/kotlin/commonizer/konan/ModuleSerializer.kt
View File

@@ -47,13 +47,13 @@ private fun writeLibrary(
moduleName = manifestData.uniqueName,
versions = manifestData.versions,
builtInsPlatform = BuiltInsPlatform.NATIVE,
nativeTargets = emptyList(), // will be overwritten with NativeSensitiveManifestData.applyTo() below
nativeTargets = emptyList(), // will be overwritten with addManifest(manifestData) below
nopack = true,
shortName = manifestData.shortName,
layout = layout
)
library.addMetadata(metadata)
manifestData.applyTo(library.base as BaseWriterImpl)
(library.base as BaseWriterImpl).addManifest(manifestData)
library.commit()
}

70
native/commonizer/src/org/jetbrains/kotlin/commonizer/konan/NativeLibrary.kt
View File

@@ -8,10 +8,12 @@
package org.jetbrains.kotlin.commonizer.konan
import gnu.trove.THashMap
import org.jetbrains.kotlin.commonizer.CommonizerTarget
import org.jetbrains.kotlin.commonizer.UniqueLibraryName
import org.jetbrains.kotlin.library.KotlinLibrary
interface NativeManifestDataProvider {
fun getManifest(libraryName: String): NativeSensitiveManifestData
fun buildManifest(libraryName: UniqueLibraryName): NativeSensitiveManifestData
}
/**
@@ -26,43 +28,61 @@ internal class NativeLibrary(
/**
* A collection of Kotlin/Native libraries for a certain Native target.
*/
internal class NativeLibrariesToCommonize(val libraries: List<NativeLibrary>) : NativeManifestDataProvider {
internal class NativeLibrariesToCommonize(
private val target: CommonizerTarget,
val libraries: List<NativeLibrary>
) : NativeManifestDataProvider {
private val manifestIndex: Map<String, NativeSensitiveManifestData> = buildManifestIndex()
override fun getManifest(libraryName: String) = manifestIndex.getValue(libraryName)
override fun buildManifest(
libraryName: String
): NativeSensitiveManifestData {
return manifestIndex.getValue(libraryName).copy(
commonizerTarget = target
)
}
companion object {
fun create(libraries: List<KotlinLibrary>) = NativeLibrariesToCommonize(libraries.map(::NativeLibrary))
internal fun create(target: CommonizerTarget, libraries: List<KotlinLibrary>) = NativeLibrariesToCommonize(
target, libraries.map(::NativeLibrary)
)
}
}
internal class CommonNativeManifestDataProvider(
libraryGroups: Collection<NativeLibrariesToCommonize>
private val target: CommonizerTarget,
private val manifests: Map<UniqueLibraryName, List<NativeSensitiveManifestData>>
) : NativeManifestDataProvider {
private val manifestIndex: Map<String, NativeSensitiveManifestData>
init {
val iterator = libraryGroups.iterator()
val index = iterator.next().buildManifestIndex()
override fun buildManifest(libraryName: UniqueLibraryName): NativeSensitiveManifestData {
val rawManifests = manifests[libraryName] ?: error("Missing manifests for $libraryName")
check(rawManifests.isNotEmpty()) { "No manifests for $libraryName" }
while (iterator.hasNext()) {
val otherIndex = iterator.next().buildManifestIndex()
otherIndex.forEach { (libraryName, otherManifestData) ->
val manifestData = index[libraryName]
if (manifestData != null) {
// merge manifests
index[libraryName] = manifestData.mergeWith(otherManifestData)
} else {
index[libraryName] = otherManifestData
}
}
}
val isInterop = rawManifests.all { it.isInterop }
manifestIndex = index
return NativeSensitiveManifestData(
uniqueName = libraryName,
versions = rawManifests.first().versions,
dependencies = rawManifests.map { it.dependencies }.reduce { acc, list -> acc.intersect(list).toList() },
isInterop = isInterop,
packageFqName = rawManifests.first().packageFqName,
exportForwardDeclarations = if (isInterop) rawManifests.map { it.exportForwardDeclarations }
.reduce { acc, list -> acc.intersect(list).toList() } else emptyList(),
nativeTargets = rawManifests.flatMapTo(mutableSetOf()) { it.nativeTargets },
shortName = rawManifests.first().shortName,
commonizerTarget = target
)
}
override fun getManifest(libraryName: String) = manifestIndex.getValue(libraryName)
}
private fun NativeLibrariesToCommonize.buildManifestIndex(): MutableMap<String, NativeSensitiveManifestData> =
internal fun NativeManifestDataProvider(target: CommonizerTarget, libraries: List<NativeLibrariesToCommonize>): NativeManifestDataProvider {
val manifestsByName = libraries
.flatMap { it.libraries }
.groupByTo(THashMap()) { it.manifestData.uniqueName }
.mapValues { (_, libraries) -> libraries.map { it.manifestData } }
return CommonNativeManifestDataProvider(target, manifestsByName)
}
private fun NativeLibrariesToCommonize.buildManifestIndex(): MutableMap<UniqueLibraryName, NativeSensitiveManifestData> =
libraries.map { it.manifestData }.associateByTo(THashMap()) { it.uniqueName }

93
native/commonizer/src/org/jetbrains/kotlin/commonizer/konan/NativeSensitiveManifestData.kt
View File

@@ -5,6 +5,7 @@
package org.jetbrains.kotlin.commonizer.konan
import org.jetbrains.kotlin.commonizer.*
import org.jetbrains.kotlin.konan.properties.propertyList
import org.jetbrains.kotlin.library.*
import org.jetbrains.kotlin.library.impl.BaseWriterImpl
@@ -14,66 +15,16 @@ import org.jetbrains.kotlin.library.impl.BaseWriterImpl
* preserved in commonized libraries (both for "common" and platform-specific library parts).
*/
data class NativeSensitiveManifestData(
val uniqueName: String,
val uniqueName: UniqueLibraryName,
val versions: KotlinLibraryVersioning,
val dependencies: List<String>,
val isInterop: Boolean,
val packageFqName: String?,
val exportForwardDeclarations: List<String>,
val nativeTargets: Collection<String>,
val shortName: String?
val shortName: String?,
val commonizerTarget: CommonizerTarget?,
) {
fun applyTo(library: BaseWriterImpl) {
library.manifestProperties[KLIB_PROPERTY_UNIQUE_NAME] = uniqueName
// note: versions can't be added here
fun addOptionalProperty(name: String, condition: Boolean, value: () -> String) =
if (condition)
library.manifestProperties[name] = value()
else
library.manifestProperties.remove(name)
addOptionalProperty(KLIB_PROPERTY_DEPENDS, dependencies.isNotEmpty()) { dependencies.joinToString(separator = " ") }
addOptionalProperty(KLIB_PROPERTY_INTEROP, isInterop) { "true" }
addOptionalProperty(KLIB_PROPERTY_PACKAGE, packageFqName != null) { packageFqName!! }
addOptionalProperty(KLIB_PROPERTY_EXPORT_FORWARD_DECLARATIONS, exportForwardDeclarations.isNotEmpty() || isInterop) {
exportForwardDeclarations.joinToString(" ")
}
addOptionalProperty(KLIB_PROPERTY_NATIVE_TARGETS, nativeTargets.isNotEmpty()) {
nativeTargets.joinToString(" ")
}
addOptionalProperty(KLIB_PROPERTY_SHORT_NAME, shortName != null) { shortName!! }
}
fun mergeWith(other: NativeSensitiveManifestData): NativeSensitiveManifestData {
if (this === other) return this
check(uniqueName == other.uniqueName)
// Merge algorithm:
// - Unite native target lists.
// - Intersect dependency lists.
// - Boolean and 'isInterop'.
// - If both libs are 'isInterop' then intersect exported forward declaration lists.
// - Other properties can be taken from 'this' manifest.
val bothAreInterop = isInterop && other.isInterop
return NativeSensitiveManifestData(
uniqueName = uniqueName,
versions = versions,
dependencies = (dependencies intersect other.dependencies).toList(),
isInterop = bothAreInterop,
packageFqName = packageFqName,
exportForwardDeclarations = if (bothAreInterop) (exportForwardDeclarations intersect other.exportForwardDeclarations).toList() else emptyList(),
nativeTargets = HashSet<String>().apply {
addAll(nativeTargets)
addAll(other.nativeTargets)
},
shortName = shortName
)
}
companion object {
fun readFrom(library: KotlinLibrary) = NativeSensitiveManifestData(
@@ -84,7 +35,41 @@ data class NativeSensitiveManifestData(
packageFqName = library.packageFqName,
exportForwardDeclarations = library.exportForwardDeclarations,
nativeTargets = library.nativeTargets,
shortName = library.shortName
shortName = library.shortName,
commonizerTarget = library.commonizerTarget?.let(::parseCommonizerTargetOrNull),
)
}
}
private inline fun BaseWriterImpl.addOptionalProperty(name: String, condition: Boolean, value: () -> String) {
if (condition) manifestProperties[name] = value()
else manifestProperties.remove(name)
}
fun BaseWriterImpl.addManifest(manifest: NativeSensitiveManifestData) {
manifestProperties[KLIB_PROPERTY_UNIQUE_NAME] = manifest.uniqueName
// note: versions can't be added here
addOptionalProperty(KLIB_PROPERTY_DEPENDS, manifest.dependencies.isNotEmpty()) { manifest.dependencies.joinToString(separator = " ") }
addOptionalProperty(KLIB_PROPERTY_INTEROP, manifest.isInterop) { "true" }
addOptionalProperty(KLIB_PROPERTY_PACKAGE, manifest.packageFqName != null) { manifest.packageFqName!! }
addOptionalProperty(KLIB_PROPERTY_EXPORT_FORWARD_DECLARATIONS, manifest.exportForwardDeclarations.isNotEmpty() || manifest.isInterop) {
manifest.exportForwardDeclarations.joinToString(" ")
}
addOptionalProperty(KLIB_PROPERTY_NATIVE_TARGETS, manifest.nativeTargets.isNotEmpty()) {
manifest.nativeTargets.sorted().joinToString(" ")
}
addOptionalProperty(KLIB_PROPERTY_SHORT_NAME, manifest.shortName != null) { manifest.shortName!! }
addOptionalProperty(KLIB_PROPERTY_COMMONIZER_TARGET, manifest.commonizerTarget != null) {
manifest.commonizerTarget?.identityString ?: error("Unexpected missing 'commonizerTarget'")
}
addOptionalProperty(KLIB_PROPERTY_COMMONIZER_NATIVE_TARGETS, manifest.commonizerTarget != null) {
manifest.commonizerTarget?.konanTargets?.map { it.name }?.sorted()?.joinToString(" ")
?: error("Unexpected missing 'commonizerTarget'")
}
}

5
native/commonizer/src/org/jetbrains/kotlin/commonizer/repository/FilesRepository.kt
View File

@@ -20,7 +20,10 @@ internal class FilesRepository(
private val librariesByKonanTargets: Map<Set<KonanTarget>, Set<NativeLibrary>> by lazy {
libraryFiles
.map(libraryLoader::invoke)
.groupBy { library -> library.manifestData.nativeTargets.map(::konanTargetOrThrow).toSet() }
.groupBy { library ->
library.manifestData.commonizerTarget?.konanTargets
?: library.manifestData.nativeTargets.map(::konanTargetOrThrow).toSet()
}
.mapValues { (_, list) -> list.toSet() }
}

4
native/commonizer/tests/org/jetbrains/kotlin/commonizer/AbstractCommonizationFromSourcesTest.kt
View File

@@ -206,10 +206,10 @@ private class AnalyzedModules(
fun toCommonizerParameters(
resultsConsumer: ResultsConsumer,
manifestDataProvider: NativeManifestDataProvider = MockNativeManifestDataProvider()
manifestDataProvider: (CommonizerTarget) -> NativeManifestDataProvider = { MockNativeManifestDataProvider(it) }
) = CommonizerParameters(
outputTarget = SharedCommonizerTarget(leafTargets.toSet()),
manifestProvider = TargetDependent(sharedTarget.withAllAncestors()) { manifestDataProvider },
manifestProvider = TargetDependent(sharedTarget.withAllAncestors(), manifestDataProvider),
dependenciesProvider = TargetDependent(sharedTarget.withAllAncestors()) { dependencyModules[it]?.let(MockModulesProvider::create) },
targetProviders = TargetDependent(leafTargets) { leafTarget ->
TargetProvider(

4
native/commonizer/tests/org/jetbrains/kotlin/commonizer/AbstractInlineSourcesCommonizationTest.kt
View File

@@ -148,11 +148,11 @@ abstract class AbstractInlineSourcesCommonizationTest : KtInlineSourceCommonizer
private fun Parameters.toCommonizerParameters(
resultsConsumer: ResultsConsumer,
manifestDataProvider: NativeManifestDataProvider = MockNativeManifestDataProvider()
manifestDataProvider: (CommonizerTarget) -> NativeManifestDataProvider = { MockNativeManifestDataProvider(it) }
): CommonizerParameters {
return CommonizerParameters(
outputTarget = outputTarget,
manifestProvider = TargetDependent(outputTarget.withAllAncestors()) { manifestDataProvider },
manifestProvider = TargetDependent(outputTarget.withAllAncestors(), manifestDataProvider),
dependenciesProvider = TargetDependent(outputTarget.withAllAncestors()) { target ->
val explicitDependencies = dependencies.getOrNull(target).orEmpty().map { module -> createModuleDescriptor(module) }
val implicitDependencies = listOfNotNull(if (target == outputTarget) DefaultBuiltIns.Instance.builtInsModule else null)

4
native/commonizer/tests/org/jetbrains/kotlin/commonizer/CommonizerFacadeTest.kt
View File

@@ -67,7 +67,7 @@ class CommonizerFacadeTest {
private fun Map<String, List<String>>.toCommonizerParameters(
resultsConsumer: ResultsConsumer,
manifestDataProvider: NativeManifestDataProvider = MockNativeManifestDataProvider()
manifestDataProvider: (CommonizerTarget) -> NativeManifestDataProvider = { MockNativeManifestDataProvider(it) }
): CommonizerParameters {
val targetDependentModuleNames = mapKeys { (targetName, _) -> LeafCommonizerTarget(targetName) }.toTargetDependent()
val sharedTarget = SharedCommonizerTarget(targetDependentModuleNames.targets.toSet())
@@ -75,7 +75,7 @@ class CommonizerFacadeTest {
return CommonizerParameters(
outputTarget = sharedTarget,
dependenciesProvider = TargetDependent(sharedTarget.withAllAncestors()) { null },
manifestProvider = TargetDependent(sharedTarget.withAllAncestors()) { manifestDataProvider },
manifestProvider = TargetDependent(sharedTarget.withAllAncestors(), manifestDataProvider),
targetProviders = targetDependentModuleNames.map { target, moduleNames ->
TargetProvider(
target = target,

12
native/commonizer/tests/org/jetbrains/kotlin/commonizer/utils/mocks.kt
View File

@@ -9,8 +9,6 @@ package org.jetbrains.kotlin.commonizer.utils
import org.jetbrains.kotlin.backend.common.serialization.metadata.KlibMetadataMonolithicSerializer
import org.jetbrains.kotlin.backend.common.serialization.metadata.KlibMetadataVersion
import org.jetbrains.kotlin.config.LanguageVersionSettingsImpl
import org.jetbrains.kotlin.descriptors.*
import org.jetbrains.kotlin.commonizer.*
import org.jetbrains.kotlin.commonizer.ModulesProvider.ModuleInfo
import org.jetbrains.kotlin.commonizer.ResultsConsumer.ModuleResult
@@ -18,6 +16,8 @@ import org.jetbrains.kotlin.commonizer.cir.*
import org.jetbrains.kotlin.commonizer.konan.NativeManifestDataProvider
import org.jetbrains.kotlin.commonizer.konan.NativeSensitiveManifestData
import org.jetbrains.kotlin.commonizer.mergedtree.*
import org.jetbrains.kotlin.config.LanguageVersionSettingsImpl
import org.jetbrains.kotlin.descriptors.*
import org.jetbrains.kotlin.library.KotlinLibraryVersioning
import org.jetbrains.kotlin.library.SerializedMetadata
import org.jetbrains.kotlin.storage.LockBasedStorageManager
@@ -148,7 +148,7 @@ internal class MockResultsConsumer : ResultsConsumer {
override fun consume(target: CommonizerTarget, moduleResult: ModuleResult) {
check(!this::status.isInitialized)
check(target !in finishedTargets) { "$target already finished"}
check(target !in finishedTargets) { "$target already finished" }
val moduleResults: ModuleResults = _modulesByTargets.getOrPut(target) { ModuleResults() }
val oldResult = moduleResults.put(moduleResult.libraryName, moduleResult)
check(oldResult == null) // to avoid accidental overwriting
@@ -169,6 +169,7 @@ internal class MockResultsConsumer : ResultsConsumer {
}
fun MockNativeManifestDataProvider(
target: CommonizerTarget,
uniqueName: String = "mock",
versions: KotlinLibraryVersioning = KotlinLibraryVersioning(null, null, null, null, null),
dependencies: List<String> = emptyList(),
@@ -178,7 +179,7 @@ fun MockNativeManifestDataProvider(
nativeTargets: Collection<String> = emptyList(),
shortName: String? = "mock"
): NativeManifestDataProvider = object : NativeManifestDataProvider {
override fun getManifest(libraryName: String): NativeSensitiveManifestData {
override fun buildManifest(libraryName: UniqueLibraryName): NativeSensitiveManifestData {
return NativeSensitiveManifestData(
uniqueName = uniqueName,
versions = versions,
@@ -187,7 +188,8 @@ fun MockNativeManifestDataProvider(
packageFqName = packageFqName,
exportForwardDeclarations = exportForwardDeclarations,
nativeTargets = nativeTargets,
shortName = shortName
shortName = shortName,
commonizerTarget = target,
)
}
}