Prevent memory leaks caused by DeprecationResolver

Use soft references for descriptors from different resolution facades
in DeprecationResolver. Provide resolution facade explicitly to prevent
de-sync.

compiler/frontend/src/org/jetbrains/kotlin/resolve/deprecation/DeprecationResolver.kt

@@ -32,28 +32,20 @@ import org.jetbrains.kotlin.types.KotlinType
 import org.jetbrains.kotlin.types.TypeUtils
 import org.jetbrains.kotlin.utils.SmartList
 import org.jetbrains.kotlin.utils.addToStdlib.safeAs
+import com.google.common.collect.*
+import org.jetbrains.kotlin.resolve.descriptorUtil.module
+import org.jetbrains.kotlin.storage.MemoizedFunctionToNotNull
+import java.util.concurrent.ConcurrentMap
 
 class DeprecationResolver(
     storageManager: StorageManager,
     private val languageVersionSettings: LanguageVersionSettings,
     private val coroutineCompatibilitySupport: CoroutineCompatibilitySupport,
-    private val deprecationSettings: DeprecationSettings
+    private val deprecationSettings: DeprecationSettings,
+    private val moduleDescriptor: ModuleDescriptor? = null,
 ) {
-    private val deprecations = storageManager.createMemoizedFunction { descriptor: DeclarationDescriptor ->
-        val deprecations = descriptor.getOwnDeprecations()
-        when {
-            deprecations.isNotEmpty() -> deprecations
-            descriptor is CallableMemberDescriptor -> listOfNotNull(deprecationByOverridden(descriptor))
-            else -> emptyList()
-        }
-    }
-
-    private val isHiddenBecauseOfKotlinVersionAccessibility = storageManager.createMemoizedFunction { descriptor: DeclarationDescriptor ->
-        descriptor.checkSinceKotlinVersionAccessibility(languageVersionSettings)
-    }
-
     fun getDeprecations(descriptor: DeclarationDescriptor): List<Deprecation> =
-        deprecations(descriptor.original)
+        memoizeBasedOnContainingModule(descriptor, deprecations, deprecationsForeignModules)
 
     fun isDeprecatedHidden(descriptor: DeclarationDescriptor): Boolean =
         getDeprecations(descriptor).any { it.deprecationLevel == DeprecationLevelValue.HIDDEN }
@@ -70,7 +62,8 @@ class DeprecationResolver(
             if (descriptor.isHiddenForResolutionEverywhereBesideSupercalls && !isSuperCall) return true
         }
 
-        val sinceKotlinAccessibility = isHiddenBecauseOfKotlinVersionAccessibility(descriptor.original)
+        val sinceKotlinAccessibility = getSinceKotlinAccessibility(descriptor)
+
         if (sinceKotlinAccessibility is SinceKotlinAccessibility.NotAccessible) return true
 
         if (sinceKotlinAccessibility is SinceKotlinAccessibility.NotAccessibleButWasExperimental) {
@@ -87,6 +80,45 @@ class DeprecationResolver(
         return isDeprecatedHidden(descriptor)
     }
 
+    private val deprecations = storageManager.createMemoizedFunction(::doGetDeprecations)
+    private val isHiddenBecauseOfKotlinVersionAccessibility = storageManager.createMemoizedFunction(::doCheckHiddenKotlinVersion)
+
+    private val deprecationsForeignModules = storageManager.createMemoizedFunction(::doGetDeprecations, newConcurrentHashMapWithWeakKeys())
+    private val isHiddenBecauseOfKotlinVersionAccessibilityForeignModules =
+        storageManager.createMemoizedFunction(::doCheckHiddenKotlinVersion, newConcurrentHashMapWithWeakKeys())
+
+    private fun doGetDeprecations(descriptor: DeclarationDescriptor): List<Deprecation> {
+        val deprecations = descriptor.getOwnDeprecations()
+        return when {
+            deprecations.isNotEmpty() -> deprecations
+            descriptor is CallableMemberDescriptor -> listOfNotNull(deprecationByOverridden(descriptor))
+            else -> emptyList()
+        }
+    }
+
+    private fun doCheckHiddenKotlinVersion(descriptor: DeclarationDescriptor): SinceKotlinAccessibility =
+        descriptor.checkSinceKotlinVersionAccessibility(languageVersionSettings)
+
+    private fun getSinceKotlinAccessibility(descriptor: DeclarationDescriptor): SinceKotlinAccessibility =
+        memoizeBasedOnContainingModule(
+            descriptor,
+            isHiddenBecauseOfKotlinVersionAccessibility,
+            isHiddenBecauseOfKotlinVersionAccessibilityForeignModules
+        )
+
+    private fun <R : Any> memoizeBasedOnContainingModule(
+        descriptor: DeclarationDescriptor,
+        forSameModule: MemoizedFunctionToNotNull<DeclarationDescriptor, R>,
+        forDifferentModule: MemoizedFunctionToNotNull<DeclarationDescriptor, R>,
+    ): R {
+        descriptor.original.let { originalDescriptor ->
+            return if (moduleDescriptor == null || originalDescriptor.module == moduleDescriptor)
+                forSameModule(originalDescriptor)
+            else
+                forDifferentModule(originalDescriptor)
+        }
+    }
+
     private fun KotlinType.deprecationsByConstituentTypes(): List<Deprecation> =
         SmartList<Deprecation>().also { deprecations ->
             TypeUtils.contains(this) { type ->
@@ -275,3 +307,7 @@ class DeprecationResolver(
         private val JAVA_DEPRECATED = FqName("java.lang.Deprecated")
     }
 }
+
+private fun <K, V> newConcurrentHashMapWithWeakKeys(): ConcurrentMap<K, V> =
+    MapMaker().concurrencyLevel(2).initialCapacity(3).weakKeys().makeMap()
+

idea/idea-analysis/src/org/jetbrains/kotlin/idea/caches/resolve/IdeaResolverForProject.kt

@@ -34,6 +34,11 @@ import org.jetbrains.kotlin.platform.jvm.isJvm
 import org.jetbrains.kotlin.platform.toTargetPlatform
 import org.jetbrains.kotlin.psi.KtFile
 import org.jetbrains.kotlin.resolve.jvm.JvmPlatformParameters
+import java.util.concurrent.atomic.AtomicLong
+
+object IdeaResolverForProjectCounter {
+    val counter = AtomicLong()
+}
 
 class IdeaResolverForProject(
     debugName: String,
@@ -53,6 +58,15 @@ class IdeaResolverForProject(
     delegateResolver,
     ServiceManager.getService(projectContext.project, IdePackageOracleFactory::class.java)
 ) {
+    init {
+        try {
+            println("IdeaResolverForProject $this")
+            //throw Exception("IdeaResolverForProject $this")
+        } catch (e: Exception) {
+            e.printStackTrace()
+        }
+    }
+
     private val builtInsCache: BuiltInsCache =
         (delegateResolver as? IdeaResolverForProject)?.builtInsCache ?: BuiltInsCache(projectContext, this)
 

idea/idea-analysis/src/org/jetbrains/kotlin/idea/caches/resolve/ProjectResolutionFacade.kt

@@ -117,7 +117,7 @@ internal class ProjectResolutionFacade(
         val modulesToCreateResolversFor = allModuleInfos.filter(moduleFilter)
 
         val resolverForProject = IdeaResolverForProject(
-            resolverDebugName,
+            "#${IdeaResolverForProjectCounter.counter.getAndIncrement()} $resolverDebugName",
             globalContext.withProject(project),
             modulesToCreateResolversFor,
             syntheticFilesByModule,

idea/idea-core/src/org/jetbrains/kotlin/idea/core/KotlinIndicesHelper.kt

@@ -26,6 +26,7 @@ import com.intellij.util.indexing.IdFilter
 import org.jetbrains.kotlin.asJava.elements.KtLightElement
 import org.jetbrains.kotlin.descriptors.*
 import org.jetbrains.kotlin.idea.caches.KotlinShortNamesCache
+import org.jetbrains.kotlin.idea.caches.resolve.getResolutionFacade
 import org.jetbrains.kotlin.idea.caches.resolve.resolveImportReference
 import org.jetbrains.kotlin.idea.caches.resolve.unsafeResolveToDescriptor
 import org.jetbrains.kotlin.idea.caches.resolve.util.getJavaMemberDescriptor
@@ -62,9 +63,9 @@ import java.util.*
 class KotlinIndicesHelper(
     private val resolutionFacade: ResolutionFacade,
     private val scope: GlobalSearchScope,
-    visibilityFilter: (DeclarationDescriptor) -> Boolean,
+    private val visibilityFilter: (DeclarationDescriptor) -> Boolean,
     private val declarationTranslator: (KtDeclaration) -> KtDeclaration? = { it },
-    applyExcludeSettings: Boolean = true,
+    private val applyExcludeSettings: Boolean = true,
     private val filterOutPrivate: Boolean = true,
     private val file: KtFile? = null
 ) {
@@ -73,11 +74,11 @@ class KotlinIndicesHelper(
     private val project = resolutionFacade.project
     private val scopeWithoutKotlin = scope.excludeKotlinSources() as GlobalSearchScope
 
-    private val descriptorFilter: (DeclarationDescriptor) -> Boolean = filter@{
-        if (resolutionFacade.frontendService<DeprecationResolver>().isHiddenInResolution(it)) return@filter false
-        if (!visibilityFilter(it)) return@filter false
-        if (applyExcludeSettings && it.isExcludedFromAutoImport(project, file)) return@filter false
-        true
+    private fun descriptorFilter(declarationDescriptor: DeclarationDescriptor, facade: ResolutionFacade? = null): Boolean {
+        if ((facade ?: resolutionFacade).frontendService<DeprecationResolver>().isHiddenInResolution(declarationDescriptor)) return false
+        if (!visibilityFilter(declarationDescriptor)) return false
+        if (applyExcludeSettings && declarationDescriptor.isExcludedFromAutoImport(project, file)) return false
+        return true
     }
 
     fun getTopLevelCallablesByName(name: String): Collection<CallableDescriptor> {
@@ -257,15 +258,20 @@ class KotlinIndicesHelper(
         return PsiShortNamesCache.getInstance(project).getClassesByName(name, scope)
             .filter { it in scope && it.containingFile != null }
             .mapNotNull { it.resolveToDescriptor(resolutionFacade) }
-            .filter(descriptorFilter)
+            .filter { descriptorFilter(it) }
             .toSet()
     }
 
     fun getKotlinEnumsByName(name: String): Collection<DeclarationDescriptor> {
         return KotlinClassShortNameIndex.getInstance()[name, project, scope]
             .filter { it is KtEnumEntry && it in scope }
-            .mapNotNull { it.unsafeResolveToDescriptor() }
-            .filter(descriptorFilter)
+            .mapNotNull {
+                val resolutionFacade = it.getResolutionFacade()
+                val resultingDescriptor = it.unsafeResolveToDescriptor(resolutionFacade)
+                if (descriptorFilter(resultingDescriptor, resolutionFacade))
+                    resultingDescriptor
+                else null
+            }
             .toSet()
     }
 
@@ -407,7 +413,9 @@ class KotlinIndicesHelper(
                     .flatMap { it.resolveToDescriptors<TypeAliasDescriptor>() }
 
             }
-            .filter(descriptorFilter)
+            .filter {
+                descriptorFilter(it)
+            }
     }
 
     fun processObjectMembers(
@@ -487,7 +495,7 @@ class KotlinIndicesHelper(
             for (field in shortNamesCache.getFieldsByName(name, scopeWithoutKotlin).filterNot { it is KtLightElement<*, *> }) {
                 if (!field.hasModifierProperty(PsiModifier.STATIC)) continue
                 if (filterOutPrivate && field.hasModifierProperty(PsiModifier.PRIVATE)) continue
-                val descriptor = field.getJavaMemberDescriptor() ?: continue
+                val descriptor = field.getJavaMemberDescriptor(resolutionFacade) ?: continue
                 if (descriptorKindFilter.accepts(descriptor) && descriptorFilter(descriptor)) {
                     processor(descriptor)
                 }

idea/performanceTests/build.gradle.kts

@@ -124,7 +124,7 @@ projectTest(taskName = "wholeProjectsPerformanceTest") {
 
     jvmArgs?.removeAll { it.startsWith("-Xmx") }
 
-    maxHeapSize = "3g"
+    maxHeapSize = "6g"
     jvmArgs("-DperformanceProjects=${System.getProperty("performanceProjects")}")
     jvmArgs("-Didea.debug.mode=true")
     jvmArgs("-DemptyProfile=${System.getProperty("emptyProfile")}")

idea/performanceTests/test/org/jetbrains/kotlin/idea/perf/HighlightWholeProjectPerformanceTest.kt

@@ -5,18 +5,34 @@
 
 package org.jetbrains.kotlin.idea.perf
 
+import com.google.common.util.concurrent.ThreadFactoryBuilder
 import com.intellij.codeInsight.daemon.impl.HighlightInfo
 import com.intellij.openapi.module.impl.ProjectLoadingErrorsHeadlessNotifier
 import com.intellij.testFramework.UsefulTestCase
 import org.jetbrains.kotlin.idea.perf.util.*
 import org.jetbrains.kotlin.idea.testFramework.ProjectOpenAction
 import org.jetbrains.kotlin.test.JUnit3RunnerWithInners
+import org.jetbrains.kotlin.utils.KotlinExceptionWithAttachments
 import org.junit.runner.RunWith
 import java.io.File
+import java.util.concurrent.ExecutionException
+import java.util.concurrent.Executors
+import java.util.concurrent.ScheduledFuture
+import java.util.concurrent.TimeUnit
 
 @RunWith(JUnit3RunnerWithInners::class)
 class HighlightWholeProjectPerformanceTest : UsefulTestCase() {
 
+    private val THREAD_FACTORY = ThreadFactoryBuilder()
+        .setNameFormat("heap-dumper-thread-%d")
+        .setDaemon(true)
+        .build()
+
+    private val heapDumpThreadService = Executors.newSingleThreadScheduledExecutor(THREAD_FACTORY)
+    private var future: ScheduledFuture<*>? = null
+
+    override fun shouldContainTempFiles(): Boolean = false
+
     override fun setUp() {
         val allowedErrorDescription = setOf(
             "Unknown artifact type: war",
@@ -33,22 +49,35 @@ class HighlightWholeProjectPerformanceTest : UsefulTestCase() {
                 }
             }, testRootDisposable
         )
+
+        future =
+            heapDumpThreadService.scheduleAtFixedRate(
+                {
+                    HeapDumper.dumpHeap("HighlightWholeProjectPerformanceTest")
+                }, 2, 60, TimeUnit.MINUTES
+            )
+    }
+
+    override fun tearDown() {
+        super.tearDown()
+
+        future?.let { it.cancel(true) }
     }
 
     fun testHighlightAllKtFilesInProject() {
         val emptyProfile = System.getProperty("emptyProfile", "false").toBoolean()
         val projectSpecs = projectSpecs()
-        suite(suiteName = "allKtFilesInProject") {
-            app {
-                warmUpProject()
+        for (projectSpec in projectSpecs) {
+            val projectName = projectSpec.name
+            val projectPath = projectSpec.path
+            suite("$projectName project") {
+                app {
+                    warmUpProject()
 
-                with(config) {
-                    warmup = 1
-                    iterations = 3
-                }
-                for (projectSpec in projectSpecs) {
-                    val projectName = projectSpec.name
-                    val projectPath = projectSpec.path
+                    with(config) {
+                        warmup = 1
+                        iterations = 3
+                    }
 
                     try {
                         project(ExternalProject(projectPath, ProjectOpenAction.GRADLE_PROJECT), refresh = true) {
@@ -77,25 +106,38 @@ class HighlightWholeProjectPerformanceTest : UsefulTestCase() {
 
                                 try {
                                     fixture(fileName).use {
-                                        measure<List<HighlightInfo>>(fileName) {
+                                        measure<List<HighlightInfo>>("highlighting", fixture = it) {
                                             test = {
                                                 highlight(it)
                                             }
                                         }
                                     }
-                                } catch (e: Exception) {
-                                    // nothing as it is already caught by perfTest
+                                } catch (e: Throwable) {
+                                    handle(e)
                                 }
                             }
                         }
-                    } catch (e: Exception) {
-                        // nothing as it is already caught by perfTest
+                    } catch (e: Throwable) {
+                        handle(e)
                     }
                 }
             }
         }
     }
 
+    private fun handle(e: Throwable) {
+        when (e) {
+            is RuntimeException, is ExecutionException -> e.cause?.let { handle(it) }
+            is Exception -> return // nothing as it is already caught by perfTest
+            is NoClassDefFoundError -> return // nothing as it is already caught by perfTest
+            is OutOfMemoryError -> {
+                HeapDumper.dumpHeap("HighlightWholeProjectPerformanceTest-OOM")
+                throw e
+            }
+            else -> throw e
+        }
+    }
+
     private fun limitedFiles(ktFiles: List<File>, partPercent: Int): Collection<File> {
         val sortedBySize = ktFiles
             .filter { it.length() > 0 }

idea/performanceTests/test/org/jetbrains/kotlin/idea/perf/util/logging.kt

@@ -26,6 +26,9 @@ fun logMessage(t: Throwable, message: () -> String) {
 
 object TeamCity {
     inline fun message(block: () -> String) {
+        // TODO: temp
+        println("--teamcity[${block()}]")
+
         println("##teamcity[${block()}]")
     }
 
@@ -48,11 +51,10 @@ object TeamCity {
             block()
         } finally {
             statValue(name, durationMs ?: -1)
-            if (errorDetails != null) {
-                testFailed(name, errorDetails)
-            } else {
-                testFinished(name, durationMs)
+            errorDetails?.let {
+                testFailed(name, it)
             }
+            testFinished(name, durationMs)
         }
     }
 

idea/src/org/jetbrains/kotlin/idea/KotlinDocumentationProvider.kt

@@ -34,6 +34,7 @@ import org.jetbrains.kotlin.idea.kdoc.KDocRenderer.appendKDocContent
 import org.jetbrains.kotlin.idea.kdoc.KDocRenderer.appendKDocSections
 import org.jetbrains.kotlin.idea.kdoc.KDocTemplate.DescriptionBodyTemplate
 import org.jetbrains.kotlin.idea.references.mainReference
+import org.jetbrains.kotlin.idea.resolve.ResolutionFacade
 import org.jetbrains.kotlin.idea.resolve.frontendService
 import org.jetbrains.kotlin.idea.util.isRunningInCidrIde
 import org.jetbrains.kotlin.js.resolve.diagnostics.findPsi
@@ -140,10 +141,11 @@ open class KotlinDocumentationProviderCompatBase : AbstractDocumentationProvider
 
     override fun getDocumentationElementForLink(psiManager: PsiManager, link: String, context: PsiElement?): PsiElement? {
         val navElement = context?.navigationElement as? KtElement ?: return null
-        val bindingContext = navElement.analyze(BodyResolveMode.PARTIAL)
+        val resolutionFacade = navElement.getResolutionFacade()
+        val bindingContext = navElement.analyze(resolutionFacade, BodyResolveMode.PARTIAL)
         val contextDescriptor = bindingContext[BindingContext.DECLARATION_TO_DESCRIPTOR, navElement] ?: return null
         val descriptors = resolveKDocLink(
-            bindingContext, navElement.getResolutionFacade(),
+            bindingContext, resolutionFacade,
             contextDescriptor, null, link.split('.')
         )
         val target = descriptors.firstOrNull() ?: return null
@@ -316,7 +318,8 @@ open class KotlinDocumentationProviderCompatBase : AbstractDocumentationProvider
         }
 
         private fun buildKotlinDeclaration(declaration: KtExpression, quickNavigation: Boolean): KDocTemplate {
-            val context = declaration.analyze(BodyResolveMode.PARTIAL)
+            val resolutionFacade = declaration.getResolutionFacade()
+            val context = declaration.analyze(resolutionFacade, BodyResolveMode.PARTIAL)
             val declarationDescriptor = context[BindingContext.DECLARATION_TO_DESCRIPTOR, declaration]
 
             if (declarationDescriptor == null) {
@@ -328,29 +331,33 @@ open class KotlinDocumentationProviderCompatBase : AbstractDocumentationProvider
                 }
             }
 
-            return buildKotlin(context, declarationDescriptor, quickNavigation, declaration)
+            return buildKotlin(context, declarationDescriptor, quickNavigation, declaration, resolutionFacade)
         }
 
         private fun renderKotlinImplicitLambdaParameter(element: KtReferenceExpression, quickNavigation: Boolean): String? {
-            val context = element.analyze(BodyResolveMode.PARTIAL)
+            val resolutionFacade = element.getResolutionFacade()
+            val context = element.analyze(resolutionFacade, BodyResolveMode.PARTIAL)
+            element.analyze(BodyResolveMode.PARTIAL)
             val target = element.mainReference.resolveToDescriptors(context).singleOrNull() as? ValueParameterDescriptor? ?: return null
-            return renderKotlin(context, target, quickNavigation, element)
+            return renderKotlin(context, target, quickNavigation, element, resolutionFacade)
         }
 
         private fun renderKotlin(
             context: BindingContext,
             declarationDescriptor: DeclarationDescriptor,
             quickNavigation: Boolean,
-            ktElement: KtElement
+            ktElement: KtElement,
+            resolutionFacade: ResolutionFacade,
         ) = buildString {
-            insert(buildKotlin(context, declarationDescriptor, quickNavigation, ktElement)) {}
+            insert(buildKotlin(context, declarationDescriptor, quickNavigation, ktElement, resolutionFacade)) {}
         }
 
         private fun buildKotlin(
             context: BindingContext,
             declarationDescriptor: DeclarationDescriptor,
             quickNavigation: Boolean,
-            ktElement: KtElement
+            ktElement: KtElement,
+            resolutionFacade: ResolutionFacade,
         ): KDocTemplate {
             @Suppress("NAME_SHADOWING")
             var declarationDescriptor = declarationDescriptor
@@ -361,7 +368,7 @@ open class KotlinDocumentationProviderCompatBase : AbstractDocumentationProvider
                 }
             }
 
-            val deprecationProvider = ktElement.getResolutionFacade().frontendService<DeprecationResolver>()
+            val deprecationProvider = resolutionFacade.frontendService<DeprecationResolver>()
 
             return KDocTemplate().apply {
                 definition {

idea/src/org/jetbrains/kotlin/idea/inspections/OverridingDeprecatedMemberInspection.kt

@@ -35,9 +35,10 @@ class OverridingDeprecatedMemberInspection : AbstractKotlinInspection() {
             }
 
             private fun registerProblemIfNeeded(declaration: KtDeclaration, targetForProblem: PsiElement) {
-                val accessorDescriptor = declaration.resolveToDescriptorIfAny() as? CallableMemberDescriptor ?: return
+                val resolutionFacade = declaration.getResolutionFacade()
+                val accessorDescriptor = declaration.resolveToDescriptorIfAny(resolutionFacade) as? CallableMemberDescriptor ?: return
 
-                val deprecationProvider = declaration.getResolutionFacade().frontendService<DeprecationResolver>()
+                val deprecationProvider = resolutionFacade.frontendService<DeprecationResolver>()
 
                 val message = deprecationProvider.getDeprecations(accessorDescriptor)
                     .firstOrNull()