diff --git a/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/optimizations/Devirtualization.kt b/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/optimizations/Devirtualization.kt index 84c485bc092..05541c0adf3 100644 --- a/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/optimizations/Devirtualization.kt +++ b/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/optimizations/Devirtualization.kt @@ -14,6 +14,7 @@ import org.jetbrains.kotlin.backend.common.lower.createIrBuilder import org.jetbrains.kotlin.backend.common.lower.irBlock import org.jetbrains.kotlin.backend.konan.* import org.jetbrains.kotlin.backend.konan.util.IntArrayList +import org.jetbrains.kotlin.backend.konan.util.LongArrayList import org.jetbrains.kotlin.descriptors.ClassKind import org.jetbrains.kotlin.ir.IrElement import org.jetbrains.kotlin.ir.builders.* @@ -148,9 +149,6 @@ internal object Devirtualization { } sealed class Node(val id: Int) { - var directEdges: IntHashSet? = null - var reversedEdges: IntArrayList? = null - var directCastEdges: MutableList? = null var reversedCastEdges: MutableList? = null @@ -163,14 +161,6 @@ internal object Devirtualization { val multiNodeSize get() = multiNodeEnd - multiNodeStart - fun addEdge(node: Node) { - if (directEdges == null) directEdges = IntHashSet() - if (directEdges!!.add(node.id)) { - if (node.reversedEdges == null) node.reversedEdges = IntArrayList() - node.reversedEdges!!.add(id) - } - } - fun addCastEdge(edge: CastEdge) { if (directCastEdges == null) directCastEdges = ArrayList(1) directCastEdges!!.add(edge) @@ -251,6 +241,11 @@ internal object Devirtualization { } } + private inline fun IntArray.forEachEdge(v: Int, block: (Int) -> Unit) { + for (i in this[v] until this[v + 1]) + block(this[i]) + } + inner class TypeHierarchy(val allTypes: Array) { private val typesSubTypes = Array(allTypes.size) { mutableListOf() } private val allInheritors = Array(allTypes.size) { BitSet() } @@ -296,7 +291,7 @@ internal object Devirtualization { fun BitSet.copy() = BitSet(this.size()).apply { this.or(this@copy) } - fun printPathToType(node: Node, type: Int) { + fun printPathToType(reversedEdges: IntArray, node: Node, type: Int) { val nodes = constraintGraph.nodes val visited = BitSet() val prev = mutableMapOf() @@ -308,20 +303,21 @@ internal object Devirtualization { val prevFront = front front = mutableListOf() for (from in prevFront) { - val reversedEdges = from.reversedEdges - if (reversedEdges != null) - for (toId in reversedEdges) { - val to = nodes[toId] - if (!visited[toId] && to.types[type]) { - visited.set(toId) - prev[to] = from - front.add(to) - if (to is Node.Source) { - source = to - break@bfs - } + var endBfs = false + reversedEdges.forEachEdge(from.id) { toId -> + val to = nodes[toId] + if (!visited[toId] && to.types[type]) { + visited.set(toId) + prev[to] = from + front.add(to) + if (to is Node.Source) { + source = to + endBfs = true + return@forEachEdge } } + } + if (endBfs) break@bfs val reversedCastEdges = from.reversedCastEdges if (reversedCastEdges != null) for (castEdge in reversedCastEdges) { @@ -352,7 +348,7 @@ internal object Devirtualization { } } - private inner class CondensationBuilder { + private inner class CondensationBuilder(val directEdges: IntArray, val reversedEdges: IntArray) { val nodes = constraintGraph.nodes val nodesCount = nodes.size val visited = BitSet(nodesCount) @@ -401,7 +397,7 @@ internal object Devirtualization { private fun findOrder(node: Node, order: IntArray) { visited.set(node.id) - node.directEdges?.forEach { + directEdges.forEachEdge(node.id) { if (!visited[it]) findOrder(nodes[it], order) } @@ -411,7 +407,7 @@ internal object Devirtualization { private fun paint(node: Node) { visited.set(node.id) multiNodes[index++] = node.id - node.reversedEdges?.forEach { + reversedEdges.forEachEdge(node.id) { if (!visited[it]) paint(nodes[it]) } @@ -421,12 +417,11 @@ internal object Devirtualization { visited.set(multiNode.id) for (v in multiNode.multiNodeStart until multiNode.multiNodeEnd) { val node = nodes[multiNodes[v]] - node.directEdges?.forEach { + directEdges.forEachEdge(node.id) { val nextMultiNode = multiNodes[nodes[it].multiNodeStart] if (!visited[nextMultiNode]) findMultiNodesOrder(nodes[nextMultiNode], order) } - } order[index++] = multiNode.id } @@ -448,16 +443,14 @@ internal object Devirtualization { val rootSet = computeRootSet(context, moduleDFG, externalModulesDFG) val nodesMap = mutableMapOf() - val constraintGraphBuilder = - ConstraintGraphBuilder(nodesMap, functions, typeHierarchy, rootSet, true) - constraintGraphBuilder.build() - val instantiatingClasses = constraintGraphBuilder.instantiatingClasses + + val (instantiatingClasses, directEdges, reversedEdges) = buildConstraintGraph(nodesMap, functions, typeHierarchy, rootSet) DEBUG_OUTPUT(0) { println("FULL CONSTRAINT GRAPH") constraintGraph.nodes.forEach { println(" NODE #${it.id}") - it.directEdges?.forEach { + directEdges.forEachEdge(it.id) { println(" EDGE: #${it}z") } it.directCastEdges?.forEach { @@ -472,17 +465,17 @@ internal object Devirtualization { constraintGraph.nodes.forEach { if (it is Node.Source) { - assert(it.reversedEdges?.isEmpty() ?: true) { "A source node #${it.id} has incoming edges" } + assert(reversedEdges[it.id] == reversedEdges[it.id + 1]) { "A source node #${it.id} has incoming edges" } assert(it.reversedCastEdges?.isEmpty() ?: true) { "A source node #${it.id} has incoming edges" } } } DEBUG_OUTPUT(0) { println("CONSTRAINT GRAPH: ${constraintGraph.nodes.size} nodes, " + - "${constraintGraph.nodes.sumBy { (it.directEdges?.size ?: 0) + (it.directCastEdges?.size ?: 0) } } edges") + "${constraintGraph.nodes.sumBy { (directEdges[it.id + 1] - directEdges[it.id]) + (it.directCastEdges?.size ?: 0) } } edges") } - val condensation = CondensationBuilder().build() + val condensation = CondensationBuilder(directEdges, reversedEdges).build() val topologicalOrder = condensation.topologicalOrder.map { constraintGraph.nodes[it] } DEBUG_OUTPUT(0) { @@ -519,7 +512,9 @@ internal object Devirtualization { continue // A source has no incoming edges. val types = BitSet() condensation.forEachNode(multiNode) { node -> - node.reversedEdges?.forEach { types.or(constraintGraph.nodes[it].types) } + reversedEdges.forEachEdge(node.id) { + types.or(constraintGraph.nodes[it].types) + } node.reversedCastEdges ?.filter { it.node.priority < node.priority } // Doesn't contradict topological order. ?.forEach { @@ -573,7 +568,7 @@ internal object Devirtualization { for (i in 0 until prevFrontSize) { marked[prevFront[i]] = false val node = constraintGraph.nodes[prevFront[i]] - node.directEdges?.forEach { distNodeId -> + directEdges.forEachEdge(node.id) { distNodeId -> val distNode = constraintGraph.nodes[distNodeId] if (marked[distNode.id]) distNode.types.or(node.types) @@ -632,7 +627,7 @@ internal object Devirtualization { (virtualCall.irCallSite?.let { ir2stringWhole(it) } ?: virtualCall.callee.toString())) println(" receiver is Virtual") - printPathToType(receiverNode, VIRTUAL_TYPE_ID) + printPathToType(reversedEdges, receiverNode, VIRTUAL_TYPE_ID) } continue @@ -652,7 +647,7 @@ internal object Devirtualization { DEBUG_OUTPUT(0) { println("Path to type $type") - printPathToType(receiverNode, it) + printPathToType(reversedEdges, receiverNode, it) } possibleReceivers.add(type) @@ -703,6 +698,83 @@ internal object Devirtualization { return AnalysisResult(result.asSequence().associateBy({ it.key }, { it.value.first }), typeHierarchy) } + // Both [directEdges] and [reversedEdges] are the array representation of a graph: + // for each node v the edges of that node are stored in edges[edges[v] until edges[v + 1]]. + private data class ConstraintGraphBuildResult(val instantiatingClasses: BitSet, + val directEdges: IntArray, val reversedEdges: IntArray) + + // Here we're dividing the build process onto two phases: + // 1. build bag of edges and direct edges array; + // 2. build reversed edges array from the direct edges array. + // This is to lower memory usage (all of these edges structures are more or less equal by size), + // and by that we're only holding references to two out of three of them. + private fun buildConstraintGraph(nodesMap: MutableMap, + functions: Map, + typeHierarchy: TypeHierarchy, + rootSet: List + ): ConstraintGraphBuildResult { + val precursor = buildConstraintGraphPrecursor(nodesMap, functions, typeHierarchy, rootSet) + return ConstraintGraphBuildResult(precursor.instantiatingClasses, precursor.directEdges, + buildReversedEdges(precursor.directEdges, precursor.reversedEdgesCount)) + } + + private class ConstraintGraphPrecursor(val instantiatingClasses: BitSet, + val directEdges: IntArray, val reversedEdgesCount: IntArrayList) + + private fun buildReversedEdges(directEdges: IntArray, reversedEdgesCount: IntArrayList): IntArray { + val numberOfNodes = constraintGraph.nodes.size + var edgesArraySize = numberOfNodes + 1 + for (v in 0 until numberOfNodes) + edgesArraySize += reversedEdgesCount[v] + val reversedEdges = IntArray(edgesArraySize) + var index = numberOfNodes + 1 + for (v in 0..numberOfNodes) { + reversedEdges[v] = index + index += reversedEdgesCount[v] + reversedEdgesCount[v] = 0 + } + for (from in 0 until numberOfNodes) { + directEdges.forEachEdge(from) { to -> + reversedEdges[reversedEdges[to] + (reversedEdgesCount[to]++)] = from + } + } + return reversedEdges + } + + private fun buildConstraintGraphPrecursor(nodesMap: MutableMap, + functions: Map, + typeHierarchy: TypeHierarchy, + rootSet: List + ): ConstraintGraphPrecursor { + val constraintGraphBuilder = ConstraintGraphBuilder(nodesMap, functions, typeHierarchy, rootSet, true) + constraintGraphBuilder.build() + val bagOfEdges = constraintGraphBuilder.bagOfEdges + val directEdgesCount = constraintGraphBuilder.directEdgesCount + val reversedEdgesCount = constraintGraphBuilder.reversedEdgesCount + val numberOfNodes = constraintGraph.nodes.size + // numberOfNodes + 1 for convenience. + directEdgesCount.reserve(numberOfNodes + 1) + reversedEdgesCount.reserve(numberOfNodes + 1) + var edgesArraySize = numberOfNodes + 1 + for (v in 0 until numberOfNodes) + edgesArraySize += directEdgesCount[v] + val directEdges = IntArray(edgesArraySize) + var index = numberOfNodes + 1 + for (v in 0..numberOfNodes) { + directEdges[v] = index + index += directEdgesCount[v] + directEdgesCount[v] = 0 + } + for (bucket in bagOfEdges) + if (bucket != null) + for (edge in bucket) { + val from = edge.toInt() + val to = (edge shr 32).toInt() + directEdges[directEdges[from] + (directEdgesCount[from]++)] = to + } + return ConstraintGraphPrecursor(constraintGraphBuilder.instantiatingClasses, directEdges, reversedEdgesCount) + } + private class ConstraintGraphVirtualCall(val caller: Function, val virtualCall: DataFlowIR.Node.VirtualCall, val arguments: List, val returnsNode: Node) @@ -720,6 +792,62 @@ internal object Devirtualization { private val virtualTypeFilter = BitSet().apply { set(VIRTUAL_TYPE_ID) } val instantiatingClasses = BitSet() + private val preliminaryNumberOfNodes = + allTypes.size + // A possible source node for each type. + functions.size * 2 + // and nodes for each function. + functions.values.sumBy { it.body.nodes.size } + // A node for each DataFlowIR.Node. + functions.values + .sumBy { function -> + function.body.nodes.count { node -> + // A cast if types are different. + node is DataFlowIR.Node.Call + && node.returnType.resolved() != node.callee.returnParameter.type.resolved() + } + } + + private fun isPrime(x: Int): Boolean { + if (x <= 3) return true + if (x % 2 == 0) return false + var r = 3 + while (r * r <= x) { + if (x % r == 0) return false + r += 2 + } + return true + } + + private fun makePrime(x: Int): Int { + var x = x + while (true) { + if (isPrime(x)) return x + ++x + } + } + + // A heuristic: the number of edges in the data flow graph + // for any reasonable program is linear in number of nodes. + val bagOfEdges = arrayOfNulls(makePrime(preliminaryNumberOfNodes * 5)) + val directEdgesCount = IntArrayList() + val reversedEdgesCount = IntArrayList() + + private fun addEdge(from: Node, to: Node) { + val fromId = from.id + val toId = to.id + val value = fromId.toLong() or (toId.toLong() shl 32) + // This is 64-bit extension of a hashing method from Knuth's "The Art of Computer Programming". + // The magic constant is the closest prime to 2^64 * phi, where phi is the golden ratio. + val bucketIdx = ((value.toULong() * 11400714819323198393UL) % bagOfEdges.size.toUInt()).toInt() + val bucket = bagOfEdges[bucketIdx] ?: LongArrayList().also { bagOfEdges[bucketIdx] = it } + for (x in bucket) + if (x == value) return + bucket.add(value) + + directEdgesCount.reserve(fromId + 1) + directEdgesCount[fromId]++ + reversedEdgesCount.reserve(toId + 1) + reversedEdgesCount[toId]++ + } + private fun concreteType(type: DataFlowIR.Type.Declared): Int { assert(!(type.isAbstract && type.isFinal)) { "Incorrect type: $type" } return if (type.isAbstract) @@ -749,9 +877,9 @@ internal object Devirtualization { val fieldType = field.type.resolved() // Some user of our library might place some value into the field. if (fieldType.isFinal) - concreteClass(fieldType).addEdge(fieldNode) + addEdge(concreteClass(fieldType), fieldNode) else - constraintGraph.virtualNode.addEdge(fieldNode) + addEdge(constraintGraph.virtualNode, fieldNode) } fieldNode } @@ -777,9 +905,8 @@ internal object Devirtualization { } else { // String arguments are implicitly put into the array parameter of
. addInstantiatingClass(symbolTable.mapType(context.irBuiltIns.stringType).resolved()) - concreteClass(symbolTable.mapType(context.irBuiltIns.stringType).resolved()).addEdge( - fieldNode(constraintGraph.arrayItemField) - ) + addEdge(concreteClass(symbolTable.mapType(context.irBuiltIns.stringType).resolved()), + fieldNode(constraintGraph.arrayItemField)) // Conservatively assume each associated object could be instantiated. context.irModule!!.acceptChildrenVoid(object: IrElementVisitorVoid { override fun visitElement(element: IrElement) { @@ -803,8 +930,8 @@ internal object Devirtualization { val functionConstraintGraph = constraintGraph.functions[symbol]!! body.nodes.forEach { dfgNodeToConstraintNode(functionConstraintGraph, it) } - functionNodesMap[body.returns]!!.addEdge(functionConstraintGraph.returns) - functionNodesMap[body.throws]!!.addEdge(functionConstraintGraph.throws) + addEdge(functionNodesMap[body.returns]!!, functionConstraintGraph.returns) + addEdge(functionNodesMap[body.throws]!!, functionConstraintGraph.throws) DEBUG_OUTPUT(0) { println("CONSTRAINT GRAPH FOR $symbol") @@ -830,10 +957,14 @@ internal object Devirtualization { for (list in typesVirtualCallSites) for (virtualCall in list) { val returnType = virtualCall.virtualCall.returnType.resolved() - if (returnType.isFinal && (virtualCall.returnsNode.reversedEdges?.size ?: 0) == 0) { + val totalIncomingEdgesToReturnsNode = + if (virtualCall.returnsNode.id >= reversedEdgesCount.size) + 0 + else reversedEdgesCount[virtualCall.returnsNode.id] + if (returnType.isFinal && totalIncomingEdgesToReturnsNode == 0) { // If we are in a library and facing final return type with no possible callees - // this type still can be returned by some user of this library, so propagate it explicitly. - concreteClass(returnType).addEdge(virtualCall.returnsNode) + addEdge(concreteClass(returnType), virtualCall.returnsNode) } } } @@ -852,7 +983,7 @@ internal object Devirtualization { constraintGraph.virtualNode // TODO: OBJC-INTEROP-GENERATED-CLASSES else concreteClass(resolvedType) - node.addEdge(parameters[index]) + addEdge(node, parameters[index]) } } @@ -884,8 +1015,8 @@ internal object Devirtualization { val callee = receiverType.calleeAt(virtualCall.virtualCall) - doCall(virtualCall.caller, callee, virtualCall.arguments, - callee.returnParameter.type.resolved()).addEdge(virtualCall.returnsNode) + addEdge(doCall(virtualCall.caller, callee, virtualCall.arguments, + callee.returnParameter.type.resolved()), virtualCall.returnsNode) } private fun checkSupertypes(type: DataFlowIR.Type.Declared, @@ -953,7 +1084,7 @@ internal object Devirtualization { " provided ${arguments.size}" } callee.parameters.forEachIndexed { index, parameter -> - arguments[index].addEdge(parameter) + addEdge(arguments[index], parameter) } return castIfNeeded(caller, callee.returns, callee.symbol.returnParameter.type.resolved(), returnType) } @@ -967,9 +1098,9 @@ internal object Devirtualization { val fictitiousReturnNode = ordinaryNode { "External$resolvedCallee" } if (returnType.isFinal) { addInstantiatingClass(returnType) - concreteClass(returnType).addEdge(fictitiousReturnNode) + addEdge(concreteClass(returnType), fictitiousReturnNode) } else { - constraintGraph.virtualNode.addEdge(fictitiousReturnNode) + addEdge(constraintGraph.virtualNode, fictitiousReturnNode) // TODO: Unconservative way - when we can use it? // TODO: OBJC-INTEROP-GENERATED-CLASSES // typeHierarchy.inheritorsOf(returnType) @@ -980,7 +1111,7 @@ internal object Devirtualization { fictitiousReturnNode } } else { - calleeConstraintGraph.throws.addEdge(caller.throws) + addEdge(calleeConstraintGraph.throws, caller.throws) doCall(caller, calleeConstraintGraph, arguments, returnType) } } @@ -1014,7 +1145,7 @@ internal object Devirtualization { value else doCast(function, value, actualType) - castedValue.addEdge(fieldNode) + addEdge(castedValue, fieldNode) } if (node is DataFlowIR.Node.Variable && node.kind != DataFlowIR.VariableKind.Temporary) { @@ -1023,7 +1154,7 @@ internal object Devirtualization { variableNode = ordinaryNode { "Variable\$${function.symbol}" } variables[node] = variableNode for (value in node.values) { - edgeToConstraintNode(value).addEdge(variableNode) + addEdge(edgeToConstraintNode(value), variableNode) } if (node.kind == DataFlowIR.VariableKind.CatchParameter) function.throws.addCastEdge(createCastEdge(variableNode, node.type.resolved())) @@ -1081,7 +1212,7 @@ internal object Devirtualization { val arguments = node.arguments.map(::edgeToConstraintNode) val receiverNode = arguments[0] if (receiverType == DataFlowIR.Type.Virtual) - constraintGraph.virtualNode.addEdge(receiverNode) + addEdge(constraintGraph.virtualNode, receiverNode) if (entryPoint == null && returnType.isFinal) { // If we are in a library and facing final return type then @@ -1095,7 +1226,7 @@ internal object Devirtualization { ConstraintGraphVirtualCall(function, node, arguments, returnsNode)) forEachBitInBoth(typeHierarchy.inheritorsOf(receiverType), instantiatingClasses) { val actualCallee = allTypes[it].calleeAt(node) - doCall(actualCallee, arguments, actualCallee.returnParameter.type.resolved()).addEdge(returnsNode) + addEdge(doCall(actualCallee, arguments, actualCallee.returnParameter.type.resolved()), returnsNode) } // Add cast to [Virtual] edge from receiver to returns, if return type is not final. // With this we're reflecting the fact that unknown function can return anything. @@ -1153,7 +1284,7 @@ internal object Devirtualization { is DataFlowIR.Node.Variable -> node.values.map { edgeToConstraintNode(it) }.let { values -> ordinaryNode { "TempVar\$${function.symbol}" }.also { node -> - values.forEach { it.addEdge(node) } + values.forEach { addEdge(it, node) } } } diff --git a/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/util/PrimitiveLists.kt b/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/util/PrimitiveLists.kt index f49bf6fae8c..09c87411b88 100644 --- a/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/util/PrimitiveLists.kt +++ b/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/util/PrimitiveLists.kt @@ -54,6 +54,53 @@ class IntArrayList : Iterable { override fun hasNext() = index < size + override fun next() = get(index++) + } +} + +class LongArrayList : Iterable { + private var array = LongArray(3) + private var length = 0 + + val size get() = length + + fun isEmpty() = size == 0 + + fun add(x: Long) { + length++ + ensureCapacity(length) + set(length - 1, x) + } + + fun get(index: Int): Long = when { + index < 0 || index >= length -> throw IndexOutOfBoundsException("Index out of range: $index") + + else -> array[index] + } + + fun set(index: Int, x: Long) = when { + index < 0 || index >= length -> throw IndexOutOfBoundsException("Index out of range: $index") + + else -> array[index] = x + } + + override operator fun iterator(): Iterator = Itr() + + private fun ensureCapacity(minCapacity: Int) { + val oldArray = array + if (minCapacity > oldArray.size) { + var newSize = oldArray.size * 3 / 2 + if (minCapacity > newSize) + newSize = minCapacity + array = oldArray.copyOf(newSize) + } + } + + private inner class Itr : Iterator { + private var index = 0 + + override fun hasNext() = index < size + override fun next() = get(index++) } } \ No newline at end of file