mirror of
https://github.com/apple/swift.git
synced 2025-12-21 12:14:44 +01:00
0f88e0f3cc
Fix innumerable latent bugs with iterator invalidation and callback invocation. Removes dead code earlier and chips away at all the redundant copies the compiler generates.
650 lines
27 KiB
C++
650 lines
27 KiB
C++
//===--- ForEachLoopUnrolling.cpp - Unroll loops over array literals ----- ===//
|
|
//
|
|
// This source file is part of the Swift.org open source project
|
|
//
|
|
// Copyright (c) 2014 - 2020 Apple Inc. and the Swift project authors
|
|
// Licensed under Apache License v2.0 with Runtime Library Exception
|
|
//
|
|
// See https://swift.org/LICENSE.txt for license information
|
|
// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors
|
|
//
|
|
//===----------------------------------------------------------------------===//
|
|
|
|
// This pass unrolls Sequence.forEach calls invoked on an array created
|
|
// from an array literal. See below for an overview of the algorithm. This is
|
|
// a function-level transformation pass that operates on ownership SIL and
|
|
// so must be applied before ownership is stripped.
|
|
//
|
|
// Algorithm overview:
|
|
//
|
|
// 1. Iterate over the body of the function and analyze calls to the array
|
|
// initializer annotated "array.uninitialized_intrinsic". This is done in
|
|
// `run` method.
|
|
//
|
|
// 2. For every "array.uninintialized_intrinsic" initializer call, try
|
|
// extracting the elements with which the array is initialized, and classify
|
|
// the uses of the array into incidental uses, forEach calls and uses that
|
|
// can write into the array. If any of the following conditions hold, give
|
|
// up and look for the next initializer call:
|
|
// - The elements stored into the array cannot be extracted.
|
|
// - The array can be modified after initialization.
|
|
// - There are no forEach calls on the array.
|
|
// - The array is too large to unroll forEach calls. This check uses
|
|
// SILModule::UnrollThreshold parameter.
|
|
// If none of the above conditions hold, procede to unrolling every forEach
|
|
// call on the array (Step 3). Step 2 is implemented in the function:
|
|
// `tryUnrollForEachCallsOverArrayLiteral`.
|
|
//
|
|
// 3. Given a forEach call, unroll it by applying the "body closure" passed to
|
|
// the call to every element of the array. There are three important
|
|
// details to this unrolling. Firstly, the SILValue of the element stored
|
|
// into the array through the store instruction (identified in step 2) may
|
|
// not be valid at the point where the unrolling must happen (i.e., where the
|
|
// forEach is called). Therefore, we need to copy_value the elements at the
|
|
// point where they are stored and ensure that the copies are alive until
|
|
// they are used by the unrolled code. The implementation actually
|
|
// makes the copies valid for the entire lifetime of the array.
|
|
//
|
|
// Secondly, the body closure uses @in_guaranteed convention for the
|
|
// parameter. Therefore, an alloc_stack is created before the unrolled code
|
|
// begins to hold the elements, and is destoryed once the unrolled code ends.
|
|
//
|
|
// Thirdly, the body closure throws. Hence, it has to be try_applied. This
|
|
// means that we need to chain the try_applies in such a way that when the
|
|
// try_apply on the element e_i completes normally it jumps to the try_apply
|
|
// of e_i+1. When the try_apply (of any element) throws, it must go to the
|
|
// error block of the original forEach call.
|
|
//
|
|
// All of this is implemented by the function `unrollForEach`.
|
|
//
|
|
// 4. Delete the forEach calls that were unrolled and clean up dead code
|
|
// resulting due to that.
|
|
//
|
|
// This transformation is illustrated below on a small example:
|
|
//
|
|
// Input:
|
|
// %initResult = apply %arrayUninitialized(...)
|
|
// (%array, %storage_ptr) = destructure_tuple %initResult
|
|
// store %elem1 at array index 1
|
|
// store %elem2 at array index 2
|
|
// ..
|
|
// try_apply %forEach(%body, %array) normal bb1, error bb2
|
|
// bb1(%res : $()):
|
|
// ..
|
|
// bb2(%error : @owned $Error):
|
|
// ...
|
|
// bb3:
|
|
// destroy_value %array
|
|
//
|
|
// Output:
|
|
//
|
|
// %initResult = apply %arrayUninitialized(...)
|
|
// (%array, %storage_ptr) = destructure_tuple %initResult
|
|
// %elem1copy = copy_value %elem1 <--
|
|
// store %elem1 at array index 1
|
|
// %elem2copy = copy_value %elem2 <--
|
|
// store %elem2 at array index 2
|
|
// ..
|
|
// alloc_stack %stack
|
|
// %elem1borrow = begin_borrow %elem1copy
|
|
// store_borrow %elem1borrow to %stack
|
|
// try_apply %body(%stack) normal normalbb1, error errbb1
|
|
//
|
|
// errbb1(%error : @owned $Error):
|
|
// end_borrow %elem1borrow
|
|
// br bb2(%error)
|
|
//
|
|
// normalbb1(%res : $()):
|
|
// end_borrow %elem1borrow
|
|
// %elem2borrow = begin_borrow %elem2copy
|
|
// store_borrow %elem2borrow to %stack
|
|
// try_apply %body(%stack) normal bb1, error errbb2
|
|
//
|
|
// errbb2(%error : @owned $Error):
|
|
// end_borrow %elem2borrow
|
|
// br bb2(%error)
|
|
//
|
|
// bb1(%res : $()):
|
|
// end_borrow %elem2borrow
|
|
// dealloc_stack %stack
|
|
// ...
|
|
// bb2(%error : @owned $Error):
|
|
// ...
|
|
// dealloc_stack %stack
|
|
// bb3:
|
|
// destroy_value %elem1copy
|
|
// destroy_value %elem2copy
|
|
// destroy_value %array
|
|
|
|
#include "swift/AST/ASTContext.h"
|
|
#include "swift/AST/Expr.h"
|
|
#include "swift/AST/Module.h"
|
|
#include "swift/AST/SemanticAttrs.h"
|
|
#include "swift/AST/SubstitutionMap.h"
|
|
#include "swift/Basic/OptimizationMode.h"
|
|
#include "swift/SIL/BasicBlockUtils.h"
|
|
#include "swift/SIL/CFG.h"
|
|
#include "swift/SIL/DebugUtils.h"
|
|
#include "swift/SIL/InstructionUtils.h"
|
|
#include "swift/SIL/OwnershipUtils.h"
|
|
#include "swift/SIL/SILBasicBlock.h"
|
|
#include "swift/SIL/SILBuilder.h"
|
|
#include "swift/SIL/SILConstants.h"
|
|
#include "swift/SIL/SILFunction.h"
|
|
#include "swift/SIL/SILInstruction.h"
|
|
#include "swift/SIL/SILModule.h"
|
|
#include "swift/SIL/TypeLowering.h"
|
|
#include "swift/SILOptimizer/Analysis/ArraySemantic.h"
|
|
#include "swift/SILOptimizer/PassManager/Passes.h"
|
|
#include "swift/SILOptimizer/PassManager/Transforms.h"
|
|
#include "swift/SILOptimizer/Utils/CFGOptUtils.h"
|
|
#include "swift/SILOptimizer/Utils/InstOptUtils.h"
|
|
#include "swift/SILOptimizer/Utils/ValueLifetime.h"
|
|
#include "llvm/ADT/DenseMap.h"
|
|
#include "llvm/ADT/SmallVector.h"
|
|
|
|
using namespace swift;
|
|
|
|
namespace {
|
|
|
|
/// A class for processing and storing information about an array
|
|
/// such as the values with which it is initialized and the kind of
|
|
/// users it has.
|
|
class ArrayInfo {
|
|
/// The array value returned by the _allocateUninitialized call.
|
|
SILValue arrayValue;
|
|
|
|
/// A map from an array index to the store instruction that initializes that
|
|
/// index.
|
|
llvm::DenseMap<uint64_t, StoreInst *> elementStoreMap;
|
|
|
|
/// List of Sequence.forEach calls invoked on the array.
|
|
SmallSetVector<TryApplyInst *, 4> forEachCalls;
|
|
|
|
/// Indicates whether the array could be modified after initialization. Note
|
|
/// that this not include modifications to the elements of the array. When
|
|
/// set, this will prevent the forEach loops from getting unrolled.
|
|
bool mayBeWritten = false;
|
|
|
|
/// Instructions that destroy the array. These must be destroy_value
|
|
/// instructions either of the \c arrayValue or the copy_values of that.
|
|
SmallVector<SILInstruction *, 4> destroys;
|
|
|
|
/// Classify uses of the array into forEach uses, read-only uses etc. and set
|
|
/// the fields of this instance appropriately. This function will recursively
|
|
/// classify the uses of borrows and copy-values of the array as well.
|
|
void classifyUsesOfArray(SILValue arrayValue);
|
|
|
|
public:
|
|
ArrayInfo() {}
|
|
|
|
/// Given an apply instruction \c apply, try to initialize this ArrayInfo
|
|
/// with it. This would succeed iff the apply instruction starts an
|
|
/// initialization pattern that is auto-generated by the compiler for array
|
|
/// literals. Return true on success and false on failure.
|
|
bool tryInitialize(ApplyInst *apply);
|
|
|
|
/// Return the SILValue of the array that is initialized.
|
|
SILValue getArrayValue() {
|
|
assert(arrayValue);
|
|
return arrayValue;
|
|
}
|
|
|
|
/// Return true iff the array could be modified after initialization.
|
|
bool mayBeModified() {
|
|
assert(arrayValue);
|
|
return mayBeWritten;
|
|
}
|
|
|
|
/// Return the forEach uses identified for the array.
|
|
ArrayRef<TryApplyInst *> getForEachUses() {
|
|
assert(arrayValue);
|
|
return ArrayRef<TryApplyInst *>(forEachCalls.begin(), forEachCalls.end());
|
|
}
|
|
|
|
/// Return the number of elements in the array.
|
|
uint64_t getNumElements() {
|
|
assert(arrayValue);
|
|
return elementStoreMap.size();
|
|
}
|
|
|
|
/// Return the store instruction that initializes the given \c index
|
|
/// of the array.
|
|
StoreInst *getElementStore(uint64_t index) {
|
|
assert(arrayValue);
|
|
return elementStoreMap[index];
|
|
}
|
|
|
|
/// Return the SIL type of the elements stored in the array.
|
|
/// \pre the array must be non-empty.
|
|
SILType getElementSILType() {
|
|
assert(getNumElements() > 0 && "cannot call this on empty arrays");
|
|
return elementStoreMap[0]->getSrc()->getType();
|
|
}
|
|
|
|
/// Add the destroy_value instructions that represents the last use of the
|
|
/// array to the parameter \c lastDestroys. The \c lastDestroys
|
|
/// added by this function are guaranteed to come after all uses of the
|
|
/// \c arrayValue and copy_values of the \c arrayValue (along all
|
|
/// "non-dead-end" blocks).
|
|
void getLastDestroys(SmallVectorImpl<DestroyValueInst *> &lastDestroys);
|
|
};
|
|
|
|
/// Given a Array-typed SIL value \c array and an instruction \c user that uses
|
|
/// the array, check whether this use actually represents a call to _fixLifetime
|
|
/// function. This would be the case if \c user is a store_borrow instruction
|
|
/// that stores into an alloc_stack which is passed to a fixLifetime
|
|
/// instruction. That is, if the following pattern holds:
|
|
///
|
|
/// %stack = alloc_stack
|
|
/// user: store_borrow %array to %stack
|
|
/// fixLifetime %stack
|
|
/// \returns the fixLifetime instruction if this is a fixLifetime use of the
|
|
/// array, nullptr otherwise.
|
|
static FixLifetimeInst *isFixLifetimeUseOfArray(SILInstruction *user,
|
|
SILValue array) {
|
|
StoreBorrowInst *storeUser = dyn_cast<StoreBorrowInst>(user);
|
|
if (!storeUser || storeUser->getSrc() != array)
|
|
return nullptr;
|
|
AllocStackInst *alloc = dyn_cast<AllocStackInst>(storeUser->getDest());
|
|
if (!alloc)
|
|
return nullptr;
|
|
auto fixLifetimeUsers = alloc->getUsersOfType<FixLifetimeInst>();
|
|
if (fixLifetimeUsers.empty())
|
|
return nullptr;
|
|
auto firstUser = fixLifetimeUsers.begin();
|
|
FixLifetimeInst *result = *firstUser;
|
|
// We need to have a unique result.
|
|
if (++firstUser != fixLifetimeUsers.end())
|
|
return nullptr;
|
|
return result;
|
|
}
|
|
|
|
/// Given an array-typed SIL value \c array and an instruction \c user that uses
|
|
/// the array, check whether this use actually represents a call to the
|
|
/// Sequence.forEach function. This would be case if \c user is a store_borrow
|
|
/// instruction that stores into an alloc_stack which is passed to a try-apply
|
|
/// of the Sequence.forEach function. That is, if the following pattern holds:
|
|
///
|
|
/// %stack = alloc_stack
|
|
/// user: store_borrow %array to %stack
|
|
/// try_apply %forEachCall(%closure, %array)
|
|
/// \returns the try-apply instruction invoking the forEach function if this is
|
|
/// a forEach use of the array, nullptr otherwise.
|
|
static TryApplyInst *isForEachUseOfArray(SILInstruction *user, SILValue array) {
|
|
StoreBorrowInst *storeUser = dyn_cast<StoreBorrowInst>(user);
|
|
if (!storeUser || storeUser->getSrc() != array)
|
|
return nullptr;
|
|
AllocStackInst *alloc = dyn_cast<AllocStackInst>(storeUser->getDest());
|
|
if (!alloc)
|
|
return nullptr;
|
|
auto applyUsers = alloc->getUsersOfType<TryApplyInst>();
|
|
if (applyUsers.empty())
|
|
return nullptr;
|
|
auto firstUser = applyUsers.begin();
|
|
TryApplyInst *apply = *firstUser;
|
|
// We need to have a unique result.
|
|
if (++firstUser != applyUsers.end())
|
|
return nullptr;
|
|
SILFunction *callee = apply->getCalleeFunction();
|
|
if (!callee || !callee->hasSemanticsAttr(semantics::SEQUENCE_FOR_EACH))
|
|
return nullptr;
|
|
return apply;
|
|
}
|
|
|
|
void ArrayInfo::classifyUsesOfArray(SILValue arrayValue) {
|
|
for (Operand *operand : arrayValue->getUses()) {
|
|
auto *user = operand->getUser();
|
|
if (isIncidentalUse(user))
|
|
continue;
|
|
// Ignore this user if it is a call to _fixLifetime. Note that this use
|
|
// will not be subsumed by InstructionUtils::isIncidentalUse check made
|
|
// above as the array would be passed indirectly.
|
|
if (isFixLifetimeUseOfArray(user, arrayValue))
|
|
continue;
|
|
// Check if this is a forEach call on the array.
|
|
if (TryApplyInst *forEachCall = isForEachUseOfArray(user, arrayValue)) {
|
|
forEachCalls.insert(forEachCall);
|
|
continue;
|
|
}
|
|
// Recursively classify begin_borrow and copy_value uses.
|
|
if (BeginBorrowInst *beginBorrow = dyn_cast<BeginBorrowInst>(user)) {
|
|
classifyUsesOfArray(beginBorrow);
|
|
continue;
|
|
}
|
|
if (CopyValueInst *copyValue = dyn_cast<CopyValueInst>(user)) {
|
|
classifyUsesOfArray(copyValue);
|
|
continue;
|
|
}
|
|
if (DestroyValueInst *destroyValue = dyn_cast<DestroyValueInst>(user)) {
|
|
destroys.push_back(destroyValue);
|
|
continue;
|
|
}
|
|
// Set mayBeWritten to true if the user could potentially modify the array.
|
|
// Note that the array elements are allowed to be modified as long
|
|
// as the array itself is not modified (which is possible with reference
|
|
// types).
|
|
ArraySemanticsCall arrayOp(user);
|
|
if (arrayOp.doesNotChangeArray())
|
|
continue;
|
|
|
|
if (arrayOp.getKind() == swift::ArrayCallKind::kArrayFinalizeIntrinsic) {
|
|
classifyUsesOfArray((ApplyInst *)arrayOp);
|
|
continue;
|
|
}
|
|
|
|
mayBeWritten = true;
|
|
}
|
|
}
|
|
|
|
bool ArrayInfo::tryInitialize(ApplyInst *apply) {
|
|
ArraySemanticsCall arrayAllocateUninitCall(
|
|
apply, semantics::ARRAY_UNINITIALIZED_INTRINSIC);
|
|
if (!arrayAllocateUninitCall)
|
|
return false;
|
|
arrayValue = arrayAllocateUninitCall.getArrayValue();
|
|
if (!arrayValue)
|
|
return false;
|
|
if (!arrayAllocateUninitCall.mapInitializationStores(elementStoreMap))
|
|
return false;
|
|
// Collect information about uses of the array value.
|
|
classifyUsesOfArray(arrayValue);
|
|
return true;
|
|
}
|
|
|
|
void ArrayInfo::getLastDestroys(
|
|
SmallVectorImpl<DestroyValueInst *> &lastDestroys) {
|
|
assert(arrayValue);
|
|
// Collect the frontier instructions of the field \c destroys, which stores
|
|
// destroy_value instructions of the \c arrayValue as well as of its
|
|
// copy_values, using ValueLifetimeAnalysis. The frontier is a list of
|
|
// instructions that mark the exits of the flow of control from the
|
|
// \c destroys.
|
|
ValueLifetimeAnalysis lifetimeAnalysis(arrayValue->getDefiningInstruction(),
|
|
destroys);
|
|
ValueLifetimeAnalysis::Frontier frontier;
|
|
lifetimeAnalysis.computeFrontier(frontier,
|
|
ValueLifetimeAnalysis::DontModifyCFG);
|
|
for (SILInstruction *frontierInst : frontier) {
|
|
// Skip frontier instructions at the start of a basic block as they do not
|
|
// follow a destroy_value of the array. Note that the goal is to collect
|
|
// the last destroys, which must always immediately preceed a frontier
|
|
// instruction as it marks the end of the use of the array.
|
|
if (frontierInst == &frontierInst->getParent()->front())
|
|
continue;
|
|
SILInstruction *inst = &*(--frontierInst->getIterator());
|
|
// This must be a destroy instruction. Moreover it must also belong to \c
|
|
// destroys.
|
|
DestroyValueInst *lastDestroy = dyn_cast<DestroyValueInst>(inst);
|
|
assert(lastDestroy);
|
|
lastDestroys.push_back(cast<DestroyValueInst>(lastDestroy));
|
|
}
|
|
}
|
|
|
|
/// Delete the forEach calls from the SIL that contains it. This function will
|
|
/// not clean up any resulting dead instructions.
|
|
static void removeForEachCall(TryApplyInst *forEachCall,
|
|
InstructionDeleter &deleter) {
|
|
AllocStackInst *allocStack =
|
|
dyn_cast<AllocStackInst>(forEachCall->getArgument(1));
|
|
assert(allocStack);
|
|
// The allocStack will be used in the forEach call and also in a store
|
|
// instruction and a dealloc_stack instruction. Force delete all of them.
|
|
deleter.recursivelyForceDeleteUsersAndFixLifetimes(allocStack);
|
|
}
|
|
|
|
/// Unroll the \c forEachCall on an array, using the information in
|
|
/// \c ArrayInfo. Once unrolled, \c forEachCall will be deleted
|
|
/// from the containing function. This function implements the following
|
|
/// transformation.
|
|
/// - If the array stores non-trivial elements, for every element stored
|
|
/// into the array, insert a copy_value of the element just before it
|
|
/// is stored. This is necessary so that an @owned SILValue for the
|
|
/// element is available at the point where the forEach is used.
|
|
///
|
|
/// - Create an alloc_stack A at the point of the forEach call. This is
|
|
/// necessary to indirectly pass the elements of the array.
|
|
///
|
|
/// For every element e_i at the index i of the array, do the following:
|
|
/// - create a new basic block b_i if i > 0. Let b_0 denote the basic block
|
|
/// that contains the forEach call.
|
|
/// - store_borrow the element e_i into the alloc_stack A. Note that we
|
|
/// can use the owned copy of e_i created in the previous step.
|
|
/// - try_apply the forEach's body closure on the the alloc_stack A.
|
|
/// If i is not the last index, jump to b_i+1 in the normal case of the
|
|
/// try_apply. If i is the last index of the array jump to the normal
|
|
/// target of the forEach call. Jump to a new error block: err_i in the
|
|
/// error case. Make err_i jump to the error target of the original
|
|
/// forEach call.
|
|
///
|
|
/// - Dealloc the alloc_stack along the normal and error targets of the
|
|
/// forEach calls.
|
|
///
|
|
/// - Destroy all the copies of the elements (if it is non-trivial) just
|
|
/// before the array's lifetime ends.
|
|
static void unrollForEach(ArrayInfo &arrayInfo, TryApplyInst *forEachCall,
|
|
InstructionDeleter &deleter) {
|
|
if (arrayInfo.getNumElements() == 0) {
|
|
// If this is an empty array, delete the forEach entirely.
|
|
removeForEachCall(forEachCall, deleter);
|
|
return;
|
|
}
|
|
|
|
SILFunction *fun = forEachCall->getFunction();
|
|
SILLocation forEachLoc = forEachCall->getLoc();
|
|
SILValue forEachBodyClosure = forEachCall->getArgument(0);
|
|
SILType arrayElementType = arrayInfo.getElementSILType();
|
|
|
|
SILFunctionType *bodyClosureType =
|
|
forEachBodyClosure->getType().getAs<SILFunctionType>();
|
|
SILParameterInfo bodyParameterInfo = bodyClosureType->getParameters()[0];
|
|
// The forEachBodyClosure must use @in_guaranteed convention for passing
|
|
// arguments.
|
|
assert(bodyParameterInfo.getConvention() ==
|
|
ParameterConvention::Indirect_In_Guaranteed &&
|
|
"forEach body closure is expected to take @in_guaranteed argument");
|
|
|
|
// Copy the elements stored into the array. This is necessary as we need to
|
|
// extend the lifetime of the stored elements at least until the forEach call,
|
|
// which will now be unrolled. The following code inserts a copy_value of the
|
|
// elements just before the point where they are stored into the array, and
|
|
// a corresponding destroy_value at the end of the lifetime of the array. In
|
|
// other words, the lifetime of the element copies are made to match the
|
|
// lifetime of the array. Even though copies can be destroyed sooner after
|
|
// they are used by the unrolled code, doing so may introduce more destroys
|
|
// than needed as the unrolled code have many branches (due to try applies)
|
|
// all of which joins later into a single path eventually.
|
|
SmallVector<SILValue, 4> elementCopies;
|
|
for (uint64_t i = 0; i < arrayInfo.getNumElements(); ++i) {
|
|
StoreInst *elementStore = arrayInfo.getElementStore(i);
|
|
// Insert the copy just before the store of the element into the array.
|
|
SILValue copy = SILBuilderWithScope(elementStore)
|
|
.emitCopyValueOperation(elementStore->getLoc(),
|
|
elementStore->getSrc());
|
|
elementCopies.push_back(copy);
|
|
}
|
|
|
|
// Destroy the copy_value of the elements before the last destroys of the
|
|
// array. It is important that the copied elements are destroyed before the
|
|
// array is destroyed. This enables other optimizations.
|
|
SmallVector<DestroyValueInst *, 4> lastDestroys;
|
|
arrayInfo.getLastDestroys(lastDestroys);
|
|
for (DestroyValueInst *destroy : lastDestroys) {
|
|
SILBuilderWithScope destroyBuilder(destroy);
|
|
for (SILValue element : elementCopies) {
|
|
destroyBuilder.emitDestroyValueOperation(destroy->getLoc(), element);
|
|
}
|
|
}
|
|
|
|
// Create alloc_stack for passing the array elements indirectly.
|
|
SILValue allocStack = SILBuilderWithScope(forEachCall)
|
|
.createAllocStack(forEachLoc, arrayElementType);
|
|
|
|
// Extract the Error and normal targets of the forEach call. Both these
|
|
// targets must be taking a phi argument.
|
|
SILBasicBlock *normalBB = forEachCall->getNormalBB();
|
|
SILBasicBlock *errorBB = forEachCall->getErrorBB();
|
|
assert(errorBB->getSILPhiArguments().size() == 1 &&
|
|
normalBB->getSILPhiArguments().size() == 1);
|
|
SILPhiArgument *normalArgument = normalBB->getSILPhiArguments()[0];
|
|
SILPhiArgument *errorArgument = errorBB->getSILPhiArguments()[0];
|
|
|
|
// A generator for creating a basic block for use as the target of the
|
|
// "normal" branch of a try_apply.
|
|
auto normalTargetGenerator = [&](SILBasicBlock *insertionBlock) {
|
|
SILBasicBlock *newBB = fun->createBasicBlockBefore(insertionBlock);
|
|
newBB->createPhiArgument(normalArgument->getType(),
|
|
normalArgument->getOwnershipKind());
|
|
return newBB;
|
|
};
|
|
|
|
// A generator for creating a basic block for use as the target of the
|
|
// "error" branch of a try_apply. The error block created here will always
|
|
// jump to the error target of the original forEach.
|
|
auto errorTargetGenerator = [&](SILBasicBlock *insertionBlock,
|
|
SILValue borrowedElem ) {
|
|
SILBasicBlock *newErrorBB = fun->createBasicBlockBefore(insertionBlock);
|
|
SILValue argument = newErrorBB->createPhiArgument(
|
|
errorArgument->getType(), errorArgument->getOwnershipKind());
|
|
// Make the errorBB jump to the error target of the original forEach.
|
|
SILBuilderWithScope builder(newErrorBB, forEachCall);
|
|
if (borrowedElem) {
|
|
builder.createEndBorrow(forEachLoc, borrowedElem);
|
|
}
|
|
builder.createBranch(forEachLoc, errorBB, argument);
|
|
return newErrorBB;
|
|
};
|
|
|
|
// The basic block to jump to in the normal case of the try_apply in each
|
|
// unrolling.
|
|
SILBasicBlock *nextNormalBB = normalBB;
|
|
|
|
// Iterate through the array indices in the reverse order and do the
|
|
// following:
|
|
// - create a new basic block b_i if i > 0. Let b_0 denote the basic block
|
|
// that contains the forEach call.
|
|
// - store_borrow the owned copy of the element e_i into the `allocStack`.
|
|
// - try_apply the forEach's body closure on the `allocStack`. The normal
|
|
// target of the try_apply is b_i+1 if i is not the last index, otherwise
|
|
// it is `normalBB`. (The normal target is captured by `nextNormalBB`.)
|
|
// Jump to a new error block: err_i in the error case. Note that all
|
|
// error blocks jump to the error target of the original forEach call.
|
|
for (uint64_t num = arrayInfo.getNumElements(); num > 0; --num) {
|
|
SILValue elementCopy = elementCopies[num - 1];
|
|
// Creating the next normal block ends the borrow scope for borrowedElem
|
|
// from the previous iteration.
|
|
SILBasicBlock *currentBB = num > 1 ? normalTargetGenerator(nextNormalBB)
|
|
: forEachCall->getParentBlock();
|
|
SILBuilderWithScope unrollBuilder(currentBB, forEachCall);
|
|
SILValue borrowedElem;
|
|
if (arrayElementType.isTrivial(*fun)) {
|
|
unrollBuilder.createStore(forEachLoc, elementCopy, allocStack,
|
|
StoreOwnershipQualifier::Trivial);
|
|
} else {
|
|
// Borrow the elementCopy and store it in the allocStack. Note that the
|
|
// element's copy is guaranteed to be alive until the array is alive.
|
|
// Therefore it is okay to use a borrow here.
|
|
borrowedElem = unrollBuilder.createBeginBorrow(forEachLoc, elementCopy);
|
|
unrollBuilder.createStoreBorrow(forEachLoc, borrowedElem, allocStack);
|
|
|
|
SILBuilderWithScope(&nextNormalBB->front(), forEachCall)
|
|
.createEndBorrow(forEachLoc, borrowedElem);
|
|
}
|
|
|
|
SILBasicBlock *errorTarget =
|
|
errorTargetGenerator(nextNormalBB, borrowedElem);
|
|
// Note that the substitution map must be empty as we are not supporting
|
|
// elements of address-only type. All elements in the array are guaranteed
|
|
// to be loadable. TODO: generalize this to address-only types.
|
|
unrollBuilder.createTryApply(forEachLoc, forEachBodyClosure,
|
|
SubstitutionMap(), allocStack,
|
|
nextNormalBB, errorTarget);
|
|
nextNormalBB = currentBB;
|
|
}
|
|
|
|
// Dealloc the stack in the normalBB and also in errorBB. Note that every
|
|
// try_apply created during the unrolling must pass through these blocks.
|
|
SILBuilderWithScope(&normalBB->front())
|
|
.createDeallocStack(forEachLoc, allocStack);
|
|
SILBuilderWithScope(&errorBB->front())
|
|
.createDeallocStack(forEachLoc, allocStack);
|
|
|
|
// Remove the forEach call as it has now been unrolled.
|
|
removeForEachCall(forEachCall, deleter);
|
|
}
|
|
|
|
/// Determine whether the cost of unrolling the forEach is within the
|
|
/// \c unrollThreshold.
|
|
static bool canUnrollForEachOfArray(ArrayInfo arrayInfo, SILModule &module) {
|
|
const uint64_t unrollThreshold = module.getOptions().UnrollThreshold;
|
|
// The cost of unrolling a forEach loop is mostly just two instructions per
|
|
// array element: one to store the element into an alloc_stack and another to
|
|
// invoke the forEach body closure with the element. Note that the copy_value
|
|
// of the element and the basic blocks created for the try-applies of the
|
|
// body closure are not counted, as these should likely get optimized away.
|
|
const uint64_t cost = 2;
|
|
return arrayInfo.getNumElements() * cost <= unrollThreshold;
|
|
}
|
|
|
|
static bool tryUnrollForEachCallsOverArrayLiteral(ApplyInst *apply,
|
|
InstructionDeleter &deleter) {
|
|
ArrayInfo arrayInfo;
|
|
if (!arrayInfo.tryInitialize(apply))
|
|
return false;
|
|
// Bail out, if the array could be modified after initialization.
|
|
if (arrayInfo.mayBeModified())
|
|
return false;
|
|
// If there are no forEach loops to unroll, return.
|
|
ArrayRef<TryApplyInst *> forEachCalls = arrayInfo.getForEachUses();
|
|
if (forEachCalls.empty())
|
|
return false;
|
|
// If the array is too large to unroll, bail out.
|
|
if (!canUnrollForEachOfArray(arrayInfo, apply->getParent()->getModule()))
|
|
return false;
|
|
for (TryApplyInst *forEachCall : forEachCalls)
|
|
unrollForEach(arrayInfo, forEachCall, deleter);
|
|
return true;
|
|
}
|
|
|
|
class ForEachLoopUnroller : public SILFunctionTransform {
|
|
|
|
~ForEachLoopUnroller() override {}
|
|
|
|
void run() override {
|
|
SILFunction &fun = *getFunction();
|
|
bool changed = false;
|
|
|
|
if (!fun.hasOwnership())
|
|
return;
|
|
|
|
InstructionDeleter deleter;
|
|
for (SILBasicBlock &bb : fun) {
|
|
for (auto instIter = bb.begin(); instIter != bb.end();) {
|
|
SILInstruction *inst = &*instIter;
|
|
ApplyInst *apply = dyn_cast<ApplyInst>(inst);
|
|
if (!apply) {
|
|
++instIter;
|
|
continue;
|
|
}
|
|
// Note that the following operation may delete a forEach call but
|
|
// would not delete this apply instruction, which is an array
|
|
// initializer. Therefore, the iterator should be valid here.
|
|
changed |= tryUnrollForEachCallsOverArrayLiteral(apply, deleter);
|
|
++instIter;
|
|
}
|
|
}
|
|
|
|
if (changed) {
|
|
deleter.cleanupDeadInstructions();
|
|
PM->invalidateAnalysis(&fun,
|
|
SILAnalysis::InvalidationKind::FunctionBody);
|
|
}
|
|
}
|
|
};
|
|
|
|
} // end anonymous namespace
|
|
|
|
SILTransform *swift::createForEachLoopUnroll() {
|
|
return new ForEachLoopUnroller();
|
|
}
|