diff --git a/include/swift/SIL/OwnershipUtils.h b/include/swift/SIL/OwnershipUtils.h index 77de2f3e92d..b9c0d68326b 100644 --- a/include/swift/SIL/OwnershipUtils.h +++ b/include/swift/SIL/OwnershipUtils.h @@ -31,6 +31,7 @@ class SILModule; class SILValue; class DeadEndBlocks; class PrunedLiveness; +struct BorrowedValue; /// Returns true if v is an address or trivial. bool isValueAddressOrTrivial(SILValue v); @@ -103,6 +104,15 @@ inline bool isForwardingConsume(SILValue value) { bool findInnerTransitiveGuaranteedUses( SILValue guaranteedValue, SmallVectorImpl *usePoints = nullptr); +/// Like findInnerTransitiveGuaranteedUses except that rather than it being a +/// precondition that the provided value not be a BorrowedValue, it is a [type- +/// system-enforced] precondition that the provided value be a BorrowedValue. +/// +/// TODO: Merge with findInnerTransitiveGuaranteedUses. +bool findInnerTransitiveGuaranteedUsesOfBorrowedValue( + BorrowedValue borrowedValue, + SmallVectorImpl *usePoints = nullptr); + /// Find leaf "use points" of a guaranteed value within its enclosing borrow /// scope (without looking through reborrows). To find the use points of the /// extended borrow scope, after looking through reborrows, use diff --git a/include/swift/SIL/SILInstruction.h b/include/swift/SIL/SILInstruction.h index 8e30d04683e..132852c65fb 100644 --- a/include/swift/SIL/SILInstruction.h +++ b/include/swift/SIL/SILInstruction.h @@ -630,6 +630,12 @@ public: /// Can this instruction abort the program in some manner? bool mayTrap() const; + /// Involves a synchronization point like a memory barrier, lock or syscall. + /// + /// TODO: We need side-effect analysis and library annotation for this to be + /// a reasonable API. For now, this is just a placeholder. + bool maySynchronize() const; + /// Returns true if the given instruction is completely identical to RHS. bool isIdenticalTo(const SILInstruction *RHS) const { return isIdenticalTo(RHS, diff --git a/include/swift/SILOptimizer/Utils/CanonicalizeBorrowScope.h b/include/swift/SILOptimizer/Utils/CanonicalizeBorrowScope.h index 3380d568ce4..e384fbbeaee 100644 --- a/include/swift/SILOptimizer/Utils/CanonicalizeBorrowScope.h +++ b/include/swift/SILOptimizer/Utils/CanonicalizeBorrowScope.h @@ -16,11 +16,8 @@ /// deleted, which in turn allows canonicalization of the outer owned values /// (via CanonicalizeOSSALifetime). /// -/// This does not shrink borrow scopes; it does not rewrite end_borrows. -/// -/// TODO: A separate utility to shrink borrow scopes should eventually run -/// before this utility. It should hoist end_borrow up to the latest "destroy -/// barrier" whenever the scope does not contain a PointerEscape. +/// This does not shrink borrow scopes; it does not rewrite end_borrows. For +/// that, see ShrinkBorrowScope. /// //===----------------------------------------------------------------------===// @@ -150,6 +147,8 @@ protected: bool consolidateBorrowScope(); }; +bool shrinkBorrowScope(BeginBorrowInst *bbi, InstructionDeleter &deleter); + } // namespace swift #endif // SWIFT_SILOPTIMIZER_UTILS_CANONICALIZEBORROWSCOPES_H diff --git a/lib/SIL/IR/SILInstruction.cpp b/lib/SIL/IR/SILInstruction.cpp index 835acb1fe77..90477dd544c 100644 --- a/lib/SIL/IR/SILInstruction.cpp +++ b/lib/SIL/IR/SILInstruction.cpp @@ -1370,6 +1370,12 @@ bool SILInstruction::mayTrap() const { } } +bool SILInstruction::maySynchronize() const { + // TODO: We need side-effect analysis and library annotation for this to be + // a reasonable API. For now, this is just a placeholder. + return isa(this); +} + bool SILInstruction::isMetaInstruction() const { // Every instruction that implements getVarInfo() should be in this list. switch (getKind()) { diff --git a/lib/SIL/Utils/OwnershipUtils.cpp b/lib/SIL/Utils/OwnershipUtils.cpp index f1aec3cf37b..abf65317d9f 100644 --- a/lib/SIL/Utils/OwnershipUtils.cpp +++ b/lib/SIL/Utils/OwnershipUtils.cpp @@ -188,6 +188,103 @@ bool swift::findInnerTransitiveGuaranteedUses( return true; } +/// Like findInnerTransitiveGuaranteedUses except that rather than it being a +/// precondition that the provided value not be a BorrowedValue, it is a [type- +/// system-enforced] precondition that the provided value be a BorrowedValue. +/// +/// TODO: Merge with findInnerTransitiveGuaranteedUses. Note that at the moment +/// the two are _almost_ identical, but not quite because the other has a +/// #if 0 and not just leaf uses but ALL uses are recorded. +bool swift::findInnerTransitiveGuaranteedUsesOfBorrowedValue( + BorrowedValue borrowedValue, SmallVectorImpl *usePoints) { + + auto recordUse = [&](Operand *use) { + if (usePoints && use->getOperandOwnership() != OperandOwnership::NonUse) { + usePoints->push_back(use); + } + }; + + // Push the value's immediate uses. + // + // TODO: The worklist can be a simple vector without any a membership check if + // destructures are changed to be represented as reborrows. Currently a + // destructure forwards multiple results! This means that the worklist could + // grow exponentially without the membership check. It's fine to do this + // membership check locally in this function (within a borrow scope) because + // it isn't needed for the immediate uses, only the transitive uses. + GraphNodeWorklist worklist; + for (Operand *use : borrowedValue.value->getUses()) { + if (use->getOperandOwnership() != OperandOwnership::NonUse) + worklist.insert(use); + } + + // --- Transitively follow forwarded uses and look for escapes. + + // usePoints grows in this loop. + while (Operand *use = worklist.pop()) { + switch (use->getOperandOwnership()) { + case OperandOwnership::NonUse: + case OperandOwnership::TrivialUse: + case OperandOwnership::ForwardingConsume: + case OperandOwnership::DestroyingConsume: + llvm_unreachable("this operand cannot handle an inner guaranteed use"); + + case OperandOwnership::ForwardingUnowned: + case OperandOwnership::PointerEscape: + return false; + + case OperandOwnership::InstantaneousUse: + case OperandOwnership::UnownedInstantaneousUse: + case OperandOwnership::BitwiseEscape: + // Reborrow only happens when this is called on a value that creates a + // borrow scope. + case OperandOwnership::Reborrow: + // EndBorrow either happens when this is called on a value that creates a + // borrow scope, or when it is pushed as a use when processing a nested + // borrow. + case OperandOwnership::EndBorrow: + recordUse(use); + break; + + case OperandOwnership::InteriorPointer: + if (InteriorPointerOperandKind::get(use) == + InteriorPointerOperandKind::Invalid) + return false; + // If our base guaranteed value does not have any consuming uses (consider + // function arguments), we need to be sure to include interior pointer + // operands since we may not get a use from a end_scope instruction. + if (InteriorPointerOperand(use).findTransitiveUses(usePoints) != + AddressUseKind::NonEscaping) { + return false; + } + break; + + case OperandOwnership::ForwardingBorrow: { + ForwardingOperand(use).visitForwardedValues([&](SILValue result) { + // Do not include transitive uses with 'none' ownership + if (result.getOwnershipKind() == OwnershipKind::None) + return true; + for (auto *resultUse : result->getUses()) { + if (resultUse->getOperandOwnership() != OperandOwnership::NonUse) { + worklist.insert(resultUse); + } + } + return true; + }); + recordUse(use); + break; + } + case OperandOwnership::Borrow: + BorrowingOperand(use).visitExtendedScopeEndingUses([&](Operand *endUse) { + recordUse(endUse); + return true; + }); + break; + } + } + return true; +} + // Find all use points of \p guaranteedValue within its borrow scope. All use // points will be dominated by \p guaranteedValue. // diff --git a/lib/SILOptimizer/Transforms/CopyPropagation.cpp b/lib/SILOptimizer/Transforms/CopyPropagation.cpp index 86f99496e63..20b9dee700a 100644 --- a/lib/SILOptimizer/Transforms/CopyPropagation.cpp +++ b/lib/SILOptimizer/Transforms/CopyPropagation.cpp @@ -75,6 +75,9 @@ struct CanonicalDefWorklist { CanonicalDefWorklist(bool canonicalizeBorrows) : canonicalizeBorrows(canonicalizeBorrows) {} + // Update the worklist for the def corresponding to \p bbi, a BeginBorrow. + void updateForBorrow(BeginBorrowInst *bbi) { borrowedValues.insert(bbi); } + // Update the worklist for the def corresponding to \p copy, which is usually // a CopyValue, but may be any owned value such as the operand of a // DestroyValue (to force lifetime shortening). @@ -414,11 +417,15 @@ void CopyPropagation::run() { InstructionDeleter deleter(std::move(callbacks)); bool changed = false; - // Driver: Find all copied defs. + GraphNodeWorklist beginBorrowsToShrink; + + // Driver: Find all copied or borrowed defs. for (auto &bb : *f) { for (auto &i : bb) { if (auto *copy = dyn_cast(&i)) { defWorklist.updateForCopy(copy); + } else if (auto *borrow = dyn_cast(&i)) { + beginBorrowsToShrink.insert(borrow); } else if (canonicalizeAll) { if (auto *destroy = dyn_cast(&i)) { defWorklist.updateForCopy(destroy->getOperand()); @@ -426,6 +433,14 @@ void CopyPropagation::run() { } } } + + // NOTE: We assume that the function is in reverse post order so visiting the + // blocks and pushing begin_borrows as we see them and then popping them + // off the end will result in shrinking inner borrow scopes first. + while (auto *bbi = beginBorrowsToShrink.pop()) { + shrinkBorrowScope(bbi, deleter); + } + // canonicalizer performs all modifications through deleter's callbacks, so we // don't need to explicitly check for changes. CanonicalizeOSSALifetime canonicalizer(pruneDebug, poisonRefs, diff --git a/lib/SILOptimizer/Utils/CMakeLists.txt b/lib/SILOptimizer/Utils/CMakeLists.txt index 0f9790ba676..6d2fd8c488e 100644 --- a/lib/SILOptimizer/Utils/CMakeLists.txt +++ b/lib/SILOptimizer/Utils/CMakeLists.txt @@ -22,6 +22,7 @@ target_sources(swiftSILOptimizer PRIVATE OptimizerStatsUtils.cpp PartialApplyCombiner.cpp PerformanceInlinerUtils.cpp + ShrinkBorrowScope.cpp SILInliner.cpp SILSSAUpdater.cpp SpecializationMangler.cpp diff --git a/lib/SILOptimizer/Utils/ShrinkBorrowScope.cpp b/lib/SILOptimizer/Utils/ShrinkBorrowScope.cpp new file mode 100644 index 00000000000..e94bf86bf10 --- /dev/null +++ b/lib/SILOptimizer/Utils/ShrinkBorrowScope.cpp @@ -0,0 +1,247 @@ +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2014 - 2021 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors +// + +#include "swift/AST/Builtins.h" +#include "swift/SIL/MemAccessUtils.h" +#include "swift/SIL/OwnershipUtils.h" +#include "swift/SIL/SILBasicBlock.h" +#include "swift/SILOptimizer/Utils/CanonicalizeBorrowScope.h" +#include "swift/SILOptimizer/Utils/InstOptUtils.h" +#include "swift/SILOptimizer/Utils/InstructionDeleter.h" + +#define DEBUG_TYPE "copy-propagation" + +using namespace swift; + +//===----------------------------------------------------------------------===// +// MARK: Local utilities +//===----------------------------------------------------------------------===// + +// TODO: Move to be member function on SILInstruction. +static SILInstruction *getPreviousInstruction(SILInstruction *inst) { + auto pos = inst->getIterator(); + return pos == inst->getParent()->begin() ? nullptr + : &*std::prev(inst->getIterator()); +} + +// TODO: Move to be member function on SILInstruction. +static SILInstruction *getNextInstruction(SILInstruction *inst) { + auto nextPos = std::next(inst->getIterator()); + return nextPos == inst->getParent()->end() ? nullptr : &*nextPos; +} + +//===----------------------------------------------------------------------===// +// MARK: ShrinkBorrowScope +//===----------------------------------------------------------------------===// + +class ShrinkBorrowScope { + // The instruction that begins this borrow scope. + BeginBorrowInst *introducer; + + InstructionDeleter &deleter; + + SmallPtrSet users; + llvm::SmallVector> + barrierInstructions; + + SmallPtrSet blocksWithReachedTops; + SmallPtrSet blocksToEndAtTop; + + // The list of blocks to look for new points at which to insert end_borrows + // in. A block must not be processed if all of its successors have not yet + // been. For that reason, it is necessary to allow the same block to be + // visited multiple times, at most once for each successor. + SmallVector worklist; + // The instructions from which the shrinking starts, the scope ending + // instructions, keyed off the block in which they appear. + llvm::SmallDenseMap startingInstructions; + +public: + ShrinkBorrowScope(BeginBorrowInst *bbi, InstructionDeleter &deleter) + : introducer(bbi), deleter(deleter) {} + + bool run(); + + bool populateUsers(); + bool initializeWorklist(); + void findBarriers(); + void rewrite(); + void createEndBorrow(SILInstruction *insertionPoint); + + bool reachedTopOfAllSuccessors(SILBasicBlock *block) { + return llvm::all_of(block->getSuccessorBlocks(), [=](auto *successor) { + return blocksWithReachedTops.contains(successor); + }); + } + + bool mayAccessPointer(SILInstruction *instruction) { + if (!instruction->mayReadOrWriteMemory()) + return false; + bool fail = false; + visitAccessedAddress(instruction, [&fail](Operand *operand) { + auto accessStorage = AccessStorage::compute(operand->get()); + if (accessStorage.getKind() != AccessRepresentation::Kind::Unidentified) + fail = true; + }); + return fail; + } + + bool mayLoadWeakOrUnowned(SILInstruction *instruction) { + // TODO: It is possible to do better here by looking at the address that is + // being loaded. + return isa(instruction) || isa(instruction); + } + + bool isDeinitBarrier(SILInstruction *instruction) { + return users.contains(instruction) || instruction->maySynchronize() || + mayAccessPointer(instruction, value) || + mayLoadWeakOrUnowned(instruction, value); + } +}; + +//===----------------------------------------------------------------------===// +// MARK: Rewrite borrow scopes +//===----------------------------------------------------------------------===// + +bool ShrinkBorrowScope::run() { + if (!BorrowedValue(introducer).isLocalScope()) + return false; + if (!populateUsers()) + return false; + if (!initializeWorklist()) + return false; + + findBarriers(); + + rewrite(); + + return true; +} + +bool ShrinkBorrowScope::populateUsers() { + SmallVector usePoints; + if (!findInnerTransitiveGuaranteedUsesOfBorrowedValue( + BorrowedValue(introducer), &usePoints)) { + // If the value produced by begin_borrow escapes, don't shrink the borrow + // scope. + return false; + } + for (auto *usePoint : usePoints) { + auto *user = usePoint->getUser(); + users.insert(user); + } + return true; +} + +bool ShrinkBorrowScope::initializeWorklist() { + llvm::SmallVector scopeEndingInsts; + BorrowedValue(introducer).getLocalScopeEndingInstructions(scopeEndingInsts); + + // Form a map of the scopeEndingInsts, keyed off the block they occur in. If + // a scope ending instruction is not an end_borrow, bail out. + for (auto *instruction : scopeEndingInsts) { + if (!isa(instruction)) + return false; + auto *block = instruction->getParent(); + worklist.push_back(block); + startingInstructions[block] = instruction; + } + + return true; +} + +void ShrinkBorrowScope::findBarriers() { + // Walk the cfg backwards from the blocks containing scope ending + // instructions, visiting only the initial blocks (which contained those + // instructions) and those blocks all of whose successors have already been + // visited. + // + // TODO: Handle loops. + while (!worklist.empty()) { + auto *block = worklist.pop_back_val(); + auto *startingInstruction = startingInstructions.lookup(block); + if (!startingInstruction && !reachedTopOfAllSuccessors(block)) { + continue; + } + if (!startingInstruction && + !tryHoistOverInstruction(block->getTerminator())) { + // This block was walked to--it was not one containing one of the initial + // end_borrow instructions. Check whether it forwards the ownership of + // the borrowed value (either directly or indirectly). If it does, we + // must not hoist the end_borrow above it. + continue; + } + for (auto *successor : block->getSuccessorBlocks()) { + blocksToEndAtTop.erase(successor); + } + + // We either have processed all successors of block or else it is a block + // which contained one of the original scope-ending instructions. Scan the + // block backwards, looking for the first deinit barrier. If we've visited + // all successors, start scanning from the terminator. If the block + // contained an original scope-ending instruction, start scanning from it. + SILInstruction *instruction = + startingInstruction ? startingInstruction : block->getTerminator(); + SILInstruction *barrier = nullptr; + while ((instruction = getPreviousInstruction(instruction))) { + if (instruction == introducer) { + barrier = instruction; + break; + } + if (isDeinitBarrier(instruction, borrowedValue.value)) { + barrier = instruction; + break; + } + } + + if (barrier) { + barrierInstructions.push_back({block, barrier}); + } else { + blocksWithReachedTops.insert(block); + blocksToEndAtTop.insert(block); + for (auto *predecessor : block->getPredecessorBlocks()) { + worklist.push_back(predecessor); + } + } + } +} + +void ShrinkBorrowScope::rewrite() { + // Remove all the original end_borrow instructions. + for (auto pair : startingInstructions) { + deleter.forceDelete(pair.getSecond()); + } + + // Insert the new end_borrow instructions that occur after deinit barriers. + for (auto pair : barrierInstructions) { + auto *insertionPoint = getNextInstruction(pair.second); + createEndBorrow(insertionPoint); + } + + // Insert the new end_borrow instructions that occur at the beginning of + // blocks which we couldn't hoist out of. + for (auto *block : blocksToEndAtTop) { + auto *insertionPoint = &*block->begin(); + createEndBorrow(insertionPoint); + } +} + +void ShrinkBorrowScope::createEndBorrow(SILInstruction *insertionPoint) { + auto builder = SILBuilderWithScope(insertionPoint); + builder.createEndBorrow( + RegularLocation::getAutoGeneratedLocation(insertionPoint->getLoc()), + introducer); +} + +bool swift::shrinkBorrowScope(BeginBorrowInst *bbi, + InstructionDeleter &deleter) { + ShrinkBorrowScope borrowShrinker(bbi, deleter); + return borrowShrinker.run(); +} diff --git a/test/SILOptimizer/copy_propagation_borrow.sil b/test/SILOptimizer/copy_propagation_borrow.sil index 830540a2a8f..a346443c166 100644 --- a/test/SILOptimizer/copy_propagation_borrow.sil +++ b/test/SILOptimizer/copy_propagation_borrow.sil @@ -520,8 +520,8 @@ bb3: // CHECK-LABEL: sil [ossa] @testEscapingForward : $@convention(method) (@guaranteed HasObject) -> () { // CHECK: begin_borrow %0 : $HasObject // CHECK: copy_value -// CHECK: destructure_struct // CHECK: end_borrow +// CHECK: destructure_struct // CHECK: ref_to_unmanaged // CHECK: destroy_value // CHECK-LABEL: } // end sil function 'testEscapingForward' diff --git a/test/SILOptimizer/copy_propagation_opaque.sil b/test/SILOptimizer/copy_propagation_opaque.sil index 972a782c1c7..dcb6c630c5e 100644 --- a/test/SILOptimizer/copy_propagation_opaque.sil +++ b/test/SILOptimizer/copy_propagation_opaque.sil @@ -416,14 +416,10 @@ bb0(%0 : @owned $T): } // CHECK-TRACE-LABEL: *** CopyPropagation: testCopyBorrow -// CHECK-TRACE: Removing destroy_value %1 : $T -// CHECK-TRACE: Removing %{{.*}} = copy_value %0 : $T // CHECK-TRACE-NOT: Removing // // CHECK-LABEL: sil [ossa] @testCopyBorrow : $@convention(thin) (@in T) -> () { // CHECK: bb0(%0 : @owned $T): -// CHECK-NEXT: %1 = begin_borrow %0 : $T -// CHECK-NEXT: end_borrow %1 : $T // CHECK-NEXT: destroy_value %0 : $T // CHECK-NEXT: tuple // CHECK-NEXT: return diff --git a/test/SILOptimizer/shrink_borrow_scope.sil b/test/SILOptimizer/shrink_borrow_scope.sil new file mode 100644 index 00000000000..45b668e90f3 --- /dev/null +++ b/test/SILOptimizer/shrink_borrow_scope.sil @@ -0,0 +1,546 @@ +// RUN: %target-sil-opt -copy-propagation -canonical-ossa-rewrite-borrows -enable-sil-verify-all %s | %FileCheck %s + +import Builtin +import Swift + +// ============================================================================= +// = DECLARATIONS {{ +// ============================================================================= + +class C { + weak var d: D? +} +class D {} +class DBox { + var d: D +} + +struct CDCase { + var c: C + var d: D +} + +class PointedTo { +} +class PointerWrapper { + var pointer: Builtin.RawPointer +} + +enum OneOfThree { case one, two, three } + +sil [ossa] @callee_guaranteed: $@convention(thin) (@guaranteed C) -> () +sil [ossa] @callee_optional_d_guaranteed: $@convention(thin) (@guaranteed Optional) -> () + +// ============================================================================= +// = DECLARATIONS }} +// ============================================================================= + +// ============================================================================= +// branching tests {{ +// ============================================================================= + +// Hoist over br. +// CHECK-LABEL: sil [ossa] @hoist_over_branch_1 : {{.*}} { +// CHECK: {{bb[0-9]+}}([[INSTANCE:%[^,]+]] : @owned $C): +// CHECK: [[LIFETIME:%[^,]+]] = begin_borrow [[INSTANCE]] +// CHECK: [[CALLEE_GUARANTEED:%[^,]+]] = function_ref @callee_guaranteed +// CHECK: {{%[^,]+}} = apply [[CALLEE_GUARANTEED]]([[LIFETIME]]) +// CHECK: end_borrow [[LIFETIME]] +// CHECK: br [[EXIT:bb[0-9]+]] +// CHECK: [[EXIT]]: +// CHECK: return [[INSTANCE]] +// CHECK-LABEL: } // end sil function 'hoist_over_branch_1' +sil [ossa] @hoist_over_branch_1 : $@convention(thin) (@owned C) -> @owned C { +entry(%instance: @owned $C): + %lifetime = begin_borrow %instance : $C + %callee_guaranteed = function_ref @callee_guaranteed : $@convention(thin) (@guaranteed C) -> () + %_ = apply %callee_guaranteed(%lifetime) : $@convention(thin) (@guaranteed C) -> () + br bl1 +bl1: + end_borrow %lifetime : $C + return %instance : $C +} + +// Hoist over cond_br. +// CHECK-LABEL: sil [ossa] @hoist_over_branch_2 : {{.*}} { +// CHECK: {{bb[0-9]+}}([[INSTANCE:%[^,]+]] : @owned $C): +// CHECK: [[LIFETIME:%[^,]+]] = begin_borrow [[INSTANCE]] +// CHECK: [[CALLEE_GUARANTEED:%[^,]+]] = function_ref @callee_guaranteed +// CHECK: {{%[^,]+}} = apply [[CALLEE_GUARANTEED]]([[LIFETIME]]) +// CHECK: end_borrow [[LIFETIME]] +// CHECK: cond_br undef, [[BL1:bb[0-9]+]], [[BL2:bb[0-9]+]] +// CHECK: [[BL1]]: +// CHECK: br [[EXIT:bb[0-9]+]] +// CHECK: [[BL2]]: +// CHECK: br [[EXIT]] +// CHECK: [[EXIT]]: +// CHECK: return [[INSTANCE]] +// CHECK-LABEL: } // end sil function 'hoist_over_branch_2' +sil [ossa] @hoist_over_branch_2 : $@convention(thin) (@owned C) -> @owned C { +entry(%instance: @owned $C): + %lifetime = begin_borrow %instance : $C + %callee_guaranteed = function_ref @callee_guaranteed : $@convention(thin) (@guaranteed C) -> () + %_ = apply %callee_guaranteed(%lifetime) : $@convention(thin) (@guaranteed C) -> () + cond_br undef, bl1, bl2 +bl1: + end_borrow %lifetime : $C + br exit +bl2: + end_borrow %lifetime : $C + br exit +exit: + return %instance : $C +} + +// Hoist over two brs. +// CHECK-LABEL: sil [ossa] @hoist_over_branch_3 : {{.*}} { +// CHECK: {{bb[0-9]+}}([[INSTANCE:%[^,]+]] : @owned $C): +// CHECK: [[LIFETIME:%[^,]+]] = begin_borrow [[INSTANCE]] +// CHECK: [[CALLEE_GUARANTEED:%[^,]+]] = function_ref @callee_guaranteed +// CHECK: {{%[^,]+}} = apply [[CALLEE_GUARANTEED]]([[LIFETIME]]) +// CHECK: end_borrow [[LIFETIME]] +// CHECK: cond_br undef, [[BL1:bb[0-9]+]], [[BL2:bb[0-9]+]] +// CHECK: [[BL1]]: +// CHECK: br [[EXIT:bb[0-9]+]] +// CHECK: [[BL2]]: +// CHECK: br [[EXIT]] +// CHECK: [[EXIT]]: +// CHECK: return [[INSTANCE]] +// CHECK-LABEL: } // end sil function 'hoist_over_branch_3' +sil [ossa] @hoist_over_branch_3 : $@convention(thin) (@owned C) -> @owned C { +entry(%instance: @owned $C): + %lifetime = begin_borrow %instance : $C + %callee_guaranteed = function_ref @callee_guaranteed : $@convention(thin) (@guaranteed C) -> () + %_ = apply %callee_guaranteed(%lifetime) : $@convention(thin) (@guaranteed C) -> () + cond_br undef, bl1, bl2 +bl1: + br exit +bl2: + br exit +exit: + end_borrow %lifetime : $C + return %instance : $C +} + +// Don't hoist over 1 / 2 brs. +// CHECK-LABEL: sil [ossa] @hoist_over_branch_4 : {{.*}} { +// CHECK: {{bb[0-9]+}}([[INSTANCE:%[^,]+]] : @owned $C): +// CHECK: [[LIFETIME:%[^,]+]] = begin_borrow [[INSTANCE]] +// CHECK: cond_br undef, [[BL1:bb[0-9]+]], [[BL2:bb[0-9]+]] +// CHECK: [[BL1]]: +// CHECK: [[CALLEE_GUARANTEED:%[^,]+]] = function_ref @callee_guaranteed +// CHECK: {{%[^,]+}} = apply [[CALLEE_GUARANTEED]]([[LIFETIME]]) +// CHECK: end_borrow [[LIFETIME]] +// CHECK: br [[EXIT:bb[0-9]+]] +// CHECK: [[BL2]]: +// CHECK: end_borrow [[LIFETIME]] +// CHECK: br [[EXIT]] +// CHECK: [[EXIT]]: +// CHECK: return [[INSTANCE]] +// CHECK-LABEL: } // end sil function 'hoist_over_branch_4' +sil [ossa] @hoist_over_branch_4 : $@convention(thin) (@owned C) -> @owned C { +entry(%instance: @owned $C): + %lifetime = begin_borrow %instance : $C + cond_br undef, bl1, bl2 +bl1: + %callee_guaranteed = function_ref @callee_guaranteed : $@convention(thin) (@guaranteed C) -> () + %_ = apply %callee_guaranteed(%lifetime) : $@convention(thin) (@guaranteed C) -> () + br exit +bl2: + br exit +exit: + end_borrow %lifetime : $C + return %instance : $C +} + +// Hoist over switch_enum destinations. +// CHECK-LABEL: sil [ossa] @hoist_over_branch_5 : {{.*}} { +// CHECK: {{bb[0-9]+}}([[INSTANCE:%[^,]+]] : @owned $C, [[CASE:%[^,]+]] : $OneOfThree): +// CHECK: [[LIFETIME:%[^,]+]] = begin_borrow [[INSTANCE]] +// CHECK: [[CALLEE_GUARANTEED:%[^,]+]] = function_ref @callee_guaranteed +// CHECK: {{%[0-9]+}} = apply [[CALLEE_GUARANTEED]]([[LIFETIME]]) +// CHECK: end_borrow [[LIFETIME]] +// CHECK: switch_enum [[CASE]] : $OneOfThree, case #OneOfThree.one!enumelt: [[ONE_DEST:bb[0-9]+]], case #OneOfThree.two!enumelt: [[TWO_DEST:bb[0-9]+]], case #OneOfThree.three!enumelt: [[THREE_DEST:bb[0-9]+]] +// CHECK: [[ONE_DEST]]: +// CHECK: br [[EXIT:bb[0-9]+]] +// CHECK: [[TWO_DEST]]: +// CHECK: br [[EXIT]] +// CHECK: [[THREE_DEST]]: +// CHECK: br [[EXIT]] +// CHECK: [[EXIT]]: +// CHECK: return [[INSTANCE]] +// CHECK-LABEL: } // end sil function 'hoist_over_branch_5' +sil [ossa] @hoist_over_branch_5 : $(@owned C, OneOfThree) -> @owned C { +entry(%instance: @owned $C, %case : $OneOfThree): + %lifetime = begin_borrow %instance : $C + %callee_guaranteed = function_ref @callee_guaranteed : $@convention(thin) (@guaranteed C) -> () + %_ = apply %callee_guaranteed(%lifetime) : $@convention(thin) (@guaranteed C) -> () + switch_enum %case : $OneOfThree, case #OneOfThree.one!enumelt: one_dest, case #OneOfThree.two!enumelt: two_dest, case #OneOfThree.three!enumelt: three_dest +one_dest: + br exit +two_dest: + br exit +three_dest: + br exit +exit: + end_borrow %lifetime : $C + return %instance : $C +} + +// Don't hoist over transformation terminator which forwards ownership of +// borrowed value. +// CHECK-LABEL: sil [ossa] @hoist_over_terminator_1 : {{.*}} { +// CHECK: {{bb[0-9]+}}([[INSTANCE:%[^,]+]] : @owned $Optional): +// CHECK: [[LIFETIME:%[^,]+]] = begin_borrow [[INSTANCE]] +// CHECK: switch_enum [[LIFETIME]] : $Optional, case #Optional.some!enumelt: [[SOME_DEST:bb[0-9]+]], case #Optional.none!enumelt: [[NONE_DEST:bb[0-9]+]] +// CHECK: [[SOME_DEST]]([[LIFETIME_2:%[^,]+]] : @guaranteed $C): +// CHECK: end_borrow [[LIFETIME]] +// CHECK: br [[EXIT:bb[0-9]+]] +// CHECK: [[NONE_DEST]]: +// CHECK: end_borrow [[LIFETIME]] +// CHECK: br [[EXIT]] +// CHECK: [[EXIT]]: +// CHECK: return [[INSTANCE]] +// CHECK-LABEL: } // end sil function 'hoist_over_terminator_1' +sil [ossa] @hoist_over_terminator_1 : $@convention(thin) (@owned Optional) -> @owned Optional { +entry(%instance_c : @owned $Optional): + %lifetime_c = begin_borrow %instance_c : $Optional + switch_enum %lifetime_c : $Optional, case #Optional.some!enumelt: some_dest, case #Optional.none!enumelt: none_dest + +some_dest(%lifetime_c_2 : @guaranteed $C): + br exit + +none_dest: + br exit + +exit: + end_borrow %lifetime_c : $Optional + return %instance_c : $Optional +} + +// Hoist over brs but don't hoist over transformation terminator which forwards +// ownership of guaranteed value which itself had forwarding ownership of the +// original borrow. +// CHECK-LABEL: sil [ossa] @hoist_over_terminator_2 : {{.*}} { +// CHECK: {{bb[0-9]+}}([[INSTANCE:%[^,]+]] : @owned $C): +// CHECK: [[LIFETIME:%[^,]+]] = begin_borrow [[INSTANCE]] +// CHECK: [[MAYBE:%[^,]+]] = enum $Optional, #Optional.some!enumelt, [[LIFETIME]] +// CHECK: switch_enum [[MAYBE]] : $Optional, case #Optional.some!enumelt: [[SOME_DEST:bb[0-9]+]], case #Optional.none!enumelt: [[NONE_DEST:bb[0-9]+]] +// CHECK: [[SOME_DEST]]([[LIFETIME_2:%[^,]+]] : @guaranteed $C): +// CHECK: end_borrow [[LIFETIME]] +// CHECK: br [[BASIC_BLOCK3:bb[0-9]+]] +// CHECK: [[NONE_DEST]]: +// CHECK: end_borrow [[LIFETIME]] +// CHECK: br [[BASIC_BLOCK3]] +// CHECK: [[BASIC_BLOCK3]]: +// CHECK: return [[INSTANCE]] +// CHECK-LABEL: } // end sil function 'hoist_over_terminator_2' +sil [ossa] @hoist_over_terminator_2 : $@convention(thin) (@owned C) -> @owned C { +entry(%instance_c : @owned $C): + %lifetime_c = begin_borrow %instance_c : $C + %maybe_c = enum $Optional, #Optional.some!enumelt, %lifetime_c : $C + switch_enum %maybe_c : $Optional, case #Optional.some!enumelt: some_dest, case #Optional.none!enumelt: none_dest + +some_dest(%lifetime_c_2 : @guaranteed $C): + br exit + +none_dest: + br exit + +exit: + end_borrow %lifetime_c : $C + return %instance_c : $C +} + +// Hoist over transformation terminator which forwards ownership of guaranteed +// value which itself had forwarding ownership of the original borrow. +// CHECK-LABEL: sil [ossa] @hoist_over_terminator_3 : {{.*}} { +// CHECK: {{bb[0-9]+}}([[INSTANCE:%[^,]+]] : @owned $C): +// CHECK: [[LIFETIME:%[^,]+]] = begin_borrow [[INSTANCE]] +// CHECK: [[MAYBE:%[^,]+]] = enum $Optional, #Optional.some!enumelt, [[LIFETIME]] +// CHECK: switch_enum [[MAYBE]] : $Optional, case #Optional.some!enumelt: [[SOME_DEST:bb[0-9]+]], case #Optional.none!enumelt: [[NONE_DEST:bb[0-9]+]] +// CHECK: [[SOME_DEST]]([[LIFETIME_2:%[^,]+]] : @guaranteed $C): +// CHECK: end_borrow [[LIFETIME]] +// CHECK: br [[EXIT:bb[0-9]+]] +// CHECK: [[NONE_DEST]]: +// CHECK: end_borrow [[LIFETIME]] +// CHECK: br [[EXIT]] +// CHECK: [[EXIT]]: +// CHECK: return [[INSTANCE]] +// CHECK-LABEL: } // end sil function 'hoist_over_terminator_3' +sil [ossa] @hoist_over_terminator_3 : $@convention(thin) (@owned C) -> @owned C { +entry(%instance_c : @owned $C): + %lifetime_c = begin_borrow %instance_c : $C + %maybe_c = enum $Optional, #Optional.some!enumelt, %lifetime_c : $C + switch_enum %maybe_c : $Optional, case #Optional.some!enumelt: some_dest, case #Optional.none!enumelt: none_dest + +some_dest(%lifetime_c_2 : @guaranteed $C): + end_borrow %lifetime_c : $C + br exit + +none_dest: + end_borrow %lifetime_c : $C + br exit + +exit: + return %instance_c : $C +} + +// Don't hoist over terminator that reborrows. +// CHECK-LABEL: sil [ossa] @hoist_over_terminator_4 : {{.*}} { +// CHECK: {{bb[0-9]+}}([[INSTANCE:%[^,]+]] : @owned $C): +// CHECK: [[LIFETIME:%[^,]+]] = begin_borrow [[INSTANCE]] +// CHECK: br [[WORK:bb[0-9]+]]([[LIFETIME]] : $C) +// CHECK: [[WORK]]([[LIFETIME_2:%[^,]+]] : @guaranteed $C): +// CHECK: cond_br undef, [[LEFT:bb[0-9]+]], [[RIGHT:bb[0-9]+]] +// CHECK: [[LEFT]]: +// CHECK: end_borrow [[LIFETIME_2]] +// CHECK: br [[EXIT:bb[0-9]+]] +// CHECK: [[RIGHT]]: +// CHECK: end_borrow [[LIFETIME_2]] +// CHECK: br [[EXIT]] +// CHECK: [[EXIT]]: +// CHECK: return [[INSTANCE]] +// CHECK-LABEL: } // end sil function 'hoist_over_terminator_4' +sil [ossa] @hoist_over_terminator_4 : $@convention(thin) (@owned C) -> @owned C { +entry(%instance_c : @owned $C): + %lifetime_c_0 = begin_borrow %instance_c : $C + br work(%lifetime_c_0 : $C) + +work(%lifetime_c : @guaranteed $C): + cond_br undef, left, right + +left: + end_borrow %lifetime_c : $C + br exit + +right: + end_borrow %lifetime_c : $C + br exit + +exit: + return %instance_c : $C +} + +// ============================================================================= +// branching tests }} +// ============================================================================= + +// ============================================================================= +// loop tests {{ +// ============================================================================= + +// Don't hoist over loop without uses. +// TODO: Eventually, we should hoist over such loops. +// CHECK-LABEL: sil [ossa] @hoist_over_loop_1 : {{.*}} { +// CHECK: {{bb[0-9]+}}([[INSTANCE:%[^,]+]] : @owned $C): +// CHECK: [[LIFETIME:%[^,]+]] = begin_borrow [[INSTANCE]] +// CHECK: [[CALLEE_GUARANTEED:%[^,]+]] = function_ref @callee_guaranteed +// CHECK: {{%[^,]+}} = apply [[CALLEE_GUARANTEED]]([[LIFETIME]]) +// CHECK: br [[LOOP_HEADER:bb[0-9]+]] +// CHECK: [[LOOP_HEADER]]: +// CHECK: br [[LOOP_BODY:bb[0-9]+]] +// CHECK: [[LOOP_BODY]]: +// CHECK: br [[LOOP_LATCH:bb[0-9]+]] +// CHECK: [[LOOP_LATCH]]: +// CHECK: cond_br undef, [[EXIT:bb[0-9]+]], [[LOOP_BACKEDGE:bb[0-9]+]] +// CHECK: [[LOOP_BACKEDGE]]: +// CHECK: br [[LOOP_HEADER]] +// CHECK: [[EXIT]]: +// CHECK: end_borrow [[LIFETIME]] +// CHECK: return [[INSTANCE]] +// CHECK-LABEL: } // end sil function 'hoist_over_loop_1' +sil [ossa] @hoist_over_loop_1 : $@convention(thin) (@owned C) -> @owned C { +entry(%instance: @owned $C): + %lifetime = begin_borrow %instance : $C + %callee_guaranteed = function_ref @callee_guaranteed : $@convention(thin) (@guaranteed C) -> () + %_ = apply %callee_guaranteed(%lifetime) : $@convention(thin) (@guaranteed C) -> () + br loop_header +loop_header: + br loop_body +loop_body: + br loop_latch +loop_latch: + cond_br undef, exit, loop_backedge +loop_backedge: + br loop_header +exit: + end_borrow %lifetime : $C + return %instance : $C +} + +// Don't hoist over loop with uses. +// CHECK-LABEL: sil [ossa] @hoist_over_loop_2 : {{.*}} { +// CHECK: {{bb[0-9]+}}([[INSTANCE:%[^,]+]] : @owned $C): +// CHECK: [[LIFETIME:%[^,]+]] = begin_borrow [[INSTANCE]] +// CHECK: br [[LOOP_HEADER:bb[0-9]+]] +// CHECK: [[LOOP_HEADER]]: +// CHECK: br [[LOOP_BODY:bb[0-9]+]] +// CHECK: [[LOOP_BODY]]: +// CHECK: [[CALLEE_GUARANTEED:%[^,]+]] = function_ref @callee_guaranteed +// CHECK: {{%[^,]+}} = apply [[CALLEE_GUARANTEED]]([[LIFETIME]]) +// CHECK: br [[LOOP_LATCH:bb[0-9]+]] +// CHECK: [[LOOP_LATCH]]: +// CHECK: cond_br undef, [[EXIT:bb[0-9]+]], [[LOOP_BACKEDGE:bb[0-9]+]] +// CHECK: [[LOOP_BACKEDGE]]: +// CHECK: br [[LOOP_HEADER]] +// CHECK: [[EXIT]]: +// CHECK: end_borrow [[LIFETIME]] +// CHECK: return [[INSTANCE]] +// CHECK-LABEL: } // end sil function 'hoist_over_loop_2' +sil [ossa] @hoist_over_loop_2 : $@convention(thin) (@owned C) -> @owned C { +entry(%instance: @owned $C): + %lifetime = begin_borrow %instance : $C + br loop_header +loop_header: + br loop_body +loop_body: + %callee_guaranteed = function_ref @callee_guaranteed : $@convention(thin) (@guaranteed C) -> () + %_ = apply %callee_guaranteed(%lifetime) : $@convention(thin) (@guaranteed C) -> () + br loop_latch +loop_latch: + cond_br undef, exit, loop_backedge +loop_backedge: + br loop_header +exit: + end_borrow %lifetime : $C + return %instance : $C +} + +// ============================================================================= +// loop tests }} +// ============================================================================= + +// ============================================================================= +// instruction tests {{ +// ============================================================================= + +// Don't hoist over struct of begin_borrow'd value. +// CHECK-LABEL: sil [ossa] @hoist_over_struct : {{.*}} { +// CHECK: {{bb[0-9]+}}([[INSTANCE_C:%[^,]+]] : @owned $C, [[INSTANCE_D:%[^,]+]] : @owned $D): +// CHECK: [[LIFETIME_C:%[^,]+]] = begin_borrow [[INSTANCE_C]] +// CHECK: [[LIFETIME_D:%[^,]+]] = begin_borrow [[INSTANCE_D]] +// CHECK: [[REGISTER_4:%[^,]+]] = struct $CDCase ([[LIFETIME_C]] : $C, [[LIFETIME_D]] : $D) +// CHECK: end_borrow [[LIFETIME_C]] +// CHECK: end_borrow [[LIFETIME_D]] +// CHECK: destroy_value [[INSTANCE_D]] +// CHECK: return [[INSTANCE_C]] +// CHECK-LABEL: } // end sil function 'hoist_over_struct' +sil [ossa] @hoist_over_struct : $@convention(thin) (@owned C, @owned D) -> @owned C { +entry(%instance_c: @owned $C, %instance_d: @owned $D): + %lifetime_c = begin_borrow %instance_c : $C + %copy_c = copy_value %lifetime_c : $C + %lifetime_d = begin_borrow %instance_d : $D + %copy_d = copy_value %lifetime_d : $D + + %struct = struct $CDCase (%lifetime_c : $C, %lifetime_d : $D) + end_borrow %lifetime_d : $D + + destroy_value %copy_d : $D + destroy_value %instance_d : $D + end_borrow %lifetime_c : $C + destroy_value %instance_c : $C + return %copy_c : $C +} + +// Don't hoist over store_weak into a field of the object being borrowed. +// CHECK-LABEL: sil [ossa] @hoist_over_store_weak_1 : {{.*}} { +// CHECK: store_weak +// CHECK: end_borrow +// CHECK-LABEL: } // end sil function 'hoist_over_store_weak_1' +sil [ossa] @hoist_over_store_weak_1 : $@convention(thin) (@owned C, @owned D) -> @owned C { +entry(%instance_c: @owned $C, %instance_d: @owned $D): + %lifetime_c = begin_borrow %instance_c : $C + %lifetime_d = begin_borrow %instance_d : $D + %copy_d = copy_value %lifetime_d : $D + %optional_d = enum $Optional, #Optional.some!enumelt, %copy_d : $D + %c_d_addr = ref_element_addr %lifetime_c : $C, #C.d + %c_d_access = begin_access [modify] [dynamic] %c_d_addr : $*@sil_weak Optional + + store_weak %optional_d to %c_d_access : $*@sil_weak Optional + end_borrow %lifetime_d : $D + + end_access %c_d_access : $*@sil_weak Optional + destroy_value %optional_d : $Optional + destroy_value %instance_d : $D + end_borrow %lifetime_c : $C + return %instance_c : $C +} + +// Don't hoist over store to an address which itself is (earlier) stored into a +// field of the object being borrowed. +// TODO: Eventually, we should be able to hoist over such a store. +// CHECK-LABEL: sil [ossa] @hoist_over_load : $@convention(thin) () -> () { +// CHECK: [[D:%[^,]+]] = alloc_ref $PointerWrapper +// CHECK: [[LIFETIME:%[^,]+]] = begin_borrow [[D]] : $PointerWrapper +// CHECK: [[ADDR:%[^,]+]] = alloc_stack $PointedTo +// CHECK: [[C:%[^,]+]] = alloc_ref $PointedTo +// CHECK: store [[C]] to [init] [[ADDR]] : $*PointedTo +// CHECK-NEXT: end_borrow [[LIFETIME]] : $PointerWrapper +// CHECK-LABEL: } // end sil function 'hoist_over_load' +sil [ossa] @hoist_over_load : $@convention(thin) () -> () { +bb0: + %d = alloc_ref $PointerWrapper + %lifetime = begin_borrow %d : $PointerWrapper + %addr = alloc_stack $PointedTo + %ptr = address_to_pointer %addr : $*PointedTo to $Builtin.RawPointer + %field = ref_element_addr %lifetime : $PointerWrapper, #PointerWrapper.pointer + + store %ptr to [trivial] %field : $*Builtin.RawPointer + + %c = alloc_ref $PointedTo + + store %c to [init] %addr : $*PointedTo + end_borrow %lifetime : $PointerWrapper + + destroy_value %d : $PointerWrapper + + destroy_addr %addr : $*PointedTo + dealloc_stack %addr : $*PointedTo + + %33 = tuple () + return %33 : $() +} + +// Don't hoist over load_weak. +// There is an ordering guarantee between a load_weak of an weak reference that +// subsequently is zeroed by a deinit; if the load_weak originally happens +// before the deinit that would deinit the weakly referenced object, it must +// always remain first. +// CHECK-LABEL: sil [ossa] @dont_hoist_over_load_weak : {{.*}} { +// CHECK: load_weak +// CHECK: end_borrow +// CHECK-LABEL: } // end sil function 'dont_hoist_over_load_weak' + +sil [ossa] @dont_hoist_over_load_weak : $@convention(thin) (@owned DBox) -> () { +entry(%instance : @owned $DBox): + %lifetime = begin_borrow [lexical] %instance : $DBox + debug_value %lifetime : $DBox, let, name "dbox" + %addr = alloc_stack [lexical] $@sil_weak Optional + %d_addr = ref_element_addr %lifetime : $DBox, #DBox.d + %d = load [copy] %d_addr : $*D + %some_d = enum $Optional, #Optional.some!enumelt, %d : $D + store_weak %some_d to [initialization] %addr : $*@sil_weak Optional + destroy_value %some_d : $Optional + + %loaded = load_weak %addr : $*@sil_weak Optional + end_borrow %lifetime : $DBox + + destroy_value %instance : $DBox + + destroy_value %loaded : $Optional + destroy_addr %addr : $*@sil_weak Optional + dealloc_stack %addr : $*@sil_weak Optional + %retval = tuple () + return %retval : $() +} + +// Don't hoist over load. +// There is an ordering guarantee between a load of an address the storage of +// which is subsequently deallocated by a deinit; if the load originally happens +// before the deinit that would dealocated the pointed-to object, it must always +// remain first. + +// ============================================================================= +// instruction tests }} +// =============================================================================