diff --git a/include/swift/SIL/SILInstruction.h b/include/swift/SIL/SILInstruction.h index 6b8c20d3e0b..5679b26cfea 100644 --- a/include/swift/SIL/SILInstruction.h +++ b/include/swift/SIL/SILInstruction.h @@ -1239,9 +1239,48 @@ public: ownershipKind = newKind; } + static bool canForwardAllOperands(SILInstruction *inst) { + switch (inst->getKind()) { + case SILInstructionKind::StructInst: + case SILInstructionKind::TupleInst: + case SILInstructionKind::LinearFunctionInst: + case SILInstructionKind::DifferentiableFunctionInst: + return true; + default: + return false; + } + } + + static bool canForwardFirstOperandOnly(SILInstruction *inst) { + if (!ForwardingInstruction::isa(inst)) { + return false; + } + return !canForwardAllOperands(inst); + } + + static bool canForwardOwnedCompatibleValuesOnly(SILInstruction *inst) { + switch (inst->getKind()) { + case SILInstructionKind::MarkUninitializedInst: + return true; + default: + return false; + } + } + + static bool canForwardGuaranteedCompatibleValuesOnly(SILInstruction *inst) { + switch (inst->getKind()) { + case SILInstructionKind::TupleExtractInst: + case SILInstructionKind::StructExtractInst: + case SILInstructionKind::DifferentiableFunctionExtractInst: + case SILInstructionKind::LinearFunctionExtractInst: + return true; + default: + return false; + } + } + /// Defined inline below due to forward declaration issues. static ForwardingInstruction *get(SILInstruction *inst); - /// Defined inline below due to forward declaration issues. static bool isa(SILInstructionKind kind); static bool isa(const SILInstruction *inst) { return isa(inst->getKind()); } static bool isa(SILNodePointer node) { @@ -1296,24 +1335,6 @@ public: } }; -inline bool -OwnershipForwardingSingleValueInstruction::classof(SILInstructionKind kind) { - switch (kind) { - case SILInstructionKind::ObjectInst: - case SILInstructionKind::EnumInst: - case SILInstructionKind::UncheckedEnumDataInst: - case SILInstructionKind::OpenExistentialRefInst: - case SILInstructionKind::InitExistentialRefInst: - case SILInstructionKind::MarkDependenceInst: - case SILInstructionKind::MoveOnlyWrapperToCopyableValueInst: - case SILInstructionKind::MoveOnlyWrapperToCopyableBoxInst: - case SILInstructionKind::CopyableToMoveOnlyWrapperValueInst: - return true; - default: - return false; - } -} - /// A value base result of a multiple value instruction. /// /// *NOTE* We want this to be a pure abstract class that does not add /any/ size @@ -10548,6 +10569,37 @@ public: } }; +inline bool +OwnershipForwardingSingleValueInstruction::classof(SILInstructionKind kind) { + switch (kind) { + case SILInstructionKind::ObjectInst: + case SILInstructionKind::EnumInst: + case SILInstructionKind::UncheckedEnumDataInst: + case SILInstructionKind::OpenExistentialRefInst: + case SILInstructionKind::InitExistentialRefInst: + case SILInstructionKind::MarkDependenceInst: + case SILInstructionKind::MoveOnlyWrapperToCopyableValueInst: + case SILInstructionKind::MoveOnlyWrapperToCopyableBoxInst: + case SILInstructionKind::CopyableToMoveOnlyWrapperValueInst: + case SILInstructionKind::MarkUninitializedInst: + case SILInstructionKind::TupleExtractInst: + case SILInstructionKind::StructExtractInst: + case SILInstructionKind::DifferentiableFunctionExtractInst: + case SILInstructionKind::LinearFunctionExtractInst: + case SILInstructionKind::OpenExistentialValueInst: + case SILInstructionKind::OpenExistentialBoxValueInst: + case SILInstructionKind::StructInst: + case SILInstructionKind::TupleInst: + case SILInstructionKind::LinearFunctionInst: + case SILInstructionKind::DifferentiableFunctionInst: + case SILInstructionKind::MarkMustCheckInst: + case SILInstructionKind::MarkUnresolvedReferenceBindingInst: + return true; + default: + return false; + } +} + /// DifferentiabilityWitnessFunctionInst - Looks up a differentiability witness /// function for a given original function. class DifferentiabilityWitnessFunctionInst @@ -10833,8 +10885,6 @@ inline ForwardingInstruction *ForwardingInstruction::get(SILInstruction *inst) { if (auto *result = dyn_cast(inst)) return result; - if (auto *result = dyn_cast(inst)) - return result; return nullptr; } diff --git a/lib/SIL/IR/SILInstructions.cpp b/lib/SIL/IR/SILInstructions.cpp index a732962f45c..b30ac0cc695 100644 --- a/lib/SIL/IR/SILInstructions.cpp +++ b/lib/SIL/IR/SILInstructions.cpp @@ -3229,13 +3229,15 @@ bool ForwardingInstruction::hasSameRepresentation(SILInstruction *inst) { } bool ForwardingInstruction::isAddressOnly(SILInstruction *inst) { - if (auto *aggregate = - dyn_cast(inst)) { + if (canForwardAllOperands(inst)) { + // All ForwardingInstructions that forward all operands are currently a + // single value instruction. + auto *aggregate = cast(inst); // If any of the operands are address-only, then the aggregate must be. return aggregate->getType().isAddressOnly(*inst->getFunction()); } // All other forwarding instructions must forward their first operand. - assert(ForwardingInstruction::isa(inst)); + assert(canForwardFirstOperandOnly(inst)); return inst->getOperand(0)->getType().isAddressOnly(*inst->getFunction()); } diff --git a/lib/SIL/Utils/MemAccessUtils.cpp b/lib/SIL/Utils/MemAccessUtils.cpp index e8e252733bc..fa42eae2646 100644 --- a/lib/SIL/Utils/MemAccessUtils.cpp +++ b/lib/SIL/Utils/MemAccessUtils.cpp @@ -899,18 +899,15 @@ SILValue swift::findOwnershipReferenceAggregate(SILValue ref) { if (!root) return root; - if (isa(root) || - isa(root) || - isa(root) || - isa(root)) { - SILInstruction *inst = root->getDefiningInstruction(); + if (auto *inst = root->getDefiningInstruction()) { + if (ForwardingInstruction::canForwardFirstOperandOnly(inst)) { + // The `enum` instruction can have no operand. + if (inst->getNumOperands() == 0) + return root; - // The `enum` instruction can have no operand. - if (inst->getNumOperands() == 0) - return root; - - root = inst->getOperand(0); - continue; + root = inst->getOperand(0); + continue; + } } if (auto *termResult = SILArgument::isTerminatorResult(root)) { if (auto *oper = termResult->forwardedTerminatorResultOperand()) { diff --git a/lib/SIL/Utils/OwnershipUtils.cpp b/lib/SIL/Utils/OwnershipUtils.cpp index 91125187eff..04b169b8574 100644 --- a/lib/SIL/Utils/OwnershipUtils.cpp +++ b/lib/SIL/Utils/OwnershipUtils.cpp @@ -117,23 +117,25 @@ bool swift::canOpcodeForwardInnerGuaranteedValues(SILValue value) { if (auto *inst = value->getDefiningInstruction()) if (auto *mixin = ForwardingInstruction::get(inst)) return mixin->preservesOwnership() && - !isa(inst); + !ForwardingInstruction::canForwardOwnedCompatibleValuesOnly(inst); return false; } bool swift::canOpcodeForwardInnerGuaranteedValues(Operand *use) { - if (auto *mixin = ForwardingInstruction::get(use->getUser())) - return mixin->preservesOwnership() && - !isa(use->getUser()); + if (auto *fwi = ForwardingInstruction::get(use->getUser())) + return fwi->preservesOwnership() && + !ForwardingInstruction::canForwardOwnedCompatibleValuesOnly( + use->getUser()); return false; } bool swift::canOpcodeForwardOwnedValues(SILValue value) { if (auto *inst = value->getDefiningInstructionOrTerminator()) { - if (auto *mixin = ForwardingInstruction::get(inst)) { - return mixin->preservesOwnership() && - !isa(inst); + if (auto *fwi = ForwardingInstruction::get(inst)) { + return fwi->preservesOwnership() && + !ForwardingInstruction::canForwardGuaranteedCompatibleValuesOnly( + inst); } } return false; @@ -141,9 +143,10 @@ bool swift::canOpcodeForwardOwnedValues(SILValue value) { bool swift::canOpcodeForwardOwnedValues(Operand *use) { auto *user = use->getUser(); - if (auto *mixin = ForwardingInstruction::get(user)) - return mixin->preservesOwnership() && - !isa(user); + if (auto *fwi = ForwardingInstruction::get(user)) + return fwi->preservesOwnership() && + !ForwardingInstruction::canForwardGuaranteedCompatibleValuesOnly( + user); return false; } @@ -1717,17 +1720,11 @@ bool swift::visitForwardedGuaranteedOperands( if (inst->getNumRealOperands() == 0) { return false; } - if (isa(inst) || - isa(inst) || - isa(inst) || - isa(inst)) { - assert(!isa(inst) - || !BorrowedValue(cast(inst)) - && "forwarded operand cannot begin a borrow scope"); + if (ForwardingInstruction::canForwardFirstOperandOnly(inst)) { visitOperand(&inst->getOperandRef(0)); return true; } - if (isa(inst)) { + if (ForwardingInstruction::canForwardAllOperands(inst)) { assert(inst->getNumOperands() > 0 && "checked above"); assert(inst->getNumOperands() == inst->getNumRealOperands() && "mixin expects all readl operands"); diff --git a/lib/SIL/Verifier/SILVerifier.cpp b/lib/SIL/Verifier/SILVerifier.cpp index 6ca4f1aa8e3..6824871b7ff 100644 --- a/lib/SIL/Verifier/SILVerifier.cpp +++ b/lib/SIL/Verifier/SILVerifier.cpp @@ -1355,14 +1355,14 @@ public: ValueOwnershipKind ownership = ForwardingInstruction::get(i)->getForwardingOwnershipKind(); - if (auto *o = dyn_cast(i)) { + if (ForwardingInstruction::canForwardOwnedCompatibleValuesOnly(i)) { ValueOwnershipKind kind = OwnershipKind::Owned; require(kind.isCompatibleWith(ownership), "OwnedFirstArgForwardingSingleValueInst's ownership kind must be " "compatible with owned"); } - if (auto *o = dyn_cast(i)) { + if (ForwardingInstruction::canForwardGuaranteedCompatibleValuesOnly(i)) { ValueOwnershipKind kind = OwnershipKind::Guaranteed; require(kind.isCompatibleWith(ownership), "GuaranteedFirstArgForwardingSingleValueInst's ownership kind " diff --git a/lib/SILOptimizer/SILCombiner/SILCombine.cpp b/lib/SILOptimizer/SILCombiner/SILCombine.cpp index 52110dcbefd..54e29359c1a 100644 --- a/lib/SILOptimizer/SILCombiner/SILCombine.cpp +++ b/lib/SILOptimizer/SILCombiner/SILCombine.cpp @@ -440,8 +440,7 @@ bool SILCombiner::doOneIteration(SILFunction &F, unsigned Iteration) { // owned parameters since chains of these values will be in the same // block. if (auto *svi = dyn_cast(I)) { - if ((isa(svi) || - isa(svi)) && + if (ForwardingInstruction::canForwardFirstOperandOnly(svi) && SILValue(svi)->getOwnershipKind() == OwnershipKind::Owned) { // Try to sink the value. If we sank the value and deleted it, // continue. If we didn't optimize or sank but we are still able to diff --git a/lib/SILOptimizer/Utils/CanonicalizeBorrowScope.cpp b/lib/SILOptimizer/Utils/CanonicalizeBorrowScope.cpp index c4f7f45f1ae..99d8e1fc824 100644 --- a/lib/SILOptimizer/Utils/CanonicalizeBorrowScope.cpp +++ b/lib/SILOptimizer/Utils/CanonicalizeBorrowScope.cpp @@ -78,8 +78,6 @@ static void deleteCopyAndMoveChain(SILValue v, InstructionDeleter &deleter) { /// OwnershipForwardingConversionInst (all kinds of ref casts) /// OwnershipForwardingMultipleValueInstruction /// (DestructureStruct, DestructureTuple) -/// AllArgOwnershipForwardingSingleValueInst -/// (Struct, Tuple) /// FirstArgOwnershipForwardingSingleValueInst /// (Object, Enum, UncheckedEnumData, Open/InitExistentialRef, /// MarkDependence) @@ -96,7 +94,7 @@ bool CanonicalizeBorrowScope::isRewritableOSSAForward(SILInstruction *inst) { if (inst->getNumOperands() != 1) return false; - if (isa(inst) + if (isa(inst) || isa(inst) || isa(inst)) { Operand *forwardedOper = &inst->getOperandRef(0); diff --git a/lib/SILOptimizer/Utils/InstOptUtils.cpp b/lib/SILOptimizer/Utils/InstOptUtils.cpp index cc1da9f76ef..430e8f38c47 100644 --- a/lib/SILOptimizer/Utils/InstOptUtils.cpp +++ b/lib/SILOptimizer/Utils/InstOptUtils.cpp @@ -1803,8 +1803,10 @@ SILValue swift::makeValueAvailable(SILValue value, SILBasicBlock *inBlock) { bool swift::tryEliminateOnlyOwnershipUsedForwardingInst( SingleValueInstruction *forwardingInst, InstModCallbacks &callbacks) { - if (!ForwardingInstruction::isa(forwardingInst) || - isa(forwardingInst)) + if (!ForwardingInstruction::isa(forwardingInst)) + return false; + + if (ForwardingInstruction::canForwardAllOperands(forwardingInst)) return false; SmallVector worklist(getNonDebugUses(forwardingInst));