diff --git a/docs/SIL.rst b/docs/SIL.rst index a258ddd10f8..4406173136d 100644 --- a/docs/SIL.rst +++ b/docs/SIL.rst @@ -3799,6 +3799,20 @@ address of the allocated memory. on stack discipline. The corresponding stack deallocation instruction is ``dealloc_pack``. +alloc_pack_metadata +``````````````````` + +:: + + sil-instruction ::= 'alloc_pack_metadata' $() + +Inserted as the last SIL lowering pass of IRGen, indicates that the next instruction +may have on-stack pack metadata allocated on its behalf. + +Notionally, ``alloc_pack_metadata`` is a stack allocation instruction. See the +section above on stack discipline. The corresponding stack deallocation +instruction is ``dealloc_pack_metadata``. + alloc_ref ````````` :: @@ -4056,6 +4070,23 @@ prior to being deallocated. on Stack Discipline above. The operand must be an ``alloc_pack`` instruction. +dealloc_pack_metadata +````````````````````` + +:: + + sil-instruction ::= 'dealloc_pack_metadata' sil-operand + + dealloc_pack_metadata $0 : $*() + +Inserted as the last SIL lowering pass of IRGen, indicates that the on-stack +pack metadata emitted on behalf of its operand (actually on behalf of the +instruction after its operand) must be cleaned up here. + +``dealloc_pack_metadata`` is a stack deallocation instruction. See the section +on Stack Discipline above. The operand must be an ``alloc_pack_metadata`` +instruction. + dealloc_box ``````````` :: diff --git a/include/swift/SIL/SILBuilder.h b/include/swift/SIL/SILBuilder.h index 644c1357389..563ba84d69b 100644 --- a/include/swift/SIL/SILBuilder.h +++ b/include/swift/SIL/SILBuilder.h @@ -436,6 +436,15 @@ public: return insert(AllocPackInst::create(getSILDebugLocation(loc), packType, getFunction())); } + AllocPackMetadataInst * + createAllocPackMetadata(SILLocation loc, + Optional elementType = llvm::None) { + return insert(new (getModule()) AllocPackMetadataInst( + getSILDebugLocation(loc), + elementType.value_or( + SILType::getEmptyTupleType(getModule().getASTContext()) + .getAddressType()))); + } AllocRefInst *createAllocRef(SILLocation Loc, SILType ObjectType, bool objc, bool canAllocOnStack, @@ -2222,6 +2231,11 @@ public: return insert(new (getModule()) DeallocPackInst(getSILDebugLocation(loc), operand)); } + DeallocPackMetadataInst *createDeallocPackMetadata(SILLocation loc, + SILValue alloc) { + return insert(new (getModule()) + DeallocPackMetadataInst(getSILDebugLocation(loc), alloc)); + } DeallocStackRefInst *createDeallocStackRef(SILLocation Loc, SILValue operand) { return insert(new (getModule()) diff --git a/include/swift/SIL/SILCloner.h b/include/swift/SIL/SILCloner.h index f20d05da4d5..518c37e449a 100644 --- a/include/swift/SIL/SILCloner.h +++ b/include/swift/SIL/SILCloner.h @@ -892,6 +892,14 @@ SILCloner::visitAllocStackInst(AllocStackInst *Inst) { recordClonedInstruction(Inst, NewInst); } +template +void SILCloner::visitAllocPackMetadataInst( + AllocPackMetadataInst *Inst) { + getBuilder().setCurrentDebugScope(getOpScope(Inst->getDebugScope())); + recordClonedInstruction(Inst, getBuilder().createAllocPackMetadata( + getOpLocation(Inst->getLoc()))); +} + template void SILCloner::visitAllocPackInst(AllocPackInst *Inst) { @@ -2856,6 +2864,15 @@ SILCloner::visitDeallocPackInst(DeallocPackInst *Inst) { getOpValue(Inst->getOperand()))); } +template +void SILCloner::visitDeallocPackMetadataInst( + DeallocPackMetadataInst *Inst) { + getBuilder().setCurrentDebugScope(getOpScope(Inst->getDebugScope())); + recordClonedInstruction( + Inst, getBuilder().createDeallocPackMetadata( + getOpLocation(Inst->getLoc()), Inst->getOperand())); +} + template void SILCloner::visitDeallocRefInst(DeallocRefInst *Inst) { diff --git a/include/swift/SIL/SILInstruction.h b/include/swift/SIL/SILInstruction.h index b09640a26ed..63ad3f1a6df 100644 --- a/include/swift/SIL/SILInstruction.h +++ b/include/swift/SIL/SILInstruction.h @@ -821,6 +821,10 @@ public: /// The first operand must be the allocating instruction. bool isDeallocatingStack() const; + /// Whether IRGen lowering of this instruction may result in emitting packs of + /// metadata or witness tables. + bool mayRequirePackMetadata() const; + /// Create a new copy of this instruction, which retains all of the operands /// and other information of this one. If an insertion point is specified, /// then the new instruction is inserted before the specified point, otherwise @@ -2275,6 +2279,27 @@ public: } }; +/// AllocPackMetadataInst - Marker instruction indicating that the next +/// instruction might allocate on-stack pack metadata +/// during IRGen. +/// +/// Only valid in lowered SIL. +class AllocPackMetadataInst final + : public NullaryInstructionWithTypeDependentOperandsBase< + SILInstructionKind::AllocPackMetadataInst, AllocPackMetadataInst, + AllocationInst> { + friend SILBuilder; + + AllocPackMetadataInst(SILDebugLocation loc, SILType elementType) + : NullaryInstructionWithTypeDependentOperandsBase( + loc, {}, elementType.getAddressType()) {} + +public: + /// The instruction which may trigger on-stack pack metadata when IRGen + /// lowering. + SILInstruction *getIntroducer() { return getNextInstruction(); } +}; + /// The base class for AllocRefInst and AllocRefDynamicInst. /// /// The first NumTailTypes operands are counts for the tail allocated @@ -8685,6 +8710,27 @@ class DeallocPackInst : : UnaryInstructionBase(debugLoc, operand) {} }; +/// DeallocPackMetadataInst - Deallocate stack memory allocated on behalf of the +/// operand by IRGen. +/// +/// Only valid in lowered SIL. +class DeallocPackMetadataInst final + : public UnaryInstructionBase { + friend SILBuilder; + + DeallocPackMetadataInst(SILDebugLocation debugLoc, SILValue alloc) + : UnaryInstructionBase(debugLoc, alloc) {} + +public: + AllocPackMetadataInst *getAllocation() { + return cast(getOperand().getDefiningInstruction()); + } + /// The instruction which may trigger on-stack pack metadata when IRGen + /// lowering. + SILInstruction *getIntroducer() { return getAllocation()->getIntroducer(); } +}; + /// Like DeallocStackInst, but for `alloc_ref [stack]`. class DeallocStackRefInst : public UnaryInstructionBase> &TailArrays, @@ -5689,6 +5693,10 @@ void IRGenSILFunction::visitDeallocPackInst(swift::DeallocPackInst *i) { deallocatePack(*this, stackAddr, allocatedType); } +void IRGenSILFunction::visitDeallocPackMetadataInst( + DeallocPackMetadataInst *i) { +} + void IRGenSILFunction::visitDeallocRefInst(swift::DeallocRefInst *i) { // Lower the operand. Explosion self = getLoweredExplosion(i->getOperand()); diff --git a/lib/SIL/IR/OperandOwnership.cpp b/lib/SIL/IR/OperandOwnership.cpp index 47e6456091d..32153e3ddb8 100644 --- a/lib/SIL/IR/OperandOwnership.cpp +++ b/lib/SIL/IR/OperandOwnership.cpp @@ -99,6 +99,7 @@ SHOULD_NEVER_VISIT_INST(AllocExistentialBox) SHOULD_NEVER_VISIT_INST(AllocGlobal) SHOULD_NEVER_VISIT_INST(AllocStack) SHOULD_NEVER_VISIT_INST(AllocPack) +SHOULD_NEVER_VISIT_INST(AllocPackMetadata) SHOULD_NEVER_VISIT_INST(PackLength) SHOULD_NEVER_VISIT_INST(DifferentiabilityWitnessFunction) SHOULD_NEVER_VISIT_INST(FloatLiteral) @@ -158,6 +159,7 @@ OPERAND_OWNERSHIP(TrivialUse, ExplicitCopyAddr) OPERAND_OWNERSHIP(TrivialUse, MarkUnresolvedMoveAddr) OPERAND_OWNERSHIP(TrivialUse, DeallocStack) OPERAND_OWNERSHIP(TrivialUse, DeallocPack) +OPERAND_OWNERSHIP(TrivialUse, DeallocPackMetadata) OPERAND_OWNERSHIP(TrivialUse, DeinitExistentialAddr) OPERAND_OWNERSHIP(TrivialUse, DestroyAddr) OPERAND_OWNERSHIP(TrivialUse, EndAccess) diff --git a/lib/SIL/IR/SILInstruction.cpp b/lib/SIL/IR/SILInstruction.cpp index f26b368346c..8c43162a976 100644 --- a/lib/SIL/IR/SILInstruction.cpp +++ b/lib/SIL/IR/SILInstruction.cpp @@ -1247,7 +1247,8 @@ namespace { bool SILInstruction::isAllocatingStack() const { if (isa(this) || - isa(this)) + isa(this) || + isa(this)) return true; if (auto *ARI = dyn_cast(this)) { @@ -1278,7 +1279,8 @@ bool SILInstruction::isAllocatingStack() const { bool SILInstruction::isDeallocatingStack() const { if (isa(this) || isa(this) || - isa(this)) + isa(this) || + isa(this)) return true; if (auto *BI = dyn_cast(this)) { @@ -1290,6 +1292,24 @@ bool SILInstruction::isDeallocatingStack() const { return false; } +bool SILInstruction::mayRequirePackMetadata() const { + switch (getKind()) { + case SILInstructionKind::AllocPackInst: + case SILInstructionKind::PartialApplyInst: + case SILInstructionKind::ApplyInst: + case SILInstructionKind::BeginApplyInst: + case SILInstructionKind::TryApplyInst: + case SILInstructionKind::DebugValueInst: + case SILInstructionKind::MetatypeInst: + case SILInstructionKind::TuplePackElementAddrInst: + case SILInstructionKind::OpenPackElementInst: + case SILInstructionKind::ClassMethodInst: + case SILInstructionKind::WitnessMethodInst: + return true; + default: + return false; + } +} /// Create a new copy of this instruction, which retains all of the operands /// and other information of this one. If an insertion point is specified, diff --git a/lib/SIL/IR/SILPrinter.cpp b/lib/SIL/IR/SILPrinter.cpp index 3f5f1a974dd..89b09af061a 100644 --- a/lib/SIL/IR/SILPrinter.cpp +++ b/lib/SIL/IR/SILPrinter.cpp @@ -1424,6 +1424,9 @@ public: void visitAllocPackInst(AllocPackInst *API) { *this << API->getType().getObjectType(); } + void visitAllocPackMetadataInst(AllocPackMetadataInst *APMI) { + *this << APMI->getType().getObjectType(); + } void printAllocRefInstBase(AllocRefInstBase *ARI) { if (ARI->isObjC()) @@ -2453,6 +2456,9 @@ public: void visitDeallocPackInst(DeallocPackInst *DI) { *this << getIDAndType(DI->getOperand()); } + void visitDeallocPackMetadataInst(DeallocPackMetadataInst *DPMI) { + *this << getIDAndType(DPMI->getOperand()); + } void visitDeallocStackRefInst(DeallocStackRefInst *ESRL) { *this << getIDAndType(ESRL->getOperand()); } diff --git a/lib/SIL/IR/ValueOwnership.cpp b/lib/SIL/IR/ValueOwnership.cpp index 1d5898202fa..4d9732ecbd3 100644 --- a/lib/SIL/IR/ValueOwnership.cpp +++ b/lib/SIL/IR/ValueOwnership.cpp @@ -94,6 +94,7 @@ CONSTANT_OWNERSHIP_INST(Owned, ObjCMetatypeToObject) CONSTANT_OWNERSHIP_INST(None, AddressToPointer) CONSTANT_OWNERSHIP_INST(None, AllocStack) CONSTANT_OWNERSHIP_INST(None, AllocPack) +CONSTANT_OWNERSHIP_INST(None, AllocPackMetadata) CONSTANT_OWNERSHIP_INST(None, PackLength) CONSTANT_OWNERSHIP_INST(None, BeginAccess) CONSTANT_OWNERSHIP_INST(None, BindMemory) diff --git a/lib/SIL/Parser/ParseSIL.cpp b/lib/SIL/Parser/ParseSIL.cpp index be1cb1fd412..19c91b00b1e 100644 --- a/lib/SIL/Parser/ParseSIL.cpp +++ b/lib/SIL/Parser/ParseSIL.cpp @@ -4770,6 +4770,14 @@ bool SILParser::parseSpecificSILInstruction(SILBuilder &B, ResultVal = B.createAllocPack(InstLoc, Ty); break; } + case SILInstructionKind::AllocPackMetadataInst: { + SILType Ty; + if (parseSILType(Ty)) + return true; + + ResultVal = B.createAllocPackMetadata(InstLoc, Ty); + break; + } case SILInstructionKind::AllocStackInst: { bool hasDynamicLifetime = false; bool isLexical = false; @@ -4898,6 +4906,11 @@ bool SILParser::parseSpecificSILInstruction(SILBuilder &B, return true; ResultVal = B.createDeallocPack(InstLoc, Val); break; + case SILInstructionKind::DeallocPackMetadataInst: + if (parseTypedValueRef(Val, B) || parseSILDebugLocation(InstLoc, B)) + return true; + ResultVal = B.createDeallocPackMetadata(InstLoc, Val); + break; case SILInstructionKind::DeallocRefInst: { if (parseTypedValueRef(Val, B) || parseSILDebugLocation(InstLoc, B)) return true; diff --git a/lib/SIL/Utils/InstructionUtils.cpp b/lib/SIL/Utils/InstructionUtils.cpp index fef40e264b4..f118afb8dfb 100644 --- a/lib/SIL/Utils/InstructionUtils.cpp +++ b/lib/SIL/Utils/InstructionUtils.cpp @@ -484,6 +484,9 @@ RuntimeEffect swift::getRuntimeEffect(SILInstruction *inst, SILType &impactType) case SILInstructionKind::DeallocStackInst: case SILInstructionKind::DeallocStackRefInst: case SILInstructionKind::DeallocPackInst: + // This instruction just destroys stack allocations where metadata pointers + // have been stored. + case SILInstructionKind::DeallocPackMetadataInst: case SILInstructionKind::AutoreleaseValueInst: case SILInstructionKind::BindMemoryInst: case SILInstructionKind::RebindMemoryInst: @@ -631,6 +634,10 @@ RuntimeEffect swift::getRuntimeEffect(SILInstruction *inst, SILType &impactType) case SILInstructionKind::AllocPackInst: // Just conservatively assume this has metadata impact. return RuntimeEffect::MetaData; + case SILInstructionKind::AllocPackMetadataInst: + // Currently this instruction has no effect but in the fullness of time it + // will have a metadata effect. + return RuntimeEffect::MetaData; case SILInstructionKind::AllocStackInst: case SILInstructionKind::ProjectBoxInst: diff --git a/lib/SIL/Verifier/SILVerifier.cpp b/lib/SIL/Verifier/SILVerifier.cpp index 9c1f7966a61..38e7075f385 100644 --- a/lib/SIL/Verifier/SILVerifier.cpp +++ b/lib/SIL/Verifier/SILVerifier.cpp @@ -6026,6 +6026,19 @@ public: "Result and operand must have the same type, today."); } + void checkAllocPackMetadataInst(AllocPackMetadataInst *apmi) { + require(apmi->getIntroducer()->mayRequirePackMetadata(), + "Introduces instruction of kind which cannot emit on-stack pack " + "metadata"); + } + + void checkDeallocPackMetadataInst(DeallocPackMetadataInst *dpmi) { + auto *apmi = dpmi->getOperand()->getDefiningInstruction(); + require(apmi, "Must have instruction operand."); + require(isa(apmi), + "Must have alloc_pack_metadata operand"); + } + void verifyEpilogBlocks(SILFunction *F) { bool FoundReturnBlock = false; bool FoundThrowBlock = false; diff --git a/lib/SILOptimizer/UtilityPasses/SerializeSILPass.cpp b/lib/SILOptimizer/UtilityPasses/SerializeSILPass.cpp index 12b3ba9d690..9cf6f15c673 100644 --- a/lib/SILOptimizer/UtilityPasses/SerializeSILPass.cpp +++ b/lib/SILOptimizer/UtilityPasses/SerializeSILPass.cpp @@ -149,6 +149,7 @@ static bool hasOpaqueArchetype(TypeExpansionContext context, switch (inst.getKind()) { case SILInstructionKind::AllocStackInst: case SILInstructionKind::AllocPackInst: + case SILInstructionKind::AllocPackMetadataInst: case SILInstructionKind::AllocRefInst: case SILInstructionKind::AllocRefDynamicInst: case SILInstructionKind::AllocBoxInst: @@ -281,6 +282,7 @@ static bool hasOpaqueArchetype(TypeExpansionContext context, case SILInstructionKind::DeallocStackInst: case SILInstructionKind::DeallocStackRefInst: case SILInstructionKind::DeallocPackInst: + case SILInstructionKind::DeallocPackMetadataInst: case SILInstructionKind::DeallocRefInst: case SILInstructionKind::DeallocPartialRefInst: case SILInstructionKind::DeallocBoxInst: diff --git a/lib/SILOptimizer/Utils/SILInliner.cpp b/lib/SILOptimizer/Utils/SILInliner.cpp index 1861648efec..aa6901d7a2b 100644 --- a/lib/SILOptimizer/Utils/SILInliner.cpp +++ b/lib/SILOptimizer/Utils/SILInliner.cpp @@ -1020,6 +1020,7 @@ InlineCost swift::instructionInlineCost(SILInstruction &I) { case SILInstructionKind::AllocRefDynamicInst: case SILInstructionKind::AllocStackInst: case SILInstructionKind::AllocPackInst: + case SILInstructionKind::AllocPackMetadataInst: case SILInstructionKind::BeginApplyInst: case SILInstructionKind::ValueMetatypeInst: case SILInstructionKind::WitnessMethodInst: @@ -1048,6 +1049,7 @@ InlineCost swift::instructionInlineCost(SILInstruction &I) { case SILInstructionKind::DeallocPartialRefInst: case SILInstructionKind::DeallocStackInst: case SILInstructionKind::DeallocPackInst: + case SILInstructionKind::DeallocPackMetadataInst: case SILInstructionKind::DeinitExistentialAddrInst: case SILInstructionKind::DeinitExistentialValueInst: case SILInstructionKind::DestroyAddrInst: diff --git a/lib/SILOptimizer/Utils/StackNesting.cpp b/lib/SILOptimizer/Utils/StackNesting.cpp index 1ec2679a818..bc388dd3fd7 100644 --- a/lib/SILOptimizer/Utils/StackNesting.cpp +++ b/lib/SILOptimizer/Utils/StackNesting.cpp @@ -230,6 +230,9 @@ static SILInstruction *createDealloc(SILInstruction *Alloc, SILType::getEmptyTupleType(context), SubstitutionMap(), {bi}); } + case SILInstructionKind::AllocPackMetadataInst: + return B.createDeallocPackMetadata(Location, + cast(Alloc)); default: llvm_unreachable("unknown stack allocation"); } diff --git a/lib/Serialization/DeserializeSIL.cpp b/lib/Serialization/DeserializeSIL.cpp index 94b4bf5b56a..57ac2f374be 100644 --- a/lib/Serialization/DeserializeSIL.cpp +++ b/lib/Serialization/DeserializeSIL.cpp @@ -1324,6 +1324,8 @@ bool SILDeserializer::readSILInstruction(SILFunction *Fn, case SILInstructionKind::DebugValueInst: case SILInstructionKind::DebugStepInst: case SILInstructionKind::TestSpecificationInst: + case SILInstructionKind::AllocPackMetadataInst: + case SILInstructionKind::DeallocPackMetadataInst: llvm_unreachable("not supported"); case SILInstructionKind::AllocBoxInst: { diff --git a/lib/Serialization/ModuleFormat.h b/lib/Serialization/ModuleFormat.h index 766faeaecdf..038c8722803 100644 --- a/lib/Serialization/ModuleFormat.h +++ b/lib/Serialization/ModuleFormat.h @@ -58,7 +58,7 @@ const uint16_t SWIFTMODULE_VERSION_MAJOR = 0; /// describe what change you made. The content of this comment isn't important; /// it just ensures a conflict if two people change the module format. /// Don't worry about adhering to the 80-column limit for this line. -const uint16_t SWIFTMODULE_VERSION_MINOR = 785; // pointer_escape SIL flag on move_value/begin_borrow/alloc_box +const uint16_t SWIFTMODULE_VERSION_MINOR = 786; // de/alloc_pack_metadata /// A standard hash seed used for all string hashes in a serialized module. /// diff --git a/lib/Serialization/SerializeSIL.cpp b/lib/Serialization/SerializeSIL.cpp index 598eca13661..81824b38f93 100644 --- a/lib/Serialization/SerializeSIL.cpp +++ b/lib/Serialization/SerializeSIL.cpp @@ -925,6 +925,11 @@ void SILSerializer::writeSILInstruction(const SILInstruction &SI) { case SILInstructionKind::TestSpecificationInst: // Instruction exists only for tests. Ignore it. return; + case SILInstructionKind::AllocPackMetadataInst: + case SILInstructionKind::DeallocPackMetadataInst: + // Shoulud never be serialized: only introduced in an IRGen pass + // (PackMetadataMarkerInserter). + return; case SILInstructionKind::UnwindInst: case SILInstructionKind::UnreachableInst: { diff --git a/test/SIL/Parser/pack_metadata.sil b/test/SIL/Parser/pack_metadata.sil new file mode 100644 index 00000000000..c7e741eead8 --- /dev/null +++ b/test/SIL/Parser/pack_metadata.sil @@ -0,0 +1,14 @@ +// RUN: %target-sil-opt -enable-sil-verify-all=true %s | %target-sil-opt -enable-sil-verify-all=true | %FileCheck %s + +// CHECK-LABEL: sil @markers : {{.*}} { +// CHECK: [[MARKER:%[^,]+]] = alloc_pack_metadata +// CHECK: dealloc_pack_metadata [[MARKER]] +// CHECK-LABEL: } // end sil function 'markers' +sil @markers : $() -> () { +entry: + %a = alloc_pack_metadata $() + apply undef() : $@convention(thin) () -> () + dealloc_pack_metadata %a : $*() + %retval = tuple () + return %retval : $() +} diff --git a/test/SIL/verifier_failures.sil b/test/SIL/verifier_failures.sil index 200dfaeb394..4ac6ab8b03c 100644 --- a/test/SIL/verifier_failures.sil +++ b/test/SIL/verifier_failures.sil @@ -17,3 +17,28 @@ sil [ossa] @end_borrow_1_addr_alloc_stack : $@convention(thin) () -> () { %retval = tuple () return %retval : $() } + +// CHECK-LABEL: Begin Error in function alloc_pack_metadata_before_tuple +// CHECK: SIL verification failed: Introduces instruction of kind which cannot emit on-stack pack metadata: +// CHECK-LABEL: End Error in function alloc_pack_metadata_before_tuple +sil @alloc_pack_metadata_before_tuple : $@convention(thin) () -> () { + %marker = alloc_pack_metadata $() + %retval = tuple () + dealloc_pack_metadata %marker : $*() + return %retval : $() +} + +// CHECK-LABEL: Begin Error in function dealloc_pack_metadata_with_bad_operand +// CHECK: SIL verification failed: stack dealloc does not match most recent stack alloc: +// CHECK-LABEL: End Error in function dealloc_pack_metadata_with_bad_operand +// CHECK-LABEL: Begin Error in function dealloc_pack_metadata_with_bad_operand +// CHECK: SIL verification failed: Must have alloc_pack_metadata operand +// CHECK-LABEL: End Error in function dealloc_pack_metadata_with_bad_operand +sil @dealloc_pack_metadata_with_bad_operand : $@convention(thin) () -> () { + %marker = alloc_pack_metadata $() + // To make the %marker legal. + %out = apply undef() : $@convention(thin) () -> () + dealloc_pack_metadata %out : $() + %retval = tuple () + return %retval : $() +}