mirror of
https://github.com/apple/swift.git
synced 2025-12-14 20:36:38 +01:00
fd369cd050
Made the code a bit more readable by adding a reader and writer type that encapsulate the layout string and offset.
750 lines
28 KiB
C++
750 lines
28 KiB
C++
//===--- RuntimeValueWitness.cpp - Value Witness Runtime Implementation---===//
|
|
//
|
|
// This source file is part of the Swift.org open source project
|
|
//
|
|
// Copyright (c) 2014 - 2017 Apple Inc. and the Swift project authors
|
|
// Licensed under Apache License v2.0 with Runtime Library Exception
|
|
//
|
|
// See https://swift.org/LICENSE.txt for license information
|
|
// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors
|
|
//
|
|
//===----------------------------------------------------------------------===//
|
|
//
|
|
// Implementations of runtime determined value witness functions
|
|
// This file is intended to be statically linked into executables until it is
|
|
// fully added to the runtime.
|
|
//
|
|
//===----------------------------------------------------------------------===//
|
|
|
|
#include "BytecodeLayouts.h"
|
|
#include "../SwiftShims/swift/shims/HeapObject.h"
|
|
#include "WeakReference.h"
|
|
#include "swift/ABI/MetadataValues.h"
|
|
#include "swift/ABI/System.h"
|
|
#include "swift/Runtime/Error.h"
|
|
#include "swift/Runtime/HeapObject.h"
|
|
#include "llvm/Support/SwapByteOrder.h"
|
|
#include <cstdint>
|
|
#include <limits>
|
|
#include <type_traits>
|
|
#if SWIFT_OBJC_INTEROP
|
|
#include "swift/Runtime/ObjCBridge.h"
|
|
#include <Block.h>
|
|
#endif
|
|
#if SWIFT_PTRAUTH
|
|
#include <ptrauth.h>
|
|
#endif
|
|
|
|
using namespace swift;
|
|
|
|
static Metadata *getExistentialTypeMetadata(OpaqueValue *object) {
|
|
return reinterpret_cast<Metadata**>(object)[NumWords_ValueBuffer];
|
|
}
|
|
|
|
template <typename FnTy>
|
|
static const FnTy readRelativeFunctionPointer(LayoutStringReader &reader) {
|
|
static_assert(std::is_pointer<FnTy>::value);
|
|
|
|
auto absolute = reader.layoutStr + reader.offset;
|
|
auto relativeOffset =
|
|
(uintptr_t)(intptr_t)(int32_t)reader.readBytes<intptr_t>();
|
|
FnTy fn;
|
|
|
|
#if SWIFT_PTRAUTH
|
|
fn = (FnTy)ptrauth_sign_unauthenticated(
|
|
(void *)((uintptr_t)absolute + relativeOffset),
|
|
ptrauth_key_function_pointer, 0);
|
|
#else
|
|
fn = (FnTy)((uintptr_t)absolute + relativeOffset);
|
|
#endif
|
|
|
|
return fn;
|
|
}
|
|
|
|
typedef Metadata *(*MetadataAccessor)(const Metadata *const *);
|
|
|
|
static const Metadata *getResilientTypeMetadata(const Metadata *metadata,
|
|
LayoutStringReader &reader) {
|
|
auto fn = readRelativeFunctionPointer<MetadataAccessor>(reader);
|
|
return fn(metadata->getGenericArgs());
|
|
}
|
|
|
|
typedef void (*DestrFn)(void*);
|
|
|
|
struct DestroyFuncAndMask {
|
|
DestrFn fn;
|
|
bool isIndirect;
|
|
};
|
|
|
|
static void skipDestroy(void* ignore) { }
|
|
|
|
static void existential_destroy(OpaqueValue* object) {
|
|
auto* metadata = getExistentialTypeMetadata(object);
|
|
if (metadata->getValueWitnesses()->isValueInline()) {
|
|
metadata->vw_destroy(object);
|
|
} else {
|
|
swift_release(*(HeapObject**)object);
|
|
}
|
|
}
|
|
|
|
template <typename Handler, typename... Params>
|
|
inline static bool handleNextRefCount(const Metadata *metadata,
|
|
LayoutStringReader &reader,
|
|
uintptr_t &addrOffset, Params... params) {
|
|
uint64_t skip = reader.readBytes<uint64_t>();
|
|
auto tag = static_cast<RefCountingKind>(skip >> 56);
|
|
skip &= ~(0xffULL << 56);
|
|
addrOffset += skip;
|
|
|
|
if (SWIFT_UNLIKELY(tag == RefCountingKind::End)) {
|
|
return false;
|
|
} else if (SWIFT_UNLIKELY(tag == RefCountingKind::Metatype)) {
|
|
auto *type = reader.readBytes<const Metadata *>();
|
|
Handler::handleMetatype(type, addrOffset, std::forward<Params>(params)...);
|
|
} else if (SWIFT_UNLIKELY(tag == RefCountingKind::Resilient)) {
|
|
auto *type = getResilientTypeMetadata(metadata, reader);
|
|
Handler::handleMetatype(type, addrOffset, std::forward<Params>(params)...);
|
|
} else if (SWIFT_UNLIKELY(tag ==
|
|
RefCountingKind::SinglePayloadEnumSimple)) {
|
|
Handler::handleSinglePayloadEnumSimple(reader, addrOffset,
|
|
std::forward<Params>(params)...);
|
|
} else if (SWIFT_UNLIKELY(tag == RefCountingKind::SinglePayloadEnumFN)) {
|
|
Handler::handleSinglePayloadEnumFN(reader, false, addrOffset,
|
|
std::forward<Params>(params)...);
|
|
} else if (SWIFT_UNLIKELY(tag ==
|
|
RefCountingKind::SinglePayloadEnumFNResolved)) {
|
|
Handler::handleSinglePayloadEnumFN(reader, true, addrOffset,
|
|
std::forward<Params>(params)...);
|
|
} else if (SWIFT_UNLIKELY(tag == RefCountingKind::SinglePayloadEnumGeneric)) {
|
|
Handler::handleSinglePayloadEnumGeneric(reader, addrOffset,
|
|
std::forward<Params>(params)...);
|
|
} else if (SWIFT_UNLIKELY(tag == RefCountingKind::MultiPayloadEnumFN)) {
|
|
Handler::handleMultiPayloadEnumFN(metadata, reader, false, addrOffset,
|
|
std::forward<Params>(params)...);
|
|
} else if (SWIFT_UNLIKELY(tag ==
|
|
RefCountingKind::MultiPayloadEnumFNResolved)) {
|
|
Handler::handleMultiPayloadEnumFN(metadata, reader, true, addrOffset,
|
|
std::forward<Params>(params)...);
|
|
} else if (SWIFT_UNLIKELY(tag == RefCountingKind::MultiPayloadEnumGeneric)) {
|
|
Handler::handleMultiPayloadEnumGeneric(metadata, reader, addrOffset,
|
|
std::forward<Params>(params)...);
|
|
} else {
|
|
Handler::handleReference(tag, addrOffset, std::forward<Params>(params)...);
|
|
}
|
|
|
|
return true;
|
|
}
|
|
|
|
template <unsigned N, typename Handler, typename... Params>
|
|
inline static void handleRefCounts(const Metadata *metadata,
|
|
LayoutStringReader &reader,
|
|
uintptr_t &addrOffset, Params... params) {
|
|
if (N == 0) {
|
|
while (handleNextRefCount<Handler>(metadata, reader, addrOffset,
|
|
std::forward<Params>(params)...)) {
|
|
}
|
|
} else {
|
|
for (int i = 0; i < N; i++) {
|
|
handleNextRefCount<Handler>(metadata, reader, addrOffset,
|
|
std::forward<Params>(params)...);
|
|
}
|
|
}
|
|
}
|
|
|
|
template <unsigned N, typename Handler, typename... Params>
|
|
inline static void handleRefCounts(const Metadata *metadata, Params... params) {
|
|
LayoutStringReader reader{metadata->getLayoutString(),
|
|
layoutStringHeaderSize};
|
|
uintptr_t addrOffset = 0;
|
|
handleRefCounts<N, Handler>(metadata, reader, addrOffset,
|
|
std::forward<Params>(params)...);
|
|
}
|
|
|
|
static uint64_t readTagBytes(uint8_t *addr, uint8_t byteCount) {
|
|
switch (byteCount) {
|
|
case 1:
|
|
return addr[0];
|
|
case 2: {
|
|
uint16_t res = 0;
|
|
memcpy(&res, addr, sizeof(uint16_t));
|
|
return res;
|
|
}
|
|
case 4: {
|
|
uint32_t res = 0;
|
|
memcpy(&res, addr, sizeof(uint32_t));
|
|
return res;
|
|
}
|
|
case 8: {
|
|
uint64_t res = 0;
|
|
memcpy(&res, addr, sizeof(uint64_t));
|
|
return res;
|
|
}
|
|
default:
|
|
swift_unreachable("Unsupported tag byte length.");
|
|
}
|
|
}
|
|
|
|
static void handleSinglePayloadEnumSimple(LayoutStringReader &reader,
|
|
uint8_t *addr,
|
|
uintptr_t &addrOffset) {
|
|
auto byteCountsAndOffset = reader.readBytes<uint64_t>();
|
|
auto extraTagBytesPattern = (uint8_t)(byteCountsAndOffset >> 62);
|
|
auto xiTagBytesPattern = ((uint8_t)(byteCountsAndOffset >> 59)) & 0x7;
|
|
auto xiTagBytesOffset =
|
|
byteCountsAndOffset & std::numeric_limits<uint32_t>::max();
|
|
|
|
if (extraTagBytesPattern) {
|
|
auto extraTagBytes = 1 << (extraTagBytesPattern - 1);
|
|
auto payloadSize = reader.readBytes<size_t>();
|
|
auto tagBytes =
|
|
readTagBytes(addr + addrOffset + payloadSize, extraTagBytes);
|
|
if (tagBytes) {
|
|
reader.skip(sizeof(uint64_t) + sizeof(size_t));
|
|
goto noPayload;
|
|
}
|
|
} else {
|
|
reader.skip(sizeof(size_t));
|
|
}
|
|
|
|
if (xiTagBytesPattern) {
|
|
auto zeroTagValue = reader.readBytes<uint64_t>();
|
|
auto xiTagValues = reader.readBytes<size_t>();
|
|
|
|
auto xiTagBytes = 1 << (xiTagBytesPattern - 1);
|
|
uint64_t tagBytes =
|
|
readTagBytes(addr + addrOffset + xiTagBytesOffset, xiTagBytes) -
|
|
zeroTagValue;
|
|
if (tagBytes >= xiTagValues) {
|
|
reader.skip(sizeof(size_t) * 2);
|
|
return;
|
|
}
|
|
} else {
|
|
reader.skip(sizeof(uint64_t) + sizeof(size_t));
|
|
}
|
|
|
|
noPayload:
|
|
auto refCountBytes = reader.readBytes<size_t>();
|
|
auto skip = reader.readBytes<size_t>();
|
|
reader.skip(refCountBytes);
|
|
addrOffset += skip;
|
|
}
|
|
|
|
typedef unsigned (*GetEnumTagFn)(const uint8_t *);
|
|
|
|
static void handleSinglePayloadEnumFN(LayoutStringReader &reader, bool resolved,
|
|
uint8_t *addr, uintptr_t &addrOffset) {
|
|
GetEnumTagFn getEnumTag;
|
|
if (resolved) {
|
|
getEnumTag = reader.readBytes<GetEnumTagFn>();
|
|
} else {
|
|
getEnumTag = readRelativeFunctionPointer<GetEnumTagFn>(reader);
|
|
}
|
|
|
|
unsigned enumTag = getEnumTag(addr + addrOffset);
|
|
|
|
if (enumTag == 0) {
|
|
reader.skip(sizeof(size_t) * 2);
|
|
} else {
|
|
auto refCountBytes = reader.readBytes<size_t>();
|
|
auto skip = reader.readBytes<size_t>();
|
|
reader.skip(refCountBytes);
|
|
addrOffset += skip;
|
|
}
|
|
}
|
|
|
|
static void handleSinglePayloadEnumGeneric(LayoutStringReader &reader,
|
|
uint8_t *addr,
|
|
uintptr_t &addrOffset) {
|
|
auto tagBytesAndOffset = reader.readBytes<uint64_t>();
|
|
auto extraTagBytesPattern = (uint8_t)(tagBytesAndOffset >> 62);
|
|
auto xiTagBytesOffset =
|
|
tagBytesAndOffset & std::numeric_limits<uint32_t>::max();
|
|
const Metadata *xiType = nullptr;
|
|
|
|
if (extraTagBytesPattern) {
|
|
auto extraTagBytes = 1 << (extraTagBytesPattern - 1);
|
|
auto payloadSize = reader.readBytes<size_t>();
|
|
auto tagBytes =
|
|
readTagBytes(addr + addrOffset + payloadSize, extraTagBytes);
|
|
if (tagBytes) {
|
|
reader.skip(sizeof(uint64_t) + sizeof(size_t));
|
|
goto noPayload;
|
|
}
|
|
} else {
|
|
reader.skip(sizeof(size_t));
|
|
}
|
|
|
|
xiType = reader.readBytes<const Metadata *>();
|
|
|
|
if (xiType) {
|
|
auto numEmptyCases = reader.readBytes<unsigned>();
|
|
|
|
auto tag = xiType->vw_getEnumTagSinglePayload(
|
|
(const OpaqueValue *)(addr + addrOffset + xiTagBytesOffset),
|
|
numEmptyCases);
|
|
if (tag == 0) {
|
|
reader.skip(sizeof(size_t) * 2);
|
|
return;
|
|
}
|
|
} else {
|
|
reader.skip(sizeof(uint64_t) + sizeof(size_t));
|
|
}
|
|
|
|
noPayload:
|
|
auto refCountBytes = reader.readBytes<size_t>();
|
|
auto skip = reader.readBytes<size_t>();
|
|
reader.skip(refCountBytes);
|
|
addrOffset += skip;
|
|
}
|
|
|
|
template <typename Handler, typename... Params>
|
|
static void handleMultiPayloadEnumFN(const Metadata *metadata,
|
|
LayoutStringReader &reader, bool resolved,
|
|
uintptr_t &addrOffset, uint8_t *addr,
|
|
Params... params) {
|
|
GetEnumTagFn getEnumTag;
|
|
if (resolved) {
|
|
getEnumTag = reader.readBytes<GetEnumTagFn>();
|
|
} else {
|
|
getEnumTag = readRelativeFunctionPointer<GetEnumTagFn>(reader);
|
|
}
|
|
|
|
size_t numPayloads = reader.readBytes<size_t>();
|
|
size_t refCountBytes = reader.readBytes<size_t>();
|
|
size_t enumSize = reader.readBytes<size_t>();
|
|
|
|
unsigned enumTag = getEnumTag(addr + addrOffset);
|
|
|
|
if (enumTag < numPayloads) {
|
|
size_t refCountOffset = reader.peekBytes<size_t>(enumTag * sizeof(size_t));
|
|
|
|
LayoutStringReader nestedReader = reader;
|
|
nestedReader.skip((numPayloads * sizeof(size_t)) + refCountOffset);
|
|
uintptr_t nestedAddrOffset = addrOffset;
|
|
handleRefCounts<0, Handler>(metadata, nestedReader, nestedAddrOffset, addr,
|
|
std::forward<Params>(params)...);
|
|
}
|
|
|
|
reader.skip(refCountBytes + (numPayloads * sizeof(size_t)));
|
|
addrOffset += enumSize;
|
|
}
|
|
|
|
template <typename Handler, typename... Params>
|
|
static void handleMultiPayloadEnumGeneric(const Metadata *metadata,
|
|
LayoutStringReader &reader,
|
|
uintptr_t &addrOffset, uint8_t *addr,
|
|
Params... params) {
|
|
auto tagBytes = reader.readBytes<size_t>();
|
|
auto numPayloads = reader.readBytes<size_t>();
|
|
auto refCountBytes = reader.readBytes<size_t>();
|
|
auto enumSize = reader.readBytes<size_t>();
|
|
auto tagBytesOffset = enumSize - tagBytes;
|
|
|
|
auto enumTag = readTagBytes(addr + addrOffset + tagBytesOffset, tagBytes);
|
|
|
|
if (enumTag < numPayloads) {
|
|
size_t refCountOffset = reader.peekBytes<size_t>(enumTag * sizeof(size_t));
|
|
|
|
LayoutStringReader nestedReader = reader;
|
|
nestedReader.skip((numPayloads * sizeof(size_t)) + refCountOffset);
|
|
uintptr_t nestedAddrOffset = addrOffset;
|
|
handleRefCounts<0, Handler>(metadata, nestedReader, nestedAddrOffset, addr,
|
|
std::forward<Params>(params)...);
|
|
}
|
|
|
|
reader.skip(refCountBytes + (numPayloads * sizeof(size_t)));
|
|
addrOffset += enumSize;
|
|
}
|
|
|
|
const DestroyFuncAndMask destroyTable[] = {
|
|
{(DestrFn)&skipDestroy, false},
|
|
{(DestrFn)&swift_errorRelease, true},
|
|
{(DestrFn)&swift_release, true},
|
|
{(DestrFn)&swift_unownedRelease, true},
|
|
{(DestrFn)&swift_weakDestroy, false},
|
|
{(DestrFn)&swift_unknownObjectRelease, true},
|
|
{(DestrFn)&swift_unknownObjectUnownedDestroy, false},
|
|
{(DestrFn)&swift_unknownObjectWeakDestroy, false},
|
|
{(DestrFn)&swift_bridgeObjectRelease, true},
|
|
#if SWIFT_OBJC_INTEROP
|
|
{(DestrFn)&_Block_release, true},
|
|
{(DestrFn)&swift_unknownObjectRelease, true},
|
|
#else
|
|
{nullptr, true},
|
|
{nullptr, true},
|
|
#endif
|
|
// TODO: how to handle Custom?
|
|
{nullptr, true},
|
|
{nullptr, true},
|
|
{nullptr, true},
|
|
{(DestrFn)&existential_destroy, false},
|
|
};
|
|
|
|
struct DestroyHandler {
|
|
static inline void handleMetatype(const Metadata *type, uintptr_t addrOffset,
|
|
uint8_t *addr) {
|
|
type->vw_destroy((OpaqueValue *)(addr + addrOffset));
|
|
}
|
|
|
|
static inline void handleSinglePayloadEnumSimple(LayoutStringReader &reader,
|
|
uintptr_t &addrOffset,
|
|
uint8_t *addr) {
|
|
::handleSinglePayloadEnumSimple(reader, addr, addrOffset);
|
|
}
|
|
|
|
static inline void handleSinglePayloadEnumFN(LayoutStringReader &reader,
|
|
bool resolved,
|
|
uintptr_t &addrOffset,
|
|
uint8_t *addr) {
|
|
::handleSinglePayloadEnumFN(reader, resolved, addr, addrOffset);
|
|
}
|
|
|
|
static inline void handleSinglePayloadEnumGeneric(LayoutStringReader &reader,
|
|
uintptr_t &addrOffset,
|
|
uint8_t *addr) {
|
|
::handleSinglePayloadEnumGeneric(reader, addr, addrOffset);
|
|
}
|
|
|
|
static inline void handleMultiPayloadEnumFN(const Metadata *metadata,
|
|
LayoutStringReader &reader,
|
|
bool resolved,
|
|
uintptr_t &addrOffset,
|
|
uint8_t *addr) {
|
|
::handleMultiPayloadEnumFN<DestroyHandler>(metadata, reader, resolved,
|
|
addrOffset, addr);
|
|
}
|
|
|
|
static inline void handleMultiPayloadEnumGeneric(const Metadata *metadata,
|
|
LayoutStringReader &reader,
|
|
uintptr_t &addrOffset,
|
|
uint8_t *addr) {
|
|
::handleMultiPayloadEnumGeneric<DestroyHandler>(metadata, reader,
|
|
addrOffset, addr);
|
|
}
|
|
|
|
static inline void handleReference(RefCountingKind tag, uintptr_t addrOffset,
|
|
uint8_t *addr) {
|
|
const auto &destroyFunc = destroyTable[static_cast<uint8_t>(tag)];
|
|
if (SWIFT_LIKELY(destroyFunc.isIndirect)) {
|
|
destroyFunc.fn(
|
|
(void *)((*(uintptr_t *)(addr + addrOffset))));
|
|
} else {
|
|
destroyFunc.fn(((void *)(addr + addrOffset)));
|
|
}
|
|
}
|
|
};
|
|
|
|
extern "C" void
|
|
swift_generic_destroy(swift::OpaqueValue *address, const Metadata *metadata) {
|
|
handleRefCounts<0, DestroyHandler>(metadata, (uint8_t *)address);
|
|
}
|
|
|
|
struct RetainFuncAndMask {
|
|
void* fn;
|
|
bool isSingle;
|
|
};
|
|
|
|
#if SWIFT_OBJC_INTEROP
|
|
void* Block_copyForwarder(void** dest, const void** src) {
|
|
*dest = _Block_copy(*src);
|
|
return *dest;
|
|
}
|
|
#endif
|
|
|
|
typedef void* (*RetainFn)(void*);
|
|
typedef void* (*CopyInitFn)(void*, void*);
|
|
|
|
void* skipRetain(void* ignore) { return nullptr; }
|
|
void* existential_initializeWithCopy(OpaqueValue* dest, OpaqueValue* src) {
|
|
auto* metadata = getExistentialTypeMetadata(src);
|
|
return metadata->vw_initializeBufferWithCopyOfBuffer((ValueBuffer*)dest,
|
|
(ValueBuffer*)src);
|
|
}
|
|
|
|
const RetainFuncAndMask retainTable[] = {
|
|
{(void*)&skipRetain, true},
|
|
{(void*)&swift_errorRetain, true},
|
|
{(void*)&swift_retain, true},
|
|
{(void*)&swift_unownedRetain, true},
|
|
{(void*)&swift_weakCopyInit, false},
|
|
{(void*)&swift_unknownObjectRetain, true},
|
|
{(void*)&swift_unknownObjectUnownedCopyInit, false},
|
|
{(void*)&swift_unknownObjectWeakCopyInit, false},
|
|
{(void*)&swift_bridgeObjectRetain, true},
|
|
#if SWIFT_OBJC_INTEROP
|
|
{(void*)&Block_copyForwarder, false},
|
|
{(void*)&objc_retain, true},
|
|
#else
|
|
{nullptr, true},
|
|
{nullptr, true},
|
|
#endif
|
|
// TODO: how to handle Custom?
|
|
{nullptr, true},
|
|
{nullptr, true},
|
|
{nullptr, true},
|
|
{(void*)&existential_initializeWithCopy, false},
|
|
};
|
|
|
|
struct CopyHandler {
|
|
static inline void handleMetatype(const Metadata *type, uintptr_t addrOffset,
|
|
uint8_t *dest, uint8_t *src) {
|
|
type->vw_initializeWithCopy((OpaqueValue*)((uintptr_t)dest + addrOffset),
|
|
(OpaqueValue*)((uintptr_t)src + addrOffset));
|
|
}
|
|
|
|
static inline void handleSinglePayloadEnumSimple(LayoutStringReader &reader,
|
|
uintptr_t &addrOffset,
|
|
uint8_t *dest,
|
|
uint8_t *src) {
|
|
::handleSinglePayloadEnumSimple(reader, src, addrOffset);
|
|
}
|
|
|
|
static inline void handleSinglePayloadEnumFN(LayoutStringReader &reader,
|
|
bool resolved,
|
|
uintptr_t &addrOffset,
|
|
uint8_t *dest, uint8_t *src) {
|
|
::handleSinglePayloadEnumFN(reader, resolved, src, addrOffset);
|
|
}
|
|
|
|
static inline void handleSinglePayloadEnumGeneric(LayoutStringReader &reader,
|
|
uintptr_t &addrOffset,
|
|
uint8_t *dest,
|
|
uint8_t *src) {
|
|
::handleSinglePayloadEnumGeneric(reader, src, addrOffset);
|
|
}
|
|
|
|
static inline void handleMultiPayloadEnumFN(const Metadata *metadata,
|
|
LayoutStringReader &reader,
|
|
bool resolved,
|
|
uintptr_t &addrOffset,
|
|
uint8_t *dest, uint8_t *src) {
|
|
::handleMultiPayloadEnumFN<CopyHandler>(metadata, reader, resolved,
|
|
addrOffset, dest, src);
|
|
}
|
|
|
|
static inline void handleMultiPayloadEnumGeneric(const Metadata *metadata,
|
|
LayoutStringReader &reader,
|
|
uintptr_t &addrOffset,
|
|
uint8_t *dest,
|
|
uint8_t *src) {
|
|
::handleMultiPayloadEnumGeneric<CopyHandler>(metadata, reader, addrOffset,
|
|
dest, src);
|
|
}
|
|
|
|
static inline void handleReference(RefCountingKind tag, uintptr_t addrOffset,
|
|
uint8_t *dest, uint8_t *src) {
|
|
const auto &retainFunc = retainTable[static_cast<uint8_t>(tag)];
|
|
if (SWIFT_LIKELY(retainFunc.isSingle)) {
|
|
((RetainFn)retainFunc.fn)(*(void**)(((uintptr_t)dest + addrOffset)));
|
|
} else {
|
|
((CopyInitFn)retainFunc.fn)((void*)((uintptr_t)dest + addrOffset),
|
|
(void*)((uintptr_t)src + addrOffset));
|
|
}
|
|
}
|
|
};
|
|
|
|
extern "C" swift::OpaqueValue *
|
|
swift_generic_initWithCopy(swift::OpaqueValue *dest, swift::OpaqueValue *src,
|
|
const Metadata *metadata) {
|
|
size_t size = metadata->vw_size();
|
|
memcpy(dest, src, size);
|
|
|
|
handleRefCounts<0, CopyHandler>(metadata, (uint8_t *)dest, (uint8_t *)src);
|
|
|
|
return dest;
|
|
}
|
|
|
|
struct TakeHandler {
|
|
static inline void handleMetatype(const Metadata *type, uintptr_t addrOffset,
|
|
uint8_t *dest, uint8_t *src) {
|
|
if (SWIFT_UNLIKELY(!type->getValueWitnesses()->isBitwiseTakable())) {
|
|
type->vw_initializeWithTake(
|
|
(OpaqueValue*)((uintptr_t)dest + addrOffset),
|
|
(OpaqueValue*)((uintptr_t)src + addrOffset));
|
|
}
|
|
}
|
|
|
|
static inline void handleSinglePayloadEnumSimple(LayoutStringReader &reader,
|
|
uintptr_t &addrOffset,
|
|
uint8_t *dest,
|
|
uint8_t *src) {
|
|
::handleSinglePayloadEnumSimple(reader, src, addrOffset);
|
|
}
|
|
|
|
static inline void handleSinglePayloadEnumFN(LayoutStringReader &reader,
|
|
bool resolved,
|
|
uintptr_t &addrOffset,
|
|
uint8_t *dest, uint8_t *src) {
|
|
::handleSinglePayloadEnumFN(reader, resolved, src, addrOffset);
|
|
}
|
|
|
|
static inline void handleSinglePayloadEnumGeneric(LayoutStringReader &reader,
|
|
uintptr_t &addrOffset,
|
|
uint8_t *dest,
|
|
uint8_t *src) {
|
|
::handleSinglePayloadEnumGeneric(reader, src, addrOffset);
|
|
}
|
|
|
|
static inline void handleMultiPayloadEnumFN(const Metadata *metadata,
|
|
LayoutStringReader &reader,
|
|
bool resolved,
|
|
uintptr_t &addrOffset,
|
|
uint8_t *dest, uint8_t *src) {
|
|
::handleMultiPayloadEnumFN<TakeHandler>(metadata, reader, resolved,
|
|
addrOffset, dest, src);
|
|
}
|
|
|
|
static inline void handleMultiPayloadEnumGeneric(const Metadata *metadata,
|
|
LayoutStringReader &reader,
|
|
uintptr_t &addrOffset,
|
|
uint8_t *dest,
|
|
uint8_t *src) {
|
|
::handleMultiPayloadEnumGeneric<TakeHandler>(metadata, reader, addrOffset,
|
|
dest, src);
|
|
}
|
|
|
|
static inline void handleReference(RefCountingKind tag, uintptr_t addrOffset,
|
|
uint8_t *dest, uint8_t *src) {
|
|
if (tag == RefCountingKind::UnknownWeak) {
|
|
swift_unknownObjectWeakTakeInit(
|
|
(WeakReference*)((uintptr_t)dest + addrOffset),
|
|
(WeakReference*)((uintptr_t)src + addrOffset));
|
|
} else if (tag == RefCountingKind::Existential) {
|
|
auto *type = getExistentialTypeMetadata(
|
|
(OpaqueValue*)((uintptr_t)src + addrOffset));
|
|
if (SWIFT_UNLIKELY(!type->getValueWitnesses()->isBitwiseTakable())) {
|
|
type->vw_initializeWithTake(
|
|
(OpaqueValue *)((uintptr_t)dest + addrOffset),
|
|
(OpaqueValue *)((uintptr_t)src + addrOffset));
|
|
}
|
|
}
|
|
}
|
|
};
|
|
|
|
extern "C" swift::OpaqueValue *
|
|
swift_generic_initWithTake(swift::OpaqueValue *dest, swift::OpaqueValue *src,
|
|
const Metadata *metadata) {
|
|
size_t size = metadata->vw_size();
|
|
|
|
memcpy(dest, src, size);
|
|
|
|
if (SWIFT_LIKELY(metadata->getValueWitnesses()->isBitwiseTakable())) {
|
|
return dest;
|
|
}
|
|
|
|
handleRefCounts<0, TakeHandler>(metadata, (uint8_t *)dest, (uint8_t *)src);
|
|
|
|
return dest;
|
|
}
|
|
|
|
extern "C" swift::OpaqueValue *
|
|
swift_generic_assignWithCopy(swift::OpaqueValue *dest, swift::OpaqueValue *src,
|
|
const Metadata *metadata) {
|
|
swift_generic_destroy(dest, metadata);
|
|
return swift_generic_initWithCopy(dest, src, metadata);
|
|
}
|
|
|
|
extern "C" swift::OpaqueValue *
|
|
swift_generic_assignWithTake(swift::OpaqueValue *dest, swift::OpaqueValue *src,
|
|
const Metadata *metadata) {
|
|
swift_generic_destroy(dest, metadata);
|
|
return swift_generic_initWithTake(dest, src, metadata);
|
|
}
|
|
|
|
void swift::swift_resolve_resilientAccessors(uint8_t *layoutStr,
|
|
size_t layoutStrOffset,
|
|
const uint8_t *fieldLayoutStr,
|
|
const Metadata *fieldType) {
|
|
LayoutStringWriter writer{layoutStr, layoutStrOffset};
|
|
LayoutStringReader reader{fieldLayoutStr, layoutStringHeaderSize};
|
|
while (true) {
|
|
size_t currentOffset = reader.offset;
|
|
uint64_t size = reader.readBytes<uint64_t>();
|
|
RefCountingKind tag = (RefCountingKind)(size >> 56);
|
|
size &= ~(0xffULL << 56);
|
|
|
|
switch (tag) {
|
|
case RefCountingKind::End:
|
|
return;
|
|
case RefCountingKind::Resilient: {
|
|
auto *type = getResilientTypeMetadata(fieldType, reader);
|
|
writer.offset = layoutStrOffset + currentOffset - layoutStringHeaderSize;
|
|
uint64_t tagAndOffset =
|
|
(((uint64_t)RefCountingKind::Metatype) << 56) | size;
|
|
writer.writeBytes(tagAndOffset);
|
|
writer.writeBytes(type);
|
|
break;
|
|
}
|
|
case RefCountingKind::Metatype:
|
|
reader.skip(sizeof(uintptr_t));
|
|
break;
|
|
case RefCountingKind::SinglePayloadEnumSimple:
|
|
reader.skip((3 * sizeof(uint64_t)) + (4 * sizeof(size_t)));
|
|
break;
|
|
|
|
case RefCountingKind::SinglePayloadEnumFN: {
|
|
auto getEnumTag = readRelativeFunctionPointer<GetEnumTagFn>(reader);
|
|
writer.offset = layoutStrOffset + currentOffset - layoutStringHeaderSize;
|
|
uint64_t tagAndOffset =
|
|
(((uint64_t)RefCountingKind::SinglePayloadEnumFNResolved) << 56) |
|
|
size;
|
|
writer.writeBytes(tagAndOffset);
|
|
writer.writeBytes(getEnumTag);
|
|
reader.skip(2 * sizeof(size_t));
|
|
break;
|
|
}
|
|
|
|
case RefCountingKind::SinglePayloadEnumFNResolved:
|
|
reader.skip(3 * sizeof(size_t));
|
|
break;
|
|
|
|
case RefCountingKind::MultiPayloadEnumFN: {
|
|
auto getEnumTag = readRelativeFunctionPointer<GetEnumTagFn>(reader);
|
|
writer.offset = layoutStrOffset + currentOffset - layoutStringHeaderSize;
|
|
uint64_t tagAndOffset =
|
|
(((uint64_t)RefCountingKind::MultiPayloadEnumFNResolved) << 56) |
|
|
size;
|
|
writer.writeBytes(tagAndOffset);
|
|
writer.writeBytes(getEnumTag);
|
|
|
|
size_t numCases = reader.readBytes<size_t>();
|
|
// skip ref count bytes
|
|
reader.skip(sizeof(size_t));
|
|
|
|
size_t casesBeginOffset =
|
|
layoutStrOffset + reader.offset + (numCases * sizeof(size_t));
|
|
|
|
for (size_t j = 0; j < numCases; j++) {
|
|
size_t caseOffset = reader.readBytes<size_t>();
|
|
const uint8_t *caseLayoutString = fieldLayoutStr + reader.offset +
|
|
(numCases * sizeof(size_t)) +
|
|
caseOffset;
|
|
swift_resolve_resilientAccessors(layoutStr,
|
|
casesBeginOffset + caseOffset,
|
|
caseLayoutString, fieldType);
|
|
}
|
|
break;
|
|
}
|
|
|
|
case RefCountingKind::MultiPayloadEnumFNResolved: {
|
|
// skip function pointer
|
|
reader.skip(sizeof(uintptr_t));
|
|
size_t numCases = reader.readBytes<size_t>();
|
|
size_t refCountBytes = reader.readBytes<size_t>();
|
|
// skip enum size, offsets and ref counts
|
|
reader.skip(sizeof(size_t) + (numCases * sizeof(size_t)) + refCountBytes);
|
|
break;
|
|
}
|
|
|
|
default:
|
|
break;
|
|
}
|
|
}
|
|
}
|
|
|
|
extern "C"
|
|
void swift_generic_instantiateLayoutString(const uint8_t* layoutStr,
|
|
Metadata* type) {
|
|
type->setLayoutString(layoutStr);
|
|
}
|