averello/swift-mirror
1
0
Fork 0
mirror of https://github.com/apple/swift.git synced 2025-12-14 20:36:38 +01:00
Code Issues Packages Projects Releases Wiki Activity
Files
17447a33783f13c63432f2bdf4df093194f0017d
swift-mirror/stdlib/public/runtime/BytecodeLayouts.cpp
Dario Rexin 923cccf1ea [Runtime] Add specialized CVW entry points for multi payload enums 
rdar://143852239

Adding these specialized entry points reduces the overhead of the witness functions by removing the first indirection.
2025-01-30 14:44:49 -08:00

2806 lines
108 KiB
C++
Raw Blame History

//===--- RuntimeValueWitness.cpp - Value Witness Runtime Implementation---===//
//
// This source file is part of the Swift.org open source project
//
// Copyright (c) 2014 - 2017 Apple Inc. and the Swift project authors
// Licensed under Apache License v2.0 with Runtime Library Exception
//
// See https://swift.org/LICENSE.txt for license information
// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors
//
//===----------------------------------------------------------------------===//
//
// Implementations of runtime determined value witness functions
// This file is intended to be statically linked into executables until it is
// fully added to the runtime.
//
//===----------------------------------------------------------------------===//
#include "BytecodeLayouts.h"
#include "../SwiftShims/swift/shims/HeapObject.h"
#include "EnumImpl.h"
#include "WeakReference.h"
#include "swift/ABI/MetadataValues.h"
#include "swift/ABI/System.h"
#include "swift/Runtime/Error.h"
#include "swift/Runtime/HeapObject.h"
#include "llvm/Support/SwapByteOrder.h"
#include <cstdint>
#include <functional>
#include <limits>
#include <optional>
#include <type_traits>
#if SWIFT_OBJC_INTEROP
#include "swift/Runtime/ObjCBridge.h"
#include <Block.h>
#endif
#if SWIFT_PTRAUTH
#include <ptrauth.h>
#endif
#include "../CompatibilityOverride/CompatibilityOverride.h"
using namespace swift;
static Metadata *getExistentialTypeMetadata(OpaqueValue *object) {
return reinterpret_cast<Metadata**>(object)[NumWords_ValueBuffer];
}
template <typename FnTy, typename Reader>
static const FnTy readRelativeFunctionPointer(Reader &reader) {
static_assert(std::is_pointer<FnTy>::value);
auto absolute = reader.getAbsolute();
auto relativeOffset =
(uintptr_t)(intptr_t)(int32_t)reader.template readBytes<intptr_t>();
FnTy fn;
#if SWIFT_PTRAUTH
fn = (FnTy)ptrauth_sign_unauthenticated(
(void *)((uintptr_t)absolute + relativeOffset),
ptrauth_key_function_pointer, 0);
#else
fn = (FnTy)((uintptr_t)absolute + relativeOffset);
#endif
return fn;
}
typedef Metadata *(*MetadataAccessor)(const Metadata *const *);
template <typename Reader>
static const Metadata *getResilientTypeMetadata(const Metadata *metadata,
Reader &reader) {
auto fn = readRelativeFunctionPointer<MetadataAccessor>(reader);
return fn(metadata->getGenericArgs());
}
static uint64_t readTagBytes(const uint8_t *addr, uint8_t byteCount) {
switch (byteCount) {
case 1:
return addr[0];
case 2: {
uint16_t res = 0;
memcpy(&res, addr, sizeof(uint16_t));
return res;
}
case 4: {
uint32_t res = 0;
memcpy(&res, addr, sizeof(uint32_t));
return res;
}
case 8: {
uint64_t res = 0;
memcpy(&res, addr, sizeof(uint64_t));
return res;
}
default:
swift_unreachable("Unsupported tag byte length.");
}
}
// This check is used to determine whether or not ObjC references can
// be tagged pointers. If they can't, they have the same spare bits
// as swift references, and we have to mask them out before passing the
// reference to ref counting operations.
static constexpr bool platformSupportsTaggedPointers() {
// Platforms that don't reserve bits for ObjC, don't support tagged
// pointers.
return _swift_abi_ObjCReservedBitsMask != 0;
}
#if defined(__APPLE__) && defined(__arm64__)
#define CONTINUE_WITH_COPY(METADATA, READER, ADDR_OFFSET, DEST, SRC) \
do { \
TAG = READER.readBytes<uint64_t>(); \
OFFSET = (TAG & ~(0xFFULL << 56)); \
if (OFFSET) { \
memcpy(DEST + ADDR_OFFSET, SRC + ADDR_OFFSET, OFFSET); \
} \
ADDR_OFFSET += OFFSET; \
TAG >>= 56; \
goto *dispatchTable[TAG]; \
} while (false)
#define CONTINUE_NO_COPY(METADATA, READER, ADDR_OFFSET, ADDR) \
do { \
TAG = READER.readBytes<uint64_t>(); \
OFFSET = (TAG & ~(0xFFULL << 56)); \
ADDR_OFFSET += OFFSET; \
TAG >>= 56; \
goto *dispatchTable[TAG]; \
} while (false)
#define handleRefCounts(FN_TABLE, CONTINUE, METADATA, READER, ADDR_OFFSET, \
...) \
do { \
while (true) { \
uint64_t TAG = 0; \
uintptr_t OFFSET = 0; \
\
_Pragma("clang diagnostic push") _Pragma( \
"clang diagnostic ignored \"-Wgnu-label-as-value\"") \
const void *dispatchTable[] = { \
&&done, &&Error, &&NativeStrong, &&NativeUnowned, \
&&NativeWeak, &&Unknown, &&UnknownUnowned, &&UnknownWeak, \
&&Bridge, &&Block, &&ObjC, &&NativeSwiftObjC, \
&&Metatype, &&Generic, &&Existential, &&Resilient, \
&&Default, &&Default, &&Default, &&Default, \
&&Default, &&Default, &&Default, \
}; \
\
[[clang::nomerge]] { \
CONTINUE(METADATA, READER, ADDR_OFFSET, __VA_ARGS__); \
} \
\
[[clang::nomerge]] { \
Error: \
\
FN_TABLE[1](METADATA, READER, ADDR_OFFSET, __VA_ARGS__); \
CONTINUE(METADATA, READER, ADDR_OFFSET, __VA_ARGS__); \
} \
[[clang::nomerge]] { \
NativeStrong: \
FN_TABLE[2](METADATA, READER, ADDR_OFFSET, __VA_ARGS__); \
CONTINUE(METADATA, READER, ADDR_OFFSET, __VA_ARGS__); \
} \
[[clang::nomerge]] { \
NativeUnowned: \
FN_TABLE[3](METADATA, READER, ADDR_OFFSET, __VA_ARGS__); \
CONTINUE(METADATA, READER, ADDR_OFFSET, __VA_ARGS__); \
} \
[[clang::nomerge]] { \
NativeWeak: \
FN_TABLE[4](METADATA, READER, ADDR_OFFSET, __VA_ARGS__); \
CONTINUE(METADATA, READER, ADDR_OFFSET, __VA_ARGS__); \
} \
[[clang::nomerge]] { \
Unknown: \
FN_TABLE[5](METADATA, READER, ADDR_OFFSET, __VA_ARGS__); \
CONTINUE(METADATA, READER, ADDR_OFFSET, __VA_ARGS__); \
} \
[[clang::nomerge]] { \
UnknownUnowned: \
FN_TABLE[6](METADATA, READER, ADDR_OFFSET, __VA_ARGS__); \
CONTINUE(METADATA, READER, ADDR_OFFSET, __VA_ARGS__); \
} \
[[clang::nomerge]] { \
UnknownWeak: \
FN_TABLE[7](METADATA, READER, ADDR_OFFSET, __VA_ARGS__); \
CONTINUE(METADATA, READER, ADDR_OFFSET, __VA_ARGS__); \
} \
[[clang::nomerge]] { \
Bridge: \
FN_TABLE[8](METADATA, READER, ADDR_OFFSET, __VA_ARGS__); \
CONTINUE(METADATA, READER, ADDR_OFFSET, __VA_ARGS__); \
} \
[[clang::nomerge]] { \
Block: \
FN_TABLE[9](METADATA, READER, ADDR_OFFSET, __VA_ARGS__); \
CONTINUE(METADATA, READER, ADDR_OFFSET, __VA_ARGS__); \
} \
[[clang::nomerge]] { \
ObjC: \
FN_TABLE[10](METADATA, READER, ADDR_OFFSET, __VA_ARGS__); \
CONTINUE(METADATA, READER, ADDR_OFFSET, __VA_ARGS__); \
} \
[[clang::nomerge]] { \
NativeSwiftObjC: \
FN_TABLE[11](METADATA, READER, ADDR_OFFSET, __VA_ARGS__); \
CONTINUE(METADATA, READER, ADDR_OFFSET, __VA_ARGS__); \
} \
[[clang::nomerge]] { \
Metatype: \
FN_TABLE[12](METADATA, READER, ADDR_OFFSET, __VA_ARGS__); \
CONTINUE(METADATA, READER, ADDR_OFFSET, __VA_ARGS__); \
} \
[[clang::nomerge]] { \
Generic: \
swift_unreachable(""); \
} \
[[clang::nomerge]] { \
Existential: \
FN_TABLE[14](METADATA, READER, ADDR_OFFSET, __VA_ARGS__); \
CONTINUE(METADATA, READER, ADDR_OFFSET, __VA_ARGS__); \
} \
[[clang::nomerge]] { \
Resilient: \
FN_TABLE[15](METADATA, READER, ADDR_OFFSET, __VA_ARGS__); \
CONTINUE(METADATA, READER, ADDR_OFFSET, __VA_ARGS__); \
} \
[[clang::nomerge]] { \
Default: \
uintptr_t _ADDR_OFFSET = ADDR_OFFSET; \
LayoutStringReader1 _READER = READER; \
FN_TABLE[TAG](METADATA, _READER, _ADDR_OFFSET, __VA_ARGS__); \
READER = _READER; \
ADDR_OFFSET = _ADDR_OFFSET; \
CONTINUE(METADATA, READER, ADDR_OFFSET, __VA_ARGS__); \
} \
_Pragma("clang diagnostic pop") \
} \
done: \
break; \
} while (false)
#endif
static void handleRefCountsDestroy(const Metadata *metadata,
LayoutStringReader1 &reader,
uintptr_t &addrOffset,
uint8_t *addr);
template <typename ...Params>
static void handleEnd(const Metadata *metadata,
LayoutStringReader1 &reader,
uintptr_t &addrOffset,
uint8_t *addr,
Params ...params) {
return;
}
static void errorDestroy(const Metadata *metadata, LayoutStringReader1 &reader,
uintptr_t &addrOffset, uint8_t *addr) {
uintptr_t object = *(uintptr_t *)(addr + addrOffset);
object &= ~_swift_abi_SwiftSpareBitsMask;
addrOffset += sizeof(SwiftError*);
swift_errorRelease((SwiftError *)object);
}
static void nativeStrongDestroy(const Metadata *metadata,
LayoutStringReader1 &reader,
uintptr_t &addrOffset, uint8_t *addr) {
HeapObject *object = (HeapObject*)((*(uintptr_t *)(addr + addrOffset)) & ~_swift_abi_SwiftSpareBitsMask);
addrOffset += sizeof(HeapObject*);
swift_release(object);
}
static void unownedDestroy(const Metadata *metadata,
LayoutStringReader1 &reader, uintptr_t &addrOffset,
uint8_t *addr) {
HeapObject *object = (HeapObject*)((*(uintptr_t *)(addr + addrOffset)) & ~_swift_abi_SwiftSpareBitsMask);
addrOffset += sizeof(HeapObject*);
swift_unownedRelease(object);
}
static void weakDestroy(const Metadata *metadata, LayoutStringReader1 &reader,
uintptr_t &addrOffset, uint8_t *addr) {
auto *object = (WeakReference *)(addr + addrOffset);
addrOffset += sizeof(WeakReference);
swift_weakDestroy(object);
}
static void unknownDestroy(const Metadata *metadata,
LayoutStringReader1 &reader, uintptr_t &addrOffset,
uint8_t *addr) {
uintptr_t object = *(uintptr_t *)(addr + addrOffset);
addrOffset += sizeof(void*);
if (!platformSupportsTaggedPointers()) {
object &= ~_swift_abi_SwiftSpareBitsMask;
}
swift_unknownObjectRelease((void *)object);
}
static void unknownUnownedDestroy(const Metadata *metadata,
LayoutStringReader1 &reader,
uintptr_t &addrOffset, uint8_t *addr) {
UnownedReference *object = (UnownedReference*)(addr + addrOffset);
addrOffset += sizeof(UnownedReference);
swift_unknownObjectUnownedDestroy(object);
}
static void unknownWeakDestroy(const Metadata *metadata,
LayoutStringReader1 &reader,
uintptr_t &addrOffset, uint8_t *addr) {
auto *object = (WeakReference *)(addr + addrOffset);
addrOffset += sizeof(WeakReference);
swift_unknownObjectWeakDestroy(object);
}
static void bridgeDestroy(const Metadata *metadata, LayoutStringReader1 &reader,
uintptr_t &addrOffset, uint8_t *addr) {
auto *object = *(void **)(addr + addrOffset);
addrOffset += sizeof(void*);
swift_bridgeObjectRelease(object);
}
static void singlePayloadEnumSimple(const Metadata *metadata,
LayoutStringReader1 &reader,
uintptr_t &addrOffset, uint8_t *addr) {
reader.modify([&](LayoutStringReader1 &reader) {
uint64_t byteCountsAndOffset;
size_t payloadSize;
uint64_t zeroTagValue;
size_t xiTagValues;
size_t refCountBytes;
size_t skip;
reader.readBytes(byteCountsAndOffset, payloadSize, zeroTagValue, xiTagValues, refCountBytes, skip);
auto extraTagBytesPattern = (uint8_t)(byteCountsAndOffset >> 62);
auto xiTagBytesPattern = ((uint8_t)(byteCountsAndOffset >> 59)) & 0x7;
auto xiTagBytesOffset =
byteCountsAndOffset & std::numeric_limits<uint32_t>::max();
if (SWIFT_UNLIKELY(extraTagBytesPattern)) {
auto extraTagBytes = 1 << (extraTagBytesPattern - 1);
auto tagBytes =
readTagBytes(addr + addrOffset + payloadSize, extraTagBytes);
if (tagBytes) {
xiTagBytesPattern = 0;
}
}
if (SWIFT_LIKELY(xiTagBytesPattern)) {
auto xiTagBytes = 1 << (xiTagBytesPattern - 1);
uint64_t tagBytes =
readTagBytes(addr + addrOffset + xiTagBytesOffset, xiTagBytes) -
zeroTagValue;
if (tagBytes >= xiTagValues) {
return;
}
}
reader.skip(refCountBytes);
addrOffset += skip;
});
}
typedef unsigned (*GetEnumTagFn)(const uint8_t *);
static void singlePayloadEnumFN(const Metadata *metadata,
LayoutStringReader1 &reader,
uintptr_t &addrOffset, uint8_t *addr) {
reader.modify([&](LayoutStringReader1 &reader) {
GetEnumTagFn getEnumTag = readRelativeFunctionPointer<GetEnumTagFn>(reader);
unsigned enumTag = getEnumTag(addr + addrOffset);
if (SWIFT_LIKELY(enumTag == 0)) {
reader.skip(sizeof(size_t) * 2);
} else {
size_t refCountBytes;
size_t skip;
reader.readBytes(refCountBytes, skip);
reader.skip(refCountBytes);
addrOffset += skip;
}
});
}
static void singlePayloadEnumFNResolved(const Metadata *metadata,
LayoutStringReader1 &reader,
uintptr_t &addrOffset, uint8_t *addr) {
reader.modify([&](LayoutStringReader1 &reader) {
GetEnumTagFn getEnumTag;
size_t refCountBytes;
size_t skip;
reader.readBytes(getEnumTag, refCountBytes, skip);
unsigned enumTag = getEnumTag(addr + addrOffset);
if (SWIFT_UNLIKELY(enumTag != 0)) {
reader.skip(refCountBytes);
addrOffset += skip;
}
});
}
static void singlePayloadEnumGeneric(const Metadata *metadata,
LayoutStringReader1 &reader,
uintptr_t &addrOffset, uint8_t *addr) {
reader.modify([&](LayoutStringReader1 &reader) {
auto tagBytesAndOffset = reader.readBytes<uint64_t>();
auto payloadSize = reader.readBytes<size_t>();
auto *xiType = reader.readBytes<const Metadata *>();
(void)reader.readBytes<unsigned>();
auto refCountBytes = reader.readBytes<size_t>();
auto skip = reader.readBytes<size_t>();
auto extraTagBytesPattern = (uint8_t)(tagBytesAndOffset >> 62);
auto xiTagBytesOffset =
tagBytesAndOffset & std::numeric_limits<uint32_t>::max();
if (SWIFT_UNLIKELY(extraTagBytesPattern)) {
auto extraTagBytes = 1 << (extraTagBytesPattern - 1);
auto tagBytes = readTagBytes(addr + addrOffset + payloadSize, extraTagBytes);
if (tagBytes) {
xiType = nullptr;
} else if (!xiType) {
// If there are no inhabitants and the extra tag bits are not set,
// we have a payload.
return;
}
}
if (SWIFT_LIKELY(xiType)) {
auto tag = xiType->vw_getEnumTagSinglePayload(
(const OpaqueValue *)(addr + addrOffset + xiTagBytesOffset),
xiType->vw_getNumExtraInhabitants());
if (SWIFT_LIKELY(tag == 0)) {
return;
}
}
reader.skip(refCountBytes);
addrOffset += skip;
});
}
template <auto HandlerFn>
static void multiPayloadEnumFN(const Metadata *metadata,
LayoutStringReader1 &reader,
uintptr_t &addrOffset, uint8_t *addr) {
size_t numPayloads;
size_t refCountBytes;
size_t enumSize;
LayoutStringReader1 nestedReader;
uintptr_t nestedAddrOffset;
unsigned enumTag;
reader.modify([&](LayoutStringReader1 &reader) {
GetEnumTagFn getEnumTag = readRelativeFunctionPointer<GetEnumTagFn>(reader);
reader.readBytes(numPayloads, refCountBytes, enumSize);
nestedReader = reader;
nestedAddrOffset = addrOffset;
enumTag = getEnumTag(addr + addrOffset);
reader.skip(refCountBytes + (numPayloads * sizeof(size_t)));
});
if (SWIFT_LIKELY(enumTag < numPayloads)) {
addrOffset += enumSize;
size_t refCountOffset = nestedReader.peekBytes<size_t>(enumTag * sizeof(size_t));
nestedReader.skip((numPayloads * sizeof(size_t)) + refCountOffset);
HandlerFn(metadata, nestedReader, nestedAddrOffset, addr);
} else {
addrOffset += enumSize;
}
}
template <auto HandlerFn>
static void multiPayloadEnumFNResolved(const Metadata *metadata,
LayoutStringReader1 &reader,
uintptr_t &addrOffset, uint8_t *addr) {
size_t numPayloads;
size_t refCountBytes;
size_t enumSize;
LayoutStringReader1 nestedReader;
uintptr_t nestedAddrOffset;
unsigned enumTag;
reader.modify([&](LayoutStringReader1 &reader) {
GetEnumTagFn getEnumTag = reader.readBytes<GetEnumTagFn>();
reader.readBytes(numPayloads, refCountBytes, enumSize);
nestedReader = reader;
nestedAddrOffset = addrOffset;
enumTag = getEnumTag(addr + addrOffset);
reader.skip(refCountBytes + (numPayloads * sizeof(size_t)));
});
if (SWIFT_LIKELY(enumTag < numPayloads)) {
addrOffset += enumSize;
size_t refCountOffset = nestedReader.peekBytes<size_t>(enumTag * sizeof(size_t));
nestedReader.skip((numPayloads * sizeof(size_t)) + refCountOffset);
HandlerFn(metadata, nestedReader, nestedAddrOffset, addr);
} else {
addrOffset += enumSize;
}
}
template <auto HandlerFn>
static void multiPayloadEnumGeneric(const Metadata *metadata,
LayoutStringReader1 &reader,
uintptr_t &addrOffset, uint8_t *addr) {
size_t tagBytes;
size_t numPayloads;
size_t refCountBytes;
size_t enumSize;
uint64_t enumTag;
uintptr_t nestedAddrOffset;
LayoutStringReader1 nestedReader;
reader.modify([&](LayoutStringReader1 &reader) {
reader.readBytes(tagBytes, numPayloads, refCountBytes, enumSize);
nestedReader = reader;
nestedAddrOffset = addrOffset;
auto tagBytesOffset = enumSize - tagBytes;
enumTag = readTagBytes(addr + addrOffset + tagBytesOffset, tagBytes);
reader.skip(refCountBytes + (numPayloads * sizeof(size_t)));
});
if (SWIFT_LIKELY(enumTag < numPayloads)) {
addrOffset += enumSize;
size_t refCountOffset = nestedReader.peekBytes<size_t>(enumTag * sizeof(size_t));
nestedReader.skip((numPayloads * sizeof(size_t)) + refCountOffset);
HandlerFn(metadata, nestedReader, nestedAddrOffset, addr);
} else {
addrOffset += enumSize;
}
}
static void singlePayloadEnumSimple(const Metadata *metadata,
LayoutStringReader1 &reader,
uintptr_t &addrOffset, uint8_t *dest,
uint8_t *src) {
reader.modify([&](LayoutStringReader1 &reader) {
uint64_t byteCountsAndOffset;
size_t payloadSize;
uint64_t zeroTagValue;
size_t xiTagValues;
size_t refCountBytes;
size_t skip;
reader.readBytes(byteCountsAndOffset, payloadSize, zeroTagValue, xiTagValues, refCountBytes, skip);
auto extraTagBytesPattern = (uint8_t)(byteCountsAndOffset >> 62);
auto xiTagBytesPattern = ((uint8_t)(byteCountsAndOffset >> 59)) & 0x7;
auto xiTagBytesOffset =
byteCountsAndOffset & std::numeric_limits<uint32_t>::max();
if (SWIFT_UNLIKELY(extraTagBytesPattern)) {
auto extraTagBytes = 1 << (extraTagBytesPattern - 1);
auto tagBytes =
readTagBytes(src + addrOffset + payloadSize, extraTagBytes);
if (tagBytes) {
xiTagBytesPattern = 0;
}
}
if (SWIFT_LIKELY(xiTagBytesPattern)) {
auto xiTagBytes = 1 << (xiTagBytesPattern - 1);
uint64_t tagBytes =
readTagBytes(src + addrOffset + xiTagBytesOffset, xiTagBytes) -
zeroTagValue;
if (tagBytes >= xiTagValues) {
return;
}
}
memcpy(dest + addrOffset, src + addrOffset, skip);
reader.skip(refCountBytes);
addrOffset += skip;
});
}
static void singlePayloadEnumFN(const Metadata *metadata,
LayoutStringReader1 &reader,
uintptr_t &addrOffset, uint8_t *dest,
uint8_t *src) {
reader.modify([&](LayoutStringReader1 &reader) {
GetEnumTagFn getEnumTag = readRelativeFunctionPointer<GetEnumTagFn>(reader);
unsigned enumTag = getEnumTag(src + addrOffset);
if (SWIFT_LIKELY(enumTag == 0)) {
reader.skip(sizeof(size_t) * 2);
} else {
size_t refCountBytes;
size_t skip;
reader.readBytes(refCountBytes, skip);
reader.skip(refCountBytes);
memcpy(dest + addrOffset, src + addrOffset, skip);
addrOffset += skip;
}
});
}
static void singlePayloadEnumFNResolved(const Metadata *metadata,
LayoutStringReader1 &reader,
uintptr_t &addrOffset, uint8_t *dest,
uint8_t *src) {
reader.modify([&](LayoutStringReader1 &reader) {
GetEnumTagFn getEnumTag;
size_t refCountBytes;
size_t skip;
reader.readBytes(getEnumTag, refCountBytes, skip);
unsigned enumTag = getEnumTag(src + addrOffset);
if (SWIFT_UNLIKELY(enumTag != 0)) {
reader.skip(refCountBytes);
memcpy(dest + addrOffset, src + addrOffset, skip);
addrOffset += skip;
}
});
}
static void singlePayloadEnumGeneric(const Metadata *metadata,
LayoutStringReader1 &reader,
uintptr_t &addrOffset, uint8_t *dest,
uint8_t *src) {
reader.modify([&](LayoutStringReader1 &reader) {
auto tagBytesAndOffset = reader.readBytes<uint64_t>();
auto payloadSize = reader.readBytes<size_t>();
auto *xiType = reader.readBytes<const Metadata *>();
(void)reader.readBytes<unsigned>(); // numEmptyCases
auto refCountBytes = reader.readBytes<size_t>();
auto skip = reader.readBytes<size_t>();
auto extraTagBytesPattern = (uint8_t)(tagBytesAndOffset >> 62);
auto xiTagBytesOffset =
tagBytesAndOffset & std::numeric_limits<uint32_t>::max();
if (SWIFT_UNLIKELY(extraTagBytesPattern)) {
auto extraTagBytes = 1 << (extraTagBytesPattern - 1);
auto tagBytes = readTagBytes(src + addrOffset + payloadSize, extraTagBytes);
if (tagBytes) {
xiType = nullptr;
} else if (!xiType) {
// If there are no inhabitants and the extra tag bits are not set,
// we have a payload.
return;
}
}
if (SWIFT_LIKELY(xiType)) {
auto tag = xiType->vw_getEnumTagSinglePayload(
(const OpaqueValue *)(src + addrOffset + xiTagBytesOffset),
xiType->vw_getNumExtraInhabitants());
if (SWIFT_LIKELY(tag == 0)) {
return;
}
}
reader.skip(refCountBytes);
memcpy(dest + addrOffset, src + addrOffset, skip);
addrOffset += skip;
});
}
template <auto HandlerFn>
static void
multiPayloadEnumFN(const Metadata *metadata, LayoutStringReader1 &reader,
uintptr_t &addrOffset, uint8_t *dest, uint8_t *src) {
size_t numPayloads;
size_t refCountBytes;
size_t enumSize;
LayoutStringReader1 nestedReader;
uintptr_t nestedAddrOffset;
unsigned enumTag;
reader.modify([&](LayoutStringReader1 &reader) {
GetEnumTagFn getEnumTag = readRelativeFunctionPointer<GetEnumTagFn>(reader);
reader.readBytes(numPayloads, refCountBytes, enumSize);
nestedReader = reader;
nestedAddrOffset = addrOffset;
enumTag = getEnumTag(src + addrOffset);
reader.skip(refCountBytes + (numPayloads * sizeof(size_t)));
});
if (SWIFT_LIKELY(enumTag < numPayloads)) {
addrOffset += enumSize;
size_t refCountOffset = nestedReader.peekBytes<size_t>(enumTag * sizeof(size_t));
nestedReader.skip((numPayloads * sizeof(size_t)) + refCountOffset);
HandlerFn(metadata, nestedReader, nestedAddrOffset, dest, src);
auto trailingBytes = addrOffset - nestedAddrOffset;
if (trailingBytes)
memcpy(dest + nestedAddrOffset, src + nestedAddrOffset, trailingBytes);
} else {
memcpy(dest + addrOffset, src + addrOffset, enumSize);
addrOffset += enumSize;
}
}
template <auto HandlerFn>
static void multiPayloadEnumFNResolved(const Metadata *metadata,
LayoutStringReader1 &reader,
uintptr_t &addrOffset, uint8_t *dest,
uint8_t *src) {
size_t numPayloads;
size_t refCountBytes;
size_t enumSize;
LayoutStringReader1 nestedReader;
uintptr_t nestedAddrOffset;
unsigned enumTag;
reader.modify([&](LayoutStringReader1 &reader) {
GetEnumTagFn getEnumTag = reader.readBytes<GetEnumTagFn>();
reader.readBytes(numPayloads, refCountBytes, enumSize);
nestedReader = reader;
nestedAddrOffset = addrOffset;
enumTag = getEnumTag(src + addrOffset);
reader.skip(refCountBytes + (numPayloads * sizeof(size_t)));
});
if (SWIFT_LIKELY(enumTag < numPayloads)) {
addrOffset += enumSize;
size_t refCountOffset = nestedReader.peekBytes<size_t>(enumTag * sizeof(size_t));
nestedReader.skip((numPayloads * sizeof(size_t)) + refCountOffset);
HandlerFn(metadata, nestedReader, nestedAddrOffset, dest, src);
auto trailingBytes = addrOffset - nestedAddrOffset;
if (trailingBytes)
memcpy(dest + nestedAddrOffset, src + nestedAddrOffset, trailingBytes);
} else {
memcpy(dest + addrOffset, src + addrOffset, enumSize);
addrOffset += enumSize;
}
}
template <auto HandlerFn>
static void
multiPayloadEnumGeneric(const Metadata *metadata, LayoutStringReader1 &reader,
uintptr_t &addrOffset, uint8_t *dest, uint8_t *src) {
size_t tagBytes;
size_t numPayloads;
size_t refCountBytes;
size_t enumSize;
uint64_t enumTag;
uintptr_t nestedAddrOffset;
LayoutStringReader1 nestedReader;
reader.modify([&](LayoutStringReader1 &reader) {
reader.readBytes(tagBytes, numPayloads, refCountBytes, enumSize);
nestedReader = reader;
nestedAddrOffset = addrOffset;
auto tagBytesOffset = enumSize - tagBytes;
enumTag = readTagBytes(src + addrOffset + tagBytesOffset, tagBytes);
reader.skip(refCountBytes + (numPayloads * sizeof(size_t)));
});
if (SWIFT_LIKELY(enumTag < numPayloads)) {
addrOffset += enumSize;
size_t refCountOffset = nestedReader.peekBytes<size_t>(enumTag * sizeof(size_t));
nestedReader.skip((numPayloads * sizeof(size_t)) + refCountOffset);
HandlerFn(metadata, nestedReader, nestedAddrOffset, dest, src);
auto trailingBytes = addrOffset - nestedAddrOffset;
if (trailingBytes)
memcpy(dest + nestedAddrOffset, src + nestedAddrOffset, trailingBytes);
} else {
memcpy(dest + addrOffset, src + addrOffset, enumSize);
addrOffset += enumSize;
}
}
static void blockDestroy(const Metadata *metadata, LayoutStringReader1 &reader,
uintptr_t &addrOffset, uint8_t *addr) {
#if SWIFT_OBJC_INTEROP
uintptr_t object = *(uintptr_t *)(addr + addrOffset);
object &= ~_swift_abi_SwiftSpareBitsMask;
addrOffset += sizeof(void*);
_Block_release((void *)object);
#else
swift_unreachable("Blocks are not available on this platform");
#endif
}
static void objcStrongDestroy(const Metadata *metadata,
LayoutStringReader1 &reader,
uintptr_t &addrOffset, uint8_t *addr) {
#if SWIFT_OBJC_INTEROP
uintptr_t object = *(uintptr_t *)(addr + addrOffset);
addrOffset += sizeof(objc_object*);
if (!platformSupportsTaggedPointers()) {
object &= ~_swift_abi_SwiftSpareBitsMask;
}
objc_release((objc_object *)object);
#else
swift_unreachable("ObjC interop is not available on this platform");
#endif
}
static void nativeSwiftObjcStrongDestroy(const Metadata *metadata,
LayoutStringReader1 &reader,
uintptr_t &addrOffset, uint8_t *addr) {
#if SWIFT_OBJC_INTEROP
uintptr_t object = *(uintptr_t *)(addr + addrOffset);
addrOffset += sizeof(objc_object *);
object &= ~_swift_abi_SwiftSpareBitsMask;
objc_release((objc_object *)object);
#else
swift_unreachable("ObjC interop is not available on this platform");
#endif
}
static void metatypeDestroy(const Metadata *metadata,
LayoutStringReader1 &reader, uintptr_t &addrOffset,
uint8_t *addr) {
auto *type = reader.readBytes<const Metadata *>();
auto *object = (OpaqueValue *)(addr + addrOffset);
addrOffset += type->vw_size();
type->vw_destroy(object);
}
static void existentialDestroy(const Metadata *metadata,
LayoutStringReader1 &reader,
uintptr_t &addrOffset, uint8_t *addr) {
OpaqueValue *object = (OpaqueValue *)(addr + addrOffset);
auto* type = getExistentialTypeMetadata(object);
addrOffset += sizeof(uintptr_t) * NumWords_ValueBuffer;
if (type->getValueWitnesses()->isValueInline()) {
type->vw_destroy(object);
} else {
swift_release(*(HeapObject**)object);
}
}
static void resilientDestroy(const Metadata *metadata,
LayoutStringReader1 &reader, uintptr_t &addrOffset,
uint8_t *addr) {
auto *type = getResilientTypeMetadata(metadata, reader);
auto *object = (OpaqueValue *)(addr + addrOffset);
addrOffset += type->vw_size();
type->vw_destroy(object);
}
typedef void (*DestrFn)(const Metadata *metadata, LayoutStringReader1 &reader,
uintptr_t &addrOffset, uint8_t *addr);
constexpr DestrFn destroyTable[] = {
&handleEnd,
&errorDestroy,
&nativeStrongDestroy,
&unownedDestroy,
&weakDestroy,
&unknownDestroy,
&unknownUnownedDestroy,
&unknownWeakDestroy,
&bridgeDestroy,
&blockDestroy,
&objcStrongDestroy,
&nativeSwiftObjcStrongDestroy,
&metatypeDestroy,
nullptr, // Generic
&existentialDestroy,
&resilientDestroy,
&singlePayloadEnumSimple,
&singlePayloadEnumFN,
&singlePayloadEnumFNResolved,
&singlePayloadEnumGeneric,
&multiPayloadEnumFN<handleRefCountsDestroy>,
&multiPayloadEnumFNResolved<handleRefCountsDestroy>,
&multiPayloadEnumGeneric<handleRefCountsDestroy>,
};
static void handleRefCountsDestroy(const Metadata *metadata,
LayoutStringReader1 &reader,
uintptr_t &addrOffset,
uint8_t *addr) {
while (true) {
auto tag = reader.readBytes<uint64_t>();
addrOffset += (tag & ~(0xFFULL << 56));
tag >>= 56;
if (SWIFT_UNLIKELY(tag == 0)) {
return;
}
destroyTable[tag](metadata, reader, addrOffset, addr);
}
}
static void swift_cvw_destroyImpl(swift::OpaqueValue *address,
const Metadata *metadata) {
const uint8_t *layoutStr = metadata->getLayoutString();
LayoutStringReader1 reader{layoutStr + layoutStringHeaderSize};
uintptr_t addrOffset = 0;
uint8_t *addr = (uint8_t *)address;
#if defined(__APPLE__) && defined(__arm64__)
handleRefCounts(destroyTable, CONTINUE_NO_COPY, metadata, reader, addrOffset,
addr);
#else
handleRefCountsDestroy(metadata, reader, addrOffset, addr);
#endif
}
void swift::swift_cvw_arrayDestroy(swift::OpaqueValue *address, size_t count,
size_t stride, const Metadata *metadata) {
const uint8_t *layoutStr = metadata->getLayoutString();
uint8_t *addr = (uint8_t *)address;
for (size_t i = 0; i < count; i++) {
LayoutStringReader1 reader{layoutStr + layoutStringHeaderSize};
uintptr_t addrOffset = i * stride;
#if defined(__APPLE__) && defined(__arm64__)
handleRefCounts(destroyTable, CONTINUE_NO_COPY, metadata, reader,
addrOffset, addr);
#else
handleRefCountsDestroy(metadata, reader, addrOffset, addr);
#endif
}
}
static void handleRefCountsInitWithCopy(const Metadata *metadata,
LayoutStringReader1 &reader,
uintptr_t &addrOffset,
uint8_t *dest,
uint8_t *src);
static void errorRetain(const Metadata *metadata, LayoutStringReader1 &reader,
uintptr_t &addrOffset, uint8_t *dest, uint8_t *src) {
uintptr_t _addrOffset = addrOffset;
uintptr_t object = *(uintptr_t *)(src + _addrOffset);
memcpy(dest + addrOffset, &object, sizeof(SwiftError*));
object &= ~_swift_abi_SwiftSpareBitsMask;
addrOffset = _addrOffset + sizeof(SwiftError *);
swift_errorRetain((SwiftError *)object);
}
static void nativeStrongRetain(const Metadata *metadata,
LayoutStringReader1 &reader,
uintptr_t &addrOffset, uint8_t *dest,
uint8_t *src) {
uintptr_t _addrOffset = addrOffset;
uintptr_t object = *(uintptr_t *)(src + _addrOffset);
memcpy(dest + _addrOffset, &object, sizeof(HeapObject*));
object &= ~_swift_abi_SwiftSpareBitsMask;
addrOffset = _addrOffset + sizeof(HeapObject *);
swift_retain((HeapObject *)object);
}
static void unownedRetain(const Metadata *metadata, LayoutStringReader1 &reader,
uintptr_t &addrOffset, uint8_t *dest, uint8_t *src) {
uintptr_t _addrOffset = addrOffset;
uintptr_t object = *(uintptr_t *)(src + _addrOffset);
memcpy(dest + _addrOffset, &object, sizeof(HeapObject*));
object &= ~_swift_abi_SwiftSpareBitsMask;
addrOffset = _addrOffset + sizeof(HeapObject *);
swift_unownedRetain((HeapObject *)object);
}
static void weakCopyInit(const Metadata *metadata, LayoutStringReader1 &reader,
uintptr_t &addrOffset, uint8_t *dest, uint8_t *src) {
uintptr_t _addrOffset = addrOffset;
auto *destObject = (WeakReference *)(dest + _addrOffset);
auto *srcObject = (WeakReference *)(src + _addrOffset);
addrOffset = _addrOffset + sizeof(WeakReference);
swift_weakCopyInit(destObject, srcObject);
}
static void unknownRetain(const Metadata *metadata, LayoutStringReader1 &reader,
uintptr_t &addrOffset, uint8_t *dest, uint8_t *src) {
uintptr_t _addrOffset = addrOffset;
uintptr_t object = *(uintptr_t *)(src + _addrOffset);
memcpy(dest + _addrOffset, &object, sizeof(void*));
addrOffset = _addrOffset + sizeof(void *);
if (!platformSupportsTaggedPointers()) {
object &= ~_swift_abi_SwiftSpareBitsMask;
}
swift_unknownObjectRetain((void *)object);
}
static void unknownUnownedCopyInit(const Metadata *metadata,
LayoutStringReader1 &reader,
uintptr_t &addrOffset, uint8_t *dest,
uint8_t *src) {
uintptr_t _addrOffset = addrOffset;
UnownedReference *objectDest = (UnownedReference*)(dest + _addrOffset);
UnownedReference *objectSrc = (UnownedReference*)(src + _addrOffset);
addrOffset = _addrOffset + sizeof(UnownedReference);
swift_unknownObjectUnownedCopyInit(objectDest, objectSrc);
}
static void unknownWeakCopyInit(const Metadata *metadata,
LayoutStringReader1 &reader,
uintptr_t &addrOffset, uint8_t *dest,
uint8_t *src) {
uintptr_t _addrOffset = addrOffset;
auto *destObject = (WeakReference *)(dest + _addrOffset);
auto *srcObject = (WeakReference *)(src + _addrOffset);
addrOffset = _addrOffset + sizeof(WeakReference);
swift_unknownObjectWeakCopyInit(destObject, srcObject);
}
static void bridgeRetain(const Metadata *metadata, LayoutStringReader1 &reader,
uintptr_t &addrOffset, uint8_t *dest, uint8_t *src) {
uintptr_t _addrOffset = addrOffset;
void *object = *(void **)(src + _addrOffset);
memcpy(dest + _addrOffset, &object, sizeof(void*));
addrOffset = _addrOffset + sizeof(void*);
swift_bridgeObjectRetain(object);
}
static void blockCopy(const Metadata *metadata, LayoutStringReader1 &reader,
uintptr_t &addrOffset, uint8_t *dest, uint8_t *src) {
#if SWIFT_OBJC_INTEROP
uintptr_t _addrOffset = addrOffset;
uintptr_t object = *(uintptr_t *)(src + _addrOffset);
memcpy(dest + _addrOffset, &object, sizeof(void *));
addrOffset = _addrOffset + sizeof(void*);
object &= ~_swift_abi_SwiftSpareBitsMask;
_Block_copy((void *)object);
#else
swift_unreachable("Blocks are not available on this platform");
#endif
}
static void objcStrongRetain(const Metadata *metadata,
LayoutStringReader1 &reader, uintptr_t &addrOffset,
uint8_t *dest, uint8_t *src) {
#if SWIFT_OBJC_INTEROP
uintptr_t _addrOffset = addrOffset;
uintptr_t object = *(uintptr_t *)(src + _addrOffset);
memcpy(dest + _addrOffset, &object, sizeof(objc_object *));
addrOffset = _addrOffset + sizeof(objc_object *);
if (!platformSupportsTaggedPointers()) {
object &= ~_swift_abi_SwiftSpareBitsMask;
}
objc_retain((objc_object *)object);
#else
swift_unreachable("ObjC interop is not available on this platform");
#endif
}
static void nativeSwiftObjcStrongRetain(const Metadata *metadata,
LayoutStringReader1 &reader,
uintptr_t &addrOffset, uint8_t *dest,
uint8_t *src) {
#if SWIFT_OBJC_INTEROP
uintptr_t _addrOffset = addrOffset;
uintptr_t object = *(uintptr_t *)(src + _addrOffset);
memcpy(dest + _addrOffset, &object, sizeof(objc_object *));
addrOffset = _addrOffset + sizeof(objc_object *);
object &= ~_swift_abi_SwiftSpareBitsMask;
objc_retain((objc_object *)object);
#else
swift_unreachable("ObjC interop is not available on this platform");
#endif
}
static void metatypeInitWithCopy(const Metadata *metadata,
LayoutStringReader1 &reader,
uintptr_t &addrOffset, uint8_t *dest,
uint8_t *src) {
uintptr_t _addrOffset = addrOffset;
auto *type = reader.readBytes<const Metadata *>();
auto *destObject = (OpaqueValue *)(dest + _addrOffset);
auto *srcObject = (OpaqueValue *)(src + _addrOffset);
addrOffset = _addrOffset + type->vw_size();
type->vw_initializeWithCopy(destObject, srcObject);
}
static void existentialInitWithCopy(const Metadata *metadata,
LayoutStringReader1 &reader,
uintptr_t &addrOffset, uint8_t *dest,
uint8_t *src) {
uintptr_t _addrOffset = addrOffset;
auto *type = getExistentialTypeMetadata((OpaqueValue*)(src + _addrOffset));
auto *destObject = (ValueBuffer *)(dest + _addrOffset);
auto *srcObject = (ValueBuffer *)(src + _addrOffset);
addrOffset = _addrOffset + (sizeof(uintptr_t) * NumWords_ValueBuffer);
type->vw_initializeBufferWithCopyOfBuffer(destObject, srcObject);
}
static void resilientInitWithCopy(const Metadata *metadata,
LayoutStringReader1 &reader,
uintptr_t &addrOffset, uint8_t *dest,
uint8_t *src) {
uintptr_t _addrOffset = addrOffset;
auto *type = getResilientTypeMetadata(metadata, reader);
auto *destObject = (OpaqueValue *)(dest + _addrOffset);
auto *srcObject = (OpaqueValue *)(src + _addrOffset);
addrOffset = _addrOffset + type->vw_size();
type->vw_initializeWithCopy(destObject, srcObject);
}
typedef void (*InitFn)(const Metadata *metadata,
LayoutStringReader1 &reader,
uintptr_t &addrOffset,
uint8_t *dest,
uint8_t *src);
constexpr InitFn initWithCopyTable[] = {
&handleEnd,
&errorRetain,
&nativeStrongRetain,
&unownedRetain,
&weakCopyInit,
&unknownRetain,
&unknownUnownedCopyInit,
&unknownWeakCopyInit,
&bridgeRetain,
&blockCopy,
&objcStrongRetain,
&nativeSwiftObjcStrongRetain,
&metatypeInitWithCopy,
nullptr, // Generic
&existentialInitWithCopy,
&resilientInitWithCopy,
&singlePayloadEnumSimple,
&singlePayloadEnumFN,
&singlePayloadEnumFNResolved,
&singlePayloadEnumGeneric,
&multiPayloadEnumFN<handleRefCountsInitWithCopy>,
&multiPayloadEnumFNResolved<handleRefCountsInitWithCopy>,
&multiPayloadEnumGeneric<handleRefCountsInitWithCopy>,
};
static void handleRefCountsInitWithCopy(const Metadata *metadata,
LayoutStringReader1 &reader,
uintptr_t &addrOffset,
uint8_t *dest,
uint8_t *src) {
while (true) {
uintptr_t _addrOffset = addrOffset;
auto tag = reader.readBytes<uint64_t>();
auto offset = (tag & ~(0xFFULL << 56));
if (offset) {
memcpy(dest + _addrOffset, src + _addrOffset, offset);
}
addrOffset = _addrOffset + offset;
tag >>= 56;
if (SWIFT_UNLIKELY(tag == 0)) {
return;
}
initWithCopyTable[tag](metadata, reader, addrOffset, dest, src);
}
}
static swift::OpaqueValue *
swift_cvw_initWithCopyImpl(swift::OpaqueValue *_dest, swift::OpaqueValue *_src,
const Metadata *metadata) {
const uint8_t *layoutStr = metadata->getLayoutString();
LayoutStringReader1 reader{layoutStr + layoutStringHeaderSize};
uintptr_t addrOffset = 0;
uint8_t *dest = (uint8_t *)_dest;
uint8_t *src = (uint8_t *)_src;
#if defined(__APPLE__) && defined(__arm64__)
handleRefCounts(initWithCopyTable, CONTINUE_WITH_COPY, metadata, reader,
addrOffset, dest, src);
#else
handleRefCountsInitWithCopy(metadata, reader, addrOffset, dest, src);
#endif
assert(addrOffset == metadata->vw_size());
return _dest;
}
void swift::swift_cvw_arrayInitWithCopy(swift::OpaqueValue *_dest,
swift::OpaqueValue *_src, size_t count,
size_t stride,
const Metadata *metadata) {
const uint8_t *layoutStr = metadata->getLayoutString();
uint8_t *dest = (uint8_t *)_dest;
uint8_t *src = (uint8_t *)_src;
for (size_t i = 0; i < count; i++) {
LayoutStringReader1 reader{layoutStr + layoutStringHeaderSize};
uintptr_t addrOffset = i * stride;
#if defined(__APPLE__) && defined(__arm64__)
handleRefCounts(initWithCopyTable, CONTINUE_WITH_COPY, metadata, reader,
addrOffset, dest, src);
#else
handleRefCountsInitWithCopy(metadata, reader, addrOffset, dest, src);
#endif
}
}
static void handleRefCountsInitWithTake(const Metadata *metadata,
LayoutStringReader1 &reader,
uintptr_t &addrOffset,
uint8_t *dest,
uint8_t *src);
static void unknownWeakInitWithTake(const Metadata *metadata,
LayoutStringReader1 &reader,
uintptr_t &addrOffset,
uint8_t *dest,
uint8_t *src) {
auto *destObject = (WeakReference *)(dest + addrOffset);
auto *srcObject = (WeakReference *)(src + addrOffset);
addrOffset += sizeof(WeakReference);
swift_unknownObjectWeakTakeInit(destObject, srcObject);
}
static void metatypeInitWithTake(const Metadata *metadata,
LayoutStringReader1 &reader,
uintptr_t &addrOffset,
uint8_t *dest,
uint8_t *src) {
uintptr_t _addrOffset = addrOffset;
auto *type = reader.readBytes<const Metadata *>();
auto *destObject = (OpaqueValue *)(dest + _addrOffset);
auto *srcObject = (OpaqueValue *)(src + _addrOffset);
addrOffset = _addrOffset + type->vw_size();
type->vw_initializeWithTake(destObject, srcObject);
}
static void existentialInitWithTake(const Metadata *metadata,
LayoutStringReader1 &reader,
uintptr_t &addrOffset,
uint8_t *dest,
uint8_t *src) {
uintptr_t _addrOffset = addrOffset;
auto* type = getExistentialTypeMetadata((OpaqueValue*)(src + addrOffset));
auto *destObject = (OpaqueValue *)(dest + _addrOffset);
auto *srcObject = (OpaqueValue *)(src + _addrOffset);
addrOffset = _addrOffset + (sizeof(uintptr_t) * NumWords_ValueBuffer);
if (type->getValueWitnesses()->isValueInline()) {
type->vw_initializeWithTake(destObject, srcObject);
} else {
memcpy(destObject, srcObject, sizeof(uintptr_t));
}
}
static void resilientInitWithTake(const Metadata *metadata,
LayoutStringReader1 &reader,
uintptr_t &addrOffset,
uint8_t *dest,
uint8_t *src) {
uintptr_t _addrOffset = addrOffset;
auto *type = getResilientTypeMetadata(metadata, reader);
auto *destObject = (OpaqueValue *)(dest + _addrOffset);
auto *srcObject = (OpaqueValue *)(src + _addrOffset);
addrOffset = _addrOffset + type->vw_size();
type->vw_initializeWithTake(destObject, srcObject);
}
static void copyingInitWithTake(const Metadata *metadata,
LayoutStringReader1 &reader,
uintptr_t &addrOffset, uint8_t *dest,
uint8_t *src) {
memcpy(dest + addrOffset, src + addrOffset, sizeof(uintptr_t));
addrOffset += sizeof(uintptr_t);
}
constexpr InitFn initWithTakeTable[] = {
&handleEnd,
&copyingInitWithTake,
&copyingInitWithTake,
&copyingInitWithTake,
&copyingInitWithTake,
&copyingInitWithTake,
&copyingInitWithTake,
&unknownWeakInitWithTake,
&copyingInitWithTake,
&copyingInitWithTake,
&copyingInitWithTake,
&copyingInitWithTake,
&metatypeInitWithTake,
nullptr, // Generic
&existentialInitWithTake,
&resilientInitWithTake,
&singlePayloadEnumSimple,
&singlePayloadEnumFN,
&singlePayloadEnumFNResolved,
&singlePayloadEnumGeneric,
&multiPayloadEnumFN<handleRefCountsInitWithTake>,
&multiPayloadEnumFNResolved<handleRefCountsInitWithTake>,
&multiPayloadEnumGeneric<handleRefCountsInitWithTake>,
};
static void handleRefCountsInitWithTake(const Metadata *metadata,
LayoutStringReader1 &reader,
uintptr_t &addrOffset,
uint8_t *dest,
uint8_t *src) {
while (true) {
uintptr_t _addrOffset = addrOffset;
auto tag = reader.readBytes<uint64_t>();
auto offset = (tag & ~(0xFFULL << 56));
if (offset) {
memcpy(dest + _addrOffset, src + _addrOffset, offset);
}
addrOffset += offset;
tag >>= 56;
if (SWIFT_UNLIKELY(tag == 0)) {
return;
}
initWithTakeTable[tag](metadata, reader, addrOffset, dest, src);
}
}
static swift::OpaqueValue *
swift_cvw_initWithTakeImpl(swift::OpaqueValue *_dest, swift::OpaqueValue *_src,
const Metadata *metadata) {
if (SWIFT_LIKELY(metadata->getValueWitnesses()->isBitwiseTakable())) {
size_t size = metadata->vw_size();
memcpy(_dest, _src, size);
return _dest;
}
uint8_t *dest = (uint8_t *)_dest;
uint8_t *src = (uint8_t *)_src;
const uint8_t *layoutStr = metadata->getLayoutString();
LayoutStringReader1 reader{layoutStr + layoutStringHeaderSize};
uintptr_t addrOffset = 0;
#if defined(__APPLE__) && defined(__arm64__)
handleRefCounts(initWithTakeTable, CONTINUE_WITH_COPY, metadata, reader,
addrOffset, dest, src);
#else
handleRefCountsInitWithTake(metadata, reader, addrOffset, dest, src);
#endif
assert(addrOffset == metadata->vw_size());
return _dest;
}
static void errorAssignWithCopy(const Metadata *metadata,
LayoutStringReader1 &reader,
uintptr_t &addrOffset, uint8_t *dest,
uint8_t *src) {
uintptr_t _addrOffset = addrOffset;
uintptr_t destObject = *(uintptr_t *)(dest + _addrOffset);
uintptr_t srcObject = *(uintptr_t *)(src + _addrOffset);
memcpy(dest + _addrOffset, &srcObject, sizeof(SwiftError *));
addrOffset = _addrOffset + sizeof(SwiftError *);
destObject &= ~_swift_abi_SwiftSpareBitsMask;
srcObject &= ~_swift_abi_SwiftSpareBitsMask;
swift_errorRelease((SwiftError *)destObject);
swift_errorRetain((SwiftError *)srcObject);
}
static void nativeStrongAssignWithCopy(const Metadata *metadata,
LayoutStringReader1 &reader,
uintptr_t &addrOffset,
uint8_t *dest,
uint8_t *src) {
uintptr_t _addrOffset = addrOffset;
uintptr_t destObject = *(uintptr_t *)(dest + _addrOffset);
uintptr_t srcObject = *(uintptr_t *)(src + _addrOffset);
memcpy(dest + _addrOffset, &srcObject, sizeof(HeapObject*));
srcObject &= ~_swift_abi_SwiftSpareBitsMask;
destObject &= ~_swift_abi_SwiftSpareBitsMask;
addrOffset = _addrOffset + sizeof(HeapObject *);
swift_release((HeapObject *)destObject);
swift_retain((HeapObject *)srcObject);
}
static void unownedAssignWithCopy(const Metadata *metadata,
LayoutStringReader1 &reader,
uintptr_t &addrOffset,
uint8_t *dest,
uint8_t *src) {
uintptr_t _addrOffset = addrOffset;
uintptr_t destObject = *(uintptr_t *)(dest + _addrOffset);
uintptr_t srcObject = *(uintptr_t *)(src + _addrOffset);
memcpy(dest + _addrOffset, &srcObject, sizeof(HeapObject*));
destObject &= ~_swift_abi_SwiftSpareBitsMask;
srcObject &= ~_swift_abi_SwiftSpareBitsMask;
addrOffset = _addrOffset + sizeof(HeapObject *);
swift_unownedRelease((HeapObject *)destObject);
swift_unownedRetain((HeapObject *)srcObject);
}
static void unknownAssignWithCopy(const Metadata *metadata,
LayoutStringReader1 &reader,
uintptr_t &addrOffset, uint8_t *dest,
uint8_t *src) {
uintptr_t _addrOffset = addrOffset;
uintptr_t destObject = *(uintptr_t *)(dest + _addrOffset);
uintptr_t srcObject = *(uintptr_t *)(src + _addrOffset);
memcpy(dest + _addrOffset, &srcObject, sizeof(void *));
addrOffset = _addrOffset + sizeof(void *);
if (!platformSupportsTaggedPointers()) {
destObject &= ~_swift_abi_SwiftSpareBitsMask;
srcObject &= ~_swift_abi_SwiftSpareBitsMask;
}
swift_unknownObjectRelease((void *)destObject);
swift_unknownObjectRetain((void *)srcObject);
}
static void bridgeAssignWithCopy(const Metadata *metadata,
LayoutStringReader1 &reader,
uintptr_t &addrOffset, uint8_t *dest,
uint8_t *src) {
uintptr_t _addrOffset = addrOffset;
void *destObject = *(void **)(dest + _addrOffset);
void *srcObject = *(void **)(src + _addrOffset);
memcpy(dest + _addrOffset, &srcObject, sizeof(void*));
addrOffset = _addrOffset + sizeof(void *);
swift_bridgeObjectRelease(destObject);
swift_bridgeObjectRetain(srcObject);
}
static void weakAssignWithCopy(const Metadata *metadata,
LayoutStringReader1 &reader,
uintptr_t &addrOffset, uint8_t *dest,
uint8_t *src) {
uintptr_t _addrOffset = addrOffset;
auto *destObject = (WeakReference *)(dest + _addrOffset);
auto *srcObject = (WeakReference *)(src + _addrOffset);
addrOffset = _addrOffset + sizeof(WeakReference);
swift_weakCopyAssign(destObject, srcObject);
}
static void unknownUnownedAssignWithCopy(const Metadata *metadata,
LayoutStringReader1 &reader,
uintptr_t &addrOffset,
uint8_t *dest,
uint8_t *src) {
uintptr_t _addrOffset = addrOffset;
UnownedReference *objectDest = (UnownedReference*)(dest + _addrOffset);
UnownedReference *objectSrc = (UnownedReference*)(src + _addrOffset);
addrOffset = _addrOffset + sizeof(UnownedReference);
swift_unknownObjectUnownedCopyAssign(objectDest, objectSrc);
}
static void unknownWeakAssignWithCopy(const Metadata *metadata,
LayoutStringReader1 &reader,
uintptr_t &addrOffset,
uint8_t *dest,
uint8_t *src) {
uintptr_t _addrOffset = addrOffset;
auto *destObject = (WeakReference *)(dest + _addrOffset);
auto *srcObject = (WeakReference *)(src + _addrOffset);
addrOffset = _addrOffset + sizeof(WeakReference);
swift_unknownObjectWeakCopyAssign(destObject, srcObject);
}
static void blockAssignWithCopy(const Metadata *metadata,
LayoutStringReader1 &reader,
uintptr_t &addrOffset,
uint8_t *dest,
uint8_t *src) {
#if SWIFT_OBJC_INTEROP
uintptr_t _addrOffset = addrOffset;
uintptr_t destObject = *(uintptr_t *)(dest + _addrOffset);
uintptr_t srcObject = *(uintptr_t *)(src + _addrOffset);
memcpy(dest + _addrOffset, &srcObject, sizeof(void *));
addrOffset = _addrOffset + sizeof(void*);
destObject &= ~_swift_abi_SwiftSpareBitsMask;
srcObject &= ~_swift_abi_SwiftSpareBitsMask;
_Block_release((void *)destObject);
_Block_copy((void *)srcObject);
#else
swift_unreachable("Blocks are not available on this platform");
#endif
}
static void objcStrongAssignWithCopy(const Metadata *metadata,
LayoutStringReader1 &reader,
uintptr_t &addrOffset,
uint8_t *dest,
uint8_t *src) {
#if SWIFT_OBJC_INTEROP
uintptr_t _addrOffset = addrOffset;
uintptr_t destObject = *(uintptr_t *)(dest + _addrOffset);
uintptr_t srcObject = *(uintptr_t *)(src + _addrOffset);
memcpy(dest + _addrOffset, &srcObject, sizeof(objc_object*));
addrOffset = _addrOffset + sizeof(objc_object*);
if (!platformSupportsTaggedPointers()) {
destObject &= ~_swift_abi_SwiftSpareBitsMask;
srcObject &= ~_swift_abi_SwiftSpareBitsMask;
}
objc_release((objc_object *)destObject);
objc_retain((objc_object *)srcObject);
#else
swift_unreachable("ObjC interop is not available on this platform");
#endif
}
static void nativeSwiftObjcStrongAssignWithCopy(const Metadata *metadata,
LayoutStringReader1 &reader,
uintptr_t &addrOffset,
uint8_t *dest, uint8_t *src) {
#if SWIFT_OBJC_INTEROP
uintptr_t _addrOffset = addrOffset;
uintptr_t destObject = *(uintptr_t *)(dest + _addrOffset);
uintptr_t srcObject = *(uintptr_t *)(src + _addrOffset);
memcpy(dest + _addrOffset, &srcObject, sizeof(objc_object *));
addrOffset = _addrOffset + sizeof(objc_object *);
destObject &= ~_swift_abi_SwiftSpareBitsMask;
objc_release((objc_object *)destObject);
srcObject &= ~_swift_abi_SwiftSpareBitsMask;
objc_retain((objc_object *)srcObject);
#else
swift_unreachable("ObjC interop is not available on this platform");
#endif
}
static void existentialAssignWithCopy(const Metadata *metadata,
LayoutStringReader1 &reader,
uintptr_t &addrOffset,
uint8_t *dest,
uint8_t *src) {
uintptr_t _addrOffset = addrOffset;
auto *srcType = getExistentialTypeMetadata((OpaqueValue*)(src + _addrOffset));
auto *destType = getExistentialTypeMetadata((OpaqueValue*)(dest + _addrOffset));
auto *destObject = (OpaqueValue *)(dest + _addrOffset);
auto *srcObject = (OpaqueValue *)(src + _addrOffset);
addrOffset = _addrOffset + (sizeof(uintptr_t) * NumWords_ValueBuffer);
if (srcType == destType) {
if (srcType->getValueWitnesses()->isValueInline()) {
srcType->vw_assignWithCopy(destObject, srcObject);
} else {
swift_release(*(HeapObject**)destObject);
memcpy(destObject, srcObject, sizeof(uintptr_t));
swift_retain(*(HeapObject**)srcObject);
}
return;
}
if (destType->getValueWitnesses()->isValueInline()) {
destType->vw_destroy(destObject);
} else {
swift_release(*(HeapObject**)destObject);
}
if (srcType->getValueWitnesses()->isValueInline()) {
srcType->vw_initializeWithCopy(destObject, srcObject);
} else {
memcpy(destObject, srcObject, sizeof(uintptr_t));
swift_retain(*(HeapObject**)srcObject);
}
}
static void metatypeAssignWithCopy(const Metadata *metadata,
LayoutStringReader1 &reader,
uintptr_t &addrOffset, uint8_t *dest,
uint8_t *src) {
uintptr_t _addrOffset = addrOffset;
auto *type = reader.readBytes<const Metadata *>();
auto *destObject = (OpaqueValue *)(dest + _addrOffset);
auto *srcObject = (OpaqueValue *)(src + _addrOffset);
addrOffset = _addrOffset + type->vw_size();
type->vw_assignWithCopy(destObject, srcObject);
}
static void resilientAssignWithCopy(const Metadata *metadata,
LayoutStringReader1 &reader,
uintptr_t &addrOffset,
uint8_t *dest,
uint8_t *src) {
uintptr_t _addrOffset = addrOffset;
auto *type = getResilientTypeMetadata(metadata, reader);
auto *destObject = (OpaqueValue *)(dest + _addrOffset);
auto *srcObject = (OpaqueValue *)(src + _addrOffset);
addrOffset = _addrOffset + type->vw_size();
type->vw_assignWithCopy(destObject, srcObject);
}
static void handleSingleRefCountDestroy(const Metadata *metadata,
LayoutStringReader1 &reader,
uintptr_t &addrOffset,
uint8_t *addr) {
auto tag = reader.readBytes<uint64_t>();
addrOffset += (tag & ~(0xFFULL << 56));
tag >>= 56;
if (SWIFT_UNLIKELY(tag == 0)) {
return;
}
destroyTable[tag](metadata, reader, addrOffset, addr);
}
static void handleSingleRefCountInitWithCopy(const Metadata *metadata,
LayoutStringReader1 &reader,
uintptr_t &addrOffset,
uint8_t *dest,
uint8_t *src) {
auto tag = reader.readBytes<uint64_t>();
auto _addrOffset = addrOffset;
auto offset = (tag & ~(0xFFULL << 56));
if (SWIFT_UNLIKELY(offset)) {
memcpy(dest + _addrOffset, src + _addrOffset, offset);
}
addrOffset = _addrOffset + offset;
tag >>= 56;
if (SWIFT_UNLIKELY(tag == 0)) {
return;
}
initWithCopyTable[tag](metadata, reader, addrOffset, dest, src);
}
static void singlePayloadEnumSimpleAssignWithCopy(const Metadata *metadata,
LayoutStringReader1 &reader,
uintptr_t &_addrOffset,
uint8_t *dest, uint8_t *src) {
uintptr_t addrOffset = _addrOffset;
reader.modify([&](LayoutStringReader1 &reader) {
uint64_t srcTagBytes = 0;
uint64_t destTagBytes = 0;
uint64_t byteCountsAndOffset;
size_t payloadSize;
uint64_t zeroTagValue;
size_t xiTagValues;
size_t refCountBytes;
size_t skip;
reader.readBytes(byteCountsAndOffset, payloadSize, zeroTagValue, xiTagValues, refCountBytes, skip);
auto extraTagBytesPattern = (uint8_t)(byteCountsAndOffset >> 62);
auto xiTagBytesPattern = ((uint8_t)(byteCountsAndOffset >> 59)) & 0x7;
auto xiTagBytesOffset =
byteCountsAndOffset & std::numeric_limits<uint32_t>::max();
if (SWIFT_UNLIKELY(extraTagBytesPattern)) {
auto extraTagBytes = 1 << (extraTagBytesPattern - 1);
srcTagBytes = readTagBytes(src + addrOffset + payloadSize, extraTagBytes);
destTagBytes = readTagBytes(dest + addrOffset + payloadSize, extraTagBytes);
}
if (SWIFT_LIKELY(xiTagBytesPattern)) {
auto xiTagBytes = 1 << (xiTagBytesPattern - 1);
srcTagBytes = srcTagBytes ? 0 :
readTagBytes(src + addrOffset + xiTagBytesOffset, xiTagBytes) -
zeroTagValue;
destTagBytes = destTagBytes ? 0 :
readTagBytes(dest + addrOffset + xiTagBytesOffset, xiTagBytes) -
zeroTagValue;
}
if (SWIFT_LIKELY(srcTagBytes >= xiTagValues &&
destTagBytes >= xiTagValues)) {
return;
} else if (srcTagBytes >= xiTagValues) {
const uint8_t *end = (reader.layoutStr + refCountBytes);
while (reader.layoutStr < end) {
handleSingleRefCountInitWithCopy(metadata, reader, addrOffset, dest, src);
}
return;
} else if (destTagBytes >= xiTagValues) {
const uint8_t *end = (reader.layoutStr + refCountBytes);
auto nestedAddrOffset = addrOffset;
while (reader.layoutStr < end) {
handleSingleRefCountDestroy(metadata, reader, nestedAddrOffset, dest);
}
} else {
reader.skip(refCountBytes);
}
memcpy(dest + addrOffset, src + addrOffset, skip);
addrOffset += skip;
});
_addrOffset = addrOffset;
}
static void singlePayloadEnumFNAssignWithCopy(const Metadata *metadata,
LayoutStringReader1 &reader,
uintptr_t &addrOffset,
uint8_t *dest,
uint8_t *src) {
reader.modify([&](LayoutStringReader1 &reader) {
GetEnumTagFn getEnumTag = readRelativeFunctionPointer<GetEnumTagFn>(reader);
size_t refCountBytes;
size_t skip;
reader.readBytes(refCountBytes, skip);
unsigned srcTag = getEnumTag(src + addrOffset);
unsigned destTag = getEnumTag(dest + addrOffset);
if (SWIFT_UNLIKELY(srcTag == 0 && destTag == 0)) {
return;
} else if (srcTag == 0) {
const uint8_t *end = (reader.layoutStr + refCountBytes);
while (reader.layoutStr < end) {
handleSingleRefCountInitWithCopy(metadata, reader, addrOffset, dest, src);
}
return;
} else if (destTag == 0) {
const uint8_t *end = (reader.layoutStr + refCountBytes);
auto nestedAddrOffset = addrOffset;
while (reader.layoutStr < end) {
handleSingleRefCountDestroy(metadata, reader, nestedAddrOffset, dest);
}
} else {
reader.skip(refCountBytes);
}
memcpy(dest + addrOffset, src + addrOffset, skip);
addrOffset += skip;
});
}
static void singlePayloadEnumFNResolvedAssignWithCopy(const Metadata *metadata,
LayoutStringReader1 &reader,
uintptr_t &addrOffset,
uint8_t *dest,
uint8_t *src) {
reader.modify([&](LayoutStringReader1 &reader) {
GetEnumTagFn getEnumTag;
size_t refCountBytes;
size_t skip;
reader.readBytes(getEnumTag, refCountBytes, skip);
unsigned srcTag = getEnumTag(src + addrOffset);
unsigned destTag = getEnumTag(dest + addrOffset);
if (SWIFT_UNLIKELY(srcTag == 0 && destTag == 0)) {
return;
} else if (srcTag == 0) {
const uint8_t *end = (reader.layoutStr + refCountBytes);
while (reader.layoutStr < end) {
handleSingleRefCountInitWithCopy(metadata, reader, addrOffset, dest, src);
}
return;
} else if (destTag == 0) {
const uint8_t *end = (reader.layoutStr + refCountBytes);
auto nestedAddrOffset = addrOffset;
while (reader.layoutStr < end) {
handleSingleRefCountDestroy(metadata, reader, nestedAddrOffset, dest);
}
} else {
reader.skip(refCountBytes);
}
memcpy(dest + addrOffset, src + addrOffset, skip);
addrOffset += skip;
});
}
static void singlePayloadEnumGenericAssignWithCopy(const Metadata *metadata,
LayoutStringReader1 &reader,
uintptr_t &addrOffset,
uint8_t *dest,
uint8_t *src) {
reader.modify([&](LayoutStringReader1 &reader) {
uint64_t srcTag = 0;
uint64_t destTag = 0;
auto tagBytesAndOffset = reader.readBytes<uint64_t>();
auto payloadSize = reader.readBytes<size_t>();
auto *xiType = reader.readBytes<const Metadata *>();
(void)reader.readBytes<unsigned>(); // numEmptyCases
auto refCountBytes = reader.readBytes<size_t>();
auto skip = reader.readBytes<size_t>();
auto extraTagBytesPattern = (uint8_t)(tagBytesAndOffset >> 62);
auto xiTagBytesOffset =
tagBytesAndOffset & std::numeric_limits<uint32_t>::max();
if (SWIFT_UNLIKELY(extraTagBytesPattern)) {
auto extraTagBytes = 1 << (extraTagBytesPattern - 1);
srcTag = readTagBytes(src + addrOffset + payloadSize, extraTagBytes);
destTag = readTagBytes(dest + addrOffset + payloadSize, extraTagBytes);
}
if (SWIFT_LIKELY(xiType)) {
if (!srcTag) {
srcTag = xiType->vw_getEnumTagSinglePayload(
(const OpaqueValue *)(src + addrOffset + xiTagBytesOffset),
xiType->vw_getNumExtraInhabitants());
}
if (!destTag) {
destTag = xiType->vw_getEnumTagSinglePayload(
(const OpaqueValue *)(dest + addrOffset + xiTagBytesOffset),
xiType->vw_getNumExtraInhabitants());
}
}
if (SWIFT_UNLIKELY(srcTag == 0 && destTag == 0)) {
return;
} else if (srcTag == 0) {
const uint8_t *end = (reader.layoutStr + refCountBytes);
while (reader.layoutStr < end) {
handleSingleRefCountInitWithCopy(metadata, reader, addrOffset, dest, src);
}
return;
} else if (destTag == 0) {
const uint8_t *end = (reader.layoutStr + refCountBytes);
auto nestedAddrOffset = addrOffset;
while (reader.layoutStr < end) {
handleSingleRefCountDestroy(metadata, reader, nestedAddrOffset, dest);
}
} else {
reader.skip(refCountBytes);
}
memcpy(dest + addrOffset, src + addrOffset, skip);
addrOffset += skip;
});
}
static void multiPayloadEnumFNAssignWithCopy(const Metadata *metadata,
LayoutStringReader1 &reader,
uintptr_t &addrOffset,
uint8_t *dest, uint8_t *src);
static void multiPayloadEnumFNResolvedAssignWithCopy(
const Metadata *metadata, LayoutStringReader1 &reader,
uintptr_t &addrOffset, uint8_t *dest, uint8_t *src) {
size_t numPayloads;
size_t refCountBytes;
size_t enumSize;
LayoutStringReader1 nestedReader;
uintptr_t nestedAddrOffset;
unsigned srcTag;
unsigned destTag;
reader.modify([&](LayoutStringReader1 &reader) {
GetEnumTagFn getEnumTag = reader.readBytes<GetEnumTagFn>();
reader.readBytes(numPayloads, refCountBytes, enumSize);
nestedReader = reader;
nestedAddrOffset = addrOffset;
srcTag = getEnumTag(src + addrOffset);
destTag = getEnumTag(dest + addrOffset);
reader.skip(refCountBytes + (numPayloads * sizeof(size_t)));
});
if (SWIFT_LIKELY(srcTag < numPayloads && destTag < numPayloads)) {
addrOffset += enumSize;
size_t srcRefCountOffset = nestedReader.peekBytes<size_t>(srcTag * sizeof(size_t));
size_t destRefCountOffset = nestedReader.peekBytes<size_t>(destTag * sizeof(size_t));
LayoutStringReader1 nestedReaderDest = nestedReader;
nestedReader.skip((numPayloads * sizeof(size_t)) + srcRefCountOffset);
nestedReaderDest.skip((numPayloads * sizeof(size_t)) + destRefCountOffset);
auto nestedAddrOffsetDest = nestedAddrOffset;
handleRefCountsDestroy(metadata, nestedReaderDest, nestedAddrOffsetDest, dest);
handleRefCountsInitWithCopy(metadata, nestedReader, nestedAddrOffset, dest, src);
auto trailingBytes = addrOffset - nestedAddrOffset;
if (trailingBytes)
memcpy(dest + nestedAddrOffset, src + nestedAddrOffset, trailingBytes);
return;
} else if (srcTag < numPayloads) {
addrOffset += enumSize;
size_t refCountOffset = nestedReader.peekBytes<size_t>(srcTag * sizeof(size_t));
nestedReader.skip((numPayloads * sizeof(size_t)) + refCountOffset);
handleRefCountsInitWithCopy(metadata, nestedReader, nestedAddrOffset, dest, src);
auto trailingBytes = addrOffset - nestedAddrOffset;
if (trailingBytes)
memcpy(dest + nestedAddrOffset, src + nestedAddrOffset, trailingBytes);
return;
} else if (destTag < numPayloads) {
size_t refCountOffset =
nestedReader.peekBytes<size_t>(destTag * sizeof(size_t));
nestedReader.skip((numPayloads * sizeof(size_t)) + refCountOffset);
handleRefCountsDestroy(metadata, nestedReader, nestedAddrOffset, dest);
}
memcpy(dest + addrOffset, src + addrOffset, enumSize);
addrOffset += enumSize;
}
static void multiPayloadEnumGenericAssignWithCopy(const Metadata *metadata,
LayoutStringReader1 &reader,
uintptr_t &addrOffset,
uint8_t *dest, uint8_t *src) {
size_t tagBytes;
size_t numPayloads;
size_t refCountBytes;
size_t enumSize;
uint64_t srcTag;
uint64_t destTag;
uintptr_t nestedAddrOffset;
LayoutStringReader1 nestedReader;
reader.modify([&](LayoutStringReader1 &reader) {
reader.readBytes(tagBytes, numPayloads, refCountBytes, enumSize);
nestedReader = reader;
nestedAddrOffset = addrOffset;
auto tagBytesOffset = enumSize - tagBytes;
srcTag = readTagBytes(src + addrOffset + tagBytesOffset, tagBytes);
destTag = readTagBytes(dest + addrOffset + tagBytesOffset, tagBytes);
reader.skip(refCountBytes + (numPayloads * sizeof(size_t)));
});
if (SWIFT_LIKELY(srcTag < numPayloads && destTag < numPayloads)) {
addrOffset += enumSize;
size_t srcRefCountOffset = nestedReader.peekBytes<size_t>(srcTag * sizeof(size_t));
size_t destRefCountOffset = nestedReader.peekBytes<size_t>(destTag * sizeof(size_t));
LayoutStringReader1 nestedReaderDest = nestedReader;
nestedReader.skip((numPayloads * sizeof(size_t)) + srcRefCountOffset);
nestedReaderDest.skip((numPayloads * sizeof(size_t)) + destRefCountOffset);
auto nestedAddrOffsetDest = nestedAddrOffset;
handleRefCountsDestroy(metadata, nestedReaderDest, nestedAddrOffsetDest, dest);
handleRefCountsInitWithCopy(metadata, nestedReader, nestedAddrOffset, dest, src);
auto trailingBytes = addrOffset - nestedAddrOffset;
if (trailingBytes)
memcpy(dest + nestedAddrOffset, src + nestedAddrOffset, trailingBytes);
return;
} else if (srcTag < numPayloads) {
addrOffset += enumSize;
size_t refCountOffset = nestedReader.peekBytes<size_t>(srcTag * sizeof(size_t));
nestedReader.skip((numPayloads * sizeof(size_t)) + refCountOffset);
handleRefCountsInitWithCopy(metadata, nestedReader, nestedAddrOffset, dest, src);
auto trailingBytes = addrOffset - nestedAddrOffset;
if (trailingBytes)
memcpy(dest + nestedAddrOffset, src + nestedAddrOffset, trailingBytes);
return;
} else if (destTag < numPayloads) {
size_t refCountOffset =
nestedReader.peekBytes<size_t>(destTag * sizeof(size_t));
nestedReader.skip((numPayloads * sizeof(size_t)) + refCountOffset);
handleRefCountsDestroy(metadata, nestedReader, nestedAddrOffset, dest);
}
memcpy(dest + addrOffset, src + addrOffset, enumSize);
addrOffset += enumSize;
}
typedef void (*AssignCPFn)(const Metadata *metadata, uint8_t *dest,
uint8_t *src);
constexpr InitFn assignWithCopyTable[] = {
&handleEnd,
&errorAssignWithCopy,
&nativeStrongAssignWithCopy,
&unownedAssignWithCopy,
&weakAssignWithCopy,
&unknownAssignWithCopy,
&unknownUnownedAssignWithCopy,
&unknownWeakAssignWithCopy,
&bridgeAssignWithCopy,
&blockAssignWithCopy,
&objcStrongAssignWithCopy,
&nativeSwiftObjcStrongAssignWithCopy,
&metatypeAssignWithCopy,
nullptr, // Generic
&existentialAssignWithCopy,
&resilientAssignWithCopy,
&singlePayloadEnumSimpleAssignWithCopy,
&singlePayloadEnumFNAssignWithCopy,
&singlePayloadEnumFNResolvedAssignWithCopy,
&singlePayloadEnumGenericAssignWithCopy,
&multiPayloadEnumFNAssignWithCopy,
&multiPayloadEnumFNResolvedAssignWithCopy,
&multiPayloadEnumGenericAssignWithCopy,
};
#if !(defined(__APPLE__) && defined(__arm64__))
static void handleRefCountsAssignWithCopy(const Metadata *metadata,
LayoutStringReader1 &reader,
uintptr_t &addrOffset, uint8_t *dest,
uint8_t *src) {
while (true) {
uintptr_t _addrOffset = addrOffset;
auto tag = reader.readBytes<uint64_t>();
auto offset = (tag & ~(0xFFULL << 56));
if (offset) {
memcpy(dest + _addrOffset, src + _addrOffset, offset);
}
addrOffset = _addrOffset + offset;
tag >>= 56;
if (SWIFT_UNLIKELY(tag == 0)) {
return;
}
assignWithCopyTable[tag](metadata, reader, addrOffset, dest, src);
}
}
#endif // !(defined(__APPLE__) && defined(__arm64__))
static void multiPayloadEnumFNAssignWithCopy(const Metadata *metadata,
LayoutStringReader1 &reader,
uintptr_t &addrOffset,
uint8_t *dest, uint8_t *src) {
size_t numPayloads;
size_t refCountBytes;
size_t enumSize;
LayoutStringReader1 nestedReader;
uintptr_t nestedAddrOffset;
unsigned srcTag;
unsigned destTag;
reader.modify([&](LayoutStringReader1 &reader) {
GetEnumTagFn getEnumTag = readRelativeFunctionPointer<GetEnumTagFn>(reader);
reader.readBytes(numPayloads, refCountBytes, enumSize);
nestedReader = reader;
nestedAddrOffset = addrOffset;
srcTag = getEnumTag(src + addrOffset);
destTag = getEnumTag(dest + addrOffset);
reader.skip(refCountBytes + (numPayloads * sizeof(size_t)));
});
if (SWIFT_LIKELY(srcTag < numPayloads && destTag < numPayloads)) {
addrOffset += enumSize;
size_t srcRefCountOffset = nestedReader.peekBytes<size_t>(srcTag * sizeof(size_t));
size_t destRefCountOffset = nestedReader.peekBytes<size_t>(destTag * sizeof(size_t));
if (srcTag == destTag) {
nestedReader.skip((numPayloads * sizeof(size_t)) + srcRefCountOffset);
#if defined(__APPLE__) && defined(__arm64__)
auto nestedAddrOffsetDest = nestedAddrOffset;
LayoutStringReader1 nestedReaderDest = nestedReader;
handleRefCounts(assignWithCopyTable, CONTINUE_WITH_COPY, metadata,
nestedReaderDest, nestedAddrOffsetDest, dest, src);
nestedAddrOffset = nestedAddrOffsetDest;
#else
handleRefCountsAssignWithCopy(metadata, nestedReader, nestedAddrOffset,
dest, src);
#endif
} else {
LayoutStringReader1 nestedReaderDest = nestedReader;
nestedReader.skip((numPayloads * sizeof(size_t)) + srcRefCountOffset);
nestedReaderDest.skip((numPayloads * sizeof(size_t)) +
destRefCountOffset);
auto nestedAddrOffsetDest = nestedAddrOffset;
handleRefCountsDestroy(metadata, nestedReaderDest, nestedAddrOffsetDest,
dest);
handleRefCountsInitWithCopy(metadata, nestedReader, nestedAddrOffset,
dest, src);
}
auto trailingBytes = addrOffset - nestedAddrOffset;
if (trailingBytes) {
memcpy(dest + nestedAddrOffset, src + nestedAddrOffset, trailingBytes);
}
return;
} else if (srcTag < numPayloads) {
addrOffset += enumSize;
size_t refCountOffset = nestedReader.peekBytes<size_t>(srcTag * sizeof(size_t));
nestedReader.skip((numPayloads * sizeof(size_t)) + refCountOffset);
handleRefCountsInitWithCopy(metadata, nestedReader, nestedAddrOffset, dest, src);
auto trailingBytes = addrOffset - nestedAddrOffset;
if (trailingBytes) {
memcpy(dest + nestedAddrOffset, src + nestedAddrOffset, trailingBytes);
}
return;
} else if (destTag < numPayloads) {
size_t refCountOffset =
nestedReader.peekBytes<size_t>(destTag * sizeof(size_t));
nestedReader.skip((numPayloads * sizeof(size_t)) + refCountOffset);
handleRefCountsDestroy(metadata, nestedReader, nestedAddrOffset, dest);
}
memcpy(dest + addrOffset, src + addrOffset, enumSize);
addrOffset += enumSize;
}
static swift::OpaqueValue *
swift_cvw_assignWithCopyImpl(swift::OpaqueValue *_dest,
swift::OpaqueValue *_src,
const Metadata *metadata) {
uint8_t *dest = (uint8_t *)_dest;
uint8_t *src = (uint8_t *)_src;
const uint8_t *layoutStr = metadata->getLayoutString();
LayoutStringReader1 reader{layoutStr + layoutStringHeaderSize};
uintptr_t addrOffset = 0;
#if defined(__APPLE__) && defined(__arm64__)
handleRefCounts(assignWithCopyTable, CONTINUE_WITH_COPY, metadata, reader,
addrOffset, dest, src);
#else
handleRefCountsAssignWithCopy(metadata, reader, addrOffset, dest, src);
#endif
assert(addrOffset == metadata->vw_size());
return _dest;
}
void swift::swift_cvw_arrayAssignWithCopy(swift::OpaqueValue *_dest,
swift::OpaqueValue *_src,
size_t count, size_t stride,
const Metadata *metadata) {
uint8_t *dest = (uint8_t *)_dest;
uint8_t *src = (uint8_t *)_src;
const uint8_t *layoutStr = metadata->getLayoutString();
for (size_t i = 0; i < count; i++) {
LayoutStringReader1 reader{layoutStr + layoutStringHeaderSize};
uintptr_t addrOffset = i * stride;
#if defined(__APPLE__) && defined(__arm64__)
handleRefCounts(assignWithCopyTable, CONTINUE_WITH_COPY, metadata, reader,
addrOffset, dest, src);
#else
handleRefCountsAssignWithCopy(metadata, reader, addrOffset, dest, src);
#endif
}
}
static swift::OpaqueValue *
swift_cvw_assignWithTakeImpl(swift::OpaqueValue *dest, swift::OpaqueValue *src,
const Metadata *metadata) {
swift_cvw_destroy(dest, metadata);
return swift_cvw_initWithTake(dest, src, metadata);
}
extern "C" unsigned
swift_cvw_singletonEnum_getEnumTag(swift::OpaqueValue *address,
const Metadata *metadata) {
return 0;
}
extern "C" void swift_cvw_singletonEnum_destructiveInjectEnumTag(
swift::OpaqueValue *address, unsigned tag, const Metadata *metadata) {
return;
}
template <typename T>
static inline T handleSinglePayloadEnumSimpleTag(
LayoutStringReader &reader, uint8_t *addr,
std::function<std::optional<T>(size_t, size_t, uint8_t)>
extraTagBytesHandler,
std::function<T(size_t, uint64_t, uint8_t, unsigned, size_t, uint8_t)>
xiHandler) {
auto byteCountsAndOffset = reader.readBytes<uint64_t>();
auto extraTagBytesPattern = (uint8_t)(byteCountsAndOffset >> 62);
auto xiTagBytesPattern = ((uint8_t)(byteCountsAndOffset >> 59)) & 0x7;
auto xiTagBytesOffset =
byteCountsAndOffset & std::numeric_limits<uint32_t>::max();
auto numExtraTagBytes = 1 << (extraTagBytesPattern - 1);
auto payloadSize = reader.readBytes<size_t>();
auto zeroTagValue = reader.readBytes<uint64_t>();
auto payloadNumExtraInhabitants = reader.readBytes<size_t>();
if (extraTagBytesPattern) {
if (auto result = extraTagBytesHandler(payloadNumExtraInhabitants,
payloadSize, numExtraTagBytes)) {
return *result;
}
}
return xiHandler(payloadNumExtraInhabitants, zeroTagValue, xiTagBytesPattern,
xiTagBytesOffset, payloadSize, numExtraTagBytes);
}
static unsigned swift_cvw_enumSimple_getEnumTagImpl(swift::OpaqueValue *address,
const Metadata *metadata) {
auto addr = reinterpret_cast<uint8_t *>(address);
LayoutStringReader reader{metadata->getLayoutString(),
layoutStringHeaderSize + sizeof(uint64_t)};
auto extraTagBytesHandler =
[addr](size_t payloadNumExtraInhabitants, size_t payloadSize,
uint8_t numExtraTagBytes) -> std::optional<unsigned> {
auto tagBytes = readTagBytes(addr + payloadSize, numExtraTagBytes);
if (tagBytes) {
unsigned caseIndexFromExtraTagBits =
payloadSize >= 4 ? 0 : (tagBytes - 1U) << (payloadSize * 8U);
unsigned caseIndexFromValue = loadEnumElement(addr, payloadSize);
unsigned noPayloadIndex =
(caseIndexFromExtraTagBits | caseIndexFromValue) +
payloadNumExtraInhabitants;
return noPayloadIndex + 1;
}
return std::nullopt;
};
auto xihandler = [addr](size_t payloadNumExtraInhabitants,
uint64_t zeroTagValue, uint8_t xiTagBytesPattern,
unsigned xiTagBytesOffset, size_t payloadSize,
uint8_t numExtraTagBytes) -> unsigned {
if (xiTagBytesPattern) {
auto xiTagBytes = 1 << (xiTagBytesPattern - 1);
uint64_t tagBytes =
readTagBytes(addr + xiTagBytesOffset, xiTagBytes) - zeroTagValue;
if (tagBytes < payloadNumExtraInhabitants) {
return tagBytes + 1;
}
}
return 0;
};
return handleSinglePayloadEnumSimpleTag<unsigned>(
reader, addr, extraTagBytesHandler, xihandler);
}
static void swift_cvw_enumSimple_destructiveInjectEnumTagImpl(
swift::OpaqueValue *address, unsigned tag, const Metadata *metadata) {
auto addr = reinterpret_cast<uint8_t *>(address);
LayoutStringReader reader{metadata->getLayoutString(),
layoutStringHeaderSize + sizeof(uint64_t)};
auto extraTagBytesHandler =
[addr, tag](size_t payloadNumExtraInhabitants, size_t payloadSize,
uint8_t numExtraTagBytes) -> std::optional<bool> {
if (tag <= payloadNumExtraInhabitants) {
return std::nullopt;
}
unsigned noPayloadIndex = tag - 1;
unsigned caseIndex = noPayloadIndex - payloadNumExtraInhabitants;
unsigned payloadIndex, extraTagIndex;
if (payloadSize >= 4) {
extraTagIndex = 1;
payloadIndex = caseIndex;
} else {
unsigned payloadBits = payloadSize * 8U;
extraTagIndex = 1U + (caseIndex >> payloadBits);
payloadIndex = caseIndex & ((1U << payloadBits) - 1U);
}
// Store into the value.
if (payloadSize)
storeEnumElement(addr, payloadIndex, payloadSize);
if (numExtraTagBytes)
storeEnumElement(addr + payloadSize, extraTagIndex, numExtraTagBytes);
return true;
};
auto xihandler = [addr, tag](size_t payloadNumExtraInhabitants,
uint64_t zeroTagValue, uint8_t xiTagBytesPattern,
unsigned xiTagBytesOffset, size_t payloadSize,
uint8_t numExtraTagBytes) -> bool {
if (xiTagBytesPattern) {
auto xiTagBytes = 1 << (xiTagBytesPattern - 1);
if (tag <= payloadNumExtraInhabitants) {
if (numExtraTagBytes != 0)
storeEnumElement(addr + payloadSize, 0, numExtraTagBytes);
if (tag == 0)
return true;
storeEnumElement(addr + xiTagBytesOffset, tag - 1 + zeroTagValue,
xiTagBytes);
}
}
return true;
};
handleSinglePayloadEnumSimpleTag<bool>(reader, addr, extraTagBytesHandler,
xihandler);
}
static unsigned swift_cvw_enumFn_getEnumTagImpl(swift::OpaqueValue *address,
const Metadata *metadata) {
auto addr = reinterpret_cast<const uint8_t *>(address);
LayoutStringReader reader{metadata->getLayoutString(),
layoutStringHeaderSize + sizeof(uint64_t)};
auto getEnumTag = readRelativeFunctionPointer<GetEnumTagFn>(reader);
return getEnumTag(addr);
}
static unsigned
swift_cvw_multiPayloadEnumGeneric_getEnumTagImpl(swift::OpaqueValue *address,
const Metadata *metadata) {
auto addr = reinterpret_cast<const uint8_t *>(address);
LayoutStringReader1 reader{metadata->getLayoutString() +
layoutStringHeaderSize + sizeof(uint64_t)};
auto tagBytes = reader.readBytes<size_t>();
auto numPayloads = reader.readBytes<size_t>();
reader.skip(sizeof(size_t));
auto enumSize = reader.readBytes<size_t>();
auto payloadSize = enumSize - tagBytes;
auto enumTag = (unsigned)readTagBytes(addr + payloadSize, tagBytes);
if (enumTag < numPayloads) {
return enumTag;
}
auto payloadValue = loadEnumElement(addr, payloadSize);
if (payloadSize >= 4) {
return numPayloads + payloadValue;
} else {
unsigned numPayloadBits = payloadSize * CHAR_BIT;
return (payloadValue | (enumTag - numPayloads) << numPayloadBits) +
numPayloads;
}
}
static void swift_cvw_multiPayloadEnumGeneric_destructiveInjectEnumTagImpl(
swift::OpaqueValue *address, unsigned tag, const Metadata *metadata) {
auto addr = reinterpret_cast<uint8_t *>(address);
LayoutStringReader reader{metadata->getLayoutString(),
layoutStringHeaderSize + sizeof(uint64_t)};
auto numTagBytes = reader.readBytes<size_t>();
auto numPayloads = reader.readBytes<size_t>();
reader.skip(sizeof(size_t));
auto enumSize = reader.readBytes<size_t>();
auto payloadSize = enumSize - numTagBytes;
if (tag < numPayloads) {
// For a payload case, store the tag after the payload area.
auto tagBytes = addr + payloadSize;
storeEnumElement(tagBytes, tag, numTagBytes);
} else {
// For an empty case, factor out the parts that go in the payload and
// tag areas.
unsigned whichEmptyCase = tag - numPayloads;
unsigned whichTag, whichPayloadValue;
if (payloadSize >= 4) {
whichTag = numPayloads;
whichPayloadValue = whichEmptyCase;
} else {
unsigned numPayloadBits = payloadSize * CHAR_BIT;
whichTag = numPayloads + (whichEmptyCase >> numPayloadBits);
whichPayloadValue = whichEmptyCase & ((1U << numPayloadBits) - 1U);
}
auto tagBytes = addr + payloadSize;
storeEnumElement(tagBytes, whichTag, numTagBytes);
storeEnumElement(addr, whichPayloadValue, payloadSize);
}
}
template <typename T>
static inline T handleSinglePayloadEnumGenericTag(
LayoutStringReader &reader, uint8_t *addr,
std::function<std::optional<T>(const Metadata *, size_t, uint8_t)>
extraTagBytesHandler,
std::function<T(const Metadata *, unsigned, unsigned, size_t, uint8_t)>
xiHandler) {
auto tagBytesAndOffset = reader.readBytes<uint64_t>();
auto extraTagBytesPattern = (uint8_t)(tagBytesAndOffset >> 62);
auto xiTagBytesOffset =
tagBytesAndOffset & std::numeric_limits<uint32_t>::max();
auto numExtraTagBytes = 1 << (extraTagBytesPattern - 1);
auto payloadSize = reader.readBytes<size_t>();
auto xiType = reader.readBytes<const Metadata *>();
if (extraTagBytesPattern) {
if (auto result =
extraTagBytesHandler(xiType, payloadSize, numExtraTagBytes)) {
return *result;
}
}
auto numEmptyCases = reader.readBytes<unsigned>();
return xiHandler(xiType, xiTagBytesOffset, numEmptyCases, payloadSize,
numExtraTagBytes);
}
static unsigned
swift_cvw_singlePayloadEnumGeneric_getEnumTagImpl(swift::OpaqueValue *address,
const Metadata *metadata) {
auto addr = reinterpret_cast<uint8_t *>(address);
LayoutStringReader reader{metadata->getLayoutString(),
layoutStringHeaderSize + sizeof(uint64_t)};
auto extraTagBytesHandler =
[addr](const Metadata *xiType, size_t payloadSize,
uint8_t numExtraTagBytes) -> std::optional<unsigned> {
auto tagBytes = readTagBytes(addr + payloadSize, numExtraTagBytes);
if (tagBytes) {
unsigned payloadNumExtraInhabitants =
xiType ? xiType->vw_getNumExtraInhabitants() : 0;
unsigned caseIndexFromExtraTagBits =
payloadSize >= 4 ? 0 : (tagBytes - 1U) << (payloadSize * 8U);
unsigned caseIndexFromValue = loadEnumElement(addr, payloadSize);
unsigned noPayloadIndex =
(caseIndexFromExtraTagBits | caseIndexFromValue) +
payloadNumExtraInhabitants;
return noPayloadIndex + 1;
}
return std::nullopt;
};
auto xihandler = [addr](const Metadata *xiType, unsigned xiTagBytesOffset,
unsigned numEmptyCases, size_t payloadSize,
uint8_t numExtraTagBytes) -> unsigned {
if (xiType) {
return xiType->vw_getEnumTagSinglePayload(
(const OpaqueValue *)(addr + xiTagBytesOffset),
xiType->vw_getNumExtraInhabitants());
}
return 0;
};
return handleSinglePayloadEnumGenericTag<unsigned>(
reader, addr, extraTagBytesHandler, xihandler);
}
static void swift_cvw_singlePayloadEnumGeneric_destructiveInjectEnumTagImpl(
swift::OpaqueValue *address, unsigned tag, const Metadata *metadata) {
auto addr = reinterpret_cast<uint8_t *>(address);
LayoutStringReader reader{metadata->getLayoutString(),
layoutStringHeaderSize + sizeof(uint64_t)};
auto extraTagBytesHandler =
[=](const Metadata *xiType, size_t payloadSize,
uint8_t numExtraTagBytes) -> std::optional<bool> {
unsigned payloadNumExtraInhabitants =
xiType ? xiType->vw_getNumExtraInhabitants() : 0;
if (tag <= payloadNumExtraInhabitants) {
return std::nullopt;
}
unsigned noPayloadIndex = tag - 1;
unsigned caseIndex = noPayloadIndex - payloadNumExtraInhabitants;
unsigned payloadIndex, extraTagIndex;
if (payloadSize >= 4) {
extraTagIndex = 1;
payloadIndex = caseIndex;
} else {
unsigned payloadBits = payloadSize * 8U;
extraTagIndex = 1U + (caseIndex >> payloadBits);
payloadIndex = caseIndex & ((1U << payloadBits) - 1U);
}
// Store into the value.
if (payloadSize)
storeEnumElement(addr, payloadIndex, payloadSize);
if (numExtraTagBytes)
storeEnumElement(addr + payloadSize, extraTagIndex, numExtraTagBytes);
return true;
};
auto xihandler = [=](const Metadata *xiType, unsigned xiTagBytesOffset,
unsigned numEmptyCases, size_t payloadSize,
uint8_t numExtraTagBytes) -> bool {
unsigned payloadNumExtraInhabitants =
xiType ? xiType->vw_getNumExtraInhabitants() : 0;
if (tag <= payloadNumExtraInhabitants) {
if (numExtraTagBytes != 0)
storeEnumElement(addr + payloadSize, 0, numExtraTagBytes);
if (tag == 0)
return true;
xiType->vw_storeEnumTagSinglePayload(
(swift::OpaqueValue *)(addr + xiTagBytesOffset), tag, numEmptyCases);
}
return true;
};
handleSinglePayloadEnumGenericTag<bool>(reader, addr, extraTagBytesHandler,
xihandler);
}
static swift::OpaqueValue *
swift_cvw_initializeBufferWithCopyOfBufferImpl(swift::ValueBuffer *dest,
swift::ValueBuffer *src,
const Metadata *metadata) {
if (metadata->getValueWitnesses()->isValueInline()) {
return swift_cvw_initWithCopy((swift::OpaqueValue *)dest,
(swift::OpaqueValue *)src, metadata);
} else {
memcpy(dest, src, sizeof(swift::HeapObject *));
swift_retain(*(swift::HeapObject **)src);
return (swift::OpaqueValue *)&(*(swift::HeapObject **)dest)[1];
}
}
void swift::swift_cvw_resolve_resilientAccessors(uint8_t *layoutStr,
size_t layoutStrOffset,
const uint8_t *fieldLayoutStr,
const Metadata *fieldType) {
LayoutStringWriter writer{layoutStr, layoutStrOffset};
LayoutStringReader reader{fieldLayoutStr, 0};
while (true) {
size_t currentOffset = reader.offset + layoutStringHeaderSize;
uint64_t size = reader.readBytes<uint64_t>();
RefCountingKind tag = (RefCountingKind)(size >> 56);
size &= ~(0xffULL << 56);
switch (tag) {
case RefCountingKind::End:
return;
case RefCountingKind::Resilient: {
auto *type = getResilientTypeMetadata(fieldType, reader);
writer.offset = layoutStrOffset + currentOffset - layoutStringHeaderSize;
uint64_t tagAndOffset =
(((uint64_t)RefCountingKind::Metatype) << 56) | size;
writer.writeBytes(tagAndOffset);
writer.writeBytes(type);
break;
}
case RefCountingKind::Metatype:
reader.skip(sizeof(uintptr_t));
break;
case RefCountingKind::SinglePayloadEnumSimple:
reader.skip((2 * sizeof(uint64_t)) + (4 * sizeof(size_t)));
break;
case RefCountingKind::SinglePayloadEnumFN: {
auto getEnumTag = readRelativeFunctionPointer<GetEnumTagFn>(reader);
writer.offset = layoutStrOffset + currentOffset - layoutStringHeaderSize;
uint64_t tagAndOffset =
(((uint64_t)RefCountingKind::SinglePayloadEnumFNResolved) << 56) |
size;
writer.writeBytes(tagAndOffset);
writer.writeBytes(getEnumTag);
reader.skip(2 * sizeof(size_t));
break;
}
case RefCountingKind::SinglePayloadEnumFNResolved:
reader.skip(3 * sizeof(size_t));
break;
case RefCountingKind::SinglePayloadEnumGeneric: {
reader.skip(sizeof(uint64_t) + // tag + offset
sizeof(uint64_t) + // extra tag bytes + XI offset
sizeof(size_t) + // payload size
sizeof(uintptr_t) + // XI metadata
sizeof(unsigned)); // num empty cases
auto refCountBytes = reader.readBytes<size_t>();
reader.skip(sizeof(size_t) + // bytes to skip if no payload case
refCountBytes);
break;
}
case RefCountingKind::MultiPayloadEnumFN: {
auto getEnumTag = readRelativeFunctionPointer<GetEnumTagFn>(reader);
writer.offset = layoutStrOffset + currentOffset - layoutStringHeaderSize;
uint64_t tagAndOffset =
(((uint64_t)RefCountingKind::MultiPayloadEnumFNResolved) << 56) |
size;
writer.writeBytes(tagAndOffset);
writer.writeBytes(getEnumTag);
size_t numCases = reader.readBytes<size_t>();
auto refCountBytes = reader.readBytes<size_t>();
// skip enum size
reader.skip(sizeof(size_t));
size_t casesBeginOffset = layoutStrOffset + reader.offset +
(numCases * sizeof(size_t));
auto fieldCasesBeginOffset = fieldLayoutStr + (numCases * sizeof(size_t)) + reader.offset;
for (size_t j = 0; j < numCases; j++) {
size_t caseOffset = reader.readBytes<size_t>();
const uint8_t *caseLayoutString = fieldCasesBeginOffset +
caseOffset;
swift_cvw_resolve_resilientAccessors(layoutStr,
casesBeginOffset + caseOffset,
caseLayoutString, fieldType);
}
reader.skip(refCountBytes);
break;
}
case RefCountingKind::MultiPayloadEnumFNResolved: {
// skip function pointer
reader.skip(sizeof(uintptr_t));
size_t numCases = reader.readBytes<size_t>();
size_t refCountBytes = reader.readBytes<size_t>();
// skip enum size, offsets and ref counts
reader.skip(sizeof(size_t) + (numCases * sizeof(size_t)) + refCountBytes);
break;
}
case RefCountingKind::MultiPayloadEnumGeneric: {
reader.skip(sizeof(size_t));
auto numPayloads = reader.readBytes<size_t>();
auto refCountBytes = reader.readBytes<size_t>();
reader.skip(sizeof(size_t) * (numPayloads + 1) + refCountBytes);
break;
}
default:
break;
}
}
}
static void swift_cvw_destroyMultiPayloadEnumFNImpl(swift::OpaqueValue *address,
const Metadata *metadata) {
const uint8_t *layoutStr = metadata->getLayoutString();
LayoutStringReader1 reader{layoutStr + layoutStringHeaderSize};
uintptr_t addrOffset = 0;
uint8_t *addr = (uint8_t *)address;
#ifndef NDEBUG
assert(reader.readBytes<uint64_t>() ==
((uint64_t)RefCountingKind::MultiPayloadEnumFN) << 56 &&
"Invalid tag, expected MultiPayloadEnumFN");
#else
reader.skip(sizeof(uint64_t));
#endif
multiPayloadEnumFN<handleRefCountsDestroy>(metadata, reader, addrOffset,
addr);
}
static swift::OpaqueValue *
swift_cvw_assignWithCopyMultiPayloadEnumFNImpl(swift::OpaqueValue *_dest,
swift::OpaqueValue *_src,
const Metadata *metadata) {
const uint8_t *layoutStr = metadata->getLayoutString();
LayoutStringReader1 reader{layoutStr + layoutStringHeaderSize};
uintptr_t addrOffset = 0;
uint8_t *dest = (uint8_t *)_dest;
uint8_t *src = (uint8_t *)_src;
#ifndef NDEBUG
assert(reader.readBytes<uint64_t>() ==
((uint64_t)RefCountingKind::MultiPayloadEnumFN) << 56 &&
"Invalid tag, expected MultiPayloadEnumFN");
#else
reader.skip(sizeof(uint64_t));
#endif
multiPayloadEnumFNAssignWithCopy(metadata, reader, addrOffset, dest, src);
return _dest;
}
static swift::OpaqueValue *
swift_cvw_assignWithTakeMultiPayloadEnumFNImpl(swift::OpaqueValue *dest,
swift::OpaqueValue *src,
const Metadata *metadata) {
swift_cvw_destroyMultiPayloadEnumFN(dest, metadata);
return swift_cvw_initWithTake(dest, src, metadata);
}
static swift::OpaqueValue *
swift_cvw_initWithCopyMultiPayloadEnumFNImpl(swift::OpaqueValue *_dest,
swift::OpaqueValue *_src,
const Metadata *metadata) {
const uint8_t *layoutStr = metadata->getLayoutString();
LayoutStringReader1 reader{layoutStr + layoutStringHeaderSize};
uintptr_t addrOffset = 0;
uint8_t *dest = (uint8_t *)_dest;
uint8_t *src = (uint8_t *)_src;
#ifndef NDEBUG
assert(reader.readBytes<uint64_t>() ==
((uint64_t)RefCountingKind::MultiPayloadEnumFN) << 56 &&
"Invalid tag, expected MultiPayloadEnumFN");
#else
reader.skip(sizeof(uint64_t));
#endif
multiPayloadEnumFN<handleRefCountsInitWithCopy>(metadata, reader, addrOffset,
dest, src);
return _dest;
}
static swift::OpaqueValue *
swift_cvw_initWithTakeMultiPayloadEnumFNImpl(swift::OpaqueValue *_dest,
swift::OpaqueValue *_src,
const Metadata *metadata) {
if (SWIFT_LIKELY(metadata->getValueWitnesses()->isBitwiseTakable())) {
size_t size = metadata->vw_size();
memcpy(_dest, _src, size);
return _dest;
}
const uint8_t *layoutStr = metadata->getLayoutString();
LayoutStringReader1 reader{layoutStr + layoutStringHeaderSize};
uintptr_t addrOffset = 0;
uint8_t *dest = (uint8_t *)_dest;
uint8_t *src = (uint8_t *)_src;
#ifndef NDEBUG
assert(reader.readBytes<uint64_t>() ==
((uint64_t)RefCountingKind::MultiPayloadEnumFN) << 56 &&
"Invalid tag, expected MultiPayloadEnumFN");
#else
reader.skip(sizeof(uint64_t));
#endif
multiPayloadEnumFN<handleRefCountsInitWithTake>(metadata, reader, addrOffset,
dest, src);
return _dest;
}
static swift::OpaqueValue *
swift_cvw_initializeBufferWithCopyOfBufferMultiPayloadEnumFNImpl(
swift::ValueBuffer *dest, swift::ValueBuffer *src,
const Metadata *metadata) {
if (metadata->getValueWitnesses()->isValueInline()) {
return swift_cvw_initWithCopyMultiPayloadEnumFN(
(swift::OpaqueValue *)dest, (swift::OpaqueValue *)src, metadata);
} else {
memcpy(dest, src, sizeof(swift::HeapObject *));
swift_retain(*(swift::HeapObject **)src);
return (swift::OpaqueValue *)&(*(swift::HeapObject **)dest)[1];
}
}
extern "C" void swift_cvw_instantiateLayoutString(const uint8_t *layoutStr,
Metadata *type) {
type->setLayoutString(layoutStr);
}
// Forwarders for compatibility reasons
extern "C" void swift_generic_destroy(swift::OpaqueValue *address,
const Metadata *metadata) {
swift_cvw_destroy(address, metadata);
}
extern "C" swift::OpaqueValue *
swift_generic_assignWithCopy(swift::OpaqueValue *dest, swift::OpaqueValue *src,
const Metadata *metadata) {
return swift_cvw_assignWithCopy(dest, src, metadata);
}
extern "C" swift::OpaqueValue *
swift_generic_assignWithTake(swift::OpaqueValue *dest, swift::OpaqueValue *src,
const Metadata *metadata) {
return swift_cvw_assignWithTake(dest, src, metadata);
}
extern "C" swift::OpaqueValue *
swift_generic_initWithCopy(swift::OpaqueValue *dest, swift::OpaqueValue *src,
const Metadata *metadata) {
return swift_cvw_initWithCopy(dest, src, metadata);
}
extern "C" swift::OpaqueValue *
swift_generic_initWithTake(swift::OpaqueValue *dest, swift::OpaqueValue *src,
const Metadata *metadata) {
return swift_cvw_initWithTake(dest, src, metadata);
}
extern "C" swift::OpaqueValue *
swift_generic_initializeBufferWithCopyOfBuffer(swift::ValueBuffer *dest,
swift::ValueBuffer *src,
const Metadata *metadata) {
return swift_cvw_initializeBufferWithCopyOfBuffer(dest, src, metadata);
}
extern "C" unsigned swift_enumSimple_getEnumTag(swift::OpaqueValue *address,
const Metadata *metadata) {
return swift_cvw_enumSimple_getEnumTag(address, metadata);
}
extern "C" void swift_enumSimple_destructiveInjectEnumTag(
swift::OpaqueValue *address, unsigned tag, const Metadata *metadata) {
swift_cvw_enumSimple_destructiveInjectEnumTag(address, tag, metadata);
}
extern "C" unsigned swift_enumFn_getEnumTag(swift::OpaqueValue *address,
const Metadata *metadata) {
return swift_cvw_enumFn_getEnumTag(address, metadata);
}
extern "C" unsigned
swift_multiPayloadEnumGeneric_getEnumTag(swift::OpaqueValue *address,
const Metadata *metadata) {
return swift_cvw_multiPayloadEnumGeneric_getEnumTag(address, metadata);
}
extern "C" void swift_multiPayloadEnumGeneric_destructiveInjectEnumTag(
swift::OpaqueValue *address, unsigned tag, const Metadata *metadata) {
swift_cvw_multiPayloadEnumGeneric_destructiveInjectEnumTag(address, tag,
metadata);
}
extern "C" unsigned
swift_singlePayloadEnumGeneric_getEnumTag(swift::OpaqueValue *address,
const Metadata *metadata) {
return swift_cvw_singlePayloadEnumGeneric_getEnumTagImpl(address, metadata);
}
extern "C" void swift_singlePayloadEnumGeneric_destructiveInjectEnumTag(
swift::OpaqueValue *address, unsigned tag, const Metadata *metadata) {
swift_cvw_singlePayloadEnumGeneric_destructiveInjectEnumTag(address, tag,
metadata);
}
extern "C" unsigned swift_singletonEnum_getEnumTag(swift::OpaqueValue *address,
const Metadata *metadata) {
return 0;
}
extern "C" void swift_singletonEnum_destructiveInjectEnumTag(
swift::OpaqueValue *address, unsigned tag, const Metadata *metadata) {
return;
}
extern "C" void swift_generic_instantiateLayoutString(const uint8_t *layoutStr,
Metadata *type) {
swift_cvw_instantiateLayoutString(layoutStr, type);
}
#define OVERRIDE_CVW COMPATIBILITY_OVERRIDE
#include "../CompatibilityOverride/CompatibilityOverrideIncludePath.h"
Reference in New Issue View Git Blame Copy Permalink