averello/swift-mirror
1
0
Fork 0
mirror of https://github.com/apple/swift.git synced 2025-12-21 12:14:44 +01:00
Code Issues Packages Projects Releases Wiki Activity

Add value witnesses for single payload enums

Browse Source 
So far single payload enums were implemented in terms of runtime functions which
internally emitted several calls to value witnesses.

This commit adds value witnesses to get and store the enum tag side stepping the
need for witness calls as this information is statically available in many cases

/// int (*getEnumTagSinglePayload)(const T* enum, UINT_TYPE emptyCases)
/// Given an instance of valid single payload enum with a payload of this
/// witness table's type (e.g Optional<ThisType>) , get the tag of the enum.

/// void (*storeEnumTagSinglePayload)(T* enum, INT_TYPE whichCase,
///                                   UINT_TYPE emptyCases)
/// Given uninitialized memory for an instance of a single payload enum with a
/// payload of this witness table's type (e.g Optional<ThisType>), store the
/// tag.

A simple 'for element in array' loop in generic code operating on a
ContigousArray of Int is ~25% faster on arm64.

rdar://31408033
This commit is contained in:
Arnold Schwaighofer
2017-10-05 13:12:40 -07:00
parent 34e602b93a
commit 43b9d13a2e
18 changed files with 995 additions and 221 deletions
Download Patch File Download Diff File Expand all files Collapse all files

181
stdlib/public/runtime/Enum.cpp
View File

@@ -20,60 +20,12 @@
#include "swift/Runtime/Enum.h"
#include "swift/Runtime/Debug.h"
#include "Private.h"
#include "EnumImpl.h"
#include <cstring>
#include <algorithm>
using namespace swift;
static unsigned getNumTagBytes(size_t size, unsigned emptyCases,
unsigned payloadCases) {
// We can use the payload area with a tag bit set somewhere outside of the
// payload area to represent cases. See how many bytes we need to cover
// all the empty cases.
unsigned numTags = payloadCases;
if (emptyCases > 0) {
if (size >= 4)
// Assume that one tag bit is enough if the precise calculation overflows
// an int32.
numTags += 1;
else {
unsigned bits = size * 8U;
unsigned casesPerTagBitValue = 1U << bits;
numTags += ((emptyCases + (casesPerTagBitValue-1U)) >> bits);
}
}
return (numTags <= 1 ? 0 :
numTags < 256 ? 1 :
numTags < 65536 ? 2 : 4);
}
/// This is a small and fast implementation of memcpy with a constant count. It
/// should be a performance win for small constant values where the function
/// can be inlined, the loop unrolled and the memory accesses merged.
template <unsigned count> static void small_memcpy(void *dest, const void *src) {
uint8_t *d8 = (uint8_t*)dest;
const uint8_t *s8 = (const uint8_t*)src;
for (unsigned i = 0; i < count; i++) {
*d8++ = *s8++;
}
}
static inline void small_memcpy(void *dest, const void *src, unsigned count) {
// This is specialization of the memcpy line below with
// specialization for values of 1, 2 and 4.
// memcpy(dst, src, count)
if (count == 1) {
small_memcpy<1>(dest, src);
} else if (count == 2) {
small_memcpy<2>(dest, src);
} else if (count == 4) {
small_memcpy<4>(dest, src);
} else {
crash("Tagbyte values should be 1, 2 or 4.");
}
}
void
swift::swift_initEnumValueWitnessTableSinglePayload(ValueWitnessTable *vwtable,
const TypeLayout *payloadLayout,
@@ -146,130 +98,33 @@ int swift::swift_getEnumCaseSinglePayload(const OpaqueValue *value,
const Metadata *payload,
unsigned emptyCases)
SWIFT_CC(RegisterPreservingCC_IMPL) {
auto *payloadWitnesses = payload->getValueWitnesses();
auto payloadSize = payloadWitnesses->getSize();
auto payloadNumExtraInhabitants = payloadWitnesses->getNumExtraInhabitants();
auto size = payloadWitnesses->getSize();
auto numExtraInhabitants = payloadWitnesses->getNumExtraInhabitants();
auto getExtraInhabitantIndex =
(static_cast<const ExtraInhabitantsValueWitnessTable *>(payloadWitnesses)
->getExtraInhabitantIndex);
// If there are extra tag bits, check them.
if (emptyCases > payloadNumExtraInhabitants) {
auto *valueAddr = reinterpret_cast<const uint8_t*>(value);
auto *extraTagBitAddr = valueAddr + payloadSize;
unsigned extraTagBits = 0;
unsigned numBytes = getNumTagBytes(payloadSize,
emptyCases-payloadNumExtraInhabitants,
1 /*payload case*/);
#if defined(__BIG_ENDIAN__)
small_memcpy(reinterpret_cast<uint8_t*>(&extraTagBits) + 4 - numBytes,
extraTagBitAddr, numBytes);
#else
small_memcpy(&extraTagBits, extraTagBitAddr, numBytes);
#endif
// If the extra tag bits are zero, we have a valid payload or
// extra inhabitant (checked below). If nonzero, form the case index from
// the extra tag value and the value stored in the payload.
if (extraTagBits > 0) {
unsigned caseIndexFromExtraTagBits = payloadSize >= 4
? 0 : (extraTagBits - 1U) << (payloadSize*8U);
// In practice we should need no more than four bytes from the payload
// area.
unsigned caseIndexFromValue = 0;
#if defined(__BIG_ENDIAN__)
unsigned numPayloadTagBytes = std::min(size_t(4), payloadSize);
memcpy(reinterpret_cast<uint8_t*>(&caseIndexFromValue) + 4 -
numPayloadTagBytes, valueAddr, numPayloadTagBytes);
#else
memcpy(&caseIndexFromValue, valueAddr,
std::min(size_t(4), payloadSize));
#endif
return (caseIndexFromExtraTagBits | caseIndexFromValue)
+ payloadNumExtraInhabitants;
}
}
// If there are extra inhabitants, see whether the payload is valid.
if (payloadNumExtraInhabitants > 0) {
return
static_cast<const ExtraInhabitantsValueWitnessTable*>(payloadWitnesses)
->getExtraInhabitantIndex(value, payload);
}
// Otherwise, we have always have a valid payload.
return -1;
return getEnumTagSinglePayloadImpl(value, emptyCases, payload, size,
numExtraInhabitants,
getExtraInhabitantIndex);
}
void swift::swift_storeEnumTagSinglePayload(OpaqueValue *value,
const Metadata *payload,
int whichCase, unsigned emptyCases)
SWIFT_CC(RegisterPreservingCC_IMPL) {
auto *payloadWitnesses = payload->getValueWitnesses();
auto payloadSize = payloadWitnesses->getSize();
unsigned payloadNumExtraInhabitants
= payloadWitnesses->getNumExtraInhabitants();
auto size = payloadWitnesses->getSize();
auto numExtraInhabitants = payloadWitnesses->getNumExtraInhabitants();
auto storeExtraInhabitant =
(static_cast<const ExtraInhabitantsValueWitnessTable *>(payloadWitnesses)
->storeExtraInhabitant);
auto *valueAddr = reinterpret_cast<uint8_t*>(value);
auto *extraTagBitAddr = valueAddr + payloadSize;
unsigned numExtraTagBytes = emptyCases > payloadNumExtraInhabitants
? getNumTagBytes(payloadSize, emptyCases - payloadNumExtraInhabitants,
1 /*payload case*/)
: 0;
// For payload or extra inhabitant cases, zero-initialize the extra tag bits,
// if any.
if (whichCase < (int)payloadNumExtraInhabitants) {
// The two most common values for numExtraTagBytes are zero and one.
// Try to avoid calling bzero by specializing for these values.
if (numExtraTagBytes != 0) {
if (numExtraTagBytes == 1) {
// Zero a single byte.
*((char*)(extraTagBitAddr)) = 0;
} else {
// Zero the buffer.
memset(extraTagBitAddr, 0, numExtraTagBytes);
}
}
// If this is the payload case, we're done.
if (whichCase == -1)
return;
// Store the extra inhabitant.
static_cast<const ExtraInhabitantsValueWitnessTable*>(payloadWitnesses)
->storeExtraInhabitant(value, whichCase, payload);
return;
}
// Factor the case index into payload and extra tag parts.
unsigned caseIndex = whichCase - payloadNumExtraInhabitants;
unsigned payloadIndex, extraTagIndex;
if (payloadSize >= 4) {
extraTagIndex = 1;
payloadIndex = caseIndex;
} else {
unsigned payloadBits = payloadSize * 8U;
extraTagIndex = 1U + (caseIndex >> payloadBits);
payloadIndex = caseIndex & ((1U << payloadBits) - 1U);
}
// Store into the value.
#if defined(__BIG_ENDIAN__)
unsigned numPayloadTagBytes = std::min(size_t(4), payloadSize);
memcpy(valueAddr,
reinterpret_cast<uint8_t*>(&payloadIndex) + 4 - numPayloadTagBytes,
numPayloadTagBytes);
if (payloadSize > 4)
memset(valueAddr + 4, 0, payloadSize - 4);
memcpy(extraTagBitAddr,
reinterpret_cast<uint8_t*>(&extraTagIndex) + 4 - numExtraTagBytes,
numExtraTagBytes);
#else
memcpy(valueAddr, &payloadIndex, std::min(size_t(4), payloadSize));
if (payloadSize > 4)
memset(valueAddr + 4, 0, payloadSize - 4);
memcpy(extraTagBitAddr, &extraTagIndex, numExtraTagBytes);
#endif
storeEnumTagSinglePayloadImpl(value, whichCase, emptyCases, payload, size,
numExtraInhabitants, storeExtraInhabitant);
}
void