mirror of
https://github.com/apple/swift.git
synced 2025-12-21 12:14:44 +01:00
37d69a4207
We were rounding the size up to the alignment when allocating a new object (swift_allocObject) but not when directly allocating memory for a non-object (swift_slowAlloc). The deallocation code wasn’t rounding the size up to the alignment at all. Overall, this meant that we would get the wrong index into the allocation cache when deallocating an object whose size got rounded in a way that affects the index. Fixes <rdar://problem/17542859>, where println(5) would fail on the 32-bit iOS simulator when building the runtime without NDEBUG (so we get extra checking) and the standard library is built without optimization (which keeps a certain 33-byte allocation on the heap). The Interpreter/SDK/objc_cast.swift test triggers this when built under those conditions. Swift SVN r19499
1066 lines
32 KiB
C++
1066 lines
32 KiB
C++
//===--- Heap.cpp - Swift Language Heap Logic -----------------------------===//
|
|
//
|
|
// This source file is part of the Swift.org open source project
|
|
//
|
|
// Copyright (c) 2014 - 2015 Apple Inc. and the Swift project authors
|
|
// Licensed under Apache License v2.0 with Runtime Library Exception
|
|
//
|
|
// See http://swift.org/LICENSE.txt for license information
|
|
// See http://swift.org/CONTRIBUTORS.txt for the list of Swift project authors
|
|
//
|
|
//===----------------------------------------------------------------------===//
|
|
//
|
|
// Implementations of the Swift heap
|
|
//
|
|
//===----------------------------------------------------------------------===//
|
|
|
|
#include "swift/Runtime/HeapObject.h"
|
|
#include "swift/Runtime/InstrumentsSupport.h"
|
|
#include "swift/Runtime/Heap.h"
|
|
#include "Private.h"
|
|
#include "Debug.h"
|
|
#include <mach/vm_statistics.h>
|
|
#include <stdio.h>
|
|
#include <stdlib.h>
|
|
#include <cassert>
|
|
#include <pthread.h>
|
|
#include <dlfcn.h>
|
|
#include <vector>
|
|
#include <functional>
|
|
#include <sys/mman.h>
|
|
#include <errno.h>
|
|
#include <unistd.h>
|
|
#include <crt_externs.h>
|
|
|
|
// FIXME when we start building with the internal SDK
|
|
//#if __has_include(<pthread/tsd_private.h>)
|
|
#if 0
|
|
#include <pthread/tsd_private.h>
|
|
#else
|
|
struct pthread_layout_offsets_s {
|
|
// always add new fields at the end
|
|
const uint16_t plo_version;
|
|
// either of the next two fields may be 0; use whichever is set
|
|
// bytes from pthread_t to base of tsd
|
|
const uint16_t plo_pthread_tsd_base_offset;
|
|
// bytes from pthread_t to a pointer to base of tsd
|
|
const uint16_t plo_pthread_tsd_base_address_offset;
|
|
const uint16_t plo_pthread_tsd_entry_size;
|
|
} /*pthread_layout_offsets*/;
|
|
#endif
|
|
|
|
using namespace swift;
|
|
|
|
static inline size_t indexToSize(AllocIndex idx);
|
|
|
|
static kern_return_t _swift_zone_enumerator(task_t task, void *,
|
|
unsigned type_mask,
|
|
vm_address_t zone_address,
|
|
memory_reader_t reader,
|
|
vm_range_recorder_t recorder);
|
|
static size_t _swift_zone_good_size(malloc_zone_t *zone, size_t size);
|
|
static boolean_t _swift_zone_check(malloc_zone_t *zone);
|
|
static void _swift_zone_print(malloc_zone_t *zone, boolean_t verbose);
|
|
static void _swift_zone_log(malloc_zone_t *zone, void *address);
|
|
static void _swift_zone_statistics(malloc_zone_t *zone,
|
|
malloc_statistics_t *stats);
|
|
|
|
static const AllocIndex badAllocIndex = AllocIndex(-1);
|
|
|
|
typedef struct AllocCacheEntry_s {
|
|
struct AllocCacheEntry_s *next;
|
|
} *AllocCacheEntry;
|
|
|
|
|
|
|
|
static const auto pthread_layout_details
|
|
= reinterpret_cast<struct pthread_layout_offsets_s *>(
|
|
dlsym(RTLD_DEFAULT, "pthread_layout_offsets"));
|
|
|
|
static void *pthreadTSDTable() {
|
|
pthread_t self = pthread_self();
|
|
auto details = pthread_layout_details;
|
|
|
|
if (details) {
|
|
assert(details->plo_pthread_tsd_entry_size == sizeof(void *));
|
|
assert((bool)details->plo_pthread_tsd_base_offset
|
|
!= (bool)details->plo_pthread_tsd_base_address_offset);
|
|
if (details->plo_pthread_tsd_base_offset) {
|
|
return (void *)((size_t)self + details->plo_pthread_tsd_base_offset);
|
|
}
|
|
return *(void **)((size_t)self
|
|
+ details->plo_pthread_tsd_base_address_offset);
|
|
}
|
|
// tested empirically on OSX 10.9 and iOS 7.1
|
|
#if __x86_64__
|
|
return (void *)((size_t)self + 28 * sizeof(void *));
|
|
#elif __i386__
|
|
return (void *)((size_t)self + 44 * sizeof(void *));
|
|
#elif __arm64__
|
|
return (void *)((size_t)self + 28 * sizeof(void *));
|
|
#elif __arm__
|
|
return (void *)((size_t)self + 41 * sizeof(void *));
|
|
#else
|
|
abort();
|
|
#endif
|
|
}
|
|
|
|
namespace {
|
|
|
|
void *mmapWrapper(size_t size, bool huge) {
|
|
int tag = VM_MAKE_TAG(huge ? VM_MEMORY_MALLOC_HUGE : VM_MEMORY_MALLOC_SMALL);
|
|
return mmap(NULL, size, PROT_READ|PROT_WRITE, MAP_ANON|MAP_PRIVATE, tag, 0);
|
|
}
|
|
|
|
// HeapMap
|
|
//
|
|
// We need a trivial map that:
|
|
// 1) doesn't use the system allocator
|
|
// 2) is easy to programmatically introspect from outside of the process
|
|
//
|
|
// This is not on the critical path, so it doesn't need to be "perfect".
|
|
//
|
|
auto tombstoneptr = (const void *)-1;
|
|
template <typename T, size_t (*hash)(const void *)>
|
|
class HeapMap {
|
|
public:
|
|
std::pair<const void *, T> *nodes;
|
|
size_t count;
|
|
size_t mask;
|
|
size_t bufSize() { return mask + 1; }
|
|
void grow() {
|
|
auto oldMask = mask;
|
|
auto oldNodes = nodes;
|
|
mask = mask * 2 + 1;
|
|
nodes = reinterpret_cast<std::pair<const void *, T> *>(
|
|
mmapWrapper(sizeof(nodes[0]) * bufSize(), false));
|
|
for (size_t i = 0; i < (oldMask + 1); i++) {
|
|
if (oldNodes[i].first == nullptr) continue;
|
|
if (oldNodes[i].first == tombstoneptr) continue;
|
|
insert(std::pair<const void *, T>(oldNodes[i].first, oldNodes[i].second));
|
|
oldNodes[i].second.~T();
|
|
}
|
|
int r = munmap(oldNodes, sizeof(oldNodes[0]) * (oldMask + 1));
|
|
assert(r == 0);
|
|
}
|
|
|
|
HeapMap() : count(0), mask(16-1) {
|
|
nodes = reinterpret_cast<std::pair<const void *, T> *>(
|
|
mmapWrapper(sizeof(nodes[0]) * bufSize(), false));
|
|
assert(nodes != nullptr);
|
|
}
|
|
~HeapMap() {
|
|
}
|
|
void insert(std::pair<const void *, T> p) {
|
|
if (size() >= (bufSize() * 3 / 4)) {
|
|
grow();
|
|
}
|
|
size_t i = hash(p.first) & mask;
|
|
do {
|
|
if (nodes[i].first == nullptr || nodes[i].first == tombstoneptr) {
|
|
nodes[i].first = p.first;
|
|
new (&nodes[i].second) T(p.second);
|
|
++count;
|
|
return;
|
|
}
|
|
} while (++i < bufSize());
|
|
for (i = 0; i < bufSize(); i++) {
|
|
if (nodes[i].first == nullptr || nodes[i].first == tombstoneptr) {
|
|
nodes[i].first = p.first;
|
|
new (&nodes[i].second) T(p.second);
|
|
++count;
|
|
return;
|
|
}
|
|
}
|
|
grow();
|
|
insert(p);
|
|
}
|
|
std::pair<const void *, T> *find(const void *key) {
|
|
size_t i = hash(key) & mask;
|
|
do {
|
|
if (nodes[i].first == key) return &nodes[i];
|
|
if (nodes[i].first == nullptr) return nullptr;
|
|
} while (++i < bufSize());
|
|
for (i = 0; i < bufSize(); i++) {
|
|
if (nodes[i].first == key) return &nodes[i];
|
|
if (nodes[i].first == nullptr) break;
|
|
}
|
|
return nullptr;
|
|
}
|
|
void erase(std::pair<const void *, T> *p) {
|
|
--count;
|
|
p->first = tombstoneptr;
|
|
p->second.~T();
|
|
}
|
|
size_t size() const { return count; }
|
|
T *operator [](const void *key) {
|
|
return &find(key)->second;
|
|
}
|
|
void forEach(std::function<void(const void *, T)> func) {
|
|
for (size_t i = 0; i < bufSize(); i++) {
|
|
if (nodes[i].first != nullptr && nodes[i].first != tombstoneptr) {
|
|
func(nodes[i].first, nodes[i].second);
|
|
}
|
|
}
|
|
}
|
|
};
|
|
|
|
static size_t arenaPointerHash(const void *pointer) {
|
|
return (size_t)pointer >> 16; // arenas are always 64KB and 64KB aligned
|
|
}
|
|
static size_t hugeAllocationPointerHash(const void *pointer) {
|
|
#ifdef __arm64__
|
|
return (size_t)pointer >> 14; // page size/alignment is 16k
|
|
#else
|
|
return (size_t)pointer >> 12; // page size/alignment is 4k
|
|
#endif
|
|
}
|
|
static size_t pthreadPointerHash(const void *pointer) {
|
|
return (size_t)pointer >> 4; // system heap is always 16-byte aligned
|
|
}
|
|
static size_t pointerHash(const void *pointer) {
|
|
#if __LP64__
|
|
return (size_t)pointer >> 3;
|
|
#else
|
|
return (size_t)pointer >> 2;
|
|
#endif
|
|
}
|
|
|
|
class Arena;
|
|
class SwiftZone {
|
|
public:
|
|
malloc_zone_t zoneShims = {
|
|
nullptr,
|
|
nullptr,
|
|
_swift_zone_size,
|
|
_swift_zone_malloc,
|
|
_swift_zone_calloc,
|
|
_swift_zone_valloc,
|
|
_swift_zone_free,
|
|
_swift_zone_realloc,
|
|
_swift_zone_destroy,
|
|
"SwiftZone",
|
|
nullptr,
|
|
nullptr,
|
|
&SwiftZone::zoneInspection,
|
|
0,
|
|
nullptr,
|
|
nullptr,
|
|
nullptr,
|
|
};
|
|
pthread_rwlock_t lock;
|
|
pthread_key_t keyOffset;
|
|
HeapMap<Arena, arenaPointerHash> arenas;
|
|
HeapMap<size_t, hugeAllocationPointerHash> hugeAllocations;
|
|
HeapMap<bool, pthreadPointerHash> threads;
|
|
AllocCacheEntry globalCache[ALLOC_CACHE_COUNT];
|
|
|
|
SwiftZone();
|
|
SwiftZone(SwiftZone const &) = delete;
|
|
SwiftZone &operator=(SwiftZone const &) = delete;
|
|
static void threadExitCleanup(void *arg);
|
|
static malloc_introspection_t zoneInspection;
|
|
|
|
// allocations are normally per-thread
|
|
// this API goes straight to the global pool
|
|
static void *globalAlloc(AllocIndex idx, uintptr_t flags);
|
|
|
|
static void *alloc_optimized(AllocIndex idx);
|
|
static void *tryAlloc_optimized(AllocIndex idx);
|
|
static void *slowAlloc_optimized(size_t size, size_t alignMask,
|
|
uintptr_t flags);
|
|
static void dealloc_optimized(void *ptr, AllocIndex idx);
|
|
static void slowDealloc_optimized(void *ptr, size_t bytes, size_t alignMask);
|
|
|
|
static void *alloc_semi_optimized(AllocIndex idx);
|
|
static void *tryAlloc_semi_optimized(AllocIndex idx);
|
|
static void dealloc_semi_optimized(void *ptr, AllocIndex idx);
|
|
|
|
static void *alloc_systemMalloc(AllocIndex idx);
|
|
static void *tryAlloc_systemMalloc(AllocIndex idx);
|
|
static void *slowAlloc_systemMalloc(size_t size, size_t alignMask,
|
|
uintptr_t flags);
|
|
static void dealloc_systemMalloc(void *ptr, AllocIndex idx);
|
|
static void slowDealloc_systemMalloc(void *ptr, size_t bytes,
|
|
size_t alignMask);
|
|
|
|
bool tryWriteLock() {
|
|
int r = pthread_rwlock_trywrlock(&lock);
|
|
assert(r == 0 || errno == EBUSY);
|
|
return r == 0;
|
|
}
|
|
void writeLock() {
|
|
int r = pthread_rwlock_wrlock(&lock);
|
|
assert(r == 0);
|
|
(void)r;
|
|
}
|
|
void writeUnlock() {
|
|
int r = pthread_rwlock_unlock(&lock);
|
|
assert(r == 0);
|
|
(void)r;
|
|
}
|
|
void readLock() {
|
|
int r = pthread_rwlock_rdlock(&lock);
|
|
assert(r == 0);
|
|
(void)r;
|
|
}
|
|
void readUnlock() {
|
|
int r = pthread_rwlock_unlock(&lock);
|
|
assert(r == 0);
|
|
(void)r;
|
|
}
|
|
void debug();
|
|
void registerThread() {
|
|
void *table = pthreadTSDTable();
|
|
auto it = threads.find(table);
|
|
if (!it) {
|
|
threads.insert(std::pair<void *, bool>(table, true));
|
|
}
|
|
}
|
|
void unregisterThread() {
|
|
auto it = threads.find(pthreadTSDTable());
|
|
if (it) {
|
|
threads.erase(it);
|
|
}
|
|
}
|
|
};
|
|
|
|
void SwiftZone::debug() {
|
|
malloc_zone_print(&zoneShims, true);
|
|
}
|
|
|
|
extern "C" pthread_key_t _swiftAllocOffset = -1;
|
|
|
|
const size_t pageSize = getpagesize();
|
|
const size_t pageMask = getpagesize() - 1;
|
|
const size_t arenaSize = 0x10000;
|
|
const size_t arenaMask = 0xFFFF;
|
|
|
|
static const void *pointerToArena(const void *pointer) {
|
|
return (const void *)((size_t)pointer & ~arenaMask);
|
|
}
|
|
|
|
size_t roundToPage(size_t size) {
|
|
return (size + pageMask) & ~pageMask;
|
|
}
|
|
|
|
#if __has_include(<os/tsd.h>)
|
|
// OS X and iOS internal version
|
|
#include <os/tsd.h>
|
|
#else
|
|
#define _os_tsd_get_direct pthread_getspecific
|
|
#define _os_tsd_set_direct pthread_setspecific
|
|
#endif
|
|
|
|
// These are implemented in FastEntryPoints.s on some platforms.
|
|
#ifndef SWIFT_HAVE_FAST_ENTRY_POINTS
|
|
|
|
static AllocCacheEntry
|
|
getAllocCacheEntry(unsigned long idx) {
|
|
assert(idx < ALLOC_CACHE_COUNT);
|
|
return (AllocCacheEntry)_os_tsd_get_direct(idx + _swiftAllocOffset);
|
|
}
|
|
|
|
static void
|
|
setAllocCacheEntry(unsigned long idx, AllocCacheEntry entry) {
|
|
assert(idx < ALLOC_CACHE_COUNT);
|
|
_os_tsd_set_direct(idx + _swiftAllocOffset, entry);
|
|
}
|
|
|
|
static AllocCacheEntry
|
|
getAllocCacheEntry_slow(unsigned long idx) {
|
|
assert(idx < ALLOC_CACHE_COUNT);
|
|
return (AllocCacheEntry)pthread_getspecific(idx + _swiftAllocOffset);
|
|
}
|
|
|
|
static void
|
|
setAllocCacheEntry_slow(unsigned long idx, AllocCacheEntry entry) {
|
|
assert(idx < ALLOC_CACHE_COUNT);
|
|
auto result = pthread_setspecific(idx + _swiftAllocOffset, entry);
|
|
assert(result == 0);
|
|
}
|
|
|
|
void *SwiftZone::alloc_optimized(AllocIndex idx) {
|
|
assert(idx < ALLOC_CACHE_COUNT);
|
|
AllocCacheEntry r = getAllocCacheEntry(idx);
|
|
if (r) {
|
|
setAllocCacheEntry(idx, r->next);
|
|
return r;
|
|
}
|
|
return globalAlloc(idx, 0);
|
|
}
|
|
|
|
void *SwiftZone::tryAlloc_optimized(AllocIndex idx) {
|
|
assert(idx < ALLOC_CACHE_COUNT);
|
|
AllocCacheEntry r = getAllocCacheEntry(idx);
|
|
if (r) {
|
|
setAllocCacheEntry(idx, r->next);
|
|
return r;
|
|
}
|
|
return globalAlloc(idx, SWIFT_TRYALLOC);
|
|
}
|
|
|
|
void SwiftZone::dealloc_optimized(void *ptr, AllocIndex idx) {
|
|
assert(idx < ALLOC_CACHE_COUNT);
|
|
auto cur = static_cast<AllocCacheEntry>(ptr);
|
|
AllocCacheEntry prev = getAllocCacheEntry(idx);
|
|
assert(cur != prev && "trivial double free!");
|
|
cur->next = prev;
|
|
setAllocCacheEntry(idx, cur);
|
|
}
|
|
|
|
void *SwiftZone::alloc_semi_optimized(AllocIndex idx) {
|
|
assert(idx < ALLOC_CACHE_COUNT);
|
|
AllocCacheEntry r = getAllocCacheEntry_slow(idx);
|
|
if (r) {
|
|
setAllocCacheEntry_slow(idx, r->next);
|
|
return r;
|
|
}
|
|
return globalAlloc(idx, 0);
|
|
}
|
|
|
|
void *SwiftZone::alloc_systemMalloc(AllocIndex idx) {
|
|
assert(idx < ALLOC_CACHE_COUNT);
|
|
size_t size = indexToSize(idx);
|
|
return malloc(size);
|
|
}
|
|
|
|
void *SwiftZone::tryAlloc_semi_optimized(AllocIndex idx) {
|
|
assert(idx < ALLOC_CACHE_COUNT);
|
|
AllocCacheEntry r = getAllocCacheEntry_slow(idx);
|
|
if (r) {
|
|
setAllocCacheEntry_slow(idx, r->next);
|
|
return r;
|
|
}
|
|
return globalAlloc(idx, SWIFT_TRYALLOC);
|
|
}
|
|
|
|
void *SwiftZone::tryAlloc_systemMalloc(AllocIndex idx) {
|
|
assert(idx < ALLOC_CACHE_COUNT);
|
|
size_t size = indexToSize(idx);
|
|
return malloc(size);
|
|
}
|
|
|
|
void SwiftZone::dealloc_semi_optimized(void *ptr, AllocIndex idx) {
|
|
assert(idx < ALLOC_CACHE_COUNT);
|
|
auto cur = static_cast<AllocCacheEntry>(ptr);
|
|
AllocCacheEntry prev = getAllocCacheEntry_slow(idx);
|
|
assert(cur != prev && "trivial double free!");
|
|
cur->next = prev;
|
|
setAllocCacheEntry_slow(idx, cur);
|
|
}
|
|
|
|
void SwiftZone::dealloc_systemMalloc(void *ptr, AllocIndex idx) {
|
|
assert(idx < ALLOC_CACHE_COUNT);
|
|
free(ptr);
|
|
}
|
|
|
|
// !SWIFT_HAVE_FAST_ENTRY_POINTS
|
|
#endif
|
|
|
|
class Arena {
|
|
public:
|
|
void *base;
|
|
uint16_t byteSize; // max 4096
|
|
uint8_t index; // max 56 or 64
|
|
Arena(size_t idx, size_t size);
|
|
static void newArena(size_t idx, size_t size);
|
|
};
|
|
|
|
} // end anonymous namespace
|
|
|
|
static SwiftZone swiftZone;
|
|
|
|
Arena::Arena(size_t idx, size_t size) : byteSize(size), index(idx) {
|
|
assert(size > 0);
|
|
base = mmapWrapper(arenaSize * 2, /*huge*/ false);
|
|
assert(base != MAP_FAILED);
|
|
void *newBase = (void *)(((size_t)base & ~arenaMask) + arenaSize);
|
|
size_t frontSlop = (size_t)newBase - (size_t)base;
|
|
size_t backSlop = arenaSize - frontSlop;
|
|
int r = munmap(base, frontSlop);
|
|
assert(r != -1);
|
|
if (backSlop) {
|
|
r = munmap((uint8_t *)newBase + arenaSize, backSlop);
|
|
assert(r != -1);
|
|
}
|
|
base = newBase;
|
|
auto headNode = (AllocCacheEntry)base;
|
|
auto node = headNode;
|
|
size_t allocations = arenaSize / size;
|
|
for (unsigned i = 1; i < allocations; i++) {
|
|
auto nextNode = (AllocCacheEntry)((uint8_t *)base + i * size);
|
|
node->next = nextNode;
|
|
node = nextNode;
|
|
}
|
|
assert(swiftZone.globalCache[idx] == NULL);
|
|
swiftZone.globalCache[idx] = headNode;
|
|
}
|
|
|
|
void Arena::newArena(size_t idx, size_t size) {
|
|
auto arena = Arena(idx, size);
|
|
swiftZone.writeLock();
|
|
swiftZone.arenas.insert(std::pair<void *, Arena>(arena.base, arena));
|
|
swiftZone.writeUnlock();
|
|
}
|
|
|
|
malloc_zone_t *swift::_swift_zone_get_shims() {
|
|
return &swiftZone.zoneShims;
|
|
}
|
|
|
|
|
|
static void _swift_zone_initImpl() {
|
|
assert(sizeof(pthread_key_t) == sizeof(long));
|
|
malloc_zone_register(&swiftZone.zoneShims);
|
|
pthread_key_t key, prev_key;
|
|
int r = pthread_key_create(&key, SwiftZone::threadExitCleanup);
|
|
assert(r == 0);
|
|
(void)r;
|
|
prev_key = key;
|
|
swiftZone.keyOffset = _swiftAllocOffset = key;
|
|
for (unsigned i = 1; i < ALLOC_CACHE_COUNT; i++) {
|
|
int r = pthread_key_create(&key, SwiftZone::threadExitCleanup);
|
|
assert(r == 0);
|
|
(void)r;
|
|
assert(key == (prev_key + 1));
|
|
prev_key = key;
|
|
}
|
|
|
|
AllocCacheEntry magic = (AllocCacheEntry)1804289383; // result from random()
|
|
setAllocCacheEntry_slow(0, magic);
|
|
if (getAllocCacheEntry(0) != magic) {
|
|
_swift_alloc = SwiftZone::alloc_semi_optimized;
|
|
_swift_tryAlloc = SwiftZone::tryAlloc_semi_optimized;
|
|
_swift_dealloc = SwiftZone::dealloc_semi_optimized;
|
|
}
|
|
setAllocCacheEntry_slow(0, NULL);
|
|
|
|
bool useSystemMalloc = false;
|
|
for (char **string = *_NSGetEnviron(); *string; string++) {
|
|
if (strncmp(*string, "Malloc", strlen("Malloc")) == 0) {
|
|
useSystemMalloc = true;
|
|
break;
|
|
}
|
|
}
|
|
const char *dyldMagic = getenv("DYLD_INSERT_LIBRARIES");
|
|
if (dyldMagic && strstr(dyldMagic, "libgmalloc")) {
|
|
useSystemMalloc = true;
|
|
}
|
|
if (useSystemMalloc) {
|
|
_swift_alloc = SwiftZone::alloc_systemMalloc;
|
|
_swift_tryAlloc = SwiftZone::tryAlloc_systemMalloc;
|
|
_swift_slowAlloc = SwiftZone::slowAlloc_systemMalloc;
|
|
_swift_dealloc = SwiftZone::dealloc_systemMalloc;
|
|
_swift_slowDealloc = SwiftZone::slowDealloc_systemMalloc;
|
|
}
|
|
}
|
|
void swift::_swift_zone_init() {
|
|
static pthread_once_t once = PTHREAD_ONCE_INIT;
|
|
int r = pthread_once(&once, _swift_zone_initImpl);
|
|
assert(r == 0);
|
|
}
|
|
SwiftZone::SwiftZone() : lock(PTHREAD_RWLOCK_INITIALIZER) {
|
|
_swift_zone_init();
|
|
}
|
|
|
|
size_t swift::_swift_zone_size(malloc_zone_t *zone, const void *pointer) {
|
|
swiftZone.readLock();
|
|
size_t value = 0;
|
|
auto it = swiftZone.arenas.find(pointerToArena(pointer));
|
|
if (it) {
|
|
value = it->second.byteSize;
|
|
} else {
|
|
auto it2 = swiftZone.hugeAllocations.find(pointer);
|
|
if (it2) {
|
|
value = it2->second;
|
|
}
|
|
}
|
|
swiftZone.readUnlock();
|
|
return value;
|
|
}
|
|
|
|
constexpr size_t _swift_zone_alignMask = 15;
|
|
|
|
void *swift::_swift_zone_malloc(malloc_zone_t *zone, size_t size) {
|
|
return SwiftZone::slowAlloc_optimized(size, _swift_zone_alignMask,
|
|
SWIFT_TRYALLOC);
|
|
}
|
|
|
|
void *swift::_swift_zone_calloc(malloc_zone_t *zone,
|
|
size_t count, size_t size) {
|
|
void *pointer = swift::_swift_zone_malloc(zone, count * size);
|
|
return pointer ? memset(pointer, 0, count * size) : pointer;
|
|
}
|
|
|
|
void *swift::_swift_zone_valloc(malloc_zone_t *zone, size_t size) {
|
|
assert((pageMask & 0xFFF) == 0xFFF); // at least 4k
|
|
return swift::_swift_zone_malloc(zone, roundToPage(size));
|
|
}
|
|
|
|
void swift::_swift_zone_free(malloc_zone_t *zone, void *pointer) {
|
|
swift::swift_slowDealloc(pointer, swift::_swift_zone_size(zone, pointer),
|
|
_swift_zone_alignMask);
|
|
}
|
|
|
|
void *swift::_swift_zone_realloc(malloc_zone_t *zone,
|
|
void *pointer, size_t size) {
|
|
auto newPointer = swift::_swift_zone_malloc(zone, size);
|
|
auto oldSize = swift::_swift_zone_size(zone, pointer);
|
|
memcpy(newPointer, pointer, size < oldSize ? size : oldSize);
|
|
swift::_swift_zone_free(zone, pointer);
|
|
return newPointer;
|
|
}
|
|
|
|
void swift::_swift_zone_destroy(malloc_zone_t *zone) {
|
|
swift::crash("The Swift heap cannot be destroyed");
|
|
}
|
|
|
|
static kern_return_t
|
|
defaultReader(task_t task, vm_address_t address, vm_size_t size, void **ptr) {
|
|
*ptr = (void *)address;
|
|
return 0;
|
|
}
|
|
|
|
template <typename T>
|
|
static void readAndDuplicate(task_t task, vm_address_t where,
|
|
memory_reader_t reader,
|
|
size_t size, T **mem) {
|
|
void *temp = nullptr;
|
|
kern_return_t error = reader(task, (vm_address_t)where, size, (void **)&temp);
|
|
assert(error == KERN_SUCCESS);
|
|
void *out = malloc(size);
|
|
memcpy(out, temp, size);
|
|
*mem = reinterpret_cast<T*>(out);
|
|
}
|
|
|
|
kern_return_t
|
|
_swift_zone_enumerator(task_t task, void *context, unsigned type_mask,
|
|
vm_address_t zone_address, memory_reader_t reader,
|
|
vm_range_recorder_t recorder) {
|
|
SwiftZone *zone = nullptr;
|
|
|
|
if (!reader) {
|
|
reader = defaultReader;
|
|
}
|
|
|
|
readAndDuplicate(task, zone_address, reader, sizeof(SwiftZone), &zone);
|
|
|
|
if ((type_mask & MALLOC_ADMIN_REGION_RANGE_TYPE)
|
|
== MALLOC_ADMIN_REGION_RANGE_TYPE) {
|
|
vm_range_t buffer[] = {
|
|
{
|
|
(vm_address_t)zone->arenas.nodes,
|
|
sizeof(*zone->arenas.nodes) * zone->arenas.bufSize()
|
|
},
|
|
{
|
|
(vm_address_t)zone->hugeAllocations.nodes,
|
|
sizeof(*zone->hugeAllocations.nodes) * zone->hugeAllocations.bufSize()
|
|
},
|
|
{
|
|
(vm_address_t)zone->threads.nodes,
|
|
sizeof(*zone->threads.nodes) * zone->threads.bufSize()
|
|
}
|
|
};
|
|
recorder(task, context, MALLOC_ADMIN_REGION_RANGE_TYPE,
|
|
buffer, sizeof(buffer) / sizeof(buffer[0]));
|
|
}
|
|
|
|
readAndDuplicate(task, (vm_address_t)zone->arenas.nodes, reader,
|
|
sizeof(*zone->arenas.nodes) * zone->arenas.bufSize(),
|
|
&zone->arenas.nodes);
|
|
|
|
readAndDuplicate(task, (vm_address_t)zone->hugeAllocations.nodes, reader,
|
|
sizeof(*zone->hugeAllocations.nodes)
|
|
* zone->hugeAllocations.bufSize(),
|
|
&zone->hugeAllocations.nodes);
|
|
|
|
if ((type_mask & MALLOC_PTR_REGION_RANGE_TYPE)
|
|
== MALLOC_PTR_REGION_RANGE_TYPE) {
|
|
zone->arenas.forEach([&] (const void *pointer, Arena arena) {
|
|
vm_range_t buffer = { (vm_address_t)arena.base, arenaSize };
|
|
recorder(task, context, MALLOC_PTR_REGION_RANGE_TYPE, &buffer, 1);
|
|
});
|
|
|
|
zone->hugeAllocations.forEach([&] (const void *pointer, size_t size) {
|
|
vm_range_t buffer = { (vm_address_t)pointer, size };
|
|
recorder(task, context, MALLOC_PTR_REGION_RANGE_TYPE, &buffer, 1);
|
|
});
|
|
}
|
|
|
|
if ((type_mask & MALLOC_PTR_IN_USE_RANGE_TYPE)
|
|
!= MALLOC_PTR_IN_USE_RANGE_TYPE) {
|
|
return KERN_SUCCESS;
|
|
}
|
|
// handle MALLOC_PTR_IN_USE_RANGE_TYPE
|
|
|
|
readAndDuplicate(task, (vm_address_t)zone->threads.nodes, reader,
|
|
sizeof(*zone->threads.nodes) * zone->threads.bufSize(),
|
|
&zone->threads.nodes);
|
|
|
|
HeapMap<bool, pointerHash> unusedBlocks;
|
|
|
|
for (unsigned i = 0; i < ALLOC_CACHE_COUNT; i++) {
|
|
AllocCacheEntry pointer = zone->globalCache[i];
|
|
kern_return_t error;
|
|
while (pointer) {
|
|
unusedBlocks.insert(std::pair<void *, bool>(pointer, true));
|
|
void **temp;
|
|
error = reader(task, (vm_address_t)&pointer->next,
|
|
sizeof(AllocCacheEntry), (void **)&temp);
|
|
assert(error == KERN_SUCCESS);
|
|
pointer = reinterpret_cast<AllocCacheEntry>(*temp);
|
|
}
|
|
}
|
|
|
|
zone->threads.forEach([&] (const void *thread, bool) {
|
|
for (unsigned i = 0; i < ALLOC_CACHE_COUNT; i++) {
|
|
void **temp = nullptr;
|
|
kern_return_t error;
|
|
// FIXME deduce the magic constant in the inferior, and then save it
|
|
// for us to look at later
|
|
error = reader(task, (vm_address_t)
|
|
&((AllocCacheEntry**)thread)[zone->keyOffset + i],
|
|
sizeof(AllocCacheEntry), (void **)&temp);
|
|
assert(error == KERN_SUCCESS);
|
|
auto pointer = reinterpret_cast<AllocCacheEntry>(*temp);
|
|
while (pointer) {
|
|
unusedBlocks.insert(std::pair<void *, bool>(pointer, true));
|
|
void **temp;
|
|
error = reader(task, (vm_address_t)&pointer->next,
|
|
sizeof(AllocCacheEntry), (void **)&temp);
|
|
assert(error == KERN_SUCCESS);
|
|
pointer = reinterpret_cast<AllocCacheEntry>(*temp);
|
|
}
|
|
}
|
|
});
|
|
|
|
zone->arenas.forEach([&] (const void *arenaBase, Arena arena) {
|
|
const size_t size = arena.byteSize;
|
|
for (size_t i = 0; (i + size) <= arenaSize; i += size) {
|
|
auto pointer = (const void *)((uint8_t *)arena.base + i);
|
|
if (!unusedBlocks.find(pointer)) {
|
|
vm_range_t buffer = { (vm_address_t)pointer, size };
|
|
recorder(task, context, MALLOC_PTR_IN_USE_RANGE_TYPE, &buffer, 1);
|
|
}
|
|
}
|
|
});
|
|
|
|
zone->hugeAllocations.forEach([&] (const void *pointer, size_t size) {
|
|
vm_range_t buffer = { (vm_address_t)pointer, size };
|
|
recorder(task, context, MALLOC_PTR_IN_USE_RANGE_TYPE, &buffer, 1);
|
|
});
|
|
|
|
return KERN_SUCCESS;
|
|
}
|
|
|
|
size_t _swift_zone_good_size(malloc_zone_t *zone, size_t size) {
|
|
// XXX -- Switch to raw swift heap calls
|
|
void *temp = _swift_zone_malloc(zone, size);
|
|
size_t r = _swift_zone_size(zone, temp);
|
|
_swift_zone_free(zone, temp);
|
|
return r;
|
|
}
|
|
|
|
boolean_t _swift_zone_check(malloc_zone_t *zone) {
|
|
return true; // we don't have any self-consistency checks; true == good/okay
|
|
}
|
|
|
|
void _swift_zone_log(malloc_zone_t *zone, void *address) {
|
|
swift::crash("Swift Zone log is unimplemented");
|
|
}
|
|
|
|
static inline size_t indexToSize(AllocIndex idx) {
|
|
size_t size;
|
|
idx++;
|
|
|
|
// we could do a table based lookup if we think it worthwhile
|
|
#ifdef __LP64__
|
|
if (idx <= 16) { size = idx << 3;
|
|
} else if (idx <= 24) { size = (idx - 8) << 4;
|
|
} else if (idx <= 32) { size = (idx - 16) << 5;
|
|
} else if (idx <= 40) { size = (idx - 24) << 6;
|
|
} else if (idx <= 48) { size = (idx - 32) << 7;
|
|
} else if (idx <= 56) { size = (idx - 40) << 8;
|
|
#else
|
|
if (idx <= 16) { size = idx << 2;
|
|
} else if (idx <= 24) { size = (idx - 8) << 3;
|
|
} else if (idx <= 32) { size = (idx - 16) << 4;
|
|
} else if (idx <= 40) { size = (idx - 24) << 5;
|
|
} else if (idx <= 48) { size = (idx - 32) << 6;
|
|
} else if (idx <= 56) { size = (idx - 40) << 7;
|
|
} else if (idx <= 64) { size = (idx - 48) << 8;
|
|
#endif
|
|
} else {
|
|
__builtin_trap();
|
|
}
|
|
|
|
return size;
|
|
}
|
|
|
|
static inline AllocIndex sizeToIndex(size_t size) {
|
|
assert(size != 0); // pass '1' if you want a placeholder object
|
|
--size;
|
|
// we could do a table based lookup if we think it worthwhile
|
|
#ifdef __LP64__
|
|
if (size < 0x80) return (size >> 3) + 0x0;
|
|
else if (size < 0x100) return (size >> 4) + 0x8;
|
|
else if (size < 0x200) return (size >> 5) + 0x10;
|
|
else if (size < 0x400) return (size >> 6) + 0x18;
|
|
else if (size < 0x800) return (size >> 7) + 0x20;
|
|
else if (size < 0x1000) return (size >> 8) + 0x28;
|
|
#else
|
|
if (size < 0x40) return (size >> 2) + 0x0;
|
|
else if (size < 0x80) return (size >> 3) + 0x8;
|
|
else if (size < 0x100) return (size >> 4) + 0x10;
|
|
else if (size < 0x200) return (size >> 5) + 0x18;
|
|
else if (size < 0x400) return (size >> 6) + 0x20;
|
|
else if (size < 0x800) return (size >> 7) + 0x28;
|
|
else if (size < 0x1000) return (size >> 8) + 0x30;
|
|
#endif
|
|
return badAllocIndex;
|
|
}
|
|
|
|
size_t swift::_swift_indexToSize(unsigned idx) {
|
|
return indexToSize(idx);
|
|
}
|
|
|
|
int swift::_swift_sizeToIndex(size_t size) {
|
|
return sizeToIndex(size);
|
|
}
|
|
|
|
|
|
__attribute__((noinline,used))
|
|
void *SwiftZone::globalAlloc(AllocIndex idx, uintptr_t flags)
|
|
{
|
|
AllocCacheEntry ptr = NULL;
|
|
size_t size;
|
|
|
|
again:
|
|
swiftZone.writeLock();
|
|
swiftZone.registerThread();
|
|
if ((ptr = swiftZone.globalCache[idx])) {
|
|
swiftZone.globalCache[idx] = swiftZone.globalCache[idx]->next;
|
|
}
|
|
// we should probably refill the cache in bulk
|
|
swiftZone.writeUnlock();
|
|
|
|
if (ptr) {
|
|
return ptr;
|
|
}
|
|
|
|
size = indexToSize(idx);
|
|
|
|
Arena::newArena(idx, size);
|
|
// FIXME -- SWIFT_TRYALLOC
|
|
goto again;
|
|
}
|
|
|
|
extern "C" LLVM_LIBRARY_VISIBILITY
|
|
void _swift_refillThreadAllocCache(AllocIndex idx, uintptr_t flags) {
|
|
void *tmp = SwiftZone::globalAlloc(idx, flags);
|
|
if (!tmp) {
|
|
return;
|
|
}
|
|
SwiftZone::dealloc_semi_optimized(tmp, idx);
|
|
}
|
|
|
|
void *SwiftZone::slowAlloc_optimized(size_t size, size_t alignMask,
|
|
uintptr_t flags) {
|
|
// llvm::RoundUpToAlignment(size, alignMask + 1) generates terrible code
|
|
size = (size + alignMask) & ~alignMask;
|
|
AllocIndex idx = sizeToIndex(size);
|
|
if (idx == badAllocIndex) {
|
|
// large allocations
|
|
void *r = mmapWrapper(size, /*huge*/ true);
|
|
if (r == MAP_FAILED) {
|
|
if (flags & SWIFT_TRYALLOC) {
|
|
return NULL;
|
|
}
|
|
swift::crash("Address space exhausted");
|
|
}
|
|
swiftZone.hugeAllocations.insert(std::pair<void *, size_t>(r, size));
|
|
return r;
|
|
}
|
|
|
|
void *r = tryAlloc_optimized(idx);
|
|
if (r) return r;
|
|
|
|
_swift_refillThreadAllocCache(idx, flags);
|
|
|
|
if (flags & SWIFT_TRYALLOC) {
|
|
return tryAlloc_optimized(idx);
|
|
} else {
|
|
return alloc_optimized(idx);
|
|
}
|
|
}
|
|
|
|
void *SwiftZone::slowAlloc_systemMalloc(size_t size, size_t alignMask,
|
|
uintptr_t flags) {
|
|
// TODO: use posix_memalign if alignMask is larger than the system guarantee.
|
|
return malloc(size);
|
|
}
|
|
|
|
void SwiftZone::slowDealloc_optimized(void *ptr, size_t bytes,
|
|
size_t alignMask) {
|
|
assert(bytes != 0);
|
|
auto size = (bytes + alignMask) & ~alignMask;
|
|
AllocIndex idx = sizeToIndex(size);
|
|
if (idx == badAllocIndex) {
|
|
swiftZone.writeLock();
|
|
auto it2 = swiftZone.hugeAllocations.find(ptr);
|
|
assert(it2);
|
|
int r = munmap(const_cast<void *>(it2->first), it2->second);
|
|
assert(r != -1);
|
|
swiftZone.hugeAllocations.erase(it2);
|
|
swiftZone.writeUnlock();
|
|
return;
|
|
}
|
|
|
|
#ifndef NDEBUG
|
|
swiftZone.readLock();
|
|
auto it = swiftZone.arenas.find(pointerToArena(ptr));
|
|
assert(it != nullptr);
|
|
assert(idx == it->second.index);
|
|
swiftZone.readUnlock();
|
|
#endif
|
|
|
|
dealloc_semi_optimized(ptr, idx);
|
|
}
|
|
|
|
void SwiftZone::slowDealloc_systemMalloc(void *ptr, size_t bytes,
|
|
size_t alignMask) {
|
|
return free(ptr);
|
|
}
|
|
|
|
void SwiftZone::threadExitCleanup(void *arg) {
|
|
(void)arg;
|
|
AllocCacheEntry threadCache[ALLOC_CACHE_COUNT];
|
|
AllocCacheEntry threadCacheTail[ALLOC_CACHE_COUNT];
|
|
for (unsigned i = 0; i < ALLOC_CACHE_COUNT; i++) {
|
|
threadCache[i] = (AllocCacheEntry)_os_tsd_get_direct(
|
|
_swiftAllocOffset + i);
|
|
if (threadCache[i] == NULL) {
|
|
continue;
|
|
}
|
|
_os_tsd_set_direct(_swiftAllocOffset + i, NULL);
|
|
AllocCacheEntry temp = threadCache[i];
|
|
while (temp->next) {
|
|
temp = temp->next;
|
|
}
|
|
threadCacheTail[i] = temp;
|
|
}
|
|
swiftZone.writeLock();
|
|
for (unsigned i = 0; i < ALLOC_CACHE_COUNT; i++) {
|
|
if (threadCache[i] == NULL) {
|
|
continue;
|
|
}
|
|
threadCacheTail[i]->next = swiftZone.globalCache[i];
|
|
swiftZone.globalCache[i] = threadCache[i];
|
|
}
|
|
swiftZone.unregisterThread();
|
|
swiftZone.writeUnlock();
|
|
}
|
|
|
|
static void
|
|
enumerateBlocks(std::function<void(const void *, size_t)> func) {
|
|
// XXX switch to a bitmap or a set
|
|
// FIXME scan other threads
|
|
HeapMap<bool, pointerHash> unusedBlocks;
|
|
for (unsigned i = 0; i < ALLOC_CACHE_COUNT; i++) {
|
|
for (AllocCacheEntry pointer = swiftZone.globalCache[i]; pointer;
|
|
pointer = pointer->next) {
|
|
unusedBlocks.insert(std::pair<void *, bool>(pointer, true));
|
|
}
|
|
for (AllocCacheEntry pointer = getAllocCacheEntry_slow(i); pointer;
|
|
pointer = pointer->next) {
|
|
unusedBlocks.insert(std::pair<void *, bool>(pointer, true));
|
|
}
|
|
}
|
|
|
|
swiftZone.readLock();
|
|
swiftZone.arenas.forEach([&](const void *key, Arena arena) {
|
|
for (size_t i = 0; (i + arena.byteSize) <= arenaSize; i += arena.byteSize) {
|
|
auto pointer = (const void *)((uint8_t *)arena.base + i);
|
|
if (!unusedBlocks[pointer]) {
|
|
func(pointer, arena.byteSize);
|
|
}
|
|
}
|
|
});
|
|
swiftZone.hugeAllocations.forEach(func);
|
|
swiftZone.readUnlock();
|
|
}
|
|
|
|
void _swift_zone_print(malloc_zone_t *zone, boolean_t verbose) {
|
|
enumerateBlocks([&](const void *pointer, size_t size) {
|
|
swift::crash("Swift Zone print not implemented");
|
|
});
|
|
}
|
|
|
|
void _swift_zone_statistics(malloc_zone_t *zone,
|
|
malloc_statistics_t *statistics) {
|
|
memset(statistics, 0, sizeof(*statistics));
|
|
enumerateBlocks([&](const void *pointer, size_t size) {
|
|
++statistics->blocks_in_use;
|
|
statistics->size_in_use += size;
|
|
if (size > statistics->max_size_in_use) {
|
|
statistics->max_size_in_use = size;
|
|
}
|
|
});
|
|
statistics->size_allocated = swiftZone.arenas.size() * arenaSize;
|
|
swiftZone.hugeAllocations.forEach([&] (const void *key, size_t size) {
|
|
statistics->size_allocated += size;
|
|
});
|
|
}
|
|
|
|
malloc_introspection_t SwiftZone::zoneInspection = {
|
|
_swift_zone_enumerator,
|
|
_swift_zone_good_size,
|
|
_swift_zone_check,
|
|
_swift_zone_print,
|
|
_swift_zone_log,
|
|
[] (malloc_zone_t *zone) {
|
|
swiftZone.writeLock();
|
|
},
|
|
[] (malloc_zone_t *zone) {
|
|
swiftZone.writeUnlock();
|
|
},
|
|
_swift_zone_statistics,
|
|
[] (malloc_zone_t *zone) -> boolean_t {
|
|
if (swiftZone.tryWriteLock()) {
|
|
swiftZone.writeUnlock(); return false;
|
|
}
|
|
return true;
|
|
},
|
|
/* Discharge checking. Present in version >= 7. */
|
|
NULL, // enable_discharge_checking
|
|
NULL, // disable_discharge_checking
|
|
NULL, // discharge
|
|
NULL, // enumerate_discharged_pointers
|
|
};
|
|
|
|
// Shims to select between the fast and
|
|
// introspectable/debugging paths at runtime
|
|
void *swift::swift_alloc(AllocIndex idx) {
|
|
return _swift_alloc(idx);
|
|
}
|
|
void *swift::swift_tryAlloc(AllocIndex idx) {
|
|
return _swift_tryAlloc(idx);
|
|
}
|
|
void *swift::swift_slowAlloc(size_t size, size_t alignMask, uintptr_t flags) {
|
|
assert(isAlignmentMask(alignMask));
|
|
return _swift_slowAlloc(size, alignMask, flags);
|
|
}
|
|
void swift::swift_dealloc(void *ptr, AllocIndex idx) {
|
|
_swift_dealloc(ptr, idx);
|
|
}
|
|
void swift::swift_slowDealloc(void *ptr, size_t bytes, size_t alignMask) {
|
|
assert(isAlignmentMask(alignMask));
|
|
return _swift_slowDealloc(ptr, bytes, alignMask);
|
|
}
|
|
|
|
auto swift::_swift_alloc = SwiftZone::alloc_optimized;
|
|
auto swift::_swift_tryAlloc = SwiftZone::tryAlloc_optimized;
|
|
auto swift::_swift_slowAlloc = SwiftZone::slowAlloc_optimized;
|
|
auto swift::_swift_dealloc = SwiftZone::dealloc_optimized;
|
|
auto swift::_swift_slowDealloc = SwiftZone::slowDealloc_optimized;
|