Add fast/basic alloc/dealloc APIs

Swift SVN r1908

lib/IRGen/IRGenModule.cpp

@@ -106,7 +106,7 @@ llvm::Constant *IRGenModule::getAllocFn() {
   llvm::Type *types[] = { HeapMetadataPtrTy, SizeTy, SizeTy };
   llvm::FunctionType *fnType =
     llvm::FunctionType::get(RefCountedPtrTy, types, false);
-  AllocFn = Module.getOrInsertFunction("swift_alloc", fnType);
+  AllocFn = Module.getOrInsertFunction("swift_allocObject", fnType);
   return AllocFn;
 }
 
@@ -153,7 +153,7 @@ llvm::Constant *IRGenModule::getDeallocFn() {
 
   llvm::FunctionType *fnType =
     llvm::FunctionType::get(VoidTy, RefCountedPtrTy, false);
-  DeallocFn = Module.getOrInsertFunction("swift_dealloc", fnType);
+  DeallocFn = Module.getOrInsertFunction("swift_deallocObject", fnType);
   return DeallocFn;
 }
 

runtime/Alloc.cpp

@@ -16,11 +16,26 @@
 
 #include "Alloc.h"
 #include <llvm/Support/MathExtras.h>
+#include <System/pthread_machdep.h>
 #include <cstdlib>
 #include <unistd.h>
 
+struct AllocCacheEntry {
+  struct AllocCacheEntry *next;
+};
+
+// XXX FIXME -- we need to clean this up when the project isn't a secret.
+// There are only 256 slots, and the latter half is basically unused. We can
+// go lower than 128, but we eventually begin to stomp on other frameworks.
+static __attribute__((address_space(256)))
+struct TSD {
+  uintptr_t junk[128];
+  AllocCacheEntry *cache[64];
+  AllocCacheEntry *rawCache[64];
+} *tsd = 0;
+
 struct SwiftHeapObject *
-swift_alloc(struct SwiftHeapMetadata *metadata,
+swift_allocObject(struct SwiftHeapMetadata *metadata,
                   size_t requiredSize,
                   size_t requiredAlignment)
 {
@@ -64,12 +79,184 @@ _swift_release_slow(struct SwiftHeapObject *object)
 {
   size_t allocSize = object->metadata->destroy(object);
   if (allocSize) {
-    swift_dealloc(object, allocSize);
+    swift_deallocObject(object, allocSize);
   }
 }
 
 void
-swift_dealloc(struct SwiftHeapObject *object, size_t allocatedSize)
+swift_deallocObject(struct SwiftHeapObject *object, size_t allocatedSize)
 {
   free(object);
 }
+
+
+// Plain old memory allocation
+
+__attribute__((noinline,used))
+static void *
+_swift_slowAlloc_fixup(size_t off, uint64_t flags)
+{
+  size_t sz;
+
+  off++;
+
+  if (off <= 16) {
+    sz = off << 3;
+  } else if (off <= 24) {
+    sz = (off - 8) << 4;
+  } else if (off <= 32) {
+    sz = (off - 16) << 5;
+  } else if (off <= 40) {
+    sz = (off - 24) << 6;
+  } else if (off <= 48) {
+    sz = (off - 32) << 7;
+  } else if (off <= 56) {
+    sz = (off - 40) << 8;
+  } else {
+    __builtin_trap();
+  }
+
+  return swift_slowAlloc(sz, flags);
+}
+
+
+
+void *
+swift_slowAlloc(size_t bytes, uint64_t flags)
+{
+  void *r;
+
+  do {
+    if (flags & SWIFT_RAWALLOC) {
+      r = malloc(bytes);
+    } else {
+      r = calloc(1, bytes);
+    }
+  } while (!r && !(flags & SWIFT_TRYALLOC));
+
+  return r;
+}
+
+void *
+swift_alloc(size_t idx)
+{
+  AllocCacheEntry *r = tsd->cache[idx];
+  if (r) {
+    tsd->cache[idx] = r->next;
+    return r;
+  }
+  return _swift_slowAlloc_fixup(idx, 0);
+}
+
+void *
+swift_rawAlloc(size_t idx)
+{
+  AllocCacheEntry *r = tsd->rawCache[idx];
+  if (r) {
+    tsd->rawCache[idx] = r->next;
+    return r;
+  }
+  return _swift_slowAlloc_fixup(idx, SWIFT_RAWALLOC);
+}
+
+void *
+swift_tryAlloc(size_t idx)
+{
+  AllocCacheEntry *r = tsd->cache[idx];
+  if (r) {
+    tsd->cache[idx] = r->next;
+    return r;
+  }
+  return _swift_slowAlloc_fixup(idx, SWIFT_TRYALLOC);
+}
+
+void *
+swift_tryRawAlloc(size_t idx)
+{
+  AllocCacheEntry *r = tsd->rawCache[idx];
+  if (r) {
+    tsd->rawCache[idx] = r->next;
+    return r;
+  }
+  return _swift_slowAlloc_fixup(idx, SWIFT_TRYALLOC|SWIFT_RAWALLOC);
+}
+
+void
+swift_dealloc(void *ptr, size_t idx)
+{
+  auto cur = static_cast<AllocCacheEntry *>(ptr);
+  AllocCacheEntry *prev = tsd->cache[idx];
+  cur->next = prev;
+  tsd->cache[idx] = cur;
+}
+
+void
+swift_rawDealloc(void *ptr, size_t idx)
+{
+  auto cur = static_cast<AllocCacheEntry *>(ptr);
+  AllocCacheEntry *prev = tsd->rawCache[idx];
+  cur->next = prev;
+  tsd->rawCache[idx] = cur;
+}
+
+void
+swift_slowDealloc(void *ptr, size_t bytes)
+{
+  size_t idx;
+
+  bytes--;
+
+  if (bytes == SIZE_MAX) {
+    // bytes was zero, therefore bucket 0
+    idx = 0;
+  } else if (bytes < 0x80) {
+    // about 90% of allocations
+    idx = (bytes >> 3);
+  } else if (bytes < 0x100) {
+    idx = (bytes >> 4) + 0x8;
+  } else if (bytes < 0x200) {
+    idx = (bytes >> 5) + 0x10;
+  } else if (bytes < 0x400) {
+    idx = (bytes >> 6) + 0x18;
+  } else if (bytes < 0x800) {
+    idx = (bytes >> 7) + 0x20;
+  } else if (bytes < 0x1000) {
+    idx = (bytes >> 8) + 0x28;
+  } else {
+    // about 1% of allocations
+    return free(ptr);
+  }
+
+  swift_dealloc(ptr, idx);
+}
+
+void
+swift_slowRawDealloc(void *ptr, size_t bytes)
+{
+  size_t idx;
+
+  bytes--;
+
+  if (bytes == SIZE_MAX) {
+    // bytes was zero, therefore bucket 0
+    idx = 0;
+  } else if (bytes < 0x80) {
+    // about 90% of allocations
+    idx = (bytes >> 3);
+  } else if (bytes < 0x100) {
+    idx = (bytes >> 4) + 0x8;
+  } else if (bytes < 0x200) {
+    idx = (bytes >> 5) + 0x10;
+  } else if (bytes < 0x400) {
+    idx = (bytes >> 6) + 0x18;
+  } else if (bytes < 0x800) {
+    idx = (bytes >> 7) + 0x20;
+  } else if (bytes < 0x1000) {
+    idx = (bytes >> 8) + 0x28;
+  } else {
+    // about 1% of allocations
+    return free(ptr);
+  }
+
+  swift_rawDealloc(ptr, idx);
+}

runtime/Alloc.h

@@ -65,9 +65,87 @@ struct SwiftHeapMetadata {
 ///
 /// POSSIBILITIES: The argument order is fair game.  It may be useful
 /// to have a variant which guarantees zero-initialized memory.
-struct SwiftHeapObject *swift_alloc(struct SwiftHeapMetadata *metadata,
-                                    size_t requiredSize,
-                                    size_t requiredAlignment);
+struct SwiftHeapObject *
+swift_allocObject(struct SwiftHeapMetadata *metadata,
+            size_t requiredSize, size_t requiredAlignment);
+
+// Allocate plain old memory, this is the generalized entry point
+//
+// The default API will wait for available memory and return zero filled.
+//
+// The "try" flag tells the runtime to not wait for memory
+// The "raw" flag allocates uninitialized memory.
+// When neither flag is needed, pass zero.
+//
+// If alignment is needed, then please round up to the desired alignment.
+// For example, a 12 byte allocation with 8 byte alignment becomes 16.
+#define SWIFT_TRYALLOC 0x0001
+#define SWIFT_RAWALLOC 0x0002
+void *
+swift_slowAlloc(size_t bytes, uint64_t flags);
+
+// These exist as fast entry points for the above slow API.
+//
+// When the compiler knows that the bytes to be allocated are constant and the
+// value is <= 4KB then the compiler precomputes an offset that the runtime uses
+// to quickly allocate/free from a per-thread cache.
+//
+// The algorithm is like so:
+//
+// if (!__builtin_constant_p(bytes) || (bytes > 0x1000)) {
+//   return swift_slowAlloc(bytes, 0);
+// }
+// if (bytes == 0) {
+//   tinyIndex = 0;
+// } else {
+//   --bytes;
+//   if (bytes < 0x80) {
+//     _slot = (bytes >> 3);
+//   } else if (bytes < 0x100) {
+//     _slot = (bytes >> 4) + 0x8;
+//   } else if (bytes < 0x200) {
+//     _slot = (bytes >> 5) + 0x10;
+//   } else if (bytes < 0x400) {
+//     _slot = (bytes >> 6) + 0x18;
+//   } else if (bytes < 0x800) {
+//     _slot = (bytes >> 7) + 0x20;
+//   } else if (bytes < 0x1000) {
+//     _slot = (bytes >> 8) + 0x28;
+//   } else {
+//     __builtin_trap();
+//   }
+// }
+// swift_alloc(tinyIndex);
+//
+// XXX FIXME -- adjust the algorithm to allow for 32-bit machines to use word
+// sized allocations on the small end of the tiny API.
+void *
+swift_alloc(size_t tinyIndex);
+void *
+swift_rawAlloc(size_t tinyIndex);
+void *
+swift_tryAlloc(size_t tinyIndex);
+void *
+swift_tryRawAlloc(size_t tinyIndex);
+
+
+// Plain old memory deallocation
+//
+// Like swift allocation tiny index trick, but for deallocation
+// If bytes is knowable and fits within the tinyIndex rule:
+void
+swift_dealloc(void *ptr, size_t tinyIndex);
+// If bytes is knowable but is large OR if bytes is not knowable,
+// then use the slow entry point and pass zero:
+void
+swift_slowDealloc(void *ptr, size_t bytes);
+
+// If the caller cannot promise to zero the object during destruction,
+// then call these corresponding APIs:
+void
+swift_rawDealloc(void *ptr, size_t tinyIndex);
+void
+swift_slowRawDealloc(void *ptr, size_t bytes);
 
 /// Atomically increments the retain count of an object.
 ///
@@ -82,13 +160,14 @@ struct SwiftHeapObject *swift_alloc(struct SwiftHeapMetadata *metadata,
 ///      - maybe a variant that can assume a non-null object
 /// It may also prove worthwhile to have this use a custom CC
 /// which preserves a larger set of registers.
-struct SwiftHeapObject *swift_retain(struct SwiftHeapObject *object);
+struct SwiftHeapObject *
+swift_retain(struct SwiftHeapObject *object);
 
 /// Atomically decrements the retain count of an object.  If the
 /// retain count reaches zero, the object is destroyed as follows:
 ///
 ///   size_t allocSize = object->metadata->destroy(object);
-///   if (allocSize) swift_dealloc(object, allocSize);
+///   if (allocSize) swift_deallocObject(object, allocSize);
 ///
 /// \param object - may be null, in which case this is a no-op
 ///
@@ -99,7 +178,8 @@ struct SwiftHeapObject *swift_retain(struct SwiftHeapObject *object);
 ///      - a variant that can safely use non-atomic operations
 ///      - maybe a variant that can assume a non-null object
 /// It's unlikely that a custom CC would be beneficial here.
-void swift_release(struct SwiftHeapObject *object);
+void
+swift_release(struct SwiftHeapObject *object);
 
 /// Deallocate the given memory; it was returned by swift_alloc
 /// but is otherwise in an unknown state.
@@ -111,7 +191,8 @@ void swift_release(struct SwiftHeapObject *object);
 /// POSSIBILITIES: It may be useful to have a variant which
 /// requires the object to have been fully zeroed from offsets
 /// sizeof(SwiftHeapObject) to allocatedSize.
-void swift_dealloc(struct SwiftHeapObject *object, size_t allocatedSize);
+void
+swift_deallocObject(struct SwiftHeapObject *object, size_t allocatedSize);
 
 #ifdef __cplusplus
 } /* extern "C" */