[Backtracing] Add an option to close all file handles (#87382)

Add an option so that when the backtracer starts in a crashing process
it will close all unnecessary file descriptors before it tries to start
gathering the backtrace as that may take time.

rdar://144402533

docs/Backtracing.rst

@@ -118,6 +118,10 @@ follows:
 |                 |         | related to the state of the backtracer.  This is |
 |                 |         | sometimes useful for testing.                    |
 +-----------------+---------+--------------------------------------------------+
+| close-fds       | false   | Close all file descriptors in the crashing       |
+|                 |         | process before starting to gather a crash log.   |
+|                 |         | (This does nothing on Windows.)                  |
++-----------------+---------+--------------------------------------------------+
 
 (*) On macOS 26 and later, this defaults to ``tty`` rather than ``yes``. On
     earlier versions, the default is ``no``.

stdlib/public/runtime/Backtrace.cpp

@@ -150,6 +150,9 @@ SWIFT_RUNTIME_STDLIB_INTERNAL BacktraceSettings _swift_backtraceSettings = {
 
   // outputPath
   NULL,
+
+  // closeFds
+  false,
 };
 
 }
@@ -782,6 +785,8 @@ _swift_processBacktracingSetting(llvm::StringRef key,
     }
   } else if (key.equals_insensitive("cache")) {
     _swift_backtraceSettings.cache = parseBoolean(value);
+  } else if (key.equals_insensitive("close-fds")) {
+    _swift_backtraceSettings.closeFds = parseBoolean(value);
   } else if (key.equals_insensitive("output-to")) {
     if (value.equals_insensitive("auto"))
       _swift_backtraceSettings.outputTo = OutputTo::Auto;

stdlib/public/runtime/BacktracePrivate.h

@@ -144,6 +144,7 @@ struct BacktraceSettings {
   bool             inBacktracer;
   const char      *swiftBacktracePath;
   const char      *outputPath;
+  bool             closeFds;
 };
 
 SWIFT_RUNTIME_STDLIB_INTERNAL BacktraceSettings _swift_backtraceSettings;

stdlib/public/runtime/CrashHandlerLinux.cpp

@@ -65,9 +65,6 @@
 
 #include "BacktracePrivate.h"
 
-// Run the memserver in a thread (0) or separate process (1)
-#define MEMSERVER_USE_PROCESS 0
-
 #ifndef lengthof
 #define lengthof(x)     (sizeof(x) / sizeof(x[0]))
 #endif
@@ -86,6 +83,7 @@ uint32_t currently_paused();
 void wait_paused(uint32_t expected, const struct timespec *timeout);
 int  memserver_start();
 int  memserver_entry(void *);
+void closeFds(int memserver_master_fd);
 
 ssize_t safe_read(int fd, void *buf, size_t len) {
   uint8_t *ptr = (uint8_t *)buf;
@@ -203,7 +201,7 @@ _swift_installCrashHandler()
 
 namespace {
 
-// Older glibc and musl don't have these two syscalls
+// Older glibc and musl don't have these syscalls
 pid_t
 gettid()
 {
@@ -215,6 +213,13 @@ tgkill(int tgid, int tid, int sig) {
   return syscall(SYS_tgkill, tgid, tid, sig);
 }
 
+#define CLOSE_RANGE_UNSHARE 0x2
+#define CLOSE_RANGE_CLOEXEC 0x4
+
+static int _close_range(unsigned int first, unsigned int last, int flags) {
+  return syscall(SYS_close_range, first, last, flags);
+}
+
 void
 reset_signal(int signum)
 {
@@ -269,6 +274,10 @@ handle_fatal_signal(int signum,
 
   _swift_displayCrashMessage(signum, pc);
 
+  if (_swift_backtraceSettings.closeFds) {
+    closeFds(fd);
+  }
+  
   // Actually start the backtracer
   if (!_swift_spawnBacktracer(&crashInfo, fd)) {
     const char *message = _swift_backtraceSettings.color == OnOffTty::On
@@ -279,12 +288,10 @@ handle_fatal_signal(int signum,
       write(STDERR_FILENO, message, strlen(message));
   }
 
-#if !MEMSERVER_USE_PROCESS
-  /* If the memserver is in-process, it may have set signal handlers,
+  /* The memserver may have set signal handlers,
      so reset SIGSEGV and SIGBUS again */
   reset_signal(SIGSEGV);
   reset_signal(SIGBUS);
-#endif
 
   // Restart the other threads
   resume_other_threads();
@@ -682,6 +689,37 @@ int memserver_fd;
 sigjmp_buf memserver_fault_buf;
 pid_t memserver_pid;
 
+#define MIN_FD_TO_CLOSE 3
+#define MAX_FD_TO_CLOSE ~0
+
+void
+closeFds(int memserver_master_fd) {
+  // We don't attempt to close either of the file descriptors for the
+  // pipe used to communicate with the memserver thread.
+
+  // Otherwise we close all file descriptors after stderr except the end of
+  // the pipe used by swift-backtrace.
+
+  int keepOpen1 = memserver_master_fd;
+  int keepOpen2 = memserver_fd;
+
+  if (keepOpen2 > keepOpen1+1) {
+    _close_range(MIN_FD_TO_CLOSE, keepOpen1-1, 0);
+    _close_range(keepOpen1+1, keepOpen2-1, 0);
+    _close_range(keepOpen2+1, MAX_FD_TO_CLOSE, 0);
+  } else if (keepOpen2+1 < keepOpen1) {
+    _close_range(MIN_FD_TO_CLOSE, keepOpen2-1, 0);
+    _close_range(keepOpen2+1, keepOpen1-1, 0);
+    _close_range(keepOpen1+1, MAX_FD_TO_CLOSE, 0);
+  } else {
+    // the file descriptors are adjacent
+    int fd1 = keepOpen2 > keepOpen1 ? keepOpen1 : keepOpen2;
+    int fd2 = keepOpen2 > keepOpen1 ? keepOpen2 : keepOpen1;
+    _close_range(MIN_FD_TO_CLOSE, fd1-1, 0);
+    _close_range(fd2+1, MAX_FD_TO_CLOSE, 0);    
+  }
+}
+
 int
 memserver_start()
 {
@@ -696,33 +734,19 @@ memserver_start()
 
   memserver_fd = fds[0];
   ret = clone(memserver_entry, memserver_stack + sizeof(memserver_stack),
-#if MEMSERVER_USE_PROCESS
-              0,
-#else
               #ifndef __musl__
               // Can't use CLONE_THREAD on musl because the clone() function
               // there returns EINVAL if we do.
               CLONE_THREAD | CLONE_SIGHAND |
               #endif
               CLONE_VM | CLONE_FILES | CLONE_FS | CLONE_IO,
-#endif
               NULL);
   if (ret < 0) {
     memserver_error("memserver_start: clone failed");
     return ret;
   }
 
-#if MEMSERVER_USE_PROCESS
-  memserver_pid = ret;
-
-  /* Tell the Yama LSM module, if it's running, that it's OK for
-     the memserver to read process memory */
-  prctl(PR_SET_PTRACER, ret);
-
-  close(fds[0]);
-#else
   memserver_pid = getpid();
-#endif
 
   return fds[1];
 }
@@ -755,10 +779,6 @@ memserver_entry(void *dummy __attribute__((unused))) {
   int fd = memserver_fd;
   int result = 1;
 
-#if MEMSERVER_USE_PROCESS || defined(__musl__)
-  prctl(PR_SET_NAME, "[backtrace]");
-#endif
-
   struct sigaction sa;
   sigfillset(&sa.sa_mask);
   sa.sa_handler = memserver_fault;

stdlib/public/runtime/CrashHandlerMacOS.cpp

@@ -195,6 +195,16 @@ resume_other_threads()
   os_unfair_lock_unlock(&crashLock);
 }
 
+#define MIN_FD_TO_CLOSE 3
+#define MAX_FD_TO_CLOSE 100
+
+void
+closeFds() {
+  for (int i = MIN_FD_TO_CLOSE; i < MAX_FD_TO_CLOSE; i++) {
+    close(i);
+  }
+}
+
 void
 handle_fatal_signal(int signum,
                     siginfo_t *pinfo,
@@ -252,6 +262,10 @@ handle_fatal_signal(int signum,
 
   _swift_displayCrashMessage(signum, pc);
 
+  if (_swift_backtraceSettings.closeFds) {
+    closeFds();
+  }
+
   /* Start the backtracer; this will suspend the process, so there's no need
      to try to suspend other threads from here. */
   if (!_swift_spawnBacktracer(&crashInfo)) {

stdlib/public/runtime/CrashHandlerWindows.cpp

@@ -187,6 +187,9 @@ LONG reallyHandleException(EXCEPTION_POINTERS *ExceptionInfo) {
   // It isn't safe to try to stop all the threads here on Windows, so we
   // delegate doing that to the backtracer process.
 
+  // Also, closing open file descriptors is
+  // not something we can easily do on Windows.
+
   if (!_swift_spawnBacktracer(&crashInfo)) {
     const char *message = _swift_backtraceSettings.color == OnOffTty::On
       ? " failed\n\n" : " failed ***\n\n";