Teach the frontend about -filelist for input files.

With this, we're out of the business of passing large numbers of input
files on the command line to the frontend, which means we no longer
overflow argv with a mere 1100 input files under whole-module optimization.

In order to make sure this doesn't happen again, I'd like to also get
this working for
- swiftmodule inputs to the merge-module build phase
- /output/ files for multithreading single-frontend builds (WMO)
- object file inputs to the linker on OS X (response files for binutils
  ld have different quoting rules)

Part 3 of https://bugs.swift.org/browse/SR-280.

include/swift/Option/FrontendOptions.td

@@ -28,6 +28,9 @@ def delayed_function_body_parsing :
 def primary_file : Separate<["-"], "primary-file">,
   HelpText<"Produce output for this file, not the whole module">;
 
+def filelist : Separate<["-"], "filelist">,
+  HelpText<"Specify source inputs in a file rather than on the command line">;
+
 def emit_module_doc : Flag<["-"], "emit-module-doc">,
   HelpText<"Emit a module documentation file based on documentation "
            "comments">;

lib/Frontend/CompilerInvocation.cpp

@@ -22,6 +22,7 @@
 #include "llvm/Option/ArgList.h"
 #include "llvm/Option/Option.h"
 #include "llvm/Support/FileSystem.h"
+#include "llvm/Support/LineIterator.h"
 #include "llvm/Support/Path.h"
 
 using namespace swift;
@@ -90,6 +91,34 @@ static void debugFailWithCrash() {
   LLVM_BUILTIN_TRAP;
 }
 
+static unsigned readInputFileList(std::vector<std::string> &inputFiles,
+                                  const llvm::opt::Arg *filelistPath,
+                                  const llvm::opt::Arg *primaryFileArg) {
+  bool foundPrimaryFile = false;
+  unsigned primaryFileIndex = 0;
+  StringRef primaryFile;
+  if (primaryFileArg)
+    primaryFile = primaryFileArg->getValue();
+
+  llvm::ErrorOr<std::unique_ptr<llvm::MemoryBuffer>> buffer =
+      llvm::MemoryBuffer::getFile(filelistPath->getValue());
+  assert(buffer && "can't read filelist; unrecoverable");
+
+  for (StringRef line : make_range(llvm::line_iterator(*buffer.get()), {})) {
+    inputFiles.push_back(line);
+    if (foundPrimaryFile)
+      continue;
+    if (line == primaryFile)
+      foundPrimaryFile = true;
+    else
+      ++primaryFileIndex;
+  }
+
+  if (primaryFileArg)
+    assert(foundPrimaryFile && "primary file not found in filelist");
+  return primaryFileIndex;
+}
+
 static bool ParseFrontendArgs(FrontendOptions &Opts, ArgList &Args,
                               DiagnosticEngine &Diags) {
   using namespace options;
@@ -142,6 +171,14 @@ static bool ParseFrontendArgs(FrontendOptions &Opts, ArgList &Args,
     }
   }
 
+  if (const Arg *A = Args.getLastArg(OPT_filelist)) {
+    const Arg *primaryFileArg = Args.getLastArg(OPT_primary_file);
+    auto primaryFileIndex = readInputFileList(Opts.InputFilenames, A,
+                                              primaryFileArg);
+    if (primaryFileArg)
+      Opts.PrimaryInput = SelectedInput(primaryFileIndex);
+    assert(!Args.hasArg(OPT_INPUT) && "mixing -filelist with inputs");
+  } else {
     for (const Arg *A : make_range(Args.filtered_begin(OPT_INPUT,
                                                        OPT_primary_file),
                                    Args.filtered_end())) {
@@ -154,6 +191,7 @@ static bool ParseFrontendArgs(FrontendOptions &Opts, ArgList &Args,
         llvm_unreachable("Unknown input-related argument!");
       }
     }
+  }
 
   Opts.ParseStdlib |= Args.hasArg(OPT_parse_stdlib);
 

test/Frontend/Inputs/filelist-other.swift

@@ -0,0 +1,4 @@
+struct Foo {}
+struct Bar {}
+
+func other() -> Bar { return Bar() }

test/Frontend/filelist.swift

@@ -0,0 +1,13 @@
+// RUN: rm %t.txt
+// RUN: echo '%S/Inputs/filelist-other.swift' >> %t.txt
+// RUN: echo '%s' >> %t.txt
+// RUN: echo '%S/../Inputs/empty.swift' >> %t.txt
+// RUN: not %target-swift-frontend -parse -filelist %t.txt -primary-file %s 2>&1 | FileCheck %s
+// RUN: not %target-swift-frontend -parse -filelist %t.txt 2>&1 | FileCheck %s
+
+func test() {
+  // Check with FileCheck because we want to see that this file is being
+  // compiled.
+  // CHECK: error: cannot convert value of type 'Bar' to specified type 'Foo'
+  let x: Foo = other()
+}

validation-test/Driver/many-inputs.swift

@@ -0,0 +1,18 @@
+// RUN: rm -rf %t && mkdir %t
+
+// This limit was chosen because multi-threaded compilation broke here on OS X
+// at one point.
+// RUN: for i in {1..1100}; do echo "public func foo$i() {}" > %t/$i.swift; echo "CHECK: foo$i" >> %t/check.txt; done
+
+// RUN: %target-build-swift -force-single-frontend-invocation -emit-library %t/*.swift -o %t/libWMO
+// RUN: nm %t/libWMO | FileCheck %t/check.txt
+
+// RUN: %target-build-swift -force-single-frontend-invocation -num-threads 1 -emit-library %t/*.swift -o %t/libWMOThreaded
+// RUN: nm %t/libWMOThreaded | FileCheck %t/check.txt
+
+// This is very slow due to process overhead. It's also doing one file at a time
+// because we don't have a good way for lit tests to claim more than one thread.
+// But it's still important to check.
+// RUN: %target-build-swift -emit-library %t/*.swift -o %t/libMultiFile
+// RUN: nm %t/libMultiFile | FileCheck %t/check.txt
+