Merge pull request #15013 from davidungar/PR-18-11-fileMap

[Batch mode] Write supplementary output file map. rdar://problem/38157859

include/swift/AST/DiagnosticsDriver.def

@@ -61,7 +61,7 @@ ERROR(error_cannot_specify__o_for_multiple_outputs,none,
       "cannot specify -o when generating multiple output files", ())
 
 ERROR(error_unable_to_load_output_file_map, none,
-      "unable to load output file map", ())
+      "unable to load output file map: %0", (StringRef))
 
 ERROR(error_no_output_file_map_specified,none,
       "no output file map specified", ())

include/swift/AST/DiagnosticsFrontend.def

@@ -133,6 +133,9 @@ ERROR(error_stdlib_not_found,Fatal,
 ERROR(error_underlying_module_not_found,none,
       "underlying Objective-C module %0 not found", (Identifier))
 
+ERROR(error_unable_to_load_supplementary_output_file_map, none,
+      "unable to load supplementary output file map '%0': %1", (StringRef, StringRef))
+
 ERROR(error_repl_requires_no_input_files,none,
   "REPL mode requires no input files", ())
 ERROR(error_mode_requires_one_input_file,none,
@@ -163,6 +166,9 @@ ERROR(error_cannot_have_input_files_with_file_list,none,
 ERROR(error_cannot_have_primary_files_with_primary_file_list,none,
 "cannot have primary input files with primary file list", ())
 
+ERROR(error_cannot_have_supplementary_outputs,none,
+"cannot have '%0' with '%1'", (StringRef, StringRef))
+
 ERROR(error_duplicate_input_file,none,
 "duplicate input file '%0'", (StringRef))
 

include/swift/Driver/Action.h

@@ -14,8 +14,8 @@
 #define SWIFT_DRIVER_ACTION_H
 
 #include "swift/Basic/LLVM.h"
-#include "swift/Driver/Types.h"
 #include "swift/Driver/Util.h"
+#include "swift/Frontend/Types.h"
 #include "llvm/ADT/ArrayRef.h"
 #include "llvm/ADT/Optional.h"
 #include "llvm/ADT/StringSwitch.h"

include/swift/Driver/Compilation.h

@@ -17,12 +17,13 @@
 #ifndef SWIFT_DRIVER_COMPILATION_H
 #define SWIFT_DRIVER_COMPILATION_H
 
-#include "swift/Driver/Driver.h"
-#include "swift/Driver/Job.h"
-#include "swift/Driver/Util.h"
 #include "swift/Basic/ArrayRefView.h"
 #include "swift/Basic/LLVM.h"
 #include "swift/Basic/Statistic.h"
+#include "swift/Driver/Driver.h"
+#include "swift/Driver/Job.h"
+#include "swift/Driver/Util.h"
+#include "swift/Frontend/OutputFileMap.h"
 #include "llvm/ADT/DenseSet.h"
 #include "llvm/ADT/StringRef.h"
 #include "llvm/Support/Chrono.h"
@@ -167,6 +168,11 @@ private:
   /// Provides a randomization seed to batch-mode partitioning, for debugging.
   unsigned BatchSeed;
 
+  /// In order to test repartitioning, set to true if
+  /// -driver-force-one-batch-repartition is present. This is cleared after the
+  /// forced repartition happens.
+  bool ForceOneBatchRepartition = false;
+
   /// True if temporary files should not be deleted.
   bool SaveTemps;
 
@@ -210,6 +216,7 @@ public:
               bool EnableIncrementalBuild = false,
               bool EnableBatchMode = false,
               unsigned BatchSeed = 0,
+              bool ForceOneBatchRepartition = false,
               bool SkipTaskExecution = false,
               bool SaveTemps = false,
               bool ShowDriverTimeCompilation = false,
@@ -272,6 +279,8 @@ public:
     return EnableBatchMode;
   }
 
+  bool getForceOneBatchRepartition() const { return ForceOneBatchRepartition; }
+
   bool getContinueBuildingAfterErrors() const {
     return ContinueBuildingAfterErrors;
   }

include/swift/Driver/Driver.h

@@ -21,9 +21,9 @@
 #include "swift/Basic/LLVM.h"
 #include "swift/Basic/OptionSet.h"
 #include "swift/Basic/Sanitizers.h"
-#include "swift/Driver/OutputFileMap.h"
-#include "swift/Driver/Types.h"
 #include "swift/Driver/Util.h"
+#include "swift/Frontend/OutputFileMap.h"
+#include "swift/Frontend/Types.h"
 #include "llvm/ADT/DenseMap.h"
 #include "llvm/ADT/StringMap.h"
 #include "llvm/ADT/StringRef.h"
@@ -50,7 +50,6 @@ namespace driver {
   class Compilation;
   class Job;
   class JobAction;
-  class OutputFileMap;
   class ToolChain;
 
 /// \brief A class encapsulating information about the outputs the driver
@@ -116,6 +115,14 @@ public:
   std::string SDKPath;
 
   OptionSet<SanitizerKind> SelectedSanitizers;
+
+  /// Might this sort of compile have explicit primary inputs?
+  /// When running a single compile for the whole module (in other words
+  /// "whole-module-optimization" mode) there must be no -primary-input's and
+  /// nothing in a (preferably non-existent) -primary-filelist. Left to its own
+  /// devices, the driver would forget to omit the primary input files, so
+  /// return a flag here.
+  bool mightHaveExplicitPrimaryInputs(const CommandOutput &Output) const;
 };
 
 class Driver {
@@ -159,6 +166,9 @@ private:
   /// Provides a randomization seed to batch-mode partitioning, for debugging.
   unsigned DriverBatchSeed = 0;
 
+  /// Forces a repartition for testing.
+  bool DriverForceOneBatchRepartition = false;
+
 public:
   Driver(StringRef DriverExecutable, StringRef Name,
          ArrayRef<const char *> Args, DiagnosticEngine &Diags);
@@ -251,7 +261,7 @@ public:
                     Compilation &C) const;
 
   /// Construct the OutputFileMap for the driver from the given arguments.
-  std::unique_ptr<OutputFileMap>
+  Optional<OutputFileMap>
   buildOutputFileMap(const llvm::opt::DerivedArgList &Args,
                      StringRef workingDirectory) const;
 

include/swift/Driver/Job.h

@@ -15,16 +15,16 @@
 
 #include "swift/Basic/LLVM.h"
 #include "swift/Driver/Action.h"
-#include "swift/Driver/OutputFileMap.h"
-#include "swift/Driver/Types.h"
 #include "swift/Driver/Util.h"
-#include "llvm/Option/Option.h"
+#include "swift/Frontend/OutputFileMap.h"
+#include "swift/Frontend/Types.h"
 #include "llvm/ADT/ArrayRef.h"
 #include "llvm/ADT/DenseMap.h"
 #include "llvm/ADT/PointerIntPair.h"
 #include "llvm/ADT/SmallSet.h"
 #include "llvm/ADT/SmallVector.h"
 #include "llvm/ADT/StringRef.h"
+#include "llvm/Option/Option.h"
 #include "llvm/Support/Chrono.h"
 #include "llvm/Support/raw_ostream.h"
 
@@ -206,9 +206,15 @@ public:
   /// the sole primary input.
   StringRef getAnyOutputForType(types::ID type) const;
 
+  /// Return the whole derived output map.
+  const OutputFileMap &getDerivedOutputMap() const;
+
   /// Return the BaseInput numbered by \p Index.
   StringRef getBaseInput(size_t Index) const;
 
+  /// Write a file map naming the outputs for each primary input.
+  void writeOutputFileMap(llvm::raw_ostream &out) const;
+
   void print(raw_ostream &Stream) const;
   void dump() const LLVM_ATTRIBUTE_USED;
 

include/swift/Driver/PrettyStackTrace.h

@@ -13,8 +13,8 @@
 #ifndef SWIFT_DRIVER_PRETTYSTACKTRACE_H
 #define SWIFT_DRIVER_PRETTYSTACKTRACE_H
 
+#include "swift/Frontend/Types.h"
 #include "llvm/Support/PrettyStackTrace.h"
-#include "swift/Driver/Types.h"
 
 namespace swift {
 namespace driver {

include/swift/Driver/ToolChain.h

@@ -13,11 +13,11 @@
 #ifndef SWIFT_DRIVER_TOOLCHAIN_H
 #define SWIFT_DRIVER_TOOLCHAIN_H
 
-#include "swift/Driver/Action.h"
-#include "swift/Driver/Types.h"
 #include "swift/Basic/LLVM.h"
-#include "llvm/Option/Option.h"
+#include "swift/Driver/Action.h"
+#include "swift/Frontend/Types.h"
 #include "llvm/ADT/Triple.h"
+#include "llvm/Option/Option.h"
 
 #include <memory>
 
@@ -54,6 +54,9 @@ protected:
   private:
     Compilation &C;
 
+    /// The limit for passing a list of files on the command line.
+    static const size_t TOO_MANY_FILES = 128;
+
   public:
     ArrayRef<const Job *> Inputs;
     ArrayRef<const Action *> InputActions;
@@ -83,6 +86,34 @@ protected:
     /// arguments.
     const char *getTemporaryFilePath(const llvm::Twine &name,
                                      StringRef suffix = "") const;
+
+    /// For frontend, merge-module, and link invocations.
+    bool shouldUseInputFileList() const;
+
+    bool shouldUsePrimaryInputFileListInFrontendInvocation() const;
+
+    bool shouldUseMainOutputFileListInFrontendInvocation() const;
+
+    bool shouldUseSupplementaryOutputFileMapInFrontendInvocation() const;
+
+    /// Reify the existing behavior that SingleCompile compile actions do not
+    /// filter, but batch-mode and single-file compilations do. Some clients are
+    /// relying on this (i.e., they pass inputs that don't have ".swift" as an
+    /// extension.) It would be nice to eliminate this distinction someday.
+    bool shouldFilterFrontendInputsByType() const;
+
+    const char *computeFrontendModeForCompile() const;
+
+    void addFrontendInputAndOutputArguments(
+        llvm::opt::ArgStringList &Arguments,
+        std::vector<FilelistInfo> &FilelistInfos) const;
+
+  private:
+    void addFrontendCommandLineInputArguments(
+        bool mayHavePrimaryInputs, bool useFileList, bool usePrimaryFileList,
+        bool filterByType, llvm::opt::ArgStringList &arguments) const;
+    void addFrontendSupplementaryOutputArguments(
+        llvm::opt::ArgStringList &arguments) const;
   };
 
   /// Packs together information chosen by toolchains to create jobs.
@@ -203,7 +234,6 @@ public:
   virtual bool sanitizerRuntimeLibExists(const llvm::opt::ArgList &args,
                                          StringRef sanitizer,
                                          bool shared=true) const;
-
 };
 } // end namespace driver
 } // end namespace swift

include/swift/Driver/Util.h

@@ -13,8 +13,8 @@
 #ifndef SWIFT_DRIVER_UTIL_H
 #define SWIFT_DRIVER_UTIL_H
 
-#include "swift/Driver/Types.h"
 #include "swift/Basic/LLVM.h"
+#include "swift/Frontend/Types.h"
 #include "llvm/ADT/SmallVector.h"
 
 namespace llvm {
@@ -46,7 +46,12 @@ namespace driver {
     enum class WhichFiles : unsigned {
       Input,
       PrimaryInputs,
-      Output
+      Output,
+      /// Batch mode frontend invocations may have so many supplementary
+      /// outputs that they don't comfortably fit as command-line arguments.
+      /// In that case, add a FilelistInfo to record the path to the file.
+      /// The type is ignored.
+      SupplementaryOutput,
     };
 
     StringRef path;

include/swift/Frontend/ArgsToFrontendOptionsConverter.h

@@ -54,15 +54,8 @@ private:
 
   bool checkUnusedSupplementaryOutputPaths() const;
 
-  /// \returns the output filenames on the command line or in the output
-  /// filelist, or an empty vector if there were neither -o's nor an output
-  /// filelist.
-  ArrayRef<std::string> getOutputFilenamesFromCommandLineOrFilelist();
-
   bool checkForUnusedOutputPaths() const;
 
-  std::vector<std::string> readOutputFileList(StringRef filelistPath) const;
-
 public:
   ArgsToFrontendOptionsConverter(DiagnosticEngine &Diags,
                                  const llvm::opt::ArgList &Args,

include/swift/Frontend/ArgsToFrontendOutputsConverter.h

@@ -130,10 +130,7 @@ private:
   /// \Return a set of supplementary output paths for each input that might
   /// produce supplementary outputs, or None to signal an error.
   /// \note
-  /// At present, only one input can produce supplementary outputs, but
+  /// Batch-mode supports multiple primary inputs.
-  /// in the future, batch-mode will support multiple primary inputs and thus,
-  /// multiple sets of supplementary outputs. This function is written with that
-  /// future in mind.
   /// \par
   /// The paths are derived from arguments
   /// such as -emit-module-path. These are not the final computed paths,
@@ -145,6 +142,11 @@ private:
   Optional<std::vector<SupplementaryOutputPaths>>
   getSupplementaryOutputPathsFromArguments() const;
 
+  /// Read a supplementary output file map file.
+  /// \returns `None` if it could not open the file map.
+  Optional<std::vector<SupplementaryOutputPaths>>
+  readSupplementaryOutputFileMap() const;
+
   /// Given an ID corresponding to supplementary output argument
   /// (e.g. -emit-module-path), collect all such paths, and ensure
   /// there are the right number of them.

include/swift/Frontend/OutputFileMap.h

@@ -14,9 +14,11 @@
 #define SWIFT_DRIVER_OUTPUTFILEMAP_H
 
 #include "swift/Basic/LLVM.h"
-#include "swift/Driver/Types.h"
+#include "swift/Frontend/Types.h"
 #include "llvm/ADT/DenseMap.h"
 #include "llvm/ADT/StringMap.h"
+#include "llvm/ADT/StringSet.h"
+#include "llvm/Support/Error.h"
 #include "llvm/Support/MemoryBuffer.h"
 #include "llvm/Support/SourceMgr.h"
 #include "llvm/Support/YAMLParser.h"
@@ -25,7 +27,6 @@
 #include <string>
 
 namespace swift {
-namespace driver {
 
 typedef llvm::DenseMap<types::ID, std::string> TypeToPathMap;
 
@@ -38,14 +39,12 @@ public:
 
   ~OutputFileMap() = default;
 
-  /// Loads an OutputFileMap from the given \p Path, if possible.
+  /// Loads an OutputFileMap from the given \p Path into the receiver, if
-  ///
+  /// possible.
-  /// When non-empty, \p workingDirectory is used to resolve relative paths in
+  static llvm::Expected<OutputFileMap> loadFromPath(StringRef Path,
-  /// the output file map.
+                                                    StringRef workingDirectory);
-  static std::unique_ptr<OutputFileMap>
-  loadFromPath(StringRef Path, StringRef workingDirectory);
 
-  static std::unique_ptr<OutputFileMap>
+  static llvm::Expected<OutputFileMap>
   loadFromBuffer(StringRef Data, StringRef workingDirectory);
 
   /// Loads an OutputFileMap from the given \p Buffer, taking ownership
@@ -53,7 +52,7 @@ public:
   ///
   /// When non-empty, \p workingDirectory is used to resolve relative paths in
   /// the output file map.
-  static std::unique_ptr<OutputFileMap>
+  static llvm::Expected<OutputFileMap>
   loadFromBuffer(std::unique_ptr<llvm::MemoryBuffer> Buffer,
                  StringRef workingDirectory);
 
@@ -74,19 +73,16 @@ public:
   /// Dump the OutputFileMap to the given \p os.
   void dump(llvm::raw_ostream &os, bool Sort = false) const;
 
+  /// Write the OutputFilemap for the \p inputs so it can be parsed.
+  void write(llvm::raw_ostream &os, ArrayRef<StringRef> inputs) const;
+
 private:
-  /// \brief Parses the given \p Buffer into the OutputFileMap, taking ownership
+  /// \brief Parses the given \p Buffer and returns either an OutputFileMap or
-  /// of \p Buffer in the process.
+  /// error, taking ownership of \p Buffer in the process.
-  ///
+  static llvm::Expected<OutputFileMap>
-  /// When non-empty, \p workingDirectory is used to resolve relative paths in
+  parse(std::unique_ptr<llvm::MemoryBuffer> Buffer, StringRef workingDirectory);
-  /// the output file map.
-  ///
-  /// \returns true on error, false on success
-  bool parse(std::unique_ptr<llvm::MemoryBuffer> Buffer,
-             StringRef workingDirectory);
 };
 
-} // end namespace driver
 } // end namespace swift
 
 #endif

include/swift/Frontend/Types.h

@@ -19,11 +19,10 @@
 #include <functional>
 
 namespace swift {
-namespace driver {
 namespace types {
   enum ID : uint8_t {
 #define TYPE(NAME, ID, TEMP_SUFFIX, FLAGS) TY_##ID,
-#include "swift/Driver/Types.def"
+#include "swift/Frontend/Types.def"
 #undef TYPE
     TY_INVALID
   };
@@ -62,13 +61,12 @@ namespace types {
   template <typename Fn>
   void forAllTypes(const Fn &fn);
 } // end namespace types
-} // end namespace driver
 } // end namespace swift
 
 namespace llvm {
   template<>
-  struct DenseMapInfo<swift::driver::types::ID> {
+  struct DenseMapInfo<swift::types::ID> {
-    using ID = swift::driver::types::ID;
+    using ID = swift::types::ID;
     static inline ID getEmptyKey() {
       return ID::TY_INVALID;
     }
@@ -85,7 +83,7 @@ namespace llvm {
 }
 
 template <typename Fn>
-void swift::driver::types::forAllTypes(const Fn &fn) {
+void swift::types::forAllTypes(const Fn &fn) {
   static_assert(std::is_constructible<std::function<void(types::ID)>,Fn>::value,
                 "must have the signature 'void(types::ID)'");
   for (uint8_t i = 0; i < static_cast<uint8_t>(TY_INVALID); ++i)

include/swift/Option/FrontendOptions.td

@@ -30,6 +30,9 @@ def primary_filelist : Separate<["-"], "primary-filelist">,
   HelpText<"Specify primary inputs in a file rather than on the command line">;
 def output_filelist : Separate<["-"], "output-filelist">,
   HelpText<"Specify outputs in a file rather than on the command line">;
+def supplementary_output_file_map : Separate<["-"], "supplementary-output-file-map">,
+  HelpText<"Specify supplementary outputs in a file rather than on the command line">;
+
 
 def emit_module_doc : Flag<["-"], "emit-module-doc">,
   HelpText<"Emit a module documentation file based on documentation "

include/swift/Option/Options.td

@@ -105,6 +105,9 @@ def driver_use_filelists : Flag<["-"], "driver-use-filelists">,
 def driver_batch_seed : Separate<["-"], "driver-batch-seed">,
   InternalDebugOpt,
   HelpText<"Use the given seed value to randomize batch-mode partitions">;
+def driver_force_one_batch_repartition : Flag<["-"], "driver-force-one-batch-repartition">,
+  InternalDebugOpt,
+  HelpText<"Force one batch repartitioning for testing">;
 
 def driver_always_rebuild_dependents :
   Flag<["-"], "driver-always-rebuild-dependents">, InternalDebugOpt,

lib/Driver/CMakeLists.txt

@@ -5,12 +5,10 @@ set(swiftDriver_sources
   Driver.cpp
   FrontendUtil.cpp
   Job.cpp
-  OutputFileMap.cpp
   ParseableOutput.cpp
   PrettyStackTrace.cpp
   ToolChain.cpp
   ToolChains.cpp
-  Types.cpp
   )
 
 set(swiftDriver_targetDefines)

lib/Driver/Compilation.cpp

@@ -26,6 +26,8 @@
 #include "swift/Driver/Job.h"
 #include "swift/Driver/ParseableOutput.h"
 #include "swift/Driver/ToolChain.h"
+#include "swift/Frontend/OutputFileMap.h"
+#include "swift/Option/Options.h"
 #include "llvm/ADT/DenseSet.h"
 #include "llvm/ADT/MapVector.h"
 #include "llvm/ADT/SetVector.h"
@@ -37,12 +39,14 @@
 #include "llvm/Support/Compiler.h"
 #include "llvm/Support/FileSystem.h"
 #include "llvm/Support/Path.h"
-#include "llvm/Support/raw_ostream.h"
 #include "llvm/Support/Timer.h"
 #include "llvm/Support/YAMLParser.h"
+#include "llvm/Support/raw_ostream.h"
 
 #include "CompilationRecord.h"
 
+#define DEBUG_TYPE "batch-mode"
+
 // Batch-mode has a sub-mode for testing that randomizes batch partitions,
 // by user-provided seed. That is the only thing randomized here.
 #include <random>
@@ -103,6 +107,7 @@ Compilation::Compilation(DiagnosticEngine &Diags,
                          bool EnableIncrementalBuild,
                          bool EnableBatchMode,
                          unsigned BatchSeed,
+                         bool ForceOneBatchRepartition,
                          bool SkipTaskExecution,
                          bool SaveTemps,
                          bool ShowDriverTimeCompilation,
@@ -119,12 +124,14 @@ Compilation::Compilation(DiagnosticEngine &Diags,
     EnableIncrementalBuild(EnableIncrementalBuild),
     EnableBatchMode(EnableBatchMode),
     BatchSeed(BatchSeed),
+    ForceOneBatchRepartition(ForceOneBatchRepartition),
     SaveTemps(SaveTemps),
     ShowDriverTimeCompilation(ShowDriverTimeCompilation),
     Stats(std::move(StatsReporter)) {
 };
 
-static bool writeFilelistIfNecessary(const Job *job, DiagnosticEngine &diags);
+static bool writeFilelistIfNecessary(const Job *job, const ArgList &args,
+                                     DiagnosticEngine &diags);
 
 using CommandSet = llvm::SmallPtrSet<const Job *, 16>;
 using CommandSetVector = llvm::SetVector<const Job*>;
@@ -254,7 +261,8 @@ namespace driver {
 
     void addPendingJobToTaskQueue(const Job *Cmd) {
       // FIXME: Failing here should not take down the whole process.
-      bool success = writeFilelistIfNecessary(Cmd, Comp.Diags);
+      bool success =
+          writeFilelistIfNecessary(Cmd, *Comp.TranslatedArgs.get(), Comp.Diags);
       assert(success && "failed to write filelist");
       (void)success;
 
@@ -787,12 +795,20 @@ namespace driver {
         });
     }
 
-    // FIXME: at the moment we're not passing OutputFileMaps to frontends, so
+    // Due to the multiplication of the number of additional files and the
-    // due to the multiplication of the number of additional files and the
     // number of files in a batch, it's pretty easy to construct too-long
     // command lines here, which will then fail to exec. We address this crudely
     // by re-forming batches with a finer partition when we overflow.
+    //
+    // Now that we're passing OutputFileMaps to frontends, this should never
+    // happen, but keep this as insurance, because the decision to pass output
+    // file maps cannot know the exact length of the command line, so may
+    // possibly fail to use the OutputFileMap.
+    //
+    // In order to be able to exercise as much of the code paths as possible,
+    // take a flag to force a retry, but only once.
     bool shouldRetryWithMorePartitions(std::vector<const Job *> const &Batches,
+                                       bool &PretendTheCommandLineIsTooLongOnce,
                                        size_t &NumPartitions) {
 
       // Stop rebatching if we can't subdivide batches any further.
@@ -801,10 +817,14 @@ namespace driver {
 
       for (auto const *B : Batches) {
         if (!llvm::sys::commandLineFitsWithinSystemLimits(B->getExecutable(),
-                                                          B->getArguments())) {
+                                                          B->getArguments()) ||
+            PretendTheCommandLineIsTooLongOnce) {
+          PretendTheCommandLineIsTooLongOnce = false;
           // To avoid redoing the batch loop too many times, repartition pretty
           // aggressively by doubling partition count / halving size.
           NumPartitions *= 2;
+          DEBUG(llvm::dbgs()
+                << "Should have used a supplementary output file map.\n");
           return true;
         }
       }
@@ -827,6 +847,8 @@ namespace driver {
       size_t NumPartitions = Comp.NumberOfParallelCommands;
       CommandSetVector Batchable, NonBatchable;
       std::vector<const Job *> Batches;
+      bool PretendTheCommandLineIsTooLongOnce =
+          Comp.getForceOneBatchRepartition();
       do {
         // We might be restarting loop; clear these before proceeding.
         Batchable.clear();
@@ -845,7 +867,8 @@ namespace driver {
           formBatchJobFromPartitionBatch(Batches, Batch);
         }
 
-      } while (shouldRetryWithMorePartitions(Batches, NumPartitions));
+      } while (shouldRetryWithMorePartitions(
+          Batches, PretendTheCommandLineIsTooLongOnce, NumPartitions));
       PendingExecution.clear();
 
       // Save batches so we can locate and decompose them on task-exit.
@@ -1089,7 +1112,8 @@ static void writeCompilationRecord(StringRef path, StringRef argsHash,
   }
 }
 
-static bool writeFilelistIfNecessary(const Job *job, DiagnosticEngine &diags) {
+static bool writeFilelistIfNecessary(const Job *job, const ArgList &args,
+                                     DiagnosticEngine &diags) {
   bool ok = true;
   for (const FilelistInfo &filelistInfo : job->getFilelistInfos()) {
     if (filelistInfo.path.empty())
@@ -1121,18 +1145,29 @@ static bool writeFilelistIfNecessary(const Job *job, DiagnosticEngine &diags) {
       }
       break;
     case FilelistInfo::WhichFiles::PrimaryInputs:
-      for (const Action *A : job->getSource().getInputs()) {
+      // Ensure that -index-file-path works in conjunction with
-        const auto *IA = cast<InputAction>(A);
+      // -driver-use-filelists. It needs to be the only primary.
-        out << IA->getInputArg().getValue() << "\n";
+      if (Arg *A = args.getLastArg(options::OPT_index_file_path))
+        out << A->getValue() << "\n";
+      else {
+        // The normal case for non-single-compile jobs.
+        for (const Action *A : job->getSource().getInputs()) {
+          const auto *IA = cast<InputAction>(A);
+          out << IA->getInputArg().getValue() << "\n";
+        }
       }
       break;
-    case FilelistInfo::WhichFiles::Output:
+    case FilelistInfo::WhichFiles::Output: {
       const CommandOutput &outputInfo = job->getOutput();
       assert(outputInfo.getPrimaryOutputType() == filelistInfo.type);
       for (auto &output : outputInfo.getPrimaryOutputFilenames())
         out << output << "\n";
       break;
     }
+    case FilelistInfo::WhichFiles::SupplementaryOutput:
+      job->getOutput().writeOutputFileMap(out);
+      break;
+    }
   }
   return ok;
 }
@@ -1171,7 +1206,7 @@ int Compilation::performSingleCommand(const Job *Cmd) {
     break;
   }
 
-  if (!writeFilelistIfNecessary(Cmd, Diags))
+  if (!writeFilelistIfNecessary(Cmd, *TranslatedArgs.get(), Diags))
     return 1;
 
   switch (Level) {

lib/Driver/Driver.cpp

@@ -17,25 +17,25 @@
 #include "swift/Driver/Driver.h"
 
 #include "ToolChains.h"
-#include "swift/Strings.h"
 #include "swift/AST/DiagnosticEngine.h"
 #include "swift/AST/DiagnosticsDriver.h"
 #include "swift/AST/DiagnosticsFrontend.h"
 #include "swift/Basic/LLVM.h"
-#include "swift/Basic/TaskQueue.h"
-#include "swift/Basic/Version.h"
 #include "swift/Basic/Range.h"
 #include "swift/Basic/Statistic.h"
+#include "swift/Basic/TaskQueue.h"
+#include "swift/Basic/Version.h"
+#include "swift/Config.h"
 #include "swift/Driver/Action.h"
 #include "swift/Driver/Compilation.h"
 #include "swift/Driver/Job.h"
-#include "swift/Driver/OutputFileMap.h"
 #include "swift/Driver/PrettyStackTrace.h"
 #include "swift/Driver/ToolChain.h"
+#include "swift/Frontend/OutputFileMap.h"
 #include "swift/Option/Options.h"
 #include "swift/Option/SanitizerOptions.h"
 #include "swift/Parse/Lexer.h"
-#include "swift/Config.h"
+#include "swift/Strings.h"
 #include "llvm/ADT/APInt.h"
 #include "llvm/ADT/DenseSet.h"
 #include "llvm/ADT/STLExtras.h"
@@ -531,6 +531,8 @@ Driver::buildCompilation(const ToolChain &TC,
                      A->getAsString(*ArgList), A->getValue());
     }
   }
+  DriverForceOneBatchRepartition =
+      ArgList->hasArg(options::OPT_driver_force_one_batch_repartition);
 
   bool Incremental = ArgList->hasArg(options::OPT_incremental);
   if (ArgList->hasArg(options::OPT_whole_module_optimization)) {
@@ -621,7 +623,7 @@ Driver::buildCompilation(const ToolChain &TC,
     // REPL mode expects no input files, so suppress the error.
     SuppressNoInputFilesError = true;
 
-  std::unique_ptr<OutputFileMap> OFM =
+  Optional<OutputFileMap> OFM =
       buildOutputFileMap(*TranslatedArgList, workingDirectory);
 
   if (Diags.hadAnyError())
@@ -704,13 +706,14 @@ Driver::buildCompilation(const ToolChain &TC,
                                                  Incremental,
                                                  BatchMode,
                                                  DriverBatchSeed,
+                                                 DriverForceOneBatchRepartition,
                                                  DriverSkipExecution,
                                                  SaveTemps,
                                                  ShowDriverTimeCompilation,
                                                  std::move(StatsReporter)));
   // Construct the graph of Actions.
   SmallVector<const Action *, 8> TopLevelActions;
-  buildActions(TopLevelActions, TC, OI, OFM.get(),
+  buildActions(TopLevelActions, TC, OI, OFM ? OFM.getPointer() : nullptr,
                rebuildEverything ? nullptr : &outOfDateMap, *C);
 
   if (Diags.hadAnyError())
@@ -721,7 +724,8 @@ Driver::buildCompilation(const ToolChain &TC,
     return nullptr;
   }
 
-  buildJobs(TopLevelActions, OI, OFM.get(), workingDirectory, TC, *C);
+  buildJobs(TopLevelActions, OI, OFM ? OFM.getPointer() : nullptr,
+            workingDirectory, TC, *C);
 
   if (DriverPrintDerivedOutputFileMap) {
     C->getDerivedOutputFileMap().dump(llvm::outs(), true);
@@ -1102,7 +1106,8 @@ void Driver::buildOutputInfo(const ToolChain &TC, const DerivedArgList &Args,
 
   } else { // DriverKind::Batch
     OI.CompilerMode = OutputInfo::Mode::StandardCompile;
-    if (Args.hasArg(options::OPT_whole_module_optimization))
+    if (Args.hasArg(options::OPT_whole_module_optimization,
+                    options::OPT_index_file))
       OI.CompilerMode = OutputInfo::Mode::SingleCompile;
     OI.CompilerOutputType = types::TY_Object;
   }
@@ -1697,20 +1702,21 @@ bool Driver::handleImmediateArgs(const ArgList &Args, const ToolChain &TC) {
   return true;
 }
 
-std::unique_ptr<OutputFileMap>
+Optional<OutputFileMap>
 Driver::buildOutputFileMap(const llvm::opt::DerivedArgList &Args,
                            StringRef workingDirectory) const {
   const Arg *A = Args.getLastArg(options::OPT_output_file_map);
   if (!A)
-    return nullptr;
+    return None;
 
   // TODO: perform some preflight checks to ensure the file exists.
-  auto OFM = OutputFileMap::loadFromPath(A->getValue(), workingDirectory);
+  llvm::Expected<OutputFileMap> OFM =
-  if (!OFM) {
+      OutputFileMap::loadFromPath(A->getValue(), workingDirectory);
-    // TODO: emit diagnostic with error string
+  if (auto Err = OFM.takeError()) {
-    Diags.diagnose(SourceLoc(), diag::error_unable_to_load_output_file_map);
+    Diags.diagnose(SourceLoc(), diag::error_unable_to_load_output_file_map,
+                   llvm::toString(std::move(Err)));
   }
-  return OFM;
+  return *OFM;
 }
 
 void Driver::buildJobs(ArrayRef<const Action *> TopLevelActions,
@@ -2650,3 +2656,17 @@ void Driver::printHelp(bool ShowHidden) const {
   getOpts().PrintHelp(llvm::outs(), Name.c_str(), "Swift compiler",
                       IncludedFlagsBitmask, ExcludedFlagsBitmask);
 }
+
+bool OutputInfo::mightHaveExplicitPrimaryInputs(
+    const CommandOutput &Output) const {
+  switch (CompilerMode) {
+  case Mode::StandardCompile:
+  case Mode::BatchModeCompile:
+    return true;
+  case Mode::SingleCompile:
+    return false;
+  case Mode::Immediate:
+  case Mode::REPL:
+    llvm_unreachable("REPL and immediate modes handled elsewhere");
+  }
+}

lib/Driver/Job.cpp

@@ -240,6 +240,10 @@ StringRef CommandOutput::getAnyOutputForType(types::ID Type) const {
   return getAdditionalOutputForType(Type);
 }
 
+const OutputFileMap &CommandOutput::getDerivedOutputMap() const {
+  return DerivedOutputMap;
+}
+
 StringRef CommandOutput::getBaseInput(size_t Index) const {
   assert(Index < Inputs.size());
   return Inputs[Index].Base;
@@ -310,6 +314,16 @@ CommandOutput::dump() const {
   llvm::errs() << '\n';
 }
 
+void CommandOutput::writeOutputFileMap(llvm::raw_ostream &out) const {
+  SmallVector<StringRef, 4> inputs;
+  for (const CommandInputPair IP : Inputs) {
+    assert(IP.Base == IP.Primary && !IP.Base.empty() &&
+           "output file maps won't work if these differ");
+    inputs.push_back(IP.Primary);
+  }
+  getDerivedOutputMap().write(out, inputs);
+}
+
 Job::~Job() = default;
 
 void Job::printArguments(raw_ostream &os,

lib/Driver/ParseableOutput.cpp

@@ -15,7 +15,7 @@
 #include "swift/Basic/JSONSerialization.h"
 #include "swift/Driver/Action.h"
 #include "swift/Driver/Job.h"
-#include "swift/Driver/Types.h"
+#include "swift/Frontend/Types.h"
 #include "llvm/Option/Arg.h"
 #include "llvm/Support/raw_ostream.h"
 

lib/Driver/PrettyStackTrace.cpp

@@ -13,7 +13,7 @@
 #include "swift/Driver/PrettyStackTrace.h"
 #include "swift/Driver/Action.h"
 #include "swift/Driver/Job.h"
-#include "swift/Driver/Types.h"
+#include "swift/Frontend/Types.h"
 #include "llvm/Option/Arg.h"
 #include "llvm/Support/raw_ostream.h"
 

lib/Driver/ToolChain.cpp

@@ -21,24 +21,23 @@
 #include "swift/Driver/Compilation.h"
 #include "swift/Driver/Driver.h"
 #include "swift/Driver/Job.h"
+#include "llvm/ADT/STLExtras.h"
 #include "llvm/ADT/SetVector.h"
 #include "llvm/Option/ArgList.h"
 #include "llvm/Support/FileSystem.h"
 #include "llvm/Support/Path.h"
 #include "llvm/Support/Program.h"
-#include "llvm/ADT/STLExtras.h"
 
 using namespace swift;
 using namespace swift::driver;
 using namespace llvm::opt;
 
-ToolChain::JobContext::JobContext(Compilation &C,
+ToolChain::JobContext::JobContext(Compilation &C, ArrayRef<const Job *> Inputs,
-                                  ArrayRef<const Job *> Inputs,
                                   ArrayRef<const Action *> InputActions,
                                   const CommandOutput &Output,
                                   const OutputInfo &OI)
-  : C(C), Inputs(Inputs), InputActions(InputActions), Output(Output),
+    : C(C), Inputs(Inputs), InputActions(InputActions), Output(Output), OI(OI),
-    OI(OI), Args(C.getArgs()) {}
+      Args(C.getArgs()) {}
 
 ArrayRef<InputPair> ToolChain::JobContext::getTopLevelInputFiles() const {
   return C.getInputFiles();

lib/Driver/ToolChains.cpp

@@ -17,6 +17,7 @@
 #include "swift/Basic/Platform.h"
 #include "swift/Basic/Range.h"
 #include "swift/Basic/TaskQueue.h"
+#include "swift/Driver/Compilation.h"
 #include "swift/Driver/Driver.h"
 #include "swift/Driver/Job.h"
 #include "swift/Frontend/Frontend.h"
@@ -37,8 +38,41 @@ using namespace swift;
 using namespace swift::driver;
 using namespace llvm::opt;
 
-/// The limit for passing a list of files on the command line.
+bool ToolChain::JobContext::shouldUseInputFileList() const {
-static const size_t TOO_MANY_FILES = 128;
+  if (Args.hasArg(options::OPT_driver_use_filelists))
+    return true;
+  return getTopLevelInputFiles().size() > TOO_MANY_FILES;
+}
+
+bool ToolChain::JobContext::shouldUsePrimaryInputFileListInFrontendInvocation()
+    const {
+  if (Args.hasArg(options::OPT_driver_use_filelists))
+    return true;
+  return InputActions.size() > TOO_MANY_FILES;
+}
+
+bool ToolChain::JobContext::shouldUseMainOutputFileListInFrontendInvocation()
+    const {
+  if (Args.hasArg(options::OPT_driver_use_filelists))
+    return true;
+  return Output.getPrimaryOutputFilenames().size() > TOO_MANY_FILES;
+}
+
+bool ToolChain::JobContext::
+    shouldUseSupplementaryOutputFileMapInFrontendInvocation() const {
+  if (Args.hasArg(options::OPT_driver_use_filelists))
+    return true;
+  static const unsigned UpperBoundOnSupplementaryOutputFileTypes =
+      types::TY_INVALID;
+  return InputActions.size() * UpperBoundOnSupplementaryOutputFileTypes >
+         TOO_MANY_FILES;
+}
+
+bool ToolChain::JobContext::shouldFilterFrontendInputsByType() const {
+  // FIXME: SingleCompile has not filtered its inputs in the past and now people
+  // rely upon that. But we would like the compilation modes to be consistent.
+  return OI.CompilerMode != OutputInfo::Mode::SingleCompile;
+}
 
 static void addInputsOfType(ArgStringList &Arguments,
                             ArrayRef<const Action *> Inputs,
@@ -214,19 +248,10 @@ static void addCommonFrontendArgs(const ToolChain &TC,
   inputArgs.AddAllArgs(arguments, options::OPT_Xllvm);
   inputArgs.AddAllArgs(arguments, options::OPT_Xcc);
 
-  addOutputsOfType(arguments, output, inputArgs,
-                   types::TY_SwiftModuleDocFile,
-                   "-emit-module-doc-path");
-
   if (llvm::sys::Process::StandardErrHasColors())
     arguments.push_back("-color-diagnostics");
-
-  addOutputsOfType(arguments, output, inputArgs,
-                   types::TY_SerializedDiagnostics,
-                   "-serialize-diagnostics-path");
 }
 
-
 ToolChain::InvocationInfo
 ToolChain::constructInvocation(const CompileJobAction &job,
                                const JobContext &context) const {
@@ -235,164 +260,15 @@ ToolChain::constructInvocation(const CompileJobAction &job,
 
   Arguments.push_back("-frontend");
 
-  // Determine the frontend mode option.
+  {
-  const char *FrontendModeOption = nullptr;
+    // Determine the frontend mode option.
-  switch (context.OI.CompilerMode) {
+    const char *FrontendModeOption = context.computeFrontendModeForCompile();
-  case OutputInfo::Mode::StandardCompile:
+    assert(FrontendModeOption != nullptr &&
-  case OutputInfo::Mode::SingleCompile:
+           "No frontend mode option specified!");
-  case OutputInfo::Mode::BatchModeCompile: {
+    Arguments.push_back(FrontendModeOption);
-    switch (context.Output.getPrimaryOutputType()) {
-    case types::TY_Object:
-      FrontendModeOption = "-c";
-      break;
-    case types::TY_PCH:
-      FrontendModeOption = "-emit-pch";
-      break;
-    case types::TY_RawSIL:
-      FrontendModeOption = "-emit-silgen";
-      break;
-    case types::TY_SIL:
-      FrontendModeOption = "-emit-sil";
-      break;
-    case types::TY_RawSIB:
-      FrontendModeOption = "-emit-sibgen";
-      break;
-    case types::TY_SIB:
-      FrontendModeOption = "-emit-sib";
-      break;
-    case types::TY_LLVM_IR:
-      FrontendModeOption = "-emit-ir";
-      break;
-    case types::TY_LLVM_BC:
-      FrontendModeOption = "-emit-bc";
-      break;
-    case types::TY_Assembly:
-      FrontendModeOption = "-S";
-      break;
-    case types::TY_SwiftModuleFile:
-      // Since this is our primary output, we need to specify the option here.
-      FrontendModeOption = "-emit-module";
-      break;
-    case types::TY_ImportedModules:
-      FrontendModeOption = "-emit-imported-modules";
-      break;
-    case types::TY_IndexData:
-      FrontendModeOption = "-typecheck";
-      break;
-    case types::TY_Remapping:
-      FrontendModeOption = "-update-code";
-      break;
-    case types::TY_Nothing:
-      // We were told to output nothing, so get the last mode option and use that.
-      if (const Arg *A = context.Args.getLastArg(options::OPT_modes_Group))
-        FrontendModeOption = A->getSpelling().data();
-      else
-        llvm_unreachable("We were told to perform a standard compile, "
-                         "but no mode option was passed to the driver.");
-      break;
-    case types::TY_Swift:
-    case types::TY_dSYM:
-    case types::TY_AutolinkFile:
-    case types::TY_Dependencies:
-    case types::TY_SwiftModuleDocFile:
-    case types::TY_ClangModuleFile:
-    case types::TY_SerializedDiagnostics:
-    case types::TY_ObjCHeader:
-    case types::TY_Image:
-    case types::TY_SwiftDeps:
-    case types::TY_ModuleTrace:
-    case types::TY_TBD:
-    case types::TY_OptRecord:
-      llvm_unreachable("Output type can never be primary output.");
-    case types::TY_INVALID:
-      llvm_unreachable("Invalid type ID");
-    }
-    break;
-  }
-  case OutputInfo::Mode::Immediate:
-  case OutputInfo::Mode::REPL:
-    llvm_unreachable("REPL and immediate modes handled elsewhere");
   }
 
-  assert(FrontendModeOption != nullptr && "No frontend mode option specified!");
+  context.addFrontendInputAndOutputArguments(Arguments, II.FilelistInfos);
-  
-  Arguments.push_back(FrontendModeOption);
-
-  // Add input arguments.
-  switch (context.OI.CompilerMode) {
-  case OutputInfo::Mode::StandardCompile:
-    assert(context.InputActions.size() == 1 &&
-           "Standard-compile mode takes exactly one input (the primary file)");
-    LLVM_FALLTHROUGH;
-  case OutputInfo::Mode::BatchModeCompile: {
-    bool UseFileList =
-      (context.Args.hasArg(options::OPT_driver_use_filelists) ||
-       context.getTopLevelInputFiles().size() > TOO_MANY_FILES);
-    bool UsePrimaryFileList =
-      ((context.OI.CompilerMode == OutputInfo::Mode::BatchModeCompile) &&
-       (context.Args.hasArg(options::OPT_driver_use_filelists) ||
-        context.InputActions.size() > TOO_MANY_FILES));
-    if (UseFileList) {
-      Arguments.push_back("-filelist");
-      Arguments.push_back(context.getAllSourcesPath());
-    }
-    if (UsePrimaryFileList) {
-      Arguments.push_back("-primary-filelist");
-      Arguments.push_back(context.getTemporaryFilePath("primaryInputs", ""));
-      II.FilelistInfos.push_back({Arguments.back(), types::TY_Swift,
-                                  FilelistInfo::WhichFiles::PrimaryInputs});
-    }
-    if (!UseFileList || !UsePrimaryFileList) {
-      llvm::StringSet<> primaries;
-      for (const Action *A : context.InputActions) {
-        const auto *IA = cast<InputAction>(A);
-        primaries.insert(IA->getInputArg().getValue());
-      }
-      for (auto inputPair : context.getTopLevelInputFiles()) {
-        if (!types::isPartOfSwiftCompilation(inputPair.first))
-          continue;
-        const char *inputName = inputPair.second->getValue();
-        if (primaries.count(inputName)) {
-          if (!UsePrimaryFileList) {
-            Arguments.push_back("-primary-file");
-            Arguments.push_back(inputName);
-          }
-        } else {
-          if (!UseFileList) {
-            Arguments.push_back(inputName);
-          }
-        }
-      }
-    }
-    break;
-  }
-  case OutputInfo::Mode::SingleCompile: {
-    StringRef PrimaryFileWrittenToCommandLine;
-    if (context.Output.getPrimaryOutputType() == types::TY_IndexData) {
-      if (Arg *A = context.Args.getLastArg(options::OPT_index_file_path)) {
-        Arguments.push_back("-primary-file");
-        Arguments.push_back(A->getValue());
-        PrimaryFileWrittenToCommandLine = A->getValue();
-      }
-    }
-    if (context.Args.hasArg(options::OPT_driver_use_filelists) ||
-        context.InputActions.size() > TOO_MANY_FILES) {
-      Arguments.push_back("-filelist");
-      Arguments.push_back(context.getAllSourcesPath());
-    } else {
-      for (const Action *A : context.InputActions) {
-        const Arg &InputArg = cast<InputAction>(A)->getInputArg();
-        // Frontend no longer tolerates redundant input files on command line.
-        if (InputArg.getValue() != PrimaryFileWrittenToCommandLine)
-          InputArg.render(context.Args, Arguments);
-      }
-    }
-    break;
-  }
-  case OutputInfo::Mode::Immediate:
-  case OutputInfo::Mode::REPL:
-    llvm_unreachable("REPL and immediate modes handled elsewhere");
-  }
 
   // Forward migrator flags.
   if (auto DataPath = context.Args.getLastArg(options::
@@ -453,33 +329,6 @@ ToolChain::constructInvocation(const CompileJobAction &job,
   Arguments.push_back("-module-name");
   Arguments.push_back(context.Args.MakeArgString(context.OI.ModuleName));
 
-  addOutputsOfType(Arguments, context.Output, context.Args,
-                   types::ID::TY_SwiftModuleFile,
-                   "-emit-module-path");
-
-  if (addOutputsOfType(Arguments, context.Output, context.Args,
-                       types::ID::TY_ObjCHeader, "-emit-objc-header-path")) {
-      assert(context.OI.CompilerMode == OutputInfo::Mode::SingleCompile &&
-             "The Swift tool should only emit an Obj-C header in single compile"
-             "mode!");
-  }
-
-  addOutputsOfType(Arguments, context.Output, context.Args,
-                   types::TY_Dependencies,
-                   "-emit-dependencies-path");
-
-  addOutputsOfType(Arguments, context.Output, context.Args,
-                   types::TY_SwiftDeps,
-                   "-emit-reference-dependencies-path");
-
-  addOutputsOfType(Arguments, context.Output, context.Args,
-                   types::TY_ModuleTrace,
-                   "-emit-loaded-module-trace-path");
-
-  addOutputsOfType(Arguments, context.Output, context.Args,
-                   types::TY_TBD,
-                   "-emit-tbd-path");
-
   addOutputsOfType(Arguments, context.Output, context.Args,
                    types::TY_OptRecord,
                    "-save-optimization-record-path");
@@ -500,8 +349,7 @@ ToolChain::constructInvocation(const CompileJobAction &job,
 
   // Add the output file argument if necessary.
   if (context.Output.getPrimaryOutputType() != types::TY_Nothing) {
-    if (context.Args.hasArg(options::OPT_driver_use_filelists) ||
+    if (context.shouldUseMainOutputFileListInFrontendInvocation()) {
-        context.Output.getPrimaryOutputFilenames().size() > TOO_MANY_FILES) {
       Arguments.push_back("-output-filelist");
       Arguments.push_back(context.getTemporaryFilePath("outputs", ""));
       II.FilelistInfos.push_back({Arguments.back(),
@@ -526,6 +374,185 @@ ToolChain::constructInvocation(const CompileJobAction &job,
   return II;
 }
 
+const char *ToolChain::JobContext::computeFrontendModeForCompile() const {
+  switch (OI.CompilerMode) {
+  case OutputInfo::Mode::StandardCompile:
+  case OutputInfo::Mode::SingleCompile:
+  case OutputInfo::Mode::BatchModeCompile:
+    break;
+  case OutputInfo::Mode::Immediate:
+  case OutputInfo::Mode::REPL:
+    llvm_unreachable("REPL and immediate modes handled elsewhere");
+  }
+  switch (Output.getPrimaryOutputType()) {
+  case types::TY_Object:
+    return "-c";
+  case types::TY_PCH:
+    return "-emit-pch";
+  case types::TY_RawSIL:
+    return "-emit-silgen";
+  case types::TY_SIL:
+    return "-emit-sil";
+  case types::TY_RawSIB:
+    return "-emit-sibgen";
+  case types::TY_SIB:
+    return "-emit-sib";
+  case types::TY_LLVM_IR:
+    return "-emit-ir";
+  case types::TY_LLVM_BC:
+    return "-emit-bc";
+  case types::TY_Assembly:
+    return "-S";
+  case types::TY_SwiftModuleFile:
+    // Since this is our primary output, we need to specify the option here.
+    return "-emit-module";
+  case types::TY_ImportedModules:
+    return "-emit-imported-modules";
+  case types::TY_IndexData:
+    return "-typecheck";
+  case types::TY_Remapping:
+    return "-update-code";
+  case types::TY_Nothing:
+    // We were told to output nothing, so get the last mode option and use that.
+    if (const Arg *A = Args.getLastArg(options::OPT_modes_Group))
+      return A->getSpelling().data();
+    else
+      llvm_unreachable("We were told to perform a standard compile, "
+                       "but no mode option was passed to the driver.");
+  case types::TY_Swift:
+  case types::TY_dSYM:
+  case types::TY_AutolinkFile:
+  case types::TY_Dependencies:
+  case types::TY_SwiftModuleDocFile:
+  case types::TY_ClangModuleFile:
+  case types::TY_SerializedDiagnostics:
+  case types::TY_ObjCHeader:
+  case types::TY_Image:
+  case types::TY_SwiftDeps:
+  case types::TY_ModuleTrace:
+  case types::TY_TBD:
+  case types::TY_OptRecord:
+    llvm_unreachable("Output type can never be primary output.");
+  case types::TY_INVALID:
+    llvm_unreachable("Invalid type ID");
+  }
+}
+
+void ToolChain::JobContext::addFrontendInputAndOutputArguments(
+    ArgStringList &Arguments, std::vector<FilelistInfo> &FilelistInfos) const {
+
+  switch (OI.CompilerMode) {
+  case OutputInfo::Mode::StandardCompile:
+    assert(InputActions.size() == 1 &&
+           "Standard-compile mode takes exactly one input (the primary file)");
+    break;
+  case OutputInfo::Mode::BatchModeCompile:
+  case OutputInfo::Mode::SingleCompile:
+    break;
+  case OutputInfo::Mode::Immediate:
+  case OutputInfo::Mode::REPL:
+    llvm_unreachable("REPL and immediate modes handled elsewhere");
+  }
+
+  const bool UseFileList = shouldUseInputFileList();
+  const bool MayHavePrimaryInputs = OI.mightHaveExplicitPrimaryInputs(Output);
+  const bool UsePrimaryFileList =
+      MayHavePrimaryInputs &&
+      shouldUsePrimaryInputFileListInFrontendInvocation();
+  const bool FilterInputsByType = shouldFilterFrontendInputsByType();
+  const bool UseSupplementaryOutputFileList =
+      shouldUseSupplementaryOutputFileMapInFrontendInvocation();
+
+  if (UseFileList) {
+    Arguments.push_back("-filelist");
+    Arguments.push_back(getAllSourcesPath());
+  }
+  if (UsePrimaryFileList) {
+    Arguments.push_back("-primary-filelist");
+    Arguments.push_back(getTemporaryFilePath("primaryInputs", ""));
+    FilelistInfos.push_back({Arguments.back(), types::TY_Swift,
+                             FilelistInfo::WhichFiles::PrimaryInputs});
+  }
+  if (!UseFileList || !UsePrimaryFileList) {
+    addFrontendCommandLineInputArguments(MayHavePrimaryInputs, UseFileList,
+                                         UsePrimaryFileList, FilterInputsByType,
+                                         Arguments);
+  }
+
+  if (UseSupplementaryOutputFileList) {
+    Arguments.push_back("-supplementary-output-file-map");
+    Arguments.push_back(getTemporaryFilePath("supplementaryOutputs", ""));
+    FilelistInfos.push_back({Arguments.back(), types::TY_INVALID,
+                             FilelistInfo::WhichFiles::SupplementaryOutput});
+  } else {
+    addFrontendSupplementaryOutputArguments(Arguments);
+  }
+}
+
+void ToolChain::JobContext::addFrontendCommandLineInputArguments(
+    const bool mayHavePrimaryInputs, const bool useFileList,
+    const bool usePrimaryFileList, const bool filterByType,
+    ArgStringList &arguments) const {
+  llvm::StringSet<> primaries;
+
+  if (mayHavePrimaryInputs) {
+    for (const Action *A : InputActions) {
+      const auto *IA = cast<InputAction>(A);
+      const llvm::opt::Arg &InArg = IA->getInputArg();
+      primaries.insert(InArg.getValue());
+    }
+  }
+  // -index-file compilations are weird. They are processed as SingleCompiles
+  // (WMO), but must indicate that there is one primary file, designated by
+  // -index-file-path.
+  if (Arg *A = Args.getLastArg(options::OPT_index_file_path)) {
+    assert(primaries.empty() &&
+           "index file jobs should be treated as single (WMO) compiles");
+    primaries.insert(A->getValue());
+  }
+  for (auto inputPair : getTopLevelInputFiles()) {
+    if (filterByType && !types::isPartOfSwiftCompilation(inputPair.first))
+      continue;
+    const char *inputName = inputPair.second->getValue();
+    const bool isPrimary = primaries.count(inputName);
+    if (isPrimary && !usePrimaryFileList) {
+      arguments.push_back("-primary-file");
+      arguments.push_back(inputName);
+    }
+    if ((!isPrimary || usePrimaryFileList) && !useFileList)
+      arguments.push_back(inputName);
+  }
+}
+
+void ToolChain::JobContext::addFrontendSupplementaryOutputArguments(
+    ArgStringList &arguments) const {
+  // FIXME: Get these and other argument strings from the same place for both
+  // driver and frontend.
+  addOutputsOfType(arguments, Output, Args, types::ID::TY_SwiftModuleFile,
+                   "-emit-module-path");
+
+  addOutputsOfType(arguments, Output, Args, types::TY_SwiftModuleDocFile,
+                   "-emit-module-doc-path");
+
+  addOutputsOfType(arguments, Output, Args, types::TY_SerializedDiagnostics,
+                   "-serialize-diagnostics-path");
+
+  if (addOutputsOfType(arguments, Output, Args, types::ID::TY_ObjCHeader,
+                       "-emit-objc-header-path")) {
+    assert(OI.CompilerMode == OutputInfo::Mode::SingleCompile &&
+           "The Swift tool should only emit an Obj-C header in single compile"
+           "mode!");
+  }
+
+  addOutputsOfType(arguments, Output, Args, types::TY_Dependencies,
+                   "-emit-dependencies-path");
+  addOutputsOfType(arguments, Output, Args, types::TY_SwiftDeps,
+                   "-emit-reference-dependencies-path");
+  addOutputsOfType(arguments, Output, Args, types::TY_ModuleTrace,
+                   "-emit-loaded-module-trace-path");
+  addOutputsOfType(arguments, Output, Args, types::TY_TBD, "-emit-tbd-path");
+}
+
 ToolChain::InvocationInfo
 ToolChain::constructInvocation(const InterpretJobAction &job,
                                const JobContext &context) const {
@@ -711,8 +738,7 @@ ToolChain::constructInvocation(const MergeModuleJobAction &job,
   Arguments.push_back("-merge-modules");
   Arguments.push_back("-emit-module");
 
-  if (context.Args.hasArg(options::OPT_driver_use_filelists) ||
+  if (context.shouldUseInputFileList()) {
-      context.Inputs.size() > TOO_MANY_FILES) {
     Arguments.push_back("-filelist");
     Arguments.push_back(context.getTemporaryFilePath("inputs", ""));
     II.FilelistInfos.push_back({Arguments.back(), types::TY_SwiftModuleFile,
@@ -747,6 +773,14 @@ ToolChain::constructInvocation(const MergeModuleJobAction &job,
 
   addCommonFrontendArgs(*this, context.OI, context.Output, context.Args,
                         Arguments);
+  addOutputsOfType(Arguments, context.Output, context.Args,
+                   types::TY_SwiftModuleDocFile, "-emit-module-doc-path");
+  addOutputsOfType(Arguments, context.Output, context.Args,
+                   types::TY_SerializedDiagnostics,
+                   "-serialize-diagnostics-path");
+  addOutputsOfType(Arguments, context.Output, context.Args,
+                   types::TY_ObjCHeader, "-emit-objc-header-path");
+
   context.Args.AddLastArg(Arguments, options::OPT_import_objc_header);
 
   Arguments.push_back("-module-name");
@@ -755,13 +789,6 @@ ToolChain::constructInvocation(const MergeModuleJobAction &job,
   assert(context.Output.getPrimaryOutputType() == types::TY_SwiftModuleFile &&
          "The MergeModule tool only produces swiftmodule files!");
 
-  auto ObjCHeaderOutputPath =
-    context.Output.getAdditionalOutputForType(types::TY_ObjCHeader);
-  if (!ObjCHeaderOutputPath.empty()) {
-    Arguments.push_back("-emit-objc-header-path");
-    Arguments.push_back(
-        context.Args.MakeArgString(ObjCHeaderOutputPath));
-  }
 
   Arguments.push_back("-o");
   Arguments.push_back(context.Args.MakeArgString(
@@ -899,6 +926,9 @@ ToolChain::constructInvocation(const GeneratePCHJobAction &job,
 
   addCommonFrontendArgs(*this, context.OI, context.Output, context.Args,
                         Arguments);
+  addOutputsOfType(Arguments, context.Output, context.Args,
+                   types::TY_SerializedDiagnostics,
+                   "-serialize-diagnostics-path");
 
   addInputsOfType(Arguments, context.InputActions, types::TY_ObjCHeader);
   context.Args.AddLastArg(Arguments, options::OPT_index_store_path);
@@ -1267,8 +1297,7 @@ toolchains::Darwin::constructInvocation(const LinkJobAction &job,
   InvocationInfo II = {LD};
   ArgStringList &Arguments = II.Arguments;
 
-  if (context.Args.hasArg(options::OPT_driver_use_filelists) ||
+  if (context.shouldUseInputFileList()) {
-      context.Inputs.size() > TOO_MANY_FILES) {
     Arguments.push_back("-filelist");
     Arguments.push_back(context.getTemporaryFilePath("inputs", "LinkFileList"));
     II.FilelistInfos.push_back(

lib/Frontend/ArgsToFrontendOutputsConverter.cpp

@@ -16,6 +16,7 @@
 #include "swift/Frontend/ArgsToFrontendInputsConverter.h"
 #include "swift/Frontend/ArgsToFrontendOptionsConverter.h"
 #include "swift/Frontend/Frontend.h"
+#include "swift/Frontend/OutputFileMap.h"
 #include "swift/Option/Options.h"
 #include "swift/Option/SanitizerOptions.h"
 #include "swift/Strings.h"
@@ -233,7 +234,9 @@ SupplementaryOutputPathsComputer::SupplementaryOutputPathsComputer(
 Optional<std::vector<SupplementaryOutputPaths>>
 SupplementaryOutputPathsComputer::computeOutputPaths() const {
   Optional<std::vector<SupplementaryOutputPaths>> pathsFromUser =
-      getSupplementaryOutputPathsFromArguments();
+      Args.hasArg(options::OPT_supplementary_output_file_map)
+          ? readSupplementaryOutputFileMap()
+          : getSupplementaryOutputPathsFromArguments();
   if (!pathsFromUser)
     return None;
 
@@ -458,3 +461,70 @@ void SupplementaryOutputPathsComputer::deriveModulePathParameters(
   mainOutputIfUsable =
       canUseMainOutputForModule && !OutputFiles.empty() ? OutputFiles[0] : "";
 }
+
+static SupplementaryOutputPaths
+createFromTypeToPathMap(const TypeToPathMap *map) {
+  SupplementaryOutputPaths paths;
+  if (!map)
+    return paths;
+  const std::pair<types::ID, std::string &> typesAndStrings[] = {
+      {types::TY_ObjCHeader, paths.ObjCHeaderOutputPath},
+      {types::TY_SwiftModuleFile, paths.ModuleOutputPath},
+      {types::TY_SwiftModuleDocFile, paths.ModuleDocOutputPath},
+      {types::TY_Dependencies, paths.DependenciesFilePath},
+      {types::TY_SwiftDeps, paths.ReferenceDependenciesFilePath},
+      {types::TY_SerializedDiagnostics, paths.SerializedDiagnosticsPath},
+      {types::TY_ModuleTrace, paths.LoadedModuleTracePath},
+      {types::TY_TBD, paths.TBDPath}};
+  for (const std::pair<types::ID, std::string &> &typeAndString :
+       typesAndStrings) {
+    auto const out = map->find(typeAndString.first);
+    typeAndString.second = out == map->end() ? "" : out->second;
+  }
+  return paths;
+}
+
+Optional<std::vector<SupplementaryOutputPaths>>
+SupplementaryOutputPathsComputer::readSupplementaryOutputFileMap() const {
+  if (Arg *A = Args.getLastArg(options::OPT_emit_objc_header_path,
+                               options::OPT_emit_module_path,
+                               options::OPT_emit_module_doc_path,
+                               options::OPT_emit_dependencies_path,
+                               options::OPT_emit_reference_dependencies_path,
+                               options::OPT_serialize_diagnostics_path,
+                               options::OPT_emit_loaded_module_trace_path,
+                               options::OPT_emit_tbd_path)) {
+    Diags.diagnose(SourceLoc(),
+                   diag::error_cannot_have_supplementary_outputs,
+                   A->getSpelling(), "-supplementary-output-file-map");
+    return None;
+  }
+  const StringRef supplementaryFileMapPath =
+      Args.getLastArgValue(options::OPT_supplementary_output_file_map);
+  llvm::ErrorOr<std::unique_ptr<llvm::MemoryBuffer>> buffer =
+      llvm::MemoryBuffer::getFile(supplementaryFileMapPath);
+  if (!buffer) {
+    Diags.diagnose(SourceLoc(), diag::cannot_open_file,
+                   supplementaryFileMapPath, buffer.getError().message());
+    return None;
+  }
+  llvm::Expected<OutputFileMap> OFM =
+      OutputFileMap::loadFromBuffer(std::move(buffer.get()), "");
+  if (auto Err = OFM.takeError()) {
+    Diags.diagnose(SourceLoc(),
+                   diag::error_unable_to_load_supplementary_output_file_map,
+                   supplementaryFileMapPath, llvm::toString(std::move(Err)));
+    return None;
+  }
+
+  std::vector<SupplementaryOutputPaths> outputPaths;
+  InputsAndOutputs.forEachInputProducingSupplementaryOutput(
+      [&](const InputFile &input) -> bool {
+        const TypeToPathMap *mapForInput =
+            OFM->getOutputMapForInput(input.file());
+        outputPaths.push_back(createFromTypeToPathMap(mapForInput));
+        return false;
+      });
+
+  return outputPaths;
+}

lib/Frontend/CMakeLists.txt

@@ -7,8 +7,10 @@ add_swift_library(swiftFrontend STATIC
   Frontend.cpp
   FrontendInputsAndOutputs.cpp
   FrontendOptions.cpp
+  OutputFileMap.cpp
   PrintingDiagnosticConsumer.cpp
   SerializedDiagnosticConsumer.cpp
+  Types.cpp
   DEPENDS
     SwiftOptions
   LINK_LIBRARIES

lib/Frontend/OutputFileMap.cpp

@@ -10,7 +10,7 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include "swift/Driver/OutputFileMap.h"
+#include "swift/Frontend/OutputFileMap.h"
 #include "llvm/ADT/SmallString.h"
 #include "llvm/ADT/StringRef.h"
 #include "llvm/Support/Path.h"
@@ -18,34 +18,28 @@
 #include <system_error>
 
 using namespace swift;
-using namespace swift::driver;
 
-std::unique_ptr<OutputFileMap>
+llvm::Expected<OutputFileMap>
 OutputFileMap::loadFromPath(StringRef Path, StringRef workingDirectory) {
   llvm::ErrorOr<std::unique_ptr<llvm::MemoryBuffer>> FileBufOrErr =
-    llvm::MemoryBuffer::getFile(Path);
+      llvm::MemoryBuffer::getFile(Path);
-  if (!FileBufOrErr)
+  if (!FileBufOrErr) {
-    return nullptr;
+    return llvm::errorCodeToError(FileBufOrErr.getError());
+  }
   return loadFromBuffer(std::move(FileBufOrErr.get()), workingDirectory);
 }
 
-std::unique_ptr<OutputFileMap>
+llvm::Expected<OutputFileMap>
 OutputFileMap::loadFromBuffer(StringRef Data, StringRef workingDirectory) {
   std::unique_ptr<llvm::MemoryBuffer> Buffer{
-    llvm::MemoryBuffer::getMemBuffer(Data)
+      llvm::MemoryBuffer::getMemBuffer(Data)};
-  };
   return loadFromBuffer(std::move(Buffer), workingDirectory);
 }
 
-std::unique_ptr<OutputFileMap>
+llvm::Expected<OutputFileMap>
 OutputFileMap::loadFromBuffer(std::unique_ptr<llvm::MemoryBuffer> Buffer,
                               StringRef workingDirectory) {
-  std::unique_ptr<OutputFileMap> OFM(new OutputFileMap());
+  return parse(std::move(Buffer), workingDirectory);
-
-  if (OFM->parse(std::move(Buffer), workingDirectory))
-    return nullptr;
-
-  return OFM;
 }
 
 const TypeToPathMap *OutputFileMap::getOutputMapForInput(StringRef Input) const{
@@ -108,21 +102,56 @@ void OutputFileMap::dump(llvm::raw_ostream &os, bool Sort) const {
   }
 }
 
-bool OutputFileMap::parse(std::unique_ptr<llvm::MemoryBuffer> Buffer,
+static void writeQuotedEscaped(llvm::raw_ostream &os,
-                          StringRef workingDirectory) {
+                               const StringRef fileName) {
+  os << "\"";
+  os.write_escaped(fileName);
+  os << "\"";
+}
+
+void OutputFileMap::write(llvm::raw_ostream &os,
+                          ArrayRef<StringRef> inputs) const {
+  for (const auto input : inputs) {
+    const TypeToPathMap *outputMap = getOutputMapForInput(input);
+    if (!outputMap)
+      continue;
+    writeQuotedEscaped(os, input);
+    os << ":\n";
+    for (auto &typeAndOutputPath : *outputMap) {
+      types::ID type = typeAndOutputPath.getFirst();
+      StringRef output = typeAndOutputPath.getSecond();
+      os << "  " << types::getTypeName(type) << ": ";
+      writeQuotedEscaped(os, output);
+      os << "\n";
+    }
+  }
+}
+
+llvm::Expected<OutputFileMap>
+OutputFileMap::parse(std::unique_ptr<llvm::MemoryBuffer> Buffer,
+                     StringRef workingDirectory) {
+  auto constructError =
+      [](const char *errorString) -> llvm::Expected<OutputFileMap> {
+    return llvm::make_error<llvm::StringError>(errorString,
+                                               llvm::inconvertibleErrorCode());
+  };
+  /// FIXME: Make the returned error strings more specific by including some of
+  /// the source.
   llvm::SourceMgr SM;
   llvm::yaml::Stream YAMLStream(Buffer->getMemBufferRef(), SM);
   auto I = YAMLStream.begin();
   if (I == YAMLStream.end())
-    return true;
+    return constructError("empty YAML stream");
 
   auto Root = I->getRoot();
   if (!Root)
-    return true;
+    return constructError("no root");
+
+  OutputFileMap OFM;
 
   auto *Map = dyn_cast<llvm::yaml::MappingNode>(Root);
   if (!Map)
-    return true;
+    return constructError("root was not a MappingNode");
 
   auto resolvePath =
       [workingDirectory](
@@ -149,19 +178,19 @@ bool OutputFileMap::parse(std::unique_ptr<llvm::MemoryBuffer> Buffer,
     llvm::yaml::Node *Value = Pair.getValue();
 
     if (!Key)
-      return true;
+      return constructError("bad key");
 
     if (!Value)
-      return true;
+      return constructError("bad value");
 
     auto *InputPath = dyn_cast<llvm::yaml::ScalarNode>(Key);
     if (!InputPath)
-      return true;
+      return constructError("input path not a scalar node");
 
     llvm::yaml::MappingNode *OutputMapNode =
       dyn_cast<llvm::yaml::MappingNode>(Value);
     if (!OutputMapNode)
-      return true;
+      return constructError("output map not a MappingNode");
 
     TypeToPathMap OutputMap;
 
@@ -171,11 +200,11 @@ bool OutputFileMap::parse(std::unique_ptr<llvm::MemoryBuffer> Buffer,
 
       auto *KindNode = dyn_cast<llvm::yaml::ScalarNode>(Key);
       if (!KindNode)
-        return true;
+        return constructError("kind not a ScalarNode");
 
       auto *Path = dyn_cast<llvm::yaml::ScalarNode>(Value);
       if (!Path)
-        return true;
+        return constructError("path not a scalar node");
 
       llvm::SmallString<16> KindStorage;
       types::ID Kind =
@@ -192,9 +221,9 @@ bool OutputFileMap::parse(std::unique_ptr<llvm::MemoryBuffer> Buffer,
     }
 
     llvm::SmallString<128> InputStorage;
-    InputToOutputsMap[resolvePath(InputPath, InputStorage)] =
+    OFM.InputToOutputsMap[resolvePath(InputPath, InputStorage)] =
         std::move(OutputMap);
   }
 
-  return false;
+  return OFM;
 }

lib/Frontend/Types.cpp

@@ -10,15 +10,14 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include "swift/Driver/Types.h"
+#include "swift/Frontend/Types.h"
 
 #include "llvm/ADT/STLExtras.h"
 #include "llvm/ADT/StringSwitch.h"
 #include "llvm/Support/ErrorHandling.h"
 
 using namespace swift;
-using namespace swift::driver;
+using namespace swift::types;
-using namespace swift::driver::types;
 
 struct TypeInfo {
   const char *Name;
@@ -29,7 +28,7 @@ struct TypeInfo {
 static const TypeInfo TypeInfos[] = {
 #define TYPE(NAME, ID, TEMP_SUFFIX, FLAGS) \
   { NAME, FLAGS, TEMP_SUFFIX },
-#include "swift/Driver/Types.def"
+#include "swift/Frontend/Types.def"
 };
 
 static const TypeInfo &getInfo(unsigned Id) {
@@ -52,7 +51,7 @@ ID types::lookupTypeForExtension(StringRef Ext) {
   return llvm::StringSwitch<types::ID>(Ext.drop_front())
 #define TYPE(NAME, ID, SUFFIX, FLAGS) \
            .Case(SUFFIX, TY_##ID)
-#include "swift/Driver/Types.def"
+#include "swift/Frontend/Types.def"
            .Default(TY_INVALID);
 }
 
@@ -60,7 +59,7 @@ ID types::lookupTypeForName(StringRef Name) {
   return llvm::StringSwitch<types::ID>(Name)
 #define TYPE(NAME, ID, SUFFIX, FLAGS) \
            .Case(NAME, TY_##ID)
-#include "swift/Driver/Types.def"
+#include "swift/Frontend/Types.def"
            .Default(TY_INVALID);
 }
 

test/Driver/Inputs/filelists/check-filelist-abc.py

@@ -38,12 +38,28 @@ with open(filelistFile, 'r') as f:
            lines[2].endswith("/c.swiftmodule\n"))
 
 if primaryFile:
-    print("Handled", os.path.basename(primaryFile))
+    print("Command-line primary", os.path.basename(primaryFile))
+
+if '-primary-filelist' in sys.argv:
+    primaryFilelistFile = sys.argv[sys.argv.index('-primary-filelist') + 1]
+    with open(primaryFilelistFile, 'r') as f:
+        lines = f.readlines()
+        assert(len(lines) == 1)
+        print("Handled", os.path.basename(lines[0]).rstrip())
 elif lines[0].endswith(".swiftmodule\n"):
     print("Handled modules")
 else:
     print("Handled all")
 
+
+if '-supplementary-output-file-map' in sys.argv:
+    supplementaryOutputMapFile = \
+        sys.argv[sys.argv.index('-supplementary-output-file-map') + 1]
+    with open(supplementaryOutputMapFile, 'r') as f:
+        lines = f.readlines()
+        for line in lines:
+            print("Supplementary", line.rstrip())
+
 if '-num-threads' in sys.argv:
     outputListFile = sys.argv[sys.argv.index('-output-filelist') + 1]
     with open(outputListFile, 'r') as f:

test/Driver/batch_mode_force_one_batch_repartition.swift

@@ -0,0 +1,12 @@
+// RUN: %empty-directory(%t)
+// RUN: touch %t/file-01.swift %t/file-02.swift %t/file-03.swift
+// RUN: echo 'public func main() {}' >%t/main.swift
+//
+// RUN: %swiftc_driver -enable-batch-mode -c -emit-module -module-name main -j 2 -driver-force-one-batch-repartition %t/file-01.swift %t/file-02.swift %t/file-03.swift %t/main.swift  -###  2>%t/shouldBeEmpty2 | %FileCheck %s  -check-prefix=CHECK-COMBINED
+// RUN: test -z "`cat %t/shouldBeEmpty1`"
+// RUN: test -z "`cat %t/shouldBeEmpty2`"
+//
+// CHECK-COMBINED: -primary-file {{.*}}/file-01.swift
+// CHECK-COMBINED-NEXT: -primary-file {{.*}}/file-02.swift
+// CHECK-COMBINED-NEXT: -primary-file {{.*}}/file-03.swift
+// CHECK-COMBINED-NEXT: -primary-file {{.*}}/main.swift

test/Driver/batch_mode_overlong_argv.swift

@@ -102,7 +102,8 @@
 // RUN: touch  %t/f_99_1.swift %t/f_99_2.swift %t/f_99_3.swift %t/f_99_4.swift %t/f_99_5.swift %t/f_99_6.swift %t/f_99_7.swift %t/f_99_8.swift %t/f_99_9.swift %t/f_99_10.swift
 // RUN: touch  %t/f_100_1.swift %t/f_100_2.swift %t/f_100_3.swift %t/f_100_4.swift %t/f_100_5.swift %t/f_100_6.swift %t/f_100_7.swift %t/f_100_8.swift %t/f_100_9.swift %t/f_100_10.swift
 // RUN: mkdir -p %t/additional/path/elements/often/make/filenames/longer/than/one/might/expect/especially/given/output/directories/deep/within/a/derived/data/folder/of/a/CI/machine/
-// RUN: %swiftc_driver -driver-show-job-lifecycle -v -c -module-name foo -o %t/additional/path/elements/often/make/filenames/longer/than/one/might/expect/especially/given/output/directories/deep/within/a/derived/data/folder/of/a/CI/machine/foo.o -emit-module -serialize-diagnostics -emit-dependencies -j 1 -enable-batch-mode %t/f_*.swift >%t/out.txt 2>&1
+// Force the repartitioning:
+// RUN: %swiftc_driver -driver-show-job-lifecycle -driver-force-one-batch-repartition -v -c -module-name foo -o %t/additional/path/elements/often/make/filenames/longer/than/one/might/expect/especially/given/output/directories/deep/within/a/derived/data/folder/of/a/CI/machine/foo.o -emit-module -serialize-diagnostics -emit-dependencies -j 1 -enable-batch-mode %t/f_*.swift >%t/out.txt 2>&1
 // RUN: %FileCheck %s <%t/out.txt
 // CHECK-NOT: unable to execute command
 // CHECK: Forming into 1 batches
@@ -110,6 +111,17 @@
 // CHECK: Forming into 2 batches
 // CHECK: Forming batch job from 500 constituents
 // CHECK: Forming batch job from 500 constituents
+//
+// Try it without the force; supplementary output file maps should obviate the repartition:
+// RUN: %swiftc_driver -driver-show-job-lifecycle -v -c -module-name foo -o %t/additional/path/elements/often/make/filenames/longer/than/one/might/expect/especially/given/output/directories/deep/within/a/derived/data/folder/of/a/CI/machine/foo.o -emit-module -serialize-diagnostics -emit-dependencies -j 1 -enable-batch-mode %t/f_*.swift >%t/out2.txt 2>&1
+// RUN: %FileCheck %s <%t/out2.txt -check-prefix=NO-REPARTITION
+// CHECK-NOT: unable to execute command
+// NO-REPARTITION: Forming into 1 batches
+// NO-REPARTITION: Forming batch job from 1000 constituents
+// NO-REPARTITION-NOT: Forming into 2 batches
+// NO-REPARTITION-NOT: Forming batch job from 500 constituents
+// NO-REPARTITION-NOT: Forming batch job from 500 constituents
+
 func thing() {
     print(1)
 }

test/Driver/batch_mode_parseable_output.swift

@@ -9,7 +9,7 @@
 // CHECK-NEXT: {
 // CHECK-NEXT:   "kind": "began",
 // CHECK-NEXT:   "name": "compile",
-// CHECK-NEXT:   "command": "{{.*}}/swift{{c?}} -frontend -c -primary-file {{.*}}/file-01.swift -primary-file {{.*}}/file-02.swift {{.*}}/file-03.swift {{.*}}/main.swift {{.*}} -emit-module-doc-path {{.*}}/file-01-[[SWIFTDOC01:[a-z0-9]+]].swiftdoc -emit-module-doc-path {{.*}}/file-02-[[SWIFTDOC02:[a-z0-9]+]].swiftdoc -module-name main -emit-module-path {{.*}}/file-01-[[MODULE01:[a-z0-9]+]].swiftmodule -emit-module-path {{.*}}/file-02-[[MODULE02:[a-z0-9]+]].swiftmodule -o {{.*}}/file-01-[[OBJ01:[a-z0-9]+]].o -o {{.*}}/file-02-[[OBJ02:[a-z0-9]+]].o",
+// CHECK-NEXT:   "command": "{{.*}}/swift{{c?}} -frontend -c -primary-file {{.*}}/file-01.swift -primary-file {{.*}}/file-02.swift {{.*}}/file-03.swift {{.*}}/main.swift -emit-module-path {{.*}}/file-01-[[MODULE01:[a-z0-9]+]].swiftmodule -emit-module-path {{.*}}/file-02-[[MODULE02:[a-z0-9]+]].swiftmodule -emit-module-doc-path {{.*}}/file-01-[[SWIFTDOC01:[a-z0-9]+]].swiftdoc -emit-module-doc-path {{.*}}/file-02-[[SWIFTDOC02:[a-z0-9]+]].swiftdoc {{.*}} -module-name main -o {{.*}}/file-01-[[OBJ01:[a-z0-9]+]].o -o {{.*}}/file-02-[[OBJ02:[a-z0-9]+]].o",
 // CHECK-NEXT:   "command_executable": "{{.*}}/swift{{c?}}",
 // CHECK-NEXT:   "command_arguments": [
 // CHECK-NEXT:     "-frontend",
@@ -20,16 +20,16 @@
 // CHECK-NEXT:     "{{.*}}/file-02.swift",
 // CHECK-NEXT:     "{{.*}}/file-03.swift",
 // CHECK-NEXT:     "{{.*}}/main.swift",
-// CHECK:          "-emit-module-doc-path",
-// CHECK-NEXT:     "{{.*}}/file-01-[[SWIFTDOC01:[a-z0-9]+]].swiftdoc",
-// CHECK-NEXT:     "-emit-module-doc-path",
-// CHECK-NEXT:     "{{.*}}/file-02-[[SWIFTDOC02:[a-z0-9]+]].swiftdoc",
-// CHECK-NEXT:     "-module-name",
-// CHECK-NEXT:     "main",
 // CHECK-NEXT:     "-emit-module-path",
 // CHECK-NEXT:     "{{.*}}/file-01-[[MODULE01:[a-z0-9]+]].swiftmodule",
 // CHECK-NEXT:     "-emit-module-path",
 // CHECK-NEXT:     "{{.*}}/file-02-[[MODULE02:[a-z0-9]+]].swiftmodule",
+// CHECK-NEXT:     "-emit-module-doc-path",
+// CHECK-NEXT:     "{{.*}}/file-01-[[SWIFTDOC01:[a-z0-9]+]].swiftdoc",
+// CHECK-NEXT:     "-emit-module-doc-path",
+// CHECK-NEXT:     "{{.*}}/file-02-[[SWIFTDOC02:[a-z0-9]+]].swiftdoc",
+// CHECK:          "-module-name",
+// CHECK-NEXT:     "main",
 // CHECK-NEXT:     "-o",
 // CHECK-NEXT:     "{{.*}}/file-01-[[OBJ01:[a-z0-9]+]].o",
 // CHECK-NEXT:     "-o",
@@ -71,7 +71,7 @@
 // CHECK-NEXT: {
 // CHECK-NEXT:   "kind": "began",
 // CHECK-NEXT:   "name": "compile",
-// CHECK-NEXT:   "command": "{{.*}}/swift{{c?}} -frontend -c {{.*}}/file-01.swift {{.*}}/file-02.swift -primary-file {{.*}}/file-03.swift -primary-file {{.*}}/main.swift {{.*}} -emit-module-doc-path {{.*}}/file-03-[[SWIFTDOC03:[a-z0-9]+]].swiftdoc -emit-module-doc-path {{.*}}/main-[[SWIFTDOCMAIN:[a-z0-9]+]].swiftdoc -module-name main -emit-module-path {{.*}}/file-03-[[MODULE03:[a-z0-9]+]].swiftmodule -emit-module-path {{.*}}/main-[[MODULEMAIN:[a-z0-9]+]].swiftmodule -o {{.*}}/file-03-[[OBJ03:[a-z0-9]+]].o -o {{.*}}/main-[[OBJMAIN:[a-z0-9]+]].o",
+// CHECK-NEXT:   "command": "{{.*}}/swift{{c?}} -frontend -c {{.*}}/file-01.swift {{.*}}/file-02.swift -primary-file {{.*}}/file-03.swift -primary-file {{.*}}/main.swift -emit-module-path {{.*}}/file-03-[[MODULE03:[a-z0-9]+]].swiftmodule -emit-module-path {{.*}}/main-[[MODULEMAIN:[a-z0-9]+]].swiftmodule -emit-module-doc-path {{.*}}/file-03-[[SWIFTDOC03:[a-z0-9]+]].swiftdoc -emit-module-doc-path {{.*}}/main-[[SWIFTDOCMAIN:[a-z0-9]+]].swiftdoc {{.*}} -module-name main -o {{.*}}/file-03-[[OBJ03:[a-z0-9]+]].o -o {{.*}}/main-[[OBJMAIN:[a-z0-9]+]].o",
 // CHECK-NEXT:   "command_executable": "{{.*}}/swift{{c?}}",
 // CHECK-NEXT:   "command_arguments": [
 // CHECK-NEXT:     "-frontend",
@@ -82,16 +82,16 @@
 // CHECK-NEXT:     "{{.*}}/file-03.swift",
 // CHECK-NEXT:     "-primary-file",
 // CHECK-NEXT:     "{{.*}}/main.swift",
-// CHECK:          "-emit-module-doc-path",
-// CHECK-NEXT:     "{{.*}}/file-03-[[SWIFTDOC03:[a-z0-9]+]].swiftdoc",
-// CHECK-NEXT:     "-emit-module-doc-path",
-// CHECK-NEXT:     "{{.*}}/main-[[SWIFTDOCMAIN:[a-z0-9]+]].swiftdoc",
-// CHECK-NEXT:     "-module-name",
-// CHECK-NEXT:     "main",
 // CHECK-NEXT:     "-emit-module-path",
 // CHECK-NEXT:     "{{.*}}/file-03-[[MODULE03:[a-z0-9]+]].swiftmodule",
 // CHECK-NEXT:     "-emit-module-path",
 // CHECK-NEXT:     "{{.*}}/main-[[MODULEMAIN:[a-z0-9]+]].swiftmodule",
+// CHECK-NEXT:     "-emit-module-doc-path",
+// CHECK-NEXT:     "{{.*}}/file-03-[[SWIFTDOC03:[a-z0-9]+]].swiftdoc",
+// CHECK-NEXT:     "-emit-module-doc-path",
+// CHECK-NEXT:     "{{.*}}/main-[[SWIFTDOCMAIN:[a-z0-9]+]].swiftdoc",
+// CHECK:          "-module-name",
+// CHECK-NEXT:     "main",
 // CHECK-NEXT:     "-o",
 // CHECK-NEXT:     "{{.*}}/file-03-[[OBJ03:[a-z0-9]+]].o",
 // CHECK-NEXT:     "-o",

test/Driver/batch_mode_with_supplementary_filelist.swift

@@ -0,0 +1,9 @@
+// RUN: %empty-directory(%t)
+// RUN: echo 'print("Hello, World!")' >%t/main.swift
+// RUN: touch %t/file-01.swift %t/file-02.swift %t/file-03.swift
+//
+// Ensure that the supplementary output filelist argument is passed to the frontend:
+//
+// RUN: %swiftc_driver -enable-batch-mode -driver-use-filelists -j2 %t/main.swift %t/file-01.swift %t/file-02.swift %t/file-03.swift -o %t/file-01.o  -o %t/file-02.o  -o %t/file-03.o -### | %FileCheck %s -check-prefix=CHECK-SUPPLEMENTARY-OUTPUT-FILELIST
+//
+// CHECK-SUPPLEMENTARY-OUTPUT-FILELIST: -supplementary-output-file-map {{.*}}/supplementaryOutputs-

test/Driver/batch_mode_with_supplementary_filelist_execution.swift

@@ -0,0 +1,12 @@
+// REQUIRES: executable_test
+// RUN: %empty-directory(%t)
+// RUN: echo 'print("Hello, World!")' >%t/main.swift
+// RUN: touch %t/file-01.swift %t/file-02.swift %t/file-03.swift
+//
+// Ensure that the supplementary output filelist argument is passed to the frontend:
+//
+// RUN: %target-build-swift -enable-batch-mode -driver-use-filelists -j2 %t/main.swift  %t/file-01.swift %t/file-02.swift %t/file-03.swift  -o %t/a.out
+// RUN: %target-run %t/a.out | %FileCheck %s -check-prefix=CHECK-HELLO-WORLD
+// CHECK-HELLO-WORLD: Hello, World!
+
+

test/Driver/driver-compile.swift

@@ -103,11 +103,13 @@
 
 // FILELIST: bin/swift
 // FILELIST: -filelist [[SOURCES:(["][^"]+|[^ ]+)sources([^"]+["]|[^ ]+)]]
-// FILELIST: -primary-file {{.*/(driver-compile.swift|empty.swift)}}
+// FILELIST: -primary-filelist  [[PRIMARY_FILELIST:(["][^"]+|[^ ]+)primaryInputs([^"]+["]|[^ ]+)]]
+// FILELIST: -supplementary-output-file-map [[SUPPLEMENTARY_OUTPUT_FILEMAP:(["][^"]+|[^ ]+)supplementaryOutputs([^"]+["]|[^ ]+)]]
 // FILELIST: -output-filelist {{[^-]}}
 // FILELIST-NEXT: bin/swift
 // FILELIST: -filelist [[SOURCES]]
-// FILELIST: -primary-file {{.*/(driver-compile.swift|empty.swift)}}
+// FILELIST: -primary-filelist  {{(["][^"]+|[^ ]+)primaryInputs([^"]+["]|[^ ]+)}}
+// FILELIST: -supplementary-output-file-map {{(["][^"]+|[^ ]+)supplementaryOutputs([^"]+["]|[^ ]+)}}
 // FILELIST: -output-filelist {{[^-]}}
 
 // UPDATE-CODE: DISTINCTIVE-PATH/usr/bin/swift

test/Driver/filelists.swift

@@ -5,15 +5,28 @@
 
 // CHECK-NOT: Handled
 // CHECK: Handled a.swift
+// CHECK-NEXT: Supplementary "./a.swift":
+// CHECK-NEXT: Supplementary swiftdoc: "./a.swiftdoc"
+// CHECK-NEXT: Supplementary swiftmodule: "./a.swiftmodule"
 // CHECK-NEXT: Handled b.swift
+// CHECK-NEXT: Supplementary "./b.swift":
+// CHECK-NEXT: Supplementary swiftdoc: "./b.swiftdoc"
+// CHECK-NEXT: Supplementary swiftmodule: "./b.swiftmodule"
 // CHECK-NEXT: Handled c.swift
+// CHECK-NEXT: Supplementary "./c.swift":
+// CHECK-NEXT: Supplementary swiftdoc: "./c.swiftdoc"
+// CHECK-NEXT: Supplementary swiftmodule: "./c.swiftmodule"
 // CHECK-NEXT: Handled modules
 // CHECK-NOT: Handled
 
+
+
 // RUN: %swiftc_driver_plain -driver-use-frontend-path %S/Inputs/filelists/check-filelist-abc.py -c %t/a.swift %t/b.swift %t/c.swift -module-name main -target x86_64-apple-macosx10.9 -driver-use-filelists -force-single-frontend-invocation 2>&1 | %FileCheck -check-prefix=CHECK-WMO %s
 
 // CHECK-WMO-NOT: Handled
 // CHECK-WMO: Handled all
+// CHECK-WMO: Supplementary "{{.*}}/a.swift":
+// CHECK-WMO: Supplementary object: "main.o"
 // CHECK-WMO-NOT: output
 // CHECK-WMO-NOT: Handled
 
@@ -29,6 +42,12 @@
 
 // CHECK-WMO-THREADED-NOT: Handled
 // CHECK-WMO-THREADED: Handled all
+// CHECK-WMO-THREADED-NEXT: Supplementary "{{.*}}/a.swift":
+// CHECK-WMO-THREADED-NEXT: Supplementary {{object|llvm-bc}}: "{{.*}}/a.{{o|bc}}"
+// CHECK-WMO-THREADED-NEXT: Supplementary "{{.*}}/b.swift":
+// CHECK-WMO-THREADED-NEXT: Supplementary {{object|llvm-bc}}: "{{.*}}/b.{{o|bc}}"
+// CHECK-WMO-THREADED-NEXT: Supplementary "{{.*}}/c.swift":
+// CHECK-WMO-THREADED-NEXT: Supplementary {{object|llvm-bc}}: "{{.*}}/c.{{o|bc}}"
 // CHECK-WMO-THREADED-NEXT: ...with output!
 // CHECK-WMO-THREADED-NOT: Handled
 

test/Driver/merge-module.swift

@@ -53,16 +53,16 @@
 
 
 // TWO-OUTPUTS: bin/swift{{c?}} -frontend
-// TWO-OUTPUTS: -emit-module-doc-path {{[^ ]*}}/merge-module-{{[^ ]*}}.swiftdoc
 // TWO-OUTPUTS: -emit-module-path [[MODULE:[^ ]+]]
+// TWO-OUTPUTS: -emit-module-doc-path {{[^ ]*}}/merge-module-{{[^ ]*}}.swiftdoc
 // TWO-OUTPUTS: -o {{[^ ]*}}/merge-module-{{[^ ]*}}.o
 // TWO-OUTPUTS: bin/swift{{c?}} -frontend
 // TWO-OUTPUTS: -emit-module [[MODULE]]
 // TWO-OUTPUTS: -o main.swiftmodule
 
 // THREE-OUTPUTS: bin/swift{{c?}} -frontend
-// THREE-OUTPUTS: -emit-module-doc-path {{[^ ]*}}/merge-module-{{[^ ]*}}.swiftdoc
 // THREE-OUTPUTS: -emit-module-path [[MODULE:[^ ]+]]
+// THREE-OUTPUTS: -emit-module-doc-path {{[^ ]*}}/merge-module-{{[^ ]*}}.swiftdoc
 // THREE-OUTPUTS: -o {{[^ ]*}}/merge-module-{{[^ ]*}}.o
 // THREE-OUTPUTS: bin/swift{{c?}} -frontend
 // THREE-OUTPUTS: -emit-module [[MODULE]]

test/Index/Store/driver-index.swift

@@ -1,2 +1,2 @@
 // RUN: %swiftc_driver -driver-print-jobs -index-file -index-file-path %s %S/Inputs/SwiftModuleA.swift %s -o %t.output_for_index -index-store-path %t.index_store -module-name driver_index 2>&1 | %FileCheck %s
-// CHECK: {{.*}}swift -frontend -typecheck {{.*}}-primary-file {{.*}}driver-index.swift {{.*}}SwiftModuleA.swift  {{.*}}-o {{.*}}.output_for_index {{.*}}-index-store-path {{.*}}.index_store -index-system-modules
+// CHECK: {{.*}}swift -frontend -typecheck {{.*}}SwiftModuleA.swift -primary-file {{.*}}driver-index.swift {{.*}}-o {{.*}}.output_for_index {{.*}}-index-store-path {{.*}}.index_store -index-system-modules