[XcodeGen] Handle 'rule' declarations and generate command line args

* Rename 'BuildRule to 'BuildEdige' because it is the official term
* NinjaParser to handle 'include' and 'rule' directives
* NinjaParser to handle parse "rule name" in 'build' correctly
* Make variable table a simple `[String: String]` and keep any bindings
  to make the substitutions possible.
* Generate command line argumets using 'command' variable in the 'rule'
  and use it as the source of truth, istead of using random known
  bindings like 'FLAGS'.

utils/swift-xcodegen/Sources/SwiftXcodeGen/BuildArgs/RunnableTargets.swift

@@ -22,7 +22,7 @@ struct RunnableTargets {
   private var targets: [RunnableTarget] = []
 
   init(from buildDir: RepoBuildDir) throws {
-    for rule in try buildDir.ninjaFile.buildRules {
+    for rule in try buildDir.ninjaFile.buildEdges {
       tryAddTarget(rule, buildDir: buildDir)
     }
   }
@@ -59,7 +59,7 @@ extension RunnableTargets {
   }
 
   private mutating func tryAddTarget(
-    _ rule: NinjaBuildFile.BuildRule, buildDir: RepoBuildDir
+    _ rule: NinjaBuildFile.BuildEdge, buildDir: RepoBuildDir
   ) {
     guard let (name, path) = getRunnablePath(for: rule.outputs),
           addedPaths.insert(path).inserted else { return }

utils/swift-xcodegen/Sources/SwiftXcodeGen/BuildArgs/SwiftDriverUtils.swift

@@ -1,126 +0,0 @@
-//===--- SwiftDriverUtils.swift -------------------------------------------===//
-//
-// This source file is part of the Swift.org open source project
-//
-// Copyright (c) 2024 Apple Inc. and the Swift project authors
-// Licensed under Apache License v2.0 with Runtime Library Exception
-//
-// See https://swift.org/LICENSE.txt for license information
-// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors
-//
-//===----------------------------------------------------------------------===//
-
-// https://github.com/swiftlang/swift-driver/blob/661e0bc74bdae4d9f6ea8a7a54015292febb0059/Sources/SwiftDriver/Utilities/StringAdditions.swift
-extension String {
-  /// Whether this string is a Swift identifier.
-  var isValidSwiftIdentifier: Bool {
-    guard let start = unicodeScalars.first else {
-      return false
-    }
-
-    let continuation = unicodeScalars.dropFirst()
-
-    return start.isValidSwiftIdentifierStart &&
-           continuation.allSatisfy { $0.isValidSwiftIdentifierContinuation }
-  }
-}
-
-extension Unicode.Scalar {
-
-  var isValidSwiftIdentifierStart: Bool {
-    guard isValidSwiftIdentifierContinuation else { return false }
-
-    if isASCIIDigit || self == "$" {
-      return false
-    }
-
-    // N1518: Recommendations for extended identifier characters for C and C++
-    // Proposed Annex X.2: Ranges of characters disallowed initially
-    if (0x0300...0x036F).contains(value) ||
-       (0x1DC0...0x1DFF).contains(value) ||
-       (0x20D0...0x20FF).contains(value) ||
-       (0xFE20...0xFE2F).contains(value) {
-      return false
-    }
-
-    return true
-  }
-
-  var isValidSwiftIdentifierContinuation: Bool {
-    if isASCII {
-      return isCIdentifierBody(allowDollar: true)
-    }
-
-    // N1518: Recommendations for extended identifier characters for C and C++
-    // Proposed Annex X.1: Ranges of characters allowed
-    return value == 0x00A8 ||
-           value == 0x00AA ||
-           value == 0x00AD ||
-           value == 0x00AF ||
-
-           (0x00B2...0x00B5).contains(value) ||
-           (0x00B7...0x00BA).contains(value) ||
-           (0x00BC...0x00BE).contains(value) ||
-           (0x00C0...0x00D6).contains(value) ||
-           (0x00D8...0x00F6).contains(value) ||
-           (0x00F8...0x00FF).contains(value) ||
-
-           (0x0100...0x167F).contains(value) ||
-           (0x1681...0x180D).contains(value) ||
-           (0x180F...0x1FFF).contains(value) ||
-
-           (0x200B...0x200D).contains(value) ||
-           (0x202A...0x202E).contains(value) ||
-           (0x203F...0x2040).contains(value) ||
-           value == 0x2054                   ||
-           (0x2060...0x206F).contains(value) ||
-
-           (0x2070...0x218F).contains(value) ||
-           (0x2460...0x24FF).contains(value) ||
-           (0x2776...0x2793).contains(value) ||
-           (0x2C00...0x2DFF).contains(value) ||
-           (0x2E80...0x2FFF).contains(value) ||
-
-           (0x3004...0x3007).contains(value) ||
-           (0x3021...0x302F).contains(value) ||
-           (0x3031...0x303F).contains(value) ||
-
-           (0x3040...0xD7FF).contains(value) ||
-
-           (0xF900...0xFD3D).contains(value) ||
-           (0xFD40...0xFDCF).contains(value) ||
-           (0xFDF0...0xFE44).contains(value) ||
-           (0xFE47...0xFFF8).contains(value) ||
-
-           (0x10000...0x1FFFD).contains(value) ||
-           (0x20000...0x2FFFD).contains(value) ||
-           (0x30000...0x3FFFD).contains(value) ||
-           (0x40000...0x4FFFD).contains(value) ||
-           (0x50000...0x5FFFD).contains(value) ||
-           (0x60000...0x6FFFD).contains(value) ||
-           (0x70000...0x7FFFD).contains(value) ||
-           (0x80000...0x8FFFD).contains(value) ||
-           (0x90000...0x9FFFD).contains(value) ||
-           (0xA0000...0xAFFFD).contains(value) ||
-           (0xB0000...0xBFFFD).contains(value) ||
-           (0xC0000...0xCFFFD).contains(value) ||
-           (0xD0000...0xDFFFD).contains(value) ||
-           (0xE0000...0xEFFFD).contains(value)
-  }
-
-  /// `true` if this character is an ASCII digit: [0-9]
-  var isASCIIDigit: Bool { (0x30...0x39).contains(value) }
-
-  /// `true` if this is a body character of a C identifier,
-  /// which is [a-zA-Z0-9_].
-  func isCIdentifierBody(allowDollar: Bool = false) -> Bool {
-    if (0x41...0x5A).contains(value) ||
-       (0x61...0x7A).contains(value) ||
-       isASCIIDigit                  ||
-       self == "_" {
-      return true
-    } else {
-      return allowDollar && self == "$"
-    }
-  }
-}

utils/swift-xcodegen/Sources/SwiftXcodeGen/BuildArgs/SwiftTargets.swift

@@ -24,7 +24,7 @@ struct SwiftTargets {
 
   init(for buildDir: RepoBuildDir) throws {
     log.debug("[*] Reading Swift targets from build.ninja")
-    for rule in try buildDir.ninjaFile.buildRules {
+    for rule in try buildDir.ninjaFile.buildEdges {
       try tryAddTarget(for: rule, buildDir: buildDir)
     }
     targets.sort(by: { $0.name < $1.name })
@@ -65,33 +65,17 @@ struct SwiftTargets {
   }
 
   private mutating func computeBuildArgs(
-    for rule: NinjaBuildFile.BuildRule
+    for edge: NinjaBuildFile.BuildEdge,
+    in ninja: NinjaBuildFile
   ) throws -> BuildArgs? {
-    var buildArgs = BuildArgs(for: .swiftc)
-    if let commandAttr = rule.attributes[.command] {
-      // We have a custom command, parse it looking for a swiftc invocation.
-      let command = try CommandParser.parseKnownCommandOnly(commandAttr.value)
-      guard let command, command.executable.knownCommand == .swiftc else {
-        return nil
-      }
-      buildArgs += command.args
-    } else if rule.attributes[.flags] != nil {
-      // Ninja separates out other arguments we need, splice them back in.
-      for key: NinjaBuildFile.Attribute.Key in [.flags, .includes, .defines] {
-        guard let attr = rule.attributes[key] else { continue }
-        buildArgs.append(attr.value)
-      }
-      // Add a module name argument if one is specified, validating to
-      // ensure it's correct since we currently have some targets with
-      // invalid module names, e.g swift-plugin-server.
-      if let moduleName = rule.attributes[.swiftModuleName]?.value,
-         moduleName.isValidSwiftIdentifier {
-        buildArgs.append("-module-name \(moduleName)")
-      }
-    } else {
+    let commandLine = try ninja.commandLine(for: edge)
+    let command = try CommandParser.parseKnownCommandOnly(commandLine)
+    guard let command, command.executable.knownCommand == .swiftc else {
       return nil
     }
 
+    var buildArgs = BuildArgs(for: .swiftc, args: command.args)
+
     // Only include known flags for now.
     buildArgs = buildArgs.filter { arg in
       if arg.flag != nil {
@@ -125,17 +109,9 @@ struct SwiftTargets {
   }
 
   func getSources(
-    from rule: NinjaBuildFile.BuildRule, buildDir: RepoBuildDir
+    from edge: NinjaBuildFile.BuildEdge, buildDir: RepoBuildDir
   ) throws -> SwiftTarget.Sources {
-    // If we have SWIFT_SOURCES defined, use it, otherwise check the rule
-    // inputs.
-    let files: [AnyPath]
-    if let sourcesStr = rule.attributes[.swiftSources]?.value {
-      files = try CommandParser.parseArguments(sourcesStr, for: .swiftc)
-        .compactMap(\.value).map(AnyPath.init)
-    } else {
-      files = rule.inputs.map(AnyPath.init)
-    }
+    let files: [AnyPath] = edge.inputs.map(AnyPath.init)
 
     // Split the files into repo sources and external sources. Repo sources
     // are those under the repo path, external sources are outside that path,
@@ -166,29 +142,29 @@ struct SwiftTargets {
   }
 
   private mutating func tryAddTarget(
-    for rule: NinjaBuildFile.BuildRule,
+    for edge: NinjaBuildFile.BuildEdge,
     buildDir: RepoBuildDir
   ) throws {
     // Phonies are only used to track aliases.
-    if rule.isPhony {
-      for output in rule.outputs {
-        outputAliases[output, default: []] += rule.inputs
+    if edge.isPhony {
+      for output in edge.outputs {
+        outputAliases[output, default: []] += edge.inputs
       }
       return
     }
 
     // Ignore build rules that don't have object file or swiftmodule outputs.
-    let forBuild = rule.outputs.contains(
+    let forBuild = edge.outputs.contains(
       where: { $0.hasExtension(.o) }
     )
-    let forModule = rule.outputs.contains(
+    let forModule = edge.outputs.contains(
       where: { $0.hasExtension(.swiftmodule) }
     )
     guard forBuild || forModule else {
       return
     }
-    let primaryOutput = rule.outputs.first!
-    let sources = try getSources(from: rule, buildDir: buildDir)
+    let primaryOutput = edge.outputs.first!
+    let sources = try getSources(from: edge, buildDir: buildDir)
     let repoSources = sources.repoSources
     let externalSources = sources.externalSources
 
@@ -198,32 +174,30 @@ struct SwiftTargets {
       return
     }
 
-    guard let buildArgs = try computeBuildArgs(for: rule) else { return }
+    guard let buildArgs = try computeBuildArgs(for: edge, in: buildDir.ninjaFile) else { return }
 
     // Pick up the module name from the arguments, or use an explicitly
     // specified module name if we have one. The latter might be invalid so
     // may not be part of the build args (e.g 'swift-plugin-server'), but is
     // fine for generation.
-    let moduleName = buildArgs.lastValue(for: .moduleName) ??
-                       rule.attributes[.swiftModuleName]?.value
+    let moduleName = buildArgs.lastValue(for: .moduleName) ?? edge.bindings[.swiftModuleName]
     guard let moduleName else {
       log.debug("! Skipping Swift target with output \(primaryOutput); no module name")
       return
     }
-    let moduleLinkName = rule.attributes[.swiftLibraryName]?.value ??
-                           buildArgs.lastValue(for: .moduleLinkName)
+    let moduleLinkName = buildArgs.lastValue(for: .moduleLinkName) ?? edge.bindings[.swiftLibraryName]
     let name = moduleLinkName ?? moduleName
 
     // Add the dependencies. We track dependencies for any input files, along
     // with any recorded swiftmodule dependencies.
     dependenciesByTargetName.withValue(for: name, default: []) { deps in
       deps.formUnion(
-        rule.inputs.filter {
+        edge.inputs.filter {
           $0.hasExtension(.swiftmodule) || $0.hasExtension(.o)
         }
       )
       deps.formUnion(
-        rule.dependencies.filter { $0.hasExtension(.swiftmodule) }
+        edge.dependencies.filter { $0.hasExtension(.swiftmodule) }
       )
     }
 
@@ -258,7 +232,7 @@ struct SwiftTargets {
       targets.append(target)
       return target
     }()
-    for output in rule.outputs {
+    for output in edge.outputs {
       targetsByOutput[output] = target
     }
     if buildRule == nil || target.buildRule == nil {

utils/swift-xcodegen/Sources/SwiftXcodeGen/Ninja/NinjaBuildFile.swift

@@ -11,88 +11,179 @@
 //===----------------------------------------------------------------------===//
 
 struct NinjaBuildFile {
-  var attributes: [Attribute.Key: Attribute]
-  var buildRules: [BuildRule] = []
+  var bindings: Bindings
+  var rules: [String: Rule]
+  var buildEdges: [BuildEdge] = []
 
   init(
-    attributes: [Attribute.Key: Attribute],
-    buildRules: [BuildRule]
+    bindings: [String: String],
+    rules: [String: Rule],
+    buildEdges: [BuildEdge]
   ) {
-    self.attributes = attributes
-    self.buildRules = buildRules
+    self.bindings = Bindings(storage: bindings)
+    self.buildEdges = buildEdges
+    self.rules = rules
   }
 }
 
 extension NinjaBuildFile {
   var buildConfiguration: BuildConfiguration? {
-    attributes[.configuration]
-      .flatMap { BuildConfiguration(rawValue: $0.value) }
+    bindings[.configuration]
+      .flatMap { BuildConfiguration(rawValue: $0) }
   }
 }
 
 extension NinjaBuildFile {
-  struct BuildRule: Hashable {
+
+  struct Bindings: Hashable {
+    let values: [String: String]
+
+    init(storage: [String : String]) {
+      self.values = storage
+    }
+
+    subscript(key: String) -> String? {
+      values[key]
+    }
+  }
+
+  struct Rule: Equatable {
+    let name: String
+    var bindings: Bindings
+
+    init(name: String, bindings: [String: String]) {
+      self.name = name
+      self.bindings = Bindings(storage: bindings)
+    }
+  }
+
+  struct BuildEdge: Hashable {
+    let ruleName: String
     let inputs: [String]
     let outputs: [String]
     let dependencies: [String]
+    var bindings: Bindings
 
-    let attributes: [Attribute.Key: Attribute]
-    private(set) var isPhony = false
+    var isPhony: Bool {
+      ruleName == "phony"
+    }
 
     init(
+      ruleName: String,
       inputs: [String], outputs: [String], dependencies: [String],
-      attributes: [Attribute.Key : Attribute]
+      bindings: [String: String]
     ) {
+      self.ruleName = ruleName
       self.inputs = inputs
       self.outputs = outputs
       self.dependencies = dependencies
-      self.attributes = attributes
+      self.bindings = Bindings(storage: bindings)
     }
 
     static func phony(for outputs: [String], inputs: [String]) -> Self {
-      var rule = Self(
-        inputs: inputs, outputs: outputs, dependencies: [], attributes: [:]
+      return Self(
+        ruleName: "phony", inputs: inputs, outputs: outputs, dependencies: [], bindings: [:]
       )
-      rule.isPhony = true
-      return rule
     }
   }
 }
 
+
+fileprivate enum NinjaCommandLineError: Error {
+  case unknownRule(String)
+  case missingCommandBinding
+}
+
 extension NinjaBuildFile {
-  struct Attribute: Hashable {
-    var key: Key
-    var value: String
+
+  func commandLine(for edge: BuildEdge) throws -> String {
+    guard let rule = self.rules[edge.ruleName] else {
+      throw NinjaCommandLineError.unknownRule(edge.ruleName)
+    }
+
+    // Helper to get a substitution value for ${key}.
+    // Note that we don't do built-in substitutions (e.g. $in, $out) for now.
+    func value(for key: String) -> String? {
+      edge.bindings[key] ?? rule.bindings[key] ?? self.bindings[key]
+    }
+
+    func eval(string: String) -> String {
+      var result = ""
+      string.scanningUTF8 { scanner in
+        while scanner.hasInput {
+          if let prefix = scanner.eat(while: { $0 != "$" }) {
+            result += String(utf8: prefix)
+          }
+          guard scanner.tryEat("$") else {
+            // Reached the end.
+            break
+          }
+
+          let substituted: String? = scanner.tryEating { scanner in
+            // Parse the variable name.
+            let key: String
+            if scanner.tryEat("{"), let keyName = scanner.eat(while: { $0 != "}" }), scanner.tryEat("}") {
+              key = String(utf8: keyName)
+            } else if let keyName = scanner.eat(while: { $0.isNinjaVarName }) {
+              key = String(utf8: keyName)
+            } else {
+              return nil
+            }
+
+            return value(for: key)
+          }
+
+          if let substituted {
+            // Recursive substitutions.
+            result += eval(string: substituted)
+          } else {
+            // Was not a variable, restore '$' and move on.
+            result += "$"
+          }
+        }
+      }
+      return result
+    }
+
+    guard let commandLine = rule.bindings["command"] else {
+      throw NinjaCommandLineError.missingCommandBinding
+    }
+    return eval(string: commandLine)
+  }
+}
+
+extension Byte {
+  fileprivate var isNinjaVarName: Bool {
+    switch self.scalar {
+    case "0"..."9", "a"..."z", "A"..."Z", "_", "-":
+      return true
+    default:
+      return false
+    }
   }
 }
 
 extension NinjaBuildFile: CustomDebugStringConvertible {
   var debugDescription: String {
-    buildRules.map(\.debugDescription).joined(separator: "\n")
+    buildEdges.map(\.debugDescription).joined(separator: "\n")
   }
 }
 
-extension NinjaBuildFile.BuildRule: CustomDebugStringConvertible {
+extension NinjaBuildFile.BuildEdge: CustomDebugStringConvertible {
   var debugDescription: String {
     """
     {
       inputs: \(inputs)
       outputs: \(outputs)
       dependencies: \(dependencies)
-      attributes: \(attributes)
+      bindings: \(bindings)
       isPhony: \(isPhony)
     }
     """
   }
 }
 
-extension NinjaBuildFile.Attribute: CustomStringConvertible {
-  var description: String {
-    "\(key.rawValue) = \(value)"
-  }
-}
-
-extension NinjaBuildFile.Attribute {
+extension NinjaBuildFile.Bindings {
   enum Key: String {
     case configuration = "CONFIGURATION"
     case defines = "DEFINES"
@@ -102,6 +193,9 @@ extension NinjaBuildFile.Attribute {
     case swiftModuleName = "SWIFT_MODULE_NAME"
     case swiftLibraryName = "SWIFT_LIBRARY_NAME"
     case swiftSources = "SWIFT_SOURCES"
-    case command = "COMMAND"
+  }
+
+  subscript(key: Key) -> String? {
+    return self[key.rawValue]
   }
 }

utils/swift-xcodegen/Sources/SwiftXcodeGen/Ninja/NinjaParser.swift

@@ -13,22 +13,26 @@
 import Foundation
 
 struct NinjaParser {
+  private let filePath: AbsolutePath
+  private let fileReader: (AbsolutePath) throws -> Data
   private var lexer: Lexer
 
-  private init(_ input: UnsafeRawBufferPointer) throws {
+  private init(input: UnsafeRawBufferPointer, filePath: AbsolutePath, fileReader: @escaping (AbsolutePath) throws -> Data) throws {
+    self.filePath = filePath
+    self.fileReader = fileReader
     self.lexer = Lexer(ByteScanner(input))
   }
 
-  static func parse(_ input: Data) throws -> NinjaBuildFile {
-    try input.withUnsafeBytes { bytes in
-      var parser = try Self(bytes)
+  static func parse(filePath: AbsolutePath, fileReader: @escaping (AbsolutePath) throws -> Data = { try $0.read() }) throws -> NinjaBuildFile {
+
+    try fileReader(filePath).withUnsafeBytes { bytes in
+      var parser = try Self(input: bytes, filePath: filePath, fileReader: fileReader)
       return try parser.parse()
     }
   }
 }
 
 fileprivate enum NinjaParseError: Error {
-  case badAttribute
   case expected(NinjaParser.Lexeme)
 }
 
@@ -66,18 +70,20 @@ fileprivate extension ByteScanner {
 }
 
 fileprivate extension NinjaParser {
-  typealias BuildRule = NinjaBuildFile.BuildRule
-  typealias Attribute = NinjaBuildFile.Attribute
+  typealias Rule = NinjaBuildFile.Rule
+  typealias BuildEdge = NinjaBuildFile.BuildEdge
 
-  struct ParsedAttribute: Hashable {
+  struct ParsedBinding: Hashable {
     var key: String
     var value: String
   }
 
   enum Lexeme: Hashable {
-    case attribute(ParsedAttribute)
+    case binding(ParsedBinding)
     case element(String)
+    case rule
     case build
+    case include
     case newline
     case colon
     case equal
@@ -132,14 +138,6 @@ fileprivate extension Byte {
   }
 }
 
-fileprivate extension NinjaBuildFile.Attribute {
-  init?(_ parsed: NinjaParser.ParsedAttribute) {
-    // Ignore unknown attributes for now.
-    guard let key = Key(rawValue: parsed.key) else { return nil }
-    self.init(key: key, value: parsed.value)
-  }
-}
-
 extension NinjaParser.Lexer {
   typealias Lexeme = NinjaParser.Lexeme
 
@@ -170,7 +168,7 @@ extension NinjaParser.Lexer {
     })
   }
 
-  private mutating func tryConsumeAttribute(key: String) -> Lexeme? {
+  private mutating func tryConsumeBinding(key: String) -> Lexeme? {
     input.tryEating { input in
       input.skip(while: \.isSpaceOrTab)
       guard input.tryEat("=") else { return nil }
@@ -178,7 +176,7 @@ extension NinjaParser.Lexer {
       guard let value = input.consumeUnescaped(while: { !$0.isNewline }) else {
         return nil
       }
-      return .attribute(.init(key: key, value: value))
+      return .binding(.init(key: key, value: value))
     }
   }
 
@@ -204,15 +202,24 @@ extension NinjaParser.Lexer {
       if c.isNinjaOperator {
         return consumeOperator()
       }
-      if isAtStartOfLine && input.tryEat(utf8: "build") {
-        return .build
+      if isAtStartOfLine {
+        // decl keywords.
+        if input.tryEat(utf8: "build") {
+          return .build
+        }
+        if input.tryEat(utf8: "rule") {
+          return .rule
+        }
+        if input.tryEat(utf8: "include") {
+          return .include
+        }
       }
       guard let element = consumeElement() else { return nil }
 
-      // If we're on a newline, check to see if we can lex an attribute.
+      // If we're on a newline, check to see if we can lex a binding.
       if isAtStartOfLine {
-        if let attr = tryConsumeAttribute(key: element) {
-          return attr
+        if let binding = tryConsumeBinding(key: element) {
+          return binding
         }
       }
       return .element(element)
@@ -233,14 +240,42 @@ fileprivate extension NinjaParser {
     while let lexeme = eat(), lexeme != .newline {}
   }
 
-  mutating func parseAttribute() throws -> ParsedAttribute? {
-    guard case let .attribute(attr) = peek else { return nil }
+  mutating func parseBinding() throws -> ParsedBinding? {
+    guard case let .binding(binding) = peek else { return nil }
     eat()
     tryEat(.newline)
-    return attr
+    return binding
   }
 
-  mutating func parseBuildRule() throws -> BuildRule? {
+  /// ```
+  /// rule rulename
+  ///   command = ...
+  ///   var = ...
+  /// ```
+  mutating func parseRule() throws -> Rule? {
+    let indent = lexer.leadingTriviaCount
+    guard tryEat(.rule) else { return nil }
+
+    guard let ruleName = tryEatElement() else {
+      throw NinjaParseError.expected(.element("<rule name>"))
+    }
+    guard tryEat(.newline) else {
+      throw NinjaParseError.expected(.newline)
+    }
+
+    var bindings: [String: String] = [:]
+    while indent < lexer.leadingTriviaCount, let binding = try parseBinding() {
+      bindings[binding.key] = binding.value
+    }
+
+    return Rule(name: ruleName, bindings: bindings)
+  }
+
+  /// ```
+  /// build out1... | implicit-out... : rulename input... | dep... || order-only-dep...
+  ///   var = ...
+  /// ```
+  mutating func parseBuildEdge() throws -> BuildEdge? {
     let indent = lexer.leadingTriviaCount
     guard tryEat(.build) else { return nil }
 
@@ -258,17 +293,16 @@ fileprivate extension NinjaParser {
       throw NinjaParseError.expected(.colon)
     }
 
-    var isPhony = false
-    var inputs: [String] = []
-    while let str = tryEatElement() {
-      if str == "phony" {
-        isPhony = true
-      } else {
-        inputs.append(str)
-      }
+    guard let ruleName = tryEatElement() else {
+      throw NinjaParseError.expected(.element("<rule name>"))
     }
 
-    if isPhony {
+    var inputs: [String] = []
+    while let str = tryEatElement() {
+      inputs.append(str)
+    }
+
+    if ruleName == "phony" {
       skipLine()
       return .phony(for: outputs, inputs: inputs)
     }
@@ -280,7 +314,7 @@ fileprivate extension NinjaParser {
         continue
       }
       if tryEat(.pipe) || tryEat(.doublePipe) {
-        // Currently we don't distinguish between implicit and explicit deps.
+        // Currently we don't distinguish between implicit deps and order-only deps.
         continue
       }
       break
@@ -289,38 +323,61 @@ fileprivate extension NinjaParser {
     // We're done with the line, skip to the next.
     skipLine()
 
-    var attributes: [Attribute.Key: Attribute] = [:]
-    while indent < lexer.leadingTriviaCount, let attr = try parseAttribute() {
-      if let attr = Attribute(attr) {
-        attributes[attr.key] = attr
-      }
+    var bindings: [String: String] = [:]
+    while indent < lexer.leadingTriviaCount, let binding = try parseBinding() {
+      bindings[binding.key] = binding.value
     }
 
-    return BuildRule(
-      inputs: inputs, 
+    return BuildEdge(
+      ruleName: ruleName,
+      inputs: inputs,
       outputs: outputs,
       dependencies: dependencies,
-      attributes: attributes
+      bindings: bindings
     )
   }
 
+  /// ```
+  /// include path/to/sub.ninja
+  /// ```
+  mutating func parseInclude() throws -> NinjaBuildFile? {
+    guard tryEat(.include) else { return nil }
+
+    guard let fileName = tryEatElement() else {
+      throw NinjaParseError.expected(.element("<path>"))
+    }
+
+    let baseDirectory = self.filePath.parentDir!
+    let path = AnyPath(fileName).absolute(in: baseDirectory)
+    return try NinjaParser.parse(filePath: path, fileReader: fileReader)
+  }
+
   mutating func parse() throws -> NinjaBuildFile {
-    var buildRules: [BuildRule] = []
-    var attributes: [Attribute.Key: Attribute] = [:]
+    var bindings: [String: String] = [:]
+    var rules: [String: Rule] = [:]
+    var buildEdges: [BuildEdge] = []
     while peek != nil {
-      if let rule = try parseBuildRule() {
-        buildRules.append(rule)
+      if let rule = try parseRule() {
+        rules[rule.name] = rule
         continue
       }
-      if let attr = try parseAttribute() {
-        if let attr = Attribute(attr) {
-          attributes[attr.key] = attr
-        }
+      if let edge = try parseBuildEdge() {
+        buildEdges.append(edge)
         continue
       }
-      // Ignore unknown bits of syntax like 'include' for now.
+      if let binding = try parseBinding() {
+        bindings[binding.key] = binding.value
+        continue
+      }
+      if let included = try parseInclude() {
+        bindings.merge(included.bindings.values, uniquingKeysWith: { _, other in other })
+        rules.merge(included.rules, uniquingKeysWith: { _, other in other })
+        buildEdges.append(contentsOf: included.buildEdges)
+        continue
+      }
+      // Ignore unknown bits of syntax like 'subninja' for now.
       eat()
     }
-    return NinjaBuildFile(attributes: attributes, buildRules: buildRules)
+    return NinjaBuildFile(bindings: bindings, rules: rules, buildEdges: buildEdges)
   }
 }

utils/swift-xcodegen/Sources/SwiftXcodeGen/Ninja/RepoBuildDir.swift

@@ -82,7 +82,7 @@ extension RepoBuildDir {
         }
 
         log.debug("[*] Reading '\(fileName)'")
-        let ninjaFile = try NinjaParser.parse(fileName.read())
+        let ninjaFile = try NinjaParser.parse(filePath: fileName)
         _ninjaFile = ninjaFile
         return ninjaFile
       }

utils/swift-xcodegen/Sources/SwiftXcodeGen/Path/AnyPath.swift

@@ -52,6 +52,15 @@ extension AnyPath {
       a
     }
   }
+
+  public func absolute(in base: AbsolutePath) -> AbsolutePath {
+    switch self {
+    case .relative(let r):
+      r.absolute(in: base)
+    case .absolute(let a):
+      a
+    }
+  }
 }
 
 extension AnyPath: Decodable {

utils/swift-xcodegen/Sources/SwiftXcodeGen/Path/RelativePath.swift

@@ -37,6 +37,11 @@ public extension RelativePath {
     .init(FileManager.default.currentDirectoryPath).appending(self)
   }
 
+  func absolute(in base: AbsolutePath) -> AbsolutePath {
+    precondition(base.isDirectory, "Expected '\(base)' to be a directory")
+    return base.appending(self)
+  }
+
   init(_ component: Component) {
     self.init(FilePath(root: nil, components: component))
   }

utils/swift-xcodegen/Tests/SwiftXcodeGenTest/NinjaParserTests.swift

@@ -53,30 +53,50 @@ fileprivate func expectEqual<T, U: Equatable>(
 
 fileprivate func assertParse(
   _ str: String,
-  attributes: [NinjaBuildFile.Attribute] = [],
-  rules: [NinjaBuildFile.BuildRule],
+  bindings: [String: String] = [:],
+  rules: [String: NinjaBuildFile.Rule] = [:],
+  edges: [NinjaBuildFile.BuildEdge],
+  file: StaticString = #file, line: UInt = #line
+) {
+  let filePath: AbsolutePath = "/tmp/build.ninja"
+  let files: [AbsolutePath: String] = [
+    filePath: str
+  ]
+  assertParse(filePath, in: files, bindings: bindings, rules: rules, edges: edges, file: file, line: line)
+}
+
+fileprivate func assertParse(
+  _ filePath: AbsolutePath,
+  in fileSystem: [AbsolutePath: String],
+  bindings: [String: String] = [:],
+  rules: [String: NinjaBuildFile.Rule] = [:],
+  edges: [NinjaBuildFile.BuildEdge],
   file: StaticString = #file, line: UInt = #line
 ) {
   do {
-    let buildFile = try NinjaParser.parse(Data(str.utf8))
-    guard rules.count == buildFile.buildRules.count else {
+    let buildFile = try NinjaParser.parse(filePath: filePath, fileReader: { Data(fileSystem[$0]!.utf8) })
+    guard edges.count == buildFile.buildEdges.count else {
       XCTFail(
-        "Expected \(rules.count) rules, got \(buildFile.buildRules.count)",
+        "Expected \(edges.count) edges, got \(buildFile.buildEdges.count)",
         file: file, line: line
       )
       return
     }
     XCTAssertEqual(
-      Dictionary(uniqueKeysWithValues: attributes.map { ($0.key, $0) }), 
-      buildFile.attributes,
+      bindings,
+      buildFile.bindings.values,
       file: file, line: line
     )
-    for (expected, actual) in zip(rules, buildFile.buildRules) {
+    XCTAssertEqual(
+      rules, buildFile.rules,
+      file: file, line: line
+    )
+    for (expected, actual) in zip(edges, buildFile.buildEdges) {
+      expectEqual(expected, actual, \.ruleName, file: file, line: line)
       expectEqual(expected, actual, \.inputs, file: file, line: line)
       expectEqual(expected, actual, \.outputs, file: file, line: line)
       expectEqual(expected, actual,  \.dependencies, file: file, line: line)
-      expectEqual(expected, actual,  \.attributes, file: file, line: line)
-      expectEqual(expected, actual,  \.isPhony, file: file, line: line)
+      expectEqual(expected, actual,  \.bindings, file: file, line: line)
 
       XCTAssertEqual(expected, actual, file: file, line: line)
     }
@@ -86,27 +106,86 @@ fileprivate func assertParse(
 }
 
 class NinjaParserTests: XCTestCase {
-  func testBuildRule() throws {
-    assertParse("""
+  func testBuildEdge() throws {
+    assertParse(
+      """
       # ignore comment, build foo.o: a.swift | dep || orderdep
       #another build comment
-      build foo.o foo.swiftmodule: a.swift | dep || orderdep
+      build foo.o foo.swiftmodule: SWIFTC a.swift | dep || orderdep
       notpartofthebuildrule
-      """, rules: [
+      """,
+      edges: [
         .init(
+          ruleName: "SWIFTC",
           inputs: ["a.swift"],
           outputs: ["foo.o", "foo.swiftmodule"],
           dependencies: ["dep", "orderdep"],
-          attributes: [:]
+          bindings: [:]
+        )
+      ]
+    )
+  }
+
+  func testRule() throws {
+    assertParse(
+      """
+      rule SWIFTC
+        command = /bin/switfc -wmo -target unknown
+        other = whatever
+      notpartoftherule
+      """,
+      rules: [
+        "SWIFTC": .init(
+          name: "SWIFTC",
+          bindings: [
+            "command": "/bin/switfc -wmo -target unknown",
+            "other": "whatever",
+          ])
+      ],
+      edges: []
+    )
+  }
+
+  func testInclude() throws {
+    let files: [AbsolutePath: String] = [
+      "/tmp/build.ninja": """
+        include path/to/sub.ninja
+        
+        build foo.swiftmodule : SWIFTC foo.swift
+        """,
+      "/tmp/path/to/sub.ninja": """
+        rule SWIFTC
+          command = /bin/swiftc $in -o $out
+        """
+    ]
+    assertParse(
+      "/tmp/build.ninja",
+      in: files,
+      rules: [
+        "SWIFTC": .init(
+          name: "SWIFTC",
+          bindings: [
+            "command": "/bin/swiftc $in -o $out",
+          ])
+      ],
+      edges: [
+        .init(
+          ruleName: "SWIFTC",
+          inputs: ["foo.swift"],
+          outputs: ["foo.swiftmodule"],
+          dependencies: [],
+          bindings: [:]
         )
       ]
     )
   }
 
   func testPhonyRule() throws {
-    assertParse("""
+    assertParse(
+      """
       build foo.swiftmodule : phony bar.swiftmodule
-      """, rules: [
+      """,
+      edges: [
         .phony(
           for: ["foo.swiftmodule"],
           inputs: ["bar.swiftmodule"]
@@ -115,13 +194,14 @@ class NinjaParserTests: XCTestCase {
     )
   }
 
-  func testAttributes() throws {
-    assertParse("""
+  func testBindings() throws {
+    assertParse(
+      """
       x = y
 
       CONFIGURATION = Debug
 
-      build foo.o: xyz foo.swift | baz.o
+      build foo.o: SWIFTC xyz foo.swift | baz.o
         UNKNOWN = foobar
         SWIFT_MODULE_NAME = foobar
 
@@ -137,28 +217,35 @@ class NinjaParserTests: XCTestCase {
         COMMAND = /bin/swiftc -I /a/b -wmo
       FLAGS = -I /c/d -wmo
 
-      """, attributes: [
-        .init(key: .configuration, value: "Debug"),
+      """,
+      bindings: [
+        "x": "y",
+
+        "CONFIGURATION": "Debug",
 
         // This is considered top-level since it's not indented.
-        .init(key: .flags, value: "-I /c/d -wmo")
+        "FLAGS": "-I /c/d -wmo"
       ],
-      rules: [
+      edges: [
         .init(
+          ruleName: "SWIFTC",
           inputs: ["xyz", "foo.swift"],
           outputs: ["foo.o"],
           dependencies: ["baz.o"],
-          attributes: [
-            .swiftModuleName: .init(key: .swiftModuleName, value: "foobar"),
-            .flags: .init(key: .flags, value: "-I /a/b -wmo"),
+          bindings: [
+            "UNKNOWN": "foobar",
+            "SWIFT_MODULE_NAME": "foobar",
+            "FLAGS": "-I /a/b -wmo",
+            "ANOTHER_UNKNOWN": "a b c",
           ]
         ),
         .init(
-          inputs: ["CUSTOM_COMMAND", "baz.swift"],
+          ruleName: "CUSTOM_COMMAND",
+          inputs: ["baz.swift"],
           outputs: ["baz.o"],
           dependencies: [],
-          attributes: [
-            .command: .init(key: .command, value: "/bin/swiftc -I /a/b -wmo"),
+          bindings: [
+            "COMMAND": "/bin/swiftc -I /a/b -wmo",
           ]
         )
       ]
@@ -167,19 +254,22 @@ class NinjaParserTests: XCTestCase {
 
   func testEscape() throws {
     for newline in ["\n", "\r", "\r\n"] {
-      assertParse("""
-        build foo.o$:: xyz$ foo$$.swift | baz$ bar.o
+      assertParse(
+        """
+        build foo.o$:: SWIFTC xyz$ foo$$.swift | baz$ bar.o
           FLAGS = -I /a$\(newline)\
                   /b -wmo
           COMMAND = swiftc$$
-        """, rules: [
+        """,
+        edges: [
           .init(
+            ruleName: "SWIFTC",
             inputs: ["xyz foo$.swift"],
             outputs: ["foo.o:"],
             dependencies: ["baz bar.o"],
-            attributes: [
-              .flags: .init(key: .flags, value: "-I /a/b -wmo"),
-              .command: .init(key: .command, value: "swiftc$")
+            bindings: [
+              "FLAGS": "-I /a/b -wmo",
+              "COMMAND": "swiftc$",
             ]
           )
         ]