[Tools] Add a tool to detect source-breaking API changes introduced from libraries. (#5236)

[Tools] Add a tool to detect source-breaking API changes introduced from libraries.

swift-api-digester is a test utility to detect source-breaking API changes
during the evolution of a swift library. The tool works on two phases:
(1) dumping library contents as a json file, and (2) comparing two json
files textually to report interesting changes.

During phase (1), the api-digester looks up every declarations inside
a module and outputs a singly-rooted tree that encloses interesting
details of the API level.

During phase (2), api-digester applies structure-information comparision
algorithms on two given singly root trees, trying to figure out, as
precise as possible, the branches/leaves in the trees that differ from
each other. Further analysis decides whether the changed leaves/branches
can be reflected as source-breaking changes for API users. If they are,
the output of api-digester will include such changes.

Also, this commit includes a regression test that make sure API changes
from the Swift stdlib are expected.

test/api-digester/lit.local.cfg

@@ -0,0 +1,6 @@
+if 'OS=macosx' not in config.available_features:
+    config.unsupported = True
+
+else:
+    config.swift_api_digester = config.inferSwiftBinary('swift-api-digester')
+    config.substitutions.append(('%api-digester', config.swift_api_digester))

test/api-digester/source-stability.swift

@@ -0,0 +1,8 @@
+// REQUIRES: OS=macosx
+// RUN: rm -rf %S/tmp && mkdir %S/tmp && mkdir %S/tmp/module-cache && mkdir %S/tmp/dummy.sdk
+// RUN: %api-digester -dump-sdk -module Swift -o %S/tmp/current-stdlib.json -module-cache-path %S/tmp/module-cache -sdk %S/tmp/dummy.sdk -swift-version 3.0
+// RUN: %api-digester -diagnose-sdk -input-paths %S/stdlib-stable.json -input-paths %S/tmp/current-stdlib.json >> %S/tmp/changes.txt
+// RUN: %clang -E -P -x c %S/source-stability.swift.expected -o - | sed '/^\s*$/d' > %S/tmp/source-stability.swift.expected
+// RUN: %clang -E -P -x c %S/tmp/changes.txt -o - | sed '/^\s*$/d' > %S/tmp/changes.txt.tmp
+// RUN: diff -u %S/tmp/source-stability.swift.expected %S/tmp/changes.txt.tmp
+// RUN: rm -rf %S/tmp

test/api-digester/source-stability.swift.expected

@@ -0,0 +1,20 @@
+
+==================================================== Removed Decls ====================================================
+Protocol BidirectionalIndexable has been removed (deprecated)
+Protocol ExpressibleByStringInterpolation has been removed (deprecated)
+Protocol Indexable has been removed (deprecated)
+Protocol IndexableBase has been removed (deprecated)
+Protocol MutableIndexable has been removed (deprecated)
+Protocol RandomAccessIndexable has been removed (deprecated)
+Protocol RangeReplaceableIndexable has been removed (deprecated)
+
+==================================================== Moved Decls ====================================================
+// rdar://28456614 Swift 3 breaking change: static function 'abs' removed from Double/Float/Float80/CGFloat
+Func Double.abs(_:) has been moved to Func abs(_:)
+Func Float.abs(_:) has been moved to Func abs(_:)
+Func Float80.abs(_:) has been moved to Func abs(_:)
+
+==================================================== Renamed Decls ====================================================
+
+==================================================== Type Changes ====================================================
+Func UnsafePointer.withMemoryRebound(to:capacity:_:) has 3rd parameter type change from (UnsafeMutablePointer<T>) throws -> Result to (UnsafePointer<T>) throws -> Result

tools/CMakeLists.txt

@@ -7,6 +7,11 @@ add_swift_tool_subdirectory(lldb-moduleimport-test)
 add_swift_tool_subdirectory(sil-extract)
 add_swift_tool_subdirectory(swift-llvm-opt)
 
+if(SWIFT_HOST_VARIANT STREQUAL "macosx")
+  # Only build Darwin-specific tools when deploying to OS X.
+  add_swift_tool_subdirectory(swift-api-digester)
+endif()
+
 if(SWIFT_BUILD_SOURCEKIT)
   add_swift_tool_subdirectory(SourceKit)
 endif()

tools/swift-api-digester/CMakeLists.txt

@@ -0,0 +1,5 @@
+add_swift_executable(swift-api-digester
+  swift-api-digester.cpp
+  LINK_LIBRARIES
+    swiftFrontend swiftIDE
+)

tools/swift-api-digester/DigesterEnums.def

@@ -0,0 +1,84 @@
+#ifndef NODE_KIND
+#define NODE_KIND(NAME)
+#endif
+
+#ifndef NODE_ANNOTATION
+#define NODE_ANNOTATION(NAME)
+#endif
+
+#ifndef DECL_ATTR
+#define DECL_ATTR(NAME)
+#endif
+
+#ifndef KEY
+#define KEY(NAME)
+#endif
+
+#ifndef KNOWN_TYPE
+#define KNOWN_TYPE(NAME)
+#endif
+
+NODE_KIND(Root)
+NODE_KIND(TypeDecl)
+NODE_KIND(TypeNominal)
+NODE_KIND(TypeFunc)
+NODE_KIND(Function)
+NODE_KIND(Constructor)
+NODE_KIND(Getter)
+NODE_KIND(Setter)
+NODE_KIND(Var)
+NODE_KIND(TypeAlias)
+NODE_KIND(Nil)
+
+NODE_ANNOTATION(Added)
+NODE_ANNOTATION(Removed)
+NODE_ANNOTATION(Updated)
+NODE_ANNOTATION(RemovingErrorParam)
+NODE_ANNOTATION(ImplicitOptionalToOptional)
+NODE_ANNOTATION(OptionalToImplicitOptional)
+NODE_ANNOTATION(WrapOptional)
+NODE_ANNOTATION(WrapImplicitOptional)
+NODE_ANNOTATION(UnwrapOptional)
+NODE_ANNOTATION(GenericParamUpCast)
+NODE_ANNOTATION(GenericParamDownCast)
+NODE_ANNOTATION(TypeAliasChangeFromInt)
+NODE_ANNOTATION(GetterToProperty)
+NODE_ANNOTATION(SetterToProperty)
+NODE_ANNOTATION(TypeRewritten)
+NODE_ANNOTATION(TypeRewrittenLeft)
+NODE_ANNOTATION(TypeRewrittenRight)
+NODE_ANNOTATION(ModernizeEnum)
+NODE_ANNOTATION(UnwrapUnmanaged)
+NODE_ANNOTATION(RemovedDecl)
+NODE_ANNOTATION(Rename)
+NODE_ANNOTATION(RenameOldName)
+NODE_ANNOTATION(RenameNewName)
+NODE_ANNOTATION(NowThrowing)
+
+DECL_ATTR(deprecated)
+
+KEY(kind)
+KEY(name)
+KEY(selfIndex)
+KEY(usr)
+KEY(location)
+KEY(children)
+KEY(printedName)
+KEY(moduleName)
+KEY(throwing)
+KEY(typeAttributes)
+KEY(declAttributes)
+KEY(declKind)
+
+KNOWN_TYPE(Optional)
+KNOWN_TYPE(ImplicitlyUnwrappedOptional)
+KNOWN_TYPE(Void)
+KNOWN_TYPE(Unmanaged)
+KNOWN_TYPE(Function)
+KNOWN_TYPE(PolymorphicFunction)
+
+#undef KNOWN_TYPE
+#undef KEY
+#undef DECL_ATTR
+#undef NODE_ANNOTATION
+#undef NODE_KIND