[test] Warning about Swift features that do not exist (#77542)

drodriguez · web-flow · commit d2d830fc0e4c · 2024-11-12T08:58:53.000-08:00
Sometimes features are removed from `Features.def`, but they are not removed from the test files. The compiler ignores every feature that does not exist. This leads to removed features still being tested, and with the introduction of #76740, `REQUIRED:`. Modify the code that generates the active feature set for testing to also generate the set of existing features, and pass this list of existing features to the verifier script. If a feature is trying to be activated and does not exist, the verifier will warn the user. - `SwiftParser` feature was renamed `ParserASTGen`, but some tests were using both spellings. Remove the mentions of `SwiftParser` in the tests. - `ImportObjcForwardDeclarations` was spelled with upper case `C` in a couple of tests. Fix it so the matching is easier. - `TransferringArgsAndResults` was an experimental feature that was removed. - Ignore the usage of inexisting feature in `swift-export-as.swift` because it seems to be what the test is actually testing. - Ignore the test `availability_define.swift` because it tests the pseudo-feature `AvailabilityMacro=` which is not part of `Features.def`.
diff --git a/test/ClangImporter/incomplete_objc_types_full_availability.swift b/test/ClangImporter/incomplete_objc_types_full_availability.swift
@@ -2,12 +2,12 @@
 // RUN: %target-clang %S/Inputs/custom-modules/IncompleteTypes/incomplete-type-library-1.m -c -o %t/incomplete-type-library-1.o
 // RUN: %target-clang %S/Inputs/custom-modules/IncompleteTypes/complete-types.m -c -o %t/complete-types.o
 
-// RUN: %target-build-swift -Xfrontend -enable-upcoming-feature -Xfrontend ImportObjCForwardDeclarations -Xfrontend -enable-objc-interop -I %S/Inputs/custom-modules/IncompleteTypes %s %t/incomplete-type-library-1.o %t/complete-types.o -Xlinker -framework -Xlinker Foundation -o %t/a.out
+// RUN: %target-build-swift -Xfrontend -enable-upcoming-feature -Xfrontend ImportObjcForwardDeclarations -Xfrontend -enable-objc-interop -I %S/Inputs/custom-modules/IncompleteTypes %s %t/incomplete-type-library-1.o %t/complete-types.o -Xlinker -framework -Xlinker Foundation -o %t/a.out
 // RUN: %target-run %t/a.out
 
 // REQUIRES: objc_interop
 // REQUIRES: executable_test
-// REQUIRES: swift_feature_ImportObjCForwardDeclarations
+// REQUIRES: swift_feature_ImportObjcForwardDeclarations
 
 import IncompleteTypeLibrary1
 import CompleteTypes
diff --git a/test/Concurrency/sending_continuation.swift b/test/Concurrency/sending_continuation.swift
@@ -1,7 +1,6 @@
-// RUN: %target-swift-frontend -emit-sil -swift-version 6 -target %target-swift-5.1-abi-triple -enable-experimental-feature TransferringArgsAndResults -verify %s -o /dev/null -parse-as-library
+// RUN: %target-swift-frontend -emit-sil -swift-version 6 -target %target-swift-5.1-abi-triple -verify %s -o /dev/null -parse-as-library
 
 // REQUIRES: concurrency
-// REQUIRES: swift_feature_TransferringArgsAndResults
 
 ////////////////////////
 // MARK: Declarations //
diff --git a/test/Misc/verify-swift-feature-testing.test-sh b/test/Misc/verify-swift-feature-testing.test-sh
@@ -1,7 +1,8 @@
 #! /usr/bin/env python3
 # -*- python -*-
-# RUN: %{python} %s '%swift_src_root'
+# RUN: %{python} %s '%swift_src_root' %existing-swift-features
 
+import json
 import pathlib
 import re
 import subprocess
@@ -11,8 +12,14 @@ import sys
 # Tests that check for the behaviour of  experimental/upcoming features, so
 # they cannot automatically be checked.
 EXCEPTIONAL_FILES = [
+    # Tests for ParserValidation not being defined in no-asserts compilers
     pathlib.Path("test/Frontend/experimental-features-no-asserts.swift"),
+    # Tests for UnknownFeature not existing
     pathlib.Path("test/Frontend/upcoming_feature.swift"),
+    # Tests for ModuleInterfaceExportAs being ignored
+    pathlib.Path("test/ModuleInterface/swift-export-as.swift"),
+    # Uses the pseudo-feature AvailabilityMacro=
+    pathlib.Path("test/Sema/availability_define.swift"),
 ]
 
 FEATURE_USAGE_RE = re.compile(
@@ -28,16 +35,19 @@ def find_test_files_with_features_usage(swift_src_root):
     # mention `-enable-experimental-feature` or `-enable-upcoming-feature`.
     # Be careful of not using REQUIRES or RUN with a colon after them or Lit will
     # pick them up.
-    output = subprocess.check_output([
-        "grep",
-        "--extended-regexp",
-        "--recursive",
-        "-e",
-        "RUN[:].*-enable-(experimental|upcoming)-feature",
-        "--files-with-matches",
-        str(swift_src_root / "test"),
-        str(swift_src_root / "validation-test"),
-    ], text=True)
+    output = subprocess.check_output(
+        [
+            "grep",
+            "--extended-regexp",
+            "--recursive",
+            "-e",
+            "RUN[:].*-enable-(experimental|upcoming)-feature",
+            "--files-with-matches",
+            str(swift_src_root / "test"),
+            str(swift_src_root / "validation-test"),
+        ],
+        text=True,
+    )
     return output.splitlines()
 
 
@@ -46,91 +56,110 @@ def find_test_files_with_marker_usage(swift_src_root):
     # that mention `swift_feature_`.
     # Be careful of not using REQUIRES with a colon after them or Lit will
     # pick them up.
-    output = subprocess.check_output([
-        "grep",
-        "--extended-regexp",
-        "--recursive",
-        "-e",
-        "REQUIRES[:].*swift_feature_",
-        "--files-with-matches",
-        str(swift_src_root / "test"),
-        str(swift_src_root / "validation-test"),
-    ], text=True)
+    output = subprocess.check_output(
+        [
+            "grep",
+            "--extended-regexp",
+            "--recursive",
+            "-e",
+            "REQUIRES[:].*swift_feature_",
+            "--files-with-matches",
+            str(swift_src_root / "test"),
+            str(swift_src_root / "validation-test"),
+        ],
+        text=True,
+    )
     return output.splitlines()
 
 
 def find_run_lines(test_file):
-    output = subprocess.check_output([
-        "grep",
-        "--extended-regexp",
-        "--no-filename",
-        "-e",
-        "RUN[:]",
-        str(test_file),
-    ], text=True)
+    output = subprocess.check_output(
+        [
+            "grep",
+            "--extended-regexp",
+            "--no-filename",
+            "-e",
+            "RUN[:]",
+            str(test_file),
+        ],
+        text=True,
+    )
     return output.splitlines()
 
 
 def find_requires_lines(test_file):
-    output = subprocess.check_output([
-        "grep",
-        "--extended-regexp",
-        "--no-filename",
-        "-e",
-        "REQUIRES[:]",
-        str(test_file),
-    ], text=True)
+    output = subprocess.check_output(
+        [
+            "grep",
+            "--extended-regexp",
+            "--no-filename",
+            "-e",
+            "REQUIRES[:]",
+            str(test_file),
+        ],
+        text=True,
+    )
     return output.splitlines()
 
 
 def check_existing_requires(test_file, feature):
-    returncode = subprocess.call([
-        "grep",
-        "--extended-regexp",
-        "--quiet",
-        "-e",
-        "REQUIRES[:].*swift_feature_" + feature,
-        str(test_file),
-    ])
+    returncode = subprocess.call(
+        [
+            "grep",
+            "--extended-regexp",
+            "--quiet",
+            "-e",
+            "REQUIRES[:].*swift_feature_" + feature,
+            str(test_file),
+        ]
+    )
     return returncode != 0
 
 
 def check_existing_feature_usage(test_file, feature):
-    returncode = subprocess.call([
-        "grep",
-        "--extended-regexp",
-        "--quiet",
-        "-e",
-        (
-            "RUN[:].*-enable-(experimental|upcoming)-feature (-Xfrontend )?"
-            + re.escape(feature)
-        ),
-        str(test_file),
-    ])
+    returncode = subprocess.call(
+        [
+            "grep",
+            "--extended-regexp",
+            "--quiet",
+            "-e",
+            (
+                "RUN[:].*-enable-(experimental|upcoming)-feature (-Xfrontend )?"
+                + re.escape(feature)
+            ),
+            str(test_file),
+        ]
+    )
     return returncode != 0
 
 
 def check_existing_error_message_checks(test_file, feature):
-    returncode = subprocess.call([
-        "grep",
-        "--extended-regexp",
-        "--quiet",
-        "-e",
-        "requires '-enable-(experimental|upcoming)-feature " + feature + "'",
-        str(test_file),
-    ])
+    returncode = subprocess.call(
+        [
+            "grep",
+            "--extended-regexp",
+            "--quiet",
+            "-e",
+            "requires '-enable-(experimental|upcoming)-feature " + feature + "'",
+            str(test_file),
+        ]
+    )
     return returncode != 0
 
 
-def check_test_file_feature_usage(test_file):
+def check_test_file_feature_usage(test_file, existing_swift_features):
     run_lines = find_run_lines(test_file)
     features = set(
-        feature
-        for line in run_lines
-        for feature in FEATURE_USAGE_RE.findall(line)
+        feature for line in run_lines for feature in FEATURE_USAGE_RE.findall(line)
     )
     num_failures = 0
     for feature in features:
+        # First, check this is a valid feature
+        if feature not in existing_swift_features:
+            print("error: {}: Unknown feature: {}".format(str(test_file), feature))
+            num_failures += 1
+            continue
+
         # No warning if the necessary `REQUIRES` is already there
         if not check_existing_requires(test_file, feature):
             continue
@@ -142,9 +171,7 @@ def check_test_file_feature_usage(test_file):
         # For everything else, print a warning and add to the invalid exit code
         print(
             "error: {}: Missing '{}: swift_feature_{}'".format(
-                str(test_file),
-                "REQUIRES",
-                feature
+                str(test_file), "REQUIRES", feature
             )
         )
         num_failures += 1
@@ -167,20 +194,20 @@ def check_test_file_marker_usage(test_file):
         # For everything else, print a warning and add to the invalid exit code
         print(
             "error: {}: Missing '-enable-experimental/upcoming-feature: {}'".format(
-                str(test_file),
-                feature
+                str(test_file), feature
             )
         )
         num_failures += 1
     return num_failures == 0
 
 
 def main():
-    if len(sys.argv) < 2:
-        print('Invalid number of arguments.')
+    if len(sys.argv) < 3:
+        print("Invalid number of arguments.")
         sys.exit(1)
 
     swift_src_root = pathlib.Path(sys.argv[1])
+    existing_swift_features = set(json.loads(sys.argv[2]))
 
     num_failures = 0
     test_files_with_features_usage = find_test_files_with_features_usage(swift_src_root)
@@ -190,7 +217,7 @@ def main():
         if test_file.relative_to(swift_src_root) in EXCEPTIONAL_FILES:
             continue
 
-        if not check_test_file_feature_usage(test_file):
+        if not check_test_file_feature_usage(test_file, existing_swift_features):
             num_failures += 1
 
     test_files_with_marker_usage = find_test_files_with_marker_usage(swift_src_root)
@@ -208,5 +235,5 @@ def main():
         sys.exit(1)
 
 
-if __name__ == '__main__':
+if __name__ == "__main__":
     main()
diff --git a/test/Sema/availability_define.swift b/test/Sema/availability_define.swift
@@ -15,7 +15,6 @@
 // RUN:   -enable-experimental-feature AvailabilityMacro="_myProject 2.5:macOS 52.5"
 
 // REQUIRES: OS=macosx
-// REQUIRES: swift_feature_AvailabilityMacro
 
 @available(_iOS53Aligned, *)
 public func onMacOS50() {}
diff --git a/test/lit.cfg b/test/lit.cfg
@@ -18,15 +18,16 @@
 #
 # -----------------------------------------------------------------------------
 
+import glob
+import json
 import os
 import platform
 import re
 import shlex
 import shutil
+import socket
 import subprocess
 import sys
-import socket
-import glob
 
 import lit
 import lit.formats
@@ -790,6 +791,8 @@ config.substitutions.append( ('%clang',
                               "%r %s" %
                                 (config.clang, clang_mcp_opt)) )
 
+config.substitutions.append(('%existing-swift-features', shlex.quote(json.dumps(list(config.existing_swift_features)))))
+
 ###
 
 def disallow(execName):
diff --git a/test/lit.swift-features.cfg.inc b/test/lit.swift-features.cfg.inc
@@ -22,12 +22,14 @@
  * statements).
  */
 
+config.existing_swift_features = set()
 def language_feature(feature_name, enabled):
+    config.existing_swift_features.add(feature_name)
     if enabled or "asserts" in config.available_features:
-        config.available_features.add(feature_name)
+        config.available_features.add("swift_feature_" + feature_name)
 
-#define UPCOMING_FEATURE(FeatureName, SENumber, Version) language_feature("swift_feature_" # FeatureName, True)
-#define EXPERIMENTAL_FEATURE(FeatureName, AvailableInProd) language_feature("swift_feature_" # FeatureName, #AvailableInProd == "true")
-#define LANGUAGE_FEATURE(FeatureName, SENumber, Description) language_feature("swift_feature_" # FeatureName, True)
+#define UPCOMING_FEATURE(FeatureName, SENumber, Version) language_feature(#FeatureName, True)
+#define EXPERIMENTAL_FEATURE(FeatureName, AvailableInProd) language_feature(#FeatureName, #AvailableInProd == "true")
+#define LANGUAGE_FEATURE(FeatureName, SENumber, Description) language_feature(#FeatureName, True)
 
 #include <swift/Basic/Features.def>
