From 168624d1d898311aee44607687ba858690516448 Mon Sep 17 00:00:00 2001 From: Paolo Tranquilli Date: Thu, 30 Mar 2023 08:13:52 +0200 Subject: [PATCH 1/2] Swift: turn extractor into a `swift-frontend` plugin This change turns the extractor into a plugin to a slightly modified `swift-extractor` binary, providing only a `swift::FrontendObserver` instance. This makes it so that we inherit the behaviours of the released `swift-frontend`, among the known solved issues are: * work around the issue preventing the Linux extractor from compiling stdlib `.swiftinterface` files, which was preventing the extractor from working on top of earlier versions of the swift SDK * `@`-prefixed parameter files are now supported Under the hood the Swift build system has been tweaked so that calls to `performFrontend` in the driver executable use a `FrontendObserver` built from a dynamic library, and the extractor now provides an alternative frontend observer dynamic library. Some special treatment needed to retain the same features: * a new `finished` method was added to `swift::FrontendObserver` to implement the `file_is_successfully_extracted` predicate * part of the swift AST library (`Identifier`) needed to be extracted into a separate dynamic library as well, as behaviour was depending on the value of static pointers that therefore needed to be shared between `swift-frontend` and the extractor * the `CODEQL_EXTRACTOR_SWIFT_RUN_UNDER` functionality has been moved to the `extractor` bash wrapper, which also makes it work better to catch crashes happening before `main` is even entered (which can happen with tracer problems) --- swift/extractor/BUILD.bazel | 15 +++- swift/extractor/extractor.sh | 16 +++- swift/extractor/main.cpp | 85 +++---------------- .../run_under/Strings.expected | 4 +- swift/third_party/load.bzl | 8 +- .../swift-llvm-support/BUILD.bazel | 2 +- .../BUILD.swift-llvm-support.bazel | 30 +++---- 7 files changed, 57 insertions(+), 103 deletions(-) diff --git a/swift/extractor/BUILD.bazel b/swift/extractor/BUILD.bazel index 804335cf65f9..00db47decc14 100644 --- a/swift/extractor/BUILD.bazel +++ b/swift/extractor/BUILD.bazel @@ -3,11 +3,12 @@ load("//misc/bazel/cmake:cmake.bzl", "generate_cmake") load("//misc/bazel:pkg_runfiles.bzl", "pkg_runfiles") swift_cc_binary( - name = "extractor.real", + name = "swiftFrontendObserver", srcs = glob([ "*.h", "*.cpp", ]), + linkshared = True, deps = [ "//swift/extractor/config", "//swift/extractor/infra", @@ -21,19 +22,27 @@ swift_cc_binary( generate_cmake( name = "cmake", - targets = [":extractor.real"], + targets = [":swiftFrontendObserver"], visibility = ["//visibility:public"], ) sh_binary( name = "extractor", srcs = ["extractor.sh"], - data = [":extractor.real"], + data = [ + ":swiftFrontendObserver", + "//swift/third_party/swift-llvm-support:swift-frontend", + ], ) pkg_runfiles( name = "pkg", srcs = [":extractor"], excludes = ["extractor.sh"], # script gets copied as "extractor", no need for the original .sh file + renames = select({ + # workaround for https://github.com/bazelbuild/bazel/issues/11082 (wrongly marked as closed) + "@platforms//os:macos": {":swiftFrontendObserver": "libswiftFrontendObserver.dylib"}, + "//conditions:default": {}, + }), visibility = ["//swift:__pkg__"], ) diff --git a/swift/extractor/extractor.sh b/swift/extractor/extractor.sh index 6fdfec50b9b7..0caac9f2abb0 100755 --- a/swift/extractor/extractor.sh +++ b/swift/extractor/extractor.sh @@ -1,9 +1,19 @@ #!/bin/bash +EXE_DIR="$(dirname "$0")" + if [[ "$(uname)" == Darwin ]]; then - export DYLD_LIBRARY_PATH=$(dirname "$0") + export DYLD_LIBRARY_PATH="$EXE_DIR" else - export LD_LIBRARY_PATH=$(dirname "$0") + export LD_LIBRARY_PATH="$EXE_DIR" +fi + +TOOL="$CODEQL_EXTRACTOR_SWIFT_RUN_UNDER" + +if [[ -n "$CODEQL_EXTRACTOR_SWIFT_RUN_UNDER_FILTER" ]]; then + if [[ ! "$*" =~ $CODEQL_EXTRACTOR_SWIFT_RUN_UNDER_FILTER ]]; then + TOOL= + fi fi -exec -a "$0" "$0.real" "$@" +exec -a swift-frontend $TOOL "$EXE_DIR/swift-frontend" "$@" diff --git a/swift/extractor/main.cpp b/swift/extractor/main.cpp index bb4f1d775c22..2857c1d446db 100644 --- a/swift/extractor/main.cpp +++ b/swift/extractor/main.cpp @@ -6,7 +6,8 @@ #include #include -#include +#include +#include #include #include "swift/extractor/SwiftExtractor.h" @@ -92,7 +93,8 @@ class Observer : public swift::FrontendObserver { codeql::extractExtractLazyDeclarations(state, compiler); } - void markSuccessfullyExtractedFiles() { + void finished(int status) override { + if (status != 0) return; codeql::SwiftLocationExtractor locExtractor{invocationTrap}; for (const auto& src : state.sourceFiles) { auto fileLabel = locExtractor.emitFile(src); @@ -102,6 +104,8 @@ class Observer : public swift::FrontendObserver { private: codeql::SwiftExtractorState state; + std::shared_ptr openInterception{ + codeql::setupFileInterception(state.configuration)}; codeql::TrapDomain invocationTrap{invocationTrapDomain(state)}; codeql::SwiftDiagnosticsConsumer diagConsumer{invocationTrap}; }; @@ -113,49 +117,6 @@ static std::string getenv_or(const char* envvar, const std::string& def) { return def; } -static bool checkRunUnderFilter(int argc, char* const* argv) { - auto runUnderFilter = getenv("CODEQL_EXTRACTOR_SWIFT_RUN_UNDER_FILTER"); - if (runUnderFilter == nullptr) { - return true; - } - std::string call = argv[0]; - for (auto i = 1; i < argc; ++i) { - call += ' '; - call += argv[i]; - } - std::regex filter{runUnderFilter, std::regex_constants::basic | std::regex_constants::nosubs}; - return std::regex_search(call, filter); -} - -// if `CODEQL_EXTRACTOR_SWIFT_RUN_UNDER` env variable is set, and either -// * `CODEQL_EXTRACTOR_SWIFT_RUN_UNDER_FILTER` is not set, or -// * it is set to a regexp matching any substring of the extractor call -// then the running process is substituted with the command (and possibly -// options) stated in `CODEQL_EXTRACTOR_SWIFT_RUN_UNDER`, followed by `argv`. -// Before calling `exec`, `CODEQL_EXTRACTOR_SWIFT_RUN_UNDER` is unset to avoid -// unpleasant loops. -// An example usage is to run the extractor under `gdbserver :1234` when the -// arguments match a given source file. -static void checkWhetherToRunUnderTool(int argc, char* const* argv) { - assert(argc > 0); - - auto runUnder = getenv("CODEQL_EXTRACTOR_SWIFT_RUN_UNDER"); - if (runUnder == nullptr || !checkRunUnderFilter(argc, argv)) { - return; - } - std::vector args; - // split RUN_UNDER value by spaces to get args vector - for (auto word = std::strtok(runUnder, " "); word != nullptr; word = std::strtok(nullptr, " ")) { - args.push_back(word); - } - // append process args, including extractor executable path - args.insert(args.end(), argv, argv + argc); - args.push_back(nullptr); - // avoid looping on this function - unsetenv("CODEQL_EXTRACTOR_SWIFT_RUN_UNDER"); - execvp(args[0], args.data()); -} - // Creates a target file that should store per-invocation info, e.g. compilation args, // compilations, diagnostics, etc. codeql::TrapDomain invocationTrapDomain(codeql::SwiftExtractorState& state) { @@ -170,39 +131,19 @@ codeql::TrapDomain invocationTrapDomain(codeql::SwiftExtractorState& state) { return std::move(maybeDomain.value()); } -codeql::SwiftExtractorConfiguration configure(int argc, char** argv) { +codeql::SwiftExtractorConfiguration configure(llvm::ArrayRef argv) { codeql::SwiftExtractorConfiguration configuration{}; configuration.trapDir = getenv_or("CODEQL_EXTRACTOR_SWIFT_TRAP_DIR", "."); configuration.sourceArchiveDir = getenv_or("CODEQL_EXTRACTOR_SWIFT_SOURCE_ARCHIVE_DIR", "."); configuration.scratchDir = getenv_or("CODEQL_EXTRACTOR_SWIFT_SCRATCH_DIR", "."); - configuration.frontendOptions.assign(argv + 1, argv + argc); + configuration.frontendOptions.assign(argv.begin() + 1, argv.end()); return configuration; } -int main(int argc, char** argv) { - checkWhetherToRunUnderTool(argc, argv); - - if (argc == 1) { - // TODO: print usage - return 1; - } - - // Required by Swift/LLVM - PROGRAM_START(argc, argv); - INITIALIZE_LLVM(); - initializeSwiftModules(); - - const auto configuration = configure(argc, argv); - - auto openInterception = codeql::setupFileInterception(configuration); - - Observer observer(configuration); - int frontend_rc = swift::performFrontend(configuration.frontendOptions, "swift-extractor", - (void*)main, &observer); - - if (frontend_rc == 0) { - observer.markSuccessfullyExtractedFiles(); - } +namespace swift { - return frontend_rc; +FrontendObserver* getFrontendObserver(llvm::ArrayRef argv) { + static Observer observer{configure(argv)}; + return &observer; } +} // namespace swift diff --git a/swift/ql/test/extractor-tests/run_under/Strings.expected b/swift/ql/test/extractor-tests/run_under/Strings.expected index 042e306b4bb0..520cb90c770d 100644 --- a/swift/ql/test/extractor-tests/run_under/Strings.expected +++ b/swift/ql/test/extractor-tests/run_under/Strings.expected @@ -1,2 +1,2 @@ -| run_under: $CODEQL_EXTRACTOR_SWIFT_ROOT/tools/$CODEQL_PLATFORM/extractor -sdk $CODEQL_EXTRACTOR_SWIFT_ROOT/qltest/$CODEQL_PLATFORM/sdk -c -primary-file filtered_in.swift | -| run_under: $CODEQL_EXTRACTOR_SWIFT_ROOT/tools/$CODEQL_PLATFORM/extractor -sdk $CODEQL_EXTRACTOR_SWIFT_ROOT/qltest/$CODEQL_PLATFORM/sdk -c -primary-file unfiltered.swift | +| run_under: $CODEQL_EXTRACTOR_SWIFT_ROOT/tools/$CODEQL_PLATFORM/swift-frontend -sdk $CODEQL_EXTRACTOR_SWIFT_ROOT/qltest/$CODEQL_PLATFORM/sdk -c -primary-file filtered_in.swift | +| run_under: $CODEQL_EXTRACTOR_SWIFT_ROOT/tools/$CODEQL_PLATFORM/swift-frontend -sdk $CODEQL_EXTRACTOR_SWIFT_ROOT/qltest/$CODEQL_PLATFORM/sdk -c -primary-file unfiltered.swift | diff --git a/swift/third_party/load.bzl b/swift/third_party/load.bzl index 4e7de8e1b7c5..07695a72e246 100644 --- a/swift/third_party/load.bzl +++ b/swift/third_party/load.bzl @@ -1,10 +1,10 @@ load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive") -_swift_prebuilt_version = "swift-5.7.3-RELEASE.142" +_swift_prebuilt_version = "swift-5.7.3-RELEASE.150" _swift_sha_map = { - "Linux-X64": "398d8de54c8775c939dff95ed5bb0e04a9308a1982b4c1900cd4a5d01223f63b", - "macOS-ARM64": "397dd67ea99b9c9455794c6eb0f1664b6179fe542c7c1d3010314a3e8a905ae4", - "macOS-X64": "4b9d8e4e89f16a7c1e7edc7893aa189b37d5b4412be724a86ef59c49d11a6f75", + "Linux-X64": "8465b5ad6b34c723786ae56478ece1a3a778c812c4d0a0521236a12c6544f57d", + "macOS-ARM64": "38b84b24366841fc2b7c5fe081d87310d8e62051884147e155bc74a929a770b7", + "macOS-X64": "1be86d88e8a9be530a8c48ba73cd000daaae9d0435ae1db9ff3a368444639b99", } _swift_arch_map = { diff --git a/swift/third_party/swift-llvm-support/BUILD.bazel b/swift/third_party/swift-llvm-support/BUILD.bazel index db5c09085a44..706589281820 100644 --- a/swift/third_party/swift-llvm-support/BUILD.bazel +++ b/swift/third_party/swift-llvm-support/BUILD.bazel @@ -17,5 +17,5 @@ _arch_override = { for arch in ("linux", "darwin_x86_64", "darwin_arm64") }), ) - for name in ("swift-llvm-support", "swift-test-sdk") + for name in ("swift-llvm-support", "swift-test-sdk", "swift-frontend") ] diff --git a/swift/third_party/swift-llvm-support/BUILD.swift-llvm-support.bazel b/swift/third_party/swift-llvm-support/BUILD.swift-llvm-support.bazel index 58796aaaacbc..784cc21a8857 100644 --- a/swift/third_party/swift-llvm-support/BUILD.swift-llvm-support.bazel +++ b/swift/third_party/swift-llvm-support/BUILD.swift-llvm-support.bazel @@ -2,23 +2,16 @@ load("@rules_pkg//:mappings.bzl", "pkg_files", "strip_prefix") cc_library( name = "swift-llvm-support", - srcs = [ - "libCodeQLSwiftFrontendTool.a", - ] + select({ - "@platforms//os:linux": [ - "libCodeQLSwiftFrontendTool.so", - "libswiftCore.so", - ], - "@platforms//os:macos": [ - "libCodeQLSwiftFrontendTool.dylib", - "libswiftCore.dylib", - "libswiftCompatibility50.a", - "libswiftCompatibility51.a", - "libswiftCompatibilityConcurrency.a", - "libswiftCompatibilityDynamicReplacements.a", - ], - }), - hdrs = glob(["include/**/*", "stdlib/**/*" ]), + srcs = glob([ + "lib*.a", + "lib*.so", + "lib*.dylib", + ]), + hdrs = glob([ + "include/**/*", + "stdlib/**/*", + ]), + includes = ["include"], linkopts = [ "-lm", "-lz", @@ -34,7 +27,6 @@ cc_library( ], "//conditions:default": [], }), - includes = [ "include" ], visibility = ["//visibility:public"], ) @@ -46,3 +38,5 @@ pkg_files( strip_prefix = strip_prefix.from_pkg(), visibility = ["//visibility:public"], ) + +exports_files(["swift-frontend"]) From d5b219b2c41ba4a4d7bf5b9fbaf8cbd86fe6d1c3 Mon Sep 17 00:00:00 2001 From: Paolo Tranquilli Date: Wed, 5 Apr 2023 09:51:40 +0200 Subject: [PATCH 2/2] Swift: update bazel version to 6.1.1 --- .bazelversion | 2 +- swift/extractor/BUILD.bazel | 5 ----- 2 files changed, 1 insertion(+), 6 deletions(-) diff --git a/.bazelversion b/.bazelversion index 0062ac971805..f3b5af39e430 100644 --- a/.bazelversion +++ b/.bazelversion @@ -1 +1 @@ -5.0.0 +6.1.1 diff --git a/swift/extractor/BUILD.bazel b/swift/extractor/BUILD.bazel index 00db47decc14..adb66cbb3e72 100644 --- a/swift/extractor/BUILD.bazel +++ b/swift/extractor/BUILD.bazel @@ -39,10 +39,5 @@ pkg_runfiles( name = "pkg", srcs = [":extractor"], excludes = ["extractor.sh"], # script gets copied as "extractor", no need for the original .sh file - renames = select({ - # workaround for https://github.com/bazelbuild/bazel/issues/11082 (wrongly marked as closed) - "@platforms//os:macos": {":swiftFrontendObserver": "libswiftFrontendObserver.dylib"}, - "//conditions:default": {}, - }), visibility = ["//swift:__pkg__"], )