Skip to content

Commit bbbbc09

Browse files
bd1976brisbd1976bris
authored
[DTLTO][LLD][COFF] Add support for Integrated Distributed ThinLTO (#148594)
This patch introduces support for Integrated Distributed ThinLTO (DTLTO) in COFF LLD. DTLTO enables the distribution of ThinLTO backend compilations via external distribution systems, such as Incredibuild, during the traditional link step: https://llvm.org/docs/DTLTO.html. Note: Bitcode members of non-thin archives are not currently supported. This will be addressed in a future change. This patch is sufficient to allow for self-hosting an LLVM build with DTLTO if thin archives are used. Testing: - LLD `lit` test coverage has been added, using a mock distributor to avoid requiring Clang. - Cross-project `lit` tests cover integration with Clang. For the design discussion of the DTLTO feature, see: #126654
1 parent 17c7c2e commit bbbbc09

File tree

10 files changed

+347
-4
lines changed

10 files changed

+347
-4
lines changed

‎cross-project-tests/dtlto/link-archive-thin.test

Lines changed: 93 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,93 @@
1+
REQUIRES: lld-link
2+
3+
## Test that a DTLTO link succeeds and outputs the expected set of files
4+
## correctly when thin archives are present.
5+
6+
RUN: rm -rf %t && split-file %s %t && cd %t
7+
8+
## Compile bitcode. -O2 is required for cross-module importing.
9+
RUN: %clang -O2 --target=x86_64-pc-windows-msvc -flto=thin -c \
10+
RUN: foo.c bar.c dog.c cat.c start.c
11+
12+
## Generate thin archives.
13+
RUN: lld-link /lib /llvmlibthin /out:foo.lib foo.o
14+
## Create this bitcode thin archive in a subdirectory to test the expansion of
15+
## the path to a bitcode file that is referenced using "..", e.g., in this case
16+
## "../bar.o".
17+
RUN: mkdir lib
18+
RUN: lld-link /lib /llvmlibthin /out:lib/bar.lib bar.o
19+
## Create this bitcode thin archive with an absolute path entry containing "..".
20+
RUN: lld-link /lib /llvmlibthin /out:dog.lib %t/lib/../dog.o
21+
RUN: lld-link /lib /llvmlibthin /out:cat.lib cat.o
22+
RUN: lld-link /lib /llvmlibthin /out:start.lib start.o
23+
24+
## Link from a different directory to ensure that thin archive member paths are
25+
## resolved correctly relative to the archive locations.
26+
RUN: mkdir %t/out && cd %t/out
27+
RUN: lld-link /subsystem:console /machine:x64 /entry:start /out:my.exe \
28+
RUN: %t/foo.lib %t/lib/bar.lib ../start.lib %t/cat.lib \
29+
RUN: /includeoptional:dog ../dog.lib \
30+
RUN: -thinlto-distributor:%python \
31+
RUN: -thinlto-distributor-arg:%llvm_src_root/utils/dtlto/local.py \
32+
RUN: -thinlto-remote-compiler:%clang \
33+
RUN: /lldsavetemps
34+
35+
## Check that the required output files have been created.
36+
RUN: ls | FileCheck %s --check-prefix=OUTPUTS --implicit-check-not=cat
37+
38+
## JSON jobs description.
39+
OUTPUTS-DAG: my.[[PID:[a-zA-Z0-9_]+]].dist-file.json
40+
41+
## Individual summary index files.
42+
OUTPUTS-DAG: start.1.[[PID]].native.o.thinlto.bc{{$}}
43+
OUTPUTS-DAG: dog.2.[[PID]].native.o.thinlto.bc{{$}}
44+
OUTPUTS-DAG: foo.3.[[PID]].native.o.thinlto.bc{{$}}
45+
OUTPUTS-DAG: bar.4.[[PID]].native.o.thinlto.bc{{$}}
46+
47+
## Native output object files.
48+
OUTPUTS-DAG: start.1.[[PID]].native.o{{$}}
49+
OUTPUTS-DAG: dog.2.[[PID]].native.o{{$}}
50+
OUTPUTS-DAG: foo.3.[[PID]].native.o{{$}}
51+
OUTPUTS-DAG: bar.4.[[PID]].native.o{{$}}
52+
53+
54+
## It is important that cross-module inlining occurs for this test to show that Clang can
55+
## successfully load the bitcode file dependencies recorded in the summary indices.
56+
## Explicitly check that the expected importing has occurred.
57+
58+
RUN: llvm-dis start.1.*.native.o.thinlto.bc -o - | \
59+
RUN: FileCheck %s --check-prefixes=FOO,BAR,START
60+
61+
RUN: llvm-dis dog.2.*.native.o.thinlto.bc -o - | \
62+
RUN: FileCheck %s --check-prefixes=FOO,BAR,DOG,START
63+
64+
RUN: llvm-dis foo.3.*.native.o.thinlto.bc -o - | \
65+
RUN: FileCheck %s --check-prefixes=FOO,BAR,START
66+
67+
RUN: llvm-dis bar.4.*.native.o.thinlto.bc -o - | \
68+
RUN: FileCheck %s --check-prefixes=FOO,BAR,START
69+
70+
FOO-DAG: foo.o
71+
BAR-DAG: bar.o
72+
DOG-DAG: dog.o
73+
START-DAG: start.o
74+
75+
76+
#--- foo.c
77+
extern int bar(int), start(int);
78+
__attribute__((retain)) int foo(int x) { return x + bar(x) + start(x); }
79+
80+
#--- bar.c
81+
extern int foo(int), start(int);
82+
__attribute__((retain)) int bar(int x) { return x + foo(x) + start(x); }
83+
84+
#--- dog.c
85+
extern int foo(int), bar(int), start(int);
86+
__attribute__((retain)) int dog(int x) { return x + foo(x) + bar(x) + start(x); }
87+
88+
#--- cat.c
89+
__attribute__((retain)) void cat(int x) {}
90+
91+
#--- start.c
92+
extern int foo(int), bar(int);
93+
__attribute__((retain)) int start(int x) { return x + foo(x) + bar(x); }

‎cross-project-tests/dtlto/link-dtlto.c

Lines changed: 41 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,41 @@
1+
// REQUIRES: lld-link
2+
3+
/// Simple test that DTLTO works with a single input bitcode file and that
4+
/// --save-temps can be applied to the remote compilation.
5+
6+
// RUN: rm -rf %t && mkdir %t && cd %t
7+
8+
// RUN: %clang --target=x86_64-pc-windows-msvc -c -flto=thin %s -o dtlto.obj
9+
10+
// RUN: lld-link /subsystem:console /entry:_start dtlto.obj \
11+
// RUN: -thinlto-distributor:%python \
12+
// RUN: -thinlto-distributor-arg:%llvm_src_root/utils/dtlto/local.py \
13+
// RUN: -thinlto-remote-compiler:%clang \
14+
// RUN: -thinlto-remote-compiler-arg:--save-temps
15+
16+
/// Check that the required output files have been created.
17+
// RUN: ls | sort | FileCheck %s
18+
19+
/// No files are expected before.
20+
// CHECK-NOT: {{.}}
21+
22+
/// Linked ELF.
23+
// CHECK: {{^}}dtlto.exe{{$}}
24+
25+
/// Produced by the bitcode compilation.
26+
// CHECK-NEXT: {{^}}dtlto.obj{{$}}
27+
28+
/// --save-temps output for the backend compilation.
29+
// CHECK-NEXT: {{^}}dtlto.s{{$}}
30+
// CHECK-NEXT: {{^}}dtlto.s.0.preopt.bc{{$}}
31+
// CHECK-NEXT: {{^}}dtlto.s.1.promote.bc{{$}}
32+
// CHECK-NEXT: {{^}}dtlto.s.2.internalize.bc{{$}}
33+
// CHECK-NEXT: {{^}}dtlto.s.3.import.bc{{$}}
34+
// CHECK-NEXT: {{^}}dtlto.s.4.opt.bc{{$}}
35+
// CHECK-NEXT: {{^}}dtlto.s.5.precodegen.bc{{$}}
36+
// CHECK-NEXT: {{^}}dtlto.s.resolution.txt{{$}}
37+
38+
/// No files are expected after.
39+
// CHECK-NOT: {{.}}
40+
41+
int _start() { return 0; }

‎cross-project-tests/lit.cfg.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -19,7 +19,7 @@
1919
config.test_format = lit.formats.ShTest(not llvm_config.use_lit_shell)
2020

2121
# suffixes: A list of file extensions to treat as test files.
22-
config.suffixes = [".c", ".cl", ".cpp", ".m"]
22+
config.suffixes = [".c", ".cl", ".cpp", ".m", ".test"]
2323

2424
# excludes: A list of directories to exclude from the testsuite. The 'Inputs'
2525
# subdirectories contain auxiliary inputs for various tests in their parent

‎lld/COFF/Config.h

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -192,6 +192,18 @@ struct Configuration {
192192
// Used for /lldltocachepolicy=policy
193193
llvm::CachePruningPolicy ltoCachePolicy;
194194

195+
// Used for /thinlto-distributor:<path>
196+
StringRef dtltoDistributor;
197+
198+
// Used for /thinlto-distributor-arg:<arg>
199+
llvm::SmallVector<llvm::StringRef, 0> dtltoDistributorArgs;
200+
201+
// Used for /thinlto-remote-compiler:<path>
202+
StringRef dtltoCompiler;
203+
204+
// Used for /thinlto-remote-compiler-arg:<arg>
205+
llvm::SmallVector<llvm::StringRef, 0> dtltoCompilerArgs;
206+
195207
// Used for /opt:[no]ltodebugpassmanager
196208
bool ltoDebugPassManager = false;
197209

‎lld/COFF/Driver.cpp

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2088,6 +2088,23 @@ void LinkerDriver::linkerMain(ArrayRef<const char *> argsArr) {
20882088
Fatal(ctx) << "/manifestinput: requires /manifest:embed";
20892089
}
20902090

2091+
// Handle /thinlto-distributor:<path>
2092+
config->dtltoDistributor = args.getLastArgValue(OPT_thinlto_distributor);
2093+
2094+
// Handle /thinlto-distributor-arg:<arg>
2095+
for (auto *arg : args.filtered(OPT_thinlto_distributor_arg))
2096+
config->dtltoDistributorArgs.push_back(arg->getValue());
2097+
2098+
// Handle /thinlto-remote-compiler:<path>
2099+
config->dtltoCompiler = args.getLastArgValue(OPT_thinlto_compiler);
2100+
if (!config->dtltoDistributor.empty() && config->dtltoCompiler.empty())
2101+
Err(ctx) << "A value must be specified for /thinlto-remote-compiler if "
2102+
"/thinlto-distributor is specified.";
2103+
2104+
// Handle /thinlto-remote-compiler-arg:<arg>
2105+
for (auto *arg : args.filtered(OPT_thinlto_compiler_arg))
2106+
config->dtltoCompilerArgs.push_back(arg->getValue());
2107+
20912108
// Handle /dwodir
20922109
config->dwoDir = args.getLastArgValue(OPT_dwodir);
20932110

‎lld/COFF/LTO.cpp

Lines changed: 10 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -110,7 +110,16 @@ BitcodeCompiler::BitcodeCompiler(COFFLinkerContext &c) : ctx(c) {
110110

111111
// Initialize ltoObj.
112112
lto::ThinBackend backend;
113-
if (ctx.config.thinLTOIndexOnly) {
113+
if (!ctx.config.dtltoDistributor.empty()) {
114+
backend = lto::createOutOfProcessThinBackend(
115+
llvm::hardware_concurrency(ctx.config.thinLTOJobs),
116+
/*OnWrite=*/nullptr,
117+
/*ShouldEmitIndexFiles=*/false,
118+
/*ShouldEmitImportFiles=*/false, ctx.config.outputFile,
119+
ctx.config.dtltoDistributor, ctx.config.dtltoDistributorArgs,
120+
ctx.config.dtltoCompiler, ctx.config.dtltoCompilerArgs,
121+
!ctx.config.saveTempsArgs.empty());
122+
} else if (ctx.config.thinLTOIndexOnly) {
114123
auto OnIndexWrite = [&](StringRef S) { thinIndices.erase(S); };
115124
backend = lto::createWriteIndexesThinBackend(
116125
llvm::hardware_concurrency(ctx.config.thinLTOJobs),

‎lld/COFF/Options.td

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -270,6 +270,17 @@ def thinlto_object_suffix_replace : P<
270270
def thinlto_prefix_replace: P<
271271
"thinlto-prefix-replace",
272272
"'old;new' replace old prefix with new prefix in ThinLTO outputs">;
273+
def thinlto_distributor : P<"thinlto-distributor",
274+
"Distributor to use for ThinLTO backend compilations. If specified, ThinLTO "
275+
"backend compilations will be distributed">;
276+
def thinlto_distributor_arg : P<"thinlto-distributor-arg",
277+
"Arguments to pass to the ThinLTO distributor">;
278+
def thinlto_compiler : P<"thinlto-remote-compiler",
279+
"Compiler for the ThinLTO distributor to invoke for ThinLTO backend "
280+
"compilations">;
281+
def thinlto_compiler_arg : P<"thinlto-remote-compiler-arg",
282+
"Compiler arguments for the ThinLTO distributor to pass for ThinLTO backend "
283+
"compilations">;
273284
def lto_obj_path : P<
274285
"lto-obj-path",
275286
"output native object for merged LTO unit to this path">;

‎lld/docs/DTLTO.rst

Lines changed: 35 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -7,8 +7,7 @@ during the traditional link step.
77

88
The implementation is documented here: https://llvm.org/docs/DTLTO.html.
99

10-
Currently, DTLTO is only supported in ELF LLD. Support will be added to other
11-
LLD flavours in the future.
10+
Currently, DTLTO is only supported in ELF and COFF LLD.
1211

1312
ELF LLD
1413
-------
@@ -40,3 +39,37 @@ The command-line interface is as follows:
4039
Some LLD LTO options (e.g., ``--lto-sample-profile=<file>``) are supported.
4140
Currently, other options are silently accepted but do not have the intended
4241
effect. Support for such options will be expanded in the future.
42+
43+
COFF LLD
44+
--------
45+
46+
The command-line interface is as follows:
47+
48+
- ``/thinlto-distributor:<path>``
49+
Specifies the file to execute as the distributor process. If specified,
50+
ThinLTO backend compilations will be distributed.
51+
52+
- ``/thinlto-remote-compiler:<path>``
53+
Specifies the path to the compiler that the distributor process will use for
54+
backend compilations. The compiler invoked must match the version of LLD.
55+
56+
- ``/thinlto-distributor-arg:<arg>``
57+
Specifies ``<arg>`` on the command line when invoking the distributor.
58+
Can be specified multiple times.
59+
60+
- ``/thinlto-remote-compiler-arg:<arg>``
61+
Appends ``<arg>`` to the remote compiler's command line.
62+
Can be specified multiple times.
63+
64+
Options that introduce extra input/output files may cause miscompilation if
65+
the distribution system does not automatically handle pushing/fetching them to
66+
remote nodes. In such cases, configure the distributor - possibly using
67+
``/thinlto-distributor-arg:`` - to manage these dependencies. See the
68+
distributor documentation for details.
69+
70+
Some LLD LTO options (e.g., ``/lto-sample-profile:<file>``) are supported.
71+
Currently, other options are silently accepted but do not have the intended
72+
effect. Support for such options could be expanded in the future.
73+
74+
Currently, there is no DTLTO command line interface supplied for ``clang-cl``,
75+
as users are expected to invoke LLD directly.

‎lld/test/COFF/dtlto/files.test

Lines changed: 71 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,71 @@
1+
REQUIRES: x86
2+
3+
## Test that the LLD options /lldsavetemps and -thinlto-emit-imports-files
4+
## function correctly with DTLTO we also check that index files
5+
## (-thinlto-emit-index-files) are not emitted with DTLTO.
6+
7+
RUN: rm -rf %t && split-file %s %t && cd %t
8+
9+
RUN: sed 's/@t1/@t2/g' t1.ll > t2.ll
10+
11+
## Generate ThinLTO bitcode files. Note that t3.bc will not be used by the
12+
## linker.
13+
RUN: opt -thinlto-bc t1.ll -o t1.bc
14+
RUN: opt -thinlto-bc t2.ll -o t2.bc
15+
RUN: cp t1.bc t3.bc
16+
17+
## Generate object files for mock.py to return.
18+
RUN: llc t1.ll --filetype=obj -o t1.obj
19+
RUN: llc t2.ll --filetype=obj -o t2.obj
20+
21+
## Create response file containing shared ThinLTO linker arguments.
22+
## -start-lib/-end-lib is used to test the special case where unused lazy
23+
## bitcode inputs result in empty index/imports files.
24+
## Note that mock.py does not do any compilation; instead, it simply writes
25+
## the contents of the object files supplied on the command line into the
26+
## output object files in job order.
27+
RUN: echo "/entry:t1 /subsystem:console \
28+
RUN: t1.bc t2.bc -start-lib t3.bc -end-lib /out:my.exe \
29+
RUN: -thinlto-distributor:\"%python\" \
30+
RUN: -thinlto-distributor-arg:\"%llvm_src_root/utils/dtlto/mock.py\" \
31+
RUN: -thinlto-distributor-arg:t1.obj \
32+
RUN: -thinlto-distributor-arg:t2.obj \
33+
RUN: -thinlto-remote-compiler:fake.exe" > l.rsp
34+
35+
## Check that without extra flags, no index/imports files are produced and
36+
## backend temp files are removed.
37+
RUN: lld-link @l.rsp
38+
RUN: ls | FileCheck %s \
39+
RUN: --check-prefixes=NOBACKEND,NOOTHERS
40+
41+
## Check that with /lldsavetemps and -thinlto-emit-imports-files backend
42+
## tempoary files are retained and no index/imports files are produced.
43+
RUN: rm -f *.imports *.thinlto.bc
44+
RUN: lld-link @l.rsp /lldsavetemps -thinlto-emit-imports-files
45+
RUN: ls | sort | FileCheck %s \
46+
RUN: --check-prefixes=BACKEND,NOOTHERS
47+
48+
## JSON jobs description, retained with --save-temps.
49+
## Note that DTLTO temporary files include a PID component.
50+
NOBACKEND-NOT: {{^}}my.[[#]].dist-file.json{{$}}
51+
BACKEND: {{^}}my.[[#]].dist-file.json{{$}}
52+
53+
## Index/imports files for t1.bc.
54+
NOOTHERS-NOT: {{^}}t1.bc.imports{{$}}
55+
NOOTHERS-NOT: {{^}}t1.bc.thinlto.bc{{$}}
56+
57+
## Index/imports files for t2.bc.
58+
NOOTHERS-NOT: {{^}}t2.bc.imports{{$}}
59+
NOOTHERS-NOT: {{^}}t2.bc.thinlto.bc{{$}}
60+
61+
## Empty index/imports files for unused t3.bc.
62+
NOOTHERS-NOT: {{^}}t3.bc.imports{{$}}
63+
NOOTHERS-NOT: {{^}}t3.bc.thinlto.bc{{$}}
64+
65+
#--- t1.ll
66+
target datalayout = "e-m:w-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
67+
target triple = "x86_64-pc-windows-msvc"
68+
69+
define void @t1() {
70+
ret void
71+
}

0 commit comments

Comments
 (0)