From 23325caf43d16e8caf77f8d4855ff568b6d143be Mon Sep 17 00:00:00 2001 From: Rich Kadel Date: Fri, 26 Mar 2021 13:02:46 -0700 Subject: [PATCH 1/3] Make rust-demangler installable Adds bootstrap rules to support installing rust-demangler. When compiling with `-Z instrument-coverage`, the coverage reports are generated by `llvm-cov`. `llvm-cov` includes a built-in demangler for C++, and an option to supply an alternate demangler. For Rust, we have `rust-demangler`, currently used in `rustc` coverage tests. Fuchsia's toolchain for Rust is built via `./x.py install`. Fuchsia is adding support for Rust coverage, and we need to include the `rust-demangler` in the installed `bin` directory. Configured rust-demangler as an in-tree extended tool. Added tests to support `./x.py test rust-demangler`. Install with extended tools by default only if `profiler = true`. --- config.toml.example | 9 +- src/bootstrap/builder.rs | 3 + src/bootstrap/dist.rs | 86 +++++++++++++++++++ src/bootstrap/install.rs | 20 +++++ src/bootstrap/tarball.rs | 5 ++ src/bootstrap/test.rs | 53 +++++++++++- src/bootstrap/tool.rs | 2 +- src/tools/rust-demangler/Cargo.toml | 6 +- src/tools/rust-demangler/README.md | 29 +++++++ src/tools/rust-demangler/src/lib.rs | 22 +++++ src/tools/rust-demangler/{ => src}/main.rs | 40 +-------- src/tools/rust-demangler/tests/lib.rs | 97 ++++++++++++++++++++++ 12 files changed, 328 insertions(+), 44 deletions(-) create mode 100644 src/tools/rust-demangler/README.md create mode 100644 src/tools/rust-demangler/src/lib.rs rename src/tools/rust-demangler/{ => src}/main.rs (76%) create mode 100644 src/tools/rust-demangler/tests/lib.rs diff --git a/config.toml.example b/config.toml.example index ee06e1bd0ba1b..d8b550d4c7423 100644 --- a/config.toml.example +++ b/config.toml.example @@ -259,10 +259,11 @@ changelog-seen = 2 # be built if `extended = true`. #extended = false -# Installs chosen set of extended tools if `extended = true`. By default builds all. -# If chosen tool failed to build the installation fails. If `extended = false`, this -# option is ignored. -#tools = ["cargo", "rls", "clippy", "rustfmt", "analysis", "src"] +# Installs chosen set of extended tools if `extended = true`. By default builds +# all extended tools except `rust-demangler`, unless the target is also being +# built with `profiler = true`. If chosen tool failed to build the installation +# fails. If `extended = false`, this option is ignored. +#tools = ["cargo", "rls", "clippy", "rustfmt", "analysis", "src"] # + "rust-demangler" if `profiler` # Verbosity level: 0 == not verbose, 1 == verbose, 2 == very verbose #verbose = 0 diff --git a/src/bootstrap/builder.rs b/src/bootstrap/builder.rs index 22a1eb6370235..86f5949550418 100644 --- a/src/bootstrap/builder.rs +++ b/src/bootstrap/builder.rs @@ -420,6 +420,7 @@ impl<'a> Builder<'a> { test::Rustfmt, test::Miri, test::Clippy, + test::RustDemangler, test::CompiletestTest, test::RustdocJSStd, test::RustdocJSNotStd, @@ -466,6 +467,7 @@ impl<'a> Builder<'a> { dist::Rls, dist::RustAnalyzer, dist::Rustfmt, + dist::RustDemangler, dist::Clippy, dist::Miri, dist::LlvmTools, @@ -481,6 +483,7 @@ impl<'a> Builder<'a> { install::Rls, install::RustAnalyzer, install::Rustfmt, + install::RustDemangler, install::Clippy, install::Miri, install::Analysis, diff --git a/src/bootstrap/dist.rs b/src/bootstrap/dist.rs index 802b5c99500cc..34243b797174b 100644 --- a/src/bootstrap/dist.rs +++ b/src/bootstrap/dist.rs @@ -1246,6 +1246,50 @@ impl Step for Rustfmt { } } +#[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)] +pub struct RustDemangler { + pub compiler: Compiler, + pub target: TargetSelection, +} + +impl Step for RustDemangler { + type Output = GeneratedTarball; + const ONLY_HOSTS: bool = true; + + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + run.path("rust-demangler") + } + + fn make_run(run: RunConfig<'_>) { + run.builder.ensure(RustDemangler { + compiler: run.builder.compiler_for( + run.builder.top_stage, + run.builder.config.build, + run.target, + ), + target: run.target, + }); + } + + fn run(self, builder: &Builder<'_>) -> GeneratedTarball { + let compiler = self.compiler; + let target = self.target; + assert!(builder.config.extended); + + let rust_demangler = builder + .ensure(tool::RustDemangler { compiler, target, extra_features: Vec::new() }) + .expect("rust-demangler expected to build - in-tree tool"); + + // Prepare the image directory + let mut tarball = Tarball::new(builder, "rust-demangler", &target.triple); + tarball.set_overlay(OverlayKind::RustDemangler); + tarball.is_preview(true); + tarball.add_file(&rust_demangler, "bin", 0o755); + tarball.add_legal_and_readme_to("share/doc/rust-demangler"); + tarball.generate() + } +} + #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)] pub struct Extended { stage: u32, @@ -1282,6 +1326,14 @@ impl Step for Extended { let rustc_installer = builder.ensure(Rustc { compiler: builder.compiler(stage, target) }); let cargo_installer = builder.ensure(Cargo { compiler, target }); let rustfmt_installer = builder.ensure(Rustfmt { compiler, target }); + let profiler = builder.config.profiler_enabled(target); + let install_rust_demangler = + builder.config.tools.as_ref().map_or(profiler, |t| t.contains("rust-demangler")); + let rust_demangler_installer = if install_rust_demangler { + Some(builder.ensure(RustDemangler { compiler, target })) + } else { + None + }; let rls_installer = builder.ensure(Rls { compiler, target }); let rust_analyzer_installer = builder.ensure(RustAnalyzer { compiler, target }); let llvm_tools_installer = builder.ensure(LlvmTools { target }); @@ -1312,6 +1364,9 @@ impl Step for Extended { tarballs.push(clippy_installer); tarballs.extend(miri_installer.clone()); tarballs.extend(rustfmt_installer.clone()); + if let Some(rust_demangler_installer) = rust_demangler_installer { + tarballs.push(rust_demangler_installer); + } tarballs.extend(llvm_tools_installer); if let Some(analysis_installer) = analysis_installer { tarballs.push(analysis_installer); @@ -1413,6 +1468,9 @@ impl Step for Extended { prepare("rust-docs"); prepare("rust-std"); prepare("rust-analysis"); + if install_rust_demangler { + prepare("rust-demangler"); + } prepare("clippy"); if rls_installer.is_some() { @@ -1476,6 +1534,9 @@ impl Step for Extended { prepare("rustc"); prepare("cargo"); prepare("rust-analysis"); + if install_rust_demangler { + prepare("rust-demangler"); + } prepare("rust-docs"); prepare("rust-std"); prepare("clippy"); @@ -1620,6 +1681,25 @@ impl Step for Extended { .arg("-t") .arg(etc.join("msi/remove-duplicates.xsl")), ); + if install_rust_demangler { + builder.run( + Command::new(&heat) + .current_dir(&exe) + .arg("dir") + .arg("rust-demangler") + .args(&heat_flags) + .arg("-cg") + .arg("RustDemanglerGroup") + .arg("-dr") + .arg("RustDemangler") + .arg("-var") + .arg("var.RustDemanglerDir") + .arg("-out") + .arg(exe.join("RustDemanglerGroup.wxs")) + .arg("-t") + .arg(etc.join("msi/remove-duplicates.xsl")), + ); + } if miri_installer.is_some() { builder.run( Command::new(&heat) @@ -1715,6 +1795,9 @@ impl Step for Extended { candle("CargoGroup.wxs".as_ref()); candle("StdGroup.wxs".as_ref()); candle("ClippyGroup.wxs".as_ref()); + if install_rust_demangler { + candle("RustDemanglerGroup.wxs".as_ref()); + } if rls_installer.is_some() { candle("RlsGroup.wxs".as_ref()); } @@ -1761,6 +1844,9 @@ impl Step for Extended { if rust_analyzer_installer.is_some() { cmd.arg("RustAnalyzerGroup.wixobj"); } + if install_rust_demangler { + cmd.arg("RustDemanglerGroup.wixobj"); + } if miri_installer.is_some() { cmd.arg("MiriGroup.wixobj"); } diff --git a/src/bootstrap/install.rs b/src/bootstrap/install.rs index b427420d57795..e034e2a2f90ec 100644 --- a/src/bootstrap/install.rs +++ b/src/bootstrap/install.rs @@ -190,6 +190,26 @@ install!((self, builder, _config), ); } }; + RustDemangler, + "rust-demangler", + Self::should_build(_config), + only_hosts: true, + { + let profiler = builder.config.profiler_enabled(self.target); + let install_rust_demangler = + builder.config.tools.as_ref().map_or(profiler, |t| t.contains("rust-demangler")); + if install_rust_demangler { + let tarball = builder.ensure( + dist::RustDemangler { compiler: self.compiler, target: self.target } + ); + install_sh(builder, "rust-demangler", self.compiler.stage, Some(self.target), &tarball); + } else { + builder.info( + &format!("skipping Install RustDemangler stage{} ({})", + self.compiler.stage, self.target), + ); + } + }; Analysis, "analysis", Self::should_build(_config), only_hosts: false, { let tarball = builder.ensure(dist::Analysis { // Find the actual compiler (handling the full bootstrap option) which diff --git a/src/bootstrap/tarball.rs b/src/bootstrap/tarball.rs index 7fb03056f1bd3..4a5d5fbf04fa8 100644 --- a/src/bootstrap/tarball.rs +++ b/src/bootstrap/tarball.rs @@ -15,6 +15,7 @@ pub(crate) enum OverlayKind { Clippy, Miri, Rustfmt, + RustDemangler, RLS, RustAnalyzer, } @@ -47,6 +48,9 @@ impl OverlayKind { "src/tools/rustfmt/LICENSE-APACHE", "src/tools/rustfmt/LICENSE-MIT", ], + OverlayKind::RustDemangler => { + &["src/tools/rust-demangler/README.md", "LICENSE-APACHE", "LICENSE-MIT"] + } OverlayKind::RLS => &[ "src/tools/rls/README.md", "src/tools/rls/LICENSE-APACHE", @@ -64,6 +68,7 @@ impl OverlayKind { match self { OverlayKind::Rust => builder.rust_version(), OverlayKind::LLVM => builder.rust_version(), + OverlayKind::RustDemangler => builder.rust_version(), OverlayKind::Cargo => { builder.cargo_info.version(builder, &builder.release_num("cargo")) } diff --git a/src/bootstrap/test.rs b/src/bootstrap/test.rs index adb0a372c64dd..7425dcb52c0f4 100644 --- a/src/bootstrap/test.rs +++ b/src/bootstrap/test.rs @@ -351,6 +351,54 @@ impl Step for Rustfmt { } } +#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)] +pub struct RustDemangler { + stage: u32, + host: TargetSelection, +} + +impl Step for RustDemangler { + type Output = (); + const ONLY_HOSTS: bool = true; + + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + run.path("src/tools/rust-demangler") + } + + fn make_run(run: RunConfig<'_>) { + run.builder.ensure(RustDemangler { stage: run.builder.top_stage, host: run.target }); + } + + /// Runs `cargo test` for rust-demangler. + fn run(self, builder: &Builder<'_>) { + let stage = self.stage; + let host = self.host; + let compiler = builder.compiler(stage, host); + + let rust_demangler = builder + .ensure(tool::RustDemangler { compiler, target: self.host, extra_features: Vec::new() }) + .expect("in-tree tool"); + let mut cargo = tool::prepare_tool_cargo( + builder, + compiler, + Mode::ToolRustc, + host, + "test", + "src/tools/rust-demangler", + SourceType::InTree, + &[], + ); + + let dir = testdir(builder, compiler.host); + t!(fs::create_dir_all(&dir)); + + cargo.env("RUST_DEMANGLER_DRIVER_PATH", rust_demangler); + cargo.add_rustc_lib_path(builder, compiler); + + builder.run(&mut cargo.into()); + } +} + #[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)] pub struct Miri { stage: u32, @@ -1126,7 +1174,10 @@ note: if you're sure you want to do this, please open an issue as to why. In the } if mode == "run-make" && suite.ends_with("fulldeps") { - cmd.arg("--rust-demangler-path").arg(builder.tool_exe(Tool::RustDemangler)); + let rust_demangler = builder + .ensure(tool::RustDemangler { compiler, target, extra_features: Vec::new() }) + .expect("in-tree tool"); + cmd.arg("--rust-demangler-path").arg(rust_demangler); } cmd.arg("--src-base").arg(builder.src.join("src/test").join(suite)); diff --git a/src/bootstrap/tool.rs b/src/bootstrap/tool.rs index 3fc3b68fd8681..bfb846f3b562f 100644 --- a/src/bootstrap/tool.rs +++ b/src/bootstrap/tool.rs @@ -368,7 +368,6 @@ bootstrap_tool!( Compiletest, "src/tools/compiletest", "compiletest", is_unstable_tool = true; BuildManifest, "src/tools/build-manifest", "build-manifest"; RemoteTestClient, "src/tools/remote-test-client", "remote-test-client"; - RustDemangler, "src/tools/rust-demangler", "rust-demangler"; RustInstaller, "src/tools/rust-installer", "fabricate", is_external_tool = true; RustdocTheme, "src/tools/rustdoc-themes", "rustdoc-themes"; ExpandYamlAnchors, "src/tools/expand-yaml-anchors", "expand-yaml-anchors"; @@ -719,6 +718,7 @@ tool_extended!((self, builder), }); self.extra_features.push("clippy".to_owned()); }; + RustDemangler, rust_demangler, "src/tools/rust-demangler", "rust-demangler", stable=false, in_tree=true, {}; Rustfmt, rustfmt, "src/tools/rustfmt", "rustfmt", stable=true, {}; RustAnalyzer, rust_analyzer, "src/tools/rust-analyzer/crates/rust-analyzer", "rust-analyzer", stable=false, {}; ); diff --git a/src/tools/rust-demangler/Cargo.toml b/src/tools/rust-demangler/Cargo.toml index ac684a3c47e42..b7bc11253191b 100644 --- a/src/tools/rust-demangler/Cargo.toml +++ b/src/tools/rust-demangler/Cargo.toml @@ -8,6 +8,10 @@ edition = "2018" regex = "1.0" rustc-demangle = "0.1.17" +[lib] +name = "rust_demangler" +doctest = false + [[bin]] name = "rust-demangler" -path = "main.rs" +test = false diff --git a/src/tools/rust-demangler/README.md b/src/tools/rust-demangler/README.md new file mode 100644 index 0000000000000..a45f5c4573250 --- /dev/null +++ b/src/tools/rust-demangler/README.md @@ -0,0 +1,29 @@ +# rust-demangler + +Demangles rustc mangled names. + +This tool uses the [rustc-demangle](https://crates.io/crates/rustc-demangle) +crate to convert an input buffer of newline-separated mangled names into their +demangled translations. + +This tool takes a list of mangled names (one per line) on standard input, and +prints a corresponding list of demangled names. The tool is designed to support +programs that can leverage a third-party demangler, such as `llvm-cov`, via the +`-Xdemangler=` option. + +To use `rust-demangler` with `llvm-cov` for example, add the `-Xdemangler=...` +option: + +```shell +$ TARGET="${PWD}/build/x86_64-unknown-linux-gnu" +$ "${TARGET}"/llvm/bin/llvm-cov show \ + --Xdemangler=path/to/rust-demangler \ + --instr-profile=main.profdata ./main --show-line-counts-or-regions +``` + +## License + +Rust-demangler is distributed under the terms of both the MIT license and the +Apache License (Version 2.0). + +See [LICENSE-APACHE](/LICENSE-APACHE) and [LICENSE-MIT](/LICENSE-MIT) for details. diff --git a/src/tools/rust-demangler/src/lib.rs b/src/tools/rust-demangler/src/lib.rs new file mode 100644 index 0000000000000..4d2911ee7f81a --- /dev/null +++ b/src/tools/rust-demangler/src/lib.rs @@ -0,0 +1,22 @@ +use regex::Regex; +use rustc_demangle::demangle; + +const REPLACE_COLONS: &str = "::"; + +pub fn create_disambiguator_re() -> Regex { + Regex::new(r"\[[a-f0-9]{5,16}\]::").unwrap() +} + +pub fn demangle_lines(buffer: &str, strip_crate_disambiguators: Option) -> Vec { + let lines = buffer.lines(); + let mut demangled_lines = Vec::new(); + for mangled in lines { + let mut demangled = demangle(mangled).to_string(); + if let Some(re) = &strip_crate_disambiguators { + demangled = re.replace_all(&demangled, REPLACE_COLONS).to_string(); + } + demangled_lines.push(demangled); + } + demangled_lines.push("".to_string()); + demangled_lines +} diff --git a/src/tools/rust-demangler/main.rs b/src/tools/rust-demangler/src/main.rs similarity index 76% rename from src/tools/rust-demangler/main.rs rename to src/tools/rust-demangler/src/main.rs index fd031ccb25249..a87369d7baab4 100644 --- a/src/tools/rust-demangler/main.rs +++ b/src/tools/rust-demangler/src/main.rs @@ -1,27 +1,5 @@ //! Demangles rustc mangled names. //! -//! This tool uses https://crates.io/crates/rustc-demangle to convert an input buffer of -//! newline-separated mangled names into their demangled translations. -//! -//! This tool can be leveraged by other applications that support third-party demanglers. -//! It takes a list of mangled names (one per line) on standard input, and prints a corresponding -//! list of demangled names. The tool is designed to support other programs that can leverage a -//! third-party demangler, such as `llvm-cov`, via the `-Xdemangler=` option. -//! -//! To use `rust-demangler`, first build the tool with: -//! -//! ```shell -//! $ ./x.py build rust-demangler -//! ``` -//! -//! Then, with `llvm-cov` for example, add the `-Xdemangler=...` option: -//! -//! ```shell -//! $ TARGET="${PWD}/build/x86_64-unknown-linux-gnu" -//! $ "${TARGET}"/llvm/bin/llvm-cov show --Xdemangler="${TARGET}"/stage0-tools-bin/rust-demangler \ -//! --instr-profile=main.profdata ./main --show-line-counts-or-regions -//! ``` -//! //! Note regarding crate disambiguators: //! //! Some demangled symbol paths can include "crate disambiguator" suffixes, represented as a large @@ -57,12 +35,9 @@ //! These disambiguators seem to have more analytical value (for instance, in coverage analysis), so //! they are not removed. -use regex::Regex; -use rustc_demangle::demangle; +use rust_demangler::*; use std::io::{self, Read, Write}; -const REPLACE_COLONS: &str = "::"; - fn main() -> io::Result<()> { // FIXME(richkadel): In Issue #77615 discussed updating the `rustc-demangle` library, to provide // an option to generate demangled names without including crate disambiguators. If that @@ -82,7 +57,7 @@ fn main() -> io::Result<()> { // and more than three leading zeros should be extremely unlikely. Conversely, it should be // sufficient to assume the zero-based indexes for closures and anonymous scopes will never // exceed the value 9999. - let mut strip_crate_disambiguators = Some(Regex::new(r"\[[a-f0-9]{5,16}\]::").unwrap()); + let mut strip_crate_disambiguators = Some(create_disambiguator_re()); let mut args = std::env::args(); let progname = args.next().unwrap(); @@ -115,16 +90,7 @@ fn main() -> io::Result<()> { let mut buffer = String::new(); io::stdin().read_to_string(&mut buffer)?; - let lines = buffer.lines(); - let mut demangled_lines = Vec::new(); - for mangled in lines { - let mut demangled = demangle(mangled).to_string(); - if let Some(re) = &strip_crate_disambiguators { - demangled = re.replace_all(&demangled, REPLACE_COLONS).to_string(); - } - demangled_lines.push(demangled); - } - demangled_lines.push("".to_string()); + let demangled_lines = demangle_lines(&buffer, strip_crate_disambiguators); io::stdout().write_all(demangled_lines.join("\n").as_bytes())?; Ok(()) } diff --git a/src/tools/rust-demangler/tests/lib.rs b/src/tools/rust-demangler/tests/lib.rs new file mode 100644 index 0000000000000..8a1647601e407 --- /dev/null +++ b/src/tools/rust-demangler/tests/lib.rs @@ -0,0 +1,97 @@ +use rust_demangler::*; + +const MANGLED_LINES: &str = r" +_RNvC6_123foo3bar +_RNqCs4fqI2P2rA04_11utf8_identsu30____7hkackfecea1cbdathfdh9hlq6y +_RNCNCNgCs6DXkGYLi8lr_2cc5spawn00B5_ +_RNCINkXs25_NgCsbmNqQUJIY6D_4core5sliceINyB9_4IterhENuNgNoBb_4iter8iterator8Iterator9rpositionNCNgNpB9_6memchr7memrchrs_0E0Bb_ +_RINbNbCskIICzLVDPPb_5alloc5alloc8box_freeDINbNiB4_5boxed5FnBoxuEp6OutputuEL_ECs1iopQbuBiw2_3std +INtC8arrayvec8ArrayVechKj7b_E +_RMCs4fqI2P2rA04_13const_genericINtB0_8UnsignedKhb_E +_RMCs4fqI2P2rA04_13const_genericINtB0_6SignedKs98_E +_RMCs4fqI2P2rA04_13const_genericINtB0_6SignedKanb_E +_RMCs4fqI2P2rA04_13const_genericINtB0_4BoolKb0_E +_RMCs4fqI2P2rA04_13const_genericINtB0_4BoolKb1_E +_RMCs4fqI2P2rA04_13const_genericINtB0_4CharKc76_E +_RMCs4fqI2P2rA04_13const_genericINtB0_4CharKca_E +_RMCs4fqI2P2rA04_13const_genericINtB0_4CharKc2202_E +_RNvNvMCs4fqI2P2rA04_13const_genericINtB4_3FooKpE3foo3FOO +_RC3foo.llvm.9D1C9369 +_RC3foo.llvm.9D1C9369@@16 +_RNvC9backtrace3foo.llvm.A5310EB9 +_RNvNtNtNtNtCs92dm3009vxr_4rand4rngs7adapter9reseeding4fork23FORK_HANDLER_REGISTERED.0.0 +"; + +#[test] +fn test_demangle_lines() { + let demangled_lines = demangle_lines(MANGLED_LINES, None); + let mut iter = demangled_lines.iter(); + assert_eq!("", iter.next().unwrap()); + assert_eq!("123foo[0]::bar", iter.next().unwrap()); + assert_eq!("utf8_idents[317d481089b8c8fe]::საჭმელად_გემრიელი_სადილი", iter.next().unwrap()); + assert_eq!("cc[4d6468d6c9fd4bb3]::spawn::{closure#0}::{closure#0}", iter.next().unwrap()); + assert_eq!( + " as core[846817f741e54dfd]::iter::iterator::Iterator>::rposition::::{closure#0}", + iter.next().unwrap() + ); + assert_eq!( + "alloc[f15a878b47eb696b]::alloc::box_free::>", + iter.next().unwrap() + ); + assert_eq!("INtC8arrayvec8ArrayVechKj7b_E", iter.next().unwrap()); + assert_eq!(">", iter.next().unwrap()); + assert_eq!(">", iter.next().unwrap()); + assert_eq!(">", iter.next().unwrap()); + assert_eq!(">", iter.next().unwrap()); + assert_eq!(">", iter.next().unwrap()); + assert_eq!(">", iter.next().unwrap()); + assert_eq!(">", iter.next().unwrap()); + assert_eq!(">", iter.next().unwrap()); + assert_eq!(">::foo::FOO", iter.next().unwrap()); + assert_eq!("foo[0]", iter.next().unwrap()); + assert_eq!("foo[0]", iter.next().unwrap()); + assert_eq!("backtrace[0]::foo", iter.next().unwrap()); + assert_eq!( + "rand[693ea8e72247470f]::rngs::adapter::reseeding::fork::FORK_HANDLER_REGISTERED.0.0", + iter.next().unwrap() + ); + assert_eq!("", iter.next().unwrap()); + assert!(iter.next().is_none()); +} + +#[test] +fn test_demangle_lines_no_crate_disambiguators() { + let demangled_lines = demangle_lines(MANGLED_LINES, Some(create_disambiguator_re())); + let mut iter = demangled_lines.iter(); + assert_eq!("", iter.next().unwrap()); + assert_eq!("123foo[0]::bar", iter.next().unwrap()); + assert_eq!("utf8_idents::საჭმელად_გემრიელი_სადილი", iter.next().unwrap()); + assert_eq!("cc::spawn::{closure#0}::{closure#0}", iter.next().unwrap()); + assert_eq!( + " as core::iter::iterator::Iterator>::rposition::::{closure#0}", + iter.next().unwrap() + ); + assert_eq!( + "alloc::alloc::box_free::>", + iter.next().unwrap() + ); + assert_eq!("INtC8arrayvec8ArrayVechKj7b_E", iter.next().unwrap()); + assert_eq!(">", iter.next().unwrap()); + assert_eq!(">", iter.next().unwrap()); + assert_eq!(">", iter.next().unwrap()); + assert_eq!(">", iter.next().unwrap()); + assert_eq!(">", iter.next().unwrap()); + assert_eq!(">", iter.next().unwrap()); + assert_eq!(">", iter.next().unwrap()); + assert_eq!(">", iter.next().unwrap()); + assert_eq!(">::foo::FOO", iter.next().unwrap()); + assert_eq!("foo[0]", iter.next().unwrap()); + assert_eq!("foo[0]", iter.next().unwrap()); + assert_eq!("backtrace[0]::foo", iter.next().unwrap()); + assert_eq!( + "rand::rngs::adapter::reseeding::fork::FORK_HANDLER_REGISTERED.0.0", + iter.next().unwrap() + ); + assert_eq!("", iter.next().unwrap()); + assert!(iter.next().is_none()); +} From c2a8bfe0ab54fa2a2682e05e59bc96d9363760f0 Mon Sep 17 00:00:00 2001 From: Rich Kadel Date: Mon, 29 Mar 2021 08:28:52 -0700 Subject: [PATCH 2/3] Changed function signature to keep buffer handling out of lib --- src/tools/rust-demangler/src/lib.rs | 5 ++--- src/tools/rust-demangler/src/main.rs | 3 ++- src/tools/rust-demangler/tests/lib.rs | 8 +++----- 3 files changed, 7 insertions(+), 9 deletions(-) diff --git a/src/tools/rust-demangler/src/lib.rs b/src/tools/rust-demangler/src/lib.rs index 4d2911ee7f81a..1d972229d9536 100644 --- a/src/tools/rust-demangler/src/lib.rs +++ b/src/tools/rust-demangler/src/lib.rs @@ -1,5 +1,6 @@ use regex::Regex; use rustc_demangle::demangle; +use std::str::Lines; const REPLACE_COLONS: &str = "::"; @@ -7,8 +8,7 @@ pub fn create_disambiguator_re() -> Regex { Regex::new(r"\[[a-f0-9]{5,16}\]::").unwrap() } -pub fn demangle_lines(buffer: &str, strip_crate_disambiguators: Option) -> Vec { - let lines = buffer.lines(); +pub fn demangle_lines(lines: Lines<'_>, strip_crate_disambiguators: Option) -> Vec { let mut demangled_lines = Vec::new(); for mangled in lines { let mut demangled = demangle(mangled).to_string(); @@ -17,6 +17,5 @@ pub fn demangle_lines(buffer: &str, strip_crate_disambiguators: Option) - } demangled_lines.push(demangled); } - demangled_lines.push("".to_string()); demangled_lines } diff --git a/src/tools/rust-demangler/src/main.rs b/src/tools/rust-demangler/src/main.rs index a87369d7baab4..1b5ef5d2442ba 100644 --- a/src/tools/rust-demangler/src/main.rs +++ b/src/tools/rust-demangler/src/main.rs @@ -90,7 +90,8 @@ fn main() -> io::Result<()> { let mut buffer = String::new(); io::stdin().read_to_string(&mut buffer)?; - let demangled_lines = demangle_lines(&buffer, strip_crate_disambiguators); + let mut demangled_lines = demangle_lines(buffer.lines(), strip_crate_disambiguators); + demangled_lines.push("".to_string()); // ensure a trailing newline io::stdout().write_all(demangled_lines.join("\n").as_bytes())?; Ok(()) } diff --git a/src/tools/rust-demangler/tests/lib.rs b/src/tools/rust-demangler/tests/lib.rs index 8a1647601e407..09752fd72123b 100644 --- a/src/tools/rust-demangler/tests/lib.rs +++ b/src/tools/rust-demangler/tests/lib.rs @@ -1,6 +1,6 @@ use rust_demangler::*; -const MANGLED_LINES: &str = r" +const MANGLED_INPUT: &str = r" _RNvC6_123foo3bar _RNqCs4fqI2P2rA04_11utf8_identsu30____7hkackfecea1cbdathfdh9hlq6y _RNCNCNgCs6DXkGYLi8lr_2cc5spawn00B5_ @@ -24,7 +24,7 @@ _RNvNtNtNtNtCs92dm3009vxr_4rand4rngs7adapter9reseeding4fork23FORK_HANDLER_REGIST #[test] fn test_demangle_lines() { - let demangled_lines = demangle_lines(MANGLED_LINES, None); + let demangled_lines = demangle_lines(MANGLED_INPUT.lines(), None); let mut iter = demangled_lines.iter(); assert_eq!("", iter.next().unwrap()); assert_eq!("123foo[0]::bar", iter.next().unwrap()); @@ -55,13 +55,12 @@ fn test_demangle_lines() { "rand[693ea8e72247470f]::rngs::adapter::reseeding::fork::FORK_HANDLER_REGISTERED.0.0", iter.next().unwrap() ); - assert_eq!("", iter.next().unwrap()); assert!(iter.next().is_none()); } #[test] fn test_demangle_lines_no_crate_disambiguators() { - let demangled_lines = demangle_lines(MANGLED_LINES, Some(create_disambiguator_re())); + let demangled_lines = demangle_lines(MANGLED_INPUT.lines(), Some(create_disambiguator_re())); let mut iter = demangled_lines.iter(); assert_eq!("", iter.next().unwrap()); assert_eq!("123foo[0]::bar", iter.next().unwrap()); @@ -92,6 +91,5 @@ fn test_demangle_lines_no_crate_disambiguators() { "rand::rngs::adapter::reseeding::fork::FORK_HANDLER_REGISTERED.0.0", iter.next().unwrap() ); - assert_eq!("", iter.next().unwrap()); assert!(iter.next().is_none()); } From ed89e6b831424ea68060c4f8b1145637d6ee15a5 Mon Sep 17 00:00:00 2001 From: Rich Kadel Date: Sat, 3 Apr 2021 17:47:32 -0700 Subject: [PATCH 3/3] Address review comments and Windows failure, and make cleaner --- src/bootstrap/dist.rs | 52 ++++++------ src/bootstrap/install.rs | 20 ++--- src/bootstrap/tarball.rs | 2 +- src/bootstrap/test.rs | 3 + src/tools/rust-demangler/README.md | 27 +++--- src/tools/rust-demangler/tests/lib.rs | 113 ++++++++++++-------------- 6 files changed, 108 insertions(+), 109 deletions(-) diff --git a/src/bootstrap/dist.rs b/src/bootstrap/dist.rs index 34243b797174b..38ebe0e52083d 100644 --- a/src/bootstrap/dist.rs +++ b/src/bootstrap/dist.rs @@ -1253,7 +1253,7 @@ pub struct RustDemangler { } impl Step for RustDemangler { - type Output = GeneratedTarball; + type Output = Option; const ONLY_HOSTS: bool = true; fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { @@ -1271,11 +1271,17 @@ impl Step for RustDemangler { }); } - fn run(self, builder: &Builder<'_>) -> GeneratedTarball { + fn run(self, builder: &Builder<'_>) -> Option { let compiler = self.compiler; let target = self.target; assert!(builder.config.extended); + // Only build this extended tool if explicitly included in `tools`, or if `profiler = true` + let profiler = builder.config.profiler_enabled(target); + if !builder.config.tools.as_ref().map_or(profiler, |t| t.contains("rust-demangler")) { + return None; + } + let rust_demangler = builder .ensure(tool::RustDemangler { compiler, target, extra_features: Vec::new() }) .expect("rust-demangler expected to build - in-tree tool"); @@ -1286,7 +1292,7 @@ impl Step for RustDemangler { tarball.is_preview(true); tarball.add_file(&rust_demangler, "bin", 0o755); tarball.add_legal_and_readme_to("share/doc/rust-demangler"); - tarball.generate() + Some(tarball.generate()) } } @@ -1326,14 +1332,7 @@ impl Step for Extended { let rustc_installer = builder.ensure(Rustc { compiler: builder.compiler(stage, target) }); let cargo_installer = builder.ensure(Cargo { compiler, target }); let rustfmt_installer = builder.ensure(Rustfmt { compiler, target }); - let profiler = builder.config.profiler_enabled(target); - let install_rust_demangler = - builder.config.tools.as_ref().map_or(profiler, |t| t.contains("rust-demangler")); - let rust_demangler_installer = if install_rust_demangler { - Some(builder.ensure(RustDemangler { compiler, target })) - } else { - None - }; + let rust_demangler_installer = builder.ensure(RustDemangler { compiler, target }); let rls_installer = builder.ensure(Rls { compiler, target }); let rust_analyzer_installer = builder.ensure(RustAnalyzer { compiler, target }); let llvm_tools_installer = builder.ensure(LlvmTools { target }); @@ -1359,14 +1358,12 @@ impl Step for Extended { let mut tarballs = Vec::new(); tarballs.push(rustc_installer); tarballs.push(cargo_installer); + tarballs.push(clippy_installer); + tarballs.extend(rust_demangler_installer.clone()); tarballs.extend(rls_installer.clone()); tarballs.extend(rust_analyzer_installer.clone()); - tarballs.push(clippy_installer); tarballs.extend(miri_installer.clone()); tarballs.extend(rustfmt_installer.clone()); - if let Some(rust_demangler_installer) = rust_demangler_installer { - tarballs.push(rust_demangler_installer); - } tarballs.extend(llvm_tools_installer); if let Some(analysis_installer) = analysis_installer { tarballs.push(analysis_installer); @@ -1421,6 +1418,9 @@ impl Step for Extended { let xform = |p: &Path| { let mut contents = t!(fs::read_to_string(p)); + if rust_demangler_installer.is_none() { + contents = filter(&contents, "rust-demangler"); + } if rls_installer.is_none() { contents = filter(&contents, "rls"); } @@ -1468,11 +1468,10 @@ impl Step for Extended { prepare("rust-docs"); prepare("rust-std"); prepare("rust-analysis"); - if install_rust_demangler { + prepare("clippy"); + if rust_demangler_installer.is_some() { prepare("rust-demangler"); } - prepare("clippy"); - if rls_installer.is_some() { prepare("rls"); } @@ -1520,6 +1519,8 @@ impl Step for Extended { "rust-analyzer-preview".to_string() } else if name == "clippy" { "clippy-preview".to_string() + } else if name == "rust-demangler" { + "rust-demangler-preview".to_string() } else if name == "miri" { "miri-preview".to_string() } else { @@ -1534,12 +1535,12 @@ impl Step for Extended { prepare("rustc"); prepare("cargo"); prepare("rust-analysis"); - if install_rust_demangler { - prepare("rust-demangler"); - } prepare("rust-docs"); prepare("rust-std"); prepare("clippy"); + if rust_demangler_installer.is_some() { + prepare("rust-demangler"); + } if rls_installer.is_some() { prepare("rls"); } @@ -1681,7 +1682,7 @@ impl Step for Extended { .arg("-t") .arg(etc.join("msi/remove-duplicates.xsl")), ); - if install_rust_demangler { + if rust_demangler_installer.is_some() { builder.run( Command::new(&heat) .current_dir(&exe) @@ -1773,6 +1774,9 @@ impl Step for Extended { .arg(&input); add_env(builder, &mut cmd, target); + if rust_demangler_installer.is_some() { + cmd.arg("-dRustDemanglerDir=rust-demangler"); + } if rls_installer.is_some() { cmd.arg("-dRlsDir=rls"); } @@ -1795,7 +1799,7 @@ impl Step for Extended { candle("CargoGroup.wxs".as_ref()); candle("StdGroup.wxs".as_ref()); candle("ClippyGroup.wxs".as_ref()); - if install_rust_demangler { + if rust_demangler_installer.is_some() { candle("RustDemanglerGroup.wxs".as_ref()); } if rls_installer.is_some() { @@ -1844,7 +1848,7 @@ impl Step for Extended { if rust_analyzer_installer.is_some() { cmd.arg("RustAnalyzerGroup.wixobj"); } - if install_rust_demangler { + if rust_demangler_installer.is_some() { cmd.arg("RustDemanglerGroup.wixobj"); } if miri_installer.is_some() { diff --git a/src/bootstrap/install.rs b/src/bootstrap/install.rs index e034e2a2f90ec..68e7dc8006726 100644 --- a/src/bootstrap/install.rs +++ b/src/bootstrap/install.rs @@ -190,18 +190,14 @@ install!((self, builder, _config), ); } }; - RustDemangler, - "rust-demangler", - Self::should_build(_config), - only_hosts: true, - { - let profiler = builder.config.profiler_enabled(self.target); - let install_rust_demangler = - builder.config.tools.as_ref().map_or(profiler, |t| t.contains("rust-demangler")); - if install_rust_demangler { - let tarball = builder.ensure( - dist::RustDemangler { compiler: self.compiler, target: self.target } - ); + RustDemangler, "rust-demangler", Self::should_build(_config), only_hosts: true, { + // Note: Even though `should_build` may return true for `extended` default tools, + // dist::RustDemangler may still return None, unless the target-dependent `profiler` config + // is also true, or the `tools` array explicitly includes "rust-demangler". + if let Some(tarball) = builder.ensure(dist::RustDemangler { + compiler: self.compiler, + target: self.target + }) { install_sh(builder, "rust-demangler", self.compiler.stage, Some(self.target), &tarball); } else { builder.info( diff --git a/src/bootstrap/tarball.rs b/src/bootstrap/tarball.rs index 4a5d5fbf04fa8..b02d7e062a524 100644 --- a/src/bootstrap/tarball.rs +++ b/src/bootstrap/tarball.rs @@ -68,7 +68,7 @@ impl OverlayKind { match self { OverlayKind::Rust => builder.rust_version(), OverlayKind::LLVM => builder.rust_version(), - OverlayKind::RustDemangler => builder.rust_version(), + OverlayKind::RustDemangler => builder.release_num("rust-demangler"), OverlayKind::Cargo => { builder.cargo_info.version(builder, &builder.release_num("cargo")) } diff --git a/src/bootstrap/test.rs b/src/bootstrap/test.rs index 7425dcb52c0f4..69d39f5e544c1 100644 --- a/src/bootstrap/test.rs +++ b/src/bootstrap/test.rs @@ -393,6 +393,9 @@ impl Step for RustDemangler { t!(fs::create_dir_all(&dir)); cargo.env("RUST_DEMANGLER_DRIVER_PATH", rust_demangler); + + cargo.arg("--").args(builder.config.cmd.test_args()); + cargo.add_rustc_lib_path(builder, compiler); builder.run(&mut cargo.into()); diff --git a/src/tools/rust-demangler/README.md b/src/tools/rust-demangler/README.md index a45f5c4573250..4e8a689a13a47 100644 --- a/src/tools/rust-demangler/README.md +++ b/src/tools/rust-demangler/README.md @@ -1,18 +1,15 @@ # rust-demangler -Demangles rustc mangled names. +_Demangles rustc mangled names._ -This tool uses the [rustc-demangle](https://crates.io/crates/rustc-demangle) -crate to convert an input buffer of newline-separated mangled names into their -demangled translations. +`rust-demangler` supports the requirements of the [`llvm-cov show -Xdemangler` +option](https://llvm.org/docs/CommandGuide/llvm-cov.html#cmdoption-llvm-cov-show-xdemangler), +to perform Rust-specific symbol demangling: -This tool takes a list of mangled names (one per line) on standard input, and -prints a corresponding list of demangled names. The tool is designed to support -programs that can leverage a third-party demangler, such as `llvm-cov`, via the -`-Xdemangler=` option. +> _The demangler is expected to read a newline-separated list of symbols from +> stdin and write a newline-separated list of the same length to stdout._ -To use `rust-demangler` with `llvm-cov` for example, add the `-Xdemangler=...` -option: +To use `rust-demangler` with `llvm-cov` for example: ```shell $ TARGET="${PWD}/build/x86_64-unknown-linux-gnu" @@ -21,6 +18,16 @@ $ "${TARGET}"/llvm/bin/llvm-cov show \ --instr-profile=main.profdata ./main --show-line-counts-or-regions ``` +`rust-demangler` is a Rust "extended tool", used in Rust compiler tests, and +optionally included in Rust distributions that enable coverage profiling. Symbol +demangling is implemented using the +[rustc-demangle](https://crates.io/crates/rustc-demangle) crate. + +_(Note, for Rust developers, the third-party tool +[`rustfilt`](https://crates.io/crates/rustfilt) also supports `llvm-cov` symbol +demangling. `rustfilt` is a more generalized tool that searches any body of +text, using pattern matching, to find and demangle Rust symbols.)_ + ## License Rust-demangler is distributed under the terms of both the MIT license and the diff --git a/src/tools/rust-demangler/tests/lib.rs b/src/tools/rust-demangler/tests/lib.rs index 09752fd72123b..5a67b42322535 100644 --- a/src/tools/rust-demangler/tests/lib.rs +++ b/src/tools/rust-demangler/tests/lib.rs @@ -22,74 +22,63 @@ _RNvC9backtrace3foo.llvm.A5310EB9 _RNvNtNtNtNtCs92dm3009vxr_4rand4rngs7adapter9reseeding4fork23FORK_HANDLER_REGISTERED.0.0 "; +const DEMANGLED_OUTPUT: &str = r" +123foo[0]::bar +utf8_idents[317d481089b8c8fe]::საჭმელად_გემრიელი_სადილი +cc[4d6468d6c9fd4bb3]::spawn::{closure#0}::{closure#0} + as core[846817f741e54dfd]::iter::iterator::Iterator>::rposition::::{closure#0} +alloc[f15a878b47eb696b]::alloc::box_free::> +INtC8arrayvec8ArrayVechKj7b_E +> +> +> +> +> +> +> +> +>::foo::FOO +foo[0] +foo[0] +backtrace[0]::foo +rand[693ea8e72247470f]::rngs::adapter::reseeding::fork::FORK_HANDLER_REGISTERED.0.0 +"; + +const DEMANGLED_OUTPUT_NO_CRATE_DISAMBIGUATORS: &str = r" +123foo[0]::bar +utf8_idents::საჭმელად_გემრიელი_სადილი +cc::spawn::{closure#0}::{closure#0} + as core::iter::iterator::Iterator>::rposition::::{closure#0} +alloc::alloc::box_free::> +INtC8arrayvec8ArrayVechKj7b_E +> +> +> +> +> +> +> +> +>::foo::FOO +foo[0] +foo[0] +backtrace[0]::foo +rand::rngs::adapter::reseeding::fork::FORK_HANDLER_REGISTERED.0.0 +"; + #[test] fn test_demangle_lines() { let demangled_lines = demangle_lines(MANGLED_INPUT.lines(), None); - let mut iter = demangled_lines.iter(); - assert_eq!("", iter.next().unwrap()); - assert_eq!("123foo[0]::bar", iter.next().unwrap()); - assert_eq!("utf8_idents[317d481089b8c8fe]::საჭმელად_გემრიელი_სადილი", iter.next().unwrap()); - assert_eq!("cc[4d6468d6c9fd4bb3]::spawn::{closure#0}::{closure#0}", iter.next().unwrap()); - assert_eq!( - " as core[846817f741e54dfd]::iter::iterator::Iterator>::rposition::::{closure#0}", - iter.next().unwrap() - ); - assert_eq!( - "alloc[f15a878b47eb696b]::alloc::box_free::>", - iter.next().unwrap() - ); - assert_eq!("INtC8arrayvec8ArrayVechKj7b_E", iter.next().unwrap()); - assert_eq!(">", iter.next().unwrap()); - assert_eq!(">", iter.next().unwrap()); - assert_eq!(">", iter.next().unwrap()); - assert_eq!(">", iter.next().unwrap()); - assert_eq!(">", iter.next().unwrap()); - assert_eq!(">", iter.next().unwrap()); - assert_eq!(">", iter.next().unwrap()); - assert_eq!(">", iter.next().unwrap()); - assert_eq!(">::foo::FOO", iter.next().unwrap()); - assert_eq!("foo[0]", iter.next().unwrap()); - assert_eq!("foo[0]", iter.next().unwrap()); - assert_eq!("backtrace[0]::foo", iter.next().unwrap()); - assert_eq!( - "rand[693ea8e72247470f]::rngs::adapter::reseeding::fork::FORK_HANDLER_REGISTERED.0.0", - iter.next().unwrap() - ); - assert!(iter.next().is_none()); + for (expected, actual) in DEMANGLED_OUTPUT.lines().zip(demangled_lines) { + assert_eq!(expected, actual); + } } #[test] fn test_demangle_lines_no_crate_disambiguators() { let demangled_lines = demangle_lines(MANGLED_INPUT.lines(), Some(create_disambiguator_re())); - let mut iter = demangled_lines.iter(); - assert_eq!("", iter.next().unwrap()); - assert_eq!("123foo[0]::bar", iter.next().unwrap()); - assert_eq!("utf8_idents::საჭმელად_გემრიელი_სადილი", iter.next().unwrap()); - assert_eq!("cc::spawn::{closure#0}::{closure#0}", iter.next().unwrap()); - assert_eq!( - " as core::iter::iterator::Iterator>::rposition::::{closure#0}", - iter.next().unwrap() - ); - assert_eq!( - "alloc::alloc::box_free::>", - iter.next().unwrap() - ); - assert_eq!("INtC8arrayvec8ArrayVechKj7b_E", iter.next().unwrap()); - assert_eq!(">", iter.next().unwrap()); - assert_eq!(">", iter.next().unwrap()); - assert_eq!(">", iter.next().unwrap()); - assert_eq!(">", iter.next().unwrap()); - assert_eq!(">", iter.next().unwrap()); - assert_eq!(">", iter.next().unwrap()); - assert_eq!(">", iter.next().unwrap()); - assert_eq!(">", iter.next().unwrap()); - assert_eq!(">::foo::FOO", iter.next().unwrap()); - assert_eq!("foo[0]", iter.next().unwrap()); - assert_eq!("foo[0]", iter.next().unwrap()); - assert_eq!("backtrace[0]::foo", iter.next().unwrap()); - assert_eq!( - "rand::rngs::adapter::reseeding::fork::FORK_HANDLER_REGISTERED.0.0", - iter.next().unwrap() - ); - assert!(iter.next().is_none()); + for (expected, actual) in DEMANGLED_OUTPUT_NO_CRATE_DISAMBIGUATORS.lines().zip(demangled_lines) + { + assert_eq!(expected, actual); + } }