From 223f0c8684a9f578ff290081280915ec9fd12ba5 Mon Sep 17 00:00:00 2001 From: Paolo Tranquilli Date: Tue, 1 Jul 2025 14:52:03 +0200 Subject: [PATCH 1/4] Rust: fix macro expansion in library code There was a mismatch between a `self.macro_context_level += 1` and the corresponding `self.macro_context_level -= 1`, which resulted in an `usize` underflow (panic in debug mode, wrong behaviour in release mode). This fixes it and adds a relevant assertion and test. In order to properly test library mode extraction, a special option enforcing that on source code as well is added. --- rust/extractor/src/config.rs | 1 + rust/extractor/src/main.rs | 13 +- rust/extractor/src/qltest.rs | 2 +- rust/extractor/src/translate/base.rs | 90 +++++++------- rust/ql/test/TestUtils.qll | 2 +- .../macro-in-library/Cargo.lock | 53 ++++++++ .../macro-in-library/PrintAst.expected | 116 ++++++++++++++++++ .../macro-in-library/PrintAst.qlref | 1 + .../macro-in-library/macro_in_library.rs | 6 + .../macro-in-library/options.yml | 1 + .../macro-in-library/proc_macro.rs | 14 +++ .../macro-in-library/test.expected | 4 + .../extractor-tests/macro-in-library/test.ql | 10 ++ 13 files changed, 264 insertions(+), 49 deletions(-) create mode 100644 rust/ql/test/extractor-tests/macro-in-library/Cargo.lock create mode 100644 rust/ql/test/extractor-tests/macro-in-library/PrintAst.expected create mode 100644 rust/ql/test/extractor-tests/macro-in-library/PrintAst.qlref create mode 100644 rust/ql/test/extractor-tests/macro-in-library/macro_in_library.rs create mode 100644 rust/ql/test/extractor-tests/macro-in-library/options.yml create mode 100644 rust/ql/test/extractor-tests/macro-in-library/proc_macro.rs create mode 100644 rust/ql/test/extractor-tests/macro-in-library/test.expected create mode 100644 rust/ql/test/extractor-tests/macro-in-library/test.ql diff --git a/rust/extractor/src/config.rs b/rust/extractor/src/config.rs index 303089b15742..a26bf790f825 100644 --- a/rust/extractor/src/config.rs +++ b/rust/extractor/src/config.rs @@ -71,6 +71,7 @@ pub struct Config { pub proc_macro_server: Option, pub skip_path_resolution: bool, pub extract_dependencies_as_source: bool, + pub force_library_mode: bool, // for testing purposes } impl Config { diff --git a/rust/extractor/src/main.rs b/rust/extractor/src/main.rs index fc9d060add81..1c8629cbbb90 100644 --- a/rust/extractor/src/main.rs +++ b/rust/extractor/src/main.rs @@ -293,6 +293,11 @@ fn main() -> anyhow::Result<()> { } else { (SourceKind::Library, ResolvePaths::No) }; + let (source_mode, source_resolve_paths) = if cfg.force_library_mode { + (library_mode, library_resolve_paths) + } else { + (SourceKind::Source, resolve_paths) + }; let mut processed_files: HashSet = HashSet::from_iter(files.iter().cloned()); for (manifest, files) in map.values().filter(|(_, files)| !files.is_empty()) { @@ -311,12 +316,10 @@ fn main() -> anyhow::Result<()> { file, &semantics, vfs, - resolve_paths, - SourceKind::Source, + source_resolve_paths, + source_mode, ), - Err(reason) => { - extractor.extract_without_semantics(file, SourceKind::Source, &reason) - } + Err(reason) => extractor.extract_without_semantics(file, source_mode, &reason), }; } for (file_id, file) in vfs.iter() { diff --git a/rust/extractor/src/qltest.rs b/rust/extractor/src/qltest.rs index 92ed9d62505c..bd9849cbb09f 100644 --- a/rust/extractor/src/qltest.rs +++ b/rust/extractor/src/qltest.rs @@ -19,7 +19,7 @@ fn dump_lib() -> anyhow::Result<()> { .iter() .map(|p| p.file_stem().expect("results of glob must have a name")) .filter(|&p| !["main", "lib", "proc_macro"].map(OsStr::new).contains(&p)) - .map(|p| format!("mod {};", p.to_string_lossy())) + .map(|p| format!("pub mod {};", p.to_string_lossy())) .join("\n"); fs::write("lib.rs", lib).context("writing lib.rs") } diff --git a/rust/extractor/src/translate/base.rs b/rust/extractor/src/translate/base.rs index 81bb13305a62..9928432f0590 100644 --- a/rust/extractor/src/translate/base.rs +++ b/rust/extractor/src/translate/base.rs @@ -24,33 +24,31 @@ use ra_ap_syntax::{ impl Emission for Translator<'_> { fn pre_emit(&mut self, node: &ast::Item) -> Option> { - self.prepare_item_expansion(node).map(Into::into) + self.item_pre_emit(node).map(Into::into) } fn post_emit(&mut self, node: &ast::Item, label: Label) { - self.emit_item_expansion(node, label); + self.item_post_emit(node, label); } } impl Emission for Translator<'_> { fn pre_emit(&mut self, node: &ast::AssocItem) -> Option> { - self.prepare_item_expansion(&node.clone().into()) - .map(Into::into) + self.item_pre_emit(&node.clone().into()).map(Into::into) } fn post_emit(&mut self, node: &ast::AssocItem, label: Label) { - self.emit_item_expansion(&node.clone().into(), label.into()); + self.item_post_emit(&node.clone().into(), label.into()); } } impl Emission for Translator<'_> { fn pre_emit(&mut self, node: &ast::ExternItem) -> Option> { - self.prepare_item_expansion(&node.clone().into()) - .map(Into::into) + self.item_pre_emit(&node.clone().into()).map(Into::into) } fn post_emit(&mut self, node: &ast::ExternItem, label: Label) { - self.emit_item_expansion(&node.clone().into(), label.into()); + self.item_post_emit(&node.clone().into(), label.into()); } } @@ -849,35 +847,6 @@ impl<'a> Translator<'a> { }) } - pub(crate) fn prepare_item_expansion( - &mut self, - node: &ast::Item, - ) -> Option> { - if self.source_kind == SourceKind::Library { - // if the item expands via an attribute macro, we want to only emit the expansion - if let Some(expanded) = self.emit_attribute_macro_expansion(node) { - // we wrap it in a dummy MacroCall to get a single Item label that can replace - // the original Item - let label = self.trap.emit(generated::MacroCall { - id: TrapId::Star, - attrs: vec![], - path: None, - token_tree: None, - }); - generated::MacroCall::emit_macro_call_expansion( - label, - expanded.into(), - &mut self.trap.writer, - ); - return Some(label); - } - } - if self.is_attribute_macro_target(node) { - self.macro_context_depth += 1; - } - None - } - fn process_item_macro_expansion( &mut self, node: &impl ast::AstNode, @@ -915,10 +884,6 @@ impl<'a> Translator<'a> { &mut self, node: &ast::Item, ) -> Option> { - if !self.is_attribute_macro_target(node) { - return None; - } - self.macro_context_depth -= 1; if self.macro_context_depth > 0 { // only expand the outermost attribute macro return None; @@ -927,7 +892,48 @@ impl<'a> Translator<'a> { self.process_item_macro_expansion(node, expansion.map(|x| x.value)) } - pub(crate) fn emit_item_expansion(&mut self, node: &ast::Item, label: Label) { + pub(crate) fn item_pre_emit( + &mut self, + node: &ast::Item, + ) -> Option> { + if !self.is_attribute_macro_target(node) { + return None; + } + if self.source_kind == SourceKind::Library { + // if the item expands via an attribute macro, we want to only emit the expansion + if let Some(expanded) = self.emit_attribute_macro_expansion(node) { + // we wrap it in a dummy MacroCall to get a single Item label that can replace + // the original Item + let label = self.trap.emit(generated::MacroCall { + id: TrapId::Star, + attrs: vec![], + path: None, + token_tree: None, + }); + generated::MacroCall::emit_macro_call_expansion( + label, + expanded.into(), + &mut self.trap.writer, + ); + return Some(label); + } + } + self.macro_context_depth += 1; + None + } + + pub(crate) fn item_post_emit(&mut self, node: &ast::Item, label: Label) { + if !self.is_attribute_macro_target(node) { + return; + } + // see `item_pre_emit`: + // if self.is_attribute_macro_target(node), then we either exited early with `Some(label)` + // and are not here, or we did self.macro_context_depth += 1 + assert!( + self.macro_context_depth > 0, + "macro_context_depth should be > 0 for an attribute macro target" + ); + self.macro_context_depth -= 1; if let Some(expanded) = self.emit_attribute_macro_expansion(node) { generated::Item::emit_attribute_macro_expansion(label, expanded, &mut self.trap.writer); } diff --git a/rust/ql/test/TestUtils.qll b/rust/ql/test/TestUtils.qll index f1e3c7294942..a68d9554d34b 100644 --- a/rust/ql/test/TestUtils.qll +++ b/rust/ql/test/TestUtils.qll @@ -6,7 +6,7 @@ predicate toBeTested(Element e) { ( not e instanceof Locatable or - e.(Locatable).fromSource() + exists(e.(Locatable).getFile().getRelativePath()) ) } diff --git a/rust/ql/test/extractor-tests/macro-in-library/Cargo.lock b/rust/ql/test/extractor-tests/macro-in-library/Cargo.lock new file mode 100644 index 000000000000..409ae76f4cd4 --- /dev/null +++ b/rust/ql/test/extractor-tests/macro-in-library/Cargo.lock @@ -0,0 +1,53 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "proc-macro2" +version = "1.0.95" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "02b3e5e68a3a1a02aad3ec490a98007cbc13c37cbe84a3cd7b8e406d76e7f778" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "proc_macro" +version = "0.0.1" +dependencies = [ + "quote", + "syn", +] + +[[package]] +name = "quote" +version = "1.0.40" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1885c039570dc00dcb4ff087a89e185fd56bae234ddc7f056a945bf36467248d" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "syn" +version = "2.0.104" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "17b6f705963418cdb9927482fa304bc562ece2fdd4f616084c50b7023b435a40" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "test" +version = "0.0.1" +dependencies = [ + "proc_macro", +] + +[[package]] +name = "unicode-ident" +version = "1.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a5f39404a5da50712a4c1eecf25e90dd62b613502b7e925fd4e4d19b5c96512" diff --git a/rust/ql/test/extractor-tests/macro-in-library/PrintAst.expected b/rust/ql/test/extractor-tests/macro-in-library/PrintAst.expected new file mode 100644 index 000000000000..a8e2990dab4f --- /dev/null +++ b/rust/ql/test/extractor-tests/macro-in-library/PrintAst.expected @@ -0,0 +1,116 @@ +lib.rs: +# 1| [SourceFile] SourceFile +# 1| getItem(0): [Module] mod macro_in_library +# 1| getName(): [Name] macro_in_library +# 1| getVisibility(): [Visibility] Visibility +macro_in_library.rs: +# 1| [SourceFile] SourceFile +# 4| getItem(1): [Function] fn bar +# 4| getParamList(): [ParamList] ParamList +# 4| getName(): [Name] bar +# 4| getVisibility(): [Visibility] Visibility +# 2| [MacroItems] MacroItems +# 2| getItem(0): [Function] fn foo +# 2| getParamList(): [ParamList] ParamList +# 2| getName(): [Name] foo +# 2| getVisibility(): [Visibility] Visibility +# 2| getItem(1): [Function] fn foo_new +# 2| getParamList(): [ParamList] ParamList +# 2| getName(): [Name] foo_new +# 2| getVisibility(): [Visibility] Visibility +proc_macro.rs: +# 1| [SourceFile] SourceFile +# 1| getItem(0): [Use] use ...::TokenStream +# 1| getUseTree(): [UseTree] ...::TokenStream +# 1| getPath(): [Path] ...::TokenStream +# 1| getQualifier(): [Path] proc_macro +# 1| getSegment(): [PathSegment] proc_macro +# 1| getIdentifier(): [NameRef] proc_macro +# 1| getSegment(): [PathSegment] TokenStream +# 1| getIdentifier(): [NameRef] TokenStream +# 2| getItem(1): [Use] use ...::quote +# 2| getUseTree(): [UseTree] ...::quote +# 2| getPath(): [Path] ...::quote +# 2| getQualifier(): [Path] quote +# 2| getSegment(): [PathSegment] quote +# 2| getIdentifier(): [NameRef] quote +# 2| getSegment(): [PathSegment] quote +# 2| getIdentifier(): [NameRef] quote +# 4| getItem(2): [Function] fn add_one +# 5| getParamList(): [ParamList] ParamList +# 5| getParam(0): [Param] : TokenStream +# 5| getTypeRepr(): [PathTypeRepr] TokenStream +# 5| getPath(): [Path] TokenStream +# 5| getSegment(): [PathSegment] TokenStream +# 5| getIdentifier(): [NameRef] TokenStream +# 5| getParam(1): [Param] : TokenStream +# 5| getTypeRepr(): [PathTypeRepr] TokenStream +# 5| getPath(): [Path] TokenStream +# 5| getSegment(): [PathSegment] TokenStream +# 5| getIdentifier(): [NameRef] TokenStream +# 4| getAttr(0): [Attr] Attr +# 4| getMeta(): [Meta] Meta +# 4| getPath(): [Path] proc_macro_attribute +# 4| getSegment(): [PathSegment] proc_macro_attribute +# 4| getIdentifier(): [NameRef] proc_macro_attribute +# 5| getName(): [Name] add_one +# 5| getRetType(): [RetTypeRepr] RetTypeRepr +# 5| getTypeRepr(): [PathTypeRepr] TokenStream +# 5| getPath(): [Path] TokenStream +# 5| getSegment(): [PathSegment] TokenStream +# 5| getIdentifier(): [NameRef] TokenStream +# 5| getVisibility(): [Visibility] Visibility +resolved/macro-in-library.testproj/trap/rust/crates/home/redsun82/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/allocator-api2-0.2.21/src/lib.rs: +# 1| [SourceFile] SourceFile +resolved/macro-in-library.testproj/trap/rust/crates/home/redsun82/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/cfg-if-1.0.0/src/lib.rs: +# 1| [SourceFile] SourceFile +resolved/macro-in-library.testproj/trap/rust/crates/home/redsun82/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/compiler_builtins-0.1.146/src/lib.rs: +# 1| [SourceFile] SourceFile +resolved/macro-in-library.testproj/trap/rust/crates/home/redsun82/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/getopts-0.2.21/src/lib.rs: +# 1| [SourceFile] SourceFile +resolved/macro-in-library.testproj/trap/rust/crates/home/redsun82/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/hashbrown-0.15.2/src/lib.rs: +# 1| [SourceFile] SourceFile +resolved/macro-in-library.testproj/trap/rust/crates/home/redsun82/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/libc-0.2.169/src/lib.rs: +# 1| [SourceFile] SourceFile +resolved/macro-in-library.testproj/trap/rust/crates/home/redsun82/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/proc-macro2-1.0.95/src/lib.rs: +# 1| [SourceFile] SourceFile +resolved/macro-in-library.testproj/trap/rust/crates/home/redsun82/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/quote-1.0.40/src/lib.rs: +# 1| [SourceFile] SourceFile +resolved/macro-in-library.testproj/trap/rust/crates/home/redsun82/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/rand-0.9.0/src/lib.rs: +# 1| [SourceFile] SourceFile +resolved/macro-in-library.testproj/trap/rust/crates/home/redsun82/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/rand_core-0.9.0/src/lib.rs: +# 1| [SourceFile] SourceFile +resolved/macro-in-library.testproj/trap/rust/crates/home/redsun82/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/rand_xorshift-0.4.0/src/lib.rs: +# 1| [SourceFile] SourceFile +resolved/macro-in-library.testproj/trap/rust/crates/home/redsun82/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/rustc-demangle-0.1.24/src/lib.rs: +# 1| [SourceFile] SourceFile +resolved/macro-in-library.testproj/trap/rust/crates/home/redsun82/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/syn-2.0.104/src/lib.rs: +# 1| [SourceFile] SourceFile +resolved/macro-in-library.testproj/trap/rust/crates/home/redsun82/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/unicode-ident-1.0.18/src/lib.rs: +# 1| [SourceFile] SourceFile +resolved/macro-in-library.testproj/trap/rust/crates/home/redsun82/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/unicode-width-0.1.14/src/lib.rs: +# 1| [SourceFile] SourceFile +resolved/macro-in-library.testproj/trap/rust/crates/home/redsun82/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/zerocopy-0.8.17/src/lib.rs: +# 1| [SourceFile] SourceFile +resolved/macro-in-library.testproj/trap/rust/crates/home/redsun82/.rustup/toolchains/1.86-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/alloc/src/lib.rs: +# 1| [SourceFile] SourceFile +resolved/macro-in-library.testproj/trap/rust/crates/home/redsun82/.rustup/toolchains/1.86-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/core/src/lib.rs: +# 1| [SourceFile] SourceFile +resolved/macro-in-library.testproj/trap/rust/crates/home/redsun82/.rustup/toolchains/1.86-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/panic_abort/src/lib.rs: +# 1| [SourceFile] SourceFile +resolved/macro-in-library.testproj/trap/rust/crates/home/redsun82/.rustup/toolchains/1.86-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/panic_unwind/src/lib.rs: +# 1| [SourceFile] SourceFile +resolved/macro-in-library.testproj/trap/rust/crates/home/redsun82/.rustup/toolchains/1.86-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/proc_macro/src/lib.rs: +# 1| [SourceFile] SourceFile +resolved/macro-in-library.testproj/trap/rust/crates/home/redsun82/.rustup/toolchains/1.86-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/std/src/lib.rs: +# 1| [SourceFile] SourceFile +resolved/macro-in-library.testproj/trap/rust/crates/home/redsun82/.rustup/toolchains/1.86-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/stdarch/crates/std_detect/src/lib.rs: +# 1| [SourceFile] SourceFile +resolved/macro-in-library.testproj/trap/rust/crates/home/redsun82/.rustup/toolchains/1.86-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/test/src/lib.rs: +# 1| [SourceFile] SourceFile +resolved/macro-in-library.testproj/trap/rust/crates/home/redsun82/.rustup/toolchains/1.86-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/unwind/src/lib.rs: +# 1| [SourceFile] SourceFile +resolved/macro-in-library.testproj/trap/rust/crateslib.rs: +# 1| [SourceFile] SourceFile +resolved/macro-in-library.testproj/trap/rust/cratesproc_macro.rs: +# 1| [SourceFile] SourceFile diff --git a/rust/ql/test/extractor-tests/macro-in-library/PrintAst.qlref b/rust/ql/test/extractor-tests/macro-in-library/PrintAst.qlref new file mode 100644 index 000000000000..ee3c14c56f15 --- /dev/null +++ b/rust/ql/test/extractor-tests/macro-in-library/PrintAst.qlref @@ -0,0 +1 @@ +utils/PrintAst.ql diff --git a/rust/ql/test/extractor-tests/macro-in-library/macro_in_library.rs b/rust/ql/test/extractor-tests/macro-in-library/macro_in_library.rs new file mode 100644 index 000000000000..aabe2f56d78e --- /dev/null +++ b/rust/ql/test/extractor-tests/macro-in-library/macro_in_library.rs @@ -0,0 +1,6 @@ +#[proc_macro::add_one] +pub fn foo() {} + +pub fn bar() { + foo_new(); +} \ No newline at end of file diff --git a/rust/ql/test/extractor-tests/macro-in-library/options.yml b/rust/ql/test/extractor-tests/macro-in-library/options.yml new file mode 100644 index 000000000000..f00676725584 --- /dev/null +++ b/rust/ql/test/extractor-tests/macro-in-library/options.yml @@ -0,0 +1 @@ +force_library_mode: true \ No newline at end of file diff --git a/rust/ql/test/extractor-tests/macro-in-library/proc_macro.rs b/rust/ql/test/extractor-tests/macro-in-library/proc_macro.rs new file mode 100644 index 000000000000..9b8d3265a9a3 --- /dev/null +++ b/rust/ql/test/extractor-tests/macro-in-library/proc_macro.rs @@ -0,0 +1,14 @@ +use proc_macro::TokenStream; +use quote::quote; + +#[proc_macro_attribute] +pub fn add_one(_attr: TokenStream, item: TokenStream) -> TokenStream { + let ast = syn::parse_macro_input!(item as syn::ItemFn); + let mut new_ast = ast.clone(); + new_ast.sig.ident = syn::Ident::new(&format!("{}_new", ast.sig.ident), ast.sig.ident.span()); + quote! { + #ast + #new_ast + }.into() +} + diff --git a/rust/ql/test/extractor-tests/macro-in-library/test.expected b/rust/ql/test/extractor-tests/macro-in-library/test.expected new file mode 100644 index 000000000000..685ce3244e35 --- /dev/null +++ b/rust/ql/test/extractor-tests/macro-in-library/test.expected @@ -0,0 +1,4 @@ +macro_items +| macro_in_library.rs:2:1:2:14 | MacroItems | 0 | macro_in_library.rs:2:1:2:14 | fn foo | +| macro_in_library.rs:2:1:2:14 | MacroItems | 1 | macro_in_library.rs:2:1:2:14 | fn foo_new | +warnings diff --git a/rust/ql/test/extractor-tests/macro-in-library/test.ql b/rust/ql/test/extractor-tests/macro-in-library/test.ql new file mode 100644 index 000000000000..985109105bf1 --- /dev/null +++ b/rust/ql/test/extractor-tests/macro-in-library/test.ql @@ -0,0 +1,10 @@ +import rust +import codeql.rust.Diagnostics + +query predicate macro_items(MacroItems c, int index, Item expanded) { + exists(c.getFile().getRelativePath()) and + not c.getLocation().getFile().getAbsolutePath().matches("%proc_macro.rs") and + expanded = c.getItem(index) +} + +query predicate warnings(ExtractionWarning w) { any() } From 63ccbec933d7f1491a4073f0ff24868ff04c8bed Mon Sep 17 00:00:00 2001 From: Paolo Tranquilli Date: Wed, 2 Jul 2025 10:19:52 +0200 Subject: [PATCH 2/4] Rust: accept language test changes --- .../canonical_path/canonical_paths.expected | 8 +-- .../canonical_paths.expected | 8 +-- .../generated/Module/Module.expected | 9 ++-- .../generated/Name/Name.expected | 4 +- .../generated/SourceFile/SourceFile.expected | 4 +- .../generated/Visibility/Visibility.expected | 1 + .../macro-expansion/PrintAst.expected | 2 + .../macro-in-library/PrintAst.expected | 54 ------------------- .../ql/test/extractor-tests/utf8/ast.expected | 7 +-- .../path-resolution/path-resolution.expected | 2 +- 10 files changed, 25 insertions(+), 74 deletions(-) diff --git a/rust/ql/test/extractor-tests/canonical_path/canonical_paths.expected b/rust/ql/test/extractor-tests/canonical_path/canonical_paths.expected index 49bf47841d0c..b44276608445 100644 --- a/rust/ql/test/extractor-tests/canonical_path/canonical_paths.expected +++ b/rust/ql/test/extractor-tests/canonical_path/canonical_paths.expected @@ -2,8 +2,8 @@ canonicalPath | anonymous.rs:3:1:32:1 | fn canonicals | test::anonymous::canonicals | | anonymous.rs:34:1:36:1 | fn other | test::anonymous::other | | {EXTERNAL LOCATION} | fn trim | ::trim | -| lib.rs:1:1:1:14 | mod anonymous | test::anonymous | -| lib.rs:2:1:2:12 | mod regular | test::regular | +| lib.rs:1:1:1:18 | mod anonymous | test::anonymous | +| lib.rs:2:1:2:16 | mod regular | test::regular | | regular.rs:1:1:2:18 | struct Struct | test::regular::Struct | | regular.rs:2:12:2:17 | fn eq | ::eq | | regular.rs:2:12:2:17 | impl ...::Eq for Struct::<...> { ... } | | @@ -42,8 +42,8 @@ canonicalPaths | anonymous.rs:26:5:31:5 | fn usage | None | None | | anonymous.rs:34:1:36:1 | fn other | repo::test | crate::anonymous::other | | anonymous.rs:35:5:35:23 | struct OtherStruct | None | None | -| lib.rs:1:1:1:14 | mod anonymous | repo::test | crate::anonymous | -| lib.rs:2:1:2:12 | mod regular | repo::test | crate::regular | +| lib.rs:1:1:1:18 | mod anonymous | repo::test | crate::anonymous | +| lib.rs:2:1:2:16 | mod regular | repo::test | crate::regular | | regular.rs:1:1:2:18 | struct Struct | repo::test | crate::regular::Struct | | regular.rs:2:12:2:17 | fn eq | repo::test | ::eq | | regular.rs:2:12:2:17 | impl ...::Eq for Struct::<...> { ... } | None | None | diff --git a/rust/ql/test/extractor-tests/canonical_path_disabled/canonical_paths.expected b/rust/ql/test/extractor-tests/canonical_path_disabled/canonical_paths.expected index d411ab289c93..905e04abee0a 100644 --- a/rust/ql/test/extractor-tests/canonical_path_disabled/canonical_paths.expected +++ b/rust/ql/test/extractor-tests/canonical_path_disabled/canonical_paths.expected @@ -2,8 +2,8 @@ canonicalPath | anonymous.rs:6:1:35:1 | fn canonicals | test::anonymous::canonicals | | anonymous.rs:37:1:39:1 | fn other | test::anonymous::other | | {EXTERNAL LOCATION} | fn trim | ::trim | -| lib.rs:1:1:1:14 | mod anonymous | test::anonymous | -| lib.rs:2:1:2:12 | mod regular | test::regular | +| lib.rs:1:1:1:18 | mod anonymous | test::anonymous | +| lib.rs:2:1:2:16 | mod regular | test::regular | | regular.rs:4:1:5:18 | struct Struct | test::regular::Struct | | regular.rs:5:12:5:17 | fn eq | ::eq | | regular.rs:5:12:5:17 | impl ...::Eq for Struct::<...> { ... } | | @@ -42,8 +42,8 @@ canonicalPaths | anonymous.rs:29:5:34:5 | fn usage | None | None | | anonymous.rs:37:1:39:1 | fn other | None | None | | anonymous.rs:38:5:38:23 | struct OtherStruct | None | None | -| lib.rs:1:1:1:14 | mod anonymous | None | None | -| lib.rs:2:1:2:12 | mod regular | None | None | +| lib.rs:1:1:1:18 | mod anonymous | None | None | +| lib.rs:2:1:2:16 | mod regular | None | None | | regular.rs:4:1:5:18 | struct Struct | None | None | | regular.rs:5:12:5:17 | fn eq | None | None | | regular.rs:5:12:5:17 | impl ...::Eq for Struct::<...> { ... } | None | None | diff --git a/rust/ql/test/extractor-tests/generated/Module/Module.expected b/rust/ql/test/extractor-tests/generated/Module/Module.expected index a742b7145d30..9fcf65f56580 100644 --- a/rust/ql/test/extractor-tests/generated/Module/Module.expected +++ b/rust/ql/test/extractor-tests/generated/Module/Module.expected @@ -1,15 +1,15 @@ instances | gen_module.rs:3:1:4:8 | mod foo | | gen_module.rs:5:1:7:1 | mod bar | -| lib.rs:1:1:1:15 | mod gen_module | +| lib.rs:1:1:1:19 | mod gen_module | getExtendedCanonicalPath | gen_module.rs:3:1:4:8 | mod foo | crate::gen_module::foo | | gen_module.rs:5:1:7:1 | mod bar | crate::gen_module::bar | -| lib.rs:1:1:1:15 | mod gen_module | crate::gen_module | +| lib.rs:1:1:1:19 | mod gen_module | crate::gen_module | getCrateOrigin | gen_module.rs:3:1:4:8 | mod foo | repo::test | | gen_module.rs:5:1:7:1 | mod bar | repo::test | -| lib.rs:1:1:1:15 | mod gen_module | repo::test | +| lib.rs:1:1:1:19 | mod gen_module | repo::test | getAttributeMacroExpansion getAttr getItemList @@ -17,5 +17,6 @@ getItemList getName | gen_module.rs:3:1:4:8 | mod foo | gen_module.rs:4:5:4:7 | foo | | gen_module.rs:5:1:7:1 | mod bar | gen_module.rs:5:5:5:7 | bar | -| lib.rs:1:1:1:15 | mod gen_module | lib.rs:1:5:1:14 | gen_module | +| lib.rs:1:1:1:19 | mod gen_module | lib.rs:1:9:1:18 | gen_module | getVisibility +| lib.rs:1:1:1:19 | mod gen_module | lib.rs:1:1:1:3 | Visibility | diff --git a/rust/ql/test/extractor-tests/generated/Name/Name.expected b/rust/ql/test/extractor-tests/generated/Name/Name.expected index 2bcd496ef2ee..315739204584 100644 --- a/rust/ql/test/extractor-tests/generated/Name/Name.expected +++ b/rust/ql/test/extractor-tests/generated/Name/Name.expected @@ -1,8 +1,8 @@ instances | gen_name.rs:3:4:3:12 | test_name | | gen_name.rs:7:9:7:11 | foo | -| lib.rs:1:5:1:12 | gen_name | +| lib.rs:1:9:1:16 | gen_name | getText | gen_name.rs:3:4:3:12 | test_name | test_name | | gen_name.rs:7:9:7:11 | foo | foo | -| lib.rs:1:5:1:12 | gen_name | gen_name | +| lib.rs:1:9:1:16 | gen_name | gen_name | diff --git a/rust/ql/test/extractor-tests/generated/SourceFile/SourceFile.expected b/rust/ql/test/extractor-tests/generated/SourceFile/SourceFile.expected index e308c26daa80..240161f481b5 100644 --- a/rust/ql/test/extractor-tests/generated/SourceFile/SourceFile.expected +++ b/rust/ql/test/extractor-tests/generated/SourceFile/SourceFile.expected @@ -1,7 +1,7 @@ instances | gen_source_file.rs:1:1:9:2 | SourceFile | -| lib.rs:1:1:1:20 | SourceFile | +| lib.rs:1:1:1:24 | SourceFile | getAttr getItem | gen_source_file.rs:1:1:9:2 | SourceFile | 0 | gen_source_file.rs:3:1:9:1 | fn test_source_file | -| lib.rs:1:1:1:20 | SourceFile | 0 | lib.rs:1:1:1:20 | mod gen_source_file | +| lib.rs:1:1:1:24 | SourceFile | 0 | lib.rs:1:1:1:24 | mod gen_source_file | diff --git a/rust/ql/test/extractor-tests/generated/Visibility/Visibility.expected b/rust/ql/test/extractor-tests/generated/Visibility/Visibility.expected index daae8776ad79..4bc7fefe4c13 100644 --- a/rust/ql/test/extractor-tests/generated/Visibility/Visibility.expected +++ b/rust/ql/test/extractor-tests/generated/Visibility/Visibility.expected @@ -1,3 +1,4 @@ instances | gen_visibility.rs:7:7:7:9 | Visibility | +| lib.rs:1:1:1:3 | Visibility | getPath diff --git a/rust/ql/test/extractor-tests/macro-expansion/PrintAst.expected b/rust/ql/test/extractor-tests/macro-expansion/PrintAst.expected index 5119e69d0430..79576902a43d 100644 --- a/rust/ql/test/extractor-tests/macro-expansion/PrintAst.expected +++ b/rust/ql/test/extractor-tests/macro-expansion/PrintAst.expected @@ -146,8 +146,10 @@ lib.rs: # 1| [SourceFile] SourceFile # 1| getItem(0): [Module] mod call # 1| getName(): [Name] call +# 1| getVisibility(): [Visibility] Visibility # 2| getItem(1): [Module] mod macro_expansion # 2| getName(): [Name] macro_expansion +# 2| getVisibility(): [Visibility] Visibility macro_expansion.rs: # 1| [SourceFile] SourceFile # 1| getItem(0): [Use] use proc_macro::{...} diff --git a/rust/ql/test/extractor-tests/macro-in-library/PrintAst.expected b/rust/ql/test/extractor-tests/macro-in-library/PrintAst.expected index a8e2990dab4f..04c5f5f28f3e 100644 --- a/rust/ql/test/extractor-tests/macro-in-library/PrintAst.expected +++ b/rust/ql/test/extractor-tests/macro-in-library/PrintAst.expected @@ -60,57 +60,3 @@ proc_macro.rs: # 5| getSegment(): [PathSegment] TokenStream # 5| getIdentifier(): [NameRef] TokenStream # 5| getVisibility(): [Visibility] Visibility -resolved/macro-in-library.testproj/trap/rust/crates/home/redsun82/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/allocator-api2-0.2.21/src/lib.rs: -# 1| [SourceFile] SourceFile -resolved/macro-in-library.testproj/trap/rust/crates/home/redsun82/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/cfg-if-1.0.0/src/lib.rs: -# 1| [SourceFile] SourceFile -resolved/macro-in-library.testproj/trap/rust/crates/home/redsun82/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/compiler_builtins-0.1.146/src/lib.rs: -# 1| [SourceFile] SourceFile -resolved/macro-in-library.testproj/trap/rust/crates/home/redsun82/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/getopts-0.2.21/src/lib.rs: -# 1| [SourceFile] SourceFile -resolved/macro-in-library.testproj/trap/rust/crates/home/redsun82/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/hashbrown-0.15.2/src/lib.rs: -# 1| [SourceFile] SourceFile -resolved/macro-in-library.testproj/trap/rust/crates/home/redsun82/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/libc-0.2.169/src/lib.rs: -# 1| [SourceFile] SourceFile -resolved/macro-in-library.testproj/trap/rust/crates/home/redsun82/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/proc-macro2-1.0.95/src/lib.rs: -# 1| [SourceFile] SourceFile -resolved/macro-in-library.testproj/trap/rust/crates/home/redsun82/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/quote-1.0.40/src/lib.rs: -# 1| [SourceFile] SourceFile -resolved/macro-in-library.testproj/trap/rust/crates/home/redsun82/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/rand-0.9.0/src/lib.rs: -# 1| [SourceFile] SourceFile -resolved/macro-in-library.testproj/trap/rust/crates/home/redsun82/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/rand_core-0.9.0/src/lib.rs: -# 1| [SourceFile] SourceFile -resolved/macro-in-library.testproj/trap/rust/crates/home/redsun82/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/rand_xorshift-0.4.0/src/lib.rs: -# 1| [SourceFile] SourceFile -resolved/macro-in-library.testproj/trap/rust/crates/home/redsun82/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/rustc-demangle-0.1.24/src/lib.rs: -# 1| [SourceFile] SourceFile -resolved/macro-in-library.testproj/trap/rust/crates/home/redsun82/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/syn-2.0.104/src/lib.rs: -# 1| [SourceFile] SourceFile -resolved/macro-in-library.testproj/trap/rust/crates/home/redsun82/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/unicode-ident-1.0.18/src/lib.rs: -# 1| [SourceFile] SourceFile -resolved/macro-in-library.testproj/trap/rust/crates/home/redsun82/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/unicode-width-0.1.14/src/lib.rs: -# 1| [SourceFile] SourceFile -resolved/macro-in-library.testproj/trap/rust/crates/home/redsun82/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/zerocopy-0.8.17/src/lib.rs: -# 1| [SourceFile] SourceFile -resolved/macro-in-library.testproj/trap/rust/crates/home/redsun82/.rustup/toolchains/1.86-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/alloc/src/lib.rs: -# 1| [SourceFile] SourceFile -resolved/macro-in-library.testproj/trap/rust/crates/home/redsun82/.rustup/toolchains/1.86-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/core/src/lib.rs: -# 1| [SourceFile] SourceFile -resolved/macro-in-library.testproj/trap/rust/crates/home/redsun82/.rustup/toolchains/1.86-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/panic_abort/src/lib.rs: -# 1| [SourceFile] SourceFile -resolved/macro-in-library.testproj/trap/rust/crates/home/redsun82/.rustup/toolchains/1.86-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/panic_unwind/src/lib.rs: -# 1| [SourceFile] SourceFile -resolved/macro-in-library.testproj/trap/rust/crates/home/redsun82/.rustup/toolchains/1.86-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/proc_macro/src/lib.rs: -# 1| [SourceFile] SourceFile -resolved/macro-in-library.testproj/trap/rust/crates/home/redsun82/.rustup/toolchains/1.86-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/std/src/lib.rs: -# 1| [SourceFile] SourceFile -resolved/macro-in-library.testproj/trap/rust/crates/home/redsun82/.rustup/toolchains/1.86-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/stdarch/crates/std_detect/src/lib.rs: -# 1| [SourceFile] SourceFile -resolved/macro-in-library.testproj/trap/rust/crates/home/redsun82/.rustup/toolchains/1.86-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/test/src/lib.rs: -# 1| [SourceFile] SourceFile -resolved/macro-in-library.testproj/trap/rust/crates/home/redsun82/.rustup/toolchains/1.86-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/unwind/src/lib.rs: -# 1| [SourceFile] SourceFile -resolved/macro-in-library.testproj/trap/rust/crateslib.rs: -# 1| [SourceFile] SourceFile -resolved/macro-in-library.testproj/trap/rust/cratesproc_macro.rs: -# 1| [SourceFile] SourceFile diff --git a/rust/ql/test/extractor-tests/utf8/ast.expected b/rust/ql/test/extractor-tests/utf8/ast.expected index a37eed8e2641..7fe2a9bd4ee7 100644 --- a/rust/ql/test/extractor-tests/utf8/ast.expected +++ b/rust/ql/test/extractor-tests/utf8/ast.expected @@ -1,6 +1,7 @@ -| lib.rs:1:1:1:21 | SourceFile | -| lib.rs:1:1:1:21 | mod utf8_identifiers | -| lib.rs:1:5:1:20 | utf8_identifiers | +| lib.rs:1:1:1:3 | Visibility | +| lib.rs:1:1:1:25 | SourceFile | +| lib.rs:1:1:1:25 | mod utf8_identifiers | +| lib.rs:1:9:1:24 | utf8_identifiers | | utf8_identifiers.rs:1:1:4:6 | fn foo | | utf8_identifiers.rs:1:1:12:2 | SourceFile | | utf8_identifiers.rs:1:4:1:6 | foo | diff --git a/rust/ql/test/library-tests/path-resolution/path-resolution.expected b/rust/ql/test/library-tests/path-resolution/path-resolution.expected index 164852626dbe..c757e29396f7 100644 --- a/rust/ql/test/library-tests/path-resolution/path-resolution.expected +++ b/rust/ql/test/library-tests/path-resolution/path-resolution.expected @@ -1,5 +1,5 @@ mod -| lib.rs:1:1:1:7 | mod my | +| lib.rs:1:1:1:11 | mod my | | main.rs:1:1:1:7 | mod my | | main.rs:7:1:7:8 | mod my2 | | main.rs:13:1:37:1 | mod m1 | From bf09c925285d6bb6660005b4d6ca29efab3cb32c Mon Sep 17 00:00:00 2001 From: Paolo Tranquilli Date: Wed, 2 Jul 2025 10:33:53 +0200 Subject: [PATCH 3/4] Rust: add location to dummy `MacroCall`s in library mode --- rust/extractor/src/translate/base.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/rust/extractor/src/translate/base.rs b/rust/extractor/src/translate/base.rs index 9928432f0590..927f5a509bf9 100644 --- a/rust/extractor/src/translate/base.rs +++ b/rust/extractor/src/translate/base.rs @@ -915,6 +915,7 @@ impl<'a> Translator<'a> { expanded.into(), &mut self.trap.writer, ); + self.emit_location(label, node); return Some(label); } } From c70198e4e44b1532b0be88167b1019b473f0c83e Mon Sep 17 00:00:00 2001 From: Paolo Tranquilli Date: Wed, 2 Jul 2025 12:25:10 +0200 Subject: [PATCH 4/4] Rust: change dummy macro call expansion --- rust/extractor/src/translate/base.rs | 6 +++--- .../rust/elements/internal/MacroCallImpl.qll | 4 +++- .../macro-in-library/PrintAst.expected | 19 ++++++++++--------- 3 files changed, 16 insertions(+), 13 deletions(-) diff --git a/rust/extractor/src/translate/base.rs b/rust/extractor/src/translate/base.rs index 927f5a509bf9..b51500291349 100644 --- a/rust/extractor/src/translate/base.rs +++ b/rust/extractor/src/translate/base.rs @@ -910,9 +910,9 @@ impl<'a> Translator<'a> { path: None, token_tree: None, }); - generated::MacroCall::emit_macro_call_expansion( - label, - expanded.into(), + generated::Item::emit_attribute_macro_expansion( + label.into(), + expanded, &mut self.trap.writer, ); self.emit_location(label, node); diff --git a/rust/ql/lib/codeql/rust/elements/internal/MacroCallImpl.qll b/rust/ql/lib/codeql/rust/elements/internal/MacroCallImpl.qll index 7afe59662e4a..27f70c77074c 100644 --- a/rust/ql/lib/codeql/rust/elements/internal/MacroCallImpl.qll +++ b/rust/ql/lib/codeql/rust/elements/internal/MacroCallImpl.qll @@ -32,7 +32,9 @@ module Impl { * ``` */ class MacroCall extends Generated::MacroCall { - override string toStringImpl() { result = this.getPath().toAbbreviatedString() + "!..." } + override string toStringImpl() { + if this.hasPath() then result = this.getPath().toAbbreviatedString() + "!..." else result = "" + } /** Gets an AST node whose location is inside the token tree belonging to this macro call. */ pragma[nomagic] diff --git a/rust/ql/test/extractor-tests/macro-in-library/PrintAst.expected b/rust/ql/test/extractor-tests/macro-in-library/PrintAst.expected index 04c5f5f28f3e..b88537912102 100644 --- a/rust/ql/test/extractor-tests/macro-in-library/PrintAst.expected +++ b/rust/ql/test/extractor-tests/macro-in-library/PrintAst.expected @@ -5,19 +5,20 @@ lib.rs: # 1| getVisibility(): [Visibility] Visibility macro_in_library.rs: # 1| [SourceFile] SourceFile +# 1| getItem(0): [MacroCall] +# 2| getAttributeMacroExpansion(): [MacroItems] MacroItems +# 2| getItem(0): [Function] fn foo +# 2| getParamList(): [ParamList] ParamList +# 2| getName(): [Name] foo +# 2| getVisibility(): [Visibility] Visibility +# 2| getItem(1): [Function] fn foo_new +# 2| getParamList(): [ParamList] ParamList +# 2| getName(): [Name] foo_new +# 2| getVisibility(): [Visibility] Visibility # 4| getItem(1): [Function] fn bar # 4| getParamList(): [ParamList] ParamList # 4| getName(): [Name] bar # 4| getVisibility(): [Visibility] Visibility -# 2| [MacroItems] MacroItems -# 2| getItem(0): [Function] fn foo -# 2| getParamList(): [ParamList] ParamList -# 2| getName(): [Name] foo -# 2| getVisibility(): [Visibility] Visibility -# 2| getItem(1): [Function] fn foo_new -# 2| getParamList(): [ParamList] ParamList -# 2| getName(): [Name] foo_new -# 2| getVisibility(): [Visibility] Visibility proc_macro.rs: # 1| [SourceFile] SourceFile # 1| getItem(0): [Use] use ...::TokenStream