Skip to content

Commit

Permalink
Merge pull request #17920 from github/redsun82/rust-files
Browse files Browse the repository at this point in the history 
Rust: exclude uncompiled files from semantics and surface semantic-less reason
  • Loading branch information
@redsun82
redsun82 authored Nov 7, 2024
2 parents 07bb60d + 64d522e commit 3488b9f
Show file tree
Hide file tree
Showing 3 changed files with 146 additions and 104 deletions.
146 changes: 86 additions & 60 deletions rust/extractor/src/main.rs
View file
Original file line number Diff line number Diff line change
@@ -1,14 +1,18 @@
use crate::rust_analyzer::path_to_file_id;
use anyhow::Context;
use archive::Archiver;
use log::info;
use ra_ap_hir::Semantics;
use ra_ap_ide_db::line_index::{LineCol, LineIndex};
use ra_ap_ide_db::RootDatabase;
use ra_ap_project_model::ProjectManifest;
use ra_ap_vfs::Vfs;
use rust_analyzer::{ParseResult, RustAnalyzer};
use std::{
collections::HashMap,
path::{Path, PathBuf},
};

mod archive;
mod config;
pub mod generated;
Expand All @@ -17,54 +21,71 @@ mod rust_analyzer;
mod translate;
pub mod trap;

fn extract(
rust_analyzer: &rust_analyzer::RustAnalyzer,
archiver: &Archiver,
traps: &trap::TrapFileProvider,
file: &std::path::Path,
) {
archiver.archive(file);
struct Extractor<'a> {
archiver: &'a Archiver,
traps: &'a trap::TrapFileProvider,
}

let ParseResult {
ast,
text,
errors,
file_id,
} = rust_analyzer.parse(file);
let line_index = LineIndex::new(text.as_ref());
let display_path = file.to_string_lossy();
let mut trap = traps.create("source", file);
let label = trap.emit_file(file);
let mut translator = translate::Translator::new(
trap,
display_path.as_ref(),
label,
line_index,
file_id,
file_id.and(rust_analyzer.semantics()),
);
impl Extractor<'_> {
fn extract(&self, rust_analyzer: &rust_analyzer::RustAnalyzer, file: &std::path::Path) {
self.archiver.archive(file);

for err in errors {
translator.emit_parse_error(&ast, &err);
}
let no_location = (LineCol { line: 0, col: 0 }, LineCol { line: 0, col: 0 });
if translator.semantics.is_none() {
translator.emit_diagnostic(
trap::DiagnosticSeverity::Warning,
"semantics".to_owned(),
"semantic analyzer unavailable".to_owned(),
"semantic analyzer unavailable: macro expansion, call graph, and type inference will be skipped.".to_owned(),
no_location,
let ParseResult {
ast,
text,
errors,
semantics_info,
} = rust_analyzer.parse(file);
let line_index = LineIndex::new(text.as_ref());
let display_path = file.to_string_lossy();
let mut trap = self.traps.create("source", file);
let label = trap.emit_file(file);
let mut translator = translate::Translator::new(
trap,
display_path.as_ref(),
label,
line_index,
semantics_info.as_ref().ok(),
);

for err in errors {
translator.emit_parse_error(&ast, &err);
}
let no_location = (LineCol { line: 0, col: 0 }, LineCol { line: 0, col: 0 });
if let Err(reason) = semantics_info {
let message = format!("semantic analyzer unavailable ({reason})");
let full_message = format!(
"{message}: macro expansion, call graph, and type inference will be skipped."
);
translator.emit_diagnostic(
trap::DiagnosticSeverity::Warning,
"semantics".to_owned(),
message,
full_message,
no_location,
);
}
translator.emit_source_file(ast);
translator.trap.commit().unwrap_or_else(|err| {
log::error!(
"Failed to write trap file for: {}: {}",
display_path,
err.to_string()
)
});
}

pub fn extract_with_semantics(
&self,
file: &Path,
semantics: &Semantics<'_, RootDatabase>,
vfs: &Vfs,
) {
self.extract(&RustAnalyzer::new(vfs, semantics), file);
}
pub fn extract_without_semantics(&self, file: &Path, reason: &str) {
self.extract(&RustAnalyzer::WithoutSemantics { reason }, file);
}
translator.emit_source_file(ast);
translator.trap.commit().unwrap_or_else(|err| {
log::error!(
"Failed to write trap file for: {}: {}",
display_path,
err.to_string()
)
});
}

fn main() -> anyhow::Result<()> {
Expand All @@ -82,6 +103,10 @@ fn main() -> anyhow::Result<()> {
let archiver = archive::Archiver {
root: cfg.source_archive_dir.clone(),
};
let extractor = Extractor {
archiver: &archiver,
traps: &traps,
};
let files: Vec<PathBuf> = cfg
.inputs
.iter()
Expand All @@ -95,38 +120,39 @@ fn main() -> anyhow::Result<()> {
.iter()
.map(|x| (x.manifest_path().parent().as_ref(), (x, Vec::new())))
.collect();
let mut other_files = Vec::new();

'outer: for file in &files {
let mut p = file.as_path();
while let Some(parent) = p.parent() {
p = parent;
if let Some((_, files)) = map.get_mut(parent) {
for ancestor in file.as_path().ancestors() {
if let Some((_, files)) = map.get_mut(ancestor) {
files.push(file);
continue 'outer;
}
}
other_files.push(file);
extractor.extract_without_semantics(file, "no manifest found");
}
for (manifest, files) in map.values() {
if files.is_empty() {
break;
}
for (manifest, files) in map.values().filter(|(_, files)| !files.is_empty()) {
if let Some((ref db, ref vfs)) = RustAnalyzer::load_workspace(manifest, &cfg.scratch_dir) {
let semantics = Semantics::new(db);
let rust_analyzer = RustAnalyzer::new(vfs, semantics);
for file in files {
extract(&rust_analyzer, &archiver, &traps, file);
let Some(id) = path_to_file_id(file, vfs) else {
extractor.extract_without_semantics(
file,
"not included in files loaded from manifest",
);
continue;
};
if semantics.file_to_module_def(id).is_none() {
extractor.extract_without_semantics(file, "not included as a module");
continue;
}
extractor.extract_with_semantics(file, &semantics, vfs);
}
} else {
for file in files {
extract(&RustAnalyzer::WithoutSemantics, &archiver, &traps, file);
extractor.extract_without_semantics(file, "unable to load manifest");
}
}
}
for file in other_files {
extract(&RustAnalyzer::WithoutSemantics, &archiver, &traps, file);
}

Ok(())
}
91 changes: 53 additions & 38 deletions rust/extractor/src/rust_analyzer.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,25 +14,34 @@ use ra_ap_span::TextRange;
use ra_ap_span::TextSize;
use ra_ap_syntax::SourceFile;
use ra_ap_syntax::SyntaxError;
use ra_ap_vfs::AbsPathBuf;
use ra_ap_vfs::Vfs;
use ra_ap_vfs::VfsPath;
use ra_ap_vfs::{AbsPathBuf, FileId};
use std::borrow::Cow;
use std::path::{Path, PathBuf};
use triomphe::Arc;
pub enum RustAnalyzer<'a> {
WithSemantics {
vfs: &'a Vfs,
semantics: Semantics<'a, RootDatabase>,
semantics: &'a Semantics<'a, RootDatabase>,
},
WithoutSemantics,
WithoutSemantics {
reason: &'a str,
},
}

pub struct FileSemanticInformation<'a> {
pub file_id: EditionedFileId,
pub semantics: &'a Semantics<'a, RootDatabase>,
}
pub struct ParseResult {

pub struct ParseResult<'a> {
pub ast: SourceFile,
pub text: Arc<str>,
pub errors: Vec<SyntaxError>,
pub file_id: Option<EditionedFileId>,
pub semantics_info: Result<FileSemanticInformation<'a>, &'a str>,
}

impl<'a> RustAnalyzer<'a> {
pub fn load_workspace(
project: &ProjectManifest,
Expand Down Expand Up @@ -61,47 +70,45 @@ impl<'a> RustAnalyzer<'a> {
}
}
}
pub fn new(vfs: &'a Vfs, semantics: Semantics<'a, RootDatabase>) -> Self {
pub fn new(vfs: &'a Vfs, semantics: &'a Semantics<'a, RootDatabase>) -> Self {
RustAnalyzer::WithSemantics { vfs, semantics }
}
pub fn semantics(&'a self) -> Option<&'a Semantics<'a, RootDatabase>> {
match self {
RustAnalyzer::WithSemantics { vfs: _, semantics } => Some(semantics),
RustAnalyzer::WithoutSemantics => None,
}
}
pub fn parse(&self, path: &Path) -> ParseResult {
if let RustAnalyzer::WithSemantics { vfs, semantics } = self {
if let Some(file_id) = Utf8PathBuf::from_path_buf(path.to_path_buf())
.ok()
.and_then(|x| AbsPathBuf::try_from(x).ok())
.map(VfsPath::from)
.and_then(|x| vfs.file_id(&x))
{
if let Ok(input) = std::panic::catch_unwind(|| semantics.db.file_text(file_id)) {
let file_id = EditionedFileId::current_edition(file_id);
let source_file = semantics.parse(file_id);
let errors = semantics
.db
.parse_errors(file_id)
.into_iter()
.flat_map(|x| x.to_vec())
.collect();
let no_semantics_reason;
match self {
RustAnalyzer::WithSemantics { vfs, semantics } => {
if let Some(file_id) = path_to_file_id(path, vfs) {
if let Ok(input) = std::panic::catch_unwind(|| semantics.db.file_text(file_id))
{
let file_id = EditionedFileId::current_edition(file_id);
let source_file = semantics.parse(file_id);
let errors = semantics
.db
.parse_errors(file_id)
.into_iter()
.flat_map(|x| x.to_vec())
.collect();

return ParseResult {
ast: source_file,
text: input,
errors,
file_id: Some(file_id),
};
} else {
log::debug!(
return ParseResult {
ast: source_file,
text: input,
errors,
semantics_info: Ok(FileSemanticInformation { file_id, semantics }),
};
}
debug!(
"No text available for file_id '{:?}', falling back to loading file '{}' from disk.",
file_id,
path.to_string_lossy()
)
);
no_semantics_reason = "no text available for the file in the project";
} else {
no_semantics_reason = "file not found in project";
}
}
RustAnalyzer::WithoutSemantics { reason } => {
no_semantics_reason = reason;
}
}
let mut errors = Vec::new();
let input = match std::fs::read(path) {
Expand All @@ -123,7 +130,7 @@ impl<'a> RustAnalyzer<'a> {
ast: parse.tree(),
text: input.as_ref().into(),
errors,
file_id: None,
semantics_info: Err(no_semantics_reason),
}
}
}
Expand Down Expand Up @@ -187,3 +194,11 @@ fn from_utf8_lossy(v: &[u8]) -> (Cow<'_, str>, Option<SyntaxError>) {

(Cow::Owned(res), Some(error))
}

pub(crate) fn path_to_file_id(path: &Path, vfs: &Vfs) -> Option<FileId> {
Utf8PathBuf::from_path_buf(path.to_path_buf())
.ok()
.and_then(|x| AbsPathBuf::try_from(x).ok())
.map(VfsPath::from)
.and_then(|x| vfs.file_id(&x))
}
13 changes: 7 additions & 6 deletions rust/extractor/src/translate/base.rs
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
use super::mappings::{AddressableAst, AddressableHir};
use crate::generated::MacroCall;
use crate::generated::{self};
use crate::rust_analyzer::FileSemanticInformation;
use crate::trap::{DiagnosticSeverity, TrapFile, TrapId};
use crate::trap::{Label, TrapClass};
use codeql_extractor::trap::{self};
Expand Down Expand Up @@ -64,16 +65,15 @@ impl<'a> Translator<'a> {
path: &'a str,
label: trap::Label,
line_index: LineIndex,
file_id: Option<EditionedFileId>,
semantics: Option<&'a Semantics<'a, RootDatabase>>,
semantic_info: Option<&FileSemanticInformation<'a>>,
) -> Translator<'a> {
Translator {
trap,
path,
label,
line_index,
file_id,
semantics,
file_id: semantic_info.map(|i| i.file_id),
semantics: semantic_info.map(|i| i.semantics),
}
}
fn location(&self, range: TextRange) -> (LineCol, LineCol) {
Expand Down Expand Up @@ -160,7 +160,7 @@ impl<'a> Translator<'a> {
self.path,
start.line + 1,
start.col + 1,
&message
&full_message
);
if severity > DiagnosticSeverity::Debug {
let location = self.trap.emit_location_label(self.label, start, end);
Expand Down Expand Up @@ -284,7 +284,8 @@ impl<'a> Translator<'a> {
range.unwrap_or_else(|| TextRange::empty(TextSize::from(0))),
));
}
} else {
} else if self.semantics.is_some() {
// let's not spam warnings if we don't have semantics, we already emitted one
let range = self.text_range_for_node(mcall);
self.emit_parse_error(
mcall,
Expand Down

0 comments on commit 3488b9f

Please sign in to comment.