Merge pull request #171 from Glyphack/import-resolver

Cargo.toml

@@ -1,7 +1,12 @@
 [workspace]
-members = ["parser", "cli", "typechecker"]
+members = ["parser", "cli", "typechecker", "ruff_python_import_resolver"]
 
 
+[workspace.dependencies]
+log = { version = "0.4.17" }
+serde = { version = "1.0.152", features = ["derive"] }
+insta = { version = "1.31.0", feature = ["filters", "glob"] }
+
 [profile.dev.package.insta]
 opt-level = 3
 

cli/src/main.rs

@@ -30,15 +30,30 @@ fn symbols(path: &PathBuf) -> std::result::Result<(), anyhow::Error> {
         source,
         followed: false,
     };
-    let mut manager = BuildManager::new(vec![initial_source], Settings::default());
+    let dir_of_path = path.parent().unwrap();
+    let python_executable = get_python_executable()?;
+    let settings = Settings { debug: true, root: dir_of_path.to_path_buf(), import_discovery: ImportDiscovery { python_executable } };
+
+    let mut manager = BuildManager::new(vec![initial_source], settings);
     manager.build();
-    let module = manager.modules.values().last().unwrap();
 
-    println!("{}", module.get_symbol_table());
+    for (name, module) in manager.modules.iter() {
+        println!("{}", name);
+        println!("{}", module.get_symbol_table());
+    }
 
     Ok(())
 }
 
+fn get_python_executable() -> Result<PathBuf> {
+    let output = std::process::Command::new("python")
+        .arg("-c")
+        .arg("import sys; print(sys.executable)")
+        .output()?;
+    let path = String::from_utf8(output.stdout)?;
+    Ok(PathBuf::from(path))
+}
+
 fn tokenize(file: &PathBuf) -> Result<()> {
     let source = fs::read_to_string(file)?;
     let mut lexer = Lexer::new(&source);
@@ -72,17 +87,10 @@ fn check(path: &PathBuf) -> Result<()> {
         source,
         followed: false,
     };
-    let options = Settings {
-        debug: true,
-        import_discovery: ImportDiscovery {
-            python_executable: PathBuf::from(
-                // TODO: discover executable
-                // Python has sys.executable to do this
-                "'/Users/shooshyari/.pyenv/versions/3.11.3/bin/python'",
-            ),
-        },
-    };
-    let mut build_manager = BuildManager::new(vec![initial_source], options);
+    let dir_of_path = path.parent().unwrap();
+    let python_executable = get_python_executable()?;
+    let settings = Settings { debug: true, root: dir_of_path.to_path_buf(), import_discovery: ImportDiscovery { python_executable } };
+    let mut build_manager = BuildManager::new(vec![initial_source], settings);
     build_manager.build();
 
     // TODO: check the build_manager errors and show to user

parser/src/parser/parser.rs

@@ -1255,7 +1255,7 @@ impl Parser {
         let mut aliases = vec![];
         while self.at(Kind::Identifier) {
             let node = self.start_node();
-            let module = self.parse_module_name();
+            let (module, _) = self.parse_module_name();
             let alias = self.parse_alias(module, node);
             aliases.push(alias);
 
@@ -1273,7 +1273,7 @@ impl Parser {
     fn parse_from_import_statement(&mut self) -> Result<Statement> {
         let import_node = self.start_node();
         self.bump(Kind::From);
-        let module = self.parse_module_name();
+        let (module, level) = self.parse_module_name();
         self.bump(Kind::Import);
         let mut aliases = vec![];
         while self.at(Kind::Identifier) {
@@ -1290,7 +1290,7 @@ impl Parser {
             node: self.finish_node(import_node),
             module,
             names: aliases,
-            level: 0,
+            level,
         }))
     }
 
@@ -1309,15 +1309,19 @@ impl Parser {
         }
     }
 
-    fn parse_module_name(&mut self) -> String {
+    fn parse_module_name(&mut self) -> (String, usize) {
+        let mut level = 0;
+        while self.eat(Kind::Dot) {
+            level += 1;
+        }
         let mut module = self.cur_token().value.to_string();
         self.bump(Kind::Identifier);
         while self.eat(Kind::Dot) {
             module.push('.');
             module.push_str(self.cur_token().value.to_string().as_str());
             self.bump(Kind::Identifier);
         }
-        module
+        (module, level)
     }
 
     // https://docs.python.org/3/library/ast.html#ast.Expr

ruff_python_import_resolver/Cargo.toml

@@ -0,0 +1,13 @@
+[package]
+name = "ruff_python_resolver"
+version = "0.0.0"
+description = "Mirror of Python module resolver for Ruff. Do not use this I don't own the code."
+edition = "2021"
+
+[dependencies]
+log = { workspace = true }
+
+[dev-dependencies]
+env_logger = "0.10.0"
+tempfile = "3.6.0"
+insta = { workspace = true }

ruff_python_import_resolver/resources/test/airflow/README.md

@@ -0,0 +1,3 @@
+# airflow
+
+This is a mock subset of the Airflow repository, used to test module resolution.

ruff_python_import_resolver/resources/test/airflow/airflow/__init__.py

@@ -0,0 +1 @@
+

ruff_python_import_resolver/resources/test/airflow/airflow/api/__init__.py

@@ -0,0 +1 @@
+

ruff_python_import_resolver/resources/test/airflow/airflow/api/common/__init__.py

@@ -0,0 +1 @@
+

ruff_python_import_resolver/resources/test/airflow/airflow/api/common/mark_tasks.py

@@ -0,0 +1,14 @@
+# Standard library.
+import os
+
+# First-party.
+from airflow.jobs.scheduler_job_runner import SchedulerJobRunner
+
+# Stub file.
+from airflow.compat.functools import cached_property
+
+# Namespace package.
+from airflow.providers.google.cloud.hooks.gcs import GCSHook
+
+# Third-party.
+from sqlalchemy.orm import Query

ruff_python_import_resolver/resources/test/airflow/airflow/compat/__init__.py

@@ -0,0 +1,16 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.

ruff_python_import_resolver/resources/test/airflow/airflow/compat/functools.py

@@ -0,0 +1 @@
+"""Empty file included to support filesystem-based resolver tests."""

ruff_python_import_resolver/resources/test/airflow/airflow/compat/functools.pyi

@@ -0,0 +1 @@
+"""Empty file included to support filesystem-based resolver tests."""

ruff_python_import_resolver/resources/test/airflow/airflow/jobs/scheduler_job_runner.py

@@ -0,0 +1 @@
+"""Empty file included to support filesystem-based resolver tests."""

ruff_python_import_resolver/resources/test/airflow/airflow/providers/google/cloud/hooks/gcs.py

@@ -0,0 +1 @@
+"""Empty file included to support filesystem-based resolver tests."""

ruff_python_import_resolver/resources/test/airflow/venv/lib/python3.11/site-packages/_watchdog_fsevents.cpython-311-darwin.so

@@ -0,0 +1 @@
+# Empty file included to support filesystem-based resolver tests.

ruff_python_import_resolver/resources/test/airflow/venv/lib/python3.11/site-packages/orjson/orjson.cpython-311-darwin.so

@@ -0,0 +1 @@
+# Empty file included to support filesystem-based resolver tests.

ruff_python_import_resolver/resources/test/airflow/venv/lib/python3.11/site-packages/sqlalchemy/__init__.py

@@ -0,0 +1 @@
+

ruff_python_import_resolver/resources/test/airflow/venv/lib/python3.11/site-packages/sqlalchemy/orm/base.py

@@ -0,0 +1 @@
+"""Empty file included to support filesystem-based resolver tests."""

ruff_python_import_resolver/resources/test/airflow/venv/lib/python3.11/site-packages/sqlalchemy/orm/dependency.py

@@ -0,0 +1 @@
+"""Empty file included to support filesystem-based resolver tests."""

ruff_python_import_resolver/resources/test/airflow/venv/lib/python3.11/site-packages/sqlalchemy/orm/query.py

@@ -0,0 +1 @@
+"""Empty file included to support filesystem-based resolver tests."""

ruff_python_import_resolver/src/config.rs

@@ -0,0 +1,18 @@
+use std::path::PathBuf;
+
+pub struct Config {
+    /// Path to use for typeshed definitions.
+    pub typeshed_path: Option<PathBuf>,
+
+    /// Path to custom typings (stub) modules.
+    pub stub_path: Option<PathBuf>,
+
+    /// Path to a directory containing one or more virtual environment
+    /// directories. This is used in conjunction with the "venv" name in
+    /// the config file to identify the python environment used for resolving
+    /// third-party modules.
+    pub venv_path: Option<PathBuf>,
+
+    /// Default venv environment.
+    pub venv: Option<PathBuf>,
+}

ruff_python_import_resolver/src/execution_environment.rs

@@ -0,0 +1,19 @@
+use std::path::PathBuf;
+
+use crate::{python_version::PythonVersion, python_platform::PythonPlatform};
+
+
+#[derive(Debug)]
+pub struct ExecutionEnvironment {
+    /// The root directory of the execution environment.
+    pub root: PathBuf,
+
+    /// The Python version of the execution environment.
+    pub python_version: PythonVersion,
+
+    /// The Python platform of the execution environment.
+    pub python_platform: PythonPlatform,
+
+    /// The extra search paths of the execution environment.
+    pub extra_paths: Vec<PathBuf>,
+}

ruff_python_import_resolver/src/host.rs

@@ -0,0 +1,43 @@
+//! Expose the host environment to the resolver.
+
+use std::path::PathBuf;
+
+use crate::python_version::PythonVersion;
+use crate::python_platform::PythonPlatform;
+
+/// A trait to expose the host environment to the resolver.
+pub trait Host {
+    /// The search paths to use when resolving Python modules.
+    fn python_search_paths(&self) -> Vec<PathBuf>;
+
+    /// The Python version to use when resolving Python modules.
+    fn python_version(&self) -> PythonVersion;
+
+    /// The OS platform to use when resolving Python modules.
+    fn python_platform(&self) -> PythonPlatform;
+}
+
+/// A host that exposes a fixed set of search paths.
+pub struct StaticHost {
+    search_paths: Vec<PathBuf>,
+}
+
+impl StaticHost {
+    pub fn new(search_paths: Vec<PathBuf>) -> Self {
+        Self { search_paths }
+    }
+}
+
+impl Host for StaticHost {
+    fn python_search_paths(&self) -> Vec<PathBuf> {
+        self.search_paths.clone()
+    }
+
+    fn python_version(&self) -> PythonVersion {
+        PythonVersion::Py312
+    }
+
+    fn python_platform(&self) -> PythonPlatform {
+        PythonPlatform::Darwin
+    }
+}