diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 971612e0..bc0fb798 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -2,6 +2,12 @@ Changelog ========= +latest +------ + +* Reimplement the graph in Rust. This is a substantial rewrite that, mostly, significantly + improves performance, but there may be certain operations that are slower than before. + 3.5 (2024-10-08) ---------------- diff --git a/rust/Cargo.lock b/rust/Cargo.lock index faeb0261..05270e54 100644 --- a/rust/Cargo.lock +++ b/rust/Cargo.lock @@ -1,29 +1,38 @@ # This file is automatically @generated by Cargo. # It is not intended for manual editing. -version = 3 +version = 4 [[package]] name = "_rustgrimp" version = "0.1.0" dependencies = [ - "log", + "bimap", + "derive-new", + "getset", + "indexmap", + "itertools", + "lazy_static", "pyo3", - "pyo3-log", "rayon", + "rustc-hash", "serde_json", + "slotmap", + "string-interner", + "tap", + "thiserror", ] [[package]] -name = "arc-swap" -version = "1.7.1" +name = "autocfg" +version = "1.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "69f7f8c3906b62b754cd5326047894316021dcfe5a194c8ea52bdd94934a3457" +checksum = "ace50bade8e6234aa140d9a2f552bbee1db4d353f69b8217bc503490fc1a9f26" [[package]] -name = "autocfg" -version = "1.3.0" +name = "bimap" +version = "0.6.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0c4b4d0bd25bd0b74681c0ad21497610ce1b7c91b1022cd21c80c6fbdd9476b0" +checksum = "230c5f1ca6a325a32553f8640d31ac9b49f2411e901e427570154868b46da4f7" [[package]] name = "cfg-if" @@ -33,9 +42,9 @@ checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" [[package]] name = "crossbeam-deque" -version = "0.8.5" +version = "0.8.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "613f8cc01fe9cf1a3eb3d7f488fd2fa8388403e97039e2f73692932e291a770d" +checksum = "9dd111b7b7f7d55b72c0a6ae361660ee5853c9af73f70c3c2ef6858b950e2e51" dependencies = [ "crossbeam-epoch", "crossbeam-utils", @@ -52,9 +61,20 @@ dependencies = [ [[package]] name = "crossbeam-utils" -version = "0.8.20" +version = "0.8.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "22ec99545bb0ed0ea7bb9b8e1e9122ea386ff8a48c0922e43f36d45ab09e0e80" +checksum = "d0a5c400df2834b80a4c3327b3aad3a4c4cd4de0629063962b03235697506a28" + +[[package]] +name = "derive-new" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2cdc8d50f426189eef89dac62fabfa0abb27d5cc008f25bf4156a0203325becc" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] [[package]] name = "either" @@ -62,35 +82,93 @@ version = "1.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "60b1af1c220855b6ceac025d3f6ecdd2b7c4894bfe9cd9bda4fbb4bc7c0d4cf0" +[[package]] +name = "equivalent" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5443807d6dff69373d433ab9ef5378ad8df50ca6298caf15de6e52e24aaf54d5" + +[[package]] +name = "foldhash" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a0d2fde1f7b3d48b8395d5f2de76c18a528bd6a9cdde438df747bfcba3e05d6f" + +[[package]] +name = "getset" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f636605b743120a8d32ed92fc27b6cde1a769f8f936c065151eb66f88ded513c" +dependencies = [ + "proc-macro-error2", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "hashbrown" +version = "0.15.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bf151400ff0baff5465007dd2f3e717f3fe502074ca563069ce3a6629d07b289" +dependencies = [ + "foldhash", +] + [[package]] name = "heck" version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" +[[package]] +name = "indexmap" +version = "2.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8c9c992b02b5b4c94ea26e32fe5bccb7aa7d9f390ab5c1221ff895bc7ea8b652" +dependencies = [ + "equivalent", + "hashbrown", +] + [[package]] name = "indoc" version = "2.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b248f5224d1d606005e02c97f5aa4e88eeb230488bcc03bc9ca4d7991399f2b5" +[[package]] +name = "itertools" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b192c782037fadd9cfa75548310488aabdbf3d2da73885b31bd0abd03351285" +dependencies = [ + "either", +] + [[package]] name = "itoa" -version = "1.0.11" +version = "1.0.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d75a2a4b1b190afb6f5425f10f6a8f959d2ea0b9c2b1d79553551850539e4674" + +[[package]] +name = "lazy_static" +version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "49f1f14873335454500d59611f1cf4a4b0f786f9ac11f4312a78e4cf2566695b" +checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" [[package]] name = "libc" -version = "0.2.155" +version = "0.2.169" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "97b3888a4aecf77e811145cadf6eef5901f4782c53886191b2f693f24761847c" +checksum = "b5aba8db14291edd000dfcc4d620c7ebfb122c613afb886ca8803fa4e128a20a" [[package]] -name = "log" -version = "0.4.22" +name = "memchr" +version = "2.7.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a7a70ba024b9dc04c27ea2f0c0548feb474ec5c54bba33a7f72f873a39d07b24" +checksum = "78ca9ab1a0babb1e7d5695e3530886289c18cf2f87ec19a575a0abdce112e3a3" [[package]] name = "memoffset" @@ -103,30 +181,52 @@ dependencies = [ [[package]] name = "once_cell" -version = "1.19.0" +version = "1.20.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3fdb12b2476b595f9358c5161aa467c2438859caa136dec86c26fdd2efe17b92" +checksum = "1261fe7e33c73b354eab43b1273a57c8f967d0391e80353e51f764ac02cf6775" [[package]] name = "portable-atomic" -version = "1.6.0" +version = "1.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "280dc24453071f1b63954171985a0b0d30058d287960968b9b2aca264c8d4ee6" + +[[package]] +name = "proc-macro-error-attr2" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "96de42df36bb9bba5542fe9f1a054b8cc87e172759a1868aa05c1f3acc89dfc5" +dependencies = [ + "proc-macro2", + "quote", +] + +[[package]] +name = "proc-macro-error2" +version = "2.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7170ef9988bc169ba16dd36a7fa041e5c4cbeb6a35b76d4c03daded371eae7c0" +checksum = "11ec05c52be0a07b08061f7dd003e7d7092e0472bc731b4af7bb1ef876109802" +dependencies = [ + "proc-macro-error-attr2", + "proc-macro2", + "quote", + "syn", +] [[package]] name = "proc-macro2" -version = "1.0.86" +version = "1.0.93" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5e719e8df665df0d1c8fbfd238015744736151d4445ec0836b8e628aae103b77" +checksum = "60946a68e5f9d28b0dc1c21bb8a97ee7d018a8b322fa57838ba31cc878e22d99" dependencies = [ "unicode-ident", ] [[package]] name = "pyo3" -version = "0.22.4" +version = "0.23.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "00e89ce2565d6044ca31a3eb79a334c3a79a841120a98f64eea9f579564cb691" +checksum = "57fe09249128b3173d092de9523eaa75136bf7ba85e0d69eca241c7939c933cc" dependencies = [ "cfg-if", "indoc", @@ -142,9 +242,9 @@ dependencies = [ [[package]] name = "pyo3-build-config" -version = "0.22.4" +version = "0.23.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d8afbaf3abd7325e08f35ffb8deb5892046fcb2608b703db6a583a5ba4cea01e" +checksum = "1cd3927b5a78757a0d71aa9dff669f903b1eb64b54142a9bd9f757f8fde65fd7" dependencies = [ "once_cell", "target-lexicon", @@ -152,30 +252,19 @@ dependencies = [ [[package]] name = "pyo3-ffi" -version = "0.22.4" +version = "0.23.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ec15a5ba277339d04763f4c23d85987a5b08cbb494860be141e6a10a8eb88022" +checksum = "dab6bb2102bd8f991e7749f130a70d05dd557613e39ed2deeee8e9ca0c4d548d" dependencies = [ "libc", "pyo3-build-config", ] -[[package]] -name = "pyo3-log" -version = "0.11.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3ac84e6eec1159bc2a575c9ae6723baa6ee9d45873e9bebad1e3ad7e8d28a443" -dependencies = [ - "arc-swap", - "log", - "pyo3", -] - [[package]] name = "pyo3-macros" -version = "0.22.4" +version = "0.23.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "15e0f01b5364bcfbb686a52fc4181d412b708a68ed20c330db9fc8d2c2bf5a43" +checksum = "91871864b353fd5ffcb3f91f2f703a22a9797c91b9ab497b1acac7b07ae509c7" dependencies = [ "proc-macro2", "pyo3-macros-backend", @@ -185,9 +274,9 @@ dependencies = [ [[package]] name = "pyo3-macros-backend" -version = "0.22.4" +version = "0.23.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a09b550200e1e5ed9176976d0060cbc2ea82dc8515da07885e7b8153a85caacb" +checksum = "43abc3b80bc20f3facd86cd3c60beed58c3e2aa26213f3cda368de39c60a27e4" dependencies = [ "heck", "proc-macro2", @@ -198,9 +287,9 @@ dependencies = [ [[package]] name = "quote" -version = "1.0.36" +version = "1.0.38" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0fa76aaf39101c457836aec0ce2316dbdc3ab723cdda1c6bd4e6ad4208acaca7" +checksum = "0e4dccaaaf89514f546c693ddc140f729f958c247918a13380cccc6078391acc" dependencies = [ "proc-macro2", ] @@ -225,6 +314,12 @@ dependencies = [ "crossbeam-utils", ] +[[package]] +name = "rustc-hash" +version = "2.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c7fb8039b3032c191086b10f11f319a6e99e1e82889c5cc6046f515c9db1d497" + [[package]] name = "ryu" version = "1.0.18" @@ -233,18 +328,18 @@ checksum = "f3cb5ba0dc43242ce17de99c180e96db90b235b8a9fdc9543c96d2209116bd9f" [[package]] name = "serde" -version = "1.0.204" +version = "1.0.217" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bc76f558e0cbb2a839d37354c575f1dc3fdc6546b5be373ba43d95f231bf7c12" +checksum = "02fc4265df13d6fa1d00ecff087228cc0a2b5f3c0e87e258d8b94a156e984c70" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.204" +version = "1.0.217" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e0cd7e117be63d3c3678776753929474f3b04a43a080c744d6b0ae2a8c28e222" +checksum = "5a9bf7cf98d04a2b28aead066b7496853d4779c9cc183c440dbac457641e19a0" dependencies = [ "proc-macro2", "quote", @@ -253,40 +348,92 @@ dependencies = [ [[package]] name = "serde_json" -version = "1.0.120" +version = "1.0.137" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4e0d21c9a8cae1235ad58a00c11cb40d4b1e5c784f1ef2c537876ed6ffd8b7c5" +checksum = "930cfb6e6abf99298aaad7d29abbef7a9999a9a8806a40088f55f0dcec03146b" dependencies = [ "itoa", + "memchr", "ryu", "serde", ] +[[package]] +name = "slotmap" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dbff4acf519f630b3a3ddcfaea6c06b42174d9a44bc70c620e9ed1649d58b82a" +dependencies = [ + "version_check", +] + +[[package]] +name = "string-interner" +version = "0.18.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a3275464d7a9f2d4cac57c89c2ef96a8524dba2864c8d6f82e3980baf136f9b" +dependencies = [ + "hashbrown", + "serde", +] + [[package]] name = "syn" -version = "2.0.70" +version = "2.0.96" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2f0209b68b3613b093e0ec905354eccaedcfe83b8cb37cbdeae64026c3064c16" +checksum = "d5d0adab1ae378d7f53bdebc67a39f1f151407ef230f0ce2883572f5d8985c80" dependencies = [ "proc-macro2", "quote", "unicode-ident", ] +[[package]] +name = "tap" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "55937e1799185b12863d447f42597ed69d9928686b8d88a1df17376a097d8369" + [[package]] name = "target-lexicon" -version = "0.12.15" +version = "0.12.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "61c41af27dd6d1e27b1b16b489db798443478cef1f06a660c96db617ba5de3b1" + +[[package]] +name = "thiserror" +version = "2.0.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d452f284b73e6d76dd36758a0c8684b1d5be31f92b89d07fd5822175732206fc" +dependencies = [ + "thiserror-impl", +] + +[[package]] +name = "thiserror-impl" +version = "2.0.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4873307b7c257eddcb50c9bedf158eb669578359fb28428bef438fec8e6ba7c2" +checksum = "26afc1baea8a989337eeb52b6e72a039780ce45c3edfcc9c5b9d112feeb173c2" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] [[package]] name = "unicode-ident" -version = "1.0.12" +version = "1.0.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3354b9ac3fae1ff6755cb6db53683adb661634f67557942dea4facebec0fee4b" +checksum = "adb9e6ca4f869e1180728b7950e35922a7fc6397f7b641499e8f3ef06e50dc83" [[package]] name = "unindent" version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c7de7d73e1754487cb58364ee906a499937a0dfabd86bcb980fa99ec8c8fa2ce" + +[[package]] +name = "version_check" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" diff --git a/rust/Cargo.toml b/rust/Cargo.toml index fca4e896..048d453e 100644 --- a/rust/Cargo.toml +++ b/rust/Cargo.toml @@ -8,14 +8,25 @@ name = "_rustgrimp" crate-type = ["cdylib", "rlib"] [dependencies] -log = "0.4.19" -pyo3-log = "0.11.0" -serde_json = "1.0.103" rayon = "1.10" +bimap = "0.6.3" +slotmap = "1.0.7" +getset = "0.1.3" +derive-new = "0.7.0" +lazy_static = "1.5.0" +string-interner = "0.18.0" +thiserror = "2.0.11" +itertools = "0.14.0" +tap = "1.0.1" +rustc-hash = "2.1.0" +indexmap = "2.7.1" [dependencies.pyo3] -version = "0.22.4" +version = "0.23.4" [features] extension-module = ["pyo3/extension-module"] default = ["extension-module"] + +[dev-dependencies] +serde_json = "1.0.137" diff --git a/rust/src/containers.rs b/rust/src/containers.rs deleted file mode 100644 index 6f17116d..00000000 --- a/rust/src/containers.rs +++ /dev/null @@ -1,14 +0,0 @@ -use crate::importgraph::ImportGraph; -use std::collections::HashSet; - -pub fn check_containers_exist<'a>( - graph: &'a ImportGraph, - containers: &'a HashSet, -) -> Result<(), String> { - for container in containers { - if !graph.contains_module(container) { - return Err(format!("Container {} does not exist.", container)); - } - } - Ok(()) -} diff --git a/rust/src/dependencies.rs b/rust/src/dependencies.rs deleted file mode 100644 index 5ed9da01..00000000 --- a/rust/src/dependencies.rs +++ /dev/null @@ -1,13 +0,0 @@ -#[derive(PartialEq, Eq, Hash, Debug)] -pub struct Route { - pub heads: Vec, - pub middle: Vec, - pub tails: Vec, -} - -#[derive(PartialEq, Eq, Hash, Debug)] -pub struct PackageDependency { - pub importer: u32, - pub imported: u32, - pub routes: Vec, -} diff --git a/rust/src/errors.rs b/rust/src/errors.rs new file mode 100644 index 00000000..e1ac1ef9 --- /dev/null +++ b/rust/src/errors.rs @@ -0,0 +1,29 @@ +use crate::exceptions::{ModuleNotPresent, NoSuchContainer}; +use pyo3::exceptions::PyValueError; +use pyo3::PyErr; +use thiserror::Error; + +#[derive(Debug, Error)] +pub enum GrimpError { + #[error("Module {0} is not present in the graph.")] + ModuleNotPresent(String), + + #[error("Container {0} does not exist.")] + NoSuchContainer(String), + + #[error("Modules have shared descendants.")] + SharedDescendants, +} + +pub type GrimpResult = Result; + +impl From for PyErr { + fn from(value: GrimpError) -> Self { + // A default mapping from `GrimpError`s to python exceptions. + match value { + GrimpError::ModuleNotPresent(_) => ModuleNotPresent::new_err(value.to_string()), + GrimpError::NoSuchContainer(_) => NoSuchContainer::new_err(value.to_string()), + GrimpError::SharedDescendants => PyValueError::new_err(value.to_string()), + } + } +} diff --git a/rust/src/exceptions.rs b/rust/src/exceptions.rs new file mode 100644 index 00000000..a8aa3e74 --- /dev/null +++ b/rust/src/exceptions.rs @@ -0,0 +1,4 @@ +use pyo3::create_exception; + +create_exception!(_rustgrimp, ModuleNotPresent, pyo3::exceptions::PyException); +create_exception!(_rustgrimp, NoSuchContainer, pyo3::exceptions::PyException); diff --git a/rust/src/graph/direct_import_queries.rs b/rust/src/graph/direct_import_queries.rs new file mode 100644 index 00000000..2023e94d --- /dev/null +++ b/rust/src/graph/direct_import_queries.rs @@ -0,0 +1,57 @@ +use crate::errors::{GrimpError, GrimpResult}; +use crate::graph::{ + ExtendWithDescendants, Graph, ImportDetails, ModuleToken, EMPTY_IMPORT_DETAILS, + EMPTY_MODULE_TOKENS, +}; +use rustc_hash::FxHashSet; + +impl Graph { + pub fn count_imports(&self) -> usize { + self.imports.values().map(|imports| imports.len()).sum() + } + + pub fn direct_import_exists( + &self, + importer: ModuleToken, + imported: ModuleToken, + as_packages: bool, + ) -> GrimpResult { + let mut importer: FxHashSet<_> = importer.into(); + let mut imported: FxHashSet<_> = imported.into(); + if as_packages { + importer.extend_with_descendants(self); + imported.extend_with_descendants(self); + if !(&importer & &imported).is_empty() { + return Err(GrimpError::SharedDescendants); + } + } + + let direct_imports = importer + .iter() + .flat_map(|module| self.imports.get(*module).unwrap().iter().cloned()) + .collect::>(); + + Ok(!(&direct_imports & &imported).is_empty()) + } + + pub fn modules_directly_imported_by(&self, importer: ModuleToken) -> &FxHashSet { + self.imports.get(importer).unwrap_or(&EMPTY_MODULE_TOKENS) + } + + pub fn modules_that_directly_import(&self, imported: ModuleToken) -> &FxHashSet { + self.reverse_imports + .get(imported) + .unwrap_or(&EMPTY_MODULE_TOKENS) + } + + pub fn get_import_details( + &self, + importer: ModuleToken, + imported: ModuleToken, + ) -> &FxHashSet { + match self.import_details.get(&(importer, imported)) { + Some(import_details) => import_details, + None => &EMPTY_IMPORT_DETAILS, + } + } +} diff --git a/rust/src/graph/graph_manipulation.rs b/rust/src/graph/graph_manipulation.rs new file mode 100644 index 00000000..41428236 --- /dev/null +++ b/rust/src/graph/graph_manipulation.rs @@ -0,0 +1,195 @@ +use crate::graph::{Graph, ImportDetails, Module, ModuleIterator, ModuleToken, MODULE_NAMES}; +use rustc_hash::FxHashSet; +use slotmap::secondary::Entry; + +impl Graph { + /// `foo.bar.baz => [foo.bar.baz, foo.bar, foo]` + pub(crate) fn module_name_to_self_and_ancestors(&self, name: &str) -> Vec { + let mut names = vec![name.to_owned()]; + while let Some(parent_name) = parent_name(names.last().unwrap()) { + names.push(parent_name); + } + names + } + + pub fn get_or_add_module(&mut self, name: &str) -> &Module { + if let Some(module) = self.get_module_by_name(name) { + let module = self.modules.get_mut(module.token).unwrap(); + module.is_invisible = false; + return module; + } + + let mut ancestor_names = self.module_name_to_self_and_ancestors(name); + + { + let mut interner = MODULE_NAMES.write().unwrap(); + let mut parent: Option = None; + while let Some(name) = ancestor_names.pop() { + let name = interner.get_or_intern(name); + if let Some(module) = self.modules_by_name.get_by_left(&name) { + parent = Some(*module) + } else { + let module = self.modules.insert_with_key(|token| Module { + token, + name, + is_invisible: !ancestor_names.is_empty(), + is_squashed: false, + }); + self.modules_by_name.insert(name, module); + self.module_parents.insert(module, parent); + self.module_children.insert(module, FxHashSet::default()); + self.imports.insert(module, FxHashSet::default()); + self.reverse_imports.insert(module, FxHashSet::default()); + if let Some(parent) = parent { + self.module_children[parent].insert(module); + } + parent = Some(module) + } + } + } + + self.get_module_by_name(name).unwrap() + } + + pub fn get_or_add_squashed_module(&mut self, module: &str) -> &Module { + let module = self.get_or_add_module(module).token(); + self.mark_module_squashed(module); + self.get_module(module).unwrap() + } + + fn mark_module_squashed(&mut self, module: ModuleToken) { + let module = self.modules.get_mut(module).unwrap(); + if !self.module_children[module.token].is_empty() { + panic!("cannot mark a module with children as squashed") + } + module.is_squashed = true; + } + + pub fn remove_module(&mut self, module: ModuleToken) { + let module = self.get_module(module); + if module.is_none() { + return; + } + let module = module.unwrap().token(); + + // TODO(peter) Remove children automatically here, or raise an error? + if !self.module_children[module].is_empty() { + for child in self.module_children[module].clone() { + self.remove_module(child); + } + } + + // Update hierarchy. + if let Some(parent) = self.module_parents[module] { + self.module_children[parent].remove(&module); + } + self.modules_by_name.remove_by_right(&module); + self.modules.remove(module); + self.module_parents.remove(module); + self.module_children.remove(module); + + // Update imports. + for imported in self.modules_directly_imported_by(module).clone() { + self.remove_import(module, imported); + } + for importer in self.modules_that_directly_import(module).clone() { + self.remove_import(importer, module); + } + self.imports.remove(module); + self.reverse_imports.remove(module); + } + + pub fn add_import(&mut self, importer: ModuleToken, imported: ModuleToken) { + self.imports + .entry(importer) + .unwrap() + .or_default() + .insert(imported); + self.reverse_imports + .entry(imported) + .unwrap() + .or_default() + .insert(importer); + } + + pub fn add_detailed_import( + &mut self, + importer: ModuleToken, + imported: ModuleToken, + line_number: usize, + line_contents: &str, + ) { + self.imports + .entry(importer) + .unwrap() + .or_default() + .insert(imported); + self.reverse_imports + .entry(imported) + .unwrap() + .or_default() + .insert(importer); + self.import_details + .entry((importer, imported)) + .or_default() + .insert(ImportDetails::new(line_number, line_contents.to_owned())); + } + + pub fn remove_import(&mut self, importer: ModuleToken, imported: ModuleToken) { + match self.imports.entry(importer).unwrap() { + Entry::Occupied(mut entry) => { + entry.get_mut().remove(&imported); + } + Entry::Vacant(_) => {} + }; + match self.reverse_imports.entry(imported).unwrap() { + Entry::Occupied(mut entry) => { + entry.get_mut().remove(&importer); + } + Entry::Vacant(_) => {} + }; + self.import_details.remove(&(importer, imported)); + } + + pub fn squash_module(&mut self, module: ModuleToken) { + // Get descendants and their imports. + let descendants: FxHashSet<_> = self.get_module_descendants(module).tokens().collect(); + + let modules_imported_by_descendants: FxHashSet<_> = descendants + .iter() + .flat_map(|descendant| { + self.modules_directly_imported_by(*descendant) + .iter() + .cloned() + }) + .collect(); + let modules_that_import_descendants: FxHashSet<_> = descendants + .iter() + .flat_map(|descendant| { + self.modules_that_directly_import(*descendant) + .iter() + .cloned() + }) + .collect(); + + // Remove any descendants. + for descendant in descendants { + self.remove_module(descendant); + } + + // Add descendants and imports to parent module. + for imported in modules_imported_by_descendants { + self.add_import(module, imported); + } + + for importer in modules_that_import_descendants { + self.add_import(importer, module); + } + + self.mark_module_squashed(module); + } +} + +fn parent_name(name: &str) -> Option { + name.rsplit_once(".").map(|(base, _)| base.to_owned()) +} diff --git a/rust/src/graph/hierarchy_queries.rs b/rust/src/graph/hierarchy_queries.rs new file mode 100644 index 00000000..41cdf570 --- /dev/null +++ b/rust/src/graph/hierarchy_queries.rs @@ -0,0 +1,50 @@ +use crate::graph::{Graph, Module, ModuleToken, MODULE_NAMES}; + +impl Graph { + pub fn get_module_by_name(&self, name: &str) -> Option<&Module> { + let interner = MODULE_NAMES.read().unwrap(); + let name = interner.get(name)?; + match self.modules_by_name.get_by_left(&name) { + Some(token) => self.get_module(*token), + None => None, + } + } + + pub fn get_module(&self, module: ModuleToken) -> Option<&Module> { + self.modules.get(module) + } + + // TODO(peter) Guarantee order? + pub fn all_modules(&self) -> impl Iterator { + self.modules.values() + } + + pub fn get_module_parent(&self, module: ModuleToken) -> Option<&Module> { + match self.module_parents.get(module) { + Some(parent) => parent.map(|parent| self.get_module(parent).unwrap()), + None => None, + } + } + + pub fn get_module_children(&self, module: ModuleToken) -> impl Iterator { + let children = match self.module_children.get(module) { + Some(children) => children + .iter() + .map(|child| self.get_module(*child).unwrap()) + .collect(), + None => Vec::new(), + }; + children.into_iter() + } + + /// Returns an iterator over the passed modules descendants. + /// + /// Parent modules will be yielded before their child modules. + pub fn get_module_descendants(&self, module: ModuleToken) -> impl Iterator { + let mut descendants = self.get_module_children(module).collect::>(); + for child in descendants.clone() { + descendants.extend(self.get_module_descendants(child.token).collect::>()) + } + descendants.into_iter() + } +} diff --git a/rust/src/graph/higher_order_queries.rs b/rust/src/graph/higher_order_queries.rs new file mode 100644 index 00000000..ebb1ba71 --- /dev/null +++ b/rust/src/graph/higher_order_queries.rs @@ -0,0 +1,210 @@ +use crate::errors::GrimpResult; +use crate::graph::{ExtendWithDescendants, Graph, ModuleToken}; +use derive_new::new; +use getset::Getters; +use itertools::Itertools; +use rayon::prelude::*; +use rustc_hash::{FxHashMap, FxHashSet}; + +use tap::prelude::*; + +#[derive(Debug, Clone, PartialEq, Eq, new, Getters)] +pub struct Level { + #[new(into)] + #[getset(get = "pub")] + layers: FxHashSet, + + #[getset(get_copy = "pub")] + independent: bool, +} + +#[derive(Debug, Clone, PartialEq, Eq, new, Getters)] +pub struct PackageDependency { + #[getset(get = "pub")] + importer: ModuleToken, + + #[getset(get = "pub")] + imported: ModuleToken, + + #[new(into)] + #[getset(get = "pub")] + routes: Vec, +} + +#[derive(Debug, Clone, PartialEq, Eq, new, Getters)] +pub struct Route { + #[new(into)] + #[getset(get = "pub")] + heads: FxHashSet, + + #[new(into)] + #[getset(get = "pub")] + middle: Vec, + + #[new(into)] + #[getset(get = "pub")] + tails: FxHashSet, +} + +impl Graph { + pub fn find_illegal_dependencies_for_layers( + &self, + levels: &[Level], + ) -> GrimpResult> { + let all_layer_modules = levels + .iter() + .flat_map(|level| level.layers().clone()) + .flat_map(|m| m.conv::>().with_descendants(self)) + .collect::>(); + + self.generate_module_permutations(levels) + .into_par_iter() + .try_fold( + Vec::new, + |mut v: Vec, (from_package, to_package)| -> GrimpResult<_> { + if let Some(dep) = self.find_illegal_dependencies( + from_package, + to_package, + &all_layer_modules, + )? { + v.push(dep); + } + Ok(v) + }, + ) + .try_reduce( + Vec::new, + |mut v: Vec, package_dependencies| { + v.extend(package_dependencies); + Ok(v) + }, + ) + } + + fn generate_module_permutations(&self, levels: &[Level]) -> Vec<(ModuleToken, ModuleToken)> { + let mut permutations = vec![]; + + for (index, level) in levels.iter().enumerate() { + for module in &level.layers { + // Should not be imported by lower layers. + for lower_level in &levels[index + 1..] { + for lower_module in &lower_level.layers { + permutations.push((*lower_module, *module)); + } + } + + // Should not import siblings (if level is independent) + if level.independent { + for sibling_module in &level.layers { + if sibling_module == module { + continue; + } + permutations.push((*module, *sibling_module)); + } + } + } + } + + permutations + } + + fn find_illegal_dependencies( + &self, + from_layer: ModuleToken, + to_layer: ModuleToken, + all_layers_modules: &FxHashSet, + ) -> GrimpResult> { + // Shortcut the detailed implementation in the case of no chains. + // This will be much faster! + if !self.chain_exists(from_layer, to_layer, true)? { + return Ok(None); + } + + let from_layer_with_descendants = from_layer.conv::>().with_descendants(self); + let to_layer_with_descendants = to_layer.conv::>().with_descendants(self); + + // Disallow chains via other layers. + let excluded_modules = + all_layers_modules - &(&from_layer_with_descendants | &to_layer_with_descendants); + + // Disallow chains via these imports. + // We'll add chains to this set as we discover them. + let mut excluded_imports = FxHashMap::default(); + + // Collect direct imports... + let mut direct_imports = vec![]; + // ...and the middles of any indirect imports. + let mut middles = vec![]; + loop { + let chain = self.find_shortest_chain_with_excluded_modules_and_imports( + &from_layer_with_descendants, + &to_layer_with_descendants, + &excluded_modules, + &excluded_imports, + )?; + + if chain.is_none() { + break; + } + let chain = chain.unwrap(); + + // Exclude this chain from further searching. + for (importer, imported) in chain.iter().tuple_windows() { + excluded_imports + .entry(*importer) + .or_default() + .insert(*imported); + } + + let (head, middle, tail) = self.split_chain(&chain); + match middle { + Some(middle) => middles.push(middle), + None => direct_imports.push((head, tail)), + } + } + + // Map to routes. + let mut routes = vec![]; + for (importer, imported) in direct_imports { + routes.push(Route::new(importer, vec![], imported)); + } + for middle in middles { + let heads = from_layer_with_descendants + .iter() + .filter(|importer| { + self.direct_import_exists(**importer, *middle.first().unwrap(), false) + .unwrap() + }) + .cloned() + .collect::>(); + let tails = to_layer_with_descendants + .iter() + .filter(|imported| { + self.direct_import_exists(*middle.last().unwrap(), **imported, false) + .unwrap() + }) + .cloned() + .collect::>(); + routes.push(Route::new(heads, middle, tails)); + } + + match routes.is_empty() { + true => Ok(None), + false => Ok(Some(PackageDependency::new(from_layer, to_layer, routes))), + } + } + + fn split_chain( + &self, + chain: &[ModuleToken], + ) -> (ModuleToken, Option>, ModuleToken) { + if chain.len() == 2 { + return (chain[0], None, chain[1]); + } + ( + chain[0], + Some(chain[1..chain.len() - 1].to_vec()), + chain[chain.len() - 1], + ) + } +} diff --git a/rust/src/graph/import_chain_queries.rs b/rust/src/graph/import_chain_queries.rs new file mode 100644 index 00000000..4f21da48 --- /dev/null +++ b/rust/src/graph/import_chain_queries.rs @@ -0,0 +1,123 @@ +use crate::errors::GrimpResult; +use crate::graph::pathfinding::{find_reach, find_shortest_path_bidirectional}; +use crate::graph::{ExtendWithDescendants, Graph, ModuleToken}; +use itertools::Itertools; +use rustc_hash::{FxHashMap, FxHashSet}; +use tap::Conv; + +impl Graph { + pub fn find_downstream_modules( + &self, + module: ModuleToken, + as_package: bool, + ) -> FxHashSet { + let mut from_modules = module.conv::>(); + if as_package { + from_modules.extend_with_descendants(self); + } + + find_reach(&self.reverse_imports, &from_modules) + } + + pub fn find_upstream_modules( + &self, + module: ModuleToken, + as_package: bool, + ) -> FxHashSet { + let mut from_modules = module.conv::>(); + if as_package { + from_modules.extend_with_descendants(self); + } + + find_reach(&self.imports, &from_modules) + } + + pub fn find_shortest_chain( + &self, + importer: ModuleToken, + imported: ModuleToken, + as_packages: bool, + ) -> GrimpResult>> { + if as_packages { + self.find_shortest_chain_with_excluded_modules_and_imports( + &importer.conv::>().with_descendants(self), + &imported.conv::>().with_descendants(self), + &FxHashSet::default(), + &FxHashMap::default(), + ) + } else { + self.find_shortest_chain_with_excluded_modules_and_imports( + &importer.conv::>(), + &imported.conv::>(), + &FxHashSet::default(), + &FxHashMap::default(), + ) + } + } + + pub(crate) fn find_shortest_chain_with_excluded_modules_and_imports( + &self, + from_modules: &FxHashSet, + to_modules: &FxHashSet, + excluded_modules: &FxHashSet, + excluded_imports: &FxHashMap>, + ) -> GrimpResult>> { + find_shortest_path_bidirectional( + self, + from_modules, + to_modules, + excluded_modules, + excluded_imports, + ) + } + + pub fn chain_exists( + &self, + importer: ModuleToken, + imported: ModuleToken, + as_packages: bool, + ) -> GrimpResult { + Ok(self + .find_shortest_chain(importer, imported, as_packages)? + .is_some()) + } + + pub fn find_shortest_chains( + &self, + importer: ModuleToken, + imported: ModuleToken, + as_packages: bool, + ) -> GrimpResult>> { + // Shortcut the detailed implementation in the case of no chains. + // This will be much faster! + if !self.chain_exists(importer, imported, as_packages)? { + return Ok(FxHashSet::default()); + } + + let mut downstream_modules = FxHashSet::from_iter([importer]); + let mut upstream_modules = FxHashSet::from_iter([imported]); + if as_packages { + downstream_modules.extend_with_descendants(self); + upstream_modules.extend_with_descendants(self); + } + let all_modules = &downstream_modules | &upstream_modules; + + let chains = downstream_modules + .iter() + .cartesian_product(upstream_modules.iter()) + .filter_map(|(downstream_module, upstream_module)| { + let excluded_modules = + &all_modules - &FxHashSet::from_iter([*downstream_module, *upstream_module]); + self.find_shortest_chain_with_excluded_modules_and_imports( + &(*downstream_module).into(), + &(*upstream_module).into(), + &excluded_modules, + &FxHashMap::default(), + ) + .unwrap() + }) + .collect(); + + Ok(chains) + } +} diff --git a/rust/src/graph/mod.rs b/rust/src/graph/mod.rs new file mode 100644 index 00000000..cdbd5380 --- /dev/null +++ b/rust/src/graph/mod.rs @@ -0,0 +1,137 @@ +use bimap::BiMap; +use derive_new::new; +use getset::{CopyGetters, Getters}; +use lazy_static::lazy_static; +use rustc_hash::{FxHashMap, FxHashSet}; +use slotmap::{new_key_type, SecondaryMap, SlotMap}; +use std::sync::RwLock; +use string_interner::backend::StringBackend; +use string_interner::{DefaultSymbol, StringInterner}; + +pub mod direct_import_queries; +pub mod graph_manipulation; +pub mod hierarchy_queries; +pub mod higher_order_queries; +pub mod import_chain_queries; + +pub(crate) mod pathfinding; + +lazy_static! { + static ref MODULE_NAMES: RwLock> = + RwLock::new(StringInterner::default()); + static ref EMPTY_MODULE_TOKENS: FxHashSet = FxHashSet::default(); + static ref EMPTY_IMPORT_DETAILS: FxHashSet = FxHashSet::default(); + static ref EMPTY_IMPORTS: FxHashSet<(ModuleToken, ModuleToken)> = FxHashSet::default(); +} + +new_key_type! { pub struct ModuleToken; } + +#[derive(Debug, Clone, PartialEq, Eq, Hash, Getters, CopyGetters)] +pub struct Module { + #[getset(get_copy = "pub")] + token: ModuleToken, + + #[getset(get_copy = "pub")] + name: DefaultSymbol, + + // Invisible modules exist in the hierarchy but haven't been explicitly added to the graph. + #[getset(get_copy = "pub")] + is_invisible: bool, + + #[getset(get_copy = "pub")] + is_squashed: bool, +} + +impl Module { + pub fn name_as_string(&self) -> String { + let interner = MODULE_NAMES.read().unwrap(); + interner.resolve(self.name).unwrap().to_owned() + } +} + +#[derive(Default, Clone)] +pub struct Graph { + // Hierarchy + modules_by_name: BiMap, + modules: SlotMap, + module_parents: SecondaryMap>, + module_children: SecondaryMap>, + // Imports + imports: SecondaryMap>, + reverse_imports: SecondaryMap>, + import_details: FxHashMap<(ModuleToken, ModuleToken), FxHashSet>, +} + +impl From for Vec { + fn from(value: ModuleToken) -> Self { + vec![value] + } +} + +impl From for FxHashSet { + fn from(value: ModuleToken) -> Self { + FxHashSet::from_iter([value]) + } +} + +pub trait ExtendWithDescendants: + Sized + Clone + IntoIterator + Extend +{ + /// Extend this collection of module tokens with all descendant items. + fn extend_with_descendants(&mut self, graph: &Graph) { + for item in self.clone().into_iter() { + let descendants = graph.get_module_descendants(item).map(|item| item.token()); + self.extend(descendants); + } + } + + /// Extend this collection of module tokens with all descendant items. + fn with_descendants(mut self, graph: &Graph) -> Self { + self.extend_with_descendants(graph); + self + } +} + +impl + Extend> + ExtendWithDescendants for T +{ +} + +pub trait ModuleIterator<'a>: Iterator + Sized { + fn tokens(self) -> impl Iterator { + self.map(|m| m.token) + } + + fn names(self) -> impl Iterator { + self.map(|m| m.name) + } + + fn names_as_strings(self) -> impl Iterator { + let interner = MODULE_NAMES.read().unwrap(); + self.map(move |m| interner.resolve(m.name).unwrap().to_owned()) + } + + fn visible(self) -> impl ModuleIterator<'a> { + self.filter(|m| !m.is_invisible) + } +} + +impl<'a, T: Iterator> ModuleIterator<'a> for T {} + +pub trait ModuleTokenIterator<'a>: Iterator + Sized { + fn into_module_iterator(self, graph: &'a Graph) -> impl ModuleIterator<'a> { + self.map(|m| graph.get_module(*m).unwrap()) + } +} + +impl<'a, T: Iterator> ModuleTokenIterator<'a> for T {} + +#[derive(Debug, Clone, PartialEq, Eq, Hash, new, Getters, CopyGetters)] +pub struct ImportDetails { + #[getset(get_copy = "pub")] + line_number: usize, + + #[new(into)] + #[getset(get = "pub")] + line_contents: String, +} diff --git a/rust/src/graph/pathfinding.rs b/rust/src/graph/pathfinding.rs new file mode 100644 index 00000000..55e9450d --- /dev/null +++ b/rust/src/graph/pathfinding.rs @@ -0,0 +1,125 @@ +use crate::errors::{GrimpError, GrimpResult}; +use crate::graph::{Graph, ModuleToken, EMPTY_MODULE_TOKENS}; +use indexmap::{IndexMap, IndexSet}; +use rustc_hash::{FxHashMap, FxHashSet, FxHasher}; +use slotmap::SecondaryMap; +use std::hash::BuildHasherDefault; + +type FxIndexSet = IndexSet>; +type FxIndexMap = IndexMap>; + +pub fn find_reach( + imports_map: &SecondaryMap>, + from_modules: &FxHashSet, +) -> FxHashSet { + let mut seen = FxIndexSet::default(); + seen.extend(from_modules.iter().cloned()); + + let mut i = 0; + while let Some(module) = seen.get_index(i) { + for next_module in imports_map.get(*module).unwrap_or(&EMPTY_MODULE_TOKENS) { + if !seen.contains(next_module) { + seen.insert(*next_module); + } + } + i += 1; + } + + &seen.into_iter().collect::>() - from_modules +} + +pub fn find_shortest_path_bidirectional( + graph: &Graph, + from_modules: &FxHashSet, + to_modules: &FxHashSet, + excluded_modules: &FxHashSet, + excluded_imports: &FxHashMap>, +) -> GrimpResult>> { + if !(from_modules & to_modules).is_empty() { + return Err(GrimpError::SharedDescendants); + } + + let mut predecessors: FxIndexMap> = from_modules + .clone() + .into_iter() + .map(|m| (m, None)) + .collect(); + let mut successors: FxIndexMap> = + to_modules.clone().into_iter().map(|m| (m, None)).collect(); + + let mut i_forwards = 0; + let mut i_backwards = 0; + let middle = 'l: loop { + for _ in 0..(predecessors.len() - i_forwards) { + let module = *predecessors.get_index(i_forwards).unwrap().0; + let next_modules = graph.imports.get(module).unwrap(); + for next_module in next_modules { + if import_is_excluded(&module, next_module, excluded_modules, excluded_imports) { + continue; + } + if !predecessors.contains_key(next_module) { + predecessors.insert(*next_module, Some(module)); + } + if successors.contains_key(next_module) { + break 'l Some(*next_module); + } + } + i_forwards += 1; + } + + for _ in 0..(successors.len() - i_backwards) { + let module = *successors.get_index(i_backwards).unwrap().0; + let next_modules = graph.reverse_imports.get(module).unwrap(); + for next_module in next_modules { + if import_is_excluded(next_module, &module, excluded_modules, excluded_imports) { + continue; + } + if !successors.contains_key(next_module) { + successors.insert(*next_module, Some(module)); + } + if predecessors.contains_key(next_module) { + break 'l Some(*next_module); + } + } + i_backwards += 1; + } + + if i_forwards == predecessors.len() && i_backwards == successors.len() { + break 'l None; + } + }; + + Ok(middle.map(|middle| { + // Path found! + // Build the path. + let mut path = vec![]; + let mut node = Some(middle); + while let Some(n) = node { + path.push(n); + node = *predecessors.get(&n).unwrap(); + } + path.reverse(); + let mut node = *successors.get(path.last().unwrap()).unwrap(); + while let Some(n) = node { + path.push(n); + node = *successors.get(&n).unwrap(); + } + path + })) +} + +fn import_is_excluded( + from_module: &ModuleToken, + to_module: &ModuleToken, + excluded_modules: &FxHashSet, + excluded_imports: &FxHashMap>, +) -> bool { + if excluded_modules.contains(to_module) { + true + } else { + excluded_imports + .get(from_module) + .unwrap_or(&EMPTY_MODULE_TOKENS) + .contains(to_module) + } +} diff --git a/rust/src/importgraph.rs b/rust/src/importgraph.rs deleted file mode 100644 index cf58aaf9..00000000 --- a/rust/src/importgraph.rs +++ /dev/null @@ -1,435 +0,0 @@ -use std::collections::hash_map::Entry::Vacant; -use std::collections::{HashMap, HashSet}; -use std::fmt; - -#[derive(Clone)] -pub struct ImportGraph<'a> { - pub names_by_id: HashMap, - pub ids_by_name: HashMap<&'a str, u32>, - pub importers_by_imported: HashMap>, - pub importeds_by_importer: HashMap>, -} - -impl<'a> ImportGraph<'a> { - pub fn new(importeds_by_importer: HashMap<&'a str, HashSet<&'a str>>) -> ImportGraph<'a> { - // Build the name/id lookup maps. - let mut names_by_id: HashMap = HashMap::new(); - let mut ids_by_name: HashMap<&'a str, u32> = HashMap::new(); - let mut current_id: u32 = 1; - for name in importeds_by_importer.keys() { - names_by_id.insert(current_id, name); - ids_by_name.insert(name, current_id); - current_id += 1; - } - - // Convert importeds_by_importer to id-based. - let mut importeds_by_importer_u32: HashMap> = HashMap::new(); - for (importer_str, importeds_strs) in importeds_by_importer.iter() { - let mut importeds_u32 = HashSet::new(); - for imported_str in importeds_strs { - importeds_u32.insert(*ids_by_name.get(imported_str).unwrap()); - } - - importeds_by_importer_u32 - .insert(*ids_by_name.get(importer_str).unwrap(), importeds_u32); - } - - let importers_by_imported_u32 = - ImportGraph::_build_importers_by_imported_u32(&importeds_by_importer_u32); - - ImportGraph { - names_by_id, - ids_by_name, - importers_by_imported: importers_by_imported_u32, - importeds_by_importer: importeds_by_importer_u32, - } - } - - fn _build_importers_by_imported_u32( - importeds_by_importer_u32: &HashMap>, - ) -> HashMap> { - // Build importers_by_imported from importeds_by_importer. - let mut importers_by_imported_u32: HashMap> = HashMap::new(); - for (importer, importeds) in importeds_by_importer_u32.iter() { - for imported in importeds { - let entry = importers_by_imported_u32.entry(*imported).or_default(); - entry.insert(*importer); - } - } - - // Check that there is an empty set for any remaining. - for importer in importeds_by_importer_u32.keys() { - importers_by_imported_u32.entry(*importer).or_default(); - } - importers_by_imported_u32 - } - - pub fn get_module_ids(&self) -> HashSet { - self.names_by_id.keys().copied().collect() - } - - pub fn contains_module(&self, module_name: &str) -> bool { - self.ids_by_name.contains_key(module_name) - } - - pub fn remove_import(&mut self, importer: &str, imported: &str) { - self.remove_import_ids(self.ids_by_name[importer], self.ids_by_name[imported]); - } - - pub fn add_import_ids(&mut self, importer: u32, imported: u32) { - let importeds = self.importeds_by_importer.get_mut(&importer).unwrap(); - importeds.insert(imported); - - let importers = self.importers_by_imported.get_mut(&imported).unwrap(); - importers.insert(importer); - } - - pub fn remove_import_ids(&mut self, importer: u32, imported: u32) { - let importeds = self.importeds_by_importer.get_mut(&importer).unwrap(); - importeds.remove(&imported); - - let importers = self.importers_by_imported.get_mut(&imported).unwrap(); - importers.remove(&importer); - } - - pub fn remove_module_by_id(&mut self, module_id: u32) { - let _module = self.names_by_id[&module_id]; - - let mut imports_to_remove = Vec::with_capacity(self.names_by_id.len()); - { - for imported_id in &self.importeds_by_importer[&module_id] { - imports_to_remove.push((module_id, *imported_id)); - } - for importer_id in &self.importers_by_imported[&module_id] { - imports_to_remove.push((*importer_id, module_id)); - } - } - - for (importer, imported) in imports_to_remove { - self.remove_import_ids(importer, imported); - } - - self.importeds_by_importer.remove(&module_id); - self.importers_by_imported.remove(&module_id); - } - - pub fn get_descendant_ids(&self, module_name: &str) -> Vec { - let mut descendant_ids = vec![]; - let namespace: String = format!("{}.", module_name); - for (candidate_name, candidate_id) in &self.ids_by_name { - if candidate_name.starts_with(&namespace) { - descendant_ids.push(*candidate_id); - } - } - descendant_ids - } - - pub fn remove_package(&mut self, module_name: &str) { - for descendant_id in self.get_descendant_ids(module_name) { - self.remove_module_by_id(descendant_id); - } - self.remove_module_by_id(self.ids_by_name[&module_name]); - } - - pub fn squash_module(&mut self, module_name: &str) { - let squashed_root_id = self.ids_by_name[module_name]; - let descendant_ids = &self.get_descendant_ids(module_name); - - // Assemble imports to add first, then add them in a second loop, - // to avoid needing to clone importeds_by_importer. - let mut imports_to_add = Vec::with_capacity(self.names_by_id.len()); - // Imports from the root. - { - for descendant_id in descendant_ids { - for imported_id in &self.importeds_by_importer[&descendant_id] { - imports_to_add.push((squashed_root_id, *imported_id)); - } - for importer_id in &self.importers_by_imported[&descendant_id] { - imports_to_add.push((*importer_id, squashed_root_id)); - } - } - } - - for (importer, imported) in imports_to_add { - self.add_import_ids(importer, imported); - } - - // Now we've added imports to/from the root, we can delete the root's descendants. - for descendant_id in descendant_ids { - self.remove_module_by_id(*descendant_id); - } - } - - pub fn pop_shortest_chains(&mut self, importer: &str, imported: &str) -> Vec> { - let mut chains = vec![]; - let importer_id = self.ids_by_name[&importer]; - let imported_id = self.ids_by_name[&imported]; - - while let Some(chain) = self.find_shortest_chain(importer_id, imported_id) { - // Remove chain - let _mods: Vec<&str> = chain.iter().map(|i| self.names_by_id[&i]).collect(); - for i in 0..chain.len() - 1 { - self.remove_import_ids(chain[i], chain[i + 1]); - } - chains.push(chain); - } - - chains - } - - pub fn find_shortest_chain(&self, importer_id: u32, imported_id: u32) -> Option> { - let results_or_none = self._search_for_path(importer_id, imported_id); - match results_or_none { - Some(results) => { - let (pred, succ, initial_w) = results; - - let mut w_or_none: Option = Some(initial_w); - // Transform results into vector. - let mut path: Vec = Vec::new(); - // From importer to w: - while w_or_none.is_some() { - let w = w_or_none.unwrap(); - path.push(w); - w_or_none = pred[&w]; - } - path.reverse(); - - // From w to imported: - w_or_none = succ[path.last().unwrap()]; - while w_or_none.is_some() { - let w = w_or_none.unwrap(); - path.push(w); - w_or_none = succ[&w]; - } - - Some(path) - } - None => None, - } - } - /// Performs a breadth first search from both source and target, meeting in the middle. - // - // Returns: - // (pred, succ, w) where - // - pred is a dictionary of predecessors from w to the source, and - // - succ is a dictionary of successors from w to the target. - // - fn _search_for_path( - &self, - importer: u32, - imported: u32, - ) -> Option<(HashMap>, HashMap>, u32)> { - if importer == imported { - Some(( - HashMap::from([(imported, None)]), - HashMap::from([(importer, None)]), - importer, - )) - } else { - let mut pred: HashMap> = HashMap::from([(importer, None)]); - let mut succ: HashMap> = HashMap::from([(imported, None)]); - - // Initialize fringes, start with forward. - let mut forward_fringe: Vec = Vec::from([importer]); - let mut reverse_fringe: Vec = Vec::from([imported]); - let mut this_level: Vec; - - while !forward_fringe.is_empty() && !reverse_fringe.is_empty() { - if forward_fringe.len() <= reverse_fringe.len() { - this_level = forward_fringe.to_vec(); - forward_fringe = Vec::new(); - for v in this_level { - for w in self.importeds_by_importer[&v].clone() { - pred.entry(w).or_insert_with(|| { - forward_fringe.push(w); - Some(v) - }); - if succ.contains_key(&w) { - // Found path. - return Some((pred, succ, w)); - } - } - } - } else { - this_level = reverse_fringe.to_vec(); - reverse_fringe = Vec::new(); - for v in this_level { - for w in self.importers_by_imported[&v].clone() { - if let Vacant(e) = succ.entry(w) { - e.insert(Some(v)); - reverse_fringe.push(w); - } - if pred.contains_key(&w) { - // Found path. - return Some((pred, succ, w)); - } - } - } - } - } - None - } - } -} - -impl fmt::Display for ImportGraph<'_> { - fn fmt(&self, dest: &mut fmt::Formatter) -> fmt::Result { - let mut strings = vec![]; - for (importer, importeds) in self.importeds_by_importer.iter() { - let mut string = format!("IMPORTER {}: ", self.names_by_id[&importer]); - for imported in importeds { - string.push_str(format!("{}, ", self.names_by_id[&imported]).as_str()); - } - strings.push(string); - } - strings.push(" ".to_string()); - for (imported, importers) in self.importers_by_imported.iter() { - let mut string = format!("IMPORTED {}: ", self.names_by_id[&imported]); - for importer in importers { - string.push_str(format!("{}, ", self.names_by_id[&importer]).as_str()); - } - strings.push(string); - } - write!(dest, "{}", strings.join("\n")) - } -} - -#[cfg(test)] -mod tests { - use super::*; - - fn _make_graph() -> ImportGraph<'static> { - ImportGraph::new(HashMap::from([ - ("blue", HashSet::from(["blue.alpha", "blue.beta", "green"])), - ("blue.alpha", HashSet::new()), - ("blue.beta", HashSet::new()), - ("green", HashSet::from(["blue.alpha", "blue.beta"])), - ])) - } - - #[test] - fn get_module_ids() { - let graph = _make_graph(); - - assert_eq!( - graph.get_module_ids(), - HashSet::from([ - *graph.ids_by_name.get("blue").unwrap(), - *graph.ids_by_name.get("blue.alpha").unwrap(), - *graph.ids_by_name.get("blue.beta").unwrap(), - *graph.ids_by_name.get("green").unwrap(), - ]) - ); - } - - #[test] - fn new_stores_importeds_by_importer_using_id() { - let graph = _make_graph(); - - let expected_importeds: HashSet = HashSet::from([ - *graph.ids_by_name.get("blue.alpha").unwrap(), - *graph.ids_by_name.get("blue.beta").unwrap(), - *graph.ids_by_name.get("green").unwrap(), - ]); - - assert_eq!( - *graph - .importeds_by_importer - .get(graph.ids_by_name.get("blue").unwrap()) - .unwrap(), - expected_importeds - ); - } - - #[test] - fn new_stores_importers_by_imported_using_id() { - let graph = _make_graph(); - - let expected_importers: HashSet = HashSet::from([ - *graph.ids_by_name.get("blue").unwrap(), - *graph.ids_by_name.get("green").unwrap(), - ]); - - assert_eq!( - *graph - .importers_by_imported - .get(graph.ids_by_name.get("blue.alpha").unwrap()) - .unwrap(), - expected_importers - ); - } - - #[test] - fn test_squash_module() { - let mut graph = ImportGraph::new(HashMap::from([ - ("blue", HashSet::from(["orange", "green"])), - ("blue.alpha", HashSet::from(["green.delta"])), - ("blue.beta", HashSet::new()), - ("green", HashSet::from(["blue.alpha", "blue.beta"])), - ("green.gamma", HashSet::new()), - ("green.delta", HashSet::new()), - ("orange", HashSet::new()), - ])); - - graph.squash_module("blue"); - - assert_eq!( - graph.importeds_by_importer[&graph.ids_by_name["blue"]], - HashSet::from([ - graph.ids_by_name["orange"], - graph.ids_by_name["green"], - graph.ids_by_name["green.delta"], - ]) - ); - assert_eq!( - graph.importeds_by_importer[&graph.ids_by_name["green"]], - HashSet::from([graph.ids_by_name["blue"],]) - ); - - assert_eq!( - graph.importers_by_imported[&graph.ids_by_name["orange"]], - HashSet::from([graph.ids_by_name["blue"],]) - ); - assert_eq!( - graph.importers_by_imported[&graph.ids_by_name["green"]], - HashSet::from([graph.ids_by_name["blue"],]) - ); - assert_eq!( - graph.importers_by_imported[&graph.ids_by_name["green.delta"]], - HashSet::from([graph.ids_by_name["blue"],]) - ); - assert_eq!( - graph.importers_by_imported[&graph.ids_by_name["blue"]], - HashSet::from([graph.ids_by_name["green"],]) - ); - } - - #[test] - fn test_find_shortest_chain() { - let blue = "blue"; - let green = "green"; - let yellow = "yellow"; - let blue_alpha = "blue.alpha"; - let blue_beta = "blue.beta"; - - let graph = ImportGraph::new(HashMap::from([ - (green, HashSet::from([blue])), - (blue_alpha, HashSet::from([blue])), - (yellow, HashSet::from([green])), - (blue_beta, HashSet::from([green])), - (blue, HashSet::new()), - ])); - - let path_or_none: Option> = - graph.find_shortest_chain(graph.ids_by_name[&yellow], graph.ids_by_name[&blue]); - - assert_eq!( - path_or_none, - Some(Vec::from([ - graph.ids_by_name[&yellow], - graph.ids_by_name[&green], - graph.ids_by_name[&blue] - ])) - ); - } -} diff --git a/rust/src/layers.rs b/rust/src/layers.rs deleted file mode 100644 index 0d5257fa..00000000 --- a/rust/src/layers.rs +++ /dev/null @@ -1,699 +0,0 @@ -use crate::dependencies::{PackageDependency, Route}; -use crate::importgraph::ImportGraph; - -use log::info; -use rayon::prelude::*; -use std::collections::HashSet; -use std::time::Instant; - -/// A group of layers at the same level in the layering. -#[derive(PartialEq, Eq, Hash, Debug)] -pub struct Level { - pub layers: Vec, - pub independent: bool, -} - -pub fn find_illegal_dependencies<'a>( - graph: &'a ImportGraph, - levels: &'a Vec, - containers: &'a HashSet, -) -> Vec { - let layers = _layers_from_levels(levels); - - _generate_module_permutations(graph, levels, containers) - .into_par_iter() - .filter_map(|(higher_layer_package, lower_layer_package, container)| { - // TODO: it's inefficient to do this for sibling layers, as we don't need - // to clone and trim the graph for identical pairs. - info!( - "Searching for import chains from {} to {}...", - lower_layer_package, higher_layer_package - ); - let now = Instant::now(); - let dependency_or_none = _search_for_package_dependency( - &higher_layer_package, - &lower_layer_package, - &layers, - &container, - graph, - ); - _log_illegal_route_count(&dependency_or_none, now.elapsed().as_secs()); - dependency_or_none - }) - .collect() -} - -/// Return every permutation of modules that exist in the graph -/// in which the second should not import the first. -fn _generate_module_permutations<'a>( - graph: &'a ImportGraph, - levels: &'a [Level], - containers: &'a HashSet, -) -> Vec<(String, String, Option)> { - let mut permutations: Vec<(String, String, Option)> = vec![]; - - let quasi_containers: Vec> = if containers.is_empty() { - vec![None] - } else { - containers.iter().map(|i| Some(i.to_string())).collect() - }; - for container in quasi_containers { - for (index, higher_level) in levels.iter().enumerate() { - for higher_layer in &higher_level.layers { - let higher_layer_module_name = _module_from_layer(higher_layer, &container); - if graph - .ids_by_name - .get(&higher_layer_module_name as &str) - .is_none() - { - continue; - } - - // Build the layers that mustn't import this higher layer. - // That includes: - // * lower layers. - // * sibling layers, if the layer is independent. - let mut layers_forbidden_to_import_higher_layer: Vec<&str> = vec![]; - - if higher_level.independent { - for potential_sibling_layer in &higher_level.layers { - if potential_sibling_layer != higher_layer { - // It's a sibling layer. - layers_forbidden_to_import_higher_layer.push(potential_sibling_layer); - } - } - } - - for lower_level in &levels[index + 1..] { - for lower_layer in &lower_level.layers { - layers_forbidden_to_import_higher_layer.push(lower_layer); - } - } - - // Now turn the layers into modules, if they exist. - for forbidden_layer in &layers_forbidden_to_import_higher_layer { - let forbidden_module_name = _module_from_layer(forbidden_layer, &container); - if let Some(_value) = graph.ids_by_name.get(&forbidden_module_name as &str) { - permutations.push(( - higher_layer_module_name.clone(), - forbidden_module_name.clone(), - container.clone(), - )); - }; - } - } - } - } - - permutations -} - -fn _module_from_layer<'a>(module: &'a str, container: &'a Option) -> String { - match container { - Some(true_container) => format!("{}.{}", true_container, module), - None => module.to_string(), - } -} - -fn _search_for_package_dependency<'a>( - higher_layer_package: &'a str, - lower_layer_package: &'a str, - layers: &'a Vec<&'a str>, - container: &'a Option, - graph: &'a ImportGraph, -) -> Option { - let mut temp_graph = graph.clone(); - _remove_other_layers( - &mut temp_graph, - layers, - container, - (higher_layer_package, lower_layer_package), - ); - let mut routes: Vec = vec![]; - - // Direct routes. - let direct_links = - _pop_direct_imports(higher_layer_package, lower_layer_package, &mut temp_graph); - for (importer, imported) in direct_links { - routes.push(Route { - heads: vec![importer], - middle: vec![], - tails: vec![imported], - }); - } - - // Indirect routes. - for indirect_route in - _get_indirect_routes(higher_layer_package, lower_layer_package, &temp_graph) - { - routes.push(indirect_route); - } - if routes.is_empty() { - None - } else { - Some(PackageDependency { - imported: graph.ids_by_name[&higher_layer_package], - importer: graph.ids_by_name[&lower_layer_package], - routes, - }) - } -} - -fn _layers_from_levels<'a>(levels: &'a Vec) -> Vec<&'a str> { - let mut layers: Vec<&str> = vec![]; - for level in levels { - layers.extend(level.layers.iter().map(|s| s.as_str())); - } - layers -} - -fn _remove_other_layers<'a>( - graph: &'a mut ImportGraph, - layers: &'a Vec<&'a str>, - container: &'a Option, - layers_to_preserve: (&'a str, &'a str), -) { - for layer in layers { - let layer_module = _module_from_layer(layer, container); - if layers_to_preserve.0 == layer_module || layers_to_preserve.1 == layer_module { - continue; - } - if graph.contains_module(&layer_module) { - graph.remove_package(&layer_module); - } - } -} - -fn _pop_direct_imports<'a>( - higher_layer_package: &'a str, - lower_layer_package: &'a str, - graph: &'a mut ImportGraph, -) -> HashSet<(u32, u32)> { - // Remove the direct imports, returning them as (importer, imported) tuples. - let mut imports = HashSet::new(); - - let higher_layer_namespace: String = format!("{}.", higher_layer_package); - let mut lower_layer_module_ids: Vec = vec![graph.ids_by_name[lower_layer_package]]; - lower_layer_module_ids.append(&mut graph.get_descendant_ids(lower_layer_package)); - - for lower_layer_module_id in lower_layer_module_ids { - let _lower = graph.names_by_id[&lower_layer_module_id]; - let imported_module_ids = graph.importeds_by_importer[&lower_layer_module_id].clone(); - for imported_module_id in imported_module_ids { - let imported_module = graph.names_by_id[&imported_module_id]; - - if imported_module.starts_with(&higher_layer_namespace) - || imported_module == higher_layer_package - { - imports.insert((lower_layer_module_id, imported_module_id)); - graph.remove_import_ids(lower_layer_module_id, imported_module_id) - } - } - } - imports -} - -fn _get_indirect_routes<'a>( - imported_package: &'a str, - importer_package: &'a str, - graph: &'a ImportGraph, -) -> Vec { - // Squashes the two packages. - // Gets a list of paths between them, called middles. - // Add the heads and tails to the middles. - let mut temp_graph = graph.clone(); - temp_graph.squash_module(imported_package); - temp_graph.squash_module(importer_package); - - let middles = _find_middles(&mut temp_graph, importer_package, imported_package); - _middles_to_routes(graph, middles, importer_package, imported_package) -} - -fn _find_middles<'a>( - graph: &'a mut ImportGraph, - importer: &'a str, - imported: &'a str, -) -> Vec> { - let mut middles = vec![]; - - for chain in graph.pop_shortest_chains(importer, imported) { - // Remove first and last element. - // TODO surely there's a better way? - let mut middle: Vec = vec![]; - let chain_length = chain.len(); - for (index, module) in chain.iter().enumerate() { - if index != 0 && index != chain_length - 1 { - middle.push(*module); - } - } - middles.push(middle); - } - - middles -} - -fn _log_illegal_route_count(dependency_or_none: &Option, duration_in_s: u64) { - let route_count = match dependency_or_none { - Some(dependency) => dependency.routes.len(), - None => 0, - }; - let pluralized = if route_count == 1 { "" } else { "s" }; - info!( - "Found {} illegal route{} in {}s.", - route_count, pluralized, duration_in_s - ); -} - -fn _middles_to_routes<'a>( - graph: &'a ImportGraph, - middles: Vec>, - importer: &'a str, - imported: &'a str, -) -> Vec { - let mut routes = vec![]; - let importer_id = graph.ids_by_name[&importer]; - let imported_id = graph.ids_by_name[&imported]; - - for middle in middles { - // Construct heads. - let mut heads: Vec = vec![]; - let first_imported_id = middle[0]; - let candidate_modules = &graph.importers_by_imported[&first_imported_id]; - for candidate_module in candidate_modules { - if importer_id == *candidate_module - || graph - .get_descendant_ids(importer) - .contains(candidate_module) - { - heads.push(*candidate_module); - } - } - - // Construct tails. - let mut tails: Vec = vec![]; - let last_importer_id = middle[middle.len() - 1]; - let candidate_modules = &graph.importeds_by_importer[&last_importer_id]; - for candidate_module in candidate_modules { - if imported_id == *candidate_module - || graph - .get_descendant_ids(imported) - .contains(candidate_module) - { - tails.push(*candidate_module); - } - } - routes.push(Route { - heads, - middle, - tails, - }) - } - - routes -} - -#[cfg(test)] -mod tests { - use super::*; - use std::collections::HashMap; - - #[test] - fn test_find_illegal_dependencies_no_container() { - let graph = ImportGraph::new(HashMap::from([ - ("low", HashSet::new()), - ("low.blue", HashSet::from(["utils"])), - ("low.green", HashSet::new()), - ("low.green.alpha", HashSet::from(["high.yellow"])), - ("mid_a", HashSet::from(["mid_b"])), - ("mid_a.orange", HashSet::new()), - ("mid_b", HashSet::from(["mid_c"])), - ("mid_b.brown", HashSet::new()), - ("mid_c", HashSet::new()), - ("mid_c.purple", HashSet::new()), - ("high", HashSet::from(["low.blue"])), - ("high.yellow", HashSet::new()), - ("high.red", HashSet::new()), - ("high.red.beta", HashSet::new()), - ("utils", HashSet::from(["high.red"])), - ])); - let levels = vec![ - Level { - independent: true, - layers: vec!["high".to_string()], - }, - Level { - independent: true, - layers: vec![ - "mid_a".to_string(), - "mid_b".to_string(), - "mid_c".to_string(), - ], - }, - Level { - independent: true, - layers: vec!["low".to_string()], - }, - ]; - let containers = HashSet::new(); - - let dependencies = find_illegal_dependencies(&graph, &levels, &containers); - - assert_eq!( - dependencies, - vec![ - PackageDependency { - importer: *graph.ids_by_name.get("low").unwrap(), - imported: *graph.ids_by_name.get("high").unwrap(), - routes: vec![ - Route { - heads: vec![*graph.ids_by_name.get("low.green.alpha").unwrap()], - middle: vec![], - tails: vec![*graph.ids_by_name.get("high.yellow").unwrap()], - }, - Route { - heads: vec![*graph.ids_by_name.get("low.blue").unwrap()], - middle: vec![*graph.ids_by_name.get("utils").unwrap()], - tails: vec![*graph.ids_by_name.get("high.red").unwrap()], - }, - ], - }, - PackageDependency { - importer: *graph.ids_by_name.get("mid_a").unwrap(), - imported: *graph.ids_by_name.get("mid_b").unwrap(), - routes: vec![Route { - heads: vec![*graph.ids_by_name.get("mid_a").unwrap()], - middle: vec![], - tails: vec![*graph.ids_by_name.get("mid_b").unwrap()], - },], - }, - PackageDependency { - importer: *graph.ids_by_name.get("mid_b").unwrap(), - imported: *graph.ids_by_name.get("mid_c").unwrap(), - routes: vec![Route { - heads: vec![*graph.ids_by_name.get("mid_b").unwrap()], - middle: vec![], - tails: vec![*graph.ids_by_name.get("mid_c").unwrap()], - },], - }, - ] - ); - } - - #[test] - fn test_find_illegal_dependencies_with_container() { - let graph = ImportGraph::new(HashMap::from([ - ("mypackage.low", HashSet::new()), - ("mypackage.low.blue", HashSet::from(["mypackage.utils"])), - ("mypackage.low.green", HashSet::new()), - ( - "mypackage.low.green.alpha", - HashSet::from(["mypackage.high.yellow"]), - ), - ("mypackage.high", HashSet::from(["mypackage.low.blue"])), - ("mypackage.high.yellow", HashSet::new()), - ("mypackage.high.red", HashSet::new()), - ("mypackage.high.red.beta", HashSet::new()), - ("mypackage.utils", HashSet::from(["mypackage.high.red"])), - ])); - let levels = vec![ - Level { - independent: true, - layers: vec!["high".to_string()], - }, - Level { - independent: true, - layers: vec!["low".to_string()], - }, - ]; - let containers = HashSet::from(["mypackage".to_string()]); - - let dependencies = find_illegal_dependencies(&graph, &levels, &containers); - - assert_eq!( - dependencies, - vec![PackageDependency { - importer: *graph.ids_by_name.get("mypackage.low").unwrap(), - imported: *graph.ids_by_name.get("mypackage.high").unwrap(), - routes: vec![ - Route { - heads: vec![*graph.ids_by_name.get("mypackage.low.green.alpha").unwrap()], - middle: vec![], - tails: vec![*graph.ids_by_name.get("mypackage.high.yellow").unwrap()], - }, - Route { - heads: vec![*graph.ids_by_name.get("mypackage.low.blue").unwrap()], - middle: vec![*graph.ids_by_name.get("mypackage.utils").unwrap()], - tails: vec![*graph.ids_by_name.get("mypackage.high.red").unwrap()], - }, - ], - }] - ); - } - - #[test] - fn test_generate_module_permutations() { - let graph = ImportGraph::new(HashMap::from([ - ("mypackage.low", HashSet::new()), - ("mypackage.low.blue", HashSet::from(["mypackage.utils"])), - ("mypackage.low.green", HashSet::new()), - ( - "mypackage.low.green.alpha", - HashSet::from(["mypackage.high.yellow"]), - ), - ("mypackage.mid_a", HashSet::new()), - ("mypackage.mid_a.foo", HashSet::new()), - ("mypackage.mid_b", HashSet::new()), - ("mypackage.mid_b.foo", HashSet::new()), - ("mypackage.mid_c", HashSet::new()), - ("mypackage.mid_c.foo", HashSet::new()), - ("mypackage.high", HashSet::from(["mypackage.low.blue"])), - ("mypackage.high.yellow", HashSet::new()), - ("mypackage.high.red", HashSet::new()), - ("mypackage.high.red.beta", HashSet::new()), - ("mypackage.utils", HashSet::from(["mypackage.high.red"])), - ])); - let levels = vec![ - Level { - independent: true, - layers: vec!["high".to_string()], - }, - Level { - independent: true, - layers: vec![ - "mid_a".to_string(), - "mid_b".to_string(), - "mid_c".to_string(), - ], - }, - Level { - independent: true, - layers: vec!["low".to_string()], - }, - ]; - let containers = HashSet::from(["mypackage".to_string()]); - - let perms = _generate_module_permutations(&graph, &levels, &containers); - - let result: HashSet<(String, String, Option)> = HashSet::from_iter(perms); - let (high, mid_a, mid_b, mid_c, low) = ( - "mypackage.high", - "mypackage.mid_a", - "mypackage.mid_b", - "mypackage.mid_c", - "mypackage.low", - ); - assert_eq!( - result, - HashSet::from_iter([ - ( - high.to_string(), - mid_a.to_string(), - Some("mypackage".to_string()) - ), - ( - high.to_string(), - mid_b.to_string(), - Some("mypackage".to_string()) - ), - ( - high.to_string(), - mid_c.to_string(), - Some("mypackage".to_string()) - ), - ( - high.to_string(), - low.to_string(), - Some("mypackage".to_string()) - ), - ( - mid_a.to_string(), - mid_b.to_string(), - Some("mypackage".to_string()) - ), - ( - mid_a.to_string(), - mid_c.to_string(), - Some("mypackage".to_string()) - ), - ( - mid_b.to_string(), - mid_a.to_string(), - Some("mypackage".to_string()) - ), - ( - mid_b.to_string(), - mid_c.to_string(), - Some("mypackage".to_string()) - ), - ( - mid_c.to_string(), - mid_a.to_string(), - Some("mypackage".to_string()) - ), - ( - mid_c.to_string(), - mid_b.to_string(), - Some("mypackage".to_string()) - ), - ( - mid_a.to_string(), - low.to_string(), - Some("mypackage".to_string()) - ), - ( - mid_b.to_string(), - low.to_string(), - Some("mypackage".to_string()) - ), - ( - mid_c.to_string(), - low.to_string(), - Some("mypackage".to_string()) - ), - ]) - ); - } - - #[test] - fn test_generate_module_permutations_sibling_layer_not_independent() { - let graph = ImportGraph::new(HashMap::from([ - ("mypackage.low", HashSet::new()), - ("mypackage.low.blue", HashSet::from(["mypackage.utils"])), - ("mypackage.low.green", HashSet::new()), - ( - "mypackage.low.green.alpha", - HashSet::from(["mypackage.high.yellow"]), - ), - ("mypackage.mid_a", HashSet::new()), - ("mypackage.mid_a.foo", HashSet::new()), - ("mypackage.mid_b", HashSet::new()), - ("mypackage.mid_b.foo", HashSet::new()), - ("mypackage.mid_c", HashSet::new()), - ("mypackage.mid_c.foo", HashSet::new()), - ("mypackage.high", HashSet::from(["mypackage.low.blue"])), - ("mypackage.high.yellow", HashSet::new()), - ("mypackage.high.red", HashSet::new()), - ("mypackage.high.red.beta", HashSet::new()), - ("mypackage.utils", HashSet::from(["mypackage.high.red"])), - ])); - let levels = vec![ - Level { - independent: true, - layers: vec!["high".to_string()], - }, - Level { - independent: false, - layers: vec![ - "mid_a".to_string(), - "mid_b".to_string(), - "mid_c".to_string(), - ], - }, - Level { - independent: true, - layers: vec!["low".to_string()], - }, - ]; - let containers = HashSet::from(["mypackage".to_string()]); - - let perms = _generate_module_permutations(&graph, &levels, &containers); - - let result: HashSet<(String, String, Option)> = HashSet::from_iter(perms); - let (high, mid_a, mid_b, mid_c, low) = ( - "mypackage.high", - "mypackage.mid_a", - "mypackage.mid_b", - "mypackage.mid_c", - "mypackage.low", - ); - assert_eq!( - result, - HashSet::from_iter([ - ( - high.to_string(), - mid_a.to_string(), - Some("mypackage".to_string()) - ), - ( - high.to_string(), - mid_b.to_string(), - Some("mypackage".to_string()) - ), - ( - high.to_string(), - mid_c.to_string(), - Some("mypackage".to_string()) - ), - ( - high.to_string(), - low.to_string(), - Some("mypackage".to_string()) - ), - ( - mid_a.to_string(), - low.to_string(), - Some("mypackage".to_string()) - ), - ( - mid_b.to_string(), - low.to_string(), - Some("mypackage".to_string()) - ), - ( - mid_c.to_string(), - low.to_string(), - Some("mypackage".to_string()) - ), - ]) - ); - } - - #[test] - fn test_layers_from_levels() { - let levels = vec![ - Level { - independent: true, - layers: vec!["high".to_string()], - }, - Level { - independent: true, - layers: vec![ - "medium_a".to_string(), - "medium_b".to_string(), - "medium_c".to_string(), - ], - }, - Level { - independent: true, - layers: vec!["low".to_string()], - }, - ]; - - let result = _layers_from_levels(&levels); - - assert_eq!( - HashSet::<&str>::from_iter(result), - HashSet::from_iter(["high", "medium_a", "medium_b", "medium_c", "low",]), - ) - } -} diff --git a/rust/src/lib.rs b/rust/src/lib.rs index 73e7038b..d293f852 100644 --- a/rust/src/lib.rs +++ b/rust/src/lib.rs @@ -1,261 +1,572 @@ -mod containers; -// TODO make these private. -pub mod dependencies; -pub mod importgraph; -pub mod layers; - -use crate::dependencies::PackageDependency; -use containers::check_containers_exist; -use importgraph::ImportGraph; -use layers::Level; -use log::info; -use pyo3::create_exception; +pub mod errors; +pub mod exceptions; +pub mod graph; + +use crate::errors::{GrimpError, GrimpResult}; +use crate::exceptions::{ModuleNotPresent, NoSuchContainer}; +use crate::graph::higher_order_queries::Level; +use crate::graph::{Graph, Module, ModuleIterator, ModuleTokenIterator}; +use derive_new::new; +use itertools::Itertools; +use pyo3::exceptions::PyValueError; use pyo3::prelude::*; -use pyo3::types::{PyDict, PyFrozenSet, PySet, PyString, PyTuple}; -use std::collections::{HashMap, HashSet}; +use pyo3::types::{IntoPyDict, PyDict, PyFrozenSet, PyList, PySet, PyString, PyTuple}; +use pyo3::IntoPyObjectExt; +use rayon::prelude::*; +use rustc_hash::FxHashSet; +use std::collections::HashSet; #[pymodule] fn _rustgrimp(py: Python<'_>, m: &Bound<'_, PyModule>) -> PyResult<()> { - pyo3_log::init(); - - m.add_function(wrap_pyfunction!(find_illegal_dependencies, m)?)?; - m.add("NoSuchContainer", py.get_type_bound::())?; + m.add_class::()?; + m.add("ModuleNotPresent", py.get_type::())?; + m.add("NoSuchContainer", py.get_type::())?; Ok(()) } -create_exception!(_rustgrimp, NoSuchContainer, pyo3::exceptions::PyException); +#[pyclass(name = "Graph")] +struct GraphWrapper { + _graph: Graph, +} -#[pyfunction] -pub fn find_illegal_dependencies<'py>( - py: Python<'py>, - levels: &Bound<'py, PyTuple>, - containers: &Bound<'py, PySet>, - importeds_by_importer: &Bound<'py, PyDict>, -) -> PyResult> { - info!("Using Rust to find illegal dependencies."); +impl GraphWrapper { + fn get_visible_module_by_name(&self, name: &str) -> Result<&Module, GrimpError> { + self._graph + .get_module_by_name(name) + .filter(|m| !m.is_invisible()) + .ok_or(GrimpError::ModuleNotPresent(name.to_owned())) + } +} - let importeds_by_importer_strings: HashMap> = - importeds_by_importer.extract()?; - let importeds_by_importer_strs = strings_to_strs_hashmap(&importeds_by_importer_strings); +/// Wrapper around the Graph struct that integrates with Python. +#[pymethods] +impl GraphWrapper { + #[new] + fn new() -> Self { + GraphWrapper { + _graph: Graph::default(), + } + } - let graph = ImportGraph::new(importeds_by_importer_strs); - let levels_rust = rustify_levels(levels); - let containers_rust: HashSet = containers.extract()?; + pub fn get_modules(&self) -> HashSet { + self._graph + .all_modules() + .visible() + .names_as_strings() + .collect() + } - if let Err(err) = check_containers_exist(&graph, &containers_rust) { - return Err(NoSuchContainer::new_err(err)); + pub fn contains_module(&self, name: &str) -> bool { + match self.get_visible_module_by_name(name) { + Ok(_) => true, + Err(GrimpError::ModuleNotPresent(_)) => false, + _ => panic!("unexpected error checking for module existence"), + } } - let dependencies = py.allow_threads(|| { - layers::find_illegal_dependencies(&graph, &levels_rust, &containers_rust) - }); + #[pyo3(signature = (module, is_squashed = false))] + pub fn add_module(&mut self, module: &str, is_squashed: bool) -> PyResult<()> { + for ancestor_module in self + ._graph + .module_name_to_self_and_ancestors(module) + .into_iter() + .skip(1) + { + if self.is_module_squashed(&ancestor_module).unwrap_or(false) { + return Err(PyValueError::new_err(format!( + "Module is a descendant of squashed module {}.", + &ancestor_module, + ))); + }; + } - convert_dependencies_to_python(py, dependencies, &graph) -} + if self.contains_module(module) && self.is_module_squashed(module)? != is_squashed { + return Err(PyValueError::new_err( + "Cannot add a squashed module when it is already present in the graph \ + as an unsquashed module, or vice versa.", + )); + } -fn strings_to_strs_hashmap<'a>( - string_map: &'a HashMap>, -) -> HashMap<&'a str, HashSet<&'a str>> { - let mut str_map: HashMap<&str, HashSet<&str>> = HashMap::new(); + match is_squashed { + false => self._graph.get_or_add_module(module), + true => self._graph.get_or_add_squashed_module(module), + }; + Ok(()) + } - for (key, set) in string_map { - let mut str_set: HashSet<&str> = HashSet::new(); - for item in set.iter() { - str_set.insert(item); + pub fn remove_module(&mut self, module: &str) { + if let Some(module) = self._graph.get_module_by_name(module) { + self._graph.remove_module(module.token()) } - str_map.insert(key.as_str(), str_set); } - str_map -} -fn rustify_levels<'a>(levels_python: &Bound<'a, PyTuple>) -> Vec { - let mut rust_levels: Vec = vec![]; - for level_python in levels_python.into_iter() { - let level_dict = level_python.downcast::().unwrap(); - let layers: HashSet = level_dict - .get_item("layers") - .unwrap() - .unwrap() - .extract() - .unwrap(); - - let independent: bool = level_dict - .get_item("independent") - .unwrap() - .unwrap() - .extract() - .unwrap(); - rust_levels.push(Level { - independent, - layers: layers.into_iter().collect(), - }); + pub fn squash_module(&mut self, module: &str) -> PyResult<()> { + let module = self.get_visible_module_by_name(module)?.token(); + self._graph.squash_module(module); + Ok(()) } - rust_levels -} -fn convert_dependencies_to_python<'py>( - py: Python<'py>, - dependencies: Vec, - graph: &ImportGraph, -) -> PyResult> { - let mut python_dependencies: Vec> = vec![]; - - for rust_dependency in dependencies { - let python_dependency = PyDict::new_bound(py); - python_dependency.set_item("imported", graph.names_by_id[&rust_dependency.imported])?; - python_dependency.set_item("importer", graph.names_by_id[&rust_dependency.importer])?; - let mut python_routes: Vec> = vec![]; - for rust_route in rust_dependency.routes { - let route = PyDict::new_bound(py); - let heads: Vec> = rust_route - .heads - .iter() - .map(|i| PyString::new_bound(py, graph.names_by_id[&i])) - .collect(); - route.set_item("heads", PyFrozenSet::new_bound(py, &heads)?)?; - let middle: Vec> = rust_route - .middle - .iter() - .map(|i| PyString::new_bound(py, graph.names_by_id[&i])) - .collect(); - route.set_item("middle", PyTuple::new_bound(py, &middle))?; - let tails: Vec> = rust_route - .tails - .iter() - .map(|i| PyString::new_bound(py, graph.names_by_id[&i])) - .collect(); - route.set_item("tails", PyFrozenSet::new_bound(py, &tails)?)?; + pub fn is_module_squashed(&self, module: &str) -> PyResult { + Ok(self.get_visible_module_by_name(module)?.is_squashed()) + } - python_routes.push(route); + #[pyo3(signature = (*, importer, imported, line_number=None, line_contents=None))] + pub fn add_import( + &mut self, + importer: &str, + imported: &str, + line_number: Option, + line_contents: Option<&str>, + ) { + let importer = self._graph.get_or_add_module(importer).token(); + let imported = self._graph.get_or_add_module(imported).token(); + match (line_number, line_contents) { + (Some(line_number), Some(line_contents)) => { + self._graph + .add_detailed_import(importer, imported, line_number, line_contents) + } + (None, None) => { + self._graph.add_import(importer, imported); + } + _ => { + // TODO handle better. + panic!("Expected line_number and line_contents, or neither."); + } } + } - python_dependency.set_item("routes", PyTuple::new_bound(py, python_routes))?; - python_dependencies.push(python_dependency) + #[pyo3(signature = (*, importer, imported))] + pub fn remove_import(&mut self, importer: &str, imported: &str) -> PyResult<()> { + let importer = self.get_visible_module_by_name(importer)?.token(); + let imported = self.get_visible_module_by_name(imported)?.token(); + self._graph.remove_import(importer, imported); + Ok(()) } - Ok(PyTuple::new_bound(py, python_dependencies)) -} + pub fn count_imports(&self) -> usize { + self._graph.count_imports() + } -#[cfg(test)] -mod tests { - use super::*; - - // Macro to easily define a python dict. - // Adapted from the hash_map! macro in https://github.com/jofas/map_macro. - macro_rules! pydict { - ($py: ident, {$($k: expr => $v: expr),*, $(,)?}) => { - { - let dict = PyDict::new_bound($py); - $( - dict.set_item($k, $v)?; - )* - dict - } - }; + pub fn find_children(&self, module: &str) -> PyResult> { + let module = self + ._graph + .get_module_by_name(module) + .ok_or(GrimpError::ModuleNotPresent(module.to_owned()))?; + Ok(self + ._graph + .get_module_children(module.token()) + .visible() + .names_as_strings() + .collect()) } - #[test] - fn test_rustify_levels_no_sibling_layers() { - pyo3::prepare_freethreaded_python(); - Python::with_gil(|py| -> PyResult<()> { - let elements = vec![ - pydict! (py, { - "independent" => true, - "layers" => HashSet::from(["high"]), - }), - pydict! (py, { - "independent" => true, - "layers" => HashSet::from(["medium"]), - }), - pydict! (py, { - "independent" => true, - "layers" => HashSet::from(["low"]), - }), - ]; - let python_levels = PyTuple::new_bound(py, elements); - - let result = rustify_levels(&python_levels); - - assert_eq!( - result, - vec![ - Level { - independent: true, - layers: vec!["high".to_string()] - }, - Level { - independent: true, - layers: vec!["medium".to_string()] - }, - Level { - independent: true, - layers: vec!["low".to_string()] - } - ] - ); - - Ok(()) - }) - .unwrap(); + pub fn find_descendants(&self, module: &str) -> PyResult> { + let module = self + ._graph + .get_module_by_name(module) + .ok_or(GrimpError::ModuleNotPresent(module.to_owned()))?; + Ok(self + ._graph + .get_module_descendants(module.token()) + .visible() + .names_as_strings() + .collect()) } - #[test] - fn test_rustify_levels_sibling_layers() { - pyo3::prepare_freethreaded_python(); - Python::with_gil(|py| -> PyResult<()> { - let elements = vec![ - pydict! (py, { - "independent" => true, - "layers" => HashSet::from(["high"]), - }), - pydict! (py, { - "independent" => true, - "layers" => HashSet::from(["blue", "green", "orange"]), - }), - pydict! (py, { - "independent" => false, - "layers" => HashSet::from(["red", "yellow"]), - }), - pydict! (py, { - "independent" => true, - "layers" => HashSet::from(["low"]), + #[pyo3(signature = (*, importer, imported, as_packages = false))] + pub fn direct_import_exists( + &self, + importer: &str, + imported: &str, + as_packages: bool, + ) -> PyResult { + let importer = self.get_visible_module_by_name(importer)?.token(); + let imported = self.get_visible_module_by_name(imported)?.token(); + Ok(self + ._graph + .direct_import_exists(importer, imported, as_packages)?) + } + + pub fn find_modules_directly_imported_by(&self, module: &str) -> PyResult> { + let module = self.get_visible_module_by_name(module)?.token(); + Ok(self + ._graph + .modules_directly_imported_by(module) + .iter() + .into_module_iterator(&self._graph) + .visible() + .names_as_strings() + .collect()) + } + + pub fn find_modules_that_directly_import(&self, module: &str) -> PyResult> { + let module = self.get_visible_module_by_name(module)?.token(); + Ok(self + ._graph + .modules_that_directly_import(module) + .iter() + .into_module_iterator(&self._graph) + .visible() + .names_as_strings() + .collect()) + } + + #[pyo3(signature = (*, importer, imported))] + pub fn get_import_details<'py>( + &self, + py: Python<'py>, + importer: &str, + imported: &str, + ) -> PyResult> { + let importer = match self._graph.get_module_by_name(importer) { + Some(module) => module, + None => return Ok(PyList::empty(py)), + }; + let imported = match self._graph.get_module_by_name(imported) { + Some(module) => module, + None => return Ok(PyList::empty(py)), + }; + + PyList::new( + py, + self._graph + .get_import_details(importer.token(), imported.token()) + .iter() + .map(|import_details| { + ImportDetails::new( + importer.name_as_string(), + imported.name_as_string(), + import_details.line_number(), + import_details.line_contents().to_owned(), + ) + }) + .sorted() + .map(|import_details| { + [ + ("importer", import_details.importer.into_py_any(py).unwrap()), + ("imported", import_details.imported.into_py_any(py).unwrap()), + ( + "line_number", + import_details.line_number.into_py_any(py).unwrap(), + ), + ( + "line_contents", + import_details.line_contents.into_py_any(py).unwrap(), + ), + ] + .into_py_dict(py) + .unwrap() }), - ]; - let python_levels = PyTuple::new_bound(py, elements); + ) + } + + #[allow(unused_variables)] + #[pyo3(signature = (module, as_package=false))] + pub fn find_downstream_modules( + &self, + module: &str, + as_package: bool, + ) -> PyResult> { + let module = self.get_visible_module_by_name(module)?.token(); + Ok(self + ._graph + .find_downstream_modules(module, as_package) + .iter() + .into_module_iterator(&self._graph) + .visible() + .names_as_strings() + .collect()) + } + + #[allow(unused_variables)] + #[pyo3(signature = (module, as_package=false))] + pub fn find_upstream_modules( + &self, + module: &str, + as_package: bool, + ) -> PyResult> { + let module = self.get_visible_module_by_name(module)?.token(); + Ok(self + ._graph + .find_upstream_modules(module, as_package) + .iter() + .into_module_iterator(&self._graph) + .visible() + .names_as_strings() + .collect()) + } + + // TODO(peter) Add `as_packages` argument here? The implementation already supports it! + pub fn find_shortest_chain( + &self, + importer: &str, + imported: &str, + ) -> PyResult>> { + let importer = self.get_visible_module_by_name(importer)?.token(); + let imported = self.get_visible_module_by_name(imported)?.token(); + Ok(self + ._graph + .find_shortest_chain(importer, imported, false)? + .map(|chain| { + chain + .iter() + .into_module_iterator(&self._graph) + .names_as_strings() + .collect() + })) + } + + #[pyo3(signature = (importer, imported, as_packages=false))] + pub fn chain_exists( + &self, + importer: &str, + imported: &str, + as_packages: bool, + ) -> PyResult { + let importer = self.get_visible_module_by_name(importer)?.token(); + let imported = self.get_visible_module_by_name(imported)?.token(); + Ok(self._graph.chain_exists(importer, imported, as_packages)?) + } + + #[pyo3(signature = (importer, imported, as_packages=true))] + pub fn find_shortest_chains<'py>( + &self, + py: Python<'py>, + importer: &str, + imported: &str, + as_packages: bool, + ) -> PyResult> { + let importer = self.get_visible_module_by_name(importer)?.token(); + let imported = self.get_visible_module_by_name(imported)?.token(); + let chains = self + ._graph + .find_shortest_chains(importer, imported, as_packages)? + .into_iter() + .map(|chain| { + PyTuple::new( + py, + chain + .iter() + .into_module_iterator(&self._graph) + .names_as_strings() + .collect::>(), + ) + .unwrap() + }); + PySet::new(py, chains) + } + + #[pyo3(signature = (layers, containers))] + pub fn find_illegal_dependencies_for_layers<'py>( + &self, + py: Python<'py>, + layers: &Bound<'py, PyTuple>, + containers: HashSet, + ) -> PyResult> { + let containers = self.parse_containers(&containers)?; + let levels_by_container = self.parse_levels_by_container(layers, &containers); + + let illegal_dependencies = levels_by_container + .into_iter() + .par_bridge() + .try_fold( + Vec::new, + |mut v: Vec, + levels| + -> GrimpResult<_> { + v.extend(self._graph.find_illegal_dependencies_for_layers(&levels)?); + Ok(v) + }, + ) + .try_reduce( + Vec::new, + |mut v: Vec, + package_dependencies| { + v.extend(package_dependencies); + Ok(v) + }, + )?; + + let illegal_dependencies = illegal_dependencies + .into_iter() + .map(|dep| { + PackageDependency::new( + self._graph + .get_module(*dep.importer()) + .unwrap() + .name_as_string(), + self._graph + .get_module(*dep.imported()) + .unwrap() + .name_as_string(), + dep.routes() + .iter() + .map(|route| { + Route::new( + route + .heads() + .iter() + .map(|m| self._graph.get_module(*m).unwrap().name_as_string()) + .collect(), + route + .middle() + .iter() + .map(|m| self._graph.get_module(*m).unwrap().name_as_string()) + .collect(), + route + .tails() + .iter() + .map(|m| self._graph.get_module(*m).unwrap().name_as_string()) + .collect(), + ) + }) + .collect(), + ) + }) + .sorted() + .collect::>(); + + self.convert_package_dependencies_to_python(py, illegal_dependencies) + } - let mut result = rustify_levels(&python_levels); + pub fn clone(&self) -> GraphWrapper { + GraphWrapper { + _graph: self._graph.clone(), + } + } +} + +impl GraphWrapper { + fn parse_containers( + &self, + containers: &HashSet, + ) -> Result, GrimpError> { + containers + .iter() + .map(|name| match self.get_visible_module_by_name(name) { + Ok(module) => Ok(module), + Err(GrimpError::ModuleNotPresent(_)) => { + Err(GrimpError::NoSuchContainer(name.into()))? + } + _ => panic!("unexpected error parsing containers"), + }) + .collect::, GrimpError>>() + } - for level in &mut result { - level.layers.sort(); + fn parse_levels_by_container( + &self, + pylevels: &Bound<'_, PyTuple>, + containers: &HashSet<&Module>, + ) -> Vec> { + let containers = match containers.is_empty() { + true => vec![None], + false => containers + .iter() + .map(|c| Some(c.name_as_string())) + .collect(), + }; + + let mut levels_by_container: Vec> = vec![]; + for container in containers { + let mut levels: Vec = vec![]; + for pylevel in pylevels.into_iter() { + let level_dict = pylevel.downcast::().unwrap(); + let layers = level_dict + .get_item("layers") + .unwrap() + .unwrap() + .extract::>() + .unwrap() + .into_iter() + .map(|name| match container.clone() { + Some(container) => format!("{}.{}", container, name), + None => name, + }) + .filter_map(|name| match self.get_visible_module_by_name(&name) { + Ok(module) => Some(module.token()), + // TODO(peter) Error here? Or silently continue (backwards compatibility?) + Err(GrimpError::ModuleNotPresent(_)) => None, + _ => panic!("unexpected error parsing levels"), + }) + .collect::>(); + + let independent = level_dict + .get_item("independent") + .unwrap() + .unwrap() + .extract::() + .unwrap(); + + levels.push(Level::new(layers, independent)); } - assert_eq!( - result, - vec![ - Level { - independent: true, - layers: vec!["high".to_string()] - }, - Level { - independent: true, - layers: vec![ - "blue".to_string(), - "green".to_string(), - "orange".to_string() - ] - }, - Level { - independent: false, - layers: vec!["red".to_string(), "yellow".to_string()] - }, - Level { - independent: true, - layers: vec!["low".to_string()] - } - ] - ); - - Ok(()) - }) - .unwrap(); + levels_by_container.push(levels); + } + + levels_by_container } + + fn convert_package_dependencies_to_python<'py>( + &self, + py: Python<'py>, + package_dependencies: Vec, + ) -> PyResult> { + let mut python_dependencies: Vec> = vec![]; + + for rust_dependency in package_dependencies { + let python_dependency = PyDict::new(py); + python_dependency.set_item("imported", &rust_dependency.imported)?; + python_dependency.set_item("importer", &rust_dependency.importer)?; + let mut python_routes: Vec> = vec![]; + for rust_route in &rust_dependency.routes { + let route = PyDict::new(py); + let heads: Vec> = rust_route + .heads + .iter() + .map(|module| PyString::new(py, module)) + .collect(); + route.set_item("heads", PyFrozenSet::new(py, &heads)?)?; + let middle: Vec> = rust_route + .middle + .iter() + .map(|module| PyString::new(py, module)) + .collect(); + route.set_item("middle", PyTuple::new(py, &middle)?)?; + let tails: Vec> = rust_route + .tails + .iter() + .map(|module| PyString::new(py, module)) + .collect(); + route.set_item("tails", PyFrozenSet::new(py, &tails)?)?; + + python_routes.push(route); + } + + python_dependency.set_item("routes", PyTuple::new(py, python_routes)?)?; + python_dependencies.push(python_dependency) + } + + PyTuple::new(py, python_dependencies) + } +} + +#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, new)] +struct ImportDetails { + importer: String, + imported: String, + line_number: usize, + line_contents: String, +} + +#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, new)] +struct PackageDependency { + importer: String, + imported: String, + routes: Vec, +} + +#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, new)] +struct Route { + heads: Vec, + middle: Vec, + tails: Vec, } diff --git a/rust/tests/large.rs b/rust/tests/large.rs index 0fff1d65..558d3b0e 100644 --- a/rust/tests/large.rs +++ b/rust/tests/large.rs @@ -1,23 +1,24 @@ -use _rustgrimp::importgraph::ImportGraph; -use _rustgrimp::layers::{find_illegal_dependencies, Level}; +use _rustgrimp::graph::higher_order_queries::Level; +use _rustgrimp::graph::Graph; +use rustc_hash::FxHashSet; use serde_json::{Map, Value}; -use std::collections::{HashMap, HashSet}; use std::fs; +use tap::Conv; #[test] fn test_large_graph_deep_layers() { let data = fs::read_to_string("tests/large_graph.json").expect("Unable to read file"); let value: Value = serde_json::from_str(&data).unwrap(); let items: &Map = value.as_object().unwrap(); - let mut importeds_by_importer: HashMap<&str, HashSet<&str>> = HashMap::new(); + + let mut graph = Graph::default(); for (importer, importeds_value) in items.iter() { - let mut importeds = HashSet::new(); + let importer = graph.get_or_add_module(importer).token(); for imported in importeds_value.as_array().unwrap() { - importeds.insert(imported.as_str().unwrap()); + let imported = graph.get_or_add_module(imported.as_str().unwrap()).token(); + graph.add_import(importer, imported); } - importeds_by_importer.insert(importer, importeds); } - let graph = ImportGraph::new(importeds_by_importer); let deep_layers = vec![ "mypackage.plugins.5634303718.1007553798.8198145119.application.3242334296.1991886645", @@ -30,16 +31,22 @@ fn test_large_graph_deep_layers() { "mypackage.plugins.5634303718.1007553798.8198145119.application.3242334296.5033127033", "mypackage.plugins.5634303718.1007553798.8198145119.application.3242334296.2454157946", ]; + let levels: Vec = deep_layers - .iter() - .map(|layer| Level { - independent: true, - layers: vec![layer.to_string()], + .into_iter() + .map(|layer| { + Level::new( + graph + .get_module_by_name(layer) + .unwrap() + .token() + .conv::>(), + true, + ) }) .collect(); - let containers = HashSet::new(); - let deps = find_illegal_dependencies(&graph, &levels, &containers); + let deps = graph.find_illegal_dependencies_for_layers(&levels).unwrap(); assert_eq!(deps.len(), 8); } diff --git a/src/grimp/adaptors/_layers.py b/src/grimp/adaptors/_layers.py deleted file mode 100644 index 7ff3c7e4..00000000 --- a/src/grimp/adaptors/_layers.py +++ /dev/null @@ -1,172 +0,0 @@ -from __future__ import annotations - -from dataclasses import dataclass -from typing import TYPE_CHECKING, Any, Iterator, Sequence, TypedDict - -from grimp import Route -from grimp import _rustgrimp as rust # type: ignore[attr-defined] - -if TYPE_CHECKING: - from grimp.adaptors.graph import ImportGraph - -from grimp.domain.analysis import PackageDependency -from grimp.exceptions import NoSuchContainer -from grimp.domain.valueobjects import Layer - - -def parse_layers(layers: Sequence[Layer | str | set[str]]) -> tuple[Layer, ...]: - """ - Convert the passed raw `layers` into `Layer`s. - """ - out_layers = [] - for layer in layers: - if isinstance(layer, Layer): - out_layers.append(layer) - elif isinstance(layer, str): - out_layers.append(Layer(layer, independent=True)) - else: - out_layers.append(Layer(*tuple(layer), independent=True)) - return tuple(out_layers) - - -def find_illegal_dependencies( - graph: ImportGraph, - layers: Sequence[Layer], - containers: set[str], -) -> set[PackageDependency]: - """ - Find dependencies that don't conform to the supplied layered architecture. - - See ImportGraph.find_illegal_dependencies_for_layers. - - The only difference between this and the method is that the containers passed in - is already a (potentially empty) set. - """ - try: - rust_package_dependency_tuple = rust.find_illegal_dependencies( - levels=tuple( - {"layers": layer.module_tails, "independent": layer.independent} - for layer in layers - ), - containers=set(containers), - importeds_by_importer=graph._importeds_by_importer, - ) - except rust.NoSuchContainer as e: - raise NoSuchContainer(str(e)) - - rust_package_dependencies = _dependencies_from_tuple(rust_package_dependency_tuple) - return rust_package_dependencies - - -class _RustRoute(TypedDict): - heads: frozenset[str] - middle: tuple[str, ...] - tails: frozenset[str] - - -class _RustPackageDependency(TypedDict): - importer: str - imported: str - routes: tuple[_RustRoute, ...] - - -def _dependencies_from_tuple( - rust_package_dependency_tuple: tuple[_RustPackageDependency, ...] -) -> set[PackageDependency]: - return { - PackageDependency( - imported=dep_dict["imported"], - importer=dep_dict["importer"], - routes=frozenset( - { - Route( - heads=route_dict["heads"], - middle=route_dict["middle"], - tails=route_dict["tails"], - ) - for route_dict in dep_dict["routes"] - } - ), - ) - for dep_dict in rust_package_dependency_tuple - } - - -class _Module: - """ - A Python module. - """ - - def __init__(self, name: str) -> None: - """ - Args: - name: The fully qualified name of a Python module, e.g. 'package.foo.bar'. - """ - self.name = name - - def __str__(self) -> str: - return self.name - - def __eq__(self, other: Any) -> bool: - if isinstance(other, self.__class__): - return hash(self) == hash(other) - else: - return False - - def __hash__(self) -> int: - return hash(str(self)) - - def is_descendant_of(self, module: "_Module") -> bool: - return self.name.startswith(f"{module.name}.") - - -@dataclass(frozen=True) -class _Link: - importer: str - imported: str - - -# A chain of modules, each of which imports the next. -_Chain = tuple[str, ...] - - -def _generate_module_permutations( - graph: ImportGraph, - layers: Sequence[str], - containers: set[str], -) -> Iterator[tuple[_Module, _Module, str | None]]: - """ - Return all possible combinations of higher level and lower level modules, in pairs. - - Each pair of modules consists of immediate children of two different layers. The first - module is in a layer higher than the layer of the second module. This means the first - module is allowed to import the second, but not the other way around. - - Returns: - module_in_higher_layer, module_in_lower_layer, container - """ - # If there are no containers, we still want to run the loop once. - quasi_containers = containers or [None] - - for container in quasi_containers: - for index, higher_layer in enumerate(layers): - higher_layer_module = _module_from_layer(higher_layer, container) - - if higher_layer_module.name not in graph.modules: - continue - - for lower_layer in layers[index + 1 :]: - lower_layer_module = _module_from_layer(lower_layer, container) - - if lower_layer_module.name not in graph.modules: - continue - - yield higher_layer_module, lower_layer_module, container - - -def _module_from_layer(layer: str, container: str | None = None) -> _Module: - if container: - name = ".".join([container, layer]) - else: - name = layer - return _Module(name) diff --git a/src/grimp/adaptors/graph.py b/src/grimp/adaptors/graph.py index 0ea9d96a..f4abe0ca 100644 --- a/src/grimp/adaptors/graph.py +++ b/src/grimp/adaptors/graph.py @@ -1,125 +1,47 @@ from __future__ import annotations - -from copy import copy -from typing import Dict, List, Optional, Sequence, Set, Tuple, cast - -from grimp.algorithms.shortest_path import bidirectional_shortest_path +from typing import List, Optional, Sequence, Set, Tuple, TypedDict +from grimp.application.ports.graph import DetailedImport +from grimp.domain.analysis import PackageDependency, Route +from grimp.domain.valueobjects import Layer +from grimp import _rustgrimp as rust # type: ignore[attr-defined] +from grimp.exceptions import ModuleNotPresent, NoSuchContainer from grimp.application.ports import graph -from grimp.domain.analysis import PackageDependency -from grimp.domain.valueobjects import Module, Layer -from grimp.exceptions import ModuleNotPresent - -from . import _layers class ImportGraph(graph.ImportGraph): """ - Pure Python implementation of the ImportGraph. + Rust-backed implementation of the ImportGraph. """ def __init__(self) -> None: - # Maps all the modules directly imported by each key. - self._importeds_by_importer: Dict[str, Set[str]] = {} - # Maps all the modules that directly import each key. - self._importers_by_imported: Dict[str, Set[str]] = {} - - self._edge_count = 0 - - # Instantiate a dict that stores the details for all direct imports. - self._import_details: Dict[str, List[graph.DetailedImport]] = {} - self._squashed_modules: Set[str] = set() - - # Dunder methods - # -------------- - - def __deepcopy__(self, memodict: Dict) -> "ImportGraph": - new_graph = ImportGraph() - new_graph._importeds_by_importer = { - key: value.copy() for key, value in self._importeds_by_importer.items() - } - new_graph._importers_by_imported = { - key: value.copy() for key, value in self._importers_by_imported.items() - } - new_graph._edge_count = self._edge_count - - # Note: this copies the dictionaries containing each import detail - # by *reference*, so be careful about mutating the import details - # dictionaries internally. - new_graph._import_details = { - key: value.copy() for key, value in self._import_details.items() - } - - new_graph._squashed_modules = self._squashed_modules.copy() - - return new_graph - - # Mechanics - # --------- + super().__init__() + self._cached_modules: Set[str] | None = None + self._rustgraph = rust.Graph() @property def modules(self) -> Set[str]: - # Note: wrapping this in a set() makes it 10 times slower to build the graph! - # As a further optimisation, we use the _StringSet type alias instead of looking up Set[str] - # when casting. - return cast(_StringSet, self._importeds_by_importer.keys()) + if self._cached_modules is None: + self._cached_modules = self._rustgraph.get_modules() + return self._cached_modules def add_module(self, module: str, is_squashed: bool = False) -> None: - ancestor_squashed_module = self._find_ancestor_squashed_module(module) - if ancestor_squashed_module: - raise ValueError( - f"Module is a descendant of squashed module {ancestor_squashed_module}." - ) - - if module in self.modules: - if self.is_module_squashed(module) != is_squashed: - raise ValueError( - "Cannot add a squashed module when it is already present in the graph as " - "an unsquashed module, or vice versa." - ) - - self._importeds_by_importer.setdefault(module, set()) - self._importers_by_imported.setdefault(module, set()) - - if is_squashed: - self._mark_module_as_squashed(module) + self._cached_modules = None + self._rustgraph.add_module(module, is_squashed) def remove_module(self, module: str) -> None: - if module not in self.modules: - # TODO: rethink this behaviour. - return - - for imported in copy(self.find_modules_directly_imported_by(module)): - self.remove_import(importer=module, imported=imported) - for importer in copy(self.find_modules_that_directly_import(module)): - self.remove_import(importer=importer, imported=module) - del self._importeds_by_importer[module] - del self._importers_by_imported[module] + self._cached_modules = None + self._rustgraph.remove_module(module) def squash_module(self, module: str) -> None: - if self.is_module_squashed(module): - return - - squashed_root = module - descendants = self.find_descendants(squashed_root) - - # Add imports to/from the root. - for descendant in descendants: - for imported_module in self.find_modules_directly_imported_by(descendant): - self.add_import(importer=squashed_root, imported=imported_module) - for importing_module in self.find_modules_that_directly_import(descendant): - self.add_import(importer=importing_module, imported=squashed_root) - - # Now we've added imports to/from the root, we can delete the root's descendants. - for descendant in descendants: - self.remove_module(descendant) - - self._mark_module_as_squashed(squashed_root) + self._cached_modules = None + if not self._rustgraph.contains_module(module): + raise ModuleNotPresent(f'"{module}" not present in the graph.') + self._rustgraph.squash_module(module) def is_module_squashed(self, module: str) -> bool: - if module not in self.modules: + if not self._rustgraph.contains_module(module): raise ModuleNotPresent(f'"{module}" not present in the graph.') - - return module in self._squashed_modules + return self._rustgraph.is_module_squashed(module) def add_import( self, @@ -129,344 +51,151 @@ def add_import( line_number: Optional[int] = None, line_contents: Optional[str] = None, ) -> None: - if any((line_number, line_contents)): - if not all((line_number, line_contents)): - raise ValueError( - "Line number and contents must be provided together, or not at all." - ) - self._import_details.setdefault(importer, []) - self._import_details[importer].append( - { - "importer": importer, - "imported": imported, - "line_number": cast(int, line_number), - "line_contents": cast(str, line_contents), - } - ) - - importer_map = self._importeds_by_importer.setdefault(importer, set()) - imported_map = self._importers_by_imported.setdefault(imported, set()) - if imported not in importer_map: - # (Alternatively could check importer in imported_map.) - importer_map.add(imported) - imported_map.add(importer) - self._edge_count += 1 - - # Also ensure they have entry in other maps. - self._importeds_by_importer.setdefault(imported, set()) - self._importers_by_imported.setdefault(importer, set()) + self._cached_modules = None + self._rustgraph.add_import( + importer=importer, + imported=imported, + line_number=line_number, + line_contents=line_contents, + ) def remove_import(self, *, importer: str, imported: str) -> None: - if imported in self._importeds_by_importer[importer]: - self._importeds_by_importer[importer].remove(imported) - self._importers_by_imported[imported].remove(importer) - self._edge_count -= 1 - - # Clean up import details. - if importer in self._import_details: - new_details = [ - details - for details in self._import_details[importer] - if details["imported"] != imported - ] - if new_details: - self._import_details[importer] = new_details - else: - del self._import_details[importer] + self._cached_modules = None + return self._rustgraph.remove_import(importer=importer, imported=imported) def count_imports(self) -> int: - return self._edge_count - - # Descendants - # ----------- + return self._rustgraph.count_imports() def find_children(self, module: str) -> Set[str]: # It doesn't make sense to find the children of a squashed module, as we don't store # the children in the graph. if self.is_module_squashed(module): raise ValueError("Cannot find children of a squashed module.") - - children = set() - for potential_child in self.modules: - if Module(potential_child).is_child_of(Module(module)): - children.add(potential_child) - return children + return self._rustgraph.find_children(module) def find_descendants(self, module: str) -> Set[str]: # It doesn't make sense to find the descendants of a squashed module, as we don't store # the descendants in the graph. if self.is_module_squashed(module): raise ValueError("Cannot find descendants of a squashed module.") - - descendants = set() - for potential_descendant in self.modules: - if Module(potential_descendant).is_descendant_of(Module(module)): - descendants.add(potential_descendant) - return descendants - - # Direct imports - # -------------- + return self._rustgraph.find_descendants(module) def direct_import_exists( self, *, importer: str, imported: str, as_packages: bool = False ) -> bool: - if not as_packages: - return imported in self.find_modules_directly_imported_by(importer) - - importer_modules = self._all_modules_in_package(importer) - imported_modules = self._all_modules_in_package(imported) - - if importer_modules & imported_modules: - # If there are shared modules between the two, one of the modules is a descendant - # of the other (or they're both the same module). This doesn't make sense in - # this context, so raise an exception. - raise ValueError("Modules have shared descendants.") - - # Return True as soon as we find a path between any of the modules in the subpackages. - for candidate_importer in importer_modules: - imported_by_importer = self.find_modules_directly_imported_by(candidate_importer) - for candidate_imported in imported_modules: - if candidate_imported in imported_by_importer: - return True - return False + return self._rustgraph.direct_import_exists( + importer=importer, imported=imported, as_packages=as_packages + ) def find_modules_directly_imported_by(self, module: str) -> Set[str]: - return self._importeds_by_importer[module] + return self._rustgraph.find_modules_directly_imported_by(module) def find_modules_that_directly_import(self, module: str) -> Set[str]: - return self._importers_by_imported[module] + if self._rustgraph.contains_module(module): + # TODO panics if module isn't in modules. + return self._rustgraph.find_modules_that_directly_import(module) + return set() - def get_import_details(self, *, importer: str, imported: str) -> List[graph.DetailedImport]: - import_details_for_importer = self._import_details.get(importer, []) - # Only include the details for the imported module. - # Note: we copy each details dictionary at this point, as our deepcopying - # only copies the dictionaries by reference. - return [i.copy() for i in import_details_for_importer if i["imported"] == imported] - - # Indirect imports - # ---------------- + def get_import_details(self, *, importer: str, imported: str) -> List[DetailedImport]: + return self._rustgraph.get_import_details( + importer=importer, + imported=imported, + ) def find_downstream_modules(self, module: str, as_package: bool = False) -> Set[str]: - # TODO optimise for as_package. - if as_package: - source_modules = self._all_modules_in_package(module) - else: - source_modules = {module} - - downstream_modules = set() - - for candidate in filter(lambda m: m not in source_modules, self.modules): - for source_module in source_modules: - if self.chain_exists(importer=candidate, imported=source_module): - downstream_modules.add(candidate) - break - - return downstream_modules + return self._rustgraph.find_downstream_modules(module, as_package) def find_upstream_modules(self, module: str, as_package: bool = False) -> Set[str]: - # TODO optimise for as_package. - if as_package: - destination_modules = self._all_modules_in_package(module) - else: - destination_modules = {module} - - upstream_modules = set() - - for candidate in filter(lambda m: m not in destination_modules, self.modules): - for destination_module in destination_modules: - if self.chain_exists(importer=destination_module, imported=candidate): - upstream_modules.add(candidate) - break - - return upstream_modules + return self._rustgraph.find_upstream_modules(module, as_package) - def find_shortest_chain(self, importer: str, imported: str) -> Optional[Tuple[str, ...]]: + def find_shortest_chain(self, importer: str, imported: str) -> tuple[str, ...] | None: for module in (importer, imported): - if module not in self.modules: + if not self._rustgraph.contains_module(module): raise ValueError(f"Module {module} is not present in the graph.") - return self._find_shortest_chain(importer=importer, imported=imported) + chain = self._rustgraph.find_shortest_chain(importer, imported) + return tuple(chain) if chain else None def find_shortest_chains( self, importer: str, imported: str, as_packages: bool = True ) -> Set[Tuple[str, ...]]: - """ - Find the shortest import chains that exist between the importer and imported, and - between any modules contained within them if as_packages is True. Only one chain per - upstream/downstream pair will be included. Any chains that are contained within other - chains in the result set will be excluded. - - The default behavior is to treat the import and imported as packages, however, if - as_packages is False, both the importer and imported will be treated as modules instead. - - Returns: - A set of tuples of strings. Each tuple is ordered from importer to imported modules. - """ - shortest_chains = set() - - upstream_modules = ( - {imported} if not as_packages else self._all_modules_in_package(imported) - ) - downstream_modules = ( - {importer} if not as_packages else self._all_modules_in_package(importer) - ) - - if upstream_modules & downstream_modules: - # If there are shared modules between the two, one of the modules is a descendant - # of the other (or they're both the same module). This doesn't make sense in - # this context, so raise an exception. - raise ValueError("Modules have shared descendants.") - - imports_between_modules = self._find_all_imports_between_modules( - upstream_modules - ) | self._find_all_imports_between_modules(downstream_modules) - self._hide_any_existing_imports(imports_between_modules) - - map_of_imports = {} - for module in upstream_modules | downstream_modules: - map_of_imports[module] = set( - (m, module) for m in self.find_modules_that_directly_import(module) - ) | set((module, m) for m in self.find_modules_directly_imported_by(module)) - for imports in map_of_imports.values(): - self._hide_any_existing_imports(imports) - - for upstream in upstream_modules: - imports_of_upstream_module = map_of_imports[upstream] - self._reveal_imports(imports_of_upstream_module) - for downstream in downstream_modules: - imports_by_downstream_module = map_of_imports[downstream] - self._reveal_imports(imports_by_downstream_module) - shortest_chain = self._find_shortest_chain(imported=upstream, importer=downstream) - if shortest_chain: - shortest_chains.add(shortest_chain) - self._hide_any_existing_imports(imports_by_downstream_module) - self._hide_any_existing_imports(imports_of_upstream_module) - - # Reveal all the hidden imports. - for imports in map_of_imports.values(): - self._reveal_imports(imports) - self._reveal_imports(imports_between_modules) - - return shortest_chains + return self._rustgraph.find_shortest_chains(importer, imported, as_packages) def chain_exists(self, importer: str, imported: str, as_packages: bool = False) -> bool: - if not as_packages: - return bool(self._find_shortest_chain(importer=importer, imported=imported)) - - upstream_modules = self._all_modules_in_package(imported) - downstream_modules = self._all_modules_in_package(importer) - - if upstream_modules & downstream_modules: - # If there are shared modules between the two, one of the modules is a descendant - # of the other (or they're both the same module). This doesn't make sense in - # this context, so raise an exception. - raise ValueError("Modules have shared descendants.") - - # Return True as soon as we find a path between any of the modules in the subpackages. - for upstream in upstream_modules: - for downstream in downstream_modules: - if self.chain_exists(imported=upstream, importer=downstream): - return True - - return False - - # High level analysis + return self._rustgraph.chain_exists(importer, imported, as_packages) def find_illegal_dependencies_for_layers( self, layers: Sequence[Layer | str | set[str]], containers: set[str] | None = None, ) -> set[PackageDependency]: - layers = _layers.parse_layers(layers) - return _layers.find_illegal_dependencies( - graph=self, layers=layers, containers=containers or set() - ) + layers = _parse_layers(layers) + try: + result = self._rustgraph.find_illegal_dependencies_for_layers( + layers=tuple( + {"layers": layer.module_tails, "independent": layer.independent} + for layer in layers + ), + containers=set(containers) if containers else set(), + ) + except rust.NoSuchContainer as e: + raise NoSuchContainer(str(e)) - # Private methods + return _dependencies_from_tuple(result) - def _find_ancestor_squashed_module(self, module: str) -> Optional[str]: - """ - Return the name of a squashed module that is an ancestor of the supplied module, or None - if no such module exists. - """ - try: - parent = Module(module).parent.name - except ValueError: - # The module has no more ancestors. - return None + # Dunder methods + # -------------- - if parent in self.modules and self.is_module_squashed(parent): - return parent - else: - return self._find_ancestor_squashed_module(parent) - - def _mark_module_as_squashed(self, module: str) -> None: - """ - Set a flag on a module in the graph that it is squashed. - """ - self._squashed_modules.add(module) - - def _all_modules_in_package(self, module: str) -> Set[str]: - """ - Return all the modules in the supplied module, including itself. - - If the module is squashed, it will be treated as a single module. - """ - importer_modules = {module} - if not self.is_module_squashed(module): - importer_modules |= self.find_descendants(module) - return importer_modules - - def _find_all_imports_between_modules(self, modules: Set[str]) -> Set[Tuple[str, str]]: - """ - Return all the imports between the supplied set of modules. - - Return: - Set of imports, in the form (importer, imported). - """ - imports = set() - for importer in modules: - for imported in self.find_modules_directly_imported_by(importer): - if imported in modules: - imports.add((importer, imported)) - return imports - - def _hide_any_existing_imports(self, imports: Set[Tuple[str, str]]) -> None: - """ - Temporarily remove the supplied direct imports from the graph. - - If an import is not in the graph, or already hidden, this will have no effect. - - Args: - imports: Set of direct imports, in the form (importer, imported). - """ - for importer, imported in tuple(imports): - if self.direct_import_exists(importer=importer, imported=imported): - # Low-level removal from import graph (but leaving other metadata in place). - self._importeds_by_importer[importer].remove(imported) - self._importers_by_imported[imported].remove(importer) - - def _reveal_imports(self, imports: Set[Tuple[str, str]]) -> None: - """ - Given a set of direct imports that were hidden by _hide_any_existing_imports, add them back. - - Args: - imports: Set of direct imports, in the form (importer, imported). - """ - for importer, imported in tuple(imports): - # Low-level addition to import graph. - self._importeds_by_importer[importer].add(imported) - self._importers_by_imported[imported].add(importer) - - def _find_shortest_chain(self, importer: str, imported: str) -> Optional[Tuple[str, ...]]: - # Similar to find_shortest_chain but without bothering to check if the modules are - # in the graph first. - return bidirectional_shortest_path( - importers_by_imported=self._importers_by_imported, - importeds_by_importer=self._importeds_by_importer, - importer=importer, - imported=imported, - ) + def __deepcopy__(self, memodict: dict) -> "ImportGraph": + new_graph = ImportGraph() + new_graph._rustgraph = self._rustgraph.clone() + return new_graph -_StringSet = Set[str] +class _RustRoute(TypedDict): + heads: frozenset[str] + middle: tuple[str, ...] + tails: frozenset[str] + + +class _RustPackageDependency(TypedDict): + importer: str + imported: str + routes: tuple[_RustRoute, ...] + + +def _parse_layers(layers: Sequence[Layer | str | set[str]]) -> tuple[Layer, ...]: + """ + Convert the passed raw `layers` into `Layer`s. + """ + out_layers = [] + for layer in layers: + if isinstance(layer, Layer): + out_layers.append(layer) + elif isinstance(layer, str): + out_layers.append(Layer(layer, independent=True)) + else: + out_layers.append(Layer(*tuple(layer), independent=True)) + return tuple(out_layers) + + +def _dependencies_from_tuple( + rust_package_dependency_tuple: tuple[_RustPackageDependency, ...] +) -> set[PackageDependency]: + return { + PackageDependency( + imported=dep_dict["imported"], + importer=dep_dict["importer"], + routes=frozenset( + { + Route( + heads=route_dict["heads"], + middle=route_dict["middle"], + tails=route_dict["tails"], + ) + for route_dict in dep_dict["routes"] + } + ), + ) + for dep_dict in rust_package_dependency_tuple + } diff --git a/tests/benchmarking/test_benchmarking.py b/tests/benchmarking/test_benchmarking.py index e5cdc964..4cedd334 100644 --- a/tests/benchmarking/test_benchmarking.py +++ b/tests/benchmarking/test_benchmarking.py @@ -8,12 +8,7 @@ def _run_benchmark(benchmark, fn, *args, **kwargs): - if hasattr(benchmark, "pedantic"): - # Running with pytest-benchmark - return benchmark.pedantic(fn, args=args, kwargs=kwargs, rounds=3) - else: - # Running with codspeed. - return benchmark(fn, *args, **kwargs) + return benchmark(fn, *args, **kwargs) @pytest.fixture(scope="module")