From 8945b91605cb21053a468edb65b8fa663b8768cf Mon Sep 17 00:00:00 2001 From: David Seddon Date: Fri, 10 Jan 2025 13:04:44 +0000 Subject: [PATCH 01/10] Implement Graph struct This is the main graph implemented in Rust - but it isn't plumbed into Python yet. --- rust/Cargo.lock | 46 + rust/Cargo.toml | 2 + rust/src/graph.rs | 2928 +++++++++++++++++++++++++++++++++++++++++++++ rust/src/lib.rs | 1 + 4 files changed, 2977 insertions(+) create mode 100644 rust/src/graph.rs diff --git a/rust/Cargo.lock b/rust/Cargo.lock index faeb0261..c9b0a5fe 100644 --- a/rust/Cargo.lock +++ b/rust/Cargo.lock @@ -6,7 +6,9 @@ version = 3 name = "_rustgrimp" version = "0.1.0" dependencies = [ + "bimap", "log", + "petgraph", "pyo3", "pyo3-log", "rayon", @@ -25,6 +27,12 @@ version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0c4b4d0bd25bd0b74681c0ad21497610ce1b7c91b1022cd21c80c6fbdd9476b0" +[[package]] +name = "bimap" +version = "0.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "230c5f1ca6a325a32553f8640d31ac9b49f2411e901e427570154868b46da4f7" + [[package]] name = "cfg-if" version = "1.0.0" @@ -62,12 +70,40 @@ version = "1.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "60b1af1c220855b6ceac025d3f6ecdd2b7c4894bfe9cd9bda4fbb4bc7c0d4cf0" +[[package]] +name = "equivalent" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5443807d6dff69373d433ab9ef5378ad8df50ca6298caf15de6e52e24aaf54d5" + +[[package]] +name = "fixedbitset" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ce7134b9999ecaf8bcd65542e436736ef32ddca1b3e06094cb6ec5755203b80" + +[[package]] +name = "hashbrown" +version = "0.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e087f84d4f86bf4b218b927129862374b72199ae7d8657835f1e89000eea4fb" + [[package]] name = "heck" version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" +[[package]] +name = "indexmap" +version = "2.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "707907fe3c25f5424cce2cb7e1cbcafee6bdbe735ca90ef77c29e84591e5b9da" +dependencies = [ + "equivalent", + "hashbrown", +] + [[package]] name = "indoc" version = "2.0.5" @@ -107,6 +143,16 @@ version = "1.19.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3fdb12b2476b595f9358c5161aa467c2438859caa136dec86c26fdd2efe17b92" +[[package]] +name = "petgraph" +version = "0.6.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b4c5cc86750666a3ed20bdaf5ca2a0344f9c67674cae0515bec2da16fbaa47db" +dependencies = [ + "fixedbitset", + "indexmap", +] + [[package]] name = "portable-atomic" version = "1.6.0" diff --git a/rust/Cargo.toml b/rust/Cargo.toml index fca4e896..3ab4be53 100644 --- a/rust/Cargo.toml +++ b/rust/Cargo.toml @@ -12,6 +12,8 @@ log = "0.4.19" pyo3-log = "0.11.0" serde_json = "1.0.103" rayon = "1.10" +petgraph = "0.6.5" +bimap = "0.6.3" [dependencies.pyo3] version = "0.22.4" diff --git a/rust/src/graph.rs b/rust/src/graph.rs new file mode 100644 index 00000000..0de65ec3 --- /dev/null +++ b/rust/src/graph.rs @@ -0,0 +1,2928 @@ +/* +Also, sensible behaviour when passing modules that don't exist in the graph. +*/ +#![allow(dead_code)] + +use bimap::BiMap; +use log::info; +use petgraph::algo::astar; +use petgraph::graph::EdgeIndex; +use petgraph::stable_graph::{NodeIndex, StableGraph}; +use petgraph::visit::{Bfs, Walker}; +use petgraph::Direction; +use rayon::prelude::*; +use std::collections::{HashMap, HashSet}; +use std::fmt; +use std::time::Instant; + +/// A group of layers at the same level in the layering. +#[derive(PartialEq, Eq, Hash, Debug)] +pub struct Level { + pub layers: Vec, + pub independent: bool, +} + +// Delimiter for Python modules. +const DELIMITER: char = '.'; + +#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)] +pub struct Module { + pub name: String, +} + +impl fmt::Display for Module { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + write!(f, "{}", self.name) + } +} + +#[derive(Debug, Clone, PartialEq)] +pub struct ModuleNotPresent { + pub module: Module, +} + +#[derive(Debug, Clone, PartialEq)] +pub struct NoSuchContainer { + pub container: String, +} + +pub struct ModulesHaveSharedDescendants {} + +impl fmt::Display for ModuleNotPresent { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + write!(f, "\"{}\" not present in the graph", self.module.name) + } +} + +impl Module { + pub fn new(name: String) -> Module { + Module { name } + } + + // Returns whether the module is a root-level package. + pub fn is_root(&self) -> bool { + !self.name.contains(DELIMITER) + } + + // Create a Module that is the parent of the passed Module. + // + // Panics if the child is a root Module. + pub fn new_parent(child: &Module) -> Module { + let parent_name = match child.name.rsplit_once(DELIMITER) { + Some((base, _)) => base.to_string(), + None => panic!("{} is a root level package", child.name), + }; + + Module::new(parent_name) + } + + // Return whether this module is a descendant of the supplied one, based on the name. + pub fn is_descendant_of(&self, module: &Module) -> bool { + let candidate = format!("{}{}", module.name, DELIMITER); + self.name.starts_with(&candidate) + } +} + +#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)] +pub struct DetailedImport { + pub importer: Module, + pub imported: Module, + pub line_number: usize, + pub line_contents: String, +} + +#[derive(Default, Clone)] +pub struct Graph { + // Bidirectional lookup between Module and NodeIndex. + hierarchy_module_indices: BiMap, + hierarchy: StableGraph, + imports_module_indices: BiMap, + imports: StableGraph, + squashed_modules: HashSet, + // Invisible modules exist in the hierarchy but haven't been explicitly added to the graph. + invisible_modules: HashSet, + detailed_imports_map: HashMap<(Module, Module), HashSet>, +} + +#[derive(PartialEq, Eq, Hash, Debug, PartialOrd, Ord)] +pub struct Route { + pub heads: Vec, + pub middle: Vec, + pub tails: Vec, +} + +#[derive(PartialEq, Eq, Hash, Debug, PartialOrd, Ord)] +pub struct PackageDependency { + pub importer: Module, + pub imported: Module, + pub routes: Vec, +} + +fn _module_from_layer(layer: &str, container: &Option) -> Module { + let module_name = match container { + Some(container) => format!("{}{}{}", container.name, DELIMITER, layer), + None => layer.to_string(), + }; + Module::new(module_name) +} + +fn _log_illegal_route_count(dependency_or_none: &Option, duration_in_s: u64) { + let route_count = match dependency_or_none { + Some(dependency) => dependency.routes.len(), + None => 0, + }; + let pluralized = if route_count == 1 { "" } else { "s" }; + info!( + "Found {} illegal route{} in {}s.", + route_count, pluralized, duration_in_s + ); +} + +impl Graph { + pub fn pretty_str(&self) -> String { + let mut hierarchy: Vec = vec![]; + let mut imports: Vec = vec![]; + + let hierarchy_module_indices: Vec<_> = self.hierarchy_module_indices.iter().collect(); + + for (parent_module, parent_index) in hierarchy_module_indices { + for child_index in self.hierarchy.neighbors(*parent_index) { + let child_module = self + .hierarchy_module_indices + .get_by_right(&child_index) + .unwrap(); + let parent_module_str = match self.invisible_modules.contains(&parent_module) { + true => format!("({})", parent_module.name), + false => parent_module.name.to_string(), + }; + let child_module_str = match self.invisible_modules.contains(&child_module) { + true => format!("({})", child_module.name), + false => child_module.name.to_string(), + }; + hierarchy.push(format!(" {} -> {}", parent_module_str, child_module_str)); + } + } + + let imports_module_indices: Vec<_> = self.imports_module_indices.iter().collect(); + + for (from_module, from_index) in imports_module_indices { + for to_index in self.imports.neighbors(*from_index) { + let to_module = self.imports_module_indices.get_by_right(&to_index).unwrap(); + imports.push(format!(" {} -> {}", from_module.name, to_module.name)); + } + } + // Assemble String. + let mut pretty = String::new(); + pretty.push_str("hierarchy:\n"); + hierarchy.sort(); + pretty.push_str(&hierarchy.join("\n")); + pretty.push_str("\nimports:\n"); + imports.sort(); + pretty.push_str(&imports.join("\n")); + pretty.push('\n'); + pretty + } + + pub fn add_module(&mut self, module: Module) { + // If this module is already in the graph, but invisible, just make it visible. + if self.invisible_modules.contains(&module) { + self.invisible_modules.remove(&module); + return; + } + // If this module is already in the graph, don't do anything. + if self.hierarchy_module_indices.get_by_left(&module).is_some() { + return; + } + + let module_index = self.hierarchy.add_node(module.clone()); + self.hierarchy_module_indices + .insert(module.clone(), module_index); + + // Add this module to the hierarchy. + if !module.is_root() { + let parent = Module::new_parent(&module); + + let parent_index = match self.hierarchy_module_indices.get_by_left(&parent) { + Some(index) => index, + None => { + // If the parent isn't already in the graph, add it, but as an invisible module. + self.add_module(parent.clone()); + self.invisible_modules.insert(parent.clone()); + self.hierarchy_module_indices.get_by_left(&parent).unwrap() + } + }; + self.hierarchy.add_edge(*parent_index, module_index, ()); + } + } + + pub fn add_squashed_module(&mut self, module: Module) { + self.add_module(module.clone()); + self.squashed_modules.insert(module); + } + + pub fn remove_module(&mut self, module: &Module) { + // Remove imports by module. + let imported_modules: Vec = self + .find_modules_directly_imported_by(module) + .iter() + .map(|m| (*m).clone()) + .collect(); + for imported_module in imported_modules { + self.remove_import(&module, &imported_module); + } + + // Remove imports of module. + let importer_modules: Vec = self + .find_modules_that_directly_import(module) + .iter() + .map(|m| (*m).clone()) + .collect(); + for importer_module in importer_modules { + self.remove_import(&importer_module, &module); + } + + // Remove module from hierarchy. + if let Some(hierarchy_index) = self.hierarchy_module_indices.get_by_left(module) { + // TODO should we check for children before removing? + // Maybe should just make invisible instead? + self.hierarchy.remove_node(*hierarchy_index); + self.hierarchy_module_indices.remove_by_left(module); + }; + } + + pub fn get_modules(&self) -> HashSet<&Module> { + self.hierarchy_module_indices + .left_values() + .filter(|module| !self.invisible_modules.contains(module)) + .collect() + } + + pub fn count_imports(&self) -> usize { + self.imports.edge_count() + } + + pub fn get_import_details( + &self, + importer: &Module, + imported: &Module, + ) -> HashSet { + let key = (importer.clone(), imported.clone()); + match self.detailed_imports_map.get(&key) { + Some(import_details) => import_details.clone(), + None => HashSet::new(), + } + } + + pub fn find_children(&self, module: &Module) -> HashSet<&Module> { + if self.invisible_modules.contains(module) { + return HashSet::new(); + } + let module_index = match self.hierarchy_module_indices.get_by_left(module) { + Some(index) => index, + // Module does not exist. + // TODO: should this return a result, to handle if module is not in graph? + None => return HashSet::new(), + }; + self.hierarchy + .neighbors(*module_index) + .map(|index| self.hierarchy_module_indices.get_by_right(&index).unwrap()) + .filter(|module| !self.invisible_modules.contains(module)) + .collect() + } + + pub fn find_descendants(&self, module: &Module) -> Result, ModuleNotPresent> { + let module_index = match self.hierarchy_module_indices.get_by_left(module) { + Some(index) => index, + None => { + return Err(ModuleNotPresent { + module: module.clone(), + }) + } + }; + Ok(Bfs::new(&self.hierarchy, *module_index) + .iter(&self.hierarchy) + .filter(|index| index != module_index) // Don't include the supplied module. + .map(|index| self.hierarchy_module_indices.get_by_right(&index).unwrap()) // This panics sometimes. + .filter(|module| !self.invisible_modules.contains(module)) + .collect()) + } + + pub fn add_import(&mut self, importer: &Module, imported: &Module) { + // Don't bother doing anything if it's already in the graph. + if self.direct_import_exists(&importer, &imported, false) { + return; + } + + self.add_module_if_not_in_hierarchy(importer); + self.add_module_if_not_in_hierarchy(imported); + + let importer_index: NodeIndex = match self.imports_module_indices.get_by_left(importer) { + Some(index) => *index, + None => { + let index = self.imports.add_node(importer.clone()); + self.imports_module_indices.insert(importer.clone(), index); + index + } + }; + let imported_index: NodeIndex = match self.imports_module_indices.get_by_left(imported) { + Some(index) => *index, + None => { + let index = self.imports.add_node(imported.clone()); + self.imports_module_indices.insert(imported.clone(), index); + index + } + }; + + self.imports.add_edge(importer_index, imported_index, ()); + // println!( + // "Added {:?} {:?} -> {:?} {:?}, edge count now {:?}", + // importer, + // importer_index, + // imported, + // imported_index, + // self.imports.edge_count() + // ); + } + + pub fn add_detailed_import(&mut self, import: &DetailedImport) { + let key = (import.importer.clone(), import.imported.clone()); + self.detailed_imports_map + .entry(key) + .or_insert_with(HashSet::new) + .insert(import.clone()); + self.add_import(&import.importer, &import.imported); + } + + pub fn remove_import(&mut self, importer: &Module, imported: &Module) { + let importer_index: NodeIndex = match self.imports_module_indices.get_by_left(importer) { + Some(index) => *index, + None => return, + }; + let imported_index: NodeIndex = match self.imports_module_indices.get_by_left(imported) { + Some(index) => *index, + None => return, + }; + let edge_index: EdgeIndex = match self.imports.find_edge(importer_index, imported_index) { + Some(index) => index, + None => return, + }; + + self.imports.remove_edge(edge_index); + + // There might be other imports to / from the modules, so don't + // remove from the indices. (TODO: does it matter if we don't clean these up + // if there are no more imports?) + // self.imports_module_indices.remove_by_left(importer); + // self.imports_module_indices.remove_by_left(importer); + + let key = (importer.clone(), imported.clone()); + + self.detailed_imports_map.remove(&key); + self.imports.remove_edge(edge_index); + } + + // Note: this will panic if importer and imported are in the same package. + #[allow(unused_variables)] + pub fn direct_import_exists( + &self, + importer: &Module, + imported: &Module, + as_packages: bool, + ) -> bool { + let graph_to_use: &Graph; + let mut graph_copy: Graph; + + if as_packages { + graph_copy = self.clone(); + graph_copy.squash_module(importer); + graph_copy.squash_module(imported); + graph_to_use = &graph_copy; + } else { + graph_to_use = self; + } + + // The modules may appear in the hierarchy, but have no imports, so we + // return false unless they're both in there. + let importer_index = match graph_to_use.imports_module_indices.get_by_left(importer) { + Some(importer_index) => *importer_index, + None => return false, + }; + let imported_index = match graph_to_use.imports_module_indices.get_by_left(imported) { + Some(imported_index) => *imported_index, + None => return false, + }; + + graph_to_use + .imports + .contains_edge(importer_index, imported_index) + } + + pub fn find_modules_that_directly_import(&self, imported: &Module) -> HashSet<&Module> { + let imported_index = match self.imports_module_indices.get_by_left(imported) { + Some(imported_index) => *imported_index, + None => return HashSet::new(), + }; + let importer_indices: HashSet = self + .imports + .neighbors_directed(imported_index, Direction::Incoming) + .collect(); + + let importers: HashSet<&Module> = importer_indices + .iter() + .map(|importer_index| { + self.imports_module_indices + .get_by_right(importer_index) + .unwrap() + }) + .collect(); + importers + } + + pub fn find_modules_directly_imported_by(&self, importer: &Module) -> HashSet<&Module> { + let importer_index = match self.imports_module_indices.get_by_left(importer) { + Some(importer_index) => *importer_index, + None => return HashSet::new(), + }; + let imported_indices: HashSet = self + .imports + .neighbors_directed(importer_index, Direction::Outgoing) + .collect(); + + let importeds: HashSet<&Module> = imported_indices + .iter() + .map(|imported_index| { + self.imports_module_indices + .get_by_right(imported_index) + .unwrap() + }) + .collect(); + importeds + } + + pub fn find_upstream_modules(&self, module: &Module, as_package: bool) -> HashSet<&Module> { + let mut upstream_modules = HashSet::new(); + + let mut modules_to_check: HashSet<&Module> = HashSet::from([module]); + if as_package { + let descendants = self.find_descendants(&module).unwrap_or(HashSet::new()); + modules_to_check.extend(descendants.into_iter()); + }; + + for module_to_check in modules_to_check.iter() { + let module_index = match self.imports_module_indices.get_by_left(module_to_check) { + Some(index) => *index, + None => continue, + }; + upstream_modules.extend( + Bfs::new(&self.imports, module_index) + .iter(&self.imports) + .map(|index| self.imports_module_indices.get_by_right(&index).unwrap()) + // Exclude any modules that we are checking. + .filter(|downstream_module| !modules_to_check.contains(downstream_module)), + ); + } + + upstream_modules + } + + pub fn find_downstream_modules(&self, module: &Module, as_package: bool) -> HashSet<&Module> { + let mut downstream_modules = HashSet::new(); + + let mut modules_to_check: HashSet<&Module> = HashSet::from([module]); + if as_package { + let descendants = self.find_descendants(&module).unwrap_or(HashSet::new()); + modules_to_check.extend(descendants.into_iter()); + }; + + for module_to_check in modules_to_check.iter() { + let module_index = match self.imports_module_indices.get_by_left(module_to_check) { + Some(index) => *index, + None => continue, + }; + + // Reverse all the edges in the graph and then do what we do in find_upstream_modules. + // Is there a way of doing this without the clone? + let mut reversed_graph = self.imports.clone(); + reversed_graph.reverse(); + + downstream_modules.extend( + Bfs::new(&reversed_graph, module_index) + .iter(&reversed_graph) + .map(|index| self.imports_module_indices.get_by_right(&index).unwrap()) + // Exclude any modules that we are checking. + .filter(|downstream_module| !modules_to_check.contains(downstream_module)), + ) + } + + downstream_modules + } + + pub fn find_shortest_chain( + &self, + importer: &Module, + imported: &Module, + ) -> Option> { + let importer_index = match self.imports_module_indices.get_by_left(importer) { + Some(index) => *index, + None => return None, // Importer has no imports to or from. + }; + let imported_index = match self.imports_module_indices.get_by_left(imported) { + Some(index) => *index, + None => return None, // Imported has no imports to or from. + }; + let path_to_imported = match astar( + &self.imports, + importer_index, + |finish| finish == imported_index, + |_e| 1, + |_| 0, + ) { + Some(path_tuple) => path_tuple.1, + None => return None, // No chain to the imported. + }; + + let mut chain: Vec<&Module> = vec![]; + for link_index in path_to_imported { + let module = self + .imports_module_indices + .get_by_right(&link_index) + .unwrap(); + chain.push(module); + } + Some(chain) + } + + // https://github.com/seddonym/grimp/blob/master/src/grimp/adaptors/graph.py#L290 + pub fn find_shortest_chains( + &self, + importer: &Module, + imported: &Module, + as_packages: bool, + ) -> Result>, String> { + let mut chains = HashSet::new(); + let mut temp_graph = self.clone(); + + let mut downstream_modules: HashSet = HashSet::from([importer.clone()]); + let mut upstream_modules: HashSet = HashSet::from([imported.clone()]); + + // TODO don't do this if module is squashed? + if as_packages { + for descendant in self.find_descendants(importer).unwrap() { + downstream_modules.insert(descendant.clone()); + } + for descendant in self.find_descendants(imported).unwrap() { + upstream_modules.insert(descendant.clone()); + } + if upstream_modules + .intersection(&downstream_modules) + .next() + .is_some() + { + return Err("Modules have shared descendants.".to_string()); + } + } + + // Remove imports within the packages. + let mut imports_to_remove: Vec<(Module, Module)> = vec![]; + for upstream_module in &upstream_modules { + for imported_module in temp_graph.find_modules_directly_imported_by(&upstream_module) { + if upstream_modules.contains(&imported_module) { + imports_to_remove.push((upstream_module.clone(), imported_module.clone())); + } + } + } + for downstream_module in &downstream_modules { + for imported_module in temp_graph.find_modules_directly_imported_by(&downstream_module) + { + if downstream_modules.contains(&imported_module) { + imports_to_remove.push((downstream_module.clone(), imported_module.clone())); + } + } + } + for (importer_to_remove, imported_to_remove) in imports_to_remove { + temp_graph.remove_import(&importer_to_remove, &imported_to_remove); + } + + // Keep track of imports into/out of upstream/downstream packages, and remove them. + let mut map_of_imports: HashMap> = HashMap::new(); + for module in upstream_modules.union(&downstream_modules) { + let mut imports_to_or_from_module = HashSet::new(); + for imported_module in temp_graph.find_modules_directly_imported_by(&module) { + imports_to_or_from_module.insert((module.clone(), imported_module.clone())); + } + for importer_module in temp_graph.find_modules_that_directly_import(&module) { + imports_to_or_from_module.insert((importer_module.clone(), module.clone())); + } + map_of_imports.insert(module.clone(), imports_to_or_from_module); + } + for imports in map_of_imports.values() { + for (importer_to_remove, imported_to_remove) in imports { + temp_graph.remove_import(&importer_to_remove, &imported_to_remove); + } + } + + for importer_module in &downstream_modules { + // Reveal imports to/from importer module. + for (importer_to_add, imported_to_add) in &map_of_imports[&importer_module] { + temp_graph.add_import(&importer_to_add, &imported_to_add); + } + for imported_module in &upstream_modules { + // Reveal imports to/from imported module. + for (importer_to_add, imported_to_add) in &map_of_imports[&imported_module] { + temp_graph.add_import(&importer_to_add, &imported_to_add); + } + if let Some(chain) = + temp_graph.find_shortest_chain(importer_module, imported_module) + { + chains.insert(chain.iter().cloned().map(|module| module.clone()).collect()); + } + // Remove imports relating to imported module again. + for (importer_to_remove, imported_to_remove) in &map_of_imports[&imported_module] { + temp_graph.remove_import(&importer_to_remove, &imported_to_remove); + } + } + // Remove imports relating to importer module again. + for (importer_to_remove, imported_to_remove) in &map_of_imports[&importer_module] { + temp_graph.remove_import(&importer_to_remove, &imported_to_remove); + } + } + Ok(chains) + } + + #[allow(unused_variables)] + pub fn chain_exists(&self, importer: &Module, imported: &Module, as_packages: bool) -> bool { + // TODO should this return a Result, so we can handle the situation the importer / imported + // having shared descendants when as_packages=true? + let mut temp_graph; + let graph = match as_packages { + true => { + temp_graph = self.clone(); + temp_graph.squash_module(importer); + temp_graph.squash_module(imported); + &temp_graph + } + false => self, + }; + graph.find_shortest_chain(importer, imported).is_some() + } + + #[allow(unused_variables)] + pub fn find_illegal_dependencies_for_layers( + &self, + levels: Vec, + containers: HashSet, + ) -> Result, NoSuchContainer> { + // Check that containers exist. + let modules = self.get_modules(); + for container in containers.iter() { + let container_module = Module::new(container.clone()); + if !modules.contains(&container_module) { + return Err(NoSuchContainer { + container: container.clone(), + }); + } + } + + let all_layers: Vec = levels + .iter() + .flat_map(|level| level.layers.iter()) + .map(|module_name| module_name.to_string()) + .collect(); + + let perms = self._generate_module_permutations(&levels, &containers); + + let mut dependencies: Vec = self + ._generate_module_permutations(&levels, &containers) + //.into_iter() + .into_par_iter() + .filter_map(|(higher_layer_package, lower_layer_package, container)| { + // TODO: it's inefficient to do this for sibling layers, as we don't need + // to clone and trim the graph for identical pairs. + info!( + "Searching for import chains from {} to {}...", + lower_layer_package, higher_layer_package + ); + let now = Instant::now(); + let dependency_or_none = self._search_for_package_dependency( + &higher_layer_package, + &lower_layer_package, + &all_layers, + &container, + ); + _log_illegal_route_count(&dependency_or_none, now.elapsed().as_secs()); + dependency_or_none + }) + .collect(); + + dependencies.sort(); + + Ok(dependencies) + } + + // Return every permutation of modules that exist in the graph + /// in which the second should not import the first. + /// The third item in the tuple is the relevant container, if used. + fn _generate_module_permutations( + &self, + levels: &Vec, + containers: &HashSet, + ) -> Vec<(Module, Module, Option)> { + let mut permutations: Vec<(Module, Module, Option)> = vec![]; + + let quasi_containers: Vec> = if containers.is_empty() { + vec![None] + } else { + containers + .iter() + .map(|i| Some(Module::new(i.to_string()))) + .collect() + }; + let all_modules = self.get_modules(); + + for quasi_container in quasi_containers { + for (index, higher_level) in levels.iter().enumerate() { + for higher_layer in &higher_level.layers { + let higher_layer_module = _module_from_layer(&higher_layer, &quasi_container); + if !all_modules.contains(&higher_layer_module) { + continue; + } + + // Build the layers that mustn't import this higher layer. + // That includes: + // * lower layers. + // * sibling layers, if the layer is independent. + let mut layers_forbidden_to_import_higher_layer: Vec = vec![]; + + // Independence + if higher_level.independent { + for potential_sibling_layer in &higher_level.layers { + let sibling_module = + _module_from_layer(&potential_sibling_layer, &quasi_container); + if sibling_module != higher_layer_module + && all_modules.contains(&sibling_module) + { + layers_forbidden_to_import_higher_layer.push(sibling_module); + } + } + } + + for lower_level in &levels[index + 1..] { + for lower_layer in &lower_level.layers { + let lower_layer_module = + _module_from_layer(&lower_layer, &quasi_container); + if all_modules.contains(&lower_layer_module) { + layers_forbidden_to_import_higher_layer.push(lower_layer_module); + } + } + } + + // Add to permutations. + for forbidden in layers_forbidden_to_import_higher_layer { + permutations.push(( + higher_layer_module.clone(), + forbidden.clone(), + quasi_container.clone(), + )); + } + } + } + } + + permutations + } + + fn _search_for_package_dependency( + &self, + higher_layer_package: &Module, + lower_layer_package: &Module, + layers: &Vec, + container: &Option, + ) -> Option { + let mut temp_graph = self.clone(); + + // Remove other layers. + let mut modules_to_remove: Vec = vec![]; + for layer in layers { + let layer_module = _module_from_layer(&layer, &container); + if layer_module != *higher_layer_package && layer_module != *lower_layer_package { + // Remove this subpackage. + match temp_graph.find_descendants(&layer_module) { + Ok(descendants) => { + for descendant in descendants { + modules_to_remove.push(descendant.clone()) + } + } + Err(_) => (), // ModuleNotPresent. + } + modules_to_remove.push(layer_module.clone()); + } + } + for module_to_remove in modules_to_remove.clone() { + temp_graph.remove_module(&module_to_remove); + } + + let mut routes: Vec = vec![]; + + // Direct routes. + // TODO: do we need to pop the imports? + // The indirect routes should cope without removing them? + let direct_links = + temp_graph._pop_direct_imports(lower_layer_package, higher_layer_package); + for (importer, imported) in direct_links { + routes.push(Route { + heads: vec![importer], + middle: vec![], + tails: vec![imported], + }); + } + + // Indirect routes. + for indirect_route in + temp_graph._find_indirect_routes(lower_layer_package, higher_layer_package) + { + routes.push(indirect_route); + } + + if routes.is_empty() { + None + } else { + Some(PackageDependency { + importer: lower_layer_package.clone(), + imported: higher_layer_package.clone(), + routes, + }) + } + } + + fn _find_indirect_routes( + &self, + importer_package: &Module, + imported_package: &Module, + ) -> Vec { + let mut routes = vec![]; + + let mut temp_graph = self.clone(); + temp_graph.squash_module(importer_package); + temp_graph.squash_module(imported_package); + + // Find middles. + let mut middles: Vec> = vec![]; + for chain in temp_graph._pop_shortest_chains(importer_package, imported_package) { + // Remove first and last element. + let mut middle: Vec = vec![]; + let chain_length = chain.len(); + for (index, module) in chain.iter().enumerate() { + if index != 0 && index != chain_length - 1 { + middle.push(module.clone()); + } + } + middles.push(middle); + } + + // Set up importer/imported package contents. + let mut importer_modules: HashSet<&Module> = HashSet::from([importer_package]); + importer_modules.extend(self.find_descendants(&importer_package).unwrap()); + let mut imported_modules: HashSet<&Module> = HashSet::from([imported_package]); + imported_modules.extend(self.find_descendants(&imported_package).unwrap()); + + // Build routes from middles. + for middle in middles { + // Construct heads. + let mut heads: Vec = vec![]; + let first_imported_module = &middle[0]; + for candidate_head in self.find_modules_that_directly_import(&first_imported_module) { + if importer_modules.contains(candidate_head) { + heads.push(candidate_head.clone()); + } + } + + // Construct tails. + let mut tails: Vec = vec![]; + let last_importer_module = &middle[middle.len() - 1]; + for candidate_tail in self.find_modules_directly_imported_by(&last_importer_module) { + if imported_modules.contains(candidate_tail) { + tails.push(candidate_tail.clone()); + } + } + + routes.push(Route { + heads, + middle, + tails, + }) + } + + routes + } + + fn _pop_shortest_chains(&mut self, importer: &Module, imported: &Module) -> Vec> { + let mut chains = vec![]; + + loop { + // TODO - defend against infinite loops somehow. + + let found_chain: Vec; + { + let chain = self.find_shortest_chain(importer, imported); + + if chain.is_none() { + break; + } + + found_chain = chain.unwrap().into_iter().cloned().collect(); + } + // Remove chain. + for i in 0..found_chain.len() - 1 { + self.remove_import(&found_chain[i], &found_chain[i + 1]); + } + chains.push(found_chain); + } + chains + } + + /// Remove the direct imports, returning them as (importer, imported) tuples. + fn _pop_direct_imports( + &mut self, + lower_layer_module: &Module, + higher_layer_module: &Module, + ) -> HashSet<(Module, Module)> { + let mut imports = HashSet::new(); + + let mut lower_layer_modules = HashSet::from([lower_layer_module.clone()]); + for descendant in self + .find_descendants(lower_layer_module) + .unwrap() + .iter() + .cloned() + { + lower_layer_modules.insert(descendant.clone()); + } + + let mut higher_layer_modules = HashSet::from([higher_layer_module.clone()]); + for descendant in self + .find_descendants(higher_layer_module) + .unwrap() + .iter() + .cloned() + { + higher_layer_modules.insert(descendant.clone()); + } + + for lower_layer_module in lower_layer_modules { + for imported_module in self.find_modules_directly_imported_by(&lower_layer_module) { + if higher_layer_modules.contains(imported_module) { + imports.insert((lower_layer_module.clone(), imported_module.clone())); + } + } + } + + // Remove imports. + for (importer, imported) in &imports { + self.remove_import(&importer, &imported) + } + + imports + } + + #[allow(unused_variables)] + pub fn squash_module(&mut self, module: &Module) { + // Get descendants and their imports. + let descendants: Vec = self + .find_descendants(module) + .unwrap() + .into_iter() + .cloned() + .collect(); + let modules_imported_by_descendants: Vec = descendants + .iter() + .flat_map(|descendant| { + self.find_modules_directly_imported_by(descendant) + .into_iter() + .cloned() + }) + .collect(); + let modules_that_import_descendants: Vec = descendants + .iter() + .flat_map(|descendant| { + self.find_modules_that_directly_import(descendant) + .into_iter() + .cloned() + }) + .collect(); + + // Remove any descendants. + for descendant in descendants { + self.remove_module(&descendant); + } + + // Add descendants and imports to parent module. + for imported in modules_imported_by_descendants { + self.add_import(module, &imported); + } + + for importer in modules_that_import_descendants { + self.add_import(&importer, module); + } + + self.squashed_modules.insert(module.clone()); + } + + pub fn is_module_squashed(&self, module: &Module) -> bool { + self.squashed_modules.contains(module) + } + + fn add_module_if_not_in_hierarchy(&mut self, module: &Module) { + if self.hierarchy_module_indices.get_by_left(module).is_none() { + self.add_module(module.clone()); + }; + if self.invisible_modules.contains(&module) { + self.invisible_modules.remove(&module); + }; + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn modules_when_empty() { + let graph = Graph::default(); + + assert_eq!(graph.get_modules(), HashSet::new()); + } + + #[test] + fn module_is_value_object() { + assert_eq!( + Module::new("mypackage".to_string()), + Module::new("mypackage".to_string()) + ); + } + + #[test] + fn add_module() { + let mypackage = Module::new("mypackage".to_string()); + let mut graph = Graph::default(); + graph.add_module(mypackage.clone()); + + let result = graph.get_modules(); + + assert_eq!(result, HashSet::from([&mypackage])); + } + + #[test] + fn add_module_doesnt_add_parent() { + let mypackage = Module::new("mypackage.foo".to_string()); + let mut graph = Graph::default(); + graph.add_module(mypackage.clone()); + + let result = graph.get_modules(); + + assert_eq!(result, HashSet::from([&mypackage])); + } + + #[test] + fn add_modules() { + let mut graph = Graph::default(); + let mypackage = Module::new("mypackage".to_string()); + let mypackage_foo = Module::new("mypackage.foo".to_string()); + graph.add_module(mypackage.clone()); + graph.add_module(mypackage_foo.clone()); + + let result = graph.get_modules(); + + assert_eq!(result, HashSet::from([&mypackage, &mypackage_foo])); + assert_eq!( + graph.pretty_str(), + " +hierarchy: + mypackage -> mypackage.foo +imports: + +" + .trim_start() + ); + } + + #[test] + fn remove_nonexistent_module() { + let mypackage = Module::new("mypackage".to_string()); + let mypackage_foo = Module::new("mypackage.foo".to_string()); + let mut graph = Graph::default(); + // Add mypackage but not mypackage.foo. + graph.add_module(mypackage.clone()); + + graph.remove_module(&mypackage_foo); + + let result = graph.get_modules(); + assert_eq!(result, HashSet::from([&mypackage])); + } + + #[test] + fn remove_existing_module_without_imports() { + let mypackage = Module::new("mypackage".to_string()); + let mypackage_foo = Module::new("mypackage.foo".to_string()); + let mypackage_foo_alpha = Module::new("mypackage.foo.alpha".to_string()); + + let mut graph = Graph::default(); + graph.add_module(mypackage.clone()); + graph.add_module(mypackage_foo.clone()); + graph.add_module(mypackage_foo_alpha.clone()); + + graph.remove_module(&mypackage_foo); + + let result = graph.get_modules(); + assert_eq!( + result, + HashSet::from([ + &mypackage, + &mypackage_foo_alpha, // To be consistent with previous versions of Grimp. + ]) + ); + } + + #[test] + fn remove_existing_module_with_imports() { + let mypackage = Module::new("mypackage".to_string()); + let mypackage_foo = Module::new("mypackage.foo".to_string()); + let mypackage_foo_alpha = Module::new("mypackage.foo.alpha".to_string()); + let importer = Module::new("importer".to_string()); + let imported = Module::new("importer".to_string()); + let mut graph = Graph::default(); + graph.add_module(mypackage.clone()); + graph.add_module(mypackage_foo.clone()); + graph.add_module(mypackage_foo_alpha.clone()); + graph.add_import(&importer, &mypackage_foo); + graph.add_import(&mypackage_foo, &imported); + + graph.remove_module(&mypackage_foo); + + let result = graph.get_modules(); + assert_eq!( + result, + HashSet::from([&mypackage, &mypackage_foo_alpha, &importer, &imported]) + ); + assert_eq!( + graph.direct_import_exists(&importer, &mypackage_foo, false), + false + ); + assert_eq!( + graph.direct_import_exists(&mypackage_foo, &imported, false), + false + ); + } + + #[test] + fn remove_importer_module_removes_import_details() { + let importer = Module::new("importer".to_string()); + let imported = Module::new("importer".to_string()); + let mut graph = Graph::default(); + graph.add_detailed_import(&DetailedImport { + importer: importer.clone(), + imported: imported.clone(), + line_number: 99, + line_contents: "-".to_string(), + }); + + graph.remove_module(&importer); + + assert_eq!( + graph.get_import_details(&importer, &imported), + HashSet::new() + ); + } + + #[test] + fn remove_imported_module_removes_import_details() { + let importer = Module::new("importer".to_string()); + let imported = Module::new("importer".to_string()); + let mut graph = Graph::default(); + graph.add_detailed_import(&DetailedImport { + importer: importer.clone(), + imported: imported.clone(), + line_number: 99, + line_contents: "-".to_string(), + }); + + graph.remove_module(&imported); + + assert_eq!( + graph.get_import_details(&importer, &imported), + HashSet::new() + ); + } + + #[test] + fn remove_import_that_exists() { + let importer = Module::new("importer".to_string()); + let imported = Module::new("importer".to_string()); + let mut graph = Graph::default(); + graph.add_import(&importer, &imported); + + graph.remove_import(&importer, &imported); + + // The import has gone... + assert_eq!( + graph.direct_import_exists(&importer, &imported, false), + false + ); + // ...but the modules are still there. + assert_eq!(graph.get_modules(), HashSet::from([&importer, &imported])); + } + + #[test] + fn remove_import_does_nothing_if_import_doesnt_exist() { + let importer = Module::new("importer".to_string()); + let imported = Module::new("importer".to_string()); + let mut graph = Graph::default(); + graph.add_module(importer.clone()); + graph.add_module(imported.clone()); + + graph.remove_import(&importer, &imported); + + // The modules are still there. + assert_eq!(graph.get_modules(), HashSet::from([&importer, &imported])); + } + + #[test] + fn remove_import_does_nothing_if_modules_dont_exist() { + let importer = Module::new("importer".to_string()); + let imported = Module::new("importer".to_string()); + let mut graph = Graph::default(); + + graph.remove_import(&importer, &imported); + } + + #[test] + fn remove_import_doesnt_affect_other_imports_from_same_modules() { + let blue = Module::new("blue".to_string()); + let green = Module::new("green".to_string()); + let yellow = Module::new("yellow".to_string()); + let red = Module::new("red".to_string()); + let mut graph = Graph::default(); + graph.add_import(&blue, &green); + graph.add_import(&blue, &yellow); + graph.add_import(&red, &blue); + + graph.remove_import(&blue, &green); + + // The other imports are still there. + assert_eq!(graph.direct_import_exists(&blue, &yellow, false), true); + assert_eq!(graph.direct_import_exists(&red, &blue, false), true); + } + + #[test] + #[should_panic(expected = "rootpackage is a root level package")] + fn new_parent_root_module() { + let root = Module::new("rootpackage".to_string()); + + Module::new_parent(&root); + } + + #[test] + fn is_root_true() { + let root = Module::new("rootpackage".to_string()); + + assert!(root.is_root()); + } + + #[test] + fn is_descendant_of_true_for_child() { + let foo = Module::new("mypackage.foo".to_string()); + let foo_bar = Module::new("mypackage.foo.bar".to_string()); + + assert!(foo_bar.is_descendant_of(&foo)); + } + + #[test] + fn is_descendant_of_false_for_parent() { + let foo = Module::new("mypackage.foo".to_string()); + let foo_bar = Module::new("mypackage.foo.bar".to_string()); + + assert_eq!(foo.is_descendant_of(&foo_bar), false); + } + + #[test] + fn is_descendant_of_true_for_grandchild() { + let foo = Module::new("mypackage.foo".to_string()); + let foo_bar_baz = Module::new("mypackage.foo.bar.baz".to_string()); + + assert!(foo_bar_baz.is_descendant_of(&foo)); + } + + #[test] + fn is_descendant_of_false_for_grandparent() { + let foo = Module::new("mypackage.foo".to_string()); + let foo_bar_baz = Module::new("mypackage.foo.bar.baz".to_string()); + + assert_eq!(foo.is_descendant_of(&foo_bar_baz), false); + } + + #[test] + fn is_root_false() { + let non_root = Module::new("rootpackage.blue".to_string()); + + assert_eq!(non_root.is_root(), false); + } + + #[test] + fn find_children_no_results() { + let mut graph = Graph::default(); + let mypackage = Module::new("mypackage".to_string()); + let mypackage_foo = Module::new("mypackage.foo".to_string()); + + graph.add_module(mypackage.clone()); + graph.add_module(mypackage_foo.clone()); + + assert_eq!(graph.find_children(&mypackage_foo), HashSet::new()); + } + + #[test] + fn find_children_one_result() { + let mut graph = Graph::default(); + let mypackage = Module::new("mypackage".to_string()); + let mypackage_foo = Module::new("mypackage.foo".to_string()); + let mypackage_bar = Module::new("mypackage.bar".to_string()); + + graph.add_module(mypackage.clone()); + graph.add_module(mypackage_foo.clone()); + graph.add_module(mypackage_bar.clone()); + + assert_eq!( + graph.find_children(&mypackage), + HashSet::from([&mypackage_foo, &mypackage_bar]) + ); + } + + #[test] + fn find_children_multiple_results() { + let mut graph = Graph::default(); + let mypackage = Module::new("mypackage".to_string()); + let mypackage_foo = Module::new("mypackage.foo".to_string()); + let mypackage_bar = Module::new("mypackage.bar".to_string()); + + graph.add_module(mypackage.clone()); + graph.add_module(mypackage_foo.clone()); + graph.add_module(mypackage_bar.clone()); + + assert_eq!( + graph.find_children(&mypackage), + HashSet::from([&mypackage_foo, &mypackage_bar]) + ); + } + + #[test] + fn find_children_returns_empty_set_with_nonexistent_module() { + let mut graph = Graph::default(); + // Note: mypackage is not in the graph. + let mypackage_foo = Module::new("mypackage.foo".to_string()); + let mypackage_bar = Module::new("mypackage.bar".to_string()); + + graph.add_module(mypackage_foo.clone()); + graph.add_module(mypackage_bar.clone()); + + assert_eq!( + graph.find_children(&Module::new("mypackage".to_string())), + HashSet::new() + ); + } + + #[test] + fn find_descendants_no_results() { + let mut graph = Graph::default(); + let mypackage = Module::new("mypackage".to_string()); + let mypackage_foo = Module::new("mypackage.foo".to_string()); + let mypackage_bar = Module::new("mypackage.bar".to_string()); + let mypackage_foo_alpha = Module::new("mypackage.foo.alpha".to_string()); + let mypackage_foo_alpha_blue = Module::new("mypackage.foo.alpha.blue".to_string()); + let mypackage_foo_alpha_green = Module::new("mypackage.foo.alpha.green".to_string()); + let mypackage_foo_beta = Module::new("mypackage.foo.beta".to_string()); + + graph.add_module(mypackage.clone()); + graph.add_module(mypackage_foo.clone()); + graph.add_module(mypackage_bar.clone()); + graph.add_module(mypackage_foo_alpha.clone()); + graph.add_module(mypackage_foo_alpha_blue.clone()); + graph.add_module(mypackage_foo_alpha_green.clone()); + graph.add_module(mypackage_foo_beta.clone()); + + assert_eq!(graph.find_descendants(&mypackage_bar), Ok(HashSet::new())); + } + + #[test] + fn find_descendants_module_not_in_graph() { + let mut graph = Graph::default(); + let blue = Module::new("blue".to_string()); + let green = Module::new("green".to_string()); + graph.add_module(blue.clone()); + + assert_eq!( + graph.find_descendants(&green), + Err(ModuleNotPresent { + module: green.clone() + }) + ); + } + + #[test] + fn find_descendants_multiple_results() { + let mut graph = Graph::default(); + let mypackage = Module::new("mypackage".to_string()); + let mypackage_foo = Module::new("mypackage.foo".to_string()); + let mypackage_bar = Module::new("mypackage.bar".to_string()); + let mypackage_foo_alpha = Module::new("mypackage.foo.alpha".to_string()); + let mypackage_foo_alpha_blue = Module::new("mypackage.foo.alpha.blue".to_string()); + let mypackage_foo_alpha_green = Module::new("mypackage.foo.alpha.green".to_string()); + let mypackage_foo_beta = Module::new("mypackage.foo.beta".to_string()); + + graph.add_module(mypackage.clone()); + graph.add_module(mypackage_foo.clone()); + graph.add_module(mypackage_bar.clone()); + graph.add_module(mypackage_foo_alpha.clone()); + graph.add_module(mypackage_foo_alpha_blue.clone()); + graph.add_module(mypackage_foo_alpha_green.clone()); + graph.add_module(mypackage_foo_beta.clone()); + + assert_eq!( + graph.find_descendants(&mypackage_foo), + Ok(HashSet::from([ + &mypackage_foo_alpha, + &mypackage_foo_alpha_blue, + &mypackage_foo_alpha_green, + &mypackage_foo_beta + ])) + ); + } + + #[test] + fn find_descendants_with_gap() { + let mut graph = Graph::default(); + let mypackage = Module::new("mypackage".to_string()); + let mypackage_foo = Module::new("mypackage.foo".to_string()); + // mypackage.foo.blue is not added. + let mypackage_foo_blue_alpha = Module::new("mypackage.foo.blue.alpha".to_string()); + let mypackage_foo_blue_alpha_one = Module::new("mypackage.foo.blue.alpha.one".to_string()); + let mypackage_foo_blue_alpha_two = Module::new("mypackage.foo.blue.alpha.two".to_string()); + let mypackage_foo_blue_beta_three = + Module::new("mypackage.foo.blue.beta.three".to_string()); + let mypackage_bar_green_alpha = Module::new("mypackage.bar.green.alpha".to_string()); + graph.add_module(mypackage.clone()); + graph.add_module(mypackage_foo.clone()); + graph.add_module(mypackage_foo_blue_alpha.clone()); + graph.add_module(mypackage_foo_blue_alpha_one.clone()); + graph.add_module(mypackage_foo_blue_alpha_two.clone()); + graph.add_module(mypackage_foo_blue_beta_three.clone()); + graph.add_module(mypackage_bar_green_alpha.clone()); + + assert_eq!( + graph.find_descendants(&mypackage_foo), + // mypackage.foo.blue is not included. + Ok(HashSet::from([ + &mypackage_foo_blue_alpha, + &mypackage_foo_blue_alpha_one, + &mypackage_foo_blue_alpha_two, + &mypackage_foo_blue_beta_three, + ])) + ); + } + + #[test] + fn find_descendants_added_in_different_order() { + let mut graph = Graph::default(); + let mypackage = Module::new("mypackage".to_string()); + let mypackage_foo = Module::new("mypackage.foo".to_string()); + let mypackage_foo_blue_alpha = Module::new("mypackage.foo.blue.alpha".to_string()); + let mypackage_foo_blue_alpha_one = Module::new("mypackage.foo.blue.alpha.one".to_string()); + let mypackage_foo_blue_alpha_two = Module::new("mypackage.foo.blue.alpha.two".to_string()); + let mypackage_foo_blue_beta_three = + Module::new("mypackage.foo.blue.beta.three".to_string()); + let mypackage_bar_green_alpha = Module::new("mypackage.bar.green.alpha".to_string()); + let mypackage_foo_blue = Module::new("mypackage.foo.blue".to_string()); + graph.add_module(mypackage.clone()); + graph.add_module(mypackage_foo.clone()); + graph.add_module(mypackage_foo_blue_alpha.clone()); + graph.add_module(mypackage_foo_blue_alpha_one.clone()); + graph.add_module(mypackage_foo_blue_alpha_two.clone()); + graph.add_module(mypackage_foo_blue_beta_three.clone()); + graph.add_module(mypackage_bar_green_alpha.clone()); + // Add the middle one at the end. + graph.add_module(mypackage_foo_blue.clone()); + + assert_eq!( + graph.find_descendants(&mypackage_foo), + Ok(HashSet::from([ + &mypackage_foo_blue, // Should be included. + &mypackage_foo_blue_alpha, + &mypackage_foo_blue_alpha_one, + &mypackage_foo_blue_alpha_two, + &mypackage_foo_blue_beta_three, + ])) + ); + } + + #[test] + fn direct_import_exists_returns_true() { + let mut graph = Graph::default(); + let mypackage = Module::new("mypackage".to_string()); + let mypackage_foo = Module::new("mypackage.foo".to_string()); + let mypackage_bar = Module::new("mypackage.bar".to_string()); + graph.add_module(mypackage.clone()); + graph.add_module(mypackage_foo.clone()); + graph.add_module(mypackage_bar.clone()); + graph.add_import(&mypackage_foo, &mypackage_bar); + + assert!(graph.direct_import_exists(&mypackage_foo, &mypackage_bar, false)); + } + + #[test] + fn add_detailed_import_adds_import() { + let mut graph = Graph::default(); + let blue = Module::new("blue".to_string()); + let green = Module::new("green".to_string()); + graph.add_module(blue.clone()); + graph.add_module(green.clone()); + let import = DetailedImport { + importer: blue.clone(), + imported: green.clone(), + line_number: 11, + line_contents: "-".to_string(), + }; + + graph.add_detailed_import(&import); + + assert_eq!(graph.direct_import_exists(&blue, &green, false), true); + } + + #[test] + fn direct_import_exists_returns_false() { + let mut graph = Graph::default(); + let mypackage = Module::new("mypackage".to_string()); + let mypackage_foo = Module::new("mypackage.foo".to_string()); + let mypackage_bar = Module::new("mypackage.bar".to_string()); + graph.add_module(mypackage.clone()); + graph.add_module(mypackage_foo.clone()); + graph.add_module(mypackage_bar.clone()); + graph.add_import(&mypackage_foo, &mypackage_bar); + + assert!(!graph.direct_import_exists(&mypackage_bar, &mypackage_foo, false)); + } + + #[test] + fn direct_import_exists_returns_false_root_to_child() { + let mut graph = Graph::default(); + let mypackage = Module::new("mypackage".to_string()); + let mypackage_foo = Module::new("mypackage.foo".to_string()); + let mypackage_bar = Module::new("mypackage.bar".to_string()); + let mypackage_foo_alpha = Module::new("mypackage.foo.alpha".to_string()); + graph.add_module(mypackage.clone()); + graph.add_module(mypackage_foo.clone()); + graph.add_module(mypackage_bar.clone()); + graph.add_module(mypackage_foo_alpha.clone()); + graph.add_import(&mypackage_bar, &mypackage_foo_alpha); + + assert_eq!( + graph.pretty_str(), + " +hierarchy: + mypackage -> mypackage.bar + mypackage -> mypackage.foo + mypackage.foo -> mypackage.foo.alpha +imports: + mypackage.bar -> mypackage.foo.alpha +" + .trim_start() + ); + assert!(!graph.direct_import_exists(&mypackage_bar, &mypackage_foo, false)); + } + + #[test] + fn add_import_with_non_existent_importer_adds_that_module() { + let mut graph = Graph::default(); + let mypackage_foo = Module::new("mypackage.foo".to_string()); + let mypackage_bar = Module::new("mypackage.bar".to_string()); + graph.add_module(mypackage_bar.clone()); + + graph.add_import(&mypackage_foo, &mypackage_bar); + + assert_eq!( + graph.get_modules(), + HashSet::from([&mypackage_bar, &mypackage_foo]) + ); + assert!(graph.direct_import_exists(&mypackage_foo, &mypackage_bar, false)); + assert_eq!( + graph.pretty_str(), + " +hierarchy: + (mypackage) -> mypackage.bar + (mypackage) -> mypackage.foo +imports: + mypackage.foo -> mypackage.bar +" + .trim_start() + ); + } + + #[test] + fn add_import_with_non_existent_imported_adds_that_module() { + let mut graph = Graph::default(); + let mypackage_foo = Module::new("mypackage.foo".to_string()); + let mypackage_bar = Module::new("mypackage.bar".to_string()); + graph.add_module(mypackage_foo.clone()); + + graph.add_import(&mypackage_foo, &mypackage_bar); + + assert_eq!( + graph.get_modules(), + HashSet::from([&mypackage_bar, &mypackage_foo]) + ); + assert!(graph.direct_import_exists(&mypackage_foo, &mypackage_bar, false)); + assert_eq!( + graph.pretty_str(), + " +hierarchy: + (mypackage) -> mypackage.bar + (mypackage) -> mypackage.foo +imports: + mypackage.foo -> mypackage.bar +" + .trim_start() + ); + } + + #[test] + fn direct_import_exists_with_as_packages_returns_false() { + let mut graph = Graph::default(); + let mypackage = Module::new("mypackage".to_string()); + let mypackage_foo = Module::new("mypackage.foo".to_string()); + let mypackage_bar = Module::new("mypackage.bar".to_string()); + let mypackage_foo_alpha = Module::new("mypackage.foo.alpha".to_string()); + let mypackage_foo_alpha_blue = Module::new("mypackage.foo.alpha.blue".to_string()); + let mypackage_foo_alpha_green = Module::new("mypackage.foo.alpha.green".to_string()); + let mypackage_foo_beta = Module::new("mypackage.foo.beta".to_string()); + graph.add_module(mypackage.clone()); + graph.add_module(mypackage_foo.clone()); + graph.add_module(mypackage_bar.clone()); + graph.add_module(mypackage_foo_alpha.clone()); + graph.add_module(mypackage_foo_alpha_blue.clone()); + graph.add_module(mypackage_foo_alpha_green.clone()); + graph.add_module(mypackage_foo_beta.clone()); + // Add an import in the other direction. + graph.add_import(&mypackage_bar, &mypackage_foo); + + assert!(!graph.direct_import_exists(&mypackage_foo, &mypackage_bar, true)); + } + + #[test] + fn direct_import_exists_with_as_packages_returns_true_between_roots() { + let mut graph = Graph::default(); + let mypackage = Module::new("mypackage".to_string()); + let mypackage_foo = Module::new("mypackage.foo".to_string()); + let mypackage_bar = Module::new("mypackage.bar".to_string()); + let mypackage_foo_alpha = Module::new("mypackage.foo.alpha".to_string()); + let mypackage_foo_alpha_blue = Module::new("mypackage.foo.alpha.blue".to_string()); + let mypackage_foo_alpha_green = Module::new("mypackage.foo.alpha.green".to_string()); + let mypackage_foo_beta = Module::new("mypackage.foo.beta".to_string()); + graph.add_module(mypackage.clone()); + graph.add_module(mypackage_foo.clone()); + graph.add_module(mypackage_bar.clone()); + graph.add_module(mypackage_foo_alpha.clone()); + graph.add_module(mypackage_foo_alpha_blue.clone()); + graph.add_module(mypackage_foo_alpha_green.clone()); + graph.add_module(mypackage_foo_beta.clone()); + graph.add_import(&mypackage_foo, &mypackage_bar); + + assert!(graph.direct_import_exists(&mypackage_foo, &mypackage_bar, true)); + } + + #[test] + fn direct_import_exists_with_as_packages_returns_true_root_to_child() { + let mut graph = Graph::default(); + let mypackage = Module::new("mypackage".to_string()); + let mypackage_foo = Module::new("mypackage.foo".to_string()); + let mypackage_bar = Module::new("mypackage.bar".to_string()); + let mypackage_foo_alpha = Module::new("mypackage.foo.alpha".to_string()); + let mypackage_foo_alpha_blue = Module::new("mypackage.foo.alpha.blue".to_string()); + let mypackage_foo_alpha_green = Module::new("mypackage.foo.alpha.green".to_string()); + let mypackage_foo_beta = Module::new("mypackage.foo.beta".to_string()); + graph.add_module(mypackage.clone()); + graph.add_module(mypackage_foo.clone()); + graph.add_module(mypackage_bar.clone()); + graph.add_module(mypackage_foo_alpha.clone()); + graph.add_module(mypackage_foo_alpha_blue.clone()); + graph.add_module(mypackage_foo_alpha_green.clone()); + graph.add_module(mypackage_foo_beta.clone()); + graph.add_import(&mypackage_bar, &mypackage_foo_alpha); + + assert!(graph.direct_import_exists(&mypackage_bar, &mypackage_foo, true)); + } + + #[test] + fn direct_import_exists_with_as_packages_returns_true_child_to_root() { + let mut graph = Graph::default(); + let mypackage = Module::new("mypackage".to_string()); + let mypackage_foo = Module::new("mypackage.foo".to_string()); + let mypackage_bar = Module::new("mypackage.bar".to_string()); + let mypackage_foo_alpha = Module::new("mypackage.foo.alpha".to_string()); + let mypackage_foo_alpha_blue = Module::new("mypackage.foo.alpha.blue".to_string()); + let mypackage_foo_alpha_green = Module::new("mypackage.foo.alpha.green".to_string()); + let mypackage_foo_beta = Module::new("mypackage.foo.beta".to_string()); + graph.add_module(mypackage.clone()); + graph.add_module(mypackage_foo.clone()); + graph.add_module(mypackage_bar.clone()); + graph.add_module(mypackage_foo_alpha.clone()); + graph.add_module(mypackage_foo_alpha_blue.clone()); + graph.add_module(mypackage_foo_alpha_green.clone()); + graph.add_module(mypackage_foo_beta.clone()); + graph.add_import(&mypackage_foo_alpha, &mypackage_bar); + + assert!(graph.direct_import_exists(&mypackage_foo, &mypackage_bar, true)); + } + + #[test] + #[should_panic] + fn direct_import_exists_within_package_panics() { + let mut graph = Graph::default(); + let ancestor = Module::new("mypackage.foo".to_string()); + let descendant = Module::new("mypackage.foo.blue.alpha".to_string()); + graph.add_import(&ancestor, &descendant); + + graph.direct_import_exists(&ancestor, &descendant, true); + } + + #[test] + fn find_modules_that_directly_import() { + let mut graph = Graph::default(); + let mypackage = Module::new("mypackage".to_string()); + let mypackage_foo = Module::new("mypackage.foo".to_string()); + let mypackage_bar = Module::new("mypackage.bar".to_string()); + let mypackage_foo_alpha = Module::new("mypackage.foo.alpha".to_string()); + let mypackage_foo_alpha_blue = Module::new("mypackage.foo.alpha.blue".to_string()); + let mypackage_foo_alpha_green = Module::new("mypackage.foo.alpha.green".to_string()); + let mypackage_foo_beta = Module::new("mypackage.foo.beta".to_string()); + let anotherpackage = Module::new("anotherpackage".to_string()); + graph.add_module(mypackage.clone()); + graph.add_module(mypackage_foo.clone()); + graph.add_module(mypackage_bar.clone()); + graph.add_module(mypackage_foo_alpha.clone()); + graph.add_module(mypackage_foo_alpha_blue.clone()); + graph.add_module(mypackage_foo_alpha_green.clone()); + graph.add_module(mypackage_foo_beta.clone()); + graph.add_import(&mypackage_foo_alpha, &mypackage_bar); + graph.add_import(&anotherpackage, &mypackage_bar); + graph.add_import(&mypackage_bar, &mypackage_foo_alpha_green); + + let result = graph.find_modules_that_directly_import(&mypackage_bar); + + assert_eq!( + result, + HashSet::from([&mypackage_foo_alpha, &anotherpackage]) + ) + } + + #[test] + fn find_modules_that_directly_import_after_removal() { + let mut graph = Graph::default(); + let blue = Module::new("blue".to_string()); + let green = Module::new("green".to_string()); + let yellow = Module::new("yellow".to_string()); + graph.add_import(&green, &blue); + graph.add_import(&yellow, &blue); + + graph.remove_import(&green, &blue); + let result = graph.find_modules_that_directly_import(&blue); + + assert_eq!(result, HashSet::from([&yellow])) + } + + #[test] + fn find_modules_directly_imported_by() { + let mut graph = Graph::default(); + let mypackage = Module::new("mypackage".to_string()); + let mypackage_foo = Module::new("mypackage.foo".to_string()); + let mypackage_bar = Module::new("mypackage.bar".to_string()); + let mypackage_foo_alpha = Module::new("mypackage.foo.alpha".to_string()); + let mypackage_foo_alpha_blue = Module::new("mypackage.foo.alpha.blue".to_string()); + let mypackage_foo_alpha_green = Module::new("mypackage.foo.alpha.green".to_string()); + let mypackage_foo_beta = Module::new("mypackage.foo.beta".to_string()); + let anotherpackage = Module::new("anotherpackage".to_string()); + graph.add_module(mypackage.clone()); + graph.add_module(mypackage_foo.clone()); + graph.add_module(mypackage_bar.clone()); + graph.add_module(mypackage_foo_alpha.clone()); + graph.add_module(mypackage_foo_alpha_blue.clone()); + graph.add_module(mypackage_foo_alpha_green.clone()); + graph.add_module(mypackage_foo_beta.clone()); + graph.add_import(&mypackage_bar, &mypackage_foo_alpha); + graph.add_import(&mypackage_bar, &anotherpackage); + graph.add_import(&mypackage_foo_alpha_green, &mypackage_bar); + + let result = graph.find_modules_directly_imported_by(&mypackage_bar); + + assert_eq!( + result, + HashSet::from([&mypackage_foo_alpha, &anotherpackage]) + ) + } + + #[test] + fn find_modules_directly_imported_by_after_removal() { + let mut graph = Graph::default(); + let blue = Module::new("blue".to_string()); + let green = Module::new("green".to_string()); + let yellow = Module::new("yellow".to_string()); + graph.add_import(&blue, &green); + graph.add_import(&blue, &yellow); + + graph.remove_import(&blue, &green); + let result = graph.find_modules_directly_imported_by(&blue); + + assert_eq!(result, HashSet::from([&yellow])) + } + + #[test] + fn squash_module_descendants() { + let mut graph = Graph::default(); + // Module we're going to squash. + let mypackage = Module::new("mypackage".to_string()); + let mypackage_blue = Module::new("mypackage.blue".to_string()); + let mypackage_blue_alpha = Module::new("mypackage.blue.alpha".to_string()); + let mypackage_blue_alpha_foo = Module::new("mypackage.blue.alpha.foo".to_string()); + let mypackage_blue_beta = Module::new("mypackage.blue.beta".to_string()); + // Other modules. + let mypackage_green = Module::new("mypackage.green".to_string()); + let mypackage_red = Module::new("mypackage.red".to_string()); + let mypackage_orange = Module::new("mypackage.orange".to_string()); + let mypackage_yellow = Module::new("mypackage.yellow".to_string()); + graph.add_module(mypackage.clone()); + graph.add_module(mypackage_blue.clone()); + // Module's descendants importing other modules. + graph.add_import(&mypackage_blue_alpha, &mypackage_green); + graph.add_import(&mypackage_blue_alpha, &mypackage_red); + graph.add_import(&mypackage_blue_alpha_foo, &mypackage_yellow); + graph.add_import(&mypackage_blue_beta, &mypackage_orange); + // Other modules importing squashed module's descendants. + graph.add_import(&mypackage_red, &mypackage_blue_alpha); + graph.add_import(&mypackage_yellow, &mypackage_blue_alpha); + graph.add_import(&mypackage_orange, &mypackage_blue_alpha_foo); + graph.add_import(&mypackage_green, &mypackage_blue_beta); + // Unrelated imports. + graph.add_import(&mypackage_green, &mypackage_orange); + assert_eq!( + graph.pretty_str(), + " +hierarchy: + mypackage -> mypackage.blue + mypackage -> mypackage.green + mypackage -> mypackage.orange + mypackage -> mypackage.red + mypackage -> mypackage.yellow + mypackage.blue -> mypackage.blue.alpha + mypackage.blue -> mypackage.blue.beta + mypackage.blue.alpha -> mypackage.blue.alpha.foo +imports: + mypackage.blue.alpha -> mypackage.green + mypackage.blue.alpha -> mypackage.red + mypackage.blue.alpha.foo -> mypackage.yellow + mypackage.blue.beta -> mypackage.orange + mypackage.green -> mypackage.blue.beta + mypackage.green -> mypackage.orange + mypackage.orange -> mypackage.blue.alpha.foo + mypackage.red -> mypackage.blue.alpha + mypackage.yellow -> mypackage.blue.alpha +" + .trim_start() + ); + + graph.squash_module(&mypackage_blue); + + assert_eq!( + graph.pretty_str(), + " +hierarchy: + mypackage -> mypackage.blue + mypackage -> mypackage.green + mypackage -> mypackage.orange + mypackage -> mypackage.red + mypackage -> mypackage.yellow +imports: + mypackage.blue -> mypackage.green + mypackage.blue -> mypackage.orange + mypackage.blue -> mypackage.red + mypackage.blue -> mypackage.yellow + mypackage.green -> mypackage.blue + mypackage.green -> mypackage.orange + mypackage.orange -> mypackage.blue + mypackage.red -> mypackage.blue + mypackage.yellow -> mypackage.blue +" + .trim_start() + ); + } + + #[test] + fn squash_module_no_descendants() { + let mut graph = Graph::default(); + let mypackage = Module::new("mypackage".to_string()); + let mypackage_blue = Module::new("mypackage.blue".to_string()); + graph.add_module(mypackage.clone()); + graph.add_module(mypackage_blue.clone()); + + graph.squash_module(&mypackage_blue); + + assert_eq!( + graph.pretty_str(), + " +hierarchy: + mypackage -> mypackage.blue +imports: + +" + .trim_start() + ); + } + + #[test] + fn find_count_imports_empty_graph() { + let graph = Graph::default(); + + let result = graph.count_imports(); + + assert_eq!(result, 0); + } + + #[test] + fn find_count_imports_modules_but_no_imports() { + let mut graph = Graph::default(); + graph.add_module(Module::new("mypackage.foo".to_string())); + graph.add_module(Module::new("mypackage.bar".to_string())); + + let result = graph.count_imports(); + + assert_eq!(result, 0); + } + + #[test] + fn find_count_imports_some_imports() { + let mut graph = Graph::default(); + let mypackage_foo = Module::new("mypackage.foo".to_string()); + let mypackage_bar = Module::new("mypackage.bar".to_string()); + let mypackage_baz = Module::new("mypackage.baz".to_string()); + graph.add_module(mypackage_foo.clone()); + graph.add_module(mypackage_bar.clone()); + graph.add_import(&mypackage_foo, &mypackage_bar); + graph.add_import(&mypackage_foo, &mypackage_baz); + + let result = graph.count_imports(); + + assert_eq!(result, 2); + } + + #[test] + fn find_count_imports_treats_two_imports_between_same_modules_as_one() { + let mut graph = Graph::default(); + let mypackage_foo = Module::new("mypackage.foo".to_string()); + let mypackage_bar = Module::new("mypackage.bar".to_string()); + graph.add_module(mypackage_foo.clone()); + graph.add_module(mypackage_bar.clone()); + graph.add_import(&mypackage_foo, &mypackage_bar); + graph.add_import(&mypackage_foo, &mypackage_bar); + + let result = graph.count_imports(); + + assert_eq!(result, 1); + } + + #[test] + fn is_module_squashed_when_not_squashed() { + let mut graph = Graph::default(); + // Module we're going to squash. + let mypackage_blue = Module::new("mypackage.blue".to_string()); + let mypackage_blue_alpha = Module::new("mypackage.blue.alpha".to_string()); + // Other module. + let mypackage_green = Module::new("mypackage.green".to_string()); + graph.add_module(mypackage_blue.clone()); + graph.add_module(mypackage_blue_alpha.clone()); + graph.add_module(mypackage_green.clone()); + graph.squash_module(&mypackage_blue); + + let result = graph.is_module_squashed(&mypackage_green); + + assert!(!result); + } + + #[test] + fn is_module_squashed_when_squashed() { + let mut graph = Graph::default(); + // Module we're going to squash. + let mypackage_blue = Module::new("mypackage.blue".to_string()); + let mypackage_blue_alpha = Module::new("mypackage.blue.alpha".to_string()); + // Other module. + let mypackage_green = Module::new("mypackage.green".to_string()); + graph.add_module(mypackage_blue.clone()); + graph.add_module(mypackage_blue_alpha.clone()); + graph.add_module(mypackage_green.clone()); + graph.squash_module(&mypackage_blue); + + let result = graph.is_module_squashed(&mypackage_blue); + + assert!(result); + } + + #[test] + fn find_upstream_modules_when_there_are_some() { + let mut graph = Graph::default(); + let mypackage = Module::new("mypackage".to_string()); + let blue = Module::new("mypackage.blue".to_string()); + let green = Module::new("mypackage.green".to_string()); + let red = Module::new("mypackage.red".to_string()); + let yellow = Module::new("mypackage.yellow".to_string()); + let purple = Module::new("mypackage.purple".to_string()); + let orange = Module::new("mypackage.orange".to_string()); + let brown = Module::new("mypackage.brown".to_string()); + graph.add_module(mypackage.clone()); + graph.add_module(blue.clone()); + graph.add_module(green.clone()); + graph.add_module(red.clone()); + graph.add_module(yellow.clone()); + graph.add_module(purple.clone()); + graph.add_module(orange.clone()); + graph.add_module(brown.clone()); + // Add the import chain we care about. + graph.add_import(&blue, &green); + graph.add_import(&blue, &red); + graph.add_import(&green, &yellow); + graph.add_import(&yellow, &purple); + // Add an import to blue. + graph.add_import(&brown, &blue); + + let result = graph.find_upstream_modules(&blue, false); + + assert_eq!(result, HashSet::from([&green, &red, &yellow, &purple])) + } + + #[test] + fn find_upstream_modules_when_module_doesnt_exist() { + let graph = Graph::default(); + let blue = Module::new("mypackage.blue".to_string()); + + let result = graph.find_upstream_modules(&blue, false); + + assert_eq!(result, HashSet::new()) + } + + #[test] + fn find_upstream_modules_as_packages() { + let mut graph = Graph::default(); + let mypackage = Module::new("mypackage".to_string()); + let blue = Module::new("mypackage.blue".to_string()); + let alpha = Module::new("mypackage.blue.alpha".to_string()); + let beta = Module::new("mypackage.blue.beta".to_string()); + let green = Module::new("mypackage.green".to_string()); + let red = Module::new("mypackage.red".to_string()); + let yellow = Module::new("mypackage.yellow".to_string()); + let purple = Module::new("mypackage.purple".to_string()); + let orange = Module::new("mypackage.orange".to_string()); + let brown = Module::new("mypackage.brown".to_string()); + graph.add_module(mypackage.clone()); + graph.add_module(blue.clone()); + graph.add_module(alpha.clone()); + graph.add_module(beta.clone()); + graph.add_module(green.clone()); + graph.add_module(red.clone()); + graph.add_module(yellow.clone()); + graph.add_module(purple.clone()); + graph.add_module(orange.clone()); + graph.add_module(brown.clone()); + // Add the import chains we care about. + graph.add_import(&blue, &green); + graph.add_import(&green, &yellow); + graph.add_import(&alpha, &purple); + graph.add_import(&purple, &brown); + // Despite being technically upstream, beta doesn't appear because it's + // in the same package. + graph.add_import(&purple, &beta); + + let result = graph.find_upstream_modules(&blue, true); + + assert_eq!(result, HashSet::from([&green, &yellow, &purple, &brown])) + } + + #[test] + fn find_downstream_modules_when_there_are_some() { + let mut graph = Graph::default(); + let mypackage = Module::new("mypackage".to_string()); + let blue = Module::new("mypackage.blue".to_string()); + let green = Module::new("mypackage.green".to_string()); + let red = Module::new("mypackage.red".to_string()); + let yellow = Module::new("mypackage.yellow".to_string()); + let purple = Module::new("mypackage.purple".to_string()); + let orange = Module::new("mypackage.orange".to_string()); + let brown = Module::new("mypackage.brown".to_string()); + graph.add_module(mypackage.clone()); + graph.add_module(blue.clone()); + graph.add_module(green.clone()); + graph.add_module(red.clone()); + graph.add_module(yellow.clone()); + graph.add_module(purple.clone()); + graph.add_module(orange.clone()); + graph.add_module(brown.clone()); + // Add the import chain we care about. + graph.add_import(&blue, &green); + graph.add_import(&blue, &red); + graph.add_import(&green, &yellow); + graph.add_import(&yellow, &purple); + // Add an import from purple. + graph.add_import(&purple, &brown); + + let result = graph.find_downstream_modules(&purple, false); + + assert_eq!(result, HashSet::from([&yellow, &green, &blue])) + } + + #[test] + fn find_downstream_modules_when_module_doesnt_exist() { + let graph = Graph::default(); + let blue = Module::new("mypackage.blue".to_string()); + + let result = graph.find_downstream_modules(&blue, false); + + assert_eq!(result, HashSet::new()) + } + + #[test] + fn find_downstream_modules_as_packages() { + let mut graph = Graph::default(); + let mypackage = Module::new("mypackage".to_string()); + let blue = Module::new("mypackage.blue".to_string()); + let alpha = Module::new("mypackage.blue.alpha".to_string()); + let beta = Module::new("mypackage.blue.beta".to_string()); + let green = Module::new("mypackage.green".to_string()); + let red = Module::new("mypackage.red".to_string()); + let yellow = Module::new("mypackage.yellow".to_string()); + let purple = Module::new("mypackage.purple".to_string()); + let orange = Module::new("mypackage.orange".to_string()); + let brown = Module::new("mypackage.brown".to_string()); + graph.add_module(mypackage.clone()); + graph.add_module(blue.clone()); + graph.add_module(alpha.clone()); + graph.add_module(beta.clone()); + graph.add_module(green.clone()); + graph.add_module(red.clone()); + graph.add_module(yellow.clone()); + graph.add_module(purple.clone()); + graph.add_module(orange.clone()); + graph.add_module(brown.clone()); + // Add the import chains we care about. + graph.add_import(&yellow, &green); + graph.add_import(&green, &blue); + graph.add_import(&brown, &purple); + graph.add_import(&purple, &alpha); + // Despite being technically downstream, beta doesn't appear because it's + // in the same package. + graph.add_import(&beta, &yellow); + + let result = graph.find_downstream_modules(&blue, true); + + assert_eq!(result, HashSet::from([&green, &yellow, &purple, &brown])) + } + + // find_shortest_chain + #[test] + fn find_shortest_chain_none() { + let mut graph = Graph::default(); + let mypackage = Module::new("mypackage".to_string()); + let blue = Module::new("mypackage.blue".to_string()); + let green = Module::new("mypackage.green".to_string()); + let purple = Module::new("mypackage.purple".to_string()); + graph.add_module(mypackage.clone()); + graph.add_module(blue.clone()); + graph.add_module(green.clone()); + graph.add_module(purple.clone()); + // Add imports that are irrelevant. + graph.add_import(&purple, &blue); + graph.add_import(&green, &purple); + + let result = graph.find_shortest_chain(&blue, &green); + + assert!(result.is_none()) + } + + #[test] + fn find_shortest_chain_one_step() { + let mut graph = Graph::default(); + let mypackage = Module::new("mypackage".to_string()); + let blue = Module::new("mypackage.blue".to_string()); + let green = Module::new("mypackage.green".to_string()); + let red = Module::new("mypackage.red".to_string()); + let purple = Module::new("mypackage.purple".to_string()); + graph.add_module(mypackage.clone()); + graph.add_module(blue.clone()); + graph.add_module(green.clone()); + graph.add_module(red.clone()); + graph.add_module(purple.clone()); + // Add the one-step chain. + graph.add_import(&blue, &green); + // Add a longer chain. + graph.add_import(&blue, &red); + graph.add_import(&red, &green); + // Add other imports that are irrelevant. + graph.add_import(&purple, &blue); + graph.add_import(&green, &purple); + + let result = graph.find_shortest_chain(&blue, &green).unwrap(); + + assert_eq!(result, vec![&blue, &green]) + } + + #[test] + fn find_shortest_chain_one_step_reverse() { + let mut graph = Graph::default(); + let blue = Module::new("mypackage.blue".to_string()); + let green = Module::new("mypackage.green".to_string()); + graph.add_module(blue.clone()); + graph.add_module(green.clone()); + // Add the one-step chain. + graph.add_import(&blue, &green); + + let result = graph.find_shortest_chain(&green, &blue); + + assert_eq!(result.is_none(), true); + } + + #[test] + fn find_shortest_chain_two_steps() { + let mut graph = Graph::default(); + let mypackage = Module::new("mypackage".to_string()); + let blue = Module::new("mypackage.blue".to_string()); + let green = Module::new("mypackage.green".to_string()); + let red = Module::new("mypackage.red".to_string()); + let orange = Module::new("mypackage.orange".to_string()); + let purple = Module::new("mypackage.purple".to_string()); + graph.add_module(mypackage.clone()); + graph.add_module(blue.clone()); + graph.add_module(green.clone()); + graph.add_module(red.clone()); + graph.add_module(orange.clone()); + graph.add_module(purple.clone()); + // Add the two-step chain. + graph.add_import(&blue, &red); + graph.add_import(&red, &green); + // Add a longer chain. + graph.add_import(&blue, &red); + graph.add_import(&red, &orange); + graph.add_import(&orange, &green); + // Add other imports that are irrelevant. + graph.add_import(&purple, &blue); + graph.add_import(&green, &purple); + + let result = graph.find_shortest_chain(&blue, &green).unwrap(); + + assert_eq!(result, vec![&blue, &red, &green]) + } + + #[test] + fn find_shortest_chain_three_steps() { + let mut graph = Graph::default(); + let mypackage = Module::new("mypackage".to_string()); + let blue = Module::new("mypackage.blue".to_string()); + let green = Module::new("mypackage.green".to_string()); + let red = Module::new("mypackage.red".to_string()); + let orange = Module::new("mypackage.orange".to_string()); + let yellow = Module::new("mypackage.yellow".to_string()); + let purple = Module::new("mypackage.purple".to_string()); + graph.add_module(mypackage.clone()); + graph.add_module(blue.clone()); + graph.add_module(green.clone()); + graph.add_module(red.clone()); + graph.add_module(orange.clone()); + graph.add_module(yellow.clone()); + graph.add_module(purple.clone()); + // Add the three-step chain. + graph.add_import(&blue, &red); + graph.add_import(&red, &orange); + graph.add_import(&orange, &green); + // Add a longer chain. + graph.add_import(&blue, &red); + graph.add_import(&red, &orange); + graph.add_import(&orange, &yellow); + graph.add_import(&yellow, &green); + // Add other imports that are irrelevant. + graph.add_import(&purple, &blue); + graph.add_import(&green, &purple); + + let result = graph.find_shortest_chain(&blue, &green).unwrap(); + + assert_eq!(result, vec![&blue, &red, &orange, &green]) + } + + // find_shortest_chains + + #[test] + fn find_shortest_chains_none() { + let mut graph = Graph::default(); + let mypackage = Module::new("mypackage".to_string()); + let blue = Module::new("mypackage.blue".to_string()); + let green = Module::new("mypackage.green".to_string()); + let purple = Module::new("mypackage.purple".to_string()); + graph.add_module(mypackage.clone()); + graph.add_module(blue.clone()); + graph.add_module(green.clone()); + graph.add_module(purple.clone()); + // Add imports that are irrelevant. + graph.add_import(&purple, &blue); + graph.add_import(&green, &purple); + + let result = graph.find_shortest_chains(&blue, &green, true); + + assert_eq!(result, Ok(HashSet::new())); + } + + #[test] + fn find_shortest_chains_between_passed_modules() { + let mut graph = Graph::default(); + let mypackage = Module::new("mypackage".to_string()); + let blue = Module::new("mypackage.blue".to_string()); + let green = Module::new("mypackage.green".to_string()); + let red = Module::new("mypackage.red".to_string()); + let purple = Module::new("mypackage.purple".to_string()); + graph.add_module(mypackage.clone()); + graph.add_module(blue.clone()); + graph.add_module(green.clone()); + graph.add_module(red.clone()); + graph.add_module(purple.clone()); + // Add a chain. + graph.add_import(&blue, &red); + graph.add_import(&red, &green); + // Add other imports that are irrelevant. + graph.add_import(&purple, &blue); + graph.add_import(&green, &purple); + + let result = graph.find_shortest_chains(&blue, &green, true); + + assert_eq!(result, Ok(HashSet::from([vec![blue, red, green],]))); + } + + #[test] + fn find_shortest_chains_between_passed_module_and_child() { + let mut graph = Graph::default(); + let mypackage = Module::new("mypackage".to_string()); + let blue = Module::new("mypackage.blue".to_string()); + let green = Module::new("mypackage.green".to_string()); + let green_alpha = Module::new("mypackage.green.alpha".to_string()); + let red = Module::new("mypackage.red".to_string()); + let purple = Module::new("mypackage.purple".to_string()); + graph.add_module(mypackage.clone()); + graph.add_module(blue.clone()); + graph.add_module(green.clone()); + graph.add_module(green_alpha.clone()); + graph.add_module(red.clone()); + graph.add_module(purple.clone()); + // Add a chain. + graph.add_import(&blue, &red); + graph.add_import(&red, &green_alpha); + // Add other imports that are irrelevant. + graph.add_import(&purple, &blue); + graph.add_import(&green, &purple); + + let result = graph.find_shortest_chains(&blue, &green, true); + + assert_eq!(result, Ok(HashSet::from([vec![blue, red, green_alpha]]))); + } + + #[test] + fn find_shortest_chains_between_passed_module_and_grandchild() { + let mut graph = Graph::default(); + let mypackage = Module::new("mypackage".to_string()); + let blue = Module::new("mypackage.blue".to_string()); + let green = Module::new("mypackage.green".to_string()); + let green_alpha = Module::new("mypackage.green.alpha".to_string()); + let green_alpha_one = Module::new("mypackage.green.alpha.one".to_string()); + let red = Module::new("mypackage.red".to_string()); + let purple = Module::new("mypackage.purple".to_string()); + graph.add_module(mypackage.clone()); + graph.add_module(blue.clone()); + graph.add_module(green.clone()); + graph.add_module(green_alpha.clone()); + graph.add_module(green_alpha_one.clone()); + graph.add_module(red.clone()); + graph.add_module(purple.clone()); + // Add a chain. + graph.add_import(&blue, &red); + graph.add_import(&red, &green_alpha_one); + // Add other imports that are irrelevant. + graph.add_import(&purple, &blue); + graph.add_import(&green, &purple); + + let result = graph.find_shortest_chains(&blue, &green, true); + + assert_eq!( + result, + Ok(HashSet::from([vec![blue, red, green_alpha_one],])) + ) + } + + #[test] + fn find_shortest_chains_between_child_and_passed_module() { + let mut graph = Graph::default(); + let mypackage = Module::new("mypackage".to_string()); + let blue = Module::new("mypackage.blue".to_string()); + let blue_alpha = Module::new("mypackage.blue.alpha".to_string()); + let green = Module::new("mypackage.green".to_string()); + let red = Module::new("mypackage.red".to_string()); + let purple = Module::new("mypackage.purple".to_string()); + graph.add_module(mypackage.clone()); + graph.add_module(blue.clone()); + graph.add_module(blue_alpha.clone()); + graph.add_module(green.clone()); + graph.add_module(red.clone()); + graph.add_module(purple.clone()); + // Add a chain. + graph.add_import(&blue_alpha, &red); + graph.add_import(&red, &green); + // Add other imports that are irrelevant. + graph.add_import(&purple, &blue); + graph.add_import(&green, &purple); + + let result = graph.find_shortest_chains(&blue, &green, true); + + assert_eq!(result, Ok(HashSet::from([vec![blue_alpha, red, green],]))); + } + + #[test] + fn find_shortest_chains_between_grandchild_and_passed_module() { + let mut graph = Graph::default(); + let mypackage = Module::new("mypackage".to_string()); + let blue = Module::new("mypackage.blue".to_string()); + let blue_alpha = Module::new("mypackage.blue.alpha".to_string()); + let blue_alpha_one = Module::new("mypackage.blue.alpha.one".to_string()); + let green = Module::new("mypackage.green".to_string()); + let red = Module::new("mypackage.red".to_string()); + let purple = Module::new("mypackage.purple".to_string()); + graph.add_module(mypackage.clone()); + graph.add_module(blue.clone()); + graph.add_module(blue_alpha.clone()); + graph.add_module(blue_alpha_one.clone()); + graph.add_module(green.clone()); + graph.add_module(red.clone()); + graph.add_module(purple.clone()); + // Add a chain. + graph.add_import(&blue_alpha_one, &red); + graph.add_import(&red, &green); + // Add other imports that are irrelevant. + graph.add_import(&purple, &blue); + graph.add_import(&green, &purple); + + let result = graph.find_shortest_chains(&blue, &green, true); + + assert_eq!( + result, + Ok(HashSet::from([vec![blue_alpha_one, red, green],])) + ) + } + + #[test] + fn chain_exists_true_as_packages_false() { + let mut graph = Graph::default(); + let mypackage = Module::new("mypackage".to_string()); + let blue = Module::new("mypackage.blue".to_string()); + let blue_alpha = Module::new("mypackage.blue.alpha".to_string()); + let blue_alpha_one = Module::new("mypackage.blue.alpha.one".to_string()); + let green = Module::new("mypackage.green".to_string()); + let red = Module::new("mypackage.red".to_string()); + let purple = Module::new("mypackage.purple".to_string()); + graph.add_module(mypackage.clone()); + graph.add_module(blue.clone()); + graph.add_module(blue_alpha.clone()); + graph.add_module(blue_alpha_one.clone()); + graph.add_module(green.clone()); + graph.add_module(red.clone()); + graph.add_module(purple.clone()); + // Add a chain. + graph.add_import(&blue_alpha_one, &red); + graph.add_import(&red, &green); + // Add other imports that are irrelevant. + graph.add_import(&purple, &blue); + graph.add_import(&green, &purple); + + let result = graph.chain_exists(&blue_alpha_one, &green, false); + + assert!(result); + } + + #[test] + fn chain_exists_false_as_packages_false() { + let mut graph = Graph::default(); + let mypackage = Module::new("mypackage".to_string()); + let blue = Module::new("mypackage.blue".to_string()); + let blue_alpha = Module::new("mypackage.blue.alpha".to_string()); + let blue_alpha_one = Module::new("mypackage.blue.alpha.one".to_string()); + let green = Module::new("mypackage.green".to_string()); + let red = Module::new("mypackage.red".to_string()); + let purple = Module::new("mypackage.purple".to_string()); + graph.add_module(mypackage.clone()); + graph.add_module(blue.clone()); + graph.add_module(blue_alpha.clone()); + graph.add_module(blue_alpha_one.clone()); + graph.add_module(green.clone()); + graph.add_module(red.clone()); + graph.add_module(purple.clone()); + // Add a chain. + graph.add_import(&blue_alpha_one, &red); + graph.add_import(&red, &green); + + let result = graph.chain_exists(&blue, &green, false); + + assert_eq!(result, false); + } + + #[test] + fn chain_exists_true_as_packages_true() { + let mut graph = Graph::default(); + let mypackage = Module::new("mypackage".to_string()); + let blue = Module::new("mypackage.blue".to_string()); + let blue_alpha = Module::new("mypackage.blue.alpha".to_string()); + let blue_alpha_one = Module::new("mypackage.blue.alpha.one".to_string()); + let green = Module::new("mypackage.green".to_string()); + let red = Module::new("mypackage.red".to_string()); + let purple = Module::new("mypackage.purple".to_string()); + graph.add_module(mypackage.clone()); + graph.add_module(blue.clone()); + graph.add_module(blue_alpha.clone()); + graph.add_module(blue_alpha_one.clone()); + graph.add_module(green.clone()); + graph.add_module(red.clone()); + graph.add_module(purple.clone()); + // Add a chain. + graph.add_import(&blue_alpha_one, &red); + graph.add_import(&red, &green); + + let result = graph.chain_exists(&blue, &green, true); + + assert_eq!(result, true); + } + + #[test] + fn chain_exists_false_as_packages_true() { + let mut graph = Graph::default(); + let mypackage = Module::new("mypackage".to_string()); + let blue = Module::new("mypackage.blue".to_string()); + let blue_alpha = Module::new("mypackage.blue.alpha".to_string()); + let blue_alpha_one = Module::new("mypackage.blue.alpha.one".to_string()); + let green = Module::new("mypackage.green".to_string()); + let red = Module::new("mypackage.red".to_string()); + let purple = Module::new("mypackage.purple".to_string()); + graph.add_module(mypackage.clone()); + graph.add_module(blue.clone()); + graph.add_module(blue_alpha.clone()); + graph.add_module(blue_alpha_one.clone()); + graph.add_module(green.clone()); + graph.add_module(red.clone()); + graph.add_module(purple.clone()); + // Add a chain. + graph.add_import(&blue_alpha_one, &red); + graph.add_import(&red, &green); + + let result = graph.chain_exists(&green, &blue, true); + + assert_eq!(result, false); + } + + #[test] + fn find_illegal_dependencies_for_layers_empty_everything() { + let graph = Graph::default(); + + let dependencies = graph.find_illegal_dependencies_for_layers(vec![], HashSet::new()); + + assert_eq!(dependencies, Ok(vec![])); + } + + #[test] + fn find_illegal_dependencies_for_layers_no_such_container() { + let graph = Graph::default(); + let container = "nonexistent_container".to_string(); + + let dependencies = + graph.find_illegal_dependencies_for_layers(vec![], HashSet::from([container.clone()])); + + assert_eq!( + dependencies, + Err(NoSuchContainer { + container: container + }) + ); + } + + #[test] + fn find_illegal_dependencies_for_layers_nonexistent_layers_no_container() { + let graph = Graph::default(); + let level = Level { + layers: vec!["nonexistent".to_string()], + independent: true, + }; + + let dependencies = graph.find_illegal_dependencies_for_layers(vec![level], HashSet::new()); + + assert_eq!(dependencies, Ok(vec![])); + } + + #[test] + fn find_illegal_dependencies_for_layers_nonexistent_layers_with_container() { + let mut graph = Graph::default(); + graph.add_module(Module::new("mypackage".to_string())); + let level = Level { + layers: vec!["nonexistent".to_string()], + independent: true, + }; + let container = "mypackage".to_string(); + + let dependencies = + graph.find_illegal_dependencies_for_layers(vec![level], HashSet::from([container])); + + assert_eq!(dependencies, Ok(vec![])); + } + + #[test] + fn find_illegal_dependencies_for_layers_no_container_direct_dependency() { + let mut graph = Graph::default(); + let high = Module::new("high".to_string()); + let low = Module::new("low".to_string()); + graph.add_import(&low, &high); + let levels = vec![ + Level { + layers: vec![high.name.clone()], + independent: true, + }, + Level { + layers: vec![low.name.clone()], + independent: true, + }, + ]; + + let dependencies = graph.find_illegal_dependencies_for_layers(levels, HashSet::new()); + + assert_eq!( + dependencies, + Ok(vec![PackageDependency { + importer: low.clone(), + imported: high.clone(), + routes: vec![Route { + heads: vec![low.clone()], + middle: vec![], + tails: vec![high.clone()], + }] + }]) + ); + } + + #[test] + fn find_illegal_dependencies_for_layers_no_container_indirect_dependency() { + let mut graph = Graph::default(); + let high = Module::new("high".to_string()); + let elsewhere = Module::new("elsewhere".to_string()); + let low = Module::new("low".to_string()); + graph.add_import(&low, &elsewhere); + graph.add_import(&elsewhere, &high); + let levels = vec![ + Level { + layers: vec![high.name.clone()], + independent: true, + }, + Level { + layers: vec![low.name.clone()], + independent: true, + }, + ]; + + let dependencies = graph.find_illegal_dependencies_for_layers(levels, HashSet::new()); + + assert_eq!( + dependencies, + Ok(vec![PackageDependency { + importer: low.clone(), + imported: high.clone(), + routes: vec![Route { + heads: vec![low.clone()], + middle: vec![elsewhere.clone()], + tails: vec![high.clone()], + }] + }]) + ); + } + + #[test] + fn find_illegal_dependencies_for_layers_containers() { + let mut graph = Graph::default(); + let blue_high = Module::new("blue.high".to_string()); + let blue_high_alpha = Module::new("blue.high.alpha".to_string()); + let blue_low = Module::new("blue.low".to_string()); + let blue_low_beta = Module::new("blue.low.beta".to_string()); + let green_high = Module::new("green.high".to_string()); + let green_high_gamma = Module::new("green.high.gamma".to_string()); + let green_low = Module::new("green.low".to_string()); + let green_low_delta = Module::new("green.low.delta".to_string()); + graph.add_module(Module::new("blue".to_string())); + graph.add_module(blue_high.clone()); + graph.add_module(blue_low.clone()); + graph.add_module(Module::new("green".to_string())); + graph.add_module(green_high.clone()); + graph.add_module(green_low.clone()); + graph.add_import(&blue_low_beta, &blue_high_alpha); + graph.add_import(&green_low_delta, &green_high_gamma); + + let levels = vec![ + Level { + layers: vec!["high".to_string()], + independent: true, + }, + Level { + layers: vec!["low".to_string()], + independent: true, + }, + ]; + let containers = HashSet::from(["blue".to_string(), "green".to_string()]); + + let dependencies = graph.find_illegal_dependencies_for_layers(levels, containers); + + assert_eq!( + dependencies, + Ok(vec![ + PackageDependency { + importer: blue_low.clone(), + imported: blue_high.clone(), + routes: vec![Route { + heads: vec![blue_low_beta.clone()], + middle: vec![], + tails: vec![blue_high_alpha.clone()], + }] + }, + PackageDependency { + importer: green_low.clone(), + imported: green_high.clone(), + routes: vec![Route { + heads: vec![green_low_delta.clone()], + middle: vec![], + tails: vec![green_high_gamma.clone()], + }] + } + ]) + ); + } + + #[test] + fn find_illegal_dependencies_for_layers_independent() { + let mut graph = Graph::default(); + let blue = Module::new("blue".to_string()); + let green = Module::new("green".to_string()); + let blue_alpha = Module::new("blue.alpha".to_string()); + let green_beta = Module::new("green.beta".to_string()); + graph.add_module(blue.clone()); + graph.add_module(green.clone()); + graph.add_import(&blue_alpha, &green_beta); + + let levels = vec![Level { + layers: vec![blue.name.clone(), green.name.clone()], + independent: true, + }]; + + let dependencies = graph.find_illegal_dependencies_for_layers(levels, HashSet::new()); + + assert_eq!( + dependencies, + Ok(vec![PackageDependency { + importer: blue.clone(), + imported: green.clone(), + routes: vec![Route { + heads: vec![blue_alpha.clone()], + middle: vec![], + tails: vec![green_beta.clone()], + }] + }]) + ); + } + + #[test] + fn get_import_details_no_modules() { + let graph = Graph::default(); + let importer = Module::new("foo".to_string()); + let imported = Module::new("bar".to_string()); + + let result = graph.get_import_details(&importer, &imported); + + assert_eq!(result, HashSet::new()); + } + + #[test] + fn get_import_details_module_without_metadata() { + let mut graph = Graph::default(); + let importer = Module::new("foo".to_string()); + let imported = Module::new("bar".to_string()); + graph.add_import(&importer, &imported); + + let result = graph.get_import_details(&importer, &imported); + + assert_eq!(result, HashSet::new()); + } + + #[test] + fn get_import_details_module_one_result() { + let mut graph = Graph::default(); + let importer = Module::new("foo".to_string()); + let imported = Module::new("bar".to_string()); + let import = DetailedImport { + importer: importer.clone(), + imported: imported.clone(), + line_number: 5, + line_contents: "import bar".to_string(), + }; + let unrelated_import = DetailedImport { + importer: importer.clone(), + imported: Module::new("baz".to_string()), + line_number: 2, + line_contents: "-".to_string(), + }; + graph.add_detailed_import(&import); + graph.add_detailed_import(&unrelated_import); + + let result = graph.get_import_details(&importer, &imported); + + assert_eq!(result, HashSet::from([import])); + } + + #[test] + fn get_import_details_module_two_results() { + let mut graph = Graph::default(); + let blue = Module::new("blue".to_string()); + let green = Module::new("green".to_string()); + let blue_to_green_a = DetailedImport { + importer: blue.clone(), + imported: green.clone(), + line_number: 5, + line_contents: "import green".to_string(), + }; + let blue_to_green_b = DetailedImport { + importer: blue.clone(), + imported: green.clone(), + line_number: 15, + line_contents: "import green".to_string(), + }; + graph.add_detailed_import(&blue_to_green_a); + graph.add_detailed_import(&blue_to_green_b); + + let result = graph.get_import_details(&blue, &green); + + assert_eq!(result, HashSet::from([blue_to_green_a, blue_to_green_b])); + } + + #[test] + fn get_import_details_after_removal() { + let mut graph = Graph::default(); + let importer = Module::new("foo".to_string()); + let imported = Module::new("bar".to_string()); + let import = DetailedImport { + importer: importer.clone(), + imported: imported.clone(), + line_number: 5, + line_contents: "import bar".to_string(), + }; + let unrelated_import = DetailedImport { + importer: importer.clone(), + imported: Module::new("baz".to_string()), + line_number: 2, + line_contents: "-".to_string(), + }; + graph.add_detailed_import(&import); + graph.add_detailed_import(&unrelated_import); + graph.remove_import(&import.importer, &import.imported); + + let result = graph.get_import_details(&importer, &imported); + + assert_eq!(result, HashSet::new()); + } + + #[test] + fn get_import_details_after_removal_of_unrelated_import() { + let mut graph = Graph::default(); + let importer = Module::new("foo".to_string()); + let imported = Module::new("bar".to_string()); + let import = DetailedImport { + importer: importer.clone(), + imported: imported.clone(), + line_number: 5, + line_contents: "import bar".to_string(), + }; + let unrelated_import = DetailedImport { + importer: importer.clone(), + imported: Module::new("baz".to_string()), + line_number: 2, + line_contents: "-".to_string(), + }; + graph.add_detailed_import(&import); + graph.add_detailed_import(&unrelated_import); + graph.remove_import(&unrelated_import.importer, &unrelated_import.imported); + + let result = graph.get_import_details(&importer, &imported); + + assert_eq!(result, HashSet::from([import])); + } +} diff --git a/rust/src/lib.rs b/rust/src/lib.rs index 73e7038b..cbc1a5ff 100644 --- a/rust/src/lib.rs +++ b/rust/src/lib.rs @@ -13,6 +13,7 @@ use pyo3::create_exception; use pyo3::prelude::*; use pyo3::types::{PyDict, PyFrozenSet, PySet, PyString, PyTuple}; use std::collections::{HashMap, HashSet}; +pub mod graph; #[pymodule] fn _rustgrimp(py: Python<'_>, m: &Bound<'_, PyModule>) -> PyResult<()> { From bbc60f2b0dffa5e30d4616ea0cdaa0a9c244f272 Mon Sep 17 00:00:00 2001 From: David Seddon Date: Fri, 10 Jan 2025 16:10:03 +0000 Subject: [PATCH 02/10] Make integration test use new API --- rust/tests/large.rs | 23 ++++++++++++++--------- 1 file changed, 14 insertions(+), 9 deletions(-) diff --git a/rust/tests/large.rs b/rust/tests/large.rs index 0fff1d65..1cdaf9ab 100644 --- a/rust/tests/large.rs +++ b/rust/tests/large.rs @@ -1,7 +1,6 @@ -use _rustgrimp::importgraph::ImportGraph; -use _rustgrimp::layers::{find_illegal_dependencies, Level}; +use _rustgrimp::graph::{Graph, Level, Module}; use serde_json::{Map, Value}; -use std::collections::{HashMap, HashSet}; +use std::collections::HashSet; use std::fs; #[test] @@ -9,15 +8,19 @@ fn test_large_graph_deep_layers() { let data = fs::read_to_string("tests/large_graph.json").expect("Unable to read file"); let value: Value = serde_json::from_str(&data).unwrap(); let items: &Map = value.as_object().unwrap(); - let mut importeds_by_importer: HashMap<&str, HashSet<&str>> = HashMap::new(); + let mut graph = Graph::default(); for (importer, importeds_value) in items.iter() { - let mut importeds = HashSet::new(); for imported in importeds_value.as_array().unwrap() { - importeds.insert(imported.as_str().unwrap()); + graph.add_import( + &Module { + name: importer.clone(), + }, + &Module { + name: imported.as_str().unwrap().to_string(), + }, + ); } - importeds_by_importer.insert(importer, importeds); } - let graph = ImportGraph::new(importeds_by_importer); let deep_layers = vec![ "mypackage.plugins.5634303718.1007553798.8198145119.application.3242334296.1991886645", @@ -39,7 +42,9 @@ fn test_large_graph_deep_layers() { .collect(); let containers = HashSet::new(); - let deps = find_illegal_dependencies(&graph, &levels, &containers); + let deps = graph + .find_illegal_dependencies_for_layers(levels, containers) + .unwrap(); assert_eq!(deps.len(), 8); } From 933df7e847a3165eb7e24d9c1db2d2075d59ac63 Mon Sep 17 00:00:00 2001 From: David Seddon Date: Fri, 10 Jan 2025 13:11:49 +0000 Subject: [PATCH 03/10] Use a Rust-backed Graph This adds a wrapper around the Rust-graph in lib.rs which handles the interface between Python and Rust. At the same time, it changes the Python ImportGraph so it uses Rust instead. --- rust/Cargo.lock | 103 ++++---- rust/Cargo.toml | 4 +- rust/src/graph.rs | 25 +- rust/src/lib.rs | 399 +++++++++++++++++++++++----- src/grimp/adaptors/graph.py | 501 ++++++++---------------------------- 5 files changed, 515 insertions(+), 517 deletions(-) diff --git a/rust/Cargo.lock b/rust/Cargo.lock index c9b0a5fe..37709188 100644 --- a/rust/Cargo.lock +++ b/rust/Cargo.lock @@ -23,9 +23,9 @@ checksum = "69f7f8c3906b62b754cd5326047894316021dcfe5a194c8ea52bdd94934a3457" [[package]] name = "autocfg" -version = "1.3.0" +version = "1.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0c4b4d0bd25bd0b74681c0ad21497610ce1b7c91b1022cd21c80c6fbdd9476b0" +checksum = "ace50bade8e6234aa140d9a2f552bbee1db4d353f69b8217bc503490fc1a9f26" [[package]] name = "bimap" @@ -41,9 +41,9 @@ checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" [[package]] name = "crossbeam-deque" -version = "0.8.5" +version = "0.8.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "613f8cc01fe9cf1a3eb3d7f488fd2fa8388403e97039e2f73692932e291a770d" +checksum = "9dd111b7b7f7d55b72c0a6ae361660ee5853c9af73f70c3c2ef6858b950e2e51" dependencies = [ "crossbeam-epoch", "crossbeam-utils", @@ -60,9 +60,9 @@ dependencies = [ [[package]] name = "crossbeam-utils" -version = "0.8.20" +version = "0.8.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "22ec99545bb0ed0ea7bb9b8e1e9122ea386ff8a48c0922e43f36d45ab09e0e80" +checksum = "d0a5c400df2834b80a4c3327b3aad3a4c4cd4de0629063962b03235697506a28" [[package]] name = "either" @@ -84,9 +84,9 @@ checksum = "0ce7134b9999ecaf8bcd65542e436736ef32ddca1b3e06094cb6ec5755203b80" [[package]] name = "hashbrown" -version = "0.15.0" +version = "0.15.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e087f84d4f86bf4b218b927129862374b72199ae7d8657835f1e89000eea4fb" +checksum = "bf151400ff0baff5465007dd2f3e717f3fe502074ca563069ce3a6629d07b289" [[package]] name = "heck" @@ -96,9 +96,9 @@ checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" [[package]] name = "indexmap" -version = "2.6.0" +version = "2.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "707907fe3c25f5424cce2cb7e1cbcafee6bdbe735ca90ef77c29e84591e5b9da" +checksum = "62f822373a4fe84d4bb149bf54e584a7f4abec90e072ed49cda0edea5b95471f" dependencies = [ "equivalent", "hashbrown", @@ -112,21 +112,27 @@ checksum = "b248f5224d1d606005e02c97f5aa4e88eeb230488bcc03bc9ca4d7991399f2b5" [[package]] name = "itoa" -version = "1.0.11" +version = "1.0.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "49f1f14873335454500d59611f1cf4a4b0f786f9ac11f4312a78e4cf2566695b" +checksum = "d75a2a4b1b190afb6f5425f10f6a8f959d2ea0b9c2b1d79553551850539e4674" [[package]] name = "libc" -version = "0.2.155" +version = "0.2.169" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "97b3888a4aecf77e811145cadf6eef5901f4782c53886191b2f693f24761847c" +checksum = "b5aba8db14291edd000dfcc4d620c7ebfb122c613afb886ca8803fa4e128a20a" [[package]] name = "log" -version = "0.4.22" +version = "0.4.25" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a7a70ba024b9dc04c27ea2f0c0548feb474ec5c54bba33a7f72f873a39d07b24" +checksum = "04cbf5b083de1c7e0222a7a51dbfdba1cbe1c6ab0b15e29fff3f6c077fd9cd9f" + +[[package]] +name = "memchr" +version = "2.7.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78ca9ab1a0babb1e7d5695e3530886289c18cf2f87ec19a575a0abdce112e3a3" [[package]] name = "memoffset" @@ -139,9 +145,9 @@ dependencies = [ [[package]] name = "once_cell" -version = "1.19.0" +version = "1.20.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3fdb12b2476b595f9358c5161aa467c2438859caa136dec86c26fdd2efe17b92" +checksum = "1261fe7e33c73b354eab43b1273a57c8f967d0391e80353e51f764ac02cf6775" [[package]] name = "petgraph" @@ -155,24 +161,24 @@ dependencies = [ [[package]] name = "portable-atomic" -version = "1.6.0" +version = "1.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7170ef9988bc169ba16dd36a7fa041e5c4cbeb6a35b76d4c03daded371eae7c0" +checksum = "280dc24453071f1b63954171985a0b0d30058d287960968b9b2aca264c8d4ee6" [[package]] name = "proc-macro2" -version = "1.0.86" +version = "1.0.93" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5e719e8df665df0d1c8fbfd238015744736151d4445ec0836b8e628aae103b77" +checksum = "60946a68e5f9d28b0dc1c21bb8a97ee7d018a8b322fa57838ba31cc878e22d99" dependencies = [ "unicode-ident", ] [[package]] name = "pyo3" -version = "0.22.4" +version = "0.23.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "00e89ce2565d6044ca31a3eb79a334c3a79a841120a98f64eea9f579564cb691" +checksum = "57fe09249128b3173d092de9523eaa75136bf7ba85e0d69eca241c7939c933cc" dependencies = [ "cfg-if", "indoc", @@ -188,9 +194,9 @@ dependencies = [ [[package]] name = "pyo3-build-config" -version = "0.22.4" +version = "0.23.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d8afbaf3abd7325e08f35ffb8deb5892046fcb2608b703db6a583a5ba4cea01e" +checksum = "1cd3927b5a78757a0d71aa9dff669f903b1eb64b54142a9bd9f757f8fde65fd7" dependencies = [ "once_cell", "target-lexicon", @@ -198,9 +204,9 @@ dependencies = [ [[package]] name = "pyo3-ffi" -version = "0.22.4" +version = "0.23.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ec15a5ba277339d04763f4c23d85987a5b08cbb494860be141e6a10a8eb88022" +checksum = "dab6bb2102bd8f991e7749f130a70d05dd557613e39ed2deeee8e9ca0c4d548d" dependencies = [ "libc", "pyo3-build-config", @@ -208,9 +214,9 @@ dependencies = [ [[package]] name = "pyo3-log" -version = "0.11.0" +version = "0.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3ac84e6eec1159bc2a575c9ae6723baa6ee9d45873e9bebad1e3ad7e8d28a443" +checksum = "be5bb22b77965a7b5394e9aae9897a0607b51df5167561ffc3b02643b4200bc7" dependencies = [ "arc-swap", "log", @@ -219,9 +225,9 @@ dependencies = [ [[package]] name = "pyo3-macros" -version = "0.22.4" +version = "0.23.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "15e0f01b5364bcfbb686a52fc4181d412b708a68ed20c330db9fc8d2c2bf5a43" +checksum = "91871864b353fd5ffcb3f91f2f703a22a9797c91b9ab497b1acac7b07ae509c7" dependencies = [ "proc-macro2", "pyo3-macros-backend", @@ -231,9 +237,9 @@ dependencies = [ [[package]] name = "pyo3-macros-backend" -version = "0.22.4" +version = "0.23.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a09b550200e1e5ed9176976d0060cbc2ea82dc8515da07885e7b8153a85caacb" +checksum = "43abc3b80bc20f3facd86cd3c60beed58c3e2aa26213f3cda368de39c60a27e4" dependencies = [ "heck", "proc-macro2", @@ -244,9 +250,9 @@ dependencies = [ [[package]] name = "quote" -version = "1.0.36" +version = "1.0.38" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0fa76aaf39101c457836aec0ce2316dbdc3ab723cdda1c6bd4e6ad4208acaca7" +checksum = "0e4dccaaaf89514f546c693ddc140f729f958c247918a13380cccc6078391acc" dependencies = [ "proc-macro2", ] @@ -279,18 +285,18 @@ checksum = "f3cb5ba0dc43242ce17de99c180e96db90b235b8a9fdc9543c96d2209116bd9f" [[package]] name = "serde" -version = "1.0.204" +version = "1.0.217" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bc76f558e0cbb2a839d37354c575f1dc3fdc6546b5be373ba43d95f231bf7c12" +checksum = "02fc4265df13d6fa1d00ecff087228cc0a2b5f3c0e87e258d8b94a156e984c70" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.204" +version = "1.0.217" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e0cd7e117be63d3c3678776753929474f3b04a43a080c744d6b0ae2a8c28e222" +checksum = "5a9bf7cf98d04a2b28aead066b7496853d4779c9cc183c440dbac457641e19a0" dependencies = [ "proc-macro2", "quote", @@ -299,20 +305,21 @@ dependencies = [ [[package]] name = "serde_json" -version = "1.0.120" +version = "1.0.135" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4e0d21c9a8cae1235ad58a00c11cb40d4b1e5c784f1ef2c537876ed6ffd8b7c5" +checksum = "2b0d7ba2887406110130a978386c4e1befb98c674b4fba677954e4db976630d9" dependencies = [ "itoa", + "memchr", "ryu", "serde", ] [[package]] name = "syn" -version = "2.0.70" +version = "2.0.96" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2f0209b68b3613b093e0ec905354eccaedcfe83b8cb37cbdeae64026c3064c16" +checksum = "d5d0adab1ae378d7f53bdebc67a39f1f151407ef230f0ce2883572f5d8985c80" dependencies = [ "proc-macro2", "quote", @@ -321,15 +328,15 @@ dependencies = [ [[package]] name = "target-lexicon" -version = "0.12.15" +version = "0.12.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4873307b7c257eddcb50c9bedf158eb669578359fb28428bef438fec8e6ba7c2" +checksum = "61c41af27dd6d1e27b1b16b489db798443478cef1f06a660c96db617ba5de3b1" [[package]] name = "unicode-ident" -version = "1.0.12" +version = "1.0.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3354b9ac3fae1ff6755cb6db53683adb661634f67557942dea4facebec0fee4b" +checksum = "adb9e6ca4f869e1180728b7950e35922a7fc6397f7b641499e8f3ef06e50dc83" [[package]] name = "unindent" diff --git a/rust/Cargo.toml b/rust/Cargo.toml index 3ab4be53..09e0e5ec 100644 --- a/rust/Cargo.toml +++ b/rust/Cargo.toml @@ -9,14 +9,14 @@ crate-type = ["cdylib", "rlib"] [dependencies] log = "0.4.19" -pyo3-log = "0.11.0" +pyo3-log = "0.12.1" serde_json = "1.0.103" rayon = "1.10" petgraph = "0.6.5" bimap = "0.6.3" [dependencies.pyo3] -version = "0.22.4" +version = "0.23.4" [features] extension-module = ["pyo3/extension-module"] diff --git a/rust/src/graph.rs b/rust/src/graph.rs index 0de65ec3..ae9cf237 100644 --- a/rust/src/graph.rs +++ b/rust/src/graph.rs @@ -1,8 +1,3 @@ -/* -Also, sensible behaviour when passing modules that don't exist in the graph. -*/ -#![allow(dead_code)] - use bimap::BiMap; use log::info; use petgraph::algo::astar; @@ -382,7 +377,6 @@ impl Graph { } // Note: this will panic if importer and imported are in the same package. - #[allow(unused_variables)] pub fn direct_import_exists( &self, importer: &Module, @@ -552,7 +546,6 @@ impl Graph { Some(chain) } - // https://github.com/seddonym/grimp/blob/master/src/grimp/adaptors/graph.py#L290 pub fn find_shortest_chains( &self, importer: &Module, @@ -649,7 +642,6 @@ impl Graph { Ok(chains) } - #[allow(unused_variables)] pub fn chain_exists(&self, importer: &Module, imported: &Module, as_packages: bool) -> bool { // TODO should this return a Result, so we can handle the situation the importer / imported // having shared descendants when as_packages=true? @@ -666,7 +658,6 @@ impl Graph { graph.find_shortest_chain(importer, imported).is_some() } - #[allow(unused_variables)] pub fn find_illegal_dependencies_for_layers( &self, levels: Vec, @@ -689,8 +680,6 @@ impl Graph { .map(|module_name| module_name.to_string()) .collect(); - let perms = self._generate_module_permutations(&levels, &containers); - let mut dependencies: Vec = self ._generate_module_permutations(&levels, &containers) //.into_iter() @@ -984,7 +973,6 @@ impl Graph { imports } - #[allow(unused_variables)] pub fn squash_module(&mut self, module: &Module) { // Get descendants and their imports. let descendants: Vec = self @@ -1031,6 +1019,19 @@ impl Graph { self.squashed_modules.contains(module) } + /// Return the squashed module that is the nearest ancestor of the supplied module, + /// if such an ancestor exists. + pub fn find_ancestor_squashed_module(&self, module: &Module) -> Option { + if module.is_root() { + return None; + } + let parent = Module::new_parent(&module); + if self.is_module_squashed(&parent) { + return Some(parent); + } + self.find_ancestor_squashed_module(&parent) + } + fn add_module_if_not_in_hierarchy(&mut self, module: &Module) { if self.hierarchy_module_indices.get_by_left(module).is_none() { self.add_module(module.clone()); diff --git a/rust/src/lib.rs b/rust/src/lib.rs index cbc1a5ff..112dbb0e 100644 --- a/rust/src/lib.rs +++ b/rust/src/lib.rs @@ -1,72 +1,342 @@ -mod containers; -// TODO make these private. -pub mod dependencies; -pub mod importgraph; -pub mod layers; - -use crate::dependencies::PackageDependency; -use containers::check_containers_exist; -use importgraph::ImportGraph; -use layers::Level; +pub mod graph; + +use crate::graph::{DetailedImport, Graph, Level, Module, PackageDependency}; use log::info; use pyo3::create_exception; +use pyo3::exceptions::PyValueError; use pyo3::prelude::*; -use pyo3::types::{PyDict, PyFrozenSet, PySet, PyString, PyTuple}; -use std::collections::{HashMap, HashSet}; -pub mod graph; +use pyo3::types::{PyDict, PyFrozenSet, PyList, PySet, PyString, PyTuple}; +use std::collections::HashSet; #[pymodule] fn _rustgrimp(py: Python<'_>, m: &Bound<'_, PyModule>) -> PyResult<()> { pyo3_log::init(); - m.add_function(wrap_pyfunction!(find_illegal_dependencies, m)?)?; - m.add("NoSuchContainer", py.get_type_bound::())?; + m.add_class::()?; + m.add("NoSuchContainer", py.get_type::())?; Ok(()) } create_exception!(_rustgrimp, NoSuchContainer, pyo3::exceptions::PyException); -#[pyfunction] -pub fn find_illegal_dependencies<'py>( - py: Python<'py>, - levels: &Bound<'py, PyTuple>, - containers: &Bound<'py, PySet>, - importeds_by_importer: &Bound<'py, PyDict>, -) -> PyResult> { - info!("Using Rust to find illegal dependencies."); +#[pyclass(name = "Graph")] +struct GraphWrapper { + _graph: Graph, +} + +/// Wrapper around the Graph struct that integrates with Python. +#[pymethods] +impl GraphWrapper { + #[new] + fn new() -> Self { + GraphWrapper { + _graph: Graph::default(), + } + } - let importeds_by_importer_strings: HashMap> = - importeds_by_importer.extract()?; - let importeds_by_importer_strs = strings_to_strs_hashmap(&importeds_by_importer_strings); + pub fn get_modules(&self) -> HashSet { + self._graph + .get_modules() + .iter() + .map(|module| module.name.clone()) + .collect() + } + + #[pyo3(signature = (module, is_squashed = false))] + pub fn add_module(&mut self, module: &str, is_squashed: bool) -> PyResult<()> { + let module_struct = Module::new(module.to_string()); - let graph = ImportGraph::new(importeds_by_importer_strs); - let levels_rust = rustify_levels(levels); - let containers_rust: HashSet = containers.extract()?; + if let Some(ancestor_squashed_module) = + self._graph.find_ancestor_squashed_module(&module_struct) + { + return Err(PyValueError::new_err(format!( + "Module is a descendant of squashed module {}.", + &ancestor_squashed_module.name + ))); + } - if let Err(err) = check_containers_exist(&graph, &containers_rust) { - return Err(NoSuchContainer::new_err(err)); + if self._graph.get_modules().contains(&module_struct) { + if self._graph.is_module_squashed(&module_struct) != is_squashed { + return Err(PyValueError::new_err( + "Cannot add a squashed module when it is already present in the graph \ + as an unsquashed module, or vice versa.", + )); + } + } + + match is_squashed { + false => self._graph.add_module(module_struct), + true => self._graph.add_squashed_module(module_struct), + }; + Ok(()) } - let dependencies = py.allow_threads(|| { - layers::find_illegal_dependencies(&graph, &levels_rust, &containers_rust) - }); + pub fn remove_module(&mut self, module: &str) { + self._graph.remove_module(&Module::new(module.to_string())); + } - convert_dependencies_to_python(py, dependencies, &graph) -} + pub fn squash_module(&mut self, module: &str) { + self._graph.squash_module(&Module::new(module.to_string())); + } + + pub fn is_module_squashed(&self, module: &str) -> bool { + self._graph + .is_module_squashed(&Module::new(module.to_string())) + } + + #[pyo3(signature = (*, importer, imported, line_number=None, line_contents=None))] + pub fn add_import( + &mut self, + importer: &str, + imported: &str, + line_number: Option, + line_contents: Option<&str>, + ) { + let importer = Module::new(importer.to_string()); + let imported = Module::new(imported.to_string()); + match (line_number, line_contents) { + (Some(line_number), Some(line_contents)) => { + self._graph.add_detailed_import(&DetailedImport { + importer: importer, + imported: imported, + line_number: line_number, + line_contents: line_contents.to_string(), + }); + } + (None, None) => { + self._graph.add_import(&importer, &imported); + } + _ => { + // TODO handle better. + panic!("Expected line_number and line_contents, or neither."); + } + } + } + + #[pyo3(signature = (*, importer, imported))] + pub fn remove_import(&mut self, importer: &str, imported: &str) { + self._graph.remove_import( + &Module::new(importer.to_string()), + &Module::new(imported.to_string()), + ); + } + + pub fn count_imports(&self) -> usize { + self._graph.count_imports() + } + + pub fn find_children(&self, module: &str) -> HashSet { + self._graph + .find_children(&Module::new(module.to_string())) + .iter() + .map(|child| child.name.clone()) + .collect() + } + + pub fn find_descendants(&self, module: &str) -> HashSet { + self._graph + .find_descendants(&Module::new(module.to_string())) + .unwrap() + .iter() + .map(|descendant| descendant.name.clone()) + .collect() + } -fn strings_to_strs_hashmap<'a>( - string_map: &'a HashMap>, -) -> HashMap<&'a str, HashSet<&'a str>> { - let mut str_map: HashMap<&str, HashSet<&str>> = HashMap::new(); + #[pyo3(signature = (*, importer, imported, as_packages = false))] + pub fn direct_import_exists( + &self, + importer: &str, + imported: &str, + as_packages: bool, + ) -> PyResult { + if as_packages { + let importer_module = Module::new(importer.to_string()); + let imported_module = Module::new(imported.to_string()); + // Raise a ValueError if they are in the same package. + // (direct_import_exists) will panic if they are passed. + // TODO - this is a simpler check than Python, is it enough? + if importer_module.is_descendant_of(&imported_module) + || imported_module.is_descendant_of(&importer_module) + { + return Err(PyValueError::new_err("Modules have shared descendants.")); + } + } + + Ok(self._graph.direct_import_exists( + &Module::new(importer.to_string()), + &Module::new(imported.to_string()), + as_packages, + )) + } + + pub fn find_modules_directly_imported_by(&self, module: &str) -> HashSet { + self._graph + .find_modules_directly_imported_by(&Module::new(module.to_string())) + .iter() + .map(|imported| imported.name.clone()) + .collect() + } + + pub fn find_modules_that_directly_import(&self, module: &str) -> HashSet { + self._graph + .find_modules_that_directly_import(&Module::new(module.to_string())) + .iter() + .map(|importer| importer.name.clone()) + .collect() + } + + #[pyo3(signature = (*, importer, imported))] + pub fn get_import_details<'py>( + &self, + py: Python<'py>, + importer: &str, + imported: &str, + ) -> PyResult> { + let mut vector: Vec> = vec![]; + + let mut rust_import_details_vec: Vec = self + ._graph + .get_import_details( + &Module::new(importer.to_string()), + &Module::new(imported.to_string()), + ) + .into_iter() + .collect(); + rust_import_details_vec.sort(); + + for detailed_import in rust_import_details_vec { + let pydict = PyDict::new(py); + pydict.set_item( + "importer".to_string(), + detailed_import.importer.name.clone(), + )?; + pydict.set_item( + "imported".to_string(), + detailed_import.imported.name.clone(), + )?; + pydict.set_item("line_number".to_string(), detailed_import.line_number)?; + pydict.set_item( + "line_contents".to_string(), + detailed_import.line_contents.clone(), + )?; + vector.push(pydict); + } + PyList::new(py, &vector) + } + + #[allow(unused_variables)] + #[pyo3(signature = (module, as_package=false))] + pub fn find_downstream_modules(&self, module: &str, as_package: bool) -> HashSet { + // Turn the Modules to Strings. + self._graph + .find_downstream_modules(&Module::new(module.to_string()), as_package) + .iter() + .map(|downstream| downstream.name.clone()) + .collect() + } + + #[allow(unused_variables)] + #[pyo3(signature = (module, as_package=false))] + pub fn find_upstream_modules(&self, module: &str, as_package: bool) -> HashSet { + self._graph + .find_upstream_modules(&Module::new(module.to_string()), as_package) + .iter() + .map(|upstream| upstream.name.clone()) + .collect() + } - for (key, set) in string_map { - let mut str_set: HashSet<&str> = HashSet::new(); - for item in set.iter() { - str_set.insert(item); + pub fn find_shortest_chain(&self, importer: &str, imported: &str) -> Option> { + let chain = self._graph.find_shortest_chain( + &Module::new(importer.to_string()), + &Module::new(imported.to_string()), + )?; + + Some(chain.iter().map(|module| module.name.clone()).collect()) + } + + #[pyo3(signature = (importer, imported, as_packages=true))] + pub fn find_shortest_chains<'py>( + &self, + py: Python<'py>, + importer: &str, + imported: &str, + as_packages: bool, + ) -> PyResult> { + let rust_chains: HashSet> = self + ._graph + .find_shortest_chains( + &Module::new(importer.to_string()), + &Module::new(imported.to_string()), + as_packages, + ) + .map_err(|string| PyValueError::new_err(string))?; + + let mut tuple_chains: Vec> = vec![]; + for rust_chain in rust_chains.iter() { + let module_names: Vec> = rust_chain + .iter() + .map(|module| PyString::new(py, &module.name)) + .collect(); + let tuple = PyTuple::new(py, &module_names)?; + tuple_chains.push(tuple); + } + PySet::new(py, &tuple_chains) + } + + #[pyo3(signature = (importer, imported, as_packages=false))] + pub fn chain_exists( + &self, + importer: &str, + imported: &str, + as_packages: bool, + ) -> PyResult { + if as_packages { + let importer_module = Module::new(importer.to_string()); + let imported_module = Module::new(imported.to_string()); + // Raise a ValueError if they are in the same package. + // TODO - this is a simpler check than Python, is it enough? + if importer_module.is_descendant_of(&imported_module) + || imported_module.is_descendant_of(&importer_module) + { + return Err(PyValueError::new_err("Modules have shared descendants.")); + } + } + Ok(self._graph.chain_exists( + &Module::new(importer.to_string()), + &Module::new(imported.to_string()), + as_packages, + )) + } + + #[allow(unused_variables)] + #[pyo3(signature = (layers, containers))] + pub fn find_illegal_dependencies_for_layers<'py>( + &self, + py: Python<'py>, + layers: &Bound<'py, PyTuple>, + containers: HashSet, + ) -> PyResult> { + info!("Using Rust to find illegal dependencies."); + let levels = rustify_levels(layers); + + println!("\nIncoming {:?}, {:?}", levels, containers); + let dependencies = py.allow_threads(|| { + self._graph + .find_illegal_dependencies_for_layers(levels, containers) + }); + match dependencies { + Ok(dependencies) => _convert_dependencies_to_python(py, &dependencies), + Err(error) => Err(NoSuchContainer::new_err(format!( + "Container {} does not exist.", + error.container + ))), + } + } + pub fn clone(&self) -> GraphWrapper { + GraphWrapper { + _graph: self._graph.clone(), } - str_map.insert(key.as_str(), str_set); } - str_map } fn rustify_levels<'a>(levels_python: &Bound<'a, PyTuple>) -> Vec { @@ -94,47 +364,46 @@ fn rustify_levels<'a>(levels_python: &Bound<'a, PyTuple>) -> Vec { rust_levels } -fn convert_dependencies_to_python<'py>( +fn _convert_dependencies_to_python<'py>( py: Python<'py>, - dependencies: Vec, - graph: &ImportGraph, + dependencies: &Vec, ) -> PyResult> { let mut python_dependencies: Vec> = vec![]; for rust_dependency in dependencies { - let python_dependency = PyDict::new_bound(py); - python_dependency.set_item("imported", graph.names_by_id[&rust_dependency.imported])?; - python_dependency.set_item("importer", graph.names_by_id[&rust_dependency.importer])?; + let python_dependency = PyDict::new(py); + python_dependency.set_item("imported", &rust_dependency.imported.name)?; + python_dependency.set_item("importer", &rust_dependency.importer.name)?; let mut python_routes: Vec> = vec![]; - for rust_route in rust_dependency.routes { - let route = PyDict::new_bound(py); + for rust_route in &rust_dependency.routes { + let route = PyDict::new(py); let heads: Vec> = rust_route .heads .iter() - .map(|i| PyString::new_bound(py, graph.names_by_id[&i])) + .map(|module| PyString::new(py, &module.name)) .collect(); - route.set_item("heads", PyFrozenSet::new_bound(py, &heads)?)?; + route.set_item("heads", PyFrozenSet::new(py, &heads)?)?; let middle: Vec> = rust_route .middle .iter() - .map(|i| PyString::new_bound(py, graph.names_by_id[&i])) + .map(|module| PyString::new(py, &module.name)) .collect(); - route.set_item("middle", PyTuple::new_bound(py, &middle))?; + route.set_item("middle", PyTuple::new(py, &middle)?)?; let tails: Vec> = rust_route .tails .iter() - .map(|i| PyString::new_bound(py, graph.names_by_id[&i])) + .map(|module| PyString::new(py, &module.name)) .collect(); - route.set_item("tails", PyFrozenSet::new_bound(py, &tails)?)?; + route.set_item("tails", PyFrozenSet::new(py, &tails)?)?; python_routes.push(route); } - python_dependency.set_item("routes", PyTuple::new_bound(py, python_routes))?; + python_dependency.set_item("routes", PyTuple::new(py, python_routes)?)?; python_dependencies.push(python_dependency) } - Ok(PyTuple::new_bound(py, python_dependencies)) + PyTuple::new(py, python_dependencies) } #[cfg(test)] @@ -146,7 +415,7 @@ mod tests { macro_rules! pydict { ($py: ident, {$($k: expr => $v: expr),*, $(,)?}) => { { - let dict = PyDict::new_bound($py); + let dict = PyDict::new($py); $( dict.set_item($k, $v)?; )* @@ -173,7 +442,7 @@ mod tests { "layers" => HashSet::from(["low"]), }), ]; - let python_levels = PyTuple::new_bound(py, elements); + let python_levels = PyTuple::new(py, elements)?; let result = rustify_levels(&python_levels); @@ -222,7 +491,7 @@ mod tests { "layers" => HashSet::from(["low"]), }), ]; - let python_levels = PyTuple::new_bound(py, elements); + let python_levels = PyTuple::new(py, elements)?; let mut result = rustify_levels(&python_levels); diff --git a/src/grimp/adaptors/graph.py b/src/grimp/adaptors/graph.py index 0ea9d96a..2c497fae 100644 --- a/src/grimp/adaptors/graph.py +++ b/src/grimp/adaptors/graph.py @@ -1,125 +1,41 @@ from __future__ import annotations - -from copy import copy -from typing import Dict, List, Optional, Sequence, Set, Tuple, cast - -from grimp.algorithms.shortest_path import bidirectional_shortest_path +from typing import List, Optional, Sequence, Set, Tuple, TypedDict +from grimp.application.ports.graph import DetailedImport +from grimp.domain.analysis import PackageDependency, Route +from grimp.domain.valueobjects import Layer +from grimp import _rustgrimp as rust # type: ignore[attr-defined] +from grimp.exceptions import ModuleNotPresent, NoSuchContainer from grimp.application.ports import graph -from grimp.domain.analysis import PackageDependency -from grimp.domain.valueobjects import Module, Layer -from grimp.exceptions import ModuleNotPresent - -from . import _layers class ImportGraph(graph.ImportGraph): """ - Pure Python implementation of the ImportGraph. + Rust-backed implementation of the ImportGraph. """ def __init__(self) -> None: - # Maps all the modules directly imported by each key. - self._importeds_by_importer: Dict[str, Set[str]] = {} - # Maps all the modules that directly import each key. - self._importers_by_imported: Dict[str, Set[str]] = {} - - self._edge_count = 0 - - # Instantiate a dict that stores the details for all direct imports. - self._import_details: Dict[str, List[graph.DetailedImport]] = {} - self._squashed_modules: Set[str] = set() - - # Dunder methods - # -------------- - - def __deepcopy__(self, memodict: Dict) -> "ImportGraph": - new_graph = ImportGraph() - new_graph._importeds_by_importer = { - key: value.copy() for key, value in self._importeds_by_importer.items() - } - new_graph._importers_by_imported = { - key: value.copy() for key, value in self._importers_by_imported.items() - } - new_graph._edge_count = self._edge_count - - # Note: this copies the dictionaries containing each import detail - # by *reference*, so be careful about mutating the import details - # dictionaries internally. - new_graph._import_details = { - key: value.copy() for key, value in self._import_details.items() - } - - new_graph._squashed_modules = self._squashed_modules.copy() - - return new_graph - - # Mechanics - # --------- + super().__init__() + self._rustgraph = rust.Graph() @property def modules(self) -> Set[str]: - # Note: wrapping this in a set() makes it 10 times slower to build the graph! - # As a further optimisation, we use the _StringSet type alias instead of looking up Set[str] - # when casting. - return cast(_StringSet, self._importeds_by_importer.keys()) + return self._rustgraph.get_modules() def add_module(self, module: str, is_squashed: bool = False) -> None: - ancestor_squashed_module = self._find_ancestor_squashed_module(module) - if ancestor_squashed_module: - raise ValueError( - f"Module is a descendant of squashed module {ancestor_squashed_module}." - ) - - if module in self.modules: - if self.is_module_squashed(module) != is_squashed: - raise ValueError( - "Cannot add a squashed module when it is already present in the graph as " - "an unsquashed module, or vice versa." - ) - - self._importeds_by_importer.setdefault(module, set()) - self._importers_by_imported.setdefault(module, set()) - - if is_squashed: - self._mark_module_as_squashed(module) + self._rustgraph.add_module(module, is_squashed) def remove_module(self, module: str) -> None: - if module not in self.modules: - # TODO: rethink this behaviour. - return - - for imported in copy(self.find_modules_directly_imported_by(module)): - self.remove_import(importer=module, imported=imported) - for importer in copy(self.find_modules_that_directly_import(module)): - self.remove_import(importer=importer, imported=module) - del self._importeds_by_importer[module] - del self._importers_by_imported[module] + self._rustgraph.remove_module(module) def squash_module(self, module: str) -> None: - if self.is_module_squashed(module): - return - - squashed_root = module - descendants = self.find_descendants(squashed_root) - - # Add imports to/from the root. - for descendant in descendants: - for imported_module in self.find_modules_directly_imported_by(descendant): - self.add_import(importer=squashed_root, imported=imported_module) - for importing_module in self.find_modules_that_directly_import(descendant): - self.add_import(importer=importing_module, imported=squashed_root) - - # Now we've added imports to/from the root, we can delete the root's descendants. - for descendant in descendants: - self.remove_module(descendant) - - self._mark_module_as_squashed(squashed_root) + if module not in self.modules: + raise ModuleNotPresent(f'"{module}" not present in the graph.') + self._rustgraph.squash_module(module) def is_module_squashed(self, module: str) -> bool: if module not in self.modules: raise ModuleNotPresent(f'"{module}" not present in the graph.') - - return module in self._squashed_modules + return self._rustgraph.is_module_squashed(module) def add_import( self, @@ -129,344 +45,149 @@ def add_import( line_number: Optional[int] = None, line_contents: Optional[str] = None, ) -> None: - if any((line_number, line_contents)): - if not all((line_number, line_contents)): - raise ValueError( - "Line number and contents must be provided together, or not at all." - ) - self._import_details.setdefault(importer, []) - self._import_details[importer].append( - { - "importer": importer, - "imported": imported, - "line_number": cast(int, line_number), - "line_contents": cast(str, line_contents), - } - ) - - importer_map = self._importeds_by_importer.setdefault(importer, set()) - imported_map = self._importers_by_imported.setdefault(imported, set()) - if imported not in importer_map: - # (Alternatively could check importer in imported_map.) - importer_map.add(imported) - imported_map.add(importer) - self._edge_count += 1 - - # Also ensure they have entry in other maps. - self._importeds_by_importer.setdefault(imported, set()) - self._importers_by_imported.setdefault(importer, set()) + self._rustgraph.add_import( + importer=importer, + imported=imported, + line_number=line_number, + line_contents=line_contents, + ) def remove_import(self, *, importer: str, imported: str) -> None: - if imported in self._importeds_by_importer[importer]: - self._importeds_by_importer[importer].remove(imported) - self._importers_by_imported[imported].remove(importer) - self._edge_count -= 1 - - # Clean up import details. - if importer in self._import_details: - new_details = [ - details - for details in self._import_details[importer] - if details["imported"] != imported - ] - if new_details: - self._import_details[importer] = new_details - else: - del self._import_details[importer] + return self._rustgraph.remove_import(importer=importer, imported=imported) def count_imports(self) -> int: - return self._edge_count - - # Descendants - # ----------- + return self._rustgraph.count_imports() def find_children(self, module: str) -> Set[str]: # It doesn't make sense to find the children of a squashed module, as we don't store # the children in the graph. if self.is_module_squashed(module): raise ValueError("Cannot find children of a squashed module.") - - children = set() - for potential_child in self.modules: - if Module(potential_child).is_child_of(Module(module)): - children.add(potential_child) - return children + return self._rustgraph.find_children(module) def find_descendants(self, module: str) -> Set[str]: # It doesn't make sense to find the descendants of a squashed module, as we don't store # the descendants in the graph. if self.is_module_squashed(module): raise ValueError("Cannot find descendants of a squashed module.") - - descendants = set() - for potential_descendant in self.modules: - if Module(potential_descendant).is_descendant_of(Module(module)): - descendants.add(potential_descendant) - return descendants - - # Direct imports - # -------------- + return self._rustgraph.find_descendants(module) def direct_import_exists( self, *, importer: str, imported: str, as_packages: bool = False ) -> bool: - if not as_packages: - return imported in self.find_modules_directly_imported_by(importer) - - importer_modules = self._all_modules_in_package(importer) - imported_modules = self._all_modules_in_package(imported) - - if importer_modules & imported_modules: - # If there are shared modules between the two, one of the modules is a descendant - # of the other (or they're both the same module). This doesn't make sense in - # this context, so raise an exception. - raise ValueError("Modules have shared descendants.") - - # Return True as soon as we find a path between any of the modules in the subpackages. - for candidate_importer in importer_modules: - imported_by_importer = self.find_modules_directly_imported_by(candidate_importer) - for candidate_imported in imported_modules: - if candidate_imported in imported_by_importer: - return True - return False + return self._rustgraph.direct_import_exists( + importer=importer, imported=imported, as_packages=as_packages + ) def find_modules_directly_imported_by(self, module: str) -> Set[str]: - return self._importeds_by_importer[module] + return self._rustgraph.find_modules_directly_imported_by(module) def find_modules_that_directly_import(self, module: str) -> Set[str]: - return self._importers_by_imported[module] - - def get_import_details(self, *, importer: str, imported: str) -> List[graph.DetailedImport]: - import_details_for_importer = self._import_details.get(importer, []) - # Only include the details for the imported module. - # Note: we copy each details dictionary at this point, as our deepcopying - # only copies the dictionaries by reference. - return [i.copy() for i in import_details_for_importer if i["imported"] == imported] + if module in self._rustgraph.get_modules(): + # TODO panics if module isn't in modules. + return self._rustgraph.find_modules_that_directly_import(module) + return set() - # Indirect imports - # ---------------- + def get_import_details(self, *, importer: str, imported: str) -> List[DetailedImport]: + return self._rustgraph.get_import_details( + importer=importer, + imported=imported, + ) def find_downstream_modules(self, module: str, as_package: bool = False) -> Set[str]: - # TODO optimise for as_package. - if as_package: - source_modules = self._all_modules_in_package(module) - else: - source_modules = {module} - - downstream_modules = set() - - for candidate in filter(lambda m: m not in source_modules, self.modules): - for source_module in source_modules: - if self.chain_exists(importer=candidate, imported=source_module): - downstream_modules.add(candidate) - break - - return downstream_modules + return self._rustgraph.find_downstream_modules(module, as_package) def find_upstream_modules(self, module: str, as_package: bool = False) -> Set[str]: - # TODO optimise for as_package. - if as_package: - destination_modules = self._all_modules_in_package(module) - else: - destination_modules = {module} - - upstream_modules = set() - - for candidate in filter(lambda m: m not in destination_modules, self.modules): - for destination_module in destination_modules: - if self.chain_exists(importer=destination_module, imported=candidate): - upstream_modules.add(candidate) - break - - return upstream_modules + return self._rustgraph.find_upstream_modules(module, as_package) - def find_shortest_chain(self, importer: str, imported: str) -> Optional[Tuple[str, ...]]: + def find_shortest_chain(self, importer: str, imported: str) -> tuple[str, ...] | None: for module in (importer, imported): if module not in self.modules: raise ValueError(f"Module {module} is not present in the graph.") - return self._find_shortest_chain(importer=importer, imported=imported) + chain = self._rustgraph.find_shortest_chain(importer, imported) + return tuple(chain) if chain else None def find_shortest_chains( self, importer: str, imported: str, as_packages: bool = True ) -> Set[Tuple[str, ...]]: - """ - Find the shortest import chains that exist between the importer and imported, and - between any modules contained within them if as_packages is True. Only one chain per - upstream/downstream pair will be included. Any chains that are contained within other - chains in the result set will be excluded. - - The default behavior is to treat the import and imported as packages, however, if - as_packages is False, both the importer and imported will be treated as modules instead. - - Returns: - A set of tuples of strings. Each tuple is ordered from importer to imported modules. - """ - shortest_chains = set() - - upstream_modules = ( - {imported} if not as_packages else self._all_modules_in_package(imported) - ) - downstream_modules = ( - {importer} if not as_packages else self._all_modules_in_package(importer) - ) - - if upstream_modules & downstream_modules: - # If there are shared modules between the two, one of the modules is a descendant - # of the other (or they're both the same module). This doesn't make sense in - # this context, so raise an exception. - raise ValueError("Modules have shared descendants.") - - imports_between_modules = self._find_all_imports_between_modules( - upstream_modules - ) | self._find_all_imports_between_modules(downstream_modules) - self._hide_any_existing_imports(imports_between_modules) - - map_of_imports = {} - for module in upstream_modules | downstream_modules: - map_of_imports[module] = set( - (m, module) for m in self.find_modules_that_directly_import(module) - ) | set((module, m) for m in self.find_modules_directly_imported_by(module)) - for imports in map_of_imports.values(): - self._hide_any_existing_imports(imports) - - for upstream in upstream_modules: - imports_of_upstream_module = map_of_imports[upstream] - self._reveal_imports(imports_of_upstream_module) - for downstream in downstream_modules: - imports_by_downstream_module = map_of_imports[downstream] - self._reveal_imports(imports_by_downstream_module) - shortest_chain = self._find_shortest_chain(imported=upstream, importer=downstream) - if shortest_chain: - shortest_chains.add(shortest_chain) - self._hide_any_existing_imports(imports_by_downstream_module) - self._hide_any_existing_imports(imports_of_upstream_module) - - # Reveal all the hidden imports. - for imports in map_of_imports.values(): - self._reveal_imports(imports) - self._reveal_imports(imports_between_modules) - - return shortest_chains + return self._rustgraph.find_shortest_chains(importer, imported, as_packages) def chain_exists(self, importer: str, imported: str, as_packages: bool = False) -> bool: - if not as_packages: - return bool(self._find_shortest_chain(importer=importer, imported=imported)) - - upstream_modules = self._all_modules_in_package(imported) - downstream_modules = self._all_modules_in_package(importer) - - if upstream_modules & downstream_modules: - # If there are shared modules between the two, one of the modules is a descendant - # of the other (or they're both the same module). This doesn't make sense in - # this context, so raise an exception. - raise ValueError("Modules have shared descendants.") - - # Return True as soon as we find a path between any of the modules in the subpackages. - for upstream in upstream_modules: - for downstream in downstream_modules: - if self.chain_exists(imported=upstream, importer=downstream): - return True - - return False - - # High level analysis + return self._rustgraph.chain_exists(importer, imported, as_packages) def find_illegal_dependencies_for_layers( self, layers: Sequence[Layer | str | set[str]], containers: set[str] | None = None, ) -> set[PackageDependency]: - layers = _layers.parse_layers(layers) - return _layers.find_illegal_dependencies( - graph=self, layers=layers, containers=containers or set() - ) + layers = _parse_layers(layers) + try: + result = self._rustgraph.find_illegal_dependencies_for_layers( + layers=tuple( + {"layers": layer.module_tails, "independent": layer.independent} + for layer in layers + ), + containers=set(containers) if containers else set(), + ) + except rust.NoSuchContainer as e: + raise NoSuchContainer(str(e)) - # Private methods + return _dependencies_from_tuple(result) - def _find_ancestor_squashed_module(self, module: str) -> Optional[str]: - """ - Return the name of a squashed module that is an ancestor of the supplied module, or None - if no such module exists. - """ - try: - parent = Module(module).parent.name - except ValueError: - # The module has no more ancestors. - return None + # Dunder methods + # -------------- - if parent in self.modules and self.is_module_squashed(parent): - return parent - else: - return self._find_ancestor_squashed_module(parent) - - def _mark_module_as_squashed(self, module: str) -> None: - """ - Set a flag on a module in the graph that it is squashed. - """ - self._squashed_modules.add(module) - - def _all_modules_in_package(self, module: str) -> Set[str]: - """ - Return all the modules in the supplied module, including itself. - - If the module is squashed, it will be treated as a single module. - """ - importer_modules = {module} - if not self.is_module_squashed(module): - importer_modules |= self.find_descendants(module) - return importer_modules - - def _find_all_imports_between_modules(self, modules: Set[str]) -> Set[Tuple[str, str]]: - """ - Return all the imports between the supplied set of modules. - - Return: - Set of imports, in the form (importer, imported). - """ - imports = set() - for importer in modules: - for imported in self.find_modules_directly_imported_by(importer): - if imported in modules: - imports.add((importer, imported)) - return imports - - def _hide_any_existing_imports(self, imports: Set[Tuple[str, str]]) -> None: - """ - Temporarily remove the supplied direct imports from the graph. - - If an import is not in the graph, or already hidden, this will have no effect. - - Args: - imports: Set of direct imports, in the form (importer, imported). - """ - for importer, imported in tuple(imports): - if self.direct_import_exists(importer=importer, imported=imported): - # Low-level removal from import graph (but leaving other metadata in place). - self._importeds_by_importer[importer].remove(imported) - self._importers_by_imported[imported].remove(importer) - - def _reveal_imports(self, imports: Set[Tuple[str, str]]) -> None: - """ - Given a set of direct imports that were hidden by _hide_any_existing_imports, add them back. - - Args: - imports: Set of direct imports, in the form (importer, imported). - """ - for importer, imported in tuple(imports): - # Low-level addition to import graph. - self._importeds_by_importer[importer].add(imported) - self._importers_by_imported[imported].add(importer) - - def _find_shortest_chain(self, importer: str, imported: str) -> Optional[Tuple[str, ...]]: - # Similar to find_shortest_chain but without bothering to check if the modules are - # in the graph first. - return bidirectional_shortest_path( - importers_by_imported=self._importers_by_imported, - importeds_by_importer=self._importeds_by_importer, - importer=importer, - imported=imported, - ) + def __deepcopy__(self, memodict: dict) -> "ImportGraph": + new_graph = ImportGraph() + new_graph._rustgraph = self._rustgraph.clone() + return new_graph -_StringSet = Set[str] +class _RustRoute(TypedDict): + heads: frozenset[str] + middle: tuple[str, ...] + tails: frozenset[str] + + +class _RustPackageDependency(TypedDict): + importer: str + imported: str + routes: tuple[_RustRoute, ...] + + +def _parse_layers(layers: Sequence[Layer | str | set[str]]) -> tuple[Layer, ...]: + """ + Convert the passed raw `layers` into `Layer`s. + """ + out_layers = [] + for layer in layers: + if isinstance(layer, Layer): + out_layers.append(layer) + elif isinstance(layer, str): + out_layers.append(Layer(layer, independent=True)) + else: + out_layers.append(Layer(*tuple(layer), independent=True)) + return tuple(out_layers) + + +def _dependencies_from_tuple( + rust_package_dependency_tuple: tuple[_RustPackageDependency, ...] +) -> set[PackageDependency]: + return { + PackageDependency( + imported=dep_dict["imported"], + importer=dep_dict["importer"], + routes=frozenset( + { + Route( + heads=route_dict["heads"], + middle=route_dict["middle"], + tails=route_dict["tails"], + ) + for route_dict in dep_dict["routes"] + } + ), + ) + for dep_dict in rust_package_dependency_tuple + } From 7e3c333075a37f73c04a9d52bd79b44f7f7947a3 Mon Sep 17 00:00:00 2001 From: David Seddon Date: Fri, 10 Jan 2025 13:14:01 +0000 Subject: [PATCH 04/10] Remove unused code --- rust/src/containers.rs | 14 - rust/src/dependencies.rs | 13 - rust/src/importgraph.rs | 435 --------------------- rust/src/layers.rs | 699 ---------------------------------- src/grimp/adaptors/_layers.py | 172 --------- 5 files changed, 1333 deletions(-) delete mode 100644 rust/src/containers.rs delete mode 100644 rust/src/dependencies.rs delete mode 100644 rust/src/importgraph.rs delete mode 100644 rust/src/layers.rs delete mode 100644 src/grimp/adaptors/_layers.py diff --git a/rust/src/containers.rs b/rust/src/containers.rs deleted file mode 100644 index 6f17116d..00000000 --- a/rust/src/containers.rs +++ /dev/null @@ -1,14 +0,0 @@ -use crate::importgraph::ImportGraph; -use std::collections::HashSet; - -pub fn check_containers_exist<'a>( - graph: &'a ImportGraph, - containers: &'a HashSet, -) -> Result<(), String> { - for container in containers { - if !graph.contains_module(container) { - return Err(format!("Container {} does not exist.", container)); - } - } - Ok(()) -} diff --git a/rust/src/dependencies.rs b/rust/src/dependencies.rs deleted file mode 100644 index 5ed9da01..00000000 --- a/rust/src/dependencies.rs +++ /dev/null @@ -1,13 +0,0 @@ -#[derive(PartialEq, Eq, Hash, Debug)] -pub struct Route { - pub heads: Vec, - pub middle: Vec, - pub tails: Vec, -} - -#[derive(PartialEq, Eq, Hash, Debug)] -pub struct PackageDependency { - pub importer: u32, - pub imported: u32, - pub routes: Vec, -} diff --git a/rust/src/importgraph.rs b/rust/src/importgraph.rs deleted file mode 100644 index cf58aaf9..00000000 --- a/rust/src/importgraph.rs +++ /dev/null @@ -1,435 +0,0 @@ -use std::collections::hash_map::Entry::Vacant; -use std::collections::{HashMap, HashSet}; -use std::fmt; - -#[derive(Clone)] -pub struct ImportGraph<'a> { - pub names_by_id: HashMap, - pub ids_by_name: HashMap<&'a str, u32>, - pub importers_by_imported: HashMap>, - pub importeds_by_importer: HashMap>, -} - -impl<'a> ImportGraph<'a> { - pub fn new(importeds_by_importer: HashMap<&'a str, HashSet<&'a str>>) -> ImportGraph<'a> { - // Build the name/id lookup maps. - let mut names_by_id: HashMap = HashMap::new(); - let mut ids_by_name: HashMap<&'a str, u32> = HashMap::new(); - let mut current_id: u32 = 1; - for name in importeds_by_importer.keys() { - names_by_id.insert(current_id, name); - ids_by_name.insert(name, current_id); - current_id += 1; - } - - // Convert importeds_by_importer to id-based. - let mut importeds_by_importer_u32: HashMap> = HashMap::new(); - for (importer_str, importeds_strs) in importeds_by_importer.iter() { - let mut importeds_u32 = HashSet::new(); - for imported_str in importeds_strs { - importeds_u32.insert(*ids_by_name.get(imported_str).unwrap()); - } - - importeds_by_importer_u32 - .insert(*ids_by_name.get(importer_str).unwrap(), importeds_u32); - } - - let importers_by_imported_u32 = - ImportGraph::_build_importers_by_imported_u32(&importeds_by_importer_u32); - - ImportGraph { - names_by_id, - ids_by_name, - importers_by_imported: importers_by_imported_u32, - importeds_by_importer: importeds_by_importer_u32, - } - } - - fn _build_importers_by_imported_u32( - importeds_by_importer_u32: &HashMap>, - ) -> HashMap> { - // Build importers_by_imported from importeds_by_importer. - let mut importers_by_imported_u32: HashMap> = HashMap::new(); - for (importer, importeds) in importeds_by_importer_u32.iter() { - for imported in importeds { - let entry = importers_by_imported_u32.entry(*imported).or_default(); - entry.insert(*importer); - } - } - - // Check that there is an empty set for any remaining. - for importer in importeds_by_importer_u32.keys() { - importers_by_imported_u32.entry(*importer).or_default(); - } - importers_by_imported_u32 - } - - pub fn get_module_ids(&self) -> HashSet { - self.names_by_id.keys().copied().collect() - } - - pub fn contains_module(&self, module_name: &str) -> bool { - self.ids_by_name.contains_key(module_name) - } - - pub fn remove_import(&mut self, importer: &str, imported: &str) { - self.remove_import_ids(self.ids_by_name[importer], self.ids_by_name[imported]); - } - - pub fn add_import_ids(&mut self, importer: u32, imported: u32) { - let importeds = self.importeds_by_importer.get_mut(&importer).unwrap(); - importeds.insert(imported); - - let importers = self.importers_by_imported.get_mut(&imported).unwrap(); - importers.insert(importer); - } - - pub fn remove_import_ids(&mut self, importer: u32, imported: u32) { - let importeds = self.importeds_by_importer.get_mut(&importer).unwrap(); - importeds.remove(&imported); - - let importers = self.importers_by_imported.get_mut(&imported).unwrap(); - importers.remove(&importer); - } - - pub fn remove_module_by_id(&mut self, module_id: u32) { - let _module = self.names_by_id[&module_id]; - - let mut imports_to_remove = Vec::with_capacity(self.names_by_id.len()); - { - for imported_id in &self.importeds_by_importer[&module_id] { - imports_to_remove.push((module_id, *imported_id)); - } - for importer_id in &self.importers_by_imported[&module_id] { - imports_to_remove.push((*importer_id, module_id)); - } - } - - for (importer, imported) in imports_to_remove { - self.remove_import_ids(importer, imported); - } - - self.importeds_by_importer.remove(&module_id); - self.importers_by_imported.remove(&module_id); - } - - pub fn get_descendant_ids(&self, module_name: &str) -> Vec { - let mut descendant_ids = vec![]; - let namespace: String = format!("{}.", module_name); - for (candidate_name, candidate_id) in &self.ids_by_name { - if candidate_name.starts_with(&namespace) { - descendant_ids.push(*candidate_id); - } - } - descendant_ids - } - - pub fn remove_package(&mut self, module_name: &str) { - for descendant_id in self.get_descendant_ids(module_name) { - self.remove_module_by_id(descendant_id); - } - self.remove_module_by_id(self.ids_by_name[&module_name]); - } - - pub fn squash_module(&mut self, module_name: &str) { - let squashed_root_id = self.ids_by_name[module_name]; - let descendant_ids = &self.get_descendant_ids(module_name); - - // Assemble imports to add first, then add them in a second loop, - // to avoid needing to clone importeds_by_importer. - let mut imports_to_add = Vec::with_capacity(self.names_by_id.len()); - // Imports from the root. - { - for descendant_id in descendant_ids { - for imported_id in &self.importeds_by_importer[&descendant_id] { - imports_to_add.push((squashed_root_id, *imported_id)); - } - for importer_id in &self.importers_by_imported[&descendant_id] { - imports_to_add.push((*importer_id, squashed_root_id)); - } - } - } - - for (importer, imported) in imports_to_add { - self.add_import_ids(importer, imported); - } - - // Now we've added imports to/from the root, we can delete the root's descendants. - for descendant_id in descendant_ids { - self.remove_module_by_id(*descendant_id); - } - } - - pub fn pop_shortest_chains(&mut self, importer: &str, imported: &str) -> Vec> { - let mut chains = vec![]; - let importer_id = self.ids_by_name[&importer]; - let imported_id = self.ids_by_name[&imported]; - - while let Some(chain) = self.find_shortest_chain(importer_id, imported_id) { - // Remove chain - let _mods: Vec<&str> = chain.iter().map(|i| self.names_by_id[&i]).collect(); - for i in 0..chain.len() - 1 { - self.remove_import_ids(chain[i], chain[i + 1]); - } - chains.push(chain); - } - - chains - } - - pub fn find_shortest_chain(&self, importer_id: u32, imported_id: u32) -> Option> { - let results_or_none = self._search_for_path(importer_id, imported_id); - match results_or_none { - Some(results) => { - let (pred, succ, initial_w) = results; - - let mut w_or_none: Option = Some(initial_w); - // Transform results into vector. - let mut path: Vec = Vec::new(); - // From importer to w: - while w_or_none.is_some() { - let w = w_or_none.unwrap(); - path.push(w); - w_or_none = pred[&w]; - } - path.reverse(); - - // From w to imported: - w_or_none = succ[path.last().unwrap()]; - while w_or_none.is_some() { - let w = w_or_none.unwrap(); - path.push(w); - w_or_none = succ[&w]; - } - - Some(path) - } - None => None, - } - } - /// Performs a breadth first search from both source and target, meeting in the middle. - // - // Returns: - // (pred, succ, w) where - // - pred is a dictionary of predecessors from w to the source, and - // - succ is a dictionary of successors from w to the target. - // - fn _search_for_path( - &self, - importer: u32, - imported: u32, - ) -> Option<(HashMap>, HashMap>, u32)> { - if importer == imported { - Some(( - HashMap::from([(imported, None)]), - HashMap::from([(importer, None)]), - importer, - )) - } else { - let mut pred: HashMap> = HashMap::from([(importer, None)]); - let mut succ: HashMap> = HashMap::from([(imported, None)]); - - // Initialize fringes, start with forward. - let mut forward_fringe: Vec = Vec::from([importer]); - let mut reverse_fringe: Vec = Vec::from([imported]); - let mut this_level: Vec; - - while !forward_fringe.is_empty() && !reverse_fringe.is_empty() { - if forward_fringe.len() <= reverse_fringe.len() { - this_level = forward_fringe.to_vec(); - forward_fringe = Vec::new(); - for v in this_level { - for w in self.importeds_by_importer[&v].clone() { - pred.entry(w).or_insert_with(|| { - forward_fringe.push(w); - Some(v) - }); - if succ.contains_key(&w) { - // Found path. - return Some((pred, succ, w)); - } - } - } - } else { - this_level = reverse_fringe.to_vec(); - reverse_fringe = Vec::new(); - for v in this_level { - for w in self.importers_by_imported[&v].clone() { - if let Vacant(e) = succ.entry(w) { - e.insert(Some(v)); - reverse_fringe.push(w); - } - if pred.contains_key(&w) { - // Found path. - return Some((pred, succ, w)); - } - } - } - } - } - None - } - } -} - -impl fmt::Display for ImportGraph<'_> { - fn fmt(&self, dest: &mut fmt::Formatter) -> fmt::Result { - let mut strings = vec![]; - for (importer, importeds) in self.importeds_by_importer.iter() { - let mut string = format!("IMPORTER {}: ", self.names_by_id[&importer]); - for imported in importeds { - string.push_str(format!("{}, ", self.names_by_id[&imported]).as_str()); - } - strings.push(string); - } - strings.push(" ".to_string()); - for (imported, importers) in self.importers_by_imported.iter() { - let mut string = format!("IMPORTED {}: ", self.names_by_id[&imported]); - for importer in importers { - string.push_str(format!("{}, ", self.names_by_id[&importer]).as_str()); - } - strings.push(string); - } - write!(dest, "{}", strings.join("\n")) - } -} - -#[cfg(test)] -mod tests { - use super::*; - - fn _make_graph() -> ImportGraph<'static> { - ImportGraph::new(HashMap::from([ - ("blue", HashSet::from(["blue.alpha", "blue.beta", "green"])), - ("blue.alpha", HashSet::new()), - ("blue.beta", HashSet::new()), - ("green", HashSet::from(["blue.alpha", "blue.beta"])), - ])) - } - - #[test] - fn get_module_ids() { - let graph = _make_graph(); - - assert_eq!( - graph.get_module_ids(), - HashSet::from([ - *graph.ids_by_name.get("blue").unwrap(), - *graph.ids_by_name.get("blue.alpha").unwrap(), - *graph.ids_by_name.get("blue.beta").unwrap(), - *graph.ids_by_name.get("green").unwrap(), - ]) - ); - } - - #[test] - fn new_stores_importeds_by_importer_using_id() { - let graph = _make_graph(); - - let expected_importeds: HashSet = HashSet::from([ - *graph.ids_by_name.get("blue.alpha").unwrap(), - *graph.ids_by_name.get("blue.beta").unwrap(), - *graph.ids_by_name.get("green").unwrap(), - ]); - - assert_eq!( - *graph - .importeds_by_importer - .get(graph.ids_by_name.get("blue").unwrap()) - .unwrap(), - expected_importeds - ); - } - - #[test] - fn new_stores_importers_by_imported_using_id() { - let graph = _make_graph(); - - let expected_importers: HashSet = HashSet::from([ - *graph.ids_by_name.get("blue").unwrap(), - *graph.ids_by_name.get("green").unwrap(), - ]); - - assert_eq!( - *graph - .importers_by_imported - .get(graph.ids_by_name.get("blue.alpha").unwrap()) - .unwrap(), - expected_importers - ); - } - - #[test] - fn test_squash_module() { - let mut graph = ImportGraph::new(HashMap::from([ - ("blue", HashSet::from(["orange", "green"])), - ("blue.alpha", HashSet::from(["green.delta"])), - ("blue.beta", HashSet::new()), - ("green", HashSet::from(["blue.alpha", "blue.beta"])), - ("green.gamma", HashSet::new()), - ("green.delta", HashSet::new()), - ("orange", HashSet::new()), - ])); - - graph.squash_module("blue"); - - assert_eq!( - graph.importeds_by_importer[&graph.ids_by_name["blue"]], - HashSet::from([ - graph.ids_by_name["orange"], - graph.ids_by_name["green"], - graph.ids_by_name["green.delta"], - ]) - ); - assert_eq!( - graph.importeds_by_importer[&graph.ids_by_name["green"]], - HashSet::from([graph.ids_by_name["blue"],]) - ); - - assert_eq!( - graph.importers_by_imported[&graph.ids_by_name["orange"]], - HashSet::from([graph.ids_by_name["blue"],]) - ); - assert_eq!( - graph.importers_by_imported[&graph.ids_by_name["green"]], - HashSet::from([graph.ids_by_name["blue"],]) - ); - assert_eq!( - graph.importers_by_imported[&graph.ids_by_name["green.delta"]], - HashSet::from([graph.ids_by_name["blue"],]) - ); - assert_eq!( - graph.importers_by_imported[&graph.ids_by_name["blue"]], - HashSet::from([graph.ids_by_name["green"],]) - ); - } - - #[test] - fn test_find_shortest_chain() { - let blue = "blue"; - let green = "green"; - let yellow = "yellow"; - let blue_alpha = "blue.alpha"; - let blue_beta = "blue.beta"; - - let graph = ImportGraph::new(HashMap::from([ - (green, HashSet::from([blue])), - (blue_alpha, HashSet::from([blue])), - (yellow, HashSet::from([green])), - (blue_beta, HashSet::from([green])), - (blue, HashSet::new()), - ])); - - let path_or_none: Option> = - graph.find_shortest_chain(graph.ids_by_name[&yellow], graph.ids_by_name[&blue]); - - assert_eq!( - path_or_none, - Some(Vec::from([ - graph.ids_by_name[&yellow], - graph.ids_by_name[&green], - graph.ids_by_name[&blue] - ])) - ); - } -} diff --git a/rust/src/layers.rs b/rust/src/layers.rs deleted file mode 100644 index 0d5257fa..00000000 --- a/rust/src/layers.rs +++ /dev/null @@ -1,699 +0,0 @@ -use crate::dependencies::{PackageDependency, Route}; -use crate::importgraph::ImportGraph; - -use log::info; -use rayon::prelude::*; -use std::collections::HashSet; -use std::time::Instant; - -/// A group of layers at the same level in the layering. -#[derive(PartialEq, Eq, Hash, Debug)] -pub struct Level { - pub layers: Vec, - pub independent: bool, -} - -pub fn find_illegal_dependencies<'a>( - graph: &'a ImportGraph, - levels: &'a Vec, - containers: &'a HashSet, -) -> Vec { - let layers = _layers_from_levels(levels); - - _generate_module_permutations(graph, levels, containers) - .into_par_iter() - .filter_map(|(higher_layer_package, lower_layer_package, container)| { - // TODO: it's inefficient to do this for sibling layers, as we don't need - // to clone and trim the graph for identical pairs. - info!( - "Searching for import chains from {} to {}...", - lower_layer_package, higher_layer_package - ); - let now = Instant::now(); - let dependency_or_none = _search_for_package_dependency( - &higher_layer_package, - &lower_layer_package, - &layers, - &container, - graph, - ); - _log_illegal_route_count(&dependency_or_none, now.elapsed().as_secs()); - dependency_or_none - }) - .collect() -} - -/// Return every permutation of modules that exist in the graph -/// in which the second should not import the first. -fn _generate_module_permutations<'a>( - graph: &'a ImportGraph, - levels: &'a [Level], - containers: &'a HashSet, -) -> Vec<(String, String, Option)> { - let mut permutations: Vec<(String, String, Option)> = vec![]; - - let quasi_containers: Vec> = if containers.is_empty() { - vec![None] - } else { - containers.iter().map(|i| Some(i.to_string())).collect() - }; - for container in quasi_containers { - for (index, higher_level) in levels.iter().enumerate() { - for higher_layer in &higher_level.layers { - let higher_layer_module_name = _module_from_layer(higher_layer, &container); - if graph - .ids_by_name - .get(&higher_layer_module_name as &str) - .is_none() - { - continue; - } - - // Build the layers that mustn't import this higher layer. - // That includes: - // * lower layers. - // * sibling layers, if the layer is independent. - let mut layers_forbidden_to_import_higher_layer: Vec<&str> = vec![]; - - if higher_level.independent { - for potential_sibling_layer in &higher_level.layers { - if potential_sibling_layer != higher_layer { - // It's a sibling layer. - layers_forbidden_to_import_higher_layer.push(potential_sibling_layer); - } - } - } - - for lower_level in &levels[index + 1..] { - for lower_layer in &lower_level.layers { - layers_forbidden_to_import_higher_layer.push(lower_layer); - } - } - - // Now turn the layers into modules, if they exist. - for forbidden_layer in &layers_forbidden_to_import_higher_layer { - let forbidden_module_name = _module_from_layer(forbidden_layer, &container); - if let Some(_value) = graph.ids_by_name.get(&forbidden_module_name as &str) { - permutations.push(( - higher_layer_module_name.clone(), - forbidden_module_name.clone(), - container.clone(), - )); - }; - } - } - } - } - - permutations -} - -fn _module_from_layer<'a>(module: &'a str, container: &'a Option) -> String { - match container { - Some(true_container) => format!("{}.{}", true_container, module), - None => module.to_string(), - } -} - -fn _search_for_package_dependency<'a>( - higher_layer_package: &'a str, - lower_layer_package: &'a str, - layers: &'a Vec<&'a str>, - container: &'a Option, - graph: &'a ImportGraph, -) -> Option { - let mut temp_graph = graph.clone(); - _remove_other_layers( - &mut temp_graph, - layers, - container, - (higher_layer_package, lower_layer_package), - ); - let mut routes: Vec = vec![]; - - // Direct routes. - let direct_links = - _pop_direct_imports(higher_layer_package, lower_layer_package, &mut temp_graph); - for (importer, imported) in direct_links { - routes.push(Route { - heads: vec![importer], - middle: vec![], - tails: vec![imported], - }); - } - - // Indirect routes. - for indirect_route in - _get_indirect_routes(higher_layer_package, lower_layer_package, &temp_graph) - { - routes.push(indirect_route); - } - if routes.is_empty() { - None - } else { - Some(PackageDependency { - imported: graph.ids_by_name[&higher_layer_package], - importer: graph.ids_by_name[&lower_layer_package], - routes, - }) - } -} - -fn _layers_from_levels<'a>(levels: &'a Vec) -> Vec<&'a str> { - let mut layers: Vec<&str> = vec![]; - for level in levels { - layers.extend(level.layers.iter().map(|s| s.as_str())); - } - layers -} - -fn _remove_other_layers<'a>( - graph: &'a mut ImportGraph, - layers: &'a Vec<&'a str>, - container: &'a Option, - layers_to_preserve: (&'a str, &'a str), -) { - for layer in layers { - let layer_module = _module_from_layer(layer, container); - if layers_to_preserve.0 == layer_module || layers_to_preserve.1 == layer_module { - continue; - } - if graph.contains_module(&layer_module) { - graph.remove_package(&layer_module); - } - } -} - -fn _pop_direct_imports<'a>( - higher_layer_package: &'a str, - lower_layer_package: &'a str, - graph: &'a mut ImportGraph, -) -> HashSet<(u32, u32)> { - // Remove the direct imports, returning them as (importer, imported) tuples. - let mut imports = HashSet::new(); - - let higher_layer_namespace: String = format!("{}.", higher_layer_package); - let mut lower_layer_module_ids: Vec = vec![graph.ids_by_name[lower_layer_package]]; - lower_layer_module_ids.append(&mut graph.get_descendant_ids(lower_layer_package)); - - for lower_layer_module_id in lower_layer_module_ids { - let _lower = graph.names_by_id[&lower_layer_module_id]; - let imported_module_ids = graph.importeds_by_importer[&lower_layer_module_id].clone(); - for imported_module_id in imported_module_ids { - let imported_module = graph.names_by_id[&imported_module_id]; - - if imported_module.starts_with(&higher_layer_namespace) - || imported_module == higher_layer_package - { - imports.insert((lower_layer_module_id, imported_module_id)); - graph.remove_import_ids(lower_layer_module_id, imported_module_id) - } - } - } - imports -} - -fn _get_indirect_routes<'a>( - imported_package: &'a str, - importer_package: &'a str, - graph: &'a ImportGraph, -) -> Vec { - // Squashes the two packages. - // Gets a list of paths between them, called middles. - // Add the heads and tails to the middles. - let mut temp_graph = graph.clone(); - temp_graph.squash_module(imported_package); - temp_graph.squash_module(importer_package); - - let middles = _find_middles(&mut temp_graph, importer_package, imported_package); - _middles_to_routes(graph, middles, importer_package, imported_package) -} - -fn _find_middles<'a>( - graph: &'a mut ImportGraph, - importer: &'a str, - imported: &'a str, -) -> Vec> { - let mut middles = vec![]; - - for chain in graph.pop_shortest_chains(importer, imported) { - // Remove first and last element. - // TODO surely there's a better way? - let mut middle: Vec = vec![]; - let chain_length = chain.len(); - for (index, module) in chain.iter().enumerate() { - if index != 0 && index != chain_length - 1 { - middle.push(*module); - } - } - middles.push(middle); - } - - middles -} - -fn _log_illegal_route_count(dependency_or_none: &Option, duration_in_s: u64) { - let route_count = match dependency_or_none { - Some(dependency) => dependency.routes.len(), - None => 0, - }; - let pluralized = if route_count == 1 { "" } else { "s" }; - info!( - "Found {} illegal route{} in {}s.", - route_count, pluralized, duration_in_s - ); -} - -fn _middles_to_routes<'a>( - graph: &'a ImportGraph, - middles: Vec>, - importer: &'a str, - imported: &'a str, -) -> Vec { - let mut routes = vec![]; - let importer_id = graph.ids_by_name[&importer]; - let imported_id = graph.ids_by_name[&imported]; - - for middle in middles { - // Construct heads. - let mut heads: Vec = vec![]; - let first_imported_id = middle[0]; - let candidate_modules = &graph.importers_by_imported[&first_imported_id]; - for candidate_module in candidate_modules { - if importer_id == *candidate_module - || graph - .get_descendant_ids(importer) - .contains(candidate_module) - { - heads.push(*candidate_module); - } - } - - // Construct tails. - let mut tails: Vec = vec![]; - let last_importer_id = middle[middle.len() - 1]; - let candidate_modules = &graph.importeds_by_importer[&last_importer_id]; - for candidate_module in candidate_modules { - if imported_id == *candidate_module - || graph - .get_descendant_ids(imported) - .contains(candidate_module) - { - tails.push(*candidate_module); - } - } - routes.push(Route { - heads, - middle, - tails, - }) - } - - routes -} - -#[cfg(test)] -mod tests { - use super::*; - use std::collections::HashMap; - - #[test] - fn test_find_illegal_dependencies_no_container() { - let graph = ImportGraph::new(HashMap::from([ - ("low", HashSet::new()), - ("low.blue", HashSet::from(["utils"])), - ("low.green", HashSet::new()), - ("low.green.alpha", HashSet::from(["high.yellow"])), - ("mid_a", HashSet::from(["mid_b"])), - ("mid_a.orange", HashSet::new()), - ("mid_b", HashSet::from(["mid_c"])), - ("mid_b.brown", HashSet::new()), - ("mid_c", HashSet::new()), - ("mid_c.purple", HashSet::new()), - ("high", HashSet::from(["low.blue"])), - ("high.yellow", HashSet::new()), - ("high.red", HashSet::new()), - ("high.red.beta", HashSet::new()), - ("utils", HashSet::from(["high.red"])), - ])); - let levels = vec![ - Level { - independent: true, - layers: vec!["high".to_string()], - }, - Level { - independent: true, - layers: vec![ - "mid_a".to_string(), - "mid_b".to_string(), - "mid_c".to_string(), - ], - }, - Level { - independent: true, - layers: vec!["low".to_string()], - }, - ]; - let containers = HashSet::new(); - - let dependencies = find_illegal_dependencies(&graph, &levels, &containers); - - assert_eq!( - dependencies, - vec![ - PackageDependency { - importer: *graph.ids_by_name.get("low").unwrap(), - imported: *graph.ids_by_name.get("high").unwrap(), - routes: vec![ - Route { - heads: vec![*graph.ids_by_name.get("low.green.alpha").unwrap()], - middle: vec![], - tails: vec![*graph.ids_by_name.get("high.yellow").unwrap()], - }, - Route { - heads: vec![*graph.ids_by_name.get("low.blue").unwrap()], - middle: vec![*graph.ids_by_name.get("utils").unwrap()], - tails: vec![*graph.ids_by_name.get("high.red").unwrap()], - }, - ], - }, - PackageDependency { - importer: *graph.ids_by_name.get("mid_a").unwrap(), - imported: *graph.ids_by_name.get("mid_b").unwrap(), - routes: vec![Route { - heads: vec![*graph.ids_by_name.get("mid_a").unwrap()], - middle: vec![], - tails: vec![*graph.ids_by_name.get("mid_b").unwrap()], - },], - }, - PackageDependency { - importer: *graph.ids_by_name.get("mid_b").unwrap(), - imported: *graph.ids_by_name.get("mid_c").unwrap(), - routes: vec![Route { - heads: vec![*graph.ids_by_name.get("mid_b").unwrap()], - middle: vec![], - tails: vec![*graph.ids_by_name.get("mid_c").unwrap()], - },], - }, - ] - ); - } - - #[test] - fn test_find_illegal_dependencies_with_container() { - let graph = ImportGraph::new(HashMap::from([ - ("mypackage.low", HashSet::new()), - ("mypackage.low.blue", HashSet::from(["mypackage.utils"])), - ("mypackage.low.green", HashSet::new()), - ( - "mypackage.low.green.alpha", - HashSet::from(["mypackage.high.yellow"]), - ), - ("mypackage.high", HashSet::from(["mypackage.low.blue"])), - ("mypackage.high.yellow", HashSet::new()), - ("mypackage.high.red", HashSet::new()), - ("mypackage.high.red.beta", HashSet::new()), - ("mypackage.utils", HashSet::from(["mypackage.high.red"])), - ])); - let levels = vec![ - Level { - independent: true, - layers: vec!["high".to_string()], - }, - Level { - independent: true, - layers: vec!["low".to_string()], - }, - ]; - let containers = HashSet::from(["mypackage".to_string()]); - - let dependencies = find_illegal_dependencies(&graph, &levels, &containers); - - assert_eq!( - dependencies, - vec![PackageDependency { - importer: *graph.ids_by_name.get("mypackage.low").unwrap(), - imported: *graph.ids_by_name.get("mypackage.high").unwrap(), - routes: vec![ - Route { - heads: vec![*graph.ids_by_name.get("mypackage.low.green.alpha").unwrap()], - middle: vec![], - tails: vec![*graph.ids_by_name.get("mypackage.high.yellow").unwrap()], - }, - Route { - heads: vec![*graph.ids_by_name.get("mypackage.low.blue").unwrap()], - middle: vec![*graph.ids_by_name.get("mypackage.utils").unwrap()], - tails: vec![*graph.ids_by_name.get("mypackage.high.red").unwrap()], - }, - ], - }] - ); - } - - #[test] - fn test_generate_module_permutations() { - let graph = ImportGraph::new(HashMap::from([ - ("mypackage.low", HashSet::new()), - ("mypackage.low.blue", HashSet::from(["mypackage.utils"])), - ("mypackage.low.green", HashSet::new()), - ( - "mypackage.low.green.alpha", - HashSet::from(["mypackage.high.yellow"]), - ), - ("mypackage.mid_a", HashSet::new()), - ("mypackage.mid_a.foo", HashSet::new()), - ("mypackage.mid_b", HashSet::new()), - ("mypackage.mid_b.foo", HashSet::new()), - ("mypackage.mid_c", HashSet::new()), - ("mypackage.mid_c.foo", HashSet::new()), - ("mypackage.high", HashSet::from(["mypackage.low.blue"])), - ("mypackage.high.yellow", HashSet::new()), - ("mypackage.high.red", HashSet::new()), - ("mypackage.high.red.beta", HashSet::new()), - ("mypackage.utils", HashSet::from(["mypackage.high.red"])), - ])); - let levels = vec![ - Level { - independent: true, - layers: vec!["high".to_string()], - }, - Level { - independent: true, - layers: vec![ - "mid_a".to_string(), - "mid_b".to_string(), - "mid_c".to_string(), - ], - }, - Level { - independent: true, - layers: vec!["low".to_string()], - }, - ]; - let containers = HashSet::from(["mypackage".to_string()]); - - let perms = _generate_module_permutations(&graph, &levels, &containers); - - let result: HashSet<(String, String, Option)> = HashSet::from_iter(perms); - let (high, mid_a, mid_b, mid_c, low) = ( - "mypackage.high", - "mypackage.mid_a", - "mypackage.mid_b", - "mypackage.mid_c", - "mypackage.low", - ); - assert_eq!( - result, - HashSet::from_iter([ - ( - high.to_string(), - mid_a.to_string(), - Some("mypackage".to_string()) - ), - ( - high.to_string(), - mid_b.to_string(), - Some("mypackage".to_string()) - ), - ( - high.to_string(), - mid_c.to_string(), - Some("mypackage".to_string()) - ), - ( - high.to_string(), - low.to_string(), - Some("mypackage".to_string()) - ), - ( - mid_a.to_string(), - mid_b.to_string(), - Some("mypackage".to_string()) - ), - ( - mid_a.to_string(), - mid_c.to_string(), - Some("mypackage".to_string()) - ), - ( - mid_b.to_string(), - mid_a.to_string(), - Some("mypackage".to_string()) - ), - ( - mid_b.to_string(), - mid_c.to_string(), - Some("mypackage".to_string()) - ), - ( - mid_c.to_string(), - mid_a.to_string(), - Some("mypackage".to_string()) - ), - ( - mid_c.to_string(), - mid_b.to_string(), - Some("mypackage".to_string()) - ), - ( - mid_a.to_string(), - low.to_string(), - Some("mypackage".to_string()) - ), - ( - mid_b.to_string(), - low.to_string(), - Some("mypackage".to_string()) - ), - ( - mid_c.to_string(), - low.to_string(), - Some("mypackage".to_string()) - ), - ]) - ); - } - - #[test] - fn test_generate_module_permutations_sibling_layer_not_independent() { - let graph = ImportGraph::new(HashMap::from([ - ("mypackage.low", HashSet::new()), - ("mypackage.low.blue", HashSet::from(["mypackage.utils"])), - ("mypackage.low.green", HashSet::new()), - ( - "mypackage.low.green.alpha", - HashSet::from(["mypackage.high.yellow"]), - ), - ("mypackage.mid_a", HashSet::new()), - ("mypackage.mid_a.foo", HashSet::new()), - ("mypackage.mid_b", HashSet::new()), - ("mypackage.mid_b.foo", HashSet::new()), - ("mypackage.mid_c", HashSet::new()), - ("mypackage.mid_c.foo", HashSet::new()), - ("mypackage.high", HashSet::from(["mypackage.low.blue"])), - ("mypackage.high.yellow", HashSet::new()), - ("mypackage.high.red", HashSet::new()), - ("mypackage.high.red.beta", HashSet::new()), - ("mypackage.utils", HashSet::from(["mypackage.high.red"])), - ])); - let levels = vec![ - Level { - independent: true, - layers: vec!["high".to_string()], - }, - Level { - independent: false, - layers: vec![ - "mid_a".to_string(), - "mid_b".to_string(), - "mid_c".to_string(), - ], - }, - Level { - independent: true, - layers: vec!["low".to_string()], - }, - ]; - let containers = HashSet::from(["mypackage".to_string()]); - - let perms = _generate_module_permutations(&graph, &levels, &containers); - - let result: HashSet<(String, String, Option)> = HashSet::from_iter(perms); - let (high, mid_a, mid_b, mid_c, low) = ( - "mypackage.high", - "mypackage.mid_a", - "mypackage.mid_b", - "mypackage.mid_c", - "mypackage.low", - ); - assert_eq!( - result, - HashSet::from_iter([ - ( - high.to_string(), - mid_a.to_string(), - Some("mypackage".to_string()) - ), - ( - high.to_string(), - mid_b.to_string(), - Some("mypackage".to_string()) - ), - ( - high.to_string(), - mid_c.to_string(), - Some("mypackage".to_string()) - ), - ( - high.to_string(), - low.to_string(), - Some("mypackage".to_string()) - ), - ( - mid_a.to_string(), - low.to_string(), - Some("mypackage".to_string()) - ), - ( - mid_b.to_string(), - low.to_string(), - Some("mypackage".to_string()) - ), - ( - mid_c.to_string(), - low.to_string(), - Some("mypackage".to_string()) - ), - ]) - ); - } - - #[test] - fn test_layers_from_levels() { - let levels = vec![ - Level { - independent: true, - layers: vec!["high".to_string()], - }, - Level { - independent: true, - layers: vec![ - "medium_a".to_string(), - "medium_b".to_string(), - "medium_c".to_string(), - ], - }, - Level { - independent: true, - layers: vec!["low".to_string()], - }, - ]; - - let result = _layers_from_levels(&levels); - - assert_eq!( - HashSet::<&str>::from_iter(result), - HashSet::from_iter(["high", "medium_a", "medium_b", "medium_c", "low",]), - ) - } -} diff --git a/src/grimp/adaptors/_layers.py b/src/grimp/adaptors/_layers.py deleted file mode 100644 index 7ff3c7e4..00000000 --- a/src/grimp/adaptors/_layers.py +++ /dev/null @@ -1,172 +0,0 @@ -from __future__ import annotations - -from dataclasses import dataclass -from typing import TYPE_CHECKING, Any, Iterator, Sequence, TypedDict - -from grimp import Route -from grimp import _rustgrimp as rust # type: ignore[attr-defined] - -if TYPE_CHECKING: - from grimp.adaptors.graph import ImportGraph - -from grimp.domain.analysis import PackageDependency -from grimp.exceptions import NoSuchContainer -from grimp.domain.valueobjects import Layer - - -def parse_layers(layers: Sequence[Layer | str | set[str]]) -> tuple[Layer, ...]: - """ - Convert the passed raw `layers` into `Layer`s. - """ - out_layers = [] - for layer in layers: - if isinstance(layer, Layer): - out_layers.append(layer) - elif isinstance(layer, str): - out_layers.append(Layer(layer, independent=True)) - else: - out_layers.append(Layer(*tuple(layer), independent=True)) - return tuple(out_layers) - - -def find_illegal_dependencies( - graph: ImportGraph, - layers: Sequence[Layer], - containers: set[str], -) -> set[PackageDependency]: - """ - Find dependencies that don't conform to the supplied layered architecture. - - See ImportGraph.find_illegal_dependencies_for_layers. - - The only difference between this and the method is that the containers passed in - is already a (potentially empty) set. - """ - try: - rust_package_dependency_tuple = rust.find_illegal_dependencies( - levels=tuple( - {"layers": layer.module_tails, "independent": layer.independent} - for layer in layers - ), - containers=set(containers), - importeds_by_importer=graph._importeds_by_importer, - ) - except rust.NoSuchContainer as e: - raise NoSuchContainer(str(e)) - - rust_package_dependencies = _dependencies_from_tuple(rust_package_dependency_tuple) - return rust_package_dependencies - - -class _RustRoute(TypedDict): - heads: frozenset[str] - middle: tuple[str, ...] - tails: frozenset[str] - - -class _RustPackageDependency(TypedDict): - importer: str - imported: str - routes: tuple[_RustRoute, ...] - - -def _dependencies_from_tuple( - rust_package_dependency_tuple: tuple[_RustPackageDependency, ...] -) -> set[PackageDependency]: - return { - PackageDependency( - imported=dep_dict["imported"], - importer=dep_dict["importer"], - routes=frozenset( - { - Route( - heads=route_dict["heads"], - middle=route_dict["middle"], - tails=route_dict["tails"], - ) - for route_dict in dep_dict["routes"] - } - ), - ) - for dep_dict in rust_package_dependency_tuple - } - - -class _Module: - """ - A Python module. - """ - - def __init__(self, name: str) -> None: - """ - Args: - name: The fully qualified name of a Python module, e.g. 'package.foo.bar'. - """ - self.name = name - - def __str__(self) -> str: - return self.name - - def __eq__(self, other: Any) -> bool: - if isinstance(other, self.__class__): - return hash(self) == hash(other) - else: - return False - - def __hash__(self) -> int: - return hash(str(self)) - - def is_descendant_of(self, module: "_Module") -> bool: - return self.name.startswith(f"{module.name}.") - - -@dataclass(frozen=True) -class _Link: - importer: str - imported: str - - -# A chain of modules, each of which imports the next. -_Chain = tuple[str, ...] - - -def _generate_module_permutations( - graph: ImportGraph, - layers: Sequence[str], - containers: set[str], -) -> Iterator[tuple[_Module, _Module, str | None]]: - """ - Return all possible combinations of higher level and lower level modules, in pairs. - - Each pair of modules consists of immediate children of two different layers. The first - module is in a layer higher than the layer of the second module. This means the first - module is allowed to import the second, but not the other way around. - - Returns: - module_in_higher_layer, module_in_lower_layer, container - """ - # If there are no containers, we still want to run the loop once. - quasi_containers = containers or [None] - - for container in quasi_containers: - for index, higher_layer in enumerate(layers): - higher_layer_module = _module_from_layer(higher_layer, container) - - if higher_layer_module.name not in graph.modules: - continue - - for lower_layer in layers[index + 1 :]: - lower_layer_module = _module_from_layer(lower_layer, container) - - if lower_layer_module.name not in graph.modules: - continue - - yield higher_layer_module, lower_layer_module, container - - -def _module_from_layer(layer: str, container: str | None = None) -> _Module: - if container: - name = ".".join([container, layer]) - else: - name = layer - return _Module(name) From c5089a00f2c3cd20abe9196abbd90a9eb93cf435 Mon Sep 17 00:00:00 2001 From: David Seddon Date: Wed, 15 Jan 2025 08:52:57 +0000 Subject: [PATCH 05/10] Use faster hash library This is faster, but not cryptographically safe. That shouldn't matter for our purposes. --- rust/Cargo.lock | 7 ++ rust/Cargo.toml | 1 + rust/src/graph.rs | 228 ++++++++++++++++++++++++++------------------ rust/src/lib.rs | 38 ++++---- rust/tests/large.rs | 4 +- 5 files changed, 161 insertions(+), 117 deletions(-) diff --git a/rust/Cargo.lock b/rust/Cargo.lock index 37709188..9773fef5 100644 --- a/rust/Cargo.lock +++ b/rust/Cargo.lock @@ -12,6 +12,7 @@ dependencies = [ "pyo3", "pyo3-log", "rayon", + "rustc-hash", "serde_json", ] @@ -277,6 +278,12 @@ dependencies = [ "crossbeam-utils", ] +[[package]] +name = "rustc-hash" +version = "2.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c7fb8039b3032c191086b10f11f319a6e99e1e82889c5cc6046f515c9db1d497" + [[package]] name = "ryu" version = "1.0.18" diff --git a/rust/Cargo.toml b/rust/Cargo.toml index 09e0e5ec..00f48bc5 100644 --- a/rust/Cargo.toml +++ b/rust/Cargo.toml @@ -14,6 +14,7 @@ serde_json = "1.0.103" rayon = "1.10" petgraph = "0.6.5" bimap = "0.6.3" +rustc-hash = "2.1.0" [dependencies.pyo3] version = "0.23.4" diff --git a/rust/src/graph.rs b/rust/src/graph.rs index ae9cf237..248e84d6 100644 --- a/rust/src/graph.rs +++ b/rust/src/graph.rs @@ -6,7 +6,7 @@ use petgraph::stable_graph::{NodeIndex, StableGraph}; use petgraph::visit::{Bfs, Walker}; use petgraph::Direction; use rayon::prelude::*; -use std::collections::{HashMap, HashSet}; +use rustc_hash::{FxHashMap, FxHashSet}; use std::fmt; use std::time::Instant; @@ -93,10 +93,10 @@ pub struct Graph { hierarchy: StableGraph, imports_module_indices: BiMap, imports: StableGraph, - squashed_modules: HashSet, + squashed_modules: FxHashSet, // Invisible modules exist in the hierarchy but haven't been explicitly added to the graph. - invisible_modules: HashSet, - detailed_imports_map: HashMap<(Module, Module), HashSet>, + invisible_modules: FxHashSet, + detailed_imports_map: FxHashMap<(Module, Module), FxHashSet>, } #[derive(PartialEq, Eq, Hash, Debug, PartialOrd, Ord)] @@ -245,7 +245,7 @@ impl Graph { }; } - pub fn get_modules(&self) -> HashSet<&Module> { + pub fn get_modules(&self) -> FxHashSet<&Module> { self.hierarchy_module_indices .left_values() .filter(|module| !self.invisible_modules.contains(module)) @@ -260,23 +260,23 @@ impl Graph { &self, importer: &Module, imported: &Module, - ) -> HashSet { + ) -> FxHashSet { let key = (importer.clone(), imported.clone()); match self.detailed_imports_map.get(&key) { Some(import_details) => import_details.clone(), - None => HashSet::new(), + None => FxHashSet::default(), } } - pub fn find_children(&self, module: &Module) -> HashSet<&Module> { + pub fn find_children(&self, module: &Module) -> FxHashSet<&Module> { if self.invisible_modules.contains(module) { - return HashSet::new(); + return FxHashSet::default(); } let module_index = match self.hierarchy_module_indices.get_by_left(module) { Some(index) => index, // Module does not exist. // TODO: should this return a result, to handle if module is not in graph? - None => return HashSet::new(), + None => return FxHashSet::default(), }; self.hierarchy .neighbors(*module_index) @@ -285,7 +285,10 @@ impl Graph { .collect() } - pub fn find_descendants(&self, module: &Module) -> Result, ModuleNotPresent> { + pub fn find_descendants( + &self, + module: &Module, + ) -> Result, ModuleNotPresent> { let module_index = match self.hierarchy_module_indices.get_by_left(module) { Some(index) => index, None => { @@ -343,7 +346,7 @@ impl Graph { let key = (import.importer.clone(), import.imported.clone()); self.detailed_imports_map .entry(key) - .or_insert_with(HashSet::new) + .or_insert_with(FxHashSet::default) .insert(import.clone()); self.add_import(&import.importer, &import.imported); } @@ -411,17 +414,17 @@ impl Graph { .contains_edge(importer_index, imported_index) } - pub fn find_modules_that_directly_import(&self, imported: &Module) -> HashSet<&Module> { + pub fn find_modules_that_directly_import(&self, imported: &Module) -> FxHashSet<&Module> { let imported_index = match self.imports_module_indices.get_by_left(imported) { Some(imported_index) => *imported_index, - None => return HashSet::new(), + None => return FxHashSet::default(), }; - let importer_indices: HashSet = self + let importer_indices: FxHashSet = self .imports .neighbors_directed(imported_index, Direction::Incoming) .collect(); - let importers: HashSet<&Module> = importer_indices + let importers: FxHashSet<&Module> = importer_indices .iter() .map(|importer_index| { self.imports_module_indices @@ -432,17 +435,17 @@ impl Graph { importers } - pub fn find_modules_directly_imported_by(&self, importer: &Module) -> HashSet<&Module> { + pub fn find_modules_directly_imported_by(&self, importer: &Module) -> FxHashSet<&Module> { let importer_index = match self.imports_module_indices.get_by_left(importer) { Some(importer_index) => *importer_index, - None => return HashSet::new(), + None => return FxHashSet::default(), }; - let imported_indices: HashSet = self + let imported_indices: FxHashSet = self .imports .neighbors_directed(importer_index, Direction::Outgoing) .collect(); - let importeds: HashSet<&Module> = imported_indices + let importeds: FxHashSet<&Module> = imported_indices .iter() .map(|imported_index| { self.imports_module_indices @@ -453,12 +456,14 @@ impl Graph { importeds } - pub fn find_upstream_modules(&self, module: &Module, as_package: bool) -> HashSet<&Module> { - let mut upstream_modules = HashSet::new(); + pub fn find_upstream_modules(&self, module: &Module, as_package: bool) -> FxHashSet<&Module> { + let mut upstream_modules = FxHashSet::default(); - let mut modules_to_check: HashSet<&Module> = HashSet::from([module]); + let mut modules_to_check: FxHashSet<&Module> = FxHashSet::from_iter([module]); if as_package { - let descendants = self.find_descendants(&module).unwrap_or(HashSet::new()); + let descendants = self + .find_descendants(&module) + .unwrap_or(FxHashSet::default()); modules_to_check.extend(descendants.into_iter()); }; @@ -479,12 +484,14 @@ impl Graph { upstream_modules } - pub fn find_downstream_modules(&self, module: &Module, as_package: bool) -> HashSet<&Module> { - let mut downstream_modules = HashSet::new(); + pub fn find_downstream_modules(&self, module: &Module, as_package: bool) -> FxHashSet<&Module> { + let mut downstream_modules = FxHashSet::default(); - let mut modules_to_check: HashSet<&Module> = HashSet::from([module]); + let mut modules_to_check: FxHashSet<&Module> = FxHashSet::from_iter([module]); if as_package { - let descendants = self.find_descendants(&module).unwrap_or(HashSet::new()); + let descendants = self + .find_descendants(&module) + .unwrap_or(FxHashSet::default()); modules_to_check.extend(descendants.into_iter()); }; @@ -551,12 +558,12 @@ impl Graph { importer: &Module, imported: &Module, as_packages: bool, - ) -> Result>, String> { - let mut chains = HashSet::new(); + ) -> Result>, String> { + let mut chains = FxHashSet::default(); let mut temp_graph = self.clone(); - let mut downstream_modules: HashSet = HashSet::from([importer.clone()]); - let mut upstream_modules: HashSet = HashSet::from([imported.clone()]); + let mut downstream_modules: FxHashSet = FxHashSet::from_iter([importer.clone()]); + let mut upstream_modules: FxHashSet = FxHashSet::from_iter([imported.clone()]); // TODO don't do this if module is squashed? if as_packages { @@ -597,9 +604,10 @@ impl Graph { } // Keep track of imports into/out of upstream/downstream packages, and remove them. - let mut map_of_imports: HashMap> = HashMap::new(); + let mut map_of_imports: FxHashMap> = + FxHashMap::default(); for module in upstream_modules.union(&downstream_modules) { - let mut imports_to_or_from_module = HashSet::new(); + let mut imports_to_or_from_module = FxHashSet::default(); for imported_module in temp_graph.find_modules_directly_imported_by(&module) { imports_to_or_from_module.insert((module.clone(), imported_module.clone())); } @@ -661,7 +669,7 @@ impl Graph { pub fn find_illegal_dependencies_for_layers( &self, levels: Vec, - containers: HashSet, + containers: FxHashSet, ) -> Result, NoSuchContainer> { // Check that containers exist. let modules = self.get_modules(); @@ -714,7 +722,7 @@ impl Graph { fn _generate_module_permutations( &self, levels: &Vec, - containers: &HashSet, + containers: &FxHashSet, ) -> Vec<(Module, Module, Option)> { let mut permutations: Vec<(Module, Module, Option)> = vec![]; @@ -869,9 +877,9 @@ impl Graph { } // Set up importer/imported package contents. - let mut importer_modules: HashSet<&Module> = HashSet::from([importer_package]); + let mut importer_modules: FxHashSet<&Module> = FxHashSet::from_iter([importer_package]); importer_modules.extend(self.find_descendants(&importer_package).unwrap()); - let mut imported_modules: HashSet<&Module> = HashSet::from([imported_package]); + let mut imported_modules: FxHashSet<&Module> = FxHashSet::from_iter([imported_package]); imported_modules.extend(self.find_descendants(&imported_package).unwrap()); // Build routes from middles. @@ -934,10 +942,10 @@ impl Graph { &mut self, lower_layer_module: &Module, higher_layer_module: &Module, - ) -> HashSet<(Module, Module)> { - let mut imports = HashSet::new(); + ) -> FxHashSet<(Module, Module)> { + let mut imports = FxHashSet::default(); - let mut lower_layer_modules = HashSet::from([lower_layer_module.clone()]); + let mut lower_layer_modules = FxHashSet::from_iter([lower_layer_module.clone()]); for descendant in self .find_descendants(lower_layer_module) .unwrap() @@ -947,7 +955,7 @@ impl Graph { lower_layer_modules.insert(descendant.clone()); } - let mut higher_layer_modules = HashSet::from([higher_layer_module.clone()]); + let mut higher_layer_modules = FxHashSet::from_iter([higher_layer_module.clone()]); for descendant in self .find_descendants(higher_layer_module) .unwrap() @@ -1050,7 +1058,7 @@ mod tests { fn modules_when_empty() { let graph = Graph::default(); - assert_eq!(graph.get_modules(), HashSet::new()); + assert_eq!(graph.get_modules(), FxHashSet::default()); } #[test] @@ -1069,7 +1077,7 @@ mod tests { let result = graph.get_modules(); - assert_eq!(result, HashSet::from([&mypackage])); + assert_eq!(result, FxHashSet::from_iter([&mypackage])); } #[test] @@ -1080,7 +1088,7 @@ mod tests { let result = graph.get_modules(); - assert_eq!(result, HashSet::from([&mypackage])); + assert_eq!(result, FxHashSet::from_iter([&mypackage])); } #[test] @@ -1093,7 +1101,7 @@ mod tests { let result = graph.get_modules(); - assert_eq!(result, HashSet::from([&mypackage, &mypackage_foo])); + assert_eq!(result, FxHashSet::from_iter([&mypackage, &mypackage_foo])); assert_eq!( graph.pretty_str(), " @@ -1117,7 +1125,7 @@ imports: graph.remove_module(&mypackage_foo); let result = graph.get_modules(); - assert_eq!(result, HashSet::from([&mypackage])); + assert_eq!(result, FxHashSet::from_iter([&mypackage])); } #[test] @@ -1136,7 +1144,7 @@ imports: let result = graph.get_modules(); assert_eq!( result, - HashSet::from([ + FxHashSet::from_iter([ &mypackage, &mypackage_foo_alpha, // To be consistent with previous versions of Grimp. ]) @@ -1162,7 +1170,7 @@ imports: let result = graph.get_modules(); assert_eq!( result, - HashSet::from([&mypackage, &mypackage_foo_alpha, &importer, &imported]) + FxHashSet::from_iter([&mypackage, &mypackage_foo_alpha, &importer, &imported]) ); assert_eq!( graph.direct_import_exists(&importer, &mypackage_foo, false), @@ -1190,7 +1198,7 @@ imports: assert_eq!( graph.get_import_details(&importer, &imported), - HashSet::new() + FxHashSet::default() ); } @@ -1210,7 +1218,7 @@ imports: assert_eq!( graph.get_import_details(&importer, &imported), - HashSet::new() + FxHashSet::default() ); } @@ -1229,7 +1237,10 @@ imports: false ); // ...but the modules are still there. - assert_eq!(graph.get_modules(), HashSet::from([&importer, &imported])); + assert_eq!( + graph.get_modules(), + FxHashSet::from_iter([&importer, &imported]) + ); } #[test] @@ -1243,7 +1254,10 @@ imports: graph.remove_import(&importer, &imported); // The modules are still there. - assert_eq!(graph.get_modules(), HashSet::from([&importer, &imported])); + assert_eq!( + graph.get_modules(), + FxHashSet::from_iter([&importer, &imported]) + ); } #[test] @@ -1336,7 +1350,7 @@ imports: graph.add_module(mypackage.clone()); graph.add_module(mypackage_foo.clone()); - assert_eq!(graph.find_children(&mypackage_foo), HashSet::new()); + assert_eq!(graph.find_children(&mypackage_foo), FxHashSet::default()); } #[test] @@ -1352,7 +1366,7 @@ imports: assert_eq!( graph.find_children(&mypackage), - HashSet::from([&mypackage_foo, &mypackage_bar]) + FxHashSet::from_iter([&mypackage_foo, &mypackage_bar]) ); } @@ -1369,7 +1383,7 @@ imports: assert_eq!( graph.find_children(&mypackage), - HashSet::from([&mypackage_foo, &mypackage_bar]) + FxHashSet::from_iter([&mypackage_foo, &mypackage_bar]) ); } @@ -1385,7 +1399,7 @@ imports: assert_eq!( graph.find_children(&Module::new("mypackage".to_string())), - HashSet::new() + FxHashSet::default() ); } @@ -1408,7 +1422,10 @@ imports: graph.add_module(mypackage_foo_alpha_green.clone()); graph.add_module(mypackage_foo_beta.clone()); - assert_eq!(graph.find_descendants(&mypackage_bar), Ok(HashSet::new())); + assert_eq!( + graph.find_descendants(&mypackage_bar), + Ok(FxHashSet::default()) + ); } #[test] @@ -1447,7 +1464,7 @@ imports: assert_eq!( graph.find_descendants(&mypackage_foo), - Ok(HashSet::from([ + Ok(FxHashSet::from_iter([ &mypackage_foo_alpha, &mypackage_foo_alpha_blue, &mypackage_foo_alpha_green, @@ -1479,7 +1496,7 @@ imports: assert_eq!( graph.find_descendants(&mypackage_foo), // mypackage.foo.blue is not included. - Ok(HashSet::from([ + Ok(FxHashSet::from_iter([ &mypackage_foo_blue_alpha, &mypackage_foo_blue_alpha_one, &mypackage_foo_blue_alpha_two, @@ -1512,7 +1529,7 @@ imports: assert_eq!( graph.find_descendants(&mypackage_foo), - Ok(HashSet::from([ + Ok(FxHashSet::from_iter([ &mypackage_foo_blue, // Should be included. &mypackage_foo_blue_alpha, &mypackage_foo_blue_alpha_one, @@ -1608,7 +1625,7 @@ imports: assert_eq!( graph.get_modules(), - HashSet::from([&mypackage_bar, &mypackage_foo]) + FxHashSet::from_iter([&mypackage_bar, &mypackage_foo]) ); assert!(graph.direct_import_exists(&mypackage_foo, &mypackage_bar, false)); assert_eq!( @@ -1635,7 +1652,7 @@ imports: assert_eq!( graph.get_modules(), - HashSet::from([&mypackage_bar, &mypackage_foo]) + FxHashSet::from_iter([&mypackage_bar, &mypackage_foo]) ); assert!(graph.direct_import_exists(&mypackage_foo, &mypackage_bar, false)); assert_eq!( @@ -1777,7 +1794,7 @@ imports: assert_eq!( result, - HashSet::from([&mypackage_foo_alpha, &anotherpackage]) + FxHashSet::from_iter([&mypackage_foo_alpha, &anotherpackage]) ) } @@ -1793,7 +1810,7 @@ imports: graph.remove_import(&green, &blue); let result = graph.find_modules_that_directly_import(&blue); - assert_eq!(result, HashSet::from([&yellow])) + assert_eq!(result, FxHashSet::from_iter([&yellow])) } #[test] @@ -1822,7 +1839,7 @@ imports: assert_eq!( result, - HashSet::from([&mypackage_foo_alpha, &anotherpackage]) + FxHashSet::from_iter([&mypackage_foo_alpha, &anotherpackage]) ) } @@ -1838,7 +1855,7 @@ imports: graph.remove_import(&blue, &green); let result = graph.find_modules_directly_imported_by(&blue); - assert_eq!(result, HashSet::from([&yellow])) + assert_eq!(result, FxHashSet::from_iter([&yellow])) } #[test] @@ -2059,7 +2076,10 @@ imports: let result = graph.find_upstream_modules(&blue, false); - assert_eq!(result, HashSet::from([&green, &red, &yellow, &purple])) + assert_eq!( + result, + FxHashSet::from_iter([&green, &red, &yellow, &purple]) + ) } #[test] @@ -2069,7 +2089,7 @@ imports: let result = graph.find_upstream_modules(&blue, false); - assert_eq!(result, HashSet::new()) + assert_eq!(result, FxHashSet::default()) } #[test] @@ -2106,7 +2126,10 @@ imports: let result = graph.find_upstream_modules(&blue, true); - assert_eq!(result, HashSet::from([&green, &yellow, &purple, &brown])) + assert_eq!( + result, + FxHashSet::from_iter([&green, &yellow, &purple, &brown]) + ) } #[test] @@ -2138,7 +2161,7 @@ imports: let result = graph.find_downstream_modules(&purple, false); - assert_eq!(result, HashSet::from([&yellow, &green, &blue])) + assert_eq!(result, FxHashSet::from_iter([&yellow, &green, &blue])) } #[test] @@ -2148,7 +2171,7 @@ imports: let result = graph.find_downstream_modules(&blue, false); - assert_eq!(result, HashSet::new()) + assert_eq!(result, FxHashSet::default()) } #[test] @@ -2185,7 +2208,10 @@ imports: let result = graph.find_downstream_modules(&blue, true); - assert_eq!(result, HashSet::from([&green, &yellow, &purple, &brown])) + assert_eq!( + result, + FxHashSet::from_iter([&green, &yellow, &purple, &brown]) + ) } // find_shortest_chain @@ -2336,7 +2362,7 @@ imports: let result = graph.find_shortest_chains(&blue, &green, true); - assert_eq!(result, Ok(HashSet::new())); + assert_eq!(result, Ok(FxHashSet::default())); } #[test] @@ -2361,7 +2387,7 @@ imports: let result = graph.find_shortest_chains(&blue, &green, true); - assert_eq!(result, Ok(HashSet::from([vec![blue, red, green],]))); + assert_eq!(result, Ok(FxHashSet::from_iter([vec![blue, red, green],]))); } #[test] @@ -2388,7 +2414,10 @@ imports: let result = graph.find_shortest_chains(&blue, &green, true); - assert_eq!(result, Ok(HashSet::from([vec![blue, red, green_alpha]]))); + assert_eq!( + result, + Ok(FxHashSet::from_iter([vec![blue, red, green_alpha]])) + ); } #[test] @@ -2419,7 +2448,7 @@ imports: assert_eq!( result, - Ok(HashSet::from([vec![blue, red, green_alpha_one],])) + Ok(FxHashSet::from_iter([vec![blue, red, green_alpha_one],])) ) } @@ -2447,7 +2476,10 @@ imports: let result = graph.find_shortest_chains(&blue, &green, true); - assert_eq!(result, Ok(HashSet::from([vec![blue_alpha, red, green],]))); + assert_eq!( + result, + Ok(FxHashSet::from_iter([vec![blue_alpha, red, green],])) + ); } #[test] @@ -2478,7 +2510,7 @@ imports: assert_eq!( result, - Ok(HashSet::from([vec![blue_alpha_one, red, green],])) + Ok(FxHashSet::from_iter([vec![blue_alpha_one, red, green],])) ) } @@ -2593,7 +2625,7 @@ imports: fn find_illegal_dependencies_for_layers_empty_everything() { let graph = Graph::default(); - let dependencies = graph.find_illegal_dependencies_for_layers(vec![], HashSet::new()); + let dependencies = graph.find_illegal_dependencies_for_layers(vec![], FxHashSet::default()); assert_eq!(dependencies, Ok(vec![])); } @@ -2603,8 +2635,10 @@ imports: let graph = Graph::default(); let container = "nonexistent_container".to_string(); - let dependencies = - graph.find_illegal_dependencies_for_layers(vec![], HashSet::from([container.clone()])); + let dependencies = graph.find_illegal_dependencies_for_layers( + vec![], + FxHashSet::from_iter([container.clone()]), + ); assert_eq!( dependencies, @@ -2622,7 +2656,8 @@ imports: independent: true, }; - let dependencies = graph.find_illegal_dependencies_for_layers(vec![level], HashSet::new()); + let dependencies = + graph.find_illegal_dependencies_for_layers(vec![level], FxHashSet::default()); assert_eq!(dependencies, Ok(vec![])); } @@ -2637,8 +2672,8 @@ imports: }; let container = "mypackage".to_string(); - let dependencies = - graph.find_illegal_dependencies_for_layers(vec![level], HashSet::from([container])); + let dependencies = graph + .find_illegal_dependencies_for_layers(vec![level], FxHashSet::from_iter([container])); assert_eq!(dependencies, Ok(vec![])); } @@ -2660,7 +2695,7 @@ imports: }, ]; - let dependencies = graph.find_illegal_dependencies_for_layers(levels, HashSet::new()); + let dependencies = graph.find_illegal_dependencies_for_layers(levels, FxHashSet::default()); assert_eq!( dependencies, @@ -2695,7 +2730,7 @@ imports: }, ]; - let dependencies = graph.find_illegal_dependencies_for_layers(levels, HashSet::new()); + let dependencies = graph.find_illegal_dependencies_for_layers(levels, FxHashSet::default()); assert_eq!( dependencies, @@ -2741,7 +2776,7 @@ imports: independent: true, }, ]; - let containers = HashSet::from(["blue".to_string(), "green".to_string()]); + let containers = FxHashSet::from_iter(["blue".to_string(), "green".to_string()]); let dependencies = graph.find_illegal_dependencies_for_layers(levels, containers); @@ -2786,7 +2821,7 @@ imports: independent: true, }]; - let dependencies = graph.find_illegal_dependencies_for_layers(levels, HashSet::new()); + let dependencies = graph.find_illegal_dependencies_for_layers(levels, FxHashSet::default()); assert_eq!( dependencies, @@ -2810,7 +2845,7 @@ imports: let result = graph.get_import_details(&importer, &imported); - assert_eq!(result, HashSet::new()); + assert_eq!(result, FxHashSet::default()); } #[test] @@ -2822,7 +2857,7 @@ imports: let result = graph.get_import_details(&importer, &imported); - assert_eq!(result, HashSet::new()); + assert_eq!(result, FxHashSet::default()); } #[test] @@ -2847,7 +2882,7 @@ imports: let result = graph.get_import_details(&importer, &imported); - assert_eq!(result, HashSet::from([import])); + assert_eq!(result, FxHashSet::from_iter([import])); } #[test] @@ -2872,7 +2907,10 @@ imports: let result = graph.get_import_details(&blue, &green); - assert_eq!(result, HashSet::from([blue_to_green_a, blue_to_green_b])); + assert_eq!( + result, + FxHashSet::from_iter([blue_to_green_a, blue_to_green_b]) + ); } #[test] @@ -2898,7 +2936,7 @@ imports: let result = graph.get_import_details(&importer, &imported); - assert_eq!(result, HashSet::new()); + assert_eq!(result, FxHashSet::default()); } #[test] @@ -2924,6 +2962,6 @@ imports: let result = graph.get_import_details(&importer, &imported); - assert_eq!(result, HashSet::from([import])); + assert_eq!(result, FxHashSet::from_iter([import])); } } diff --git a/rust/src/lib.rs b/rust/src/lib.rs index 112dbb0e..10d52631 100644 --- a/rust/src/lib.rs +++ b/rust/src/lib.rs @@ -6,7 +6,7 @@ use pyo3::create_exception; use pyo3::exceptions::PyValueError; use pyo3::prelude::*; use pyo3::types::{PyDict, PyFrozenSet, PyList, PySet, PyString, PyTuple}; -use std::collections::HashSet; +use rustc_hash::FxHashSet; #[pymodule] fn _rustgrimp(py: Python<'_>, m: &Bound<'_, PyModule>) -> PyResult<()> { @@ -34,7 +34,7 @@ impl GraphWrapper { } } - pub fn get_modules(&self) -> HashSet { + pub fn get_modules(&self) -> FxHashSet { self._graph .get_modules() .iter() @@ -125,7 +125,7 @@ impl GraphWrapper { self._graph.count_imports() } - pub fn find_children(&self, module: &str) -> HashSet { + pub fn find_children(&self, module: &str) -> FxHashSet { self._graph .find_children(&Module::new(module.to_string())) .iter() @@ -133,7 +133,7 @@ impl GraphWrapper { .collect() } - pub fn find_descendants(&self, module: &str) -> HashSet { + pub fn find_descendants(&self, module: &str) -> FxHashSet { self._graph .find_descendants(&Module::new(module.to_string())) .unwrap() @@ -169,7 +169,7 @@ impl GraphWrapper { )) } - pub fn find_modules_directly_imported_by(&self, module: &str) -> HashSet { + pub fn find_modules_directly_imported_by(&self, module: &str) -> FxHashSet { self._graph .find_modules_directly_imported_by(&Module::new(module.to_string())) .iter() @@ -177,7 +177,7 @@ impl GraphWrapper { .collect() } - pub fn find_modules_that_directly_import(&self, module: &str) -> HashSet { + pub fn find_modules_that_directly_import(&self, module: &str) -> FxHashSet { self._graph .find_modules_that_directly_import(&Module::new(module.to_string())) .iter() @@ -226,7 +226,7 @@ impl GraphWrapper { #[allow(unused_variables)] #[pyo3(signature = (module, as_package=false))] - pub fn find_downstream_modules(&self, module: &str, as_package: bool) -> HashSet { + pub fn find_downstream_modules(&self, module: &str, as_package: bool) -> FxHashSet { // Turn the Modules to Strings. self._graph .find_downstream_modules(&Module::new(module.to_string()), as_package) @@ -237,7 +237,7 @@ impl GraphWrapper { #[allow(unused_variables)] #[pyo3(signature = (module, as_package=false))] - pub fn find_upstream_modules(&self, module: &str, as_package: bool) -> HashSet { + pub fn find_upstream_modules(&self, module: &str, as_package: bool) -> FxHashSet { self._graph .find_upstream_modules(&Module::new(module.to_string()), as_package) .iter() @@ -262,7 +262,7 @@ impl GraphWrapper { imported: &str, as_packages: bool, ) -> PyResult> { - let rust_chains: HashSet> = self + let rust_chains: FxHashSet> = self ._graph .find_shortest_chains( &Module::new(importer.to_string()), @@ -308,18 +308,16 @@ impl GraphWrapper { )) } - #[allow(unused_variables)] #[pyo3(signature = (layers, containers))] pub fn find_illegal_dependencies_for_layers<'py>( &self, py: Python<'py>, layers: &Bound<'py, PyTuple>, - containers: HashSet, + containers: FxHashSet, ) -> PyResult> { info!("Using Rust to find illegal dependencies."); let levels = rustify_levels(layers); - println!("\nIncoming {:?}, {:?}", levels, containers); let dependencies = py.allow_threads(|| { self._graph .find_illegal_dependencies_for_layers(levels, containers) @@ -343,7 +341,7 @@ fn rustify_levels<'a>(levels_python: &Bound<'a, PyTuple>) -> Vec { let mut rust_levels: Vec = vec![]; for level_python in levels_python.into_iter() { let level_dict = level_python.downcast::().unwrap(); - let layers: HashSet = level_dict + let layers: FxHashSet = level_dict .get_item("layers") .unwrap() .unwrap() @@ -431,15 +429,15 @@ mod tests { let elements = vec![ pydict! (py, { "independent" => true, - "layers" => HashSet::from(["high"]), + "layers" => FxHashSet::from_iter(["high"]), }), pydict! (py, { "independent" => true, - "layers" => HashSet::from(["medium"]), + "layers" => FxHashSet::from_iter(["medium"]), }), pydict! (py, { "independent" => true, - "layers" => HashSet::from(["low"]), + "layers" => FxHashSet::from_iter(["low"]), }), ]; let python_levels = PyTuple::new(py, elements)?; @@ -476,19 +474,19 @@ mod tests { let elements = vec![ pydict! (py, { "independent" => true, - "layers" => HashSet::from(["high"]), + "layers" => FxHashSet::from_iter(["high"]), }), pydict! (py, { "independent" => true, - "layers" => HashSet::from(["blue", "green", "orange"]), + "layers" => FxHashSet::from_iter(["blue", "green", "orange"]), }), pydict! (py, { "independent" => false, - "layers" => HashSet::from(["red", "yellow"]), + "layers" => FxHashSet::from_iter(["red", "yellow"]), }), pydict! (py, { "independent" => true, - "layers" => HashSet::from(["low"]), + "layers" => FxHashSet::from_iter(["low"]), }), ]; let python_levels = PyTuple::new(py, elements)?; diff --git a/rust/tests/large.rs b/rust/tests/large.rs index 1cdaf9ab..5eaca91b 100644 --- a/rust/tests/large.rs +++ b/rust/tests/large.rs @@ -1,6 +1,6 @@ use _rustgrimp::graph::{Graph, Level, Module}; +use rustc_hash::FxHashSet; use serde_json::{Map, Value}; -use std::collections::HashSet; use std::fs; #[test] @@ -40,7 +40,7 @@ fn test_large_graph_deep_layers() { layers: vec![layer.to_string()], }) .collect(); - let containers = HashSet::new(); + let containers = FxHashSet::default(); let deps = graph .find_illegal_dependencies_for_layers(levels, containers) From 631a5efd9713f161f57e5664d7021ab4e09c6965 Mon Sep 17 00:00:00 2001 From: Peter Byfield Date: Wed, 22 Jan 2025 09:43:43 +0100 Subject: [PATCH 06/10] Do not set number of rounds for local benchmark pytest-benchmark will actually determine a suitable number of rounds+iterations automatically (typically much more than 3!). With only 3 rounds the results are quite noisy. --- tests/benchmarking/test_benchmarking.py | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/tests/benchmarking/test_benchmarking.py b/tests/benchmarking/test_benchmarking.py index e5cdc964..4cedd334 100644 --- a/tests/benchmarking/test_benchmarking.py +++ b/tests/benchmarking/test_benchmarking.py @@ -8,12 +8,7 @@ def _run_benchmark(benchmark, fn, *args, **kwargs): - if hasattr(benchmark, "pedantic"): - # Running with pytest-benchmark - return benchmark.pedantic(fn, args=args, kwargs=kwargs, rounds=3) - else: - # Running with codspeed. - return benchmark(fn, *args, **kwargs) + return benchmark(fn, *args, **kwargs) @pytest.fixture(scope="module") From 1452847c06bed7f57942a50e9cf4b4fec2dd0584 Mon Sep 17 00:00:00 2001 From: Peter Byfield Date: Fri, 17 Jan 2025 15:37:36 +0100 Subject: [PATCH 07/10] Reimplement import graph --- rust/Cargo.lock | 171 +- rust/Cargo.toml | 16 +- rust/src/errors.rs | 29 + rust/src/exceptions.rs | 4 + rust/src/graph.rs | 2967 ----------------------- rust/src/graph/direct_import_queries.rs | 57 + rust/src/graph/graph_manipulation.rs | 195 ++ rust/src/graph/hierarchy_queries.rs | 50 + rust/src/graph/higher_order_queries.rs | 210 ++ rust/src/graph/import_chain_queries.rs | 123 + rust/src/graph/mod.rs | 137 ++ rust/src/graph/pathfinding.rs | 125 + rust/src/lib.rs | 777 +++--- rust/tests/large.rs | 36 +- src/grimp/adaptors/graph.py | 13 +- src/grimp/application/ports/graph.py | 7 + 16 files changed, 1517 insertions(+), 3400 deletions(-) create mode 100644 rust/src/errors.rs create mode 100644 rust/src/exceptions.rs delete mode 100644 rust/src/graph.rs create mode 100644 rust/src/graph/direct_import_queries.rs create mode 100644 rust/src/graph/graph_manipulation.rs create mode 100644 rust/src/graph/hierarchy_queries.rs create mode 100644 rust/src/graph/higher_order_queries.rs create mode 100644 rust/src/graph/import_chain_queries.rs create mode 100644 rust/src/graph/mod.rs create mode 100644 rust/src/graph/pathfinding.rs diff --git a/rust/Cargo.lock b/rust/Cargo.lock index 9773fef5..05270e54 100644 --- a/rust/Cargo.lock +++ b/rust/Cargo.lock @@ -1,27 +1,27 @@ # This file is automatically @generated by Cargo. # It is not intended for manual editing. -version = 3 +version = 4 [[package]] name = "_rustgrimp" version = "0.1.0" dependencies = [ "bimap", - "log", - "petgraph", + "derive-new", + "getset", + "indexmap", + "itertools", + "lazy_static", "pyo3", - "pyo3-log", "rayon", "rustc-hash", "serde_json", + "slotmap", + "string-interner", + "tap", + "thiserror", ] -[[package]] -name = "arc-swap" -version = "1.7.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "69f7f8c3906b62b754cd5326047894316021dcfe5a194c8ea52bdd94934a3457" - [[package]] name = "autocfg" version = "1.4.0" @@ -65,6 +65,17 @@ version = "0.8.21" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d0a5c400df2834b80a4c3327b3aad3a4c4cd4de0629063962b03235697506a28" +[[package]] +name = "derive-new" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2cdc8d50f426189eef89dac62fabfa0abb27d5cc008f25bf4156a0203325becc" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "either" version = "1.13.0" @@ -78,16 +89,31 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5443807d6dff69373d433ab9ef5378ad8df50ca6298caf15de6e52e24aaf54d5" [[package]] -name = "fixedbitset" -version = "0.4.2" +name = "foldhash" +version = "0.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0ce7134b9999ecaf8bcd65542e436736ef32ddca1b3e06094cb6ec5755203b80" +checksum = "a0d2fde1f7b3d48b8395d5f2de76c18a528bd6a9cdde438df747bfcba3e05d6f" + +[[package]] +name = "getset" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f636605b743120a8d32ed92fc27b6cde1a769f8f936c065151eb66f88ded513c" +dependencies = [ + "proc-macro-error2", + "proc-macro2", + "quote", + "syn", +] [[package]] name = "hashbrown" version = "0.15.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bf151400ff0baff5465007dd2f3e717f3fe502074ca563069ce3a6629d07b289" +dependencies = [ + "foldhash", +] [[package]] name = "heck" @@ -97,9 +123,9 @@ checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" [[package]] name = "indexmap" -version = "2.7.0" +version = "2.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "62f822373a4fe84d4bb149bf54e584a7f4abec90e072ed49cda0edea5b95471f" +checksum = "8c9c992b02b5b4c94ea26e32fe5bccb7aa7d9f390ab5c1221ff895bc7ea8b652" dependencies = [ "equivalent", "hashbrown", @@ -111,6 +137,15 @@ version = "2.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b248f5224d1d606005e02c97f5aa4e88eeb230488bcc03bc9ca4d7991399f2b5" +[[package]] +name = "itertools" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b192c782037fadd9cfa75548310488aabdbf3d2da73885b31bd0abd03351285" +dependencies = [ + "either", +] + [[package]] name = "itoa" version = "1.0.14" @@ -118,16 +153,16 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d75a2a4b1b190afb6f5425f10f6a8f959d2ea0b9c2b1d79553551850539e4674" [[package]] -name = "libc" -version = "0.2.169" +name = "lazy_static" +version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b5aba8db14291edd000dfcc4d620c7ebfb122c613afb886ca8803fa4e128a20a" +checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" [[package]] -name = "log" -version = "0.4.25" +name = "libc" +version = "0.2.169" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "04cbf5b083de1c7e0222a7a51dbfdba1cbe1c6ab0b15e29fff3f6c077fd9cd9f" +checksum = "b5aba8db14291edd000dfcc4d620c7ebfb122c613afb886ca8803fa4e128a20a" [[package]] name = "memchr" @@ -151,20 +186,32 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1261fe7e33c73b354eab43b1273a57c8f967d0391e80353e51f764ac02cf6775" [[package]] -name = "petgraph" -version = "0.6.5" +name = "portable-atomic" +version = "1.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "280dc24453071f1b63954171985a0b0d30058d287960968b9b2aca264c8d4ee6" + +[[package]] +name = "proc-macro-error-attr2" +version = "2.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b4c5cc86750666a3ed20bdaf5ca2a0344f9c67674cae0515bec2da16fbaa47db" +checksum = "96de42df36bb9bba5542fe9f1a054b8cc87e172759a1868aa05c1f3acc89dfc5" dependencies = [ - "fixedbitset", - "indexmap", + "proc-macro2", + "quote", ] [[package]] -name = "portable-atomic" -version = "1.10.0" +name = "proc-macro-error2" +version = "2.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "280dc24453071f1b63954171985a0b0d30058d287960968b9b2aca264c8d4ee6" +checksum = "11ec05c52be0a07b08061f7dd003e7d7092e0472bc731b4af7bb1ef876109802" +dependencies = [ + "proc-macro-error-attr2", + "proc-macro2", + "quote", + "syn", +] [[package]] name = "proc-macro2" @@ -213,17 +260,6 @@ dependencies = [ "pyo3-build-config", ] -[[package]] -name = "pyo3-log" -version = "0.12.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "be5bb22b77965a7b5394e9aae9897a0607b51df5167561ffc3b02643b4200bc7" -dependencies = [ - "arc-swap", - "log", - "pyo3", -] - [[package]] name = "pyo3-macros" version = "0.23.4" @@ -312,9 +348,9 @@ dependencies = [ [[package]] name = "serde_json" -version = "1.0.135" +version = "1.0.137" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2b0d7ba2887406110130a978386c4e1befb98c674b4fba677954e4db976630d9" +checksum = "930cfb6e6abf99298aaad7d29abbef7a9999a9a8806a40088f55f0dcec03146b" dependencies = [ "itoa", "memchr", @@ -322,6 +358,25 @@ dependencies = [ "serde", ] +[[package]] +name = "slotmap" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dbff4acf519f630b3a3ddcfaea6c06b42174d9a44bc70c620e9ed1649d58b82a" +dependencies = [ + "version_check", +] + +[[package]] +name = "string-interner" +version = "0.18.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a3275464d7a9f2d4cac57c89c2ef96a8524dba2864c8d6f82e3980baf136f9b" +dependencies = [ + "hashbrown", + "serde", +] + [[package]] name = "syn" version = "2.0.96" @@ -333,12 +388,38 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "tap" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "55937e1799185b12863d447f42597ed69d9928686b8d88a1df17376a097d8369" + [[package]] name = "target-lexicon" version = "0.12.16" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "61c41af27dd6d1e27b1b16b489db798443478cef1f06a660c96db617ba5de3b1" +[[package]] +name = "thiserror" +version = "2.0.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d452f284b73e6d76dd36758a0c8684b1d5be31f92b89d07fd5822175732206fc" +dependencies = [ + "thiserror-impl", +] + +[[package]] +name = "thiserror-impl" +version = "2.0.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "26afc1baea8a989337eeb52b6e72a039780ce45c3edfcc9c5b9d112feeb173c2" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "unicode-ident" version = "1.0.14" @@ -350,3 +431,9 @@ name = "unindent" version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c7de7d73e1754487cb58364ee906a499937a0dfabd86bcb980fa99ec8c8fa2ce" + +[[package]] +name = "version_check" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" diff --git a/rust/Cargo.toml b/rust/Cargo.toml index 00f48bc5..048d453e 100644 --- a/rust/Cargo.toml +++ b/rust/Cargo.toml @@ -8,13 +8,18 @@ name = "_rustgrimp" crate-type = ["cdylib", "rlib"] [dependencies] -log = "0.4.19" -pyo3-log = "0.12.1" -serde_json = "1.0.103" rayon = "1.10" -petgraph = "0.6.5" bimap = "0.6.3" +slotmap = "1.0.7" +getset = "0.1.3" +derive-new = "0.7.0" +lazy_static = "1.5.0" +string-interner = "0.18.0" +thiserror = "2.0.11" +itertools = "0.14.0" +tap = "1.0.1" rustc-hash = "2.1.0" +indexmap = "2.7.1" [dependencies.pyo3] version = "0.23.4" @@ -22,3 +27,6 @@ version = "0.23.4" [features] extension-module = ["pyo3/extension-module"] default = ["extension-module"] + +[dev-dependencies] +serde_json = "1.0.137" diff --git a/rust/src/errors.rs b/rust/src/errors.rs new file mode 100644 index 00000000..e1ac1ef9 --- /dev/null +++ b/rust/src/errors.rs @@ -0,0 +1,29 @@ +use crate::exceptions::{ModuleNotPresent, NoSuchContainer}; +use pyo3::exceptions::PyValueError; +use pyo3::PyErr; +use thiserror::Error; + +#[derive(Debug, Error)] +pub enum GrimpError { + #[error("Module {0} is not present in the graph.")] + ModuleNotPresent(String), + + #[error("Container {0} does not exist.")] + NoSuchContainer(String), + + #[error("Modules have shared descendants.")] + SharedDescendants, +} + +pub type GrimpResult = Result; + +impl From for PyErr { + fn from(value: GrimpError) -> Self { + // A default mapping from `GrimpError`s to python exceptions. + match value { + GrimpError::ModuleNotPresent(_) => ModuleNotPresent::new_err(value.to_string()), + GrimpError::NoSuchContainer(_) => NoSuchContainer::new_err(value.to_string()), + GrimpError::SharedDescendants => PyValueError::new_err(value.to_string()), + } + } +} diff --git a/rust/src/exceptions.rs b/rust/src/exceptions.rs new file mode 100644 index 00000000..a8aa3e74 --- /dev/null +++ b/rust/src/exceptions.rs @@ -0,0 +1,4 @@ +use pyo3::create_exception; + +create_exception!(_rustgrimp, ModuleNotPresent, pyo3::exceptions::PyException); +create_exception!(_rustgrimp, NoSuchContainer, pyo3::exceptions::PyException); diff --git a/rust/src/graph.rs b/rust/src/graph.rs deleted file mode 100644 index 248e84d6..00000000 --- a/rust/src/graph.rs +++ /dev/null @@ -1,2967 +0,0 @@ -use bimap::BiMap; -use log::info; -use petgraph::algo::astar; -use petgraph::graph::EdgeIndex; -use petgraph::stable_graph::{NodeIndex, StableGraph}; -use petgraph::visit::{Bfs, Walker}; -use petgraph::Direction; -use rayon::prelude::*; -use rustc_hash::{FxHashMap, FxHashSet}; -use std::fmt; -use std::time::Instant; - -/// A group of layers at the same level in the layering. -#[derive(PartialEq, Eq, Hash, Debug)] -pub struct Level { - pub layers: Vec, - pub independent: bool, -} - -// Delimiter for Python modules. -const DELIMITER: char = '.'; - -#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)] -pub struct Module { - pub name: String, -} - -impl fmt::Display for Module { - fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { - write!(f, "{}", self.name) - } -} - -#[derive(Debug, Clone, PartialEq)] -pub struct ModuleNotPresent { - pub module: Module, -} - -#[derive(Debug, Clone, PartialEq)] -pub struct NoSuchContainer { - pub container: String, -} - -pub struct ModulesHaveSharedDescendants {} - -impl fmt::Display for ModuleNotPresent { - fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { - write!(f, "\"{}\" not present in the graph", self.module.name) - } -} - -impl Module { - pub fn new(name: String) -> Module { - Module { name } - } - - // Returns whether the module is a root-level package. - pub fn is_root(&self) -> bool { - !self.name.contains(DELIMITER) - } - - // Create a Module that is the parent of the passed Module. - // - // Panics if the child is a root Module. - pub fn new_parent(child: &Module) -> Module { - let parent_name = match child.name.rsplit_once(DELIMITER) { - Some((base, _)) => base.to_string(), - None => panic!("{} is a root level package", child.name), - }; - - Module::new(parent_name) - } - - // Return whether this module is a descendant of the supplied one, based on the name. - pub fn is_descendant_of(&self, module: &Module) -> bool { - let candidate = format!("{}{}", module.name, DELIMITER); - self.name.starts_with(&candidate) - } -} - -#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)] -pub struct DetailedImport { - pub importer: Module, - pub imported: Module, - pub line_number: usize, - pub line_contents: String, -} - -#[derive(Default, Clone)] -pub struct Graph { - // Bidirectional lookup between Module and NodeIndex. - hierarchy_module_indices: BiMap, - hierarchy: StableGraph, - imports_module_indices: BiMap, - imports: StableGraph, - squashed_modules: FxHashSet, - // Invisible modules exist in the hierarchy but haven't been explicitly added to the graph. - invisible_modules: FxHashSet, - detailed_imports_map: FxHashMap<(Module, Module), FxHashSet>, -} - -#[derive(PartialEq, Eq, Hash, Debug, PartialOrd, Ord)] -pub struct Route { - pub heads: Vec, - pub middle: Vec, - pub tails: Vec, -} - -#[derive(PartialEq, Eq, Hash, Debug, PartialOrd, Ord)] -pub struct PackageDependency { - pub importer: Module, - pub imported: Module, - pub routes: Vec, -} - -fn _module_from_layer(layer: &str, container: &Option) -> Module { - let module_name = match container { - Some(container) => format!("{}{}{}", container.name, DELIMITER, layer), - None => layer.to_string(), - }; - Module::new(module_name) -} - -fn _log_illegal_route_count(dependency_or_none: &Option, duration_in_s: u64) { - let route_count = match dependency_or_none { - Some(dependency) => dependency.routes.len(), - None => 0, - }; - let pluralized = if route_count == 1 { "" } else { "s" }; - info!( - "Found {} illegal route{} in {}s.", - route_count, pluralized, duration_in_s - ); -} - -impl Graph { - pub fn pretty_str(&self) -> String { - let mut hierarchy: Vec = vec![]; - let mut imports: Vec = vec![]; - - let hierarchy_module_indices: Vec<_> = self.hierarchy_module_indices.iter().collect(); - - for (parent_module, parent_index) in hierarchy_module_indices { - for child_index in self.hierarchy.neighbors(*parent_index) { - let child_module = self - .hierarchy_module_indices - .get_by_right(&child_index) - .unwrap(); - let parent_module_str = match self.invisible_modules.contains(&parent_module) { - true => format!("({})", parent_module.name), - false => parent_module.name.to_string(), - }; - let child_module_str = match self.invisible_modules.contains(&child_module) { - true => format!("({})", child_module.name), - false => child_module.name.to_string(), - }; - hierarchy.push(format!(" {} -> {}", parent_module_str, child_module_str)); - } - } - - let imports_module_indices: Vec<_> = self.imports_module_indices.iter().collect(); - - for (from_module, from_index) in imports_module_indices { - for to_index in self.imports.neighbors(*from_index) { - let to_module = self.imports_module_indices.get_by_right(&to_index).unwrap(); - imports.push(format!(" {} -> {}", from_module.name, to_module.name)); - } - } - // Assemble String. - let mut pretty = String::new(); - pretty.push_str("hierarchy:\n"); - hierarchy.sort(); - pretty.push_str(&hierarchy.join("\n")); - pretty.push_str("\nimports:\n"); - imports.sort(); - pretty.push_str(&imports.join("\n")); - pretty.push('\n'); - pretty - } - - pub fn add_module(&mut self, module: Module) { - // If this module is already in the graph, but invisible, just make it visible. - if self.invisible_modules.contains(&module) { - self.invisible_modules.remove(&module); - return; - } - // If this module is already in the graph, don't do anything. - if self.hierarchy_module_indices.get_by_left(&module).is_some() { - return; - } - - let module_index = self.hierarchy.add_node(module.clone()); - self.hierarchy_module_indices - .insert(module.clone(), module_index); - - // Add this module to the hierarchy. - if !module.is_root() { - let parent = Module::new_parent(&module); - - let parent_index = match self.hierarchy_module_indices.get_by_left(&parent) { - Some(index) => index, - None => { - // If the parent isn't already in the graph, add it, but as an invisible module. - self.add_module(parent.clone()); - self.invisible_modules.insert(parent.clone()); - self.hierarchy_module_indices.get_by_left(&parent).unwrap() - } - }; - self.hierarchy.add_edge(*parent_index, module_index, ()); - } - } - - pub fn add_squashed_module(&mut self, module: Module) { - self.add_module(module.clone()); - self.squashed_modules.insert(module); - } - - pub fn remove_module(&mut self, module: &Module) { - // Remove imports by module. - let imported_modules: Vec = self - .find_modules_directly_imported_by(module) - .iter() - .map(|m| (*m).clone()) - .collect(); - for imported_module in imported_modules { - self.remove_import(&module, &imported_module); - } - - // Remove imports of module. - let importer_modules: Vec = self - .find_modules_that_directly_import(module) - .iter() - .map(|m| (*m).clone()) - .collect(); - for importer_module in importer_modules { - self.remove_import(&importer_module, &module); - } - - // Remove module from hierarchy. - if let Some(hierarchy_index) = self.hierarchy_module_indices.get_by_left(module) { - // TODO should we check for children before removing? - // Maybe should just make invisible instead? - self.hierarchy.remove_node(*hierarchy_index); - self.hierarchy_module_indices.remove_by_left(module); - }; - } - - pub fn get_modules(&self) -> FxHashSet<&Module> { - self.hierarchy_module_indices - .left_values() - .filter(|module| !self.invisible_modules.contains(module)) - .collect() - } - - pub fn count_imports(&self) -> usize { - self.imports.edge_count() - } - - pub fn get_import_details( - &self, - importer: &Module, - imported: &Module, - ) -> FxHashSet { - let key = (importer.clone(), imported.clone()); - match self.detailed_imports_map.get(&key) { - Some(import_details) => import_details.clone(), - None => FxHashSet::default(), - } - } - - pub fn find_children(&self, module: &Module) -> FxHashSet<&Module> { - if self.invisible_modules.contains(module) { - return FxHashSet::default(); - } - let module_index = match self.hierarchy_module_indices.get_by_left(module) { - Some(index) => index, - // Module does not exist. - // TODO: should this return a result, to handle if module is not in graph? - None => return FxHashSet::default(), - }; - self.hierarchy - .neighbors(*module_index) - .map(|index| self.hierarchy_module_indices.get_by_right(&index).unwrap()) - .filter(|module| !self.invisible_modules.contains(module)) - .collect() - } - - pub fn find_descendants( - &self, - module: &Module, - ) -> Result, ModuleNotPresent> { - let module_index = match self.hierarchy_module_indices.get_by_left(module) { - Some(index) => index, - None => { - return Err(ModuleNotPresent { - module: module.clone(), - }) - } - }; - Ok(Bfs::new(&self.hierarchy, *module_index) - .iter(&self.hierarchy) - .filter(|index| index != module_index) // Don't include the supplied module. - .map(|index| self.hierarchy_module_indices.get_by_right(&index).unwrap()) // This panics sometimes. - .filter(|module| !self.invisible_modules.contains(module)) - .collect()) - } - - pub fn add_import(&mut self, importer: &Module, imported: &Module) { - // Don't bother doing anything if it's already in the graph. - if self.direct_import_exists(&importer, &imported, false) { - return; - } - - self.add_module_if_not_in_hierarchy(importer); - self.add_module_if_not_in_hierarchy(imported); - - let importer_index: NodeIndex = match self.imports_module_indices.get_by_left(importer) { - Some(index) => *index, - None => { - let index = self.imports.add_node(importer.clone()); - self.imports_module_indices.insert(importer.clone(), index); - index - } - }; - let imported_index: NodeIndex = match self.imports_module_indices.get_by_left(imported) { - Some(index) => *index, - None => { - let index = self.imports.add_node(imported.clone()); - self.imports_module_indices.insert(imported.clone(), index); - index - } - }; - - self.imports.add_edge(importer_index, imported_index, ()); - // println!( - // "Added {:?} {:?} -> {:?} {:?}, edge count now {:?}", - // importer, - // importer_index, - // imported, - // imported_index, - // self.imports.edge_count() - // ); - } - - pub fn add_detailed_import(&mut self, import: &DetailedImport) { - let key = (import.importer.clone(), import.imported.clone()); - self.detailed_imports_map - .entry(key) - .or_insert_with(FxHashSet::default) - .insert(import.clone()); - self.add_import(&import.importer, &import.imported); - } - - pub fn remove_import(&mut self, importer: &Module, imported: &Module) { - let importer_index: NodeIndex = match self.imports_module_indices.get_by_left(importer) { - Some(index) => *index, - None => return, - }; - let imported_index: NodeIndex = match self.imports_module_indices.get_by_left(imported) { - Some(index) => *index, - None => return, - }; - let edge_index: EdgeIndex = match self.imports.find_edge(importer_index, imported_index) { - Some(index) => index, - None => return, - }; - - self.imports.remove_edge(edge_index); - - // There might be other imports to / from the modules, so don't - // remove from the indices. (TODO: does it matter if we don't clean these up - // if there are no more imports?) - // self.imports_module_indices.remove_by_left(importer); - // self.imports_module_indices.remove_by_left(importer); - - let key = (importer.clone(), imported.clone()); - - self.detailed_imports_map.remove(&key); - self.imports.remove_edge(edge_index); - } - - // Note: this will panic if importer and imported are in the same package. - pub fn direct_import_exists( - &self, - importer: &Module, - imported: &Module, - as_packages: bool, - ) -> bool { - let graph_to_use: &Graph; - let mut graph_copy: Graph; - - if as_packages { - graph_copy = self.clone(); - graph_copy.squash_module(importer); - graph_copy.squash_module(imported); - graph_to_use = &graph_copy; - } else { - graph_to_use = self; - } - - // The modules may appear in the hierarchy, but have no imports, so we - // return false unless they're both in there. - let importer_index = match graph_to_use.imports_module_indices.get_by_left(importer) { - Some(importer_index) => *importer_index, - None => return false, - }; - let imported_index = match graph_to_use.imports_module_indices.get_by_left(imported) { - Some(imported_index) => *imported_index, - None => return false, - }; - - graph_to_use - .imports - .contains_edge(importer_index, imported_index) - } - - pub fn find_modules_that_directly_import(&self, imported: &Module) -> FxHashSet<&Module> { - let imported_index = match self.imports_module_indices.get_by_left(imported) { - Some(imported_index) => *imported_index, - None => return FxHashSet::default(), - }; - let importer_indices: FxHashSet = self - .imports - .neighbors_directed(imported_index, Direction::Incoming) - .collect(); - - let importers: FxHashSet<&Module> = importer_indices - .iter() - .map(|importer_index| { - self.imports_module_indices - .get_by_right(importer_index) - .unwrap() - }) - .collect(); - importers - } - - pub fn find_modules_directly_imported_by(&self, importer: &Module) -> FxHashSet<&Module> { - let importer_index = match self.imports_module_indices.get_by_left(importer) { - Some(importer_index) => *importer_index, - None => return FxHashSet::default(), - }; - let imported_indices: FxHashSet = self - .imports - .neighbors_directed(importer_index, Direction::Outgoing) - .collect(); - - let importeds: FxHashSet<&Module> = imported_indices - .iter() - .map(|imported_index| { - self.imports_module_indices - .get_by_right(imported_index) - .unwrap() - }) - .collect(); - importeds - } - - pub fn find_upstream_modules(&self, module: &Module, as_package: bool) -> FxHashSet<&Module> { - let mut upstream_modules = FxHashSet::default(); - - let mut modules_to_check: FxHashSet<&Module> = FxHashSet::from_iter([module]); - if as_package { - let descendants = self - .find_descendants(&module) - .unwrap_or(FxHashSet::default()); - modules_to_check.extend(descendants.into_iter()); - }; - - for module_to_check in modules_to_check.iter() { - let module_index = match self.imports_module_indices.get_by_left(module_to_check) { - Some(index) => *index, - None => continue, - }; - upstream_modules.extend( - Bfs::new(&self.imports, module_index) - .iter(&self.imports) - .map(|index| self.imports_module_indices.get_by_right(&index).unwrap()) - // Exclude any modules that we are checking. - .filter(|downstream_module| !modules_to_check.contains(downstream_module)), - ); - } - - upstream_modules - } - - pub fn find_downstream_modules(&self, module: &Module, as_package: bool) -> FxHashSet<&Module> { - let mut downstream_modules = FxHashSet::default(); - - let mut modules_to_check: FxHashSet<&Module> = FxHashSet::from_iter([module]); - if as_package { - let descendants = self - .find_descendants(&module) - .unwrap_or(FxHashSet::default()); - modules_to_check.extend(descendants.into_iter()); - }; - - for module_to_check in modules_to_check.iter() { - let module_index = match self.imports_module_indices.get_by_left(module_to_check) { - Some(index) => *index, - None => continue, - }; - - // Reverse all the edges in the graph and then do what we do in find_upstream_modules. - // Is there a way of doing this without the clone? - let mut reversed_graph = self.imports.clone(); - reversed_graph.reverse(); - - downstream_modules.extend( - Bfs::new(&reversed_graph, module_index) - .iter(&reversed_graph) - .map(|index| self.imports_module_indices.get_by_right(&index).unwrap()) - // Exclude any modules that we are checking. - .filter(|downstream_module| !modules_to_check.contains(downstream_module)), - ) - } - - downstream_modules - } - - pub fn find_shortest_chain( - &self, - importer: &Module, - imported: &Module, - ) -> Option> { - let importer_index = match self.imports_module_indices.get_by_left(importer) { - Some(index) => *index, - None => return None, // Importer has no imports to or from. - }; - let imported_index = match self.imports_module_indices.get_by_left(imported) { - Some(index) => *index, - None => return None, // Imported has no imports to or from. - }; - let path_to_imported = match astar( - &self.imports, - importer_index, - |finish| finish == imported_index, - |_e| 1, - |_| 0, - ) { - Some(path_tuple) => path_tuple.1, - None => return None, // No chain to the imported. - }; - - let mut chain: Vec<&Module> = vec![]; - for link_index in path_to_imported { - let module = self - .imports_module_indices - .get_by_right(&link_index) - .unwrap(); - chain.push(module); - } - Some(chain) - } - - pub fn find_shortest_chains( - &self, - importer: &Module, - imported: &Module, - as_packages: bool, - ) -> Result>, String> { - let mut chains = FxHashSet::default(); - let mut temp_graph = self.clone(); - - let mut downstream_modules: FxHashSet = FxHashSet::from_iter([importer.clone()]); - let mut upstream_modules: FxHashSet = FxHashSet::from_iter([imported.clone()]); - - // TODO don't do this if module is squashed? - if as_packages { - for descendant in self.find_descendants(importer).unwrap() { - downstream_modules.insert(descendant.clone()); - } - for descendant in self.find_descendants(imported).unwrap() { - upstream_modules.insert(descendant.clone()); - } - if upstream_modules - .intersection(&downstream_modules) - .next() - .is_some() - { - return Err("Modules have shared descendants.".to_string()); - } - } - - // Remove imports within the packages. - let mut imports_to_remove: Vec<(Module, Module)> = vec![]; - for upstream_module in &upstream_modules { - for imported_module in temp_graph.find_modules_directly_imported_by(&upstream_module) { - if upstream_modules.contains(&imported_module) { - imports_to_remove.push((upstream_module.clone(), imported_module.clone())); - } - } - } - for downstream_module in &downstream_modules { - for imported_module in temp_graph.find_modules_directly_imported_by(&downstream_module) - { - if downstream_modules.contains(&imported_module) { - imports_to_remove.push((downstream_module.clone(), imported_module.clone())); - } - } - } - for (importer_to_remove, imported_to_remove) in imports_to_remove { - temp_graph.remove_import(&importer_to_remove, &imported_to_remove); - } - - // Keep track of imports into/out of upstream/downstream packages, and remove them. - let mut map_of_imports: FxHashMap> = - FxHashMap::default(); - for module in upstream_modules.union(&downstream_modules) { - let mut imports_to_or_from_module = FxHashSet::default(); - for imported_module in temp_graph.find_modules_directly_imported_by(&module) { - imports_to_or_from_module.insert((module.clone(), imported_module.clone())); - } - for importer_module in temp_graph.find_modules_that_directly_import(&module) { - imports_to_or_from_module.insert((importer_module.clone(), module.clone())); - } - map_of_imports.insert(module.clone(), imports_to_or_from_module); - } - for imports in map_of_imports.values() { - for (importer_to_remove, imported_to_remove) in imports { - temp_graph.remove_import(&importer_to_remove, &imported_to_remove); - } - } - - for importer_module in &downstream_modules { - // Reveal imports to/from importer module. - for (importer_to_add, imported_to_add) in &map_of_imports[&importer_module] { - temp_graph.add_import(&importer_to_add, &imported_to_add); - } - for imported_module in &upstream_modules { - // Reveal imports to/from imported module. - for (importer_to_add, imported_to_add) in &map_of_imports[&imported_module] { - temp_graph.add_import(&importer_to_add, &imported_to_add); - } - if let Some(chain) = - temp_graph.find_shortest_chain(importer_module, imported_module) - { - chains.insert(chain.iter().cloned().map(|module| module.clone()).collect()); - } - // Remove imports relating to imported module again. - for (importer_to_remove, imported_to_remove) in &map_of_imports[&imported_module] { - temp_graph.remove_import(&importer_to_remove, &imported_to_remove); - } - } - // Remove imports relating to importer module again. - for (importer_to_remove, imported_to_remove) in &map_of_imports[&importer_module] { - temp_graph.remove_import(&importer_to_remove, &imported_to_remove); - } - } - Ok(chains) - } - - pub fn chain_exists(&self, importer: &Module, imported: &Module, as_packages: bool) -> bool { - // TODO should this return a Result, so we can handle the situation the importer / imported - // having shared descendants when as_packages=true? - let mut temp_graph; - let graph = match as_packages { - true => { - temp_graph = self.clone(); - temp_graph.squash_module(importer); - temp_graph.squash_module(imported); - &temp_graph - } - false => self, - }; - graph.find_shortest_chain(importer, imported).is_some() - } - - pub fn find_illegal_dependencies_for_layers( - &self, - levels: Vec, - containers: FxHashSet, - ) -> Result, NoSuchContainer> { - // Check that containers exist. - let modules = self.get_modules(); - for container in containers.iter() { - let container_module = Module::new(container.clone()); - if !modules.contains(&container_module) { - return Err(NoSuchContainer { - container: container.clone(), - }); - } - } - - let all_layers: Vec = levels - .iter() - .flat_map(|level| level.layers.iter()) - .map(|module_name| module_name.to_string()) - .collect(); - - let mut dependencies: Vec = self - ._generate_module_permutations(&levels, &containers) - //.into_iter() - .into_par_iter() - .filter_map(|(higher_layer_package, lower_layer_package, container)| { - // TODO: it's inefficient to do this for sibling layers, as we don't need - // to clone and trim the graph for identical pairs. - info!( - "Searching for import chains from {} to {}...", - lower_layer_package, higher_layer_package - ); - let now = Instant::now(); - let dependency_or_none = self._search_for_package_dependency( - &higher_layer_package, - &lower_layer_package, - &all_layers, - &container, - ); - _log_illegal_route_count(&dependency_or_none, now.elapsed().as_secs()); - dependency_or_none - }) - .collect(); - - dependencies.sort(); - - Ok(dependencies) - } - - // Return every permutation of modules that exist in the graph - /// in which the second should not import the first. - /// The third item in the tuple is the relevant container, if used. - fn _generate_module_permutations( - &self, - levels: &Vec, - containers: &FxHashSet, - ) -> Vec<(Module, Module, Option)> { - let mut permutations: Vec<(Module, Module, Option)> = vec![]; - - let quasi_containers: Vec> = if containers.is_empty() { - vec![None] - } else { - containers - .iter() - .map(|i| Some(Module::new(i.to_string()))) - .collect() - }; - let all_modules = self.get_modules(); - - for quasi_container in quasi_containers { - for (index, higher_level) in levels.iter().enumerate() { - for higher_layer in &higher_level.layers { - let higher_layer_module = _module_from_layer(&higher_layer, &quasi_container); - if !all_modules.contains(&higher_layer_module) { - continue; - } - - // Build the layers that mustn't import this higher layer. - // That includes: - // * lower layers. - // * sibling layers, if the layer is independent. - let mut layers_forbidden_to_import_higher_layer: Vec = vec![]; - - // Independence - if higher_level.independent { - for potential_sibling_layer in &higher_level.layers { - let sibling_module = - _module_from_layer(&potential_sibling_layer, &quasi_container); - if sibling_module != higher_layer_module - && all_modules.contains(&sibling_module) - { - layers_forbidden_to_import_higher_layer.push(sibling_module); - } - } - } - - for lower_level in &levels[index + 1..] { - for lower_layer in &lower_level.layers { - let lower_layer_module = - _module_from_layer(&lower_layer, &quasi_container); - if all_modules.contains(&lower_layer_module) { - layers_forbidden_to_import_higher_layer.push(lower_layer_module); - } - } - } - - // Add to permutations. - for forbidden in layers_forbidden_to_import_higher_layer { - permutations.push(( - higher_layer_module.clone(), - forbidden.clone(), - quasi_container.clone(), - )); - } - } - } - } - - permutations - } - - fn _search_for_package_dependency( - &self, - higher_layer_package: &Module, - lower_layer_package: &Module, - layers: &Vec, - container: &Option, - ) -> Option { - let mut temp_graph = self.clone(); - - // Remove other layers. - let mut modules_to_remove: Vec = vec![]; - for layer in layers { - let layer_module = _module_from_layer(&layer, &container); - if layer_module != *higher_layer_package && layer_module != *lower_layer_package { - // Remove this subpackage. - match temp_graph.find_descendants(&layer_module) { - Ok(descendants) => { - for descendant in descendants { - modules_to_remove.push(descendant.clone()) - } - } - Err(_) => (), // ModuleNotPresent. - } - modules_to_remove.push(layer_module.clone()); - } - } - for module_to_remove in modules_to_remove.clone() { - temp_graph.remove_module(&module_to_remove); - } - - let mut routes: Vec = vec![]; - - // Direct routes. - // TODO: do we need to pop the imports? - // The indirect routes should cope without removing them? - let direct_links = - temp_graph._pop_direct_imports(lower_layer_package, higher_layer_package); - for (importer, imported) in direct_links { - routes.push(Route { - heads: vec![importer], - middle: vec![], - tails: vec![imported], - }); - } - - // Indirect routes. - for indirect_route in - temp_graph._find_indirect_routes(lower_layer_package, higher_layer_package) - { - routes.push(indirect_route); - } - - if routes.is_empty() { - None - } else { - Some(PackageDependency { - importer: lower_layer_package.clone(), - imported: higher_layer_package.clone(), - routes, - }) - } - } - - fn _find_indirect_routes( - &self, - importer_package: &Module, - imported_package: &Module, - ) -> Vec { - let mut routes = vec![]; - - let mut temp_graph = self.clone(); - temp_graph.squash_module(importer_package); - temp_graph.squash_module(imported_package); - - // Find middles. - let mut middles: Vec> = vec![]; - for chain in temp_graph._pop_shortest_chains(importer_package, imported_package) { - // Remove first and last element. - let mut middle: Vec = vec![]; - let chain_length = chain.len(); - for (index, module) in chain.iter().enumerate() { - if index != 0 && index != chain_length - 1 { - middle.push(module.clone()); - } - } - middles.push(middle); - } - - // Set up importer/imported package contents. - let mut importer_modules: FxHashSet<&Module> = FxHashSet::from_iter([importer_package]); - importer_modules.extend(self.find_descendants(&importer_package).unwrap()); - let mut imported_modules: FxHashSet<&Module> = FxHashSet::from_iter([imported_package]); - imported_modules.extend(self.find_descendants(&imported_package).unwrap()); - - // Build routes from middles. - for middle in middles { - // Construct heads. - let mut heads: Vec = vec![]; - let first_imported_module = &middle[0]; - for candidate_head in self.find_modules_that_directly_import(&first_imported_module) { - if importer_modules.contains(candidate_head) { - heads.push(candidate_head.clone()); - } - } - - // Construct tails. - let mut tails: Vec = vec![]; - let last_importer_module = &middle[middle.len() - 1]; - for candidate_tail in self.find_modules_directly_imported_by(&last_importer_module) { - if imported_modules.contains(candidate_tail) { - tails.push(candidate_tail.clone()); - } - } - - routes.push(Route { - heads, - middle, - tails, - }) - } - - routes - } - - fn _pop_shortest_chains(&mut self, importer: &Module, imported: &Module) -> Vec> { - let mut chains = vec![]; - - loop { - // TODO - defend against infinite loops somehow. - - let found_chain: Vec; - { - let chain = self.find_shortest_chain(importer, imported); - - if chain.is_none() { - break; - } - - found_chain = chain.unwrap().into_iter().cloned().collect(); - } - // Remove chain. - for i in 0..found_chain.len() - 1 { - self.remove_import(&found_chain[i], &found_chain[i + 1]); - } - chains.push(found_chain); - } - chains - } - - /// Remove the direct imports, returning them as (importer, imported) tuples. - fn _pop_direct_imports( - &mut self, - lower_layer_module: &Module, - higher_layer_module: &Module, - ) -> FxHashSet<(Module, Module)> { - let mut imports = FxHashSet::default(); - - let mut lower_layer_modules = FxHashSet::from_iter([lower_layer_module.clone()]); - for descendant in self - .find_descendants(lower_layer_module) - .unwrap() - .iter() - .cloned() - { - lower_layer_modules.insert(descendant.clone()); - } - - let mut higher_layer_modules = FxHashSet::from_iter([higher_layer_module.clone()]); - for descendant in self - .find_descendants(higher_layer_module) - .unwrap() - .iter() - .cloned() - { - higher_layer_modules.insert(descendant.clone()); - } - - for lower_layer_module in lower_layer_modules { - for imported_module in self.find_modules_directly_imported_by(&lower_layer_module) { - if higher_layer_modules.contains(imported_module) { - imports.insert((lower_layer_module.clone(), imported_module.clone())); - } - } - } - - // Remove imports. - for (importer, imported) in &imports { - self.remove_import(&importer, &imported) - } - - imports - } - - pub fn squash_module(&mut self, module: &Module) { - // Get descendants and their imports. - let descendants: Vec = self - .find_descendants(module) - .unwrap() - .into_iter() - .cloned() - .collect(); - let modules_imported_by_descendants: Vec = descendants - .iter() - .flat_map(|descendant| { - self.find_modules_directly_imported_by(descendant) - .into_iter() - .cloned() - }) - .collect(); - let modules_that_import_descendants: Vec = descendants - .iter() - .flat_map(|descendant| { - self.find_modules_that_directly_import(descendant) - .into_iter() - .cloned() - }) - .collect(); - - // Remove any descendants. - for descendant in descendants { - self.remove_module(&descendant); - } - - // Add descendants and imports to parent module. - for imported in modules_imported_by_descendants { - self.add_import(module, &imported); - } - - for importer in modules_that_import_descendants { - self.add_import(&importer, module); - } - - self.squashed_modules.insert(module.clone()); - } - - pub fn is_module_squashed(&self, module: &Module) -> bool { - self.squashed_modules.contains(module) - } - - /// Return the squashed module that is the nearest ancestor of the supplied module, - /// if such an ancestor exists. - pub fn find_ancestor_squashed_module(&self, module: &Module) -> Option { - if module.is_root() { - return None; - } - let parent = Module::new_parent(&module); - if self.is_module_squashed(&parent) { - return Some(parent); - } - self.find_ancestor_squashed_module(&parent) - } - - fn add_module_if_not_in_hierarchy(&mut self, module: &Module) { - if self.hierarchy_module_indices.get_by_left(module).is_none() { - self.add_module(module.clone()); - }; - if self.invisible_modules.contains(&module) { - self.invisible_modules.remove(&module); - }; - } -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn modules_when_empty() { - let graph = Graph::default(); - - assert_eq!(graph.get_modules(), FxHashSet::default()); - } - - #[test] - fn module_is_value_object() { - assert_eq!( - Module::new("mypackage".to_string()), - Module::new("mypackage".to_string()) - ); - } - - #[test] - fn add_module() { - let mypackage = Module::new("mypackage".to_string()); - let mut graph = Graph::default(); - graph.add_module(mypackage.clone()); - - let result = graph.get_modules(); - - assert_eq!(result, FxHashSet::from_iter([&mypackage])); - } - - #[test] - fn add_module_doesnt_add_parent() { - let mypackage = Module::new("mypackage.foo".to_string()); - let mut graph = Graph::default(); - graph.add_module(mypackage.clone()); - - let result = graph.get_modules(); - - assert_eq!(result, FxHashSet::from_iter([&mypackage])); - } - - #[test] - fn add_modules() { - let mut graph = Graph::default(); - let mypackage = Module::new("mypackage".to_string()); - let mypackage_foo = Module::new("mypackage.foo".to_string()); - graph.add_module(mypackage.clone()); - graph.add_module(mypackage_foo.clone()); - - let result = graph.get_modules(); - - assert_eq!(result, FxHashSet::from_iter([&mypackage, &mypackage_foo])); - assert_eq!( - graph.pretty_str(), - " -hierarchy: - mypackage -> mypackage.foo -imports: - -" - .trim_start() - ); - } - - #[test] - fn remove_nonexistent_module() { - let mypackage = Module::new("mypackage".to_string()); - let mypackage_foo = Module::new("mypackage.foo".to_string()); - let mut graph = Graph::default(); - // Add mypackage but not mypackage.foo. - graph.add_module(mypackage.clone()); - - graph.remove_module(&mypackage_foo); - - let result = graph.get_modules(); - assert_eq!(result, FxHashSet::from_iter([&mypackage])); - } - - #[test] - fn remove_existing_module_without_imports() { - let mypackage = Module::new("mypackage".to_string()); - let mypackage_foo = Module::new("mypackage.foo".to_string()); - let mypackage_foo_alpha = Module::new("mypackage.foo.alpha".to_string()); - - let mut graph = Graph::default(); - graph.add_module(mypackage.clone()); - graph.add_module(mypackage_foo.clone()); - graph.add_module(mypackage_foo_alpha.clone()); - - graph.remove_module(&mypackage_foo); - - let result = graph.get_modules(); - assert_eq!( - result, - FxHashSet::from_iter([ - &mypackage, - &mypackage_foo_alpha, // To be consistent with previous versions of Grimp. - ]) - ); - } - - #[test] - fn remove_existing_module_with_imports() { - let mypackage = Module::new("mypackage".to_string()); - let mypackage_foo = Module::new("mypackage.foo".to_string()); - let mypackage_foo_alpha = Module::new("mypackage.foo.alpha".to_string()); - let importer = Module::new("importer".to_string()); - let imported = Module::new("importer".to_string()); - let mut graph = Graph::default(); - graph.add_module(mypackage.clone()); - graph.add_module(mypackage_foo.clone()); - graph.add_module(mypackage_foo_alpha.clone()); - graph.add_import(&importer, &mypackage_foo); - graph.add_import(&mypackage_foo, &imported); - - graph.remove_module(&mypackage_foo); - - let result = graph.get_modules(); - assert_eq!( - result, - FxHashSet::from_iter([&mypackage, &mypackage_foo_alpha, &importer, &imported]) - ); - assert_eq!( - graph.direct_import_exists(&importer, &mypackage_foo, false), - false - ); - assert_eq!( - graph.direct_import_exists(&mypackage_foo, &imported, false), - false - ); - } - - #[test] - fn remove_importer_module_removes_import_details() { - let importer = Module::new("importer".to_string()); - let imported = Module::new("importer".to_string()); - let mut graph = Graph::default(); - graph.add_detailed_import(&DetailedImport { - importer: importer.clone(), - imported: imported.clone(), - line_number: 99, - line_contents: "-".to_string(), - }); - - graph.remove_module(&importer); - - assert_eq!( - graph.get_import_details(&importer, &imported), - FxHashSet::default() - ); - } - - #[test] - fn remove_imported_module_removes_import_details() { - let importer = Module::new("importer".to_string()); - let imported = Module::new("importer".to_string()); - let mut graph = Graph::default(); - graph.add_detailed_import(&DetailedImport { - importer: importer.clone(), - imported: imported.clone(), - line_number: 99, - line_contents: "-".to_string(), - }); - - graph.remove_module(&imported); - - assert_eq!( - graph.get_import_details(&importer, &imported), - FxHashSet::default() - ); - } - - #[test] - fn remove_import_that_exists() { - let importer = Module::new("importer".to_string()); - let imported = Module::new("importer".to_string()); - let mut graph = Graph::default(); - graph.add_import(&importer, &imported); - - graph.remove_import(&importer, &imported); - - // The import has gone... - assert_eq!( - graph.direct_import_exists(&importer, &imported, false), - false - ); - // ...but the modules are still there. - assert_eq!( - graph.get_modules(), - FxHashSet::from_iter([&importer, &imported]) - ); - } - - #[test] - fn remove_import_does_nothing_if_import_doesnt_exist() { - let importer = Module::new("importer".to_string()); - let imported = Module::new("importer".to_string()); - let mut graph = Graph::default(); - graph.add_module(importer.clone()); - graph.add_module(imported.clone()); - - graph.remove_import(&importer, &imported); - - // The modules are still there. - assert_eq!( - graph.get_modules(), - FxHashSet::from_iter([&importer, &imported]) - ); - } - - #[test] - fn remove_import_does_nothing_if_modules_dont_exist() { - let importer = Module::new("importer".to_string()); - let imported = Module::new("importer".to_string()); - let mut graph = Graph::default(); - - graph.remove_import(&importer, &imported); - } - - #[test] - fn remove_import_doesnt_affect_other_imports_from_same_modules() { - let blue = Module::new("blue".to_string()); - let green = Module::new("green".to_string()); - let yellow = Module::new("yellow".to_string()); - let red = Module::new("red".to_string()); - let mut graph = Graph::default(); - graph.add_import(&blue, &green); - graph.add_import(&blue, &yellow); - graph.add_import(&red, &blue); - - graph.remove_import(&blue, &green); - - // The other imports are still there. - assert_eq!(graph.direct_import_exists(&blue, &yellow, false), true); - assert_eq!(graph.direct_import_exists(&red, &blue, false), true); - } - - #[test] - #[should_panic(expected = "rootpackage is a root level package")] - fn new_parent_root_module() { - let root = Module::new("rootpackage".to_string()); - - Module::new_parent(&root); - } - - #[test] - fn is_root_true() { - let root = Module::new("rootpackage".to_string()); - - assert!(root.is_root()); - } - - #[test] - fn is_descendant_of_true_for_child() { - let foo = Module::new("mypackage.foo".to_string()); - let foo_bar = Module::new("mypackage.foo.bar".to_string()); - - assert!(foo_bar.is_descendant_of(&foo)); - } - - #[test] - fn is_descendant_of_false_for_parent() { - let foo = Module::new("mypackage.foo".to_string()); - let foo_bar = Module::new("mypackage.foo.bar".to_string()); - - assert_eq!(foo.is_descendant_of(&foo_bar), false); - } - - #[test] - fn is_descendant_of_true_for_grandchild() { - let foo = Module::new("mypackage.foo".to_string()); - let foo_bar_baz = Module::new("mypackage.foo.bar.baz".to_string()); - - assert!(foo_bar_baz.is_descendant_of(&foo)); - } - - #[test] - fn is_descendant_of_false_for_grandparent() { - let foo = Module::new("mypackage.foo".to_string()); - let foo_bar_baz = Module::new("mypackage.foo.bar.baz".to_string()); - - assert_eq!(foo.is_descendant_of(&foo_bar_baz), false); - } - - #[test] - fn is_root_false() { - let non_root = Module::new("rootpackage.blue".to_string()); - - assert_eq!(non_root.is_root(), false); - } - - #[test] - fn find_children_no_results() { - let mut graph = Graph::default(); - let mypackage = Module::new("mypackage".to_string()); - let mypackage_foo = Module::new("mypackage.foo".to_string()); - - graph.add_module(mypackage.clone()); - graph.add_module(mypackage_foo.clone()); - - assert_eq!(graph.find_children(&mypackage_foo), FxHashSet::default()); - } - - #[test] - fn find_children_one_result() { - let mut graph = Graph::default(); - let mypackage = Module::new("mypackage".to_string()); - let mypackage_foo = Module::new("mypackage.foo".to_string()); - let mypackage_bar = Module::new("mypackage.bar".to_string()); - - graph.add_module(mypackage.clone()); - graph.add_module(mypackage_foo.clone()); - graph.add_module(mypackage_bar.clone()); - - assert_eq!( - graph.find_children(&mypackage), - FxHashSet::from_iter([&mypackage_foo, &mypackage_bar]) - ); - } - - #[test] - fn find_children_multiple_results() { - let mut graph = Graph::default(); - let mypackage = Module::new("mypackage".to_string()); - let mypackage_foo = Module::new("mypackage.foo".to_string()); - let mypackage_bar = Module::new("mypackage.bar".to_string()); - - graph.add_module(mypackage.clone()); - graph.add_module(mypackage_foo.clone()); - graph.add_module(mypackage_bar.clone()); - - assert_eq!( - graph.find_children(&mypackage), - FxHashSet::from_iter([&mypackage_foo, &mypackage_bar]) - ); - } - - #[test] - fn find_children_returns_empty_set_with_nonexistent_module() { - let mut graph = Graph::default(); - // Note: mypackage is not in the graph. - let mypackage_foo = Module::new("mypackage.foo".to_string()); - let mypackage_bar = Module::new("mypackage.bar".to_string()); - - graph.add_module(mypackage_foo.clone()); - graph.add_module(mypackage_bar.clone()); - - assert_eq!( - graph.find_children(&Module::new("mypackage".to_string())), - FxHashSet::default() - ); - } - - #[test] - fn find_descendants_no_results() { - let mut graph = Graph::default(); - let mypackage = Module::new("mypackage".to_string()); - let mypackage_foo = Module::new("mypackage.foo".to_string()); - let mypackage_bar = Module::new("mypackage.bar".to_string()); - let mypackage_foo_alpha = Module::new("mypackage.foo.alpha".to_string()); - let mypackage_foo_alpha_blue = Module::new("mypackage.foo.alpha.blue".to_string()); - let mypackage_foo_alpha_green = Module::new("mypackage.foo.alpha.green".to_string()); - let mypackage_foo_beta = Module::new("mypackage.foo.beta".to_string()); - - graph.add_module(mypackage.clone()); - graph.add_module(mypackage_foo.clone()); - graph.add_module(mypackage_bar.clone()); - graph.add_module(mypackage_foo_alpha.clone()); - graph.add_module(mypackage_foo_alpha_blue.clone()); - graph.add_module(mypackage_foo_alpha_green.clone()); - graph.add_module(mypackage_foo_beta.clone()); - - assert_eq!( - graph.find_descendants(&mypackage_bar), - Ok(FxHashSet::default()) - ); - } - - #[test] - fn find_descendants_module_not_in_graph() { - let mut graph = Graph::default(); - let blue = Module::new("blue".to_string()); - let green = Module::new("green".to_string()); - graph.add_module(blue.clone()); - - assert_eq!( - graph.find_descendants(&green), - Err(ModuleNotPresent { - module: green.clone() - }) - ); - } - - #[test] - fn find_descendants_multiple_results() { - let mut graph = Graph::default(); - let mypackage = Module::new("mypackage".to_string()); - let mypackage_foo = Module::new("mypackage.foo".to_string()); - let mypackage_bar = Module::new("mypackage.bar".to_string()); - let mypackage_foo_alpha = Module::new("mypackage.foo.alpha".to_string()); - let mypackage_foo_alpha_blue = Module::new("mypackage.foo.alpha.blue".to_string()); - let mypackage_foo_alpha_green = Module::new("mypackage.foo.alpha.green".to_string()); - let mypackage_foo_beta = Module::new("mypackage.foo.beta".to_string()); - - graph.add_module(mypackage.clone()); - graph.add_module(mypackage_foo.clone()); - graph.add_module(mypackage_bar.clone()); - graph.add_module(mypackage_foo_alpha.clone()); - graph.add_module(mypackage_foo_alpha_blue.clone()); - graph.add_module(mypackage_foo_alpha_green.clone()); - graph.add_module(mypackage_foo_beta.clone()); - - assert_eq!( - graph.find_descendants(&mypackage_foo), - Ok(FxHashSet::from_iter([ - &mypackage_foo_alpha, - &mypackage_foo_alpha_blue, - &mypackage_foo_alpha_green, - &mypackage_foo_beta - ])) - ); - } - - #[test] - fn find_descendants_with_gap() { - let mut graph = Graph::default(); - let mypackage = Module::new("mypackage".to_string()); - let mypackage_foo = Module::new("mypackage.foo".to_string()); - // mypackage.foo.blue is not added. - let mypackage_foo_blue_alpha = Module::new("mypackage.foo.blue.alpha".to_string()); - let mypackage_foo_blue_alpha_one = Module::new("mypackage.foo.blue.alpha.one".to_string()); - let mypackage_foo_blue_alpha_two = Module::new("mypackage.foo.blue.alpha.two".to_string()); - let mypackage_foo_blue_beta_three = - Module::new("mypackage.foo.blue.beta.three".to_string()); - let mypackage_bar_green_alpha = Module::new("mypackage.bar.green.alpha".to_string()); - graph.add_module(mypackage.clone()); - graph.add_module(mypackage_foo.clone()); - graph.add_module(mypackage_foo_blue_alpha.clone()); - graph.add_module(mypackage_foo_blue_alpha_one.clone()); - graph.add_module(mypackage_foo_blue_alpha_two.clone()); - graph.add_module(mypackage_foo_blue_beta_three.clone()); - graph.add_module(mypackage_bar_green_alpha.clone()); - - assert_eq!( - graph.find_descendants(&mypackage_foo), - // mypackage.foo.blue is not included. - Ok(FxHashSet::from_iter([ - &mypackage_foo_blue_alpha, - &mypackage_foo_blue_alpha_one, - &mypackage_foo_blue_alpha_two, - &mypackage_foo_blue_beta_three, - ])) - ); - } - - #[test] - fn find_descendants_added_in_different_order() { - let mut graph = Graph::default(); - let mypackage = Module::new("mypackage".to_string()); - let mypackage_foo = Module::new("mypackage.foo".to_string()); - let mypackage_foo_blue_alpha = Module::new("mypackage.foo.blue.alpha".to_string()); - let mypackage_foo_blue_alpha_one = Module::new("mypackage.foo.blue.alpha.one".to_string()); - let mypackage_foo_blue_alpha_two = Module::new("mypackage.foo.blue.alpha.two".to_string()); - let mypackage_foo_blue_beta_three = - Module::new("mypackage.foo.blue.beta.three".to_string()); - let mypackage_bar_green_alpha = Module::new("mypackage.bar.green.alpha".to_string()); - let mypackage_foo_blue = Module::new("mypackage.foo.blue".to_string()); - graph.add_module(mypackage.clone()); - graph.add_module(mypackage_foo.clone()); - graph.add_module(mypackage_foo_blue_alpha.clone()); - graph.add_module(mypackage_foo_blue_alpha_one.clone()); - graph.add_module(mypackage_foo_blue_alpha_two.clone()); - graph.add_module(mypackage_foo_blue_beta_three.clone()); - graph.add_module(mypackage_bar_green_alpha.clone()); - // Add the middle one at the end. - graph.add_module(mypackage_foo_blue.clone()); - - assert_eq!( - graph.find_descendants(&mypackage_foo), - Ok(FxHashSet::from_iter([ - &mypackage_foo_blue, // Should be included. - &mypackage_foo_blue_alpha, - &mypackage_foo_blue_alpha_one, - &mypackage_foo_blue_alpha_two, - &mypackage_foo_blue_beta_three, - ])) - ); - } - - #[test] - fn direct_import_exists_returns_true() { - let mut graph = Graph::default(); - let mypackage = Module::new("mypackage".to_string()); - let mypackage_foo = Module::new("mypackage.foo".to_string()); - let mypackage_bar = Module::new("mypackage.bar".to_string()); - graph.add_module(mypackage.clone()); - graph.add_module(mypackage_foo.clone()); - graph.add_module(mypackage_bar.clone()); - graph.add_import(&mypackage_foo, &mypackage_bar); - - assert!(graph.direct_import_exists(&mypackage_foo, &mypackage_bar, false)); - } - - #[test] - fn add_detailed_import_adds_import() { - let mut graph = Graph::default(); - let blue = Module::new("blue".to_string()); - let green = Module::new("green".to_string()); - graph.add_module(blue.clone()); - graph.add_module(green.clone()); - let import = DetailedImport { - importer: blue.clone(), - imported: green.clone(), - line_number: 11, - line_contents: "-".to_string(), - }; - - graph.add_detailed_import(&import); - - assert_eq!(graph.direct_import_exists(&blue, &green, false), true); - } - - #[test] - fn direct_import_exists_returns_false() { - let mut graph = Graph::default(); - let mypackage = Module::new("mypackage".to_string()); - let mypackage_foo = Module::new("mypackage.foo".to_string()); - let mypackage_bar = Module::new("mypackage.bar".to_string()); - graph.add_module(mypackage.clone()); - graph.add_module(mypackage_foo.clone()); - graph.add_module(mypackage_bar.clone()); - graph.add_import(&mypackage_foo, &mypackage_bar); - - assert!(!graph.direct_import_exists(&mypackage_bar, &mypackage_foo, false)); - } - - #[test] - fn direct_import_exists_returns_false_root_to_child() { - let mut graph = Graph::default(); - let mypackage = Module::new("mypackage".to_string()); - let mypackage_foo = Module::new("mypackage.foo".to_string()); - let mypackage_bar = Module::new("mypackage.bar".to_string()); - let mypackage_foo_alpha = Module::new("mypackage.foo.alpha".to_string()); - graph.add_module(mypackage.clone()); - graph.add_module(mypackage_foo.clone()); - graph.add_module(mypackage_bar.clone()); - graph.add_module(mypackage_foo_alpha.clone()); - graph.add_import(&mypackage_bar, &mypackage_foo_alpha); - - assert_eq!( - graph.pretty_str(), - " -hierarchy: - mypackage -> mypackage.bar - mypackage -> mypackage.foo - mypackage.foo -> mypackage.foo.alpha -imports: - mypackage.bar -> mypackage.foo.alpha -" - .trim_start() - ); - assert!(!graph.direct_import_exists(&mypackage_bar, &mypackage_foo, false)); - } - - #[test] - fn add_import_with_non_existent_importer_adds_that_module() { - let mut graph = Graph::default(); - let mypackage_foo = Module::new("mypackage.foo".to_string()); - let mypackage_bar = Module::new("mypackage.bar".to_string()); - graph.add_module(mypackage_bar.clone()); - - graph.add_import(&mypackage_foo, &mypackage_bar); - - assert_eq!( - graph.get_modules(), - FxHashSet::from_iter([&mypackage_bar, &mypackage_foo]) - ); - assert!(graph.direct_import_exists(&mypackage_foo, &mypackage_bar, false)); - assert_eq!( - graph.pretty_str(), - " -hierarchy: - (mypackage) -> mypackage.bar - (mypackage) -> mypackage.foo -imports: - mypackage.foo -> mypackage.bar -" - .trim_start() - ); - } - - #[test] - fn add_import_with_non_existent_imported_adds_that_module() { - let mut graph = Graph::default(); - let mypackage_foo = Module::new("mypackage.foo".to_string()); - let mypackage_bar = Module::new("mypackage.bar".to_string()); - graph.add_module(mypackage_foo.clone()); - - graph.add_import(&mypackage_foo, &mypackage_bar); - - assert_eq!( - graph.get_modules(), - FxHashSet::from_iter([&mypackage_bar, &mypackage_foo]) - ); - assert!(graph.direct_import_exists(&mypackage_foo, &mypackage_bar, false)); - assert_eq!( - graph.pretty_str(), - " -hierarchy: - (mypackage) -> mypackage.bar - (mypackage) -> mypackage.foo -imports: - mypackage.foo -> mypackage.bar -" - .trim_start() - ); - } - - #[test] - fn direct_import_exists_with_as_packages_returns_false() { - let mut graph = Graph::default(); - let mypackage = Module::new("mypackage".to_string()); - let mypackage_foo = Module::new("mypackage.foo".to_string()); - let mypackage_bar = Module::new("mypackage.bar".to_string()); - let mypackage_foo_alpha = Module::new("mypackage.foo.alpha".to_string()); - let mypackage_foo_alpha_blue = Module::new("mypackage.foo.alpha.blue".to_string()); - let mypackage_foo_alpha_green = Module::new("mypackage.foo.alpha.green".to_string()); - let mypackage_foo_beta = Module::new("mypackage.foo.beta".to_string()); - graph.add_module(mypackage.clone()); - graph.add_module(mypackage_foo.clone()); - graph.add_module(mypackage_bar.clone()); - graph.add_module(mypackage_foo_alpha.clone()); - graph.add_module(mypackage_foo_alpha_blue.clone()); - graph.add_module(mypackage_foo_alpha_green.clone()); - graph.add_module(mypackage_foo_beta.clone()); - // Add an import in the other direction. - graph.add_import(&mypackage_bar, &mypackage_foo); - - assert!(!graph.direct_import_exists(&mypackage_foo, &mypackage_bar, true)); - } - - #[test] - fn direct_import_exists_with_as_packages_returns_true_between_roots() { - let mut graph = Graph::default(); - let mypackage = Module::new("mypackage".to_string()); - let mypackage_foo = Module::new("mypackage.foo".to_string()); - let mypackage_bar = Module::new("mypackage.bar".to_string()); - let mypackage_foo_alpha = Module::new("mypackage.foo.alpha".to_string()); - let mypackage_foo_alpha_blue = Module::new("mypackage.foo.alpha.blue".to_string()); - let mypackage_foo_alpha_green = Module::new("mypackage.foo.alpha.green".to_string()); - let mypackage_foo_beta = Module::new("mypackage.foo.beta".to_string()); - graph.add_module(mypackage.clone()); - graph.add_module(mypackage_foo.clone()); - graph.add_module(mypackage_bar.clone()); - graph.add_module(mypackage_foo_alpha.clone()); - graph.add_module(mypackage_foo_alpha_blue.clone()); - graph.add_module(mypackage_foo_alpha_green.clone()); - graph.add_module(mypackage_foo_beta.clone()); - graph.add_import(&mypackage_foo, &mypackage_bar); - - assert!(graph.direct_import_exists(&mypackage_foo, &mypackage_bar, true)); - } - - #[test] - fn direct_import_exists_with_as_packages_returns_true_root_to_child() { - let mut graph = Graph::default(); - let mypackage = Module::new("mypackage".to_string()); - let mypackage_foo = Module::new("mypackage.foo".to_string()); - let mypackage_bar = Module::new("mypackage.bar".to_string()); - let mypackage_foo_alpha = Module::new("mypackage.foo.alpha".to_string()); - let mypackage_foo_alpha_blue = Module::new("mypackage.foo.alpha.blue".to_string()); - let mypackage_foo_alpha_green = Module::new("mypackage.foo.alpha.green".to_string()); - let mypackage_foo_beta = Module::new("mypackage.foo.beta".to_string()); - graph.add_module(mypackage.clone()); - graph.add_module(mypackage_foo.clone()); - graph.add_module(mypackage_bar.clone()); - graph.add_module(mypackage_foo_alpha.clone()); - graph.add_module(mypackage_foo_alpha_blue.clone()); - graph.add_module(mypackage_foo_alpha_green.clone()); - graph.add_module(mypackage_foo_beta.clone()); - graph.add_import(&mypackage_bar, &mypackage_foo_alpha); - - assert!(graph.direct_import_exists(&mypackage_bar, &mypackage_foo, true)); - } - - #[test] - fn direct_import_exists_with_as_packages_returns_true_child_to_root() { - let mut graph = Graph::default(); - let mypackage = Module::new("mypackage".to_string()); - let mypackage_foo = Module::new("mypackage.foo".to_string()); - let mypackage_bar = Module::new("mypackage.bar".to_string()); - let mypackage_foo_alpha = Module::new("mypackage.foo.alpha".to_string()); - let mypackage_foo_alpha_blue = Module::new("mypackage.foo.alpha.blue".to_string()); - let mypackage_foo_alpha_green = Module::new("mypackage.foo.alpha.green".to_string()); - let mypackage_foo_beta = Module::new("mypackage.foo.beta".to_string()); - graph.add_module(mypackage.clone()); - graph.add_module(mypackage_foo.clone()); - graph.add_module(mypackage_bar.clone()); - graph.add_module(mypackage_foo_alpha.clone()); - graph.add_module(mypackage_foo_alpha_blue.clone()); - graph.add_module(mypackage_foo_alpha_green.clone()); - graph.add_module(mypackage_foo_beta.clone()); - graph.add_import(&mypackage_foo_alpha, &mypackage_bar); - - assert!(graph.direct_import_exists(&mypackage_foo, &mypackage_bar, true)); - } - - #[test] - #[should_panic] - fn direct_import_exists_within_package_panics() { - let mut graph = Graph::default(); - let ancestor = Module::new("mypackage.foo".to_string()); - let descendant = Module::new("mypackage.foo.blue.alpha".to_string()); - graph.add_import(&ancestor, &descendant); - - graph.direct_import_exists(&ancestor, &descendant, true); - } - - #[test] - fn find_modules_that_directly_import() { - let mut graph = Graph::default(); - let mypackage = Module::new("mypackage".to_string()); - let mypackage_foo = Module::new("mypackage.foo".to_string()); - let mypackage_bar = Module::new("mypackage.bar".to_string()); - let mypackage_foo_alpha = Module::new("mypackage.foo.alpha".to_string()); - let mypackage_foo_alpha_blue = Module::new("mypackage.foo.alpha.blue".to_string()); - let mypackage_foo_alpha_green = Module::new("mypackage.foo.alpha.green".to_string()); - let mypackage_foo_beta = Module::new("mypackage.foo.beta".to_string()); - let anotherpackage = Module::new("anotherpackage".to_string()); - graph.add_module(mypackage.clone()); - graph.add_module(mypackage_foo.clone()); - graph.add_module(mypackage_bar.clone()); - graph.add_module(mypackage_foo_alpha.clone()); - graph.add_module(mypackage_foo_alpha_blue.clone()); - graph.add_module(mypackage_foo_alpha_green.clone()); - graph.add_module(mypackage_foo_beta.clone()); - graph.add_import(&mypackage_foo_alpha, &mypackage_bar); - graph.add_import(&anotherpackage, &mypackage_bar); - graph.add_import(&mypackage_bar, &mypackage_foo_alpha_green); - - let result = graph.find_modules_that_directly_import(&mypackage_bar); - - assert_eq!( - result, - FxHashSet::from_iter([&mypackage_foo_alpha, &anotherpackage]) - ) - } - - #[test] - fn find_modules_that_directly_import_after_removal() { - let mut graph = Graph::default(); - let blue = Module::new("blue".to_string()); - let green = Module::new("green".to_string()); - let yellow = Module::new("yellow".to_string()); - graph.add_import(&green, &blue); - graph.add_import(&yellow, &blue); - - graph.remove_import(&green, &blue); - let result = graph.find_modules_that_directly_import(&blue); - - assert_eq!(result, FxHashSet::from_iter([&yellow])) - } - - #[test] - fn find_modules_directly_imported_by() { - let mut graph = Graph::default(); - let mypackage = Module::new("mypackage".to_string()); - let mypackage_foo = Module::new("mypackage.foo".to_string()); - let mypackage_bar = Module::new("mypackage.bar".to_string()); - let mypackage_foo_alpha = Module::new("mypackage.foo.alpha".to_string()); - let mypackage_foo_alpha_blue = Module::new("mypackage.foo.alpha.blue".to_string()); - let mypackage_foo_alpha_green = Module::new("mypackage.foo.alpha.green".to_string()); - let mypackage_foo_beta = Module::new("mypackage.foo.beta".to_string()); - let anotherpackage = Module::new("anotherpackage".to_string()); - graph.add_module(mypackage.clone()); - graph.add_module(mypackage_foo.clone()); - graph.add_module(mypackage_bar.clone()); - graph.add_module(mypackage_foo_alpha.clone()); - graph.add_module(mypackage_foo_alpha_blue.clone()); - graph.add_module(mypackage_foo_alpha_green.clone()); - graph.add_module(mypackage_foo_beta.clone()); - graph.add_import(&mypackage_bar, &mypackage_foo_alpha); - graph.add_import(&mypackage_bar, &anotherpackage); - graph.add_import(&mypackage_foo_alpha_green, &mypackage_bar); - - let result = graph.find_modules_directly_imported_by(&mypackage_bar); - - assert_eq!( - result, - FxHashSet::from_iter([&mypackage_foo_alpha, &anotherpackage]) - ) - } - - #[test] - fn find_modules_directly_imported_by_after_removal() { - let mut graph = Graph::default(); - let blue = Module::new("blue".to_string()); - let green = Module::new("green".to_string()); - let yellow = Module::new("yellow".to_string()); - graph.add_import(&blue, &green); - graph.add_import(&blue, &yellow); - - graph.remove_import(&blue, &green); - let result = graph.find_modules_directly_imported_by(&blue); - - assert_eq!(result, FxHashSet::from_iter([&yellow])) - } - - #[test] - fn squash_module_descendants() { - let mut graph = Graph::default(); - // Module we're going to squash. - let mypackage = Module::new("mypackage".to_string()); - let mypackage_blue = Module::new("mypackage.blue".to_string()); - let mypackage_blue_alpha = Module::new("mypackage.blue.alpha".to_string()); - let mypackage_blue_alpha_foo = Module::new("mypackage.blue.alpha.foo".to_string()); - let mypackage_blue_beta = Module::new("mypackage.blue.beta".to_string()); - // Other modules. - let mypackage_green = Module::new("mypackage.green".to_string()); - let mypackage_red = Module::new("mypackage.red".to_string()); - let mypackage_orange = Module::new("mypackage.orange".to_string()); - let mypackage_yellow = Module::new("mypackage.yellow".to_string()); - graph.add_module(mypackage.clone()); - graph.add_module(mypackage_blue.clone()); - // Module's descendants importing other modules. - graph.add_import(&mypackage_blue_alpha, &mypackage_green); - graph.add_import(&mypackage_blue_alpha, &mypackage_red); - graph.add_import(&mypackage_blue_alpha_foo, &mypackage_yellow); - graph.add_import(&mypackage_blue_beta, &mypackage_orange); - // Other modules importing squashed module's descendants. - graph.add_import(&mypackage_red, &mypackage_blue_alpha); - graph.add_import(&mypackage_yellow, &mypackage_blue_alpha); - graph.add_import(&mypackage_orange, &mypackage_blue_alpha_foo); - graph.add_import(&mypackage_green, &mypackage_blue_beta); - // Unrelated imports. - graph.add_import(&mypackage_green, &mypackage_orange); - assert_eq!( - graph.pretty_str(), - " -hierarchy: - mypackage -> mypackage.blue - mypackage -> mypackage.green - mypackage -> mypackage.orange - mypackage -> mypackage.red - mypackage -> mypackage.yellow - mypackage.blue -> mypackage.blue.alpha - mypackage.blue -> mypackage.blue.beta - mypackage.blue.alpha -> mypackage.blue.alpha.foo -imports: - mypackage.blue.alpha -> mypackage.green - mypackage.blue.alpha -> mypackage.red - mypackage.blue.alpha.foo -> mypackage.yellow - mypackage.blue.beta -> mypackage.orange - mypackage.green -> mypackage.blue.beta - mypackage.green -> mypackage.orange - mypackage.orange -> mypackage.blue.alpha.foo - mypackage.red -> mypackage.blue.alpha - mypackage.yellow -> mypackage.blue.alpha -" - .trim_start() - ); - - graph.squash_module(&mypackage_blue); - - assert_eq!( - graph.pretty_str(), - " -hierarchy: - mypackage -> mypackage.blue - mypackage -> mypackage.green - mypackage -> mypackage.orange - mypackage -> mypackage.red - mypackage -> mypackage.yellow -imports: - mypackage.blue -> mypackage.green - mypackage.blue -> mypackage.orange - mypackage.blue -> mypackage.red - mypackage.blue -> mypackage.yellow - mypackage.green -> mypackage.blue - mypackage.green -> mypackage.orange - mypackage.orange -> mypackage.blue - mypackage.red -> mypackage.blue - mypackage.yellow -> mypackage.blue -" - .trim_start() - ); - } - - #[test] - fn squash_module_no_descendants() { - let mut graph = Graph::default(); - let mypackage = Module::new("mypackage".to_string()); - let mypackage_blue = Module::new("mypackage.blue".to_string()); - graph.add_module(mypackage.clone()); - graph.add_module(mypackage_blue.clone()); - - graph.squash_module(&mypackage_blue); - - assert_eq!( - graph.pretty_str(), - " -hierarchy: - mypackage -> mypackage.blue -imports: - -" - .trim_start() - ); - } - - #[test] - fn find_count_imports_empty_graph() { - let graph = Graph::default(); - - let result = graph.count_imports(); - - assert_eq!(result, 0); - } - - #[test] - fn find_count_imports_modules_but_no_imports() { - let mut graph = Graph::default(); - graph.add_module(Module::new("mypackage.foo".to_string())); - graph.add_module(Module::new("mypackage.bar".to_string())); - - let result = graph.count_imports(); - - assert_eq!(result, 0); - } - - #[test] - fn find_count_imports_some_imports() { - let mut graph = Graph::default(); - let mypackage_foo = Module::new("mypackage.foo".to_string()); - let mypackage_bar = Module::new("mypackage.bar".to_string()); - let mypackage_baz = Module::new("mypackage.baz".to_string()); - graph.add_module(mypackage_foo.clone()); - graph.add_module(mypackage_bar.clone()); - graph.add_import(&mypackage_foo, &mypackage_bar); - graph.add_import(&mypackage_foo, &mypackage_baz); - - let result = graph.count_imports(); - - assert_eq!(result, 2); - } - - #[test] - fn find_count_imports_treats_two_imports_between_same_modules_as_one() { - let mut graph = Graph::default(); - let mypackage_foo = Module::new("mypackage.foo".to_string()); - let mypackage_bar = Module::new("mypackage.bar".to_string()); - graph.add_module(mypackage_foo.clone()); - graph.add_module(mypackage_bar.clone()); - graph.add_import(&mypackage_foo, &mypackage_bar); - graph.add_import(&mypackage_foo, &mypackage_bar); - - let result = graph.count_imports(); - - assert_eq!(result, 1); - } - - #[test] - fn is_module_squashed_when_not_squashed() { - let mut graph = Graph::default(); - // Module we're going to squash. - let mypackage_blue = Module::new("mypackage.blue".to_string()); - let mypackage_blue_alpha = Module::new("mypackage.blue.alpha".to_string()); - // Other module. - let mypackage_green = Module::new("mypackage.green".to_string()); - graph.add_module(mypackage_blue.clone()); - graph.add_module(mypackage_blue_alpha.clone()); - graph.add_module(mypackage_green.clone()); - graph.squash_module(&mypackage_blue); - - let result = graph.is_module_squashed(&mypackage_green); - - assert!(!result); - } - - #[test] - fn is_module_squashed_when_squashed() { - let mut graph = Graph::default(); - // Module we're going to squash. - let mypackage_blue = Module::new("mypackage.blue".to_string()); - let mypackage_blue_alpha = Module::new("mypackage.blue.alpha".to_string()); - // Other module. - let mypackage_green = Module::new("mypackage.green".to_string()); - graph.add_module(mypackage_blue.clone()); - graph.add_module(mypackage_blue_alpha.clone()); - graph.add_module(mypackage_green.clone()); - graph.squash_module(&mypackage_blue); - - let result = graph.is_module_squashed(&mypackage_blue); - - assert!(result); - } - - #[test] - fn find_upstream_modules_when_there_are_some() { - let mut graph = Graph::default(); - let mypackage = Module::new("mypackage".to_string()); - let blue = Module::new("mypackage.blue".to_string()); - let green = Module::new("mypackage.green".to_string()); - let red = Module::new("mypackage.red".to_string()); - let yellow = Module::new("mypackage.yellow".to_string()); - let purple = Module::new("mypackage.purple".to_string()); - let orange = Module::new("mypackage.orange".to_string()); - let brown = Module::new("mypackage.brown".to_string()); - graph.add_module(mypackage.clone()); - graph.add_module(blue.clone()); - graph.add_module(green.clone()); - graph.add_module(red.clone()); - graph.add_module(yellow.clone()); - graph.add_module(purple.clone()); - graph.add_module(orange.clone()); - graph.add_module(brown.clone()); - // Add the import chain we care about. - graph.add_import(&blue, &green); - graph.add_import(&blue, &red); - graph.add_import(&green, &yellow); - graph.add_import(&yellow, &purple); - // Add an import to blue. - graph.add_import(&brown, &blue); - - let result = graph.find_upstream_modules(&blue, false); - - assert_eq!( - result, - FxHashSet::from_iter([&green, &red, &yellow, &purple]) - ) - } - - #[test] - fn find_upstream_modules_when_module_doesnt_exist() { - let graph = Graph::default(); - let blue = Module::new("mypackage.blue".to_string()); - - let result = graph.find_upstream_modules(&blue, false); - - assert_eq!(result, FxHashSet::default()) - } - - #[test] - fn find_upstream_modules_as_packages() { - let mut graph = Graph::default(); - let mypackage = Module::new("mypackage".to_string()); - let blue = Module::new("mypackage.blue".to_string()); - let alpha = Module::new("mypackage.blue.alpha".to_string()); - let beta = Module::new("mypackage.blue.beta".to_string()); - let green = Module::new("mypackage.green".to_string()); - let red = Module::new("mypackage.red".to_string()); - let yellow = Module::new("mypackage.yellow".to_string()); - let purple = Module::new("mypackage.purple".to_string()); - let orange = Module::new("mypackage.orange".to_string()); - let brown = Module::new("mypackage.brown".to_string()); - graph.add_module(mypackage.clone()); - graph.add_module(blue.clone()); - graph.add_module(alpha.clone()); - graph.add_module(beta.clone()); - graph.add_module(green.clone()); - graph.add_module(red.clone()); - graph.add_module(yellow.clone()); - graph.add_module(purple.clone()); - graph.add_module(orange.clone()); - graph.add_module(brown.clone()); - // Add the import chains we care about. - graph.add_import(&blue, &green); - graph.add_import(&green, &yellow); - graph.add_import(&alpha, &purple); - graph.add_import(&purple, &brown); - // Despite being technically upstream, beta doesn't appear because it's - // in the same package. - graph.add_import(&purple, &beta); - - let result = graph.find_upstream_modules(&blue, true); - - assert_eq!( - result, - FxHashSet::from_iter([&green, &yellow, &purple, &brown]) - ) - } - - #[test] - fn find_downstream_modules_when_there_are_some() { - let mut graph = Graph::default(); - let mypackage = Module::new("mypackage".to_string()); - let blue = Module::new("mypackage.blue".to_string()); - let green = Module::new("mypackage.green".to_string()); - let red = Module::new("mypackage.red".to_string()); - let yellow = Module::new("mypackage.yellow".to_string()); - let purple = Module::new("mypackage.purple".to_string()); - let orange = Module::new("mypackage.orange".to_string()); - let brown = Module::new("mypackage.brown".to_string()); - graph.add_module(mypackage.clone()); - graph.add_module(blue.clone()); - graph.add_module(green.clone()); - graph.add_module(red.clone()); - graph.add_module(yellow.clone()); - graph.add_module(purple.clone()); - graph.add_module(orange.clone()); - graph.add_module(brown.clone()); - // Add the import chain we care about. - graph.add_import(&blue, &green); - graph.add_import(&blue, &red); - graph.add_import(&green, &yellow); - graph.add_import(&yellow, &purple); - // Add an import from purple. - graph.add_import(&purple, &brown); - - let result = graph.find_downstream_modules(&purple, false); - - assert_eq!(result, FxHashSet::from_iter([&yellow, &green, &blue])) - } - - #[test] - fn find_downstream_modules_when_module_doesnt_exist() { - let graph = Graph::default(); - let blue = Module::new("mypackage.blue".to_string()); - - let result = graph.find_downstream_modules(&blue, false); - - assert_eq!(result, FxHashSet::default()) - } - - #[test] - fn find_downstream_modules_as_packages() { - let mut graph = Graph::default(); - let mypackage = Module::new("mypackage".to_string()); - let blue = Module::new("mypackage.blue".to_string()); - let alpha = Module::new("mypackage.blue.alpha".to_string()); - let beta = Module::new("mypackage.blue.beta".to_string()); - let green = Module::new("mypackage.green".to_string()); - let red = Module::new("mypackage.red".to_string()); - let yellow = Module::new("mypackage.yellow".to_string()); - let purple = Module::new("mypackage.purple".to_string()); - let orange = Module::new("mypackage.orange".to_string()); - let brown = Module::new("mypackage.brown".to_string()); - graph.add_module(mypackage.clone()); - graph.add_module(blue.clone()); - graph.add_module(alpha.clone()); - graph.add_module(beta.clone()); - graph.add_module(green.clone()); - graph.add_module(red.clone()); - graph.add_module(yellow.clone()); - graph.add_module(purple.clone()); - graph.add_module(orange.clone()); - graph.add_module(brown.clone()); - // Add the import chains we care about. - graph.add_import(&yellow, &green); - graph.add_import(&green, &blue); - graph.add_import(&brown, &purple); - graph.add_import(&purple, &alpha); - // Despite being technically downstream, beta doesn't appear because it's - // in the same package. - graph.add_import(&beta, &yellow); - - let result = graph.find_downstream_modules(&blue, true); - - assert_eq!( - result, - FxHashSet::from_iter([&green, &yellow, &purple, &brown]) - ) - } - - // find_shortest_chain - #[test] - fn find_shortest_chain_none() { - let mut graph = Graph::default(); - let mypackage = Module::new("mypackage".to_string()); - let blue = Module::new("mypackage.blue".to_string()); - let green = Module::new("mypackage.green".to_string()); - let purple = Module::new("mypackage.purple".to_string()); - graph.add_module(mypackage.clone()); - graph.add_module(blue.clone()); - graph.add_module(green.clone()); - graph.add_module(purple.clone()); - // Add imports that are irrelevant. - graph.add_import(&purple, &blue); - graph.add_import(&green, &purple); - - let result = graph.find_shortest_chain(&blue, &green); - - assert!(result.is_none()) - } - - #[test] - fn find_shortest_chain_one_step() { - let mut graph = Graph::default(); - let mypackage = Module::new("mypackage".to_string()); - let blue = Module::new("mypackage.blue".to_string()); - let green = Module::new("mypackage.green".to_string()); - let red = Module::new("mypackage.red".to_string()); - let purple = Module::new("mypackage.purple".to_string()); - graph.add_module(mypackage.clone()); - graph.add_module(blue.clone()); - graph.add_module(green.clone()); - graph.add_module(red.clone()); - graph.add_module(purple.clone()); - // Add the one-step chain. - graph.add_import(&blue, &green); - // Add a longer chain. - graph.add_import(&blue, &red); - graph.add_import(&red, &green); - // Add other imports that are irrelevant. - graph.add_import(&purple, &blue); - graph.add_import(&green, &purple); - - let result = graph.find_shortest_chain(&blue, &green).unwrap(); - - assert_eq!(result, vec![&blue, &green]) - } - - #[test] - fn find_shortest_chain_one_step_reverse() { - let mut graph = Graph::default(); - let blue = Module::new("mypackage.blue".to_string()); - let green = Module::new("mypackage.green".to_string()); - graph.add_module(blue.clone()); - graph.add_module(green.clone()); - // Add the one-step chain. - graph.add_import(&blue, &green); - - let result = graph.find_shortest_chain(&green, &blue); - - assert_eq!(result.is_none(), true); - } - - #[test] - fn find_shortest_chain_two_steps() { - let mut graph = Graph::default(); - let mypackage = Module::new("mypackage".to_string()); - let blue = Module::new("mypackage.blue".to_string()); - let green = Module::new("mypackage.green".to_string()); - let red = Module::new("mypackage.red".to_string()); - let orange = Module::new("mypackage.orange".to_string()); - let purple = Module::new("mypackage.purple".to_string()); - graph.add_module(mypackage.clone()); - graph.add_module(blue.clone()); - graph.add_module(green.clone()); - graph.add_module(red.clone()); - graph.add_module(orange.clone()); - graph.add_module(purple.clone()); - // Add the two-step chain. - graph.add_import(&blue, &red); - graph.add_import(&red, &green); - // Add a longer chain. - graph.add_import(&blue, &red); - graph.add_import(&red, &orange); - graph.add_import(&orange, &green); - // Add other imports that are irrelevant. - graph.add_import(&purple, &blue); - graph.add_import(&green, &purple); - - let result = graph.find_shortest_chain(&blue, &green).unwrap(); - - assert_eq!(result, vec![&blue, &red, &green]) - } - - #[test] - fn find_shortest_chain_three_steps() { - let mut graph = Graph::default(); - let mypackage = Module::new("mypackage".to_string()); - let blue = Module::new("mypackage.blue".to_string()); - let green = Module::new("mypackage.green".to_string()); - let red = Module::new("mypackage.red".to_string()); - let orange = Module::new("mypackage.orange".to_string()); - let yellow = Module::new("mypackage.yellow".to_string()); - let purple = Module::new("mypackage.purple".to_string()); - graph.add_module(mypackage.clone()); - graph.add_module(blue.clone()); - graph.add_module(green.clone()); - graph.add_module(red.clone()); - graph.add_module(orange.clone()); - graph.add_module(yellow.clone()); - graph.add_module(purple.clone()); - // Add the three-step chain. - graph.add_import(&blue, &red); - graph.add_import(&red, &orange); - graph.add_import(&orange, &green); - // Add a longer chain. - graph.add_import(&blue, &red); - graph.add_import(&red, &orange); - graph.add_import(&orange, &yellow); - graph.add_import(&yellow, &green); - // Add other imports that are irrelevant. - graph.add_import(&purple, &blue); - graph.add_import(&green, &purple); - - let result = graph.find_shortest_chain(&blue, &green).unwrap(); - - assert_eq!(result, vec![&blue, &red, &orange, &green]) - } - - // find_shortest_chains - - #[test] - fn find_shortest_chains_none() { - let mut graph = Graph::default(); - let mypackage = Module::new("mypackage".to_string()); - let blue = Module::new("mypackage.blue".to_string()); - let green = Module::new("mypackage.green".to_string()); - let purple = Module::new("mypackage.purple".to_string()); - graph.add_module(mypackage.clone()); - graph.add_module(blue.clone()); - graph.add_module(green.clone()); - graph.add_module(purple.clone()); - // Add imports that are irrelevant. - graph.add_import(&purple, &blue); - graph.add_import(&green, &purple); - - let result = graph.find_shortest_chains(&blue, &green, true); - - assert_eq!(result, Ok(FxHashSet::default())); - } - - #[test] - fn find_shortest_chains_between_passed_modules() { - let mut graph = Graph::default(); - let mypackage = Module::new("mypackage".to_string()); - let blue = Module::new("mypackage.blue".to_string()); - let green = Module::new("mypackage.green".to_string()); - let red = Module::new("mypackage.red".to_string()); - let purple = Module::new("mypackage.purple".to_string()); - graph.add_module(mypackage.clone()); - graph.add_module(blue.clone()); - graph.add_module(green.clone()); - graph.add_module(red.clone()); - graph.add_module(purple.clone()); - // Add a chain. - graph.add_import(&blue, &red); - graph.add_import(&red, &green); - // Add other imports that are irrelevant. - graph.add_import(&purple, &blue); - graph.add_import(&green, &purple); - - let result = graph.find_shortest_chains(&blue, &green, true); - - assert_eq!(result, Ok(FxHashSet::from_iter([vec![blue, red, green],]))); - } - - #[test] - fn find_shortest_chains_between_passed_module_and_child() { - let mut graph = Graph::default(); - let mypackage = Module::new("mypackage".to_string()); - let blue = Module::new("mypackage.blue".to_string()); - let green = Module::new("mypackage.green".to_string()); - let green_alpha = Module::new("mypackage.green.alpha".to_string()); - let red = Module::new("mypackage.red".to_string()); - let purple = Module::new("mypackage.purple".to_string()); - graph.add_module(mypackage.clone()); - graph.add_module(blue.clone()); - graph.add_module(green.clone()); - graph.add_module(green_alpha.clone()); - graph.add_module(red.clone()); - graph.add_module(purple.clone()); - // Add a chain. - graph.add_import(&blue, &red); - graph.add_import(&red, &green_alpha); - // Add other imports that are irrelevant. - graph.add_import(&purple, &blue); - graph.add_import(&green, &purple); - - let result = graph.find_shortest_chains(&blue, &green, true); - - assert_eq!( - result, - Ok(FxHashSet::from_iter([vec![blue, red, green_alpha]])) - ); - } - - #[test] - fn find_shortest_chains_between_passed_module_and_grandchild() { - let mut graph = Graph::default(); - let mypackage = Module::new("mypackage".to_string()); - let blue = Module::new("mypackage.blue".to_string()); - let green = Module::new("mypackage.green".to_string()); - let green_alpha = Module::new("mypackage.green.alpha".to_string()); - let green_alpha_one = Module::new("mypackage.green.alpha.one".to_string()); - let red = Module::new("mypackage.red".to_string()); - let purple = Module::new("mypackage.purple".to_string()); - graph.add_module(mypackage.clone()); - graph.add_module(blue.clone()); - graph.add_module(green.clone()); - graph.add_module(green_alpha.clone()); - graph.add_module(green_alpha_one.clone()); - graph.add_module(red.clone()); - graph.add_module(purple.clone()); - // Add a chain. - graph.add_import(&blue, &red); - graph.add_import(&red, &green_alpha_one); - // Add other imports that are irrelevant. - graph.add_import(&purple, &blue); - graph.add_import(&green, &purple); - - let result = graph.find_shortest_chains(&blue, &green, true); - - assert_eq!( - result, - Ok(FxHashSet::from_iter([vec![blue, red, green_alpha_one],])) - ) - } - - #[test] - fn find_shortest_chains_between_child_and_passed_module() { - let mut graph = Graph::default(); - let mypackage = Module::new("mypackage".to_string()); - let blue = Module::new("mypackage.blue".to_string()); - let blue_alpha = Module::new("mypackage.blue.alpha".to_string()); - let green = Module::new("mypackage.green".to_string()); - let red = Module::new("mypackage.red".to_string()); - let purple = Module::new("mypackage.purple".to_string()); - graph.add_module(mypackage.clone()); - graph.add_module(blue.clone()); - graph.add_module(blue_alpha.clone()); - graph.add_module(green.clone()); - graph.add_module(red.clone()); - graph.add_module(purple.clone()); - // Add a chain. - graph.add_import(&blue_alpha, &red); - graph.add_import(&red, &green); - // Add other imports that are irrelevant. - graph.add_import(&purple, &blue); - graph.add_import(&green, &purple); - - let result = graph.find_shortest_chains(&blue, &green, true); - - assert_eq!( - result, - Ok(FxHashSet::from_iter([vec![blue_alpha, red, green],])) - ); - } - - #[test] - fn find_shortest_chains_between_grandchild_and_passed_module() { - let mut graph = Graph::default(); - let mypackage = Module::new("mypackage".to_string()); - let blue = Module::new("mypackage.blue".to_string()); - let blue_alpha = Module::new("mypackage.blue.alpha".to_string()); - let blue_alpha_one = Module::new("mypackage.blue.alpha.one".to_string()); - let green = Module::new("mypackage.green".to_string()); - let red = Module::new("mypackage.red".to_string()); - let purple = Module::new("mypackage.purple".to_string()); - graph.add_module(mypackage.clone()); - graph.add_module(blue.clone()); - graph.add_module(blue_alpha.clone()); - graph.add_module(blue_alpha_one.clone()); - graph.add_module(green.clone()); - graph.add_module(red.clone()); - graph.add_module(purple.clone()); - // Add a chain. - graph.add_import(&blue_alpha_one, &red); - graph.add_import(&red, &green); - // Add other imports that are irrelevant. - graph.add_import(&purple, &blue); - graph.add_import(&green, &purple); - - let result = graph.find_shortest_chains(&blue, &green, true); - - assert_eq!( - result, - Ok(FxHashSet::from_iter([vec![blue_alpha_one, red, green],])) - ) - } - - #[test] - fn chain_exists_true_as_packages_false() { - let mut graph = Graph::default(); - let mypackage = Module::new("mypackage".to_string()); - let blue = Module::new("mypackage.blue".to_string()); - let blue_alpha = Module::new("mypackage.blue.alpha".to_string()); - let blue_alpha_one = Module::new("mypackage.blue.alpha.one".to_string()); - let green = Module::new("mypackage.green".to_string()); - let red = Module::new("mypackage.red".to_string()); - let purple = Module::new("mypackage.purple".to_string()); - graph.add_module(mypackage.clone()); - graph.add_module(blue.clone()); - graph.add_module(blue_alpha.clone()); - graph.add_module(blue_alpha_one.clone()); - graph.add_module(green.clone()); - graph.add_module(red.clone()); - graph.add_module(purple.clone()); - // Add a chain. - graph.add_import(&blue_alpha_one, &red); - graph.add_import(&red, &green); - // Add other imports that are irrelevant. - graph.add_import(&purple, &blue); - graph.add_import(&green, &purple); - - let result = graph.chain_exists(&blue_alpha_one, &green, false); - - assert!(result); - } - - #[test] - fn chain_exists_false_as_packages_false() { - let mut graph = Graph::default(); - let mypackage = Module::new("mypackage".to_string()); - let blue = Module::new("mypackage.blue".to_string()); - let blue_alpha = Module::new("mypackage.blue.alpha".to_string()); - let blue_alpha_one = Module::new("mypackage.blue.alpha.one".to_string()); - let green = Module::new("mypackage.green".to_string()); - let red = Module::new("mypackage.red".to_string()); - let purple = Module::new("mypackage.purple".to_string()); - graph.add_module(mypackage.clone()); - graph.add_module(blue.clone()); - graph.add_module(blue_alpha.clone()); - graph.add_module(blue_alpha_one.clone()); - graph.add_module(green.clone()); - graph.add_module(red.clone()); - graph.add_module(purple.clone()); - // Add a chain. - graph.add_import(&blue_alpha_one, &red); - graph.add_import(&red, &green); - - let result = graph.chain_exists(&blue, &green, false); - - assert_eq!(result, false); - } - - #[test] - fn chain_exists_true_as_packages_true() { - let mut graph = Graph::default(); - let mypackage = Module::new("mypackage".to_string()); - let blue = Module::new("mypackage.blue".to_string()); - let blue_alpha = Module::new("mypackage.blue.alpha".to_string()); - let blue_alpha_one = Module::new("mypackage.blue.alpha.one".to_string()); - let green = Module::new("mypackage.green".to_string()); - let red = Module::new("mypackage.red".to_string()); - let purple = Module::new("mypackage.purple".to_string()); - graph.add_module(mypackage.clone()); - graph.add_module(blue.clone()); - graph.add_module(blue_alpha.clone()); - graph.add_module(blue_alpha_one.clone()); - graph.add_module(green.clone()); - graph.add_module(red.clone()); - graph.add_module(purple.clone()); - // Add a chain. - graph.add_import(&blue_alpha_one, &red); - graph.add_import(&red, &green); - - let result = graph.chain_exists(&blue, &green, true); - - assert_eq!(result, true); - } - - #[test] - fn chain_exists_false_as_packages_true() { - let mut graph = Graph::default(); - let mypackage = Module::new("mypackage".to_string()); - let blue = Module::new("mypackage.blue".to_string()); - let blue_alpha = Module::new("mypackage.blue.alpha".to_string()); - let blue_alpha_one = Module::new("mypackage.blue.alpha.one".to_string()); - let green = Module::new("mypackage.green".to_string()); - let red = Module::new("mypackage.red".to_string()); - let purple = Module::new("mypackage.purple".to_string()); - graph.add_module(mypackage.clone()); - graph.add_module(blue.clone()); - graph.add_module(blue_alpha.clone()); - graph.add_module(blue_alpha_one.clone()); - graph.add_module(green.clone()); - graph.add_module(red.clone()); - graph.add_module(purple.clone()); - // Add a chain. - graph.add_import(&blue_alpha_one, &red); - graph.add_import(&red, &green); - - let result = graph.chain_exists(&green, &blue, true); - - assert_eq!(result, false); - } - - #[test] - fn find_illegal_dependencies_for_layers_empty_everything() { - let graph = Graph::default(); - - let dependencies = graph.find_illegal_dependencies_for_layers(vec![], FxHashSet::default()); - - assert_eq!(dependencies, Ok(vec![])); - } - - #[test] - fn find_illegal_dependencies_for_layers_no_such_container() { - let graph = Graph::default(); - let container = "nonexistent_container".to_string(); - - let dependencies = graph.find_illegal_dependencies_for_layers( - vec![], - FxHashSet::from_iter([container.clone()]), - ); - - assert_eq!( - dependencies, - Err(NoSuchContainer { - container: container - }) - ); - } - - #[test] - fn find_illegal_dependencies_for_layers_nonexistent_layers_no_container() { - let graph = Graph::default(); - let level = Level { - layers: vec!["nonexistent".to_string()], - independent: true, - }; - - let dependencies = - graph.find_illegal_dependencies_for_layers(vec![level], FxHashSet::default()); - - assert_eq!(dependencies, Ok(vec![])); - } - - #[test] - fn find_illegal_dependencies_for_layers_nonexistent_layers_with_container() { - let mut graph = Graph::default(); - graph.add_module(Module::new("mypackage".to_string())); - let level = Level { - layers: vec!["nonexistent".to_string()], - independent: true, - }; - let container = "mypackage".to_string(); - - let dependencies = graph - .find_illegal_dependencies_for_layers(vec![level], FxHashSet::from_iter([container])); - - assert_eq!(dependencies, Ok(vec![])); - } - - #[test] - fn find_illegal_dependencies_for_layers_no_container_direct_dependency() { - let mut graph = Graph::default(); - let high = Module::new("high".to_string()); - let low = Module::new("low".to_string()); - graph.add_import(&low, &high); - let levels = vec![ - Level { - layers: vec![high.name.clone()], - independent: true, - }, - Level { - layers: vec![low.name.clone()], - independent: true, - }, - ]; - - let dependencies = graph.find_illegal_dependencies_for_layers(levels, FxHashSet::default()); - - assert_eq!( - dependencies, - Ok(vec![PackageDependency { - importer: low.clone(), - imported: high.clone(), - routes: vec![Route { - heads: vec![low.clone()], - middle: vec![], - tails: vec![high.clone()], - }] - }]) - ); - } - - #[test] - fn find_illegal_dependencies_for_layers_no_container_indirect_dependency() { - let mut graph = Graph::default(); - let high = Module::new("high".to_string()); - let elsewhere = Module::new("elsewhere".to_string()); - let low = Module::new("low".to_string()); - graph.add_import(&low, &elsewhere); - graph.add_import(&elsewhere, &high); - let levels = vec![ - Level { - layers: vec![high.name.clone()], - independent: true, - }, - Level { - layers: vec![low.name.clone()], - independent: true, - }, - ]; - - let dependencies = graph.find_illegal_dependencies_for_layers(levels, FxHashSet::default()); - - assert_eq!( - dependencies, - Ok(vec![PackageDependency { - importer: low.clone(), - imported: high.clone(), - routes: vec![Route { - heads: vec![low.clone()], - middle: vec![elsewhere.clone()], - tails: vec![high.clone()], - }] - }]) - ); - } - - #[test] - fn find_illegal_dependencies_for_layers_containers() { - let mut graph = Graph::default(); - let blue_high = Module::new("blue.high".to_string()); - let blue_high_alpha = Module::new("blue.high.alpha".to_string()); - let blue_low = Module::new("blue.low".to_string()); - let blue_low_beta = Module::new("blue.low.beta".to_string()); - let green_high = Module::new("green.high".to_string()); - let green_high_gamma = Module::new("green.high.gamma".to_string()); - let green_low = Module::new("green.low".to_string()); - let green_low_delta = Module::new("green.low.delta".to_string()); - graph.add_module(Module::new("blue".to_string())); - graph.add_module(blue_high.clone()); - graph.add_module(blue_low.clone()); - graph.add_module(Module::new("green".to_string())); - graph.add_module(green_high.clone()); - graph.add_module(green_low.clone()); - graph.add_import(&blue_low_beta, &blue_high_alpha); - graph.add_import(&green_low_delta, &green_high_gamma); - - let levels = vec![ - Level { - layers: vec!["high".to_string()], - independent: true, - }, - Level { - layers: vec!["low".to_string()], - independent: true, - }, - ]; - let containers = FxHashSet::from_iter(["blue".to_string(), "green".to_string()]); - - let dependencies = graph.find_illegal_dependencies_for_layers(levels, containers); - - assert_eq!( - dependencies, - Ok(vec![ - PackageDependency { - importer: blue_low.clone(), - imported: blue_high.clone(), - routes: vec![Route { - heads: vec![blue_low_beta.clone()], - middle: vec![], - tails: vec![blue_high_alpha.clone()], - }] - }, - PackageDependency { - importer: green_low.clone(), - imported: green_high.clone(), - routes: vec![Route { - heads: vec![green_low_delta.clone()], - middle: vec![], - tails: vec![green_high_gamma.clone()], - }] - } - ]) - ); - } - - #[test] - fn find_illegal_dependencies_for_layers_independent() { - let mut graph = Graph::default(); - let blue = Module::new("blue".to_string()); - let green = Module::new("green".to_string()); - let blue_alpha = Module::new("blue.alpha".to_string()); - let green_beta = Module::new("green.beta".to_string()); - graph.add_module(blue.clone()); - graph.add_module(green.clone()); - graph.add_import(&blue_alpha, &green_beta); - - let levels = vec![Level { - layers: vec![blue.name.clone(), green.name.clone()], - independent: true, - }]; - - let dependencies = graph.find_illegal_dependencies_for_layers(levels, FxHashSet::default()); - - assert_eq!( - dependencies, - Ok(vec![PackageDependency { - importer: blue.clone(), - imported: green.clone(), - routes: vec![Route { - heads: vec![blue_alpha.clone()], - middle: vec![], - tails: vec![green_beta.clone()], - }] - }]) - ); - } - - #[test] - fn get_import_details_no_modules() { - let graph = Graph::default(); - let importer = Module::new("foo".to_string()); - let imported = Module::new("bar".to_string()); - - let result = graph.get_import_details(&importer, &imported); - - assert_eq!(result, FxHashSet::default()); - } - - #[test] - fn get_import_details_module_without_metadata() { - let mut graph = Graph::default(); - let importer = Module::new("foo".to_string()); - let imported = Module::new("bar".to_string()); - graph.add_import(&importer, &imported); - - let result = graph.get_import_details(&importer, &imported); - - assert_eq!(result, FxHashSet::default()); - } - - #[test] - fn get_import_details_module_one_result() { - let mut graph = Graph::default(); - let importer = Module::new("foo".to_string()); - let imported = Module::new("bar".to_string()); - let import = DetailedImport { - importer: importer.clone(), - imported: imported.clone(), - line_number: 5, - line_contents: "import bar".to_string(), - }; - let unrelated_import = DetailedImport { - importer: importer.clone(), - imported: Module::new("baz".to_string()), - line_number: 2, - line_contents: "-".to_string(), - }; - graph.add_detailed_import(&import); - graph.add_detailed_import(&unrelated_import); - - let result = graph.get_import_details(&importer, &imported); - - assert_eq!(result, FxHashSet::from_iter([import])); - } - - #[test] - fn get_import_details_module_two_results() { - let mut graph = Graph::default(); - let blue = Module::new("blue".to_string()); - let green = Module::new("green".to_string()); - let blue_to_green_a = DetailedImport { - importer: blue.clone(), - imported: green.clone(), - line_number: 5, - line_contents: "import green".to_string(), - }; - let blue_to_green_b = DetailedImport { - importer: blue.clone(), - imported: green.clone(), - line_number: 15, - line_contents: "import green".to_string(), - }; - graph.add_detailed_import(&blue_to_green_a); - graph.add_detailed_import(&blue_to_green_b); - - let result = graph.get_import_details(&blue, &green); - - assert_eq!( - result, - FxHashSet::from_iter([blue_to_green_a, blue_to_green_b]) - ); - } - - #[test] - fn get_import_details_after_removal() { - let mut graph = Graph::default(); - let importer = Module::new("foo".to_string()); - let imported = Module::new("bar".to_string()); - let import = DetailedImport { - importer: importer.clone(), - imported: imported.clone(), - line_number: 5, - line_contents: "import bar".to_string(), - }; - let unrelated_import = DetailedImport { - importer: importer.clone(), - imported: Module::new("baz".to_string()), - line_number: 2, - line_contents: "-".to_string(), - }; - graph.add_detailed_import(&import); - graph.add_detailed_import(&unrelated_import); - graph.remove_import(&import.importer, &import.imported); - - let result = graph.get_import_details(&importer, &imported); - - assert_eq!(result, FxHashSet::default()); - } - - #[test] - fn get_import_details_after_removal_of_unrelated_import() { - let mut graph = Graph::default(); - let importer = Module::new("foo".to_string()); - let imported = Module::new("bar".to_string()); - let import = DetailedImport { - importer: importer.clone(), - imported: imported.clone(), - line_number: 5, - line_contents: "import bar".to_string(), - }; - let unrelated_import = DetailedImport { - importer: importer.clone(), - imported: Module::new("baz".to_string()), - line_number: 2, - line_contents: "-".to_string(), - }; - graph.add_detailed_import(&import); - graph.add_detailed_import(&unrelated_import); - graph.remove_import(&unrelated_import.importer, &unrelated_import.imported); - - let result = graph.get_import_details(&importer, &imported); - - assert_eq!(result, FxHashSet::from_iter([import])); - } -} diff --git a/rust/src/graph/direct_import_queries.rs b/rust/src/graph/direct_import_queries.rs new file mode 100644 index 00000000..2023e94d --- /dev/null +++ b/rust/src/graph/direct_import_queries.rs @@ -0,0 +1,57 @@ +use crate::errors::{GrimpError, GrimpResult}; +use crate::graph::{ + ExtendWithDescendants, Graph, ImportDetails, ModuleToken, EMPTY_IMPORT_DETAILS, + EMPTY_MODULE_TOKENS, +}; +use rustc_hash::FxHashSet; + +impl Graph { + pub fn count_imports(&self) -> usize { + self.imports.values().map(|imports| imports.len()).sum() + } + + pub fn direct_import_exists( + &self, + importer: ModuleToken, + imported: ModuleToken, + as_packages: bool, + ) -> GrimpResult { + let mut importer: FxHashSet<_> = importer.into(); + let mut imported: FxHashSet<_> = imported.into(); + if as_packages { + importer.extend_with_descendants(self); + imported.extend_with_descendants(self); + if !(&importer & &imported).is_empty() { + return Err(GrimpError::SharedDescendants); + } + } + + let direct_imports = importer + .iter() + .flat_map(|module| self.imports.get(*module).unwrap().iter().cloned()) + .collect::>(); + + Ok(!(&direct_imports & &imported).is_empty()) + } + + pub fn modules_directly_imported_by(&self, importer: ModuleToken) -> &FxHashSet { + self.imports.get(importer).unwrap_or(&EMPTY_MODULE_TOKENS) + } + + pub fn modules_that_directly_import(&self, imported: ModuleToken) -> &FxHashSet { + self.reverse_imports + .get(imported) + .unwrap_or(&EMPTY_MODULE_TOKENS) + } + + pub fn get_import_details( + &self, + importer: ModuleToken, + imported: ModuleToken, + ) -> &FxHashSet { + match self.import_details.get(&(importer, imported)) { + Some(import_details) => import_details, + None => &EMPTY_IMPORT_DETAILS, + } + } +} diff --git a/rust/src/graph/graph_manipulation.rs b/rust/src/graph/graph_manipulation.rs new file mode 100644 index 00000000..41428236 --- /dev/null +++ b/rust/src/graph/graph_manipulation.rs @@ -0,0 +1,195 @@ +use crate::graph::{Graph, ImportDetails, Module, ModuleIterator, ModuleToken, MODULE_NAMES}; +use rustc_hash::FxHashSet; +use slotmap::secondary::Entry; + +impl Graph { + /// `foo.bar.baz => [foo.bar.baz, foo.bar, foo]` + pub(crate) fn module_name_to_self_and_ancestors(&self, name: &str) -> Vec { + let mut names = vec![name.to_owned()]; + while let Some(parent_name) = parent_name(names.last().unwrap()) { + names.push(parent_name); + } + names + } + + pub fn get_or_add_module(&mut self, name: &str) -> &Module { + if let Some(module) = self.get_module_by_name(name) { + let module = self.modules.get_mut(module.token).unwrap(); + module.is_invisible = false; + return module; + } + + let mut ancestor_names = self.module_name_to_self_and_ancestors(name); + + { + let mut interner = MODULE_NAMES.write().unwrap(); + let mut parent: Option = None; + while let Some(name) = ancestor_names.pop() { + let name = interner.get_or_intern(name); + if let Some(module) = self.modules_by_name.get_by_left(&name) { + parent = Some(*module) + } else { + let module = self.modules.insert_with_key(|token| Module { + token, + name, + is_invisible: !ancestor_names.is_empty(), + is_squashed: false, + }); + self.modules_by_name.insert(name, module); + self.module_parents.insert(module, parent); + self.module_children.insert(module, FxHashSet::default()); + self.imports.insert(module, FxHashSet::default()); + self.reverse_imports.insert(module, FxHashSet::default()); + if let Some(parent) = parent { + self.module_children[parent].insert(module); + } + parent = Some(module) + } + } + } + + self.get_module_by_name(name).unwrap() + } + + pub fn get_or_add_squashed_module(&mut self, module: &str) -> &Module { + let module = self.get_or_add_module(module).token(); + self.mark_module_squashed(module); + self.get_module(module).unwrap() + } + + fn mark_module_squashed(&mut self, module: ModuleToken) { + let module = self.modules.get_mut(module).unwrap(); + if !self.module_children[module.token].is_empty() { + panic!("cannot mark a module with children as squashed") + } + module.is_squashed = true; + } + + pub fn remove_module(&mut self, module: ModuleToken) { + let module = self.get_module(module); + if module.is_none() { + return; + } + let module = module.unwrap().token(); + + // TODO(peter) Remove children automatically here, or raise an error? + if !self.module_children[module].is_empty() { + for child in self.module_children[module].clone() { + self.remove_module(child); + } + } + + // Update hierarchy. + if let Some(parent) = self.module_parents[module] { + self.module_children[parent].remove(&module); + } + self.modules_by_name.remove_by_right(&module); + self.modules.remove(module); + self.module_parents.remove(module); + self.module_children.remove(module); + + // Update imports. + for imported in self.modules_directly_imported_by(module).clone() { + self.remove_import(module, imported); + } + for importer in self.modules_that_directly_import(module).clone() { + self.remove_import(importer, module); + } + self.imports.remove(module); + self.reverse_imports.remove(module); + } + + pub fn add_import(&mut self, importer: ModuleToken, imported: ModuleToken) { + self.imports + .entry(importer) + .unwrap() + .or_default() + .insert(imported); + self.reverse_imports + .entry(imported) + .unwrap() + .or_default() + .insert(importer); + } + + pub fn add_detailed_import( + &mut self, + importer: ModuleToken, + imported: ModuleToken, + line_number: usize, + line_contents: &str, + ) { + self.imports + .entry(importer) + .unwrap() + .or_default() + .insert(imported); + self.reverse_imports + .entry(imported) + .unwrap() + .or_default() + .insert(importer); + self.import_details + .entry((importer, imported)) + .or_default() + .insert(ImportDetails::new(line_number, line_contents.to_owned())); + } + + pub fn remove_import(&mut self, importer: ModuleToken, imported: ModuleToken) { + match self.imports.entry(importer).unwrap() { + Entry::Occupied(mut entry) => { + entry.get_mut().remove(&imported); + } + Entry::Vacant(_) => {} + }; + match self.reverse_imports.entry(imported).unwrap() { + Entry::Occupied(mut entry) => { + entry.get_mut().remove(&importer); + } + Entry::Vacant(_) => {} + }; + self.import_details.remove(&(importer, imported)); + } + + pub fn squash_module(&mut self, module: ModuleToken) { + // Get descendants and their imports. + let descendants: FxHashSet<_> = self.get_module_descendants(module).tokens().collect(); + + let modules_imported_by_descendants: FxHashSet<_> = descendants + .iter() + .flat_map(|descendant| { + self.modules_directly_imported_by(*descendant) + .iter() + .cloned() + }) + .collect(); + let modules_that_import_descendants: FxHashSet<_> = descendants + .iter() + .flat_map(|descendant| { + self.modules_that_directly_import(*descendant) + .iter() + .cloned() + }) + .collect(); + + // Remove any descendants. + for descendant in descendants { + self.remove_module(descendant); + } + + // Add descendants and imports to parent module. + for imported in modules_imported_by_descendants { + self.add_import(module, imported); + } + + for importer in modules_that_import_descendants { + self.add_import(importer, module); + } + + self.mark_module_squashed(module); + } +} + +fn parent_name(name: &str) -> Option { + name.rsplit_once(".").map(|(base, _)| base.to_owned()) +} diff --git a/rust/src/graph/hierarchy_queries.rs b/rust/src/graph/hierarchy_queries.rs new file mode 100644 index 00000000..41cdf570 --- /dev/null +++ b/rust/src/graph/hierarchy_queries.rs @@ -0,0 +1,50 @@ +use crate::graph::{Graph, Module, ModuleToken, MODULE_NAMES}; + +impl Graph { + pub fn get_module_by_name(&self, name: &str) -> Option<&Module> { + let interner = MODULE_NAMES.read().unwrap(); + let name = interner.get(name)?; + match self.modules_by_name.get_by_left(&name) { + Some(token) => self.get_module(*token), + None => None, + } + } + + pub fn get_module(&self, module: ModuleToken) -> Option<&Module> { + self.modules.get(module) + } + + // TODO(peter) Guarantee order? + pub fn all_modules(&self) -> impl Iterator { + self.modules.values() + } + + pub fn get_module_parent(&self, module: ModuleToken) -> Option<&Module> { + match self.module_parents.get(module) { + Some(parent) => parent.map(|parent| self.get_module(parent).unwrap()), + None => None, + } + } + + pub fn get_module_children(&self, module: ModuleToken) -> impl Iterator { + let children = match self.module_children.get(module) { + Some(children) => children + .iter() + .map(|child| self.get_module(*child).unwrap()) + .collect(), + None => Vec::new(), + }; + children.into_iter() + } + + /// Returns an iterator over the passed modules descendants. + /// + /// Parent modules will be yielded before their child modules. + pub fn get_module_descendants(&self, module: ModuleToken) -> impl Iterator { + let mut descendants = self.get_module_children(module).collect::>(); + for child in descendants.clone() { + descendants.extend(self.get_module_descendants(child.token).collect::>()) + } + descendants.into_iter() + } +} diff --git a/rust/src/graph/higher_order_queries.rs b/rust/src/graph/higher_order_queries.rs new file mode 100644 index 00000000..ebb1ba71 --- /dev/null +++ b/rust/src/graph/higher_order_queries.rs @@ -0,0 +1,210 @@ +use crate::errors::GrimpResult; +use crate::graph::{ExtendWithDescendants, Graph, ModuleToken}; +use derive_new::new; +use getset::Getters; +use itertools::Itertools; +use rayon::prelude::*; +use rustc_hash::{FxHashMap, FxHashSet}; + +use tap::prelude::*; + +#[derive(Debug, Clone, PartialEq, Eq, new, Getters)] +pub struct Level { + #[new(into)] + #[getset(get = "pub")] + layers: FxHashSet, + + #[getset(get_copy = "pub")] + independent: bool, +} + +#[derive(Debug, Clone, PartialEq, Eq, new, Getters)] +pub struct PackageDependency { + #[getset(get = "pub")] + importer: ModuleToken, + + #[getset(get = "pub")] + imported: ModuleToken, + + #[new(into)] + #[getset(get = "pub")] + routes: Vec, +} + +#[derive(Debug, Clone, PartialEq, Eq, new, Getters)] +pub struct Route { + #[new(into)] + #[getset(get = "pub")] + heads: FxHashSet, + + #[new(into)] + #[getset(get = "pub")] + middle: Vec, + + #[new(into)] + #[getset(get = "pub")] + tails: FxHashSet, +} + +impl Graph { + pub fn find_illegal_dependencies_for_layers( + &self, + levels: &[Level], + ) -> GrimpResult> { + let all_layer_modules = levels + .iter() + .flat_map(|level| level.layers().clone()) + .flat_map(|m| m.conv::>().with_descendants(self)) + .collect::>(); + + self.generate_module_permutations(levels) + .into_par_iter() + .try_fold( + Vec::new, + |mut v: Vec, (from_package, to_package)| -> GrimpResult<_> { + if let Some(dep) = self.find_illegal_dependencies( + from_package, + to_package, + &all_layer_modules, + )? { + v.push(dep); + } + Ok(v) + }, + ) + .try_reduce( + Vec::new, + |mut v: Vec, package_dependencies| { + v.extend(package_dependencies); + Ok(v) + }, + ) + } + + fn generate_module_permutations(&self, levels: &[Level]) -> Vec<(ModuleToken, ModuleToken)> { + let mut permutations = vec![]; + + for (index, level) in levels.iter().enumerate() { + for module in &level.layers { + // Should not be imported by lower layers. + for lower_level in &levels[index + 1..] { + for lower_module in &lower_level.layers { + permutations.push((*lower_module, *module)); + } + } + + // Should not import siblings (if level is independent) + if level.independent { + for sibling_module in &level.layers { + if sibling_module == module { + continue; + } + permutations.push((*module, *sibling_module)); + } + } + } + } + + permutations + } + + fn find_illegal_dependencies( + &self, + from_layer: ModuleToken, + to_layer: ModuleToken, + all_layers_modules: &FxHashSet, + ) -> GrimpResult> { + // Shortcut the detailed implementation in the case of no chains. + // This will be much faster! + if !self.chain_exists(from_layer, to_layer, true)? { + return Ok(None); + } + + let from_layer_with_descendants = from_layer.conv::>().with_descendants(self); + let to_layer_with_descendants = to_layer.conv::>().with_descendants(self); + + // Disallow chains via other layers. + let excluded_modules = + all_layers_modules - &(&from_layer_with_descendants | &to_layer_with_descendants); + + // Disallow chains via these imports. + // We'll add chains to this set as we discover them. + let mut excluded_imports = FxHashMap::default(); + + // Collect direct imports... + let mut direct_imports = vec![]; + // ...and the middles of any indirect imports. + let mut middles = vec![]; + loop { + let chain = self.find_shortest_chain_with_excluded_modules_and_imports( + &from_layer_with_descendants, + &to_layer_with_descendants, + &excluded_modules, + &excluded_imports, + )?; + + if chain.is_none() { + break; + } + let chain = chain.unwrap(); + + // Exclude this chain from further searching. + for (importer, imported) in chain.iter().tuple_windows() { + excluded_imports + .entry(*importer) + .or_default() + .insert(*imported); + } + + let (head, middle, tail) = self.split_chain(&chain); + match middle { + Some(middle) => middles.push(middle), + None => direct_imports.push((head, tail)), + } + } + + // Map to routes. + let mut routes = vec![]; + for (importer, imported) in direct_imports { + routes.push(Route::new(importer, vec![], imported)); + } + for middle in middles { + let heads = from_layer_with_descendants + .iter() + .filter(|importer| { + self.direct_import_exists(**importer, *middle.first().unwrap(), false) + .unwrap() + }) + .cloned() + .collect::>(); + let tails = to_layer_with_descendants + .iter() + .filter(|imported| { + self.direct_import_exists(*middle.last().unwrap(), **imported, false) + .unwrap() + }) + .cloned() + .collect::>(); + routes.push(Route::new(heads, middle, tails)); + } + + match routes.is_empty() { + true => Ok(None), + false => Ok(Some(PackageDependency::new(from_layer, to_layer, routes))), + } + } + + fn split_chain( + &self, + chain: &[ModuleToken], + ) -> (ModuleToken, Option>, ModuleToken) { + if chain.len() == 2 { + return (chain[0], None, chain[1]); + } + ( + chain[0], + Some(chain[1..chain.len() - 1].to_vec()), + chain[chain.len() - 1], + ) + } +} diff --git a/rust/src/graph/import_chain_queries.rs b/rust/src/graph/import_chain_queries.rs new file mode 100644 index 00000000..4f21da48 --- /dev/null +++ b/rust/src/graph/import_chain_queries.rs @@ -0,0 +1,123 @@ +use crate::errors::GrimpResult; +use crate::graph::pathfinding::{find_reach, find_shortest_path_bidirectional}; +use crate::graph::{ExtendWithDescendants, Graph, ModuleToken}; +use itertools::Itertools; +use rustc_hash::{FxHashMap, FxHashSet}; +use tap::Conv; + +impl Graph { + pub fn find_downstream_modules( + &self, + module: ModuleToken, + as_package: bool, + ) -> FxHashSet { + let mut from_modules = module.conv::>(); + if as_package { + from_modules.extend_with_descendants(self); + } + + find_reach(&self.reverse_imports, &from_modules) + } + + pub fn find_upstream_modules( + &self, + module: ModuleToken, + as_package: bool, + ) -> FxHashSet { + let mut from_modules = module.conv::>(); + if as_package { + from_modules.extend_with_descendants(self); + } + + find_reach(&self.imports, &from_modules) + } + + pub fn find_shortest_chain( + &self, + importer: ModuleToken, + imported: ModuleToken, + as_packages: bool, + ) -> GrimpResult>> { + if as_packages { + self.find_shortest_chain_with_excluded_modules_and_imports( + &importer.conv::>().with_descendants(self), + &imported.conv::>().with_descendants(self), + &FxHashSet::default(), + &FxHashMap::default(), + ) + } else { + self.find_shortest_chain_with_excluded_modules_and_imports( + &importer.conv::>(), + &imported.conv::>(), + &FxHashSet::default(), + &FxHashMap::default(), + ) + } + } + + pub(crate) fn find_shortest_chain_with_excluded_modules_and_imports( + &self, + from_modules: &FxHashSet, + to_modules: &FxHashSet, + excluded_modules: &FxHashSet, + excluded_imports: &FxHashMap>, + ) -> GrimpResult>> { + find_shortest_path_bidirectional( + self, + from_modules, + to_modules, + excluded_modules, + excluded_imports, + ) + } + + pub fn chain_exists( + &self, + importer: ModuleToken, + imported: ModuleToken, + as_packages: bool, + ) -> GrimpResult { + Ok(self + .find_shortest_chain(importer, imported, as_packages)? + .is_some()) + } + + pub fn find_shortest_chains( + &self, + importer: ModuleToken, + imported: ModuleToken, + as_packages: bool, + ) -> GrimpResult>> { + // Shortcut the detailed implementation in the case of no chains. + // This will be much faster! + if !self.chain_exists(importer, imported, as_packages)? { + return Ok(FxHashSet::default()); + } + + let mut downstream_modules = FxHashSet::from_iter([importer]); + let mut upstream_modules = FxHashSet::from_iter([imported]); + if as_packages { + downstream_modules.extend_with_descendants(self); + upstream_modules.extend_with_descendants(self); + } + let all_modules = &downstream_modules | &upstream_modules; + + let chains = downstream_modules + .iter() + .cartesian_product(upstream_modules.iter()) + .filter_map(|(downstream_module, upstream_module)| { + let excluded_modules = + &all_modules - &FxHashSet::from_iter([*downstream_module, *upstream_module]); + self.find_shortest_chain_with_excluded_modules_and_imports( + &(*downstream_module).into(), + &(*upstream_module).into(), + &excluded_modules, + &FxHashMap::default(), + ) + .unwrap() + }) + .collect(); + + Ok(chains) + } +} diff --git a/rust/src/graph/mod.rs b/rust/src/graph/mod.rs new file mode 100644 index 00000000..cdbd5380 --- /dev/null +++ b/rust/src/graph/mod.rs @@ -0,0 +1,137 @@ +use bimap::BiMap; +use derive_new::new; +use getset::{CopyGetters, Getters}; +use lazy_static::lazy_static; +use rustc_hash::{FxHashMap, FxHashSet}; +use slotmap::{new_key_type, SecondaryMap, SlotMap}; +use std::sync::RwLock; +use string_interner::backend::StringBackend; +use string_interner::{DefaultSymbol, StringInterner}; + +pub mod direct_import_queries; +pub mod graph_manipulation; +pub mod hierarchy_queries; +pub mod higher_order_queries; +pub mod import_chain_queries; + +pub(crate) mod pathfinding; + +lazy_static! { + static ref MODULE_NAMES: RwLock> = + RwLock::new(StringInterner::default()); + static ref EMPTY_MODULE_TOKENS: FxHashSet = FxHashSet::default(); + static ref EMPTY_IMPORT_DETAILS: FxHashSet = FxHashSet::default(); + static ref EMPTY_IMPORTS: FxHashSet<(ModuleToken, ModuleToken)> = FxHashSet::default(); +} + +new_key_type! { pub struct ModuleToken; } + +#[derive(Debug, Clone, PartialEq, Eq, Hash, Getters, CopyGetters)] +pub struct Module { + #[getset(get_copy = "pub")] + token: ModuleToken, + + #[getset(get_copy = "pub")] + name: DefaultSymbol, + + // Invisible modules exist in the hierarchy but haven't been explicitly added to the graph. + #[getset(get_copy = "pub")] + is_invisible: bool, + + #[getset(get_copy = "pub")] + is_squashed: bool, +} + +impl Module { + pub fn name_as_string(&self) -> String { + let interner = MODULE_NAMES.read().unwrap(); + interner.resolve(self.name).unwrap().to_owned() + } +} + +#[derive(Default, Clone)] +pub struct Graph { + // Hierarchy + modules_by_name: BiMap, + modules: SlotMap, + module_parents: SecondaryMap>, + module_children: SecondaryMap>, + // Imports + imports: SecondaryMap>, + reverse_imports: SecondaryMap>, + import_details: FxHashMap<(ModuleToken, ModuleToken), FxHashSet>, +} + +impl From for Vec { + fn from(value: ModuleToken) -> Self { + vec![value] + } +} + +impl From for FxHashSet { + fn from(value: ModuleToken) -> Self { + FxHashSet::from_iter([value]) + } +} + +pub trait ExtendWithDescendants: + Sized + Clone + IntoIterator + Extend +{ + /// Extend this collection of module tokens with all descendant items. + fn extend_with_descendants(&mut self, graph: &Graph) { + for item in self.clone().into_iter() { + let descendants = graph.get_module_descendants(item).map(|item| item.token()); + self.extend(descendants); + } + } + + /// Extend this collection of module tokens with all descendant items. + fn with_descendants(mut self, graph: &Graph) -> Self { + self.extend_with_descendants(graph); + self + } +} + +impl + Extend> + ExtendWithDescendants for T +{ +} + +pub trait ModuleIterator<'a>: Iterator + Sized { + fn tokens(self) -> impl Iterator { + self.map(|m| m.token) + } + + fn names(self) -> impl Iterator { + self.map(|m| m.name) + } + + fn names_as_strings(self) -> impl Iterator { + let interner = MODULE_NAMES.read().unwrap(); + self.map(move |m| interner.resolve(m.name).unwrap().to_owned()) + } + + fn visible(self) -> impl ModuleIterator<'a> { + self.filter(|m| !m.is_invisible) + } +} + +impl<'a, T: Iterator> ModuleIterator<'a> for T {} + +pub trait ModuleTokenIterator<'a>: Iterator + Sized { + fn into_module_iterator(self, graph: &'a Graph) -> impl ModuleIterator<'a> { + self.map(|m| graph.get_module(*m).unwrap()) + } +} + +impl<'a, T: Iterator> ModuleTokenIterator<'a> for T {} + +#[derive(Debug, Clone, PartialEq, Eq, Hash, new, Getters, CopyGetters)] +pub struct ImportDetails { + #[getset(get_copy = "pub")] + line_number: usize, + + #[new(into)] + #[getset(get = "pub")] + line_contents: String, +} diff --git a/rust/src/graph/pathfinding.rs b/rust/src/graph/pathfinding.rs new file mode 100644 index 00000000..55e9450d --- /dev/null +++ b/rust/src/graph/pathfinding.rs @@ -0,0 +1,125 @@ +use crate::errors::{GrimpError, GrimpResult}; +use crate::graph::{Graph, ModuleToken, EMPTY_MODULE_TOKENS}; +use indexmap::{IndexMap, IndexSet}; +use rustc_hash::{FxHashMap, FxHashSet, FxHasher}; +use slotmap::SecondaryMap; +use std::hash::BuildHasherDefault; + +type FxIndexSet = IndexSet>; +type FxIndexMap = IndexMap>; + +pub fn find_reach( + imports_map: &SecondaryMap>, + from_modules: &FxHashSet, +) -> FxHashSet { + let mut seen = FxIndexSet::default(); + seen.extend(from_modules.iter().cloned()); + + let mut i = 0; + while let Some(module) = seen.get_index(i) { + for next_module in imports_map.get(*module).unwrap_or(&EMPTY_MODULE_TOKENS) { + if !seen.contains(next_module) { + seen.insert(*next_module); + } + } + i += 1; + } + + &seen.into_iter().collect::>() - from_modules +} + +pub fn find_shortest_path_bidirectional( + graph: &Graph, + from_modules: &FxHashSet, + to_modules: &FxHashSet, + excluded_modules: &FxHashSet, + excluded_imports: &FxHashMap>, +) -> GrimpResult>> { + if !(from_modules & to_modules).is_empty() { + return Err(GrimpError::SharedDescendants); + } + + let mut predecessors: FxIndexMap> = from_modules + .clone() + .into_iter() + .map(|m| (m, None)) + .collect(); + let mut successors: FxIndexMap> = + to_modules.clone().into_iter().map(|m| (m, None)).collect(); + + let mut i_forwards = 0; + let mut i_backwards = 0; + let middle = 'l: loop { + for _ in 0..(predecessors.len() - i_forwards) { + let module = *predecessors.get_index(i_forwards).unwrap().0; + let next_modules = graph.imports.get(module).unwrap(); + for next_module in next_modules { + if import_is_excluded(&module, next_module, excluded_modules, excluded_imports) { + continue; + } + if !predecessors.contains_key(next_module) { + predecessors.insert(*next_module, Some(module)); + } + if successors.contains_key(next_module) { + break 'l Some(*next_module); + } + } + i_forwards += 1; + } + + for _ in 0..(successors.len() - i_backwards) { + let module = *successors.get_index(i_backwards).unwrap().0; + let next_modules = graph.reverse_imports.get(module).unwrap(); + for next_module in next_modules { + if import_is_excluded(next_module, &module, excluded_modules, excluded_imports) { + continue; + } + if !successors.contains_key(next_module) { + successors.insert(*next_module, Some(module)); + } + if predecessors.contains_key(next_module) { + break 'l Some(*next_module); + } + } + i_backwards += 1; + } + + if i_forwards == predecessors.len() && i_backwards == successors.len() { + break 'l None; + } + }; + + Ok(middle.map(|middle| { + // Path found! + // Build the path. + let mut path = vec![]; + let mut node = Some(middle); + while let Some(n) = node { + path.push(n); + node = *predecessors.get(&n).unwrap(); + } + path.reverse(); + let mut node = *successors.get(path.last().unwrap()).unwrap(); + while let Some(n) = node { + path.push(n); + node = *successors.get(&n).unwrap(); + } + path + })) +} + +fn import_is_excluded( + from_module: &ModuleToken, + to_module: &ModuleToken, + excluded_modules: &FxHashSet, + excluded_imports: &FxHashMap>, +) -> bool { + if excluded_modules.contains(to_module) { + true + } else { + excluded_imports + .get(from_module) + .unwrap_or(&EMPTY_MODULE_TOKENS) + .contains(to_module) + } +} diff --git a/rust/src/lib.rs b/rust/src/lib.rs index 10d52631..d293f852 100644 --- a/rust/src/lib.rs +++ b/rust/src/lib.rs @@ -1,29 +1,43 @@ +pub mod errors; +pub mod exceptions; pub mod graph; -use crate::graph::{DetailedImport, Graph, Level, Module, PackageDependency}; -use log::info; -use pyo3::create_exception; +use crate::errors::{GrimpError, GrimpResult}; +use crate::exceptions::{ModuleNotPresent, NoSuchContainer}; +use crate::graph::higher_order_queries::Level; +use crate::graph::{Graph, Module, ModuleIterator, ModuleTokenIterator}; +use derive_new::new; +use itertools::Itertools; use pyo3::exceptions::PyValueError; use pyo3::prelude::*; -use pyo3::types::{PyDict, PyFrozenSet, PyList, PySet, PyString, PyTuple}; +use pyo3::types::{IntoPyDict, PyDict, PyFrozenSet, PyList, PySet, PyString, PyTuple}; +use pyo3::IntoPyObjectExt; +use rayon::prelude::*; use rustc_hash::FxHashSet; +use std::collections::HashSet; #[pymodule] fn _rustgrimp(py: Python<'_>, m: &Bound<'_, PyModule>) -> PyResult<()> { - pyo3_log::init(); - m.add_class::()?; + m.add("ModuleNotPresent", py.get_type::())?; m.add("NoSuchContainer", py.get_type::())?; Ok(()) } -create_exception!(_rustgrimp, NoSuchContainer, pyo3::exceptions::PyException); - #[pyclass(name = "Graph")] struct GraphWrapper { _graph: Graph, } +impl GraphWrapper { + fn get_visible_module_by_name(&self, name: &str) -> Result<&Module, GrimpError> { + self._graph + .get_module_by_name(name) + .filter(|m| !m.is_invisible()) + .ok_or(GrimpError::ModuleNotPresent(name.to_owned())) + } +} + /// Wrapper around the Graph struct that integrates with Python. #[pymethods] impl GraphWrapper { @@ -34,54 +48,66 @@ impl GraphWrapper { } } - pub fn get_modules(&self) -> FxHashSet { + pub fn get_modules(&self) -> HashSet { self._graph - .get_modules() - .iter() - .map(|module| module.name.clone()) + .all_modules() + .visible() + .names_as_strings() .collect() } + pub fn contains_module(&self, name: &str) -> bool { + match self.get_visible_module_by_name(name) { + Ok(_) => true, + Err(GrimpError::ModuleNotPresent(_)) => false, + _ => panic!("unexpected error checking for module existence"), + } + } + #[pyo3(signature = (module, is_squashed = false))] pub fn add_module(&mut self, module: &str, is_squashed: bool) -> PyResult<()> { - let module_struct = Module::new(module.to_string()); - - if let Some(ancestor_squashed_module) = - self._graph.find_ancestor_squashed_module(&module_struct) + for ancestor_module in self + ._graph + .module_name_to_self_and_ancestors(module) + .into_iter() + .skip(1) { - return Err(PyValueError::new_err(format!( - "Module is a descendant of squashed module {}.", - &ancestor_squashed_module.name - ))); + if self.is_module_squashed(&ancestor_module).unwrap_or(false) { + return Err(PyValueError::new_err(format!( + "Module is a descendant of squashed module {}.", + &ancestor_module, + ))); + }; } - if self._graph.get_modules().contains(&module_struct) { - if self._graph.is_module_squashed(&module_struct) != is_squashed { - return Err(PyValueError::new_err( - "Cannot add a squashed module when it is already present in the graph \ - as an unsquashed module, or vice versa.", - )); - } + if self.contains_module(module) && self.is_module_squashed(module)? != is_squashed { + return Err(PyValueError::new_err( + "Cannot add a squashed module when it is already present in the graph \ + as an unsquashed module, or vice versa.", + )); } match is_squashed { - false => self._graph.add_module(module_struct), - true => self._graph.add_squashed_module(module_struct), + false => self._graph.get_or_add_module(module), + true => self._graph.get_or_add_squashed_module(module), }; Ok(()) } pub fn remove_module(&mut self, module: &str) { - self._graph.remove_module(&Module::new(module.to_string())); + if let Some(module) = self._graph.get_module_by_name(module) { + self._graph.remove_module(module.token()) + } } - pub fn squash_module(&mut self, module: &str) { - self._graph.squash_module(&Module::new(module.to_string())); + pub fn squash_module(&mut self, module: &str) -> PyResult<()> { + let module = self.get_visible_module_by_name(module)?.token(); + self._graph.squash_module(module); + Ok(()) } - pub fn is_module_squashed(&self, module: &str) -> bool { - self._graph - .is_module_squashed(&Module::new(module.to_string())) + pub fn is_module_squashed(&self, module: &str) -> PyResult { + Ok(self.get_visible_module_by_name(module)?.is_squashed()) } #[pyo3(signature = (*, importer, imported, line_number=None, line_contents=None))] @@ -92,19 +118,15 @@ impl GraphWrapper { line_number: Option, line_contents: Option<&str>, ) { - let importer = Module::new(importer.to_string()); - let imported = Module::new(imported.to_string()); + let importer = self._graph.get_or_add_module(importer).token(); + let imported = self._graph.get_or_add_module(imported).token(); match (line_number, line_contents) { (Some(line_number), Some(line_contents)) => { - self._graph.add_detailed_import(&DetailedImport { - importer: importer, - imported: imported, - line_number: line_number, - line_contents: line_contents.to_string(), - }); + self._graph + .add_detailed_import(importer, imported, line_number, line_contents) } (None, None) => { - self._graph.add_import(&importer, &imported); + self._graph.add_import(importer, imported); } _ => { // TODO handle better. @@ -114,32 +136,41 @@ impl GraphWrapper { } #[pyo3(signature = (*, importer, imported))] - pub fn remove_import(&mut self, importer: &str, imported: &str) { - self._graph.remove_import( - &Module::new(importer.to_string()), - &Module::new(imported.to_string()), - ); + pub fn remove_import(&mut self, importer: &str, imported: &str) -> PyResult<()> { + let importer = self.get_visible_module_by_name(importer)?.token(); + let imported = self.get_visible_module_by_name(imported)?.token(); + self._graph.remove_import(importer, imported); + Ok(()) } pub fn count_imports(&self) -> usize { self._graph.count_imports() } - pub fn find_children(&self, module: &str) -> FxHashSet { - self._graph - .find_children(&Module::new(module.to_string())) - .iter() - .map(|child| child.name.clone()) - .collect() + pub fn find_children(&self, module: &str) -> PyResult> { + let module = self + ._graph + .get_module_by_name(module) + .ok_or(GrimpError::ModuleNotPresent(module.to_owned()))?; + Ok(self + ._graph + .get_module_children(module.token()) + .visible() + .names_as_strings() + .collect()) } - pub fn find_descendants(&self, module: &str) -> FxHashSet { - self._graph - .find_descendants(&Module::new(module.to_string())) - .unwrap() - .iter() - .map(|descendant| descendant.name.clone()) - .collect() + pub fn find_descendants(&self, module: &str) -> PyResult> { + let module = self + ._graph + .get_module_by_name(module) + .ok_or(GrimpError::ModuleNotPresent(module.to_owned()))?; + Ok(self + ._graph + .get_module_descendants(module.token()) + .visible() + .names_as_strings() + .collect()) } #[pyo3(signature = (*, importer, imported, as_packages = false))] @@ -149,40 +180,35 @@ impl GraphWrapper { imported: &str, as_packages: bool, ) -> PyResult { - if as_packages { - let importer_module = Module::new(importer.to_string()); - let imported_module = Module::new(imported.to_string()); - // Raise a ValueError if they are in the same package. - // (direct_import_exists) will panic if they are passed. - // TODO - this is a simpler check than Python, is it enough? - if importer_module.is_descendant_of(&imported_module) - || imported_module.is_descendant_of(&importer_module) - { - return Err(PyValueError::new_err("Modules have shared descendants.")); - } - } - - Ok(self._graph.direct_import_exists( - &Module::new(importer.to_string()), - &Module::new(imported.to_string()), - as_packages, - )) + let importer = self.get_visible_module_by_name(importer)?.token(); + let imported = self.get_visible_module_by_name(imported)?.token(); + Ok(self + ._graph + .direct_import_exists(importer, imported, as_packages)?) } - pub fn find_modules_directly_imported_by(&self, module: &str) -> FxHashSet { - self._graph - .find_modules_directly_imported_by(&Module::new(module.to_string())) + pub fn find_modules_directly_imported_by(&self, module: &str) -> PyResult> { + let module = self.get_visible_module_by_name(module)?.token(); + Ok(self + ._graph + .modules_directly_imported_by(module) .iter() - .map(|imported| imported.name.clone()) - .collect() + .into_module_iterator(&self._graph) + .visible() + .names_as_strings() + .collect()) } - pub fn find_modules_that_directly_import(&self, module: &str) -> FxHashSet { - self._graph - .find_modules_that_directly_import(&Module::new(module.to_string())) + pub fn find_modules_that_directly_import(&self, module: &str) -> PyResult> { + let module = self.get_visible_module_by_name(module)?.token(); + Ok(self + ._graph + .modules_that_directly_import(module) .iter() - .map(|importer| importer.name.clone()) - .collect() + .into_module_iterator(&self._graph) + .visible() + .names_as_strings() + .collect()) } #[pyo3(signature = (*, importer, imported))] @@ -192,95 +218,102 @@ impl GraphWrapper { importer: &str, imported: &str, ) -> PyResult> { - let mut vector: Vec> = vec![]; + let importer = match self._graph.get_module_by_name(importer) { + Some(module) => module, + None => return Ok(PyList::empty(py)), + }; + let imported = match self._graph.get_module_by_name(imported) { + Some(module) => module, + None => return Ok(PyList::empty(py)), + }; - let mut rust_import_details_vec: Vec = self - ._graph - .get_import_details( - &Module::new(importer.to_string()), - &Module::new(imported.to_string()), - ) - .into_iter() - .collect(); - rust_import_details_vec.sort(); - - for detailed_import in rust_import_details_vec { - let pydict = PyDict::new(py); - pydict.set_item( - "importer".to_string(), - detailed_import.importer.name.clone(), - )?; - pydict.set_item( - "imported".to_string(), - detailed_import.imported.name.clone(), - )?; - pydict.set_item("line_number".to_string(), detailed_import.line_number)?; - pydict.set_item( - "line_contents".to_string(), - detailed_import.line_contents.clone(), - )?; - vector.push(pydict); - } - PyList::new(py, &vector) + PyList::new( + py, + self._graph + .get_import_details(importer.token(), imported.token()) + .iter() + .map(|import_details| { + ImportDetails::new( + importer.name_as_string(), + imported.name_as_string(), + import_details.line_number(), + import_details.line_contents().to_owned(), + ) + }) + .sorted() + .map(|import_details| { + [ + ("importer", import_details.importer.into_py_any(py).unwrap()), + ("imported", import_details.imported.into_py_any(py).unwrap()), + ( + "line_number", + import_details.line_number.into_py_any(py).unwrap(), + ), + ( + "line_contents", + import_details.line_contents.into_py_any(py).unwrap(), + ), + ] + .into_py_dict(py) + .unwrap() + }), + ) } #[allow(unused_variables)] #[pyo3(signature = (module, as_package=false))] - pub fn find_downstream_modules(&self, module: &str, as_package: bool) -> FxHashSet { - // Turn the Modules to Strings. - self._graph - .find_downstream_modules(&Module::new(module.to_string()), as_package) + pub fn find_downstream_modules( + &self, + module: &str, + as_package: bool, + ) -> PyResult> { + let module = self.get_visible_module_by_name(module)?.token(); + Ok(self + ._graph + .find_downstream_modules(module, as_package) .iter() - .map(|downstream| downstream.name.clone()) - .collect() + .into_module_iterator(&self._graph) + .visible() + .names_as_strings() + .collect()) } #[allow(unused_variables)] #[pyo3(signature = (module, as_package=false))] - pub fn find_upstream_modules(&self, module: &str, as_package: bool) -> FxHashSet { - self._graph - .find_upstream_modules(&Module::new(module.to_string()), as_package) + pub fn find_upstream_modules( + &self, + module: &str, + as_package: bool, + ) -> PyResult> { + let module = self.get_visible_module_by_name(module)?.token(); + Ok(self + ._graph + .find_upstream_modules(module, as_package) .iter() - .map(|upstream| upstream.name.clone()) - .collect() - } - - pub fn find_shortest_chain(&self, importer: &str, imported: &str) -> Option> { - let chain = self._graph.find_shortest_chain( - &Module::new(importer.to_string()), - &Module::new(imported.to_string()), - )?; - - Some(chain.iter().map(|module| module.name.clone()).collect()) + .into_module_iterator(&self._graph) + .visible() + .names_as_strings() + .collect()) } - #[pyo3(signature = (importer, imported, as_packages=true))] - pub fn find_shortest_chains<'py>( + // TODO(peter) Add `as_packages` argument here? The implementation already supports it! + pub fn find_shortest_chain( &self, - py: Python<'py>, importer: &str, imported: &str, - as_packages: bool, - ) -> PyResult> { - let rust_chains: FxHashSet> = self + ) -> PyResult>> { + let importer = self.get_visible_module_by_name(importer)?.token(); + let imported = self.get_visible_module_by_name(imported)?.token(); + Ok(self ._graph - .find_shortest_chains( - &Module::new(importer.to_string()), - &Module::new(imported.to_string()), - as_packages, - ) - .map_err(|string| PyValueError::new_err(string))?; - - let mut tuple_chains: Vec> = vec![]; - for rust_chain in rust_chains.iter() { - let module_names: Vec> = rust_chain - .iter() - .map(|module| PyString::new(py, &module.name)) - .collect(); - let tuple = PyTuple::new(py, &module_names)?; - tuple_chains.push(tuple); - } - PySet::new(py, &tuple_chains) + .find_shortest_chain(importer, imported, false)? + .map(|chain| { + chain + .iter() + .into_module_iterator(&self._graph) + .names_as_strings() + .collect() + })) } #[pyo3(signature = (importer, imported, as_packages=false))] @@ -290,22 +323,37 @@ impl GraphWrapper { imported: &str, as_packages: bool, ) -> PyResult { - if as_packages { - let importer_module = Module::new(importer.to_string()); - let imported_module = Module::new(imported.to_string()); - // Raise a ValueError if they are in the same package. - // TODO - this is a simpler check than Python, is it enough? - if importer_module.is_descendant_of(&imported_module) - || imported_module.is_descendant_of(&importer_module) - { - return Err(PyValueError::new_err("Modules have shared descendants.")); - } - } - Ok(self._graph.chain_exists( - &Module::new(importer.to_string()), - &Module::new(imported.to_string()), - as_packages, - )) + let importer = self.get_visible_module_by_name(importer)?.token(); + let imported = self.get_visible_module_by_name(imported)?.token(); + Ok(self._graph.chain_exists(importer, imported, as_packages)?) + } + + #[pyo3(signature = (importer, imported, as_packages=true))] + pub fn find_shortest_chains<'py>( + &self, + py: Python<'py>, + importer: &str, + imported: &str, + as_packages: bool, + ) -> PyResult> { + let importer = self.get_visible_module_by_name(importer)?.token(); + let imported = self.get_visible_module_by_name(imported)?.token(); + let chains = self + ._graph + .find_shortest_chains(importer, imported, as_packages)? + .into_iter() + .map(|chain| { + PyTuple::new( + py, + chain + .iter() + .into_module_iterator(&self._graph) + .names_as_strings() + .collect::>(), + ) + .unwrap() + }); + PySet::new(py, chains) } #[pyo3(signature = (layers, containers))] @@ -313,23 +361,74 @@ impl GraphWrapper { &self, py: Python<'py>, layers: &Bound<'py, PyTuple>, - containers: FxHashSet, + containers: HashSet, ) -> PyResult> { - info!("Using Rust to find illegal dependencies."); - let levels = rustify_levels(layers); + let containers = self.parse_containers(&containers)?; + let levels_by_container = self.parse_levels_by_container(layers, &containers); - let dependencies = py.allow_threads(|| { - self._graph - .find_illegal_dependencies_for_layers(levels, containers) - }); - match dependencies { - Ok(dependencies) => _convert_dependencies_to_python(py, &dependencies), - Err(error) => Err(NoSuchContainer::new_err(format!( - "Container {} does not exist.", - error.container - ))), - } + let illegal_dependencies = levels_by_container + .into_iter() + .par_bridge() + .try_fold( + Vec::new, + |mut v: Vec, + levels| + -> GrimpResult<_> { + v.extend(self._graph.find_illegal_dependencies_for_layers(&levels)?); + Ok(v) + }, + ) + .try_reduce( + Vec::new, + |mut v: Vec, + package_dependencies| { + v.extend(package_dependencies); + Ok(v) + }, + )?; + + let illegal_dependencies = illegal_dependencies + .into_iter() + .map(|dep| { + PackageDependency::new( + self._graph + .get_module(*dep.importer()) + .unwrap() + .name_as_string(), + self._graph + .get_module(*dep.imported()) + .unwrap() + .name_as_string(), + dep.routes() + .iter() + .map(|route| { + Route::new( + route + .heads() + .iter() + .map(|m| self._graph.get_module(*m).unwrap().name_as_string()) + .collect(), + route + .middle() + .iter() + .map(|m| self._graph.get_module(*m).unwrap().name_as_string()) + .collect(), + route + .tails() + .iter() + .map(|m| self._graph.get_module(*m).unwrap().name_as_string()) + .collect(), + ) + }) + .collect(), + ) + }) + .sorted() + .collect::>(); + + self.convert_package_dependencies_to_python(py, illegal_dependencies) } + pub fn clone(&self) -> GraphWrapper { GraphWrapper { _graph: self._graph.clone(), @@ -337,193 +436,137 @@ impl GraphWrapper { } } -fn rustify_levels<'a>(levels_python: &Bound<'a, PyTuple>) -> Vec { - let mut rust_levels: Vec = vec![]; - for level_python in levels_python.into_iter() { - let level_dict = level_python.downcast::().unwrap(); - let layers: FxHashSet = level_dict - .get_item("layers") - .unwrap() - .unwrap() - .extract() - .unwrap(); - - let independent: bool = level_dict - .get_item("independent") - .unwrap() - .unwrap() - .extract() - .unwrap(); - rust_levels.push(Level { - independent, - layers: layers.into_iter().collect(), - }); +impl GraphWrapper { + fn parse_containers( + &self, + containers: &HashSet, + ) -> Result, GrimpError> { + containers + .iter() + .map(|name| match self.get_visible_module_by_name(name) { + Ok(module) => Ok(module), + Err(GrimpError::ModuleNotPresent(_)) => { + Err(GrimpError::NoSuchContainer(name.into()))? + } + _ => panic!("unexpected error parsing containers"), + }) + .collect::, GrimpError>>() } - rust_levels -} -fn _convert_dependencies_to_python<'py>( - py: Python<'py>, - dependencies: &Vec, -) -> PyResult> { - let mut python_dependencies: Vec> = vec![]; - - for rust_dependency in dependencies { - let python_dependency = PyDict::new(py); - python_dependency.set_item("imported", &rust_dependency.imported.name)?; - python_dependency.set_item("importer", &rust_dependency.importer.name)?; - let mut python_routes: Vec> = vec![]; - for rust_route in &rust_dependency.routes { - let route = PyDict::new(py); - let heads: Vec> = rust_route - .heads - .iter() - .map(|module| PyString::new(py, &module.name)) - .collect(); - route.set_item("heads", PyFrozenSet::new(py, &heads)?)?; - let middle: Vec> = rust_route - .middle - .iter() - .map(|module| PyString::new(py, &module.name)) - .collect(); - route.set_item("middle", PyTuple::new(py, &middle)?)?; - let tails: Vec> = rust_route - .tails + fn parse_levels_by_container( + &self, + pylevels: &Bound<'_, PyTuple>, + containers: &HashSet<&Module>, + ) -> Vec> { + let containers = match containers.is_empty() { + true => vec![None], + false => containers .iter() - .map(|module| PyString::new(py, &module.name)) - .collect(); - route.set_item("tails", PyFrozenSet::new(py, &tails)?)?; + .map(|c| Some(c.name_as_string())) + .collect(), + }; - python_routes.push(route); + let mut levels_by_container: Vec> = vec![]; + for container in containers { + let mut levels: Vec = vec![]; + for pylevel in pylevels.into_iter() { + let level_dict = pylevel.downcast::().unwrap(); + let layers = level_dict + .get_item("layers") + .unwrap() + .unwrap() + .extract::>() + .unwrap() + .into_iter() + .map(|name| match container.clone() { + Some(container) => format!("{}.{}", container, name), + None => name, + }) + .filter_map(|name| match self.get_visible_module_by_name(&name) { + Ok(module) => Some(module.token()), + // TODO(peter) Error here? Or silently continue (backwards compatibility?) + Err(GrimpError::ModuleNotPresent(_)) => None, + _ => panic!("unexpected error parsing levels"), + }) + .collect::>(); + + let independent = level_dict + .get_item("independent") + .unwrap() + .unwrap() + .extract::() + .unwrap(); + + levels.push(Level::new(layers, independent)); + } + levels_by_container.push(levels); } - python_dependency.set_item("routes", PyTuple::new(py, python_routes)?)?; - python_dependencies.push(python_dependency) + levels_by_container } - PyTuple::new(py, python_dependencies) -} - -#[cfg(test)] -mod tests { - use super::*; - - // Macro to easily define a python dict. - // Adapted from the hash_map! macro in https://github.com/jofas/map_macro. - macro_rules! pydict { - ($py: ident, {$($k: expr => $v: expr),*, $(,)?}) => { - { - let dict = PyDict::new($py); - $( - dict.set_item($k, $v)?; - )* - dict + fn convert_package_dependencies_to_python<'py>( + &self, + py: Python<'py>, + package_dependencies: Vec, + ) -> PyResult> { + let mut python_dependencies: Vec> = vec![]; + + for rust_dependency in package_dependencies { + let python_dependency = PyDict::new(py); + python_dependency.set_item("imported", &rust_dependency.imported)?; + python_dependency.set_item("importer", &rust_dependency.importer)?; + let mut python_routes: Vec> = vec![]; + for rust_route in &rust_dependency.routes { + let route = PyDict::new(py); + let heads: Vec> = rust_route + .heads + .iter() + .map(|module| PyString::new(py, module)) + .collect(); + route.set_item("heads", PyFrozenSet::new(py, &heads)?)?; + let middle: Vec> = rust_route + .middle + .iter() + .map(|module| PyString::new(py, module)) + .collect(); + route.set_item("middle", PyTuple::new(py, &middle)?)?; + let tails: Vec> = rust_route + .tails + .iter() + .map(|module| PyString::new(py, module)) + .collect(); + route.set_item("tails", PyFrozenSet::new(py, &tails)?)?; + + python_routes.push(route); } - }; - } - #[test] - fn test_rustify_levels_no_sibling_layers() { - pyo3::prepare_freethreaded_python(); - Python::with_gil(|py| -> PyResult<()> { - let elements = vec![ - pydict! (py, { - "independent" => true, - "layers" => FxHashSet::from_iter(["high"]), - }), - pydict! (py, { - "independent" => true, - "layers" => FxHashSet::from_iter(["medium"]), - }), - pydict! (py, { - "independent" => true, - "layers" => FxHashSet::from_iter(["low"]), - }), - ]; - let python_levels = PyTuple::new(py, elements)?; - - let result = rustify_levels(&python_levels); - - assert_eq!( - result, - vec![ - Level { - independent: true, - layers: vec!["high".to_string()] - }, - Level { - independent: true, - layers: vec!["medium".to_string()] - }, - Level { - independent: true, - layers: vec!["low".to_string()] - } - ] - ); - - Ok(()) - }) - .unwrap(); + python_dependency.set_item("routes", PyTuple::new(py, python_routes)?)?; + python_dependencies.push(python_dependency) + } + + PyTuple::new(py, python_dependencies) } +} - #[test] - fn test_rustify_levels_sibling_layers() { - pyo3::prepare_freethreaded_python(); - Python::with_gil(|py| -> PyResult<()> { - let elements = vec![ - pydict! (py, { - "independent" => true, - "layers" => FxHashSet::from_iter(["high"]), - }), - pydict! (py, { - "independent" => true, - "layers" => FxHashSet::from_iter(["blue", "green", "orange"]), - }), - pydict! (py, { - "independent" => false, - "layers" => FxHashSet::from_iter(["red", "yellow"]), - }), - pydict! (py, { - "independent" => true, - "layers" => FxHashSet::from_iter(["low"]), - }), - ]; - let python_levels = PyTuple::new(py, elements)?; +#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, new)] +struct ImportDetails { + importer: String, + imported: String, + line_number: usize, + line_contents: String, +} - let mut result = rustify_levels(&python_levels); +#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, new)] +struct PackageDependency { + importer: String, + imported: String, + routes: Vec, +} - for level in &mut result { - level.layers.sort(); - } - assert_eq!( - result, - vec![ - Level { - independent: true, - layers: vec!["high".to_string()] - }, - Level { - independent: true, - layers: vec![ - "blue".to_string(), - "green".to_string(), - "orange".to_string() - ] - }, - Level { - independent: false, - layers: vec!["red".to_string(), "yellow".to_string()] - }, - Level { - independent: true, - layers: vec!["low".to_string()] - } - ] - ); - - Ok(()) - }) - .unwrap(); - } +#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, new)] +struct Route { + heads: Vec, + middle: Vec, + tails: Vec, } diff --git a/rust/tests/large.rs b/rust/tests/large.rs index 5eaca91b..558d3b0e 100644 --- a/rust/tests/large.rs +++ b/rust/tests/large.rs @@ -1,24 +1,22 @@ -use _rustgrimp::graph::{Graph, Level, Module}; +use _rustgrimp::graph::higher_order_queries::Level; +use _rustgrimp::graph::Graph; use rustc_hash::FxHashSet; use serde_json::{Map, Value}; use std::fs; +use tap::Conv; #[test] fn test_large_graph_deep_layers() { let data = fs::read_to_string("tests/large_graph.json").expect("Unable to read file"); let value: Value = serde_json::from_str(&data).unwrap(); let items: &Map = value.as_object().unwrap(); + let mut graph = Graph::default(); for (importer, importeds_value) in items.iter() { + let importer = graph.get_or_add_module(importer).token(); for imported in importeds_value.as_array().unwrap() { - graph.add_import( - &Module { - name: importer.clone(), - }, - &Module { - name: imported.as_str().unwrap().to_string(), - }, - ); + let imported = graph.get_or_add_module(imported.as_str().unwrap()).token(); + graph.add_import(importer, imported); } } @@ -33,18 +31,22 @@ fn test_large_graph_deep_layers() { "mypackage.plugins.5634303718.1007553798.8198145119.application.3242334296.5033127033", "mypackage.plugins.5634303718.1007553798.8198145119.application.3242334296.2454157946", ]; + let levels: Vec = deep_layers - .iter() - .map(|layer| Level { - independent: true, - layers: vec![layer.to_string()], + .into_iter() + .map(|layer| { + Level::new( + graph + .get_module_by_name(layer) + .unwrap() + .token() + .conv::>(), + true, + ) }) .collect(); - let containers = FxHashSet::default(); - let deps = graph - .find_illegal_dependencies_for_layers(levels, containers) - .unwrap(); + let deps = graph.find_illegal_dependencies_for_layers(&levels).unwrap(); assert_eq!(deps.len(), 8); } diff --git a/src/grimp/adaptors/graph.py b/src/grimp/adaptors/graph.py index 2c497fae..abadbec8 100644 --- a/src/grimp/adaptors/graph.py +++ b/src/grimp/adaptors/graph.py @@ -17,10 +17,17 @@ def __init__(self) -> None: super().__init__() self._rustgraph = rust.Graph() + # TODO(peter) Be wary about `if X in graph.modules` since we need to + # convert the entire graph from rust -> python, which can hurt performance. + # Prefer `graph.contains_module`. Is there a way to make this clearer, it feels like + # a performance footgun. @property def modules(self) -> Set[str]: return self._rustgraph.get_modules() + def contains_module(self, module: str) -> bool: + return self._rustgraph.contains_module(module) + def add_module(self, module: str, is_squashed: bool = False) -> None: self._rustgraph.add_module(module, is_squashed) @@ -28,12 +35,12 @@ def remove_module(self, module: str) -> None: self._rustgraph.remove_module(module) def squash_module(self, module: str) -> None: - if module not in self.modules: + if not self.contains_module(module): raise ModuleNotPresent(f'"{module}" not present in the graph.') self._rustgraph.squash_module(module) def is_module_squashed(self, module: str) -> bool: - if module not in self.modules: + if not self.contains_module(module): raise ModuleNotPresent(f'"{module}" not present in the graph.') return self._rustgraph.is_module_squashed(module) @@ -102,7 +109,7 @@ def find_upstream_modules(self, module: str, as_package: bool = False) -> Set[st def find_shortest_chain(self, importer: str, imported: str) -> tuple[str, ...] | None: for module in (importer, imported): - if module not in self.modules: + if not self.contains_module(module): raise ValueError(f"Module {module} is not present in the graph.") chain = self._rustgraph.find_shortest_chain(importer, imported) diff --git a/src/grimp/application/ports/graph.py b/src/grimp/application/ports/graph.py index cf9ec1bb..b05f8006 100644 --- a/src/grimp/application/ports/graph.py +++ b/src/grimp/application/ports/graph.py @@ -32,6 +32,13 @@ def modules(self) -> Set[str]: """ raise NotImplementedError + @abc.abstractmethod + def contains_module(self, module: str) -> bool: + """ + Determine whether a module exists within the graph. + """ + raise NotImplementedError + @abc.abstractmethod def add_module(self, module: str, is_squashed: bool = False) -> None: """ From 95fd61ad50964232b6b24c99f7ae92b59ea66655 Mon Sep 17 00:00:00 2001 From: David Seddon Date: Fri, 7 Feb 2025 12:12:53 +0000 Subject: [PATCH 08/10] Cache modules on Python graph Accessing graph.modules is more expensive than it looks. --- src/grimp/adaptors/graph.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/src/grimp/adaptors/graph.py b/src/grimp/adaptors/graph.py index abadbec8..85448605 100644 --- a/src/grimp/adaptors/graph.py +++ b/src/grimp/adaptors/graph.py @@ -15,6 +15,7 @@ class ImportGraph(graph.ImportGraph): def __init__(self) -> None: super().__init__() + self._cached_modules: Set[str] | None = None self._rustgraph = rust.Graph() # TODO(peter) Be wary about `if X in graph.modules` since we need to @@ -23,18 +24,23 @@ def __init__(self) -> None: # a performance footgun. @property def modules(self) -> Set[str]: - return self._rustgraph.get_modules() + if self._cached_modules is None: + self._cached_modules = self._rustgraph.get_modules() + return self._cached_modules def contains_module(self, module: str) -> bool: return self._rustgraph.contains_module(module) def add_module(self, module: str, is_squashed: bool = False) -> None: + self._cached_modules = None self._rustgraph.add_module(module, is_squashed) def remove_module(self, module: str) -> None: + self._cached_modules = None self._rustgraph.remove_module(module) def squash_module(self, module: str) -> None: + self._cached_modules = None if not self.contains_module(module): raise ModuleNotPresent(f'"{module}" not present in the graph.') self._rustgraph.squash_module(module) @@ -52,6 +58,7 @@ def add_import( line_number: Optional[int] = None, line_contents: Optional[str] = None, ) -> None: + self._cached_modules = None self._rustgraph.add_import( importer=importer, imported=imported, @@ -60,6 +67,7 @@ def add_import( ) def remove_import(self, *, importer: str, imported: str) -> None: + self._cached_modules = None return self._rustgraph.remove_import(importer=importer, imported=imported) def count_imports(self) -> int: From a08d69824e7ae3eb1c81dec437d59cf8b2a85c32 Mon Sep 17 00:00:00 2001 From: Peter Byfield Date: Fri, 7 Feb 2025 13:53:15 +0100 Subject: [PATCH 09/10] Remove contains_module from public API --- src/grimp/adaptors/graph.py | 15 ++++----------- src/grimp/application/ports/graph.py | 7 ------- 2 files changed, 4 insertions(+), 18 deletions(-) diff --git a/src/grimp/adaptors/graph.py b/src/grimp/adaptors/graph.py index 85448605..f4abe0ca 100644 --- a/src/grimp/adaptors/graph.py +++ b/src/grimp/adaptors/graph.py @@ -18,19 +18,12 @@ def __init__(self) -> None: self._cached_modules: Set[str] | None = None self._rustgraph = rust.Graph() - # TODO(peter) Be wary about `if X in graph.modules` since we need to - # convert the entire graph from rust -> python, which can hurt performance. - # Prefer `graph.contains_module`. Is there a way to make this clearer, it feels like - # a performance footgun. @property def modules(self) -> Set[str]: if self._cached_modules is None: self._cached_modules = self._rustgraph.get_modules() return self._cached_modules - def contains_module(self, module: str) -> bool: - return self._rustgraph.contains_module(module) - def add_module(self, module: str, is_squashed: bool = False) -> None: self._cached_modules = None self._rustgraph.add_module(module, is_squashed) @@ -41,12 +34,12 @@ def remove_module(self, module: str) -> None: def squash_module(self, module: str) -> None: self._cached_modules = None - if not self.contains_module(module): + if not self._rustgraph.contains_module(module): raise ModuleNotPresent(f'"{module}" not present in the graph.') self._rustgraph.squash_module(module) def is_module_squashed(self, module: str) -> bool: - if not self.contains_module(module): + if not self._rustgraph.contains_module(module): raise ModuleNotPresent(f'"{module}" not present in the graph.') return self._rustgraph.is_module_squashed(module) @@ -98,7 +91,7 @@ def find_modules_directly_imported_by(self, module: str) -> Set[str]: return self._rustgraph.find_modules_directly_imported_by(module) def find_modules_that_directly_import(self, module: str) -> Set[str]: - if module in self._rustgraph.get_modules(): + if self._rustgraph.contains_module(module): # TODO panics if module isn't in modules. return self._rustgraph.find_modules_that_directly_import(module) return set() @@ -117,7 +110,7 @@ def find_upstream_modules(self, module: str, as_package: bool = False) -> Set[st def find_shortest_chain(self, importer: str, imported: str) -> tuple[str, ...] | None: for module in (importer, imported): - if not self.contains_module(module): + if not self._rustgraph.contains_module(module): raise ValueError(f"Module {module} is not present in the graph.") chain = self._rustgraph.find_shortest_chain(importer, imported) diff --git a/src/grimp/application/ports/graph.py b/src/grimp/application/ports/graph.py index b05f8006..cf9ec1bb 100644 --- a/src/grimp/application/ports/graph.py +++ b/src/grimp/application/ports/graph.py @@ -32,13 +32,6 @@ def modules(self) -> Set[str]: """ raise NotImplementedError - @abc.abstractmethod - def contains_module(self, module: str) -> bool: - """ - Determine whether a module exists within the graph. - """ - raise NotImplementedError - @abc.abstractmethod def add_module(self, module: str, is_squashed: bool = False) -> None: """ From ec7e7f945d65d8ef6d6969df6f60f213a28dbf63 Mon Sep 17 00:00:00 2001 From: David Seddon Date: Fri, 7 Feb 2025 14:49:47 +0000 Subject: [PATCH 10/10] Add to changelog --- CHANGELOG.rst | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 971612e0..bc0fb798 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -2,6 +2,12 @@ Changelog ========= +latest +------ + +* Reimplement the graph in Rust. This is a substantial rewrite that, mostly, significantly + improves performance, but there may be certain operations that are slower than before. + 3.5 (2024-10-08) ----------------