Expand workspace to cover all crates and enable rust testing (#1684)

dulinriley · facebook-github-bot · commit 066212f0c710 · 2025-10-30T09:29:43.000-07:00
Summary:

Expand Github rust testing to the whole workspace of crates in monarch.
Tests that do not pass in Github are marked as fb-only for now. Many of them can
be fixed easily, but we can turn on the majority of tests right away.

Reviewed By: colin2328

Differential Revision: D85676520
diff --git a/.config/nextest.toml b/.config/nextest.toml
@@ -0,0 +1,6 @@
+[profile.ci]
+# Do not cancel the test run on the first failure.
+fail-fast = false
+
+[profile.ci.junit]
+path = "junit.xml"
diff --git a/.github/workflows/test-gpu-rust.yml b/.github/workflows/test-gpu-rust.yml
@@ -56,9 +56,18 @@ jobs:
 
         # Run GPU Rust tests
         echo "Running OSS Rust tests..."
-        # TODO: fix broken tests, then update to `cargo test --no-fail-fast`
-        cargo test -p monarch_rdma
         # Uses cargo nextest to run tests in separate processes, which better matches
         # internal buck test behavior.
-        # TODO: increase coverage to more crates.
-        cargo nextest run -p hyperactor --no-fail-fast
+        # The CI profile is configured in .config/nextest.toml
+        # Exclude filter is for packages that don't build in Github Actions yet.
+        # * monarch_messages: monarch/target/debug/deps/monarch_messages-...:
+        #   /lib64/libm.so.6: version `GLIBC_2.29' not found
+        #   (required by /meta-pytorch/monarch/libtorch/lib/libtorch_cpu.so)
+        cargo nextest run --workspace --profile ci \
+          --exclude monarch_messages \
+          --exclude monarch_tensor_worker \
+          --exclude monarch_simulator_lib \
+          --exclude torch-sys \
+          --exclude torch-sys-cuda
+        # Copy the test results to the expected location
+        cp target/nextest/ci/junit.xml "${RUNNER_TEST_RESUILTS_DIR:-test-results}/junit.xml"
diff --git a/Cargo.toml b/Cargo.toml
@@ -1,6 +1,7 @@
 [workspace]
 resolver = "2"
 members = [
+    "build_utils",
     "controller",
     "cuda-sys",
     "erased_lifetime",
@@ -10,11 +11,22 @@ members = [
     "hyperactor_multiprocess",
     "hyperactor_mesh",
     "hyperactor_mesh_macros",
-    "ndslice",
+    "hyperactor_telemetry",
+    "monarch_conda",
     "monarch_extension",
-    "monarch_tensor_worker",
+    "monarch_hyperactor",
+    "monarch_messages",
+    "monarch_perfetto_trace",
     "monarch_rdma",
+    "monarch_simulator",
+    "monarch_tensor_worker",
+    "monarch_types",
     "nccl-sys",
+    "ndslice",
+    "preempt_rwlock",
     "rdmaxcel-sys",
+    "serde_multipart",
+    "timed_test",
     "torch-sys",
+    "torch-sys-cuda",
 ]
diff --git a/controller/src/lib.rs b/controller/src/lib.rs
@@ -660,6 +660,7 @@ mod tests {
     use monarch_messages::worker::CallFunctionParams;
     use monarch_messages::worker::WorkerMessage;
     use monarch_types::PyTree;
+    use timed_test::async_timed_test;
     use torch_sys::RValue;
 
     use super::*;
@@ -855,7 +856,7 @@ mod tests {
         );
     }
 
-    #[tokio::test]
+    #[async_timed_test(30)]
     async fn worker_timeout() {
         tokio::time::pause();
         let timeout_secs = 3;
@@ -1838,7 +1839,7 @@ mod tests {
 
     hyperactor::remote!(PanickingActor);
 
-    #[tokio::test]
+    #[async_timed_test(timeout_secs = 30)]
     async fn test_supervision_fault() {
         // Start system actor.
         let timeout: Duration = Duration::from_secs(6);
diff --git a/hyperactor_mesh/src/actor_mesh.rs b/hyperactor_mesh/src/actor_mesh.rs
@@ -1482,18 +1482,19 @@ mod tests {
 
         use crate::alloc::process::ProcessAllocator;
 
+        #[cfg(feature = "fb")]
         fn process_allocator() -> ProcessAllocator {
             ProcessAllocator::new(Command::new(crate::testresource::get(
                 "monarch/hyperactor_mesh/bootstrap",
             )))
         }
 
-        #[cfg(fbcode_build)] // we use an external binary, produced by buck
+        #[cfg(feature = "fb")] // we use an external binary, produced by buck
         actor_mesh_test_suite!(process_allocator());
 
         // This test is concerned with correctly reporting failures
         // when message sizes exceed configured limits.
-        #[cfg(fbcode_build)]
+        #[cfg(feature = "fb")]
         //#[tracing_test::traced_test]
         #[async_timed_test(timeout_secs = 30)]
         async fn test_oversized_frames() {
@@ -1603,7 +1604,7 @@ mod tests {
         // Set this test only for `mod process` because it relies on a
         // trick to emulate router failure that only works when using
         // non-local allocators.
-        #[cfg(fbcode_build)]
+        #[cfg(feature = "fb")]
         #[tokio::test]
         async fn test_router_undeliverable_return() {
             // Test that an undeliverable message received by a
@@ -1928,6 +1929,7 @@ mod tests {
         use crate::sel;
 
         #[tokio::test]
+        #[cfg(feature = "fb")]
         async fn test_basic() {
             let instance = v1::testing::instance().await;
             let host_mesh = v1::testing::host_mesh(extent!(host = 4)).await;
diff --git a/hyperactor_mesh/src/alloc.rs b/hyperactor_mesh/src/alloc.rs
@@ -914,6 +914,7 @@ pub(crate) mod testing {
     /// a proc that does not time out when it is asked to wait for
     /// a stuck actor.
     #[tokio::test]
+    #[cfg(feature = "fb")]
     async fn test_allocator_stuck_task() {
         // Override config.
         // Use temporary config for this test
diff --git a/hyperactor_mesh/src/alloc/process.rs b/hyperactor_mesh/src/alloc/process.rs
@@ -644,11 +644,12 @@ impl Drop for ProcessAlloc {
 mod tests {
     use super::*;
 
-    #[cfg(fbcode_build)] // we use an external binary, produced by buck
+    #[cfg(feature = "fb")] // we use an external binary, produced by buck
     crate::alloc_test_suite!(ProcessAllocator::new(Command::new(
         crate::testresource::get("monarch/hyperactor_mesh/bootstrap")
     )));
 
+    #[cfg(feature = "fb")]
     #[tokio::test]
     async fn test_sigterm_on_group_fail() {
         let bootstrap_binary = crate::testresource::get("monarch/hyperactor_mesh/bootstrap");
diff --git a/hyperactor_mesh/src/alloc/remoteprocess.rs b/hyperactor_mesh/src/alloc/remoteprocess.rs
@@ -2055,6 +2055,7 @@ mod test_alloc {
     use super::*;
 
     #[async_timed_test(timeout_secs = 60)]
+    #[cfg(feature = "fb")]
     async fn test_alloc_simple() {
         // Use temporary config for this test
         let config = hyperactor::config::global::lock();
@@ -2184,6 +2185,7 @@ mod test_alloc {
     }
 
     #[async_timed_test(timeout_secs = 60)]
+    #[cfg(feature = "fb")]
     async fn test_alloc_host_failure() {
         // Use temporary config for this test
         let config = hyperactor::config::global::lock();
@@ -2315,6 +2317,7 @@ mod test_alloc {
     }
 
     #[async_timed_test(timeout_secs = 15)]
+    #[cfg(feature = "fb")]
     async fn test_alloc_inner_alloc_failure() {
         // SAFETY: Test happens in single-threaded code.
         unsafe {
@@ -2450,6 +2453,7 @@ mod test_alloc {
 
     #[tracing_test::traced_test]
     #[async_timed_test(timeout_secs = 60)]
+    #[cfg(feature = "fb")]
     async fn test_remote_process_alloc_signal_handler() {
         let num_proc_meshes = 5;
         let hosts_per_proc_mesh = 5;
diff --git a/hyperactor_mesh/src/bootstrap.rs b/hyperactor_mesh/src/bootstrap.rs
@@ -1382,6 +1382,7 @@ impl BootstrapCommand {
     /// bootstrap processes under proc manager control. Not available
     /// outside of test builds.
     #[cfg(test)]
+    #[cfg(feature = "fb")]
     pub(crate) fn test() -> Self {
         Self {
             program: crate::testresource::get("monarch/hyperactor_mesh/bootstrap"),
@@ -3218,6 +3219,7 @@ mod tests {
     }
 
     #[tokio::test]
+    #[cfg(feature = "fb")]
     async fn bootstrap_handle_terminate_graceful() {
         // Create a root direct-addressed proc + client instance.
         let root = hyperactor::Proc::direct(ChannelTransport::Unix.any(), "root".to_string())
@@ -3281,6 +3283,7 @@ mod tests {
     }
 
     #[tokio::test]
+    #[cfg(feature = "fb")]
     async fn bootstrap_handle_kill_forced() {
         // Root proc + client instance (so the child can dial back).
         let root = hyperactor::Proc::direct(ChannelTransport::Unix.any(), "root".to_string())
@@ -3330,6 +3333,7 @@ mod tests {
     }
 
     #[tokio::test]
+    #[cfg(feature = "fb")]
     async fn bootstrap_cannonical_simple() {
         // SAFETY: unit-test scoped
         unsafe {
diff --git a/hyperactor_mesh/src/proc_mesh.rs b/hyperactor_mesh/src/proc_mesh.rs
@@ -1264,6 +1264,7 @@ mod tests {
         use crate::sel;
 
         #[tokio::test]
+        #[cfg(feature = "fb")]
         async fn test_basic() {
             let instance = v1::testing::instance().await;
             let ext = extent!(host = 4);
diff --git a/hyperactor_mesh/src/testresource.rs b/hyperactor_mesh/src/testresource.rs
@@ -16,6 +16,7 @@ use std::path::PathBuf;
 ///
 /// We should convert these tests to integration tests, so that cargo can
 /// also manage the binaries.
+#[cfg(feature = "fb")]
 pub fn get<S>(name: S) -> PathBuf
 where
     S: AsRef<str>,
diff --git a/hyperactor_mesh/src/v1/actor_mesh.rs b/hyperactor_mesh/src/v1/actor_mesh.rs
@@ -467,6 +467,7 @@ mod tests {
     use crate::v1::testing;
 
     #[tokio::test]
+    #[cfg(feature = "fb")]
     async fn test_actor_mesh_ref_lazy_materialization() {
         // 1) Bring up procs and spawn actors.
         let instance = testing::instance().await;
@@ -566,6 +567,7 @@ mod tests {
     }
 
     #[async_timed_test(timeout_secs = 30)]
+    #[cfg(feature = "fb")]
     async fn test_actor_states_with_panic() {
         hyperactor_telemetry::initialize_logging_for_test();
 
@@ -632,6 +634,7 @@ mod tests {
     }
 
     #[async_timed_test(timeout_secs = 30)]
+    #[cfg(feature = "fb")]
     async fn test_actor_states_with_process_exit() {
         hyperactor_telemetry::initialize_logging_for_test();
 
@@ -699,6 +702,7 @@ mod tests {
     }
 
     #[async_timed_test(timeout_secs = 30)]
+    #[cfg(feature = "fb")]
     async fn test_actor_states_on_sliced_mesh() {
         hyperactor_telemetry::initialize_logging_for_test();
 
@@ -772,6 +776,7 @@ mod tests {
     }
 
     #[async_timed_test(timeout_secs = 30)]
+    #[cfg(feature = "fb")]
     async fn test_cast() {
         let config = hyperactor::config::global::lock();
         let _guard = config.override_key(crate::bootstrap::MESH_BOOTSTRAP_ENABLE_PDEATHSIG, false);
diff --git a/hyperactor_mesh/src/v1/host_mesh.rs b/hyperactor_mesh/src/v1/host_mesh.rs
@@ -1115,6 +1115,7 @@ mod tests {
     }
 
     #[tokio::test]
+    #[cfg(feature = "fb")]
     async fn test_allocate() {
         let config = hyperactor::config::global::lock();
         let _guard = config.override_key(crate::bootstrap::MESH_BOOTSTRAP_ENABLE_PDEATHSIG, false);
@@ -1225,6 +1226,7 @@ mod tests {
     }
 
     #[tokio::test]
+    #[cfg(feature = "fb")]
     async fn test_extrinsic_allocation() {
         let config = hyperactor::config::global::lock();
         let _guard = config.override_key(crate::bootstrap::MESH_BOOTSTRAP_ENABLE_PDEATHSIG, false);
@@ -1268,6 +1270,7 @@ mod tests {
     }
 
     #[tokio::test]
+    #[cfg(feature = "fb")]
     async fn test_failing_proc_allocation() {
         let program = crate::testresource::get("monarch/hyperactor_mesh/bootstrap");
 
@@ -1301,6 +1304,7 @@ mod tests {
     }
 
     #[tokio::test]
+    #[cfg(feature = "fb")]
     async fn test_halting_proc_allocation() {
         let config = config::global::lock();
         let _guard1 = config.override_key(PROC_SPAWN_MAX_IDLE, Duration::from_secs(5));
@@ -1345,6 +1349,7 @@ mod tests {
     }
 
     #[tokio::test]
+    #[cfg(feature = "fb")]
     async fn test_client_config_override() {
         let config = hyperactor::config::global::lock();
         let _guard1 = config.override_key(crate::bootstrap::MESH_BOOTSTRAP_ENABLE_PDEATHSIG, false);
diff --git a/hyperactor_mesh/src/v1/host_mesh/mesh_agent.rs b/hyperactor_mesh/src/v1/host_mesh/mesh_agent.rs
@@ -521,6 +521,7 @@ mod tests {
     use crate::resource::GetStateClient;
 
     #[tokio::test]
+    #[cfg(feature = "fb")]
     async fn test_basic() {
         let (host, _handle) = Host::serve(
             BootstrapProcManager::new(BootstrapCommand::test()),
diff --git a/hyperactor_mesh/src/v1/proc_mesh.rs b/hyperactor_mesh/src/v1/proc_mesh.rs
@@ -1032,6 +1032,7 @@ mod tests {
     }
 
     #[async_timed_test(timeout_secs = 30)]
+    #[cfg(feature = "fb")]
     async fn test_spawn_actor() {
         hyperactor_telemetry::initialize_logging(hyperactor::clock::ClockKind::default());
 
@@ -1044,6 +1045,7 @@ mod tests {
     }
 
     #[tokio::test]
+    #[cfg(feature = "fb")]
     async fn test_failing_spawn_actor() {
         hyperactor_telemetry::initialize_logging(hyperactor::clock::ClockKind::default());
 
diff --git a/hyperactor_mesh/src/v1/testing.rs b/hyperactor_mesh/src/v1/testing.rs
@@ -45,6 +45,7 @@ pub async fn instance() -> &'static Instance<()> {
     INSTANCE.get_or_init(fresh_instance).await
 }
 
+#[cfg(feature = "fb")]
 pub async fn proc_meshes(cx: &impl context::Actor, extent: Extent) -> Vec<ProcMesh> {
     let mut meshes = Vec::new();
 
@@ -87,6 +88,7 @@ pub async fn proc_meshes(cx: &impl context::Actor, extent: Extent) -> Vec<ProcMe
 }
 
 /// Return different alloc implementations with the provided extent.
+#[cfg(feature = "fb")]
 pub async fn allocs(extent: Extent) -> Vec<Box<dyn Alloc + Send + Sync>> {
     let spec = AllocSpec {
         extent: extent.clone(),
@@ -134,6 +136,7 @@ pub async fn local_proc_mesh(extent: Extent) -> (ProcMesh, Instance<()>, DialMai
 }
 
 /// Create a host mesh using multiple processes running on the test machine.
+#[cfg(feature = "fb")]
 pub async fn host_mesh(extent: Extent) -> HostMesh {
     let mut allocator = ProcessAllocator::new(Command::new(crate::testresource::get(
         "monarch/hyperactor_mesh/bootstrap",
diff --git a/hyperactor_mesh/test/process_allocator_cleanup/process_allocator_cleanup.rs b/hyperactor_mesh/test/process_allocator_cleanup/process_allocator_cleanup.rs
@@ -27,6 +27,7 @@ use tokio::time::timeout;
 
 /// Test that ProcessAllocator children are cleaned up when parent is killed
 #[tokio::test]
+#[cfg_attr(not(feature = "fb"), ignore)]
 async fn test_process_allocator_child_cleanup() {
     let test_binary_path = buck_resources::get("monarch/hyperactor_mesh/test_bin").unwrap();
     eprintln!("Starting test process allocator at: {:?}", test_binary_path);
diff --git a/hyperactor_multiprocess/src/proc_actor.rs b/hyperactor_multiprocess/src/proc_actor.rs
@@ -1043,6 +1043,7 @@ mod tests {
 
     #[tracing_test::traced_test]
     #[tokio::test]
+    #[cfg_attr(not(feature = "fb"), ignore)]
     async fn test_stop_timeout() {
         let Bootstrapped {
             server_handle,
@@ -1488,6 +1489,7 @@ mod tests {
 
     #[tracing_test::traced_test]
     #[tokio::test]
+    #[cfg_attr(not(feature = "fb"), ignore)]
     async fn test_proc_actor_mailbox_admin_message() {
         // Verify that proc actors update their address books on first
         // contact, and that no additional updates are triggered for
diff --git a/monarch_hyperactor/src/channel.rs b/monarch_hyperactor/src/channel.rs
@@ -158,6 +158,8 @@ mod tests {
     use super::*;
 
     #[test]
+    // TODO: OSS: failed to retrieve ipv6 address
+    #[cfg_attr(not(feature = "fb"), ignore)]
     fn test_channel_any_and_parse() -> PyResult<()> {
         // just make sure any() and parse() calls work for all transports
         for transport in [
diff --git a/monarch_hyperactor/src/code_sync/rsync.rs b/monarch_hyperactor/src/code_sync/rsync.rs
diff --git a/monarch_hyperactor/tests/code_sync/auto_reload.rs b/monarch_hyperactor/tests/code_sync/auto_reload.rs
diff --git a/ndslice/src/reshape.rs b/ndslice/src/reshape.rs
diff --git a/scripts/common-setup.sh b/scripts/common-setup.sh
