Expand workspace to cover all crates and enable rust testing

.config/nextest.toml

@@ -0,0 +1,6 @@
+[profile.ci]
+# Do not cancel the test run on the first failure.
+fail-fast = false
+
+[profile.ci.junit]
+path = "junit.xml"

.github/workflows/test-gpu-rust.yml

@@ -56,9 +56,21 @@ jobs:
 
         # Run GPU Rust tests
         echo "Running OSS Rust tests..."
-        # TODO: fix broken tests, then update to `cargo test --no-fail-fast`
-        cargo test -p monarch_rdma
         # Uses cargo nextest to run tests in separate processes, which better matches
         # internal buck test behavior.
-        # TODO: increase coverage to more crates.
-        cargo nextest run -p hyperactor --no-fail-fast
+        # The CI profile is configured in .config/nextest.toml
+        # Exclude filter is for packages that don't build in Github Actions yet.
+        # * monarch_messages: monarch/target/debug/deps/monarch_messages-...:
+        #   /lib64/libm.so.6: version `GLIBC_2.29' not found
+        #   (required by /meta-pytorch/monarch/libtorch/lib/libtorch_cpu.so)
+        cargo nextest run --workspace --profile ci \
+          --exclude monarch_messages \
+          --exclude monarch_tensor_worker \
+          --exclude monarch_simulator_lib \
+          --exclude torch-sys \
+          --exclude torch-sys-cuda
+        # Copy the test results to the expected location
+        # TODO: error in pytest-results-action, TypeError: results.testsuites.testsuite.testcase is not iterable
+        # Don't try to parse these results for now.
+        # mkdir -p "${RUNNER_TEST_RESULTS_DIR:-test-results}"
+        # cp target/nextest/ci/junit.xml "${RUNNER_TEST_RESULTS_DIR:-test-results}/junit.xml"

Cargo.toml

@@ -1,6 +1,7 @@
 [workspace]
 resolver = "2"
 members = [
+    "build_utils",
     "controller",
     "cuda-sys",
     "erased_lifetime",
@@ -10,11 +11,22 @@ members = [
     "hyperactor_multiprocess",
     "hyperactor_mesh",
     "hyperactor_mesh_macros",
-    "ndslice",
+    "hyperactor_telemetry",
+    "monarch_conda",
     "monarch_extension",
-    "monarch_tensor_worker",
+    "monarch_hyperactor",
+    "monarch_messages",
+    "monarch_perfetto_trace",
     "monarch_rdma",
+    "monarch_simulator",
+    "monarch_tensor_worker",
+    "monarch_types",
     "nccl-sys",
+    "ndslice",
+    "preempt_rwlock",
     "rdmaxcel-sys",
+    "serde_multipart",
+    "timed_test",
     "torch-sys",
+    "torch-sys-cuda",
 ]

controller/Cargo.toml

@@ -36,4 +36,8 @@ tracing = { version = "0.1.41", features = ["attributes", "valuable"] }
 
 [dev-dependencies]
 monarch_types = { version = "0.0.0", path = "../monarch_types" }
+timed_test = { version = "0.0.0", path = "../timed_test" }
 torch-sys = { version = "0.0.0", path = "../torch-sys" }
+
+[lints]
+rust = { unexpected_cfgs = { check-cfg = ["cfg(fbcode_build)"], level = "warn" } }

controller/src/lib.rs

@@ -660,6 +660,7 @@ mod tests {
     use monarch_messages::worker::CallFunctionParams;
     use monarch_messages::worker::WorkerMessage;
     use monarch_types::PyTree;
+    use timed_test::async_timed_test;
     use torch_sys::RValue;
 
     use super::*;
@@ -1838,7 +1839,9 @@ mod tests {
 
     hyperactor::remote!(PanickingActor);
 
-    #[tokio::test]
+    #[async_timed_test(timeout_secs = 30)]
+    // times out (both internal and external).
+    #[cfg_attr(not(fbcode_build), ignore)]
     async fn test_supervision_fault() {
         // Start system actor.
         let timeout: Duration = Duration::from_secs(6);

hyperactor/Cargo.toml

@@ -100,3 +100,6 @@ tracing-test = { version = "0.2.3", features = ["no-env-filter"] }
 [features]
 default = []
 stdio-write-probe = []
+
+[lints]
+rust = { unexpected_cfgs = { check-cfg = ["cfg(fbcode_build)"], level = "warn" } }

hyperactor/src/channel.rs

@@ -1143,7 +1143,7 @@ mod tests {
 
     #[tokio::test]
     // TODO: OSS: called `Result::unwrap()` on an `Err` value: Server(Listen(Tcp([::1]:0), Os { code: 99, kind: AddrNotAvailable, message: "Cannot assign requested address" }))
-    #[cfg_attr(not(feature = "fb"), ignore)]
+    #[cfg_attr(not(fbcode_build), ignore)]
     async fn test_dial_serve() {
         for addr in addrs() {
             let (listen_addr, mut rx) = crate::channel::serve::<i32>(addr).unwrap();
@@ -1155,7 +1155,7 @@ mod tests {
 
     #[tokio::test]
     // TODO: OSS: called `Result::unwrap()` on an `Err` value: Server(Listen(Tcp([::1]:0), Os { code: 99, kind: AddrNotAvailable, message: "Cannot assign requested address" }))
-    #[cfg_attr(not(feature = "fb"), ignore)]
+    #[cfg_attr(not(fbcode_build), ignore)]
     async fn test_send() {
         let config = crate::config::global::lock();
 

hyperactor/src/channel/net.rs

@@ -2572,7 +2572,7 @@ mod tests {
     #[tracing_test::traced_test]
     #[async_timed_test(timeout_secs = 60)]
     // TODO: OSS: called `Result::unwrap()` on an `Err` value: Listen(Tcp([::1]:0), Os { code: 99, kind: AddrNotAvailable, message: "Cannot assign requested address" })
-    #[cfg_attr(not(feature = "fb"), ignore)]
+    #[cfg_attr(not(fbcode_build), ignore)]
     async fn test_tcp_basic() {
         let (addr, mut rx) = tcp::serve::<u64>("[::1]:0".parse().unwrap()).unwrap();
         {
@@ -2605,7 +2605,7 @@ mod tests {
     // The message size is limited by CODEC_MAX_FRAME_LENGTH.
     #[async_timed_test(timeout_secs = 5)]
     // TODO: OSS: called `Result::unwrap()` on an `Err` value: Listen(Tcp([::1]:0), Os { code: 99, kind: AddrNotAvailable, message: "Cannot assign requested address" })
-    #[cfg_attr(not(feature = "fb"), ignore)]
+    #[cfg_attr(not(fbcode_build), ignore)]
     async fn test_tcp_message_size() {
         let default_size_in_bytes = 100 * 1024 * 1024;
         // Use temporary config for this test
@@ -2635,7 +2635,7 @@ mod tests {
 
     #[async_timed_test(timeout_secs = 30)]
     // TODO: OSS: called `Result::unwrap()` on an `Err` value: Listen(Tcp([::1]:0), Os { code: 99, kind: AddrNotAvailable, message: "Cannot assign requested address" })
-    #[cfg_attr(not(feature = "fb"), ignore)]
+    #[cfg_attr(not(fbcode_build), ignore)]
     async fn test_ack_flush() {
         let config = config::global::lock();
         // Set a large value to effectively prevent acks from being sent except
@@ -2659,7 +2659,7 @@ mod tests {
     #[tracing_test::traced_test]
     #[tokio::test]
     // TODO: OSS: failed to retrieve ipv6 address
-    #[cfg_attr(not(feature = "fb"), ignore)]
+    #[cfg_attr(not(fbcode_build), ignore)]
     async fn test_meta_tls_basic() {
         let addr = ChannelAddr::any(ChannelTransport::MetaTls(TlsMode::IpV6));
         let meta_addr = match addr {
@@ -3273,7 +3273,7 @@ mod tests {
     #[tracing_test::traced_test]
     #[tokio::test]
     // TODO: OSS: The logs_assert function returned an error: expected log not found
-    #[cfg_attr(not(feature = "fb"), ignore)]
+    #[cfg_attr(not(fbcode_build), ignore)]
     async fn test_tcp_tx_delivery_timeout() {
         // This link always fails to connect.
         let link = MockLink::<u64>::fail_connects();
@@ -3699,15 +3699,15 @@ mod tests {
     #[tracing_test::traced_test]
     #[async_timed_test(timeout_secs = 30)]
     // TODO: OSS: The logs_assert function returned an error: expected log not found
-    #[cfg_attr(not(feature = "fb"), ignore)]
+    #[cfg_attr(not(fbcode_build), ignore)]
     async fn test_ack_exceeded_limit_with_connected_link() {
         verify_ack_exceeded_limit(false).await;
     }
 
     #[tracing_test::traced_test]
     #[async_timed_test(timeout_secs = 30)]
     // TODO: OSS: The logs_assert function returned an error: expected log not found
-    #[cfg_attr(not(feature = "fb"), ignore)]
+    #[cfg_attr(not(fbcode_build), ignore)]
     async fn test_ack_exceeded_limit_with_broken_link() {
         verify_ack_exceeded_limit(true).await;
     }
@@ -3878,7 +3878,7 @@ mod tests {
 
     #[async_timed_test(timeout_secs = 300)]
     // TODO: OSS: called `Result::unwrap()` on an `Err` value: Listen(Tcp([::1]:0), Os { code: 99, kind: AddrNotAvailable, message: "Cannot assign requested address" })
-    #[cfg_attr(not(feature = "fb"), ignore)]
+    #[cfg_attr(not(fbcode_build), ignore)]
     async fn test_tcp_throughput() {
         let config = config::global::lock();
         let _guard =
@@ -3930,7 +3930,7 @@ mod tests {
     #[tracing_test::traced_test]
     #[async_timed_test(timeout_secs = 60)]
     // TODO: OSS: The logs_assert function returned an error: expected log not found
-    #[cfg_attr(not(feature = "fb"), ignore)]
+    #[cfg_attr(not(fbcode_build), ignore)]
     async fn test_net_tx_closed_on_server_reject() {
         let link = MockLink::<u64>::new();
         let receiver_storage = link.receiver_storage();

hyperactor/src/config.rs

@@ -314,7 +314,7 @@ mod tests {
     #[tracing_test::traced_test]
     #[test]
     // TODO: OSS: The logs_assert function returned an error: missing log lines: {"# export HYPERACTOR_DEFAULT_ENCODING=serde_multipart", ...}
-    #[cfg_attr(not(feature = "fb"), ignore)]
+    #[cfg_attr(not(fbcode_build), ignore)]
     fn test_from_env() {
         // Set environment variables
         // SAFETY: TODO: Audit that the environment access only happens in single-threaded code.

hyperactor/src/host.rs

@@ -1280,7 +1280,7 @@ mod tests {
 
     #[tokio::test]
     // TODO: OSS: called `Result::unwrap()` on an `Err` value: ReadFailed { manifest_path: "/meta-pytorch/monarch/target/debug/deps/hyperactor-0e1fe83af739d976.resources.json", source: Os { code: 2, kind: NotFound, message: "No such file or directory" } }
-    #[cfg_attr(not(feature = "fb"), ignore)]
+    #[cfg_attr(not(fbcode_build), ignore)]
     async fn test_process_proc_manager() {
         hyperactor_telemetry::initialize_logging(crate::clock::ClockKind::default());
 

hyperactor/src/mailbox.rs

@@ -3371,7 +3371,7 @@ mod tests {
 
     #[async_timed_test(timeout_secs = 30)]
     // TODO: OSS: this test is flaky in OSS. Need to repo and fix it.
-    #[cfg_attr(not(feature = "fb"), ignore)]
+    #[cfg_attr(not(fbcode_build), ignore)]
     async fn test_split_port_id_no_reducer() {
         let Setup {
             mut receiver,
@@ -3457,7 +3457,7 @@ mod tests {
 
     #[async_timed_test(timeout_secs = 30)]
     // TODO: OSS: this test is flaky in OSS. Need to repo and fix it.
-    #[cfg_attr(not(feature = "fb"), ignore)]
+    #[cfg_attr(not(fbcode_build), ignore)]
     async fn test_split_port_id_every_n_messages() {
         let config = crate::config::global::lock();
         let _config_guard = config.override_key(

hyperactor_mesh/src/actor_mesh.rs

@@ -1496,6 +1496,7 @@ mod tests {
 
         use crate::alloc::process::ProcessAllocator;
 
+        #[cfg(fbcode_build)]
         fn process_allocator() -> ProcessAllocator {
             ProcessAllocator::new(Command::new(crate::testresource::get(
                 "monarch/hyperactor_mesh/bootstrap",
@@ -1947,6 +1948,7 @@ mod tests {
         use crate::sel;
 
         #[tokio::test]
+        #[cfg(fbcode_build)]
         async fn test_basic() {
             let instance = v1::testing::instance().await;
             let host_mesh = v1::testing::host_mesh(extent!(host = 4)).await;

hyperactor_mesh/src/alloc.rs

@@ -931,6 +931,7 @@ pub(crate) mod testing {
     /// a proc that does not time out when it is asked to wait for
     /// a stuck actor.
     #[tokio::test]
+    #[cfg(fbcode_build)]
     async fn test_allocator_stuck_task() {
         // Override config.
         // Use temporary config for this test

hyperactor_mesh/src/alloc/process.rs

@@ -699,6 +699,7 @@ mod tests {
         crate::testresource::get("monarch/hyperactor_mesh/bootstrap")
     )));
 
+    #[cfg(fbcode_build)]
     #[tokio::test]
     async fn test_sigterm_on_group_fail() {
         let bootstrap_binary = crate::testresource::get("monarch/hyperactor_mesh/bootstrap");

hyperactor_mesh/src/alloc/remoteprocess.rs

@@ -2194,6 +2194,7 @@ mod test_alloc {
     use super::*;
 
     #[async_timed_test(timeout_secs = 60)]
+    #[cfg(fbcode_build)]
     async fn test_alloc_simple() {
         // Use temporary config for this test
         let config = hyperactor::config::global::lock();
@@ -2324,6 +2325,7 @@ mod test_alloc {
     }
 
     #[async_timed_test(timeout_secs = 60)]
+    #[cfg(fbcode_build)]
     async fn test_alloc_host_failure() {
         // Use temporary config for this test
         let config = hyperactor::config::global::lock();
@@ -2456,6 +2458,7 @@ mod test_alloc {
     }
 
     #[async_timed_test(timeout_secs = 15)]
+    #[cfg(fbcode_build)]
     async fn test_alloc_inner_alloc_failure() {
         // SAFETY: Test happens in single-threaded code.
         unsafe {
@@ -2592,6 +2595,7 @@ mod test_alloc {
 
     #[tracing_test::traced_test]
     #[async_timed_test(timeout_secs = 60)]
+    #[cfg(fbcode_build)]
     async fn test_remote_process_alloc_signal_handler() {
         let num_proc_meshes = 5;
         let hosts_per_proc_mesh = 5;

hyperactor_mesh/src/bootstrap.rs

@@ -1506,6 +1506,7 @@ impl BootstrapCommand {
     /// bootstrap processes under proc manager control. Not available
     /// outside of test builds.
     #[cfg(test)]
+    #[cfg(fbcode_build)]
     pub(crate) fn test() -> Self {
         Self {
             program: crate::testresource::get("monarch/hyperactor_mesh/bootstrap"),
@@ -3422,6 +3423,7 @@ mod tests {
     }
 
     #[tokio::test]
+    #[cfg(fbcode_build)]
     async fn bootstrap_handle_terminate_graceful() {
         // Create a root direct-addressed proc + client instance.
         let root = hyperactor::Proc::direct(ChannelTransport::Unix.any(), "root".to_string())
@@ -3485,6 +3487,7 @@ mod tests {
     }
 
     #[tokio::test]
+    #[cfg(fbcode_build)]
     async fn bootstrap_handle_kill_forced() {
         // Root proc + client instance (so the child can dial back).
         let root = hyperactor::Proc::direct(ChannelTransport::Unix.any(), "root".to_string())
@@ -3534,6 +3537,7 @@ mod tests {
     }
 
     #[tokio::test]
+    #[cfg(fbcode_build)]
     async fn bootstrap_canonical_simple() {
         // SAFETY: unit-test scoped
         unsafe {

hyperactor_mesh/src/proc_mesh.rs

@@ -1265,6 +1265,7 @@ mod tests {
         use crate::sel;
 
         #[tokio::test]
+        #[cfg(fbcode_build)]
         async fn test_basic() {
             let instance = v1::testing::instance().await;
             let ext = extent!(host = 4);

hyperactor_mesh/src/testresource.rs

@@ -16,6 +16,7 @@ use std::path::PathBuf;
 ///
 /// We should convert these tests to integration tests, so that cargo can
 /// also manage the binaries.
+#[cfg(fbcode_build)]
 pub fn get<S>(name: S) -> PathBuf
 where
     S: AsRef<str>,

hyperactor_mesh/src/v1/actor_mesh.rs

@@ -467,6 +467,7 @@ mod tests {
     use crate::v1::testing;
 
     #[tokio::test]
+    #[cfg(fbcode_build)]
     async fn test_actor_mesh_ref_lazy_materialization() {
         // 1) Bring up procs and spawn actors.
         let instance = testing::instance().await;
@@ -566,6 +567,7 @@ mod tests {
     }
 
     #[async_timed_test(timeout_secs = 30)]
+    #[cfg(fbcode_build)]
     async fn test_actor_states_with_panic() {
         hyperactor_telemetry::initialize_logging_for_test();
 
@@ -632,6 +634,7 @@ mod tests {
     }
 
     #[async_timed_test(timeout_secs = 30)]
+    #[cfg(fbcode_build)]
     async fn test_actor_states_with_process_exit() {
         hyperactor_telemetry::initialize_logging_for_test();
 
@@ -699,6 +702,7 @@ mod tests {
     }
 
     #[async_timed_test(timeout_secs = 30)]
+    #[cfg(fbcode_build)]
     async fn test_actor_states_on_sliced_mesh() {
         hyperactor_telemetry::initialize_logging_for_test();
 
@@ -772,6 +776,7 @@ mod tests {
     }
 
     #[async_timed_test(timeout_secs = 30)]
+    #[cfg(fbcode_build)]
     async fn test_cast() {
         let config = hyperactor::config::global::lock();
         let _guard = config.override_key(crate::bootstrap::MESH_BOOTSTRAP_ENABLE_PDEATHSIG, false);