Code review

Author: mjacar

python/ray/tests/test_runtime_env.py

@@ -9,6 +9,7 @@
 import pytest
 
 import ray
+from ray.exceptions import RuntimeEnvSetupError
 from ray.runtime_env import RuntimeEnv, RuntimeEnvConfig
 
 
@@ -167,5 +168,30 @@ def run(runtime_env):
         run(runtime_env)
 
 
+def test_large_runtime_env_fails_fast(start_cluster_shared):
+    """
+    Tests that a task with a runtime_env that is too large fails quickly
+    instead of hanging. This is a regression test for #47432.
+    """
+    cluster, address = start_cluster_shared
+    ray.init(address)
+
+    # Create a runtime_env with a very large environment variable to trigger
+    # the E2BIG error. The exact limit for process arguments depends on the OS,
+    # but ~400KB should exceed it on most systems.
+    large_env_vars = {"MY_HUGE_VAR": "X" * 4096 * 100}
+    runtime_env = {"env_vars": large_env_vars}
+
+    @ray.remote
+    def f():
+        # This code should not be reached.
+        return 1
+
+    # The PR ensures the E2BIG error from the raylet is propagated to the
+    # driver, which should raise a RuntimeEnvSetupError.
+    with pytest.raises(RuntimeEnvSetupError):
+        ray.get(f.options(runtime_env=runtime_env).remote())
+
+
 if __name__ == "__main__":
     sys.exit(pytest.main(["-sv", __file__]))

src/ray/raylet/worker_pool.cc

@@ -530,14 +530,10 @@ std::tuple<Process, StartupToken> WorkerPool::StartWorkerProcess(
   // Start a process and measure the startup time.
   Process proc = StartProcess(worker_command_args, env, ec);
   if (ec) {
-    if (ec.value() == E2BIG) {
-      RAY_LOG(ERROR) << "E2BIG error occurred when starting worker process. Worker "
+    RAY_CHECK(ec.value() == E2BIG);
+    RAY_LOG(WARNING) << "E2BIG error occurred when starting worker process. Worker "
                         "command arguments likely too long.";
-      *status = PopWorkerStatus::ArgumentListTooLong;
-      return {Process(), (StartupToken)-1};
-    }
-    RAY_LOG(ERROR) << "Failed to start worker process: " << ec.message();
-    *status = PopWorkerStatus::RuntimeEnvCreationFailed;
+    *status = PopWorkerStatus::ArgumentListTooLong;
     return {Process(), (StartupToken)-1};
   }
   stats::NumWorkersStarted.Record(1);
@@ -684,8 +680,10 @@ Process WorkerPool::StartProcess(const std::vector<std::string> &worker_command_
 
   Process child(argv.data(), io_service_, ec, /*decouple=*/false, env);
   if (!child.IsValid() || ec) {
-    // errorcode 24: Too many files. This is caused by ulimit.
-    if (ec.value() == 24) {
+    if (ec.value() == E2BIG) {
+      // Do nothing here; the error code `ec` will be propagated to the caller.
+    } else if (ec.value() == 24) {
+      // errorcode 24: Too many files. This is caused by ulimit.
       RAY_LOG(FATAL) << "Too many workers, failed to create a file. Try setting "
                      << "`ulimit -n <num_files>` then restart Ray.";
     } else {

src/ray/raylet/worker_pool.h

@@ -51,30 +51,25 @@ using WorkerCommandMap =
 
 enum PopWorkerStatus {
   // OK.
-  // A registered worker will be returned with callback.
   OK = 0,
   // Job config is not found.
-  // A nullptr worker will be returned with callback.
   JobConfigMissing = 1,
   // Worker process startup rate is limited.
-  // A nullptr worker will be returned with callback.
   TooManyStartingWorkerProcesses = 2,
   // Worker process has been started, but the worker did not register at the raylet within
   // the timeout.
-  // A nullptr worker will be returned with callback.
   WorkerPendingRegistration = 3,
   // Any fails of runtime env creation.
-  // A nullptr worker will be returned with callback.
   RuntimeEnvCreationFailed = 4,
   // The task's job has finished.
-  // A nullptr worker will be returned with callback.
   JobFinished = 5,
   // The worker process failed to launch because the OS returned an `E2BIG`
   // (Argument list too long) error. This typically occurs when a `runtime_env`
   // is so large that its serialized context exceeds the kernel's command-line
   // argument size limit.
-  // A nullptr worker will be returned with callback.
   ArgumentListTooLong = 6,
+  // The worker process failed to launch.
+  WorkerLaunchFailed = 7,
 };
 
 /// \param[in] worker The started worker instance. Nullptr if worker is not started.