Skip to content

Commit bb1eccb

Browse files
samluryemeta-codesync[bot]
authored andcommitted
Add bootstrap command hint when proc bootstrap fails (#1719)
Summary: Pull Request resolved: #1719 If proc bootstrap (e.g., `monarch/_src/actor/bootstrap_main.py`) fails, provide both the original error message and a hint that indicates that the user might need to look at their proc bootstrap command, and points them to the correct API. ghstack-source-id: 320149478 exported-using-ghexport Reviewed By: mariusae Differential Revision: D85903691 fbshipit-source-id: f27d2f36c4351adfc11795ea646a5d9098775e5e
1 parent f8d2fed commit bb1eccb

File tree

1 file changed

+40
-32
lines changed

1 file changed

+40
-32
lines changed

python/monarch/_src/actor/bootstrap_main.py

Lines changed: 40 additions & 32 deletions
Original file line numberDiff line numberDiff line change
@@ -37,40 +37,48 @@ async def main() -> None:
3737

3838

3939
def invoke_main() -> None:
40-
# if this is invoked with the stdout piped somewhere, then print
41-
# changes its buffering behavior. So we default to the standard
42-
# behavior of std out as if it were a terminal.
43-
sys.stdout.reconfigure(line_buffering=True)
44-
global bootstrap_main
45-
46-
# TODO: figure out what from worker_main.py we should reproduce here.
47-
48-
from monarch._src.actor.telemetry import TracingForwarder # noqa
49-
50-
if os.environ.get("MONARCH_ERROR_DURING_BOOTSTRAP_FOR_TESTING") == "1":
51-
raise RuntimeError("Error during bootstrap for testing")
52-
53-
# forward logs to rust tracing. Defaults to on.
54-
if os.environ.get("MONARCH_PYTHON_LOG_TRACING", "1") == "1":
55-
# we can stream python logs now; no need to forward them to rust processes
56-
pass
57-
# install opentelemetry tracing
58-
5940
try:
60-
with (
61-
importlib.resources.as_file(
62-
importlib.resources.files("monarch") / "py-spy"
63-
) as pyspy,
64-
):
65-
if pyspy.exists():
66-
os.environ["PYSPY_BIN"] = str(pyspy)
67-
# fallback to using local py-spy
41+
# if this is invoked with the stdout piped somewhere, then print
42+
# changes its buffering behavior. So we default to the standard
43+
# behavior of std out as if it were a terminal.
44+
sys.stdout.reconfigure(line_buffering=True)
45+
global bootstrap_main
46+
47+
# TODO: figure out what from worker_main.py we should reproduce here.
48+
49+
from monarch._src.actor.telemetry import TracingForwarder # noqa
50+
51+
if os.environ.get("MONARCH_ERROR_DURING_BOOTSTRAP_FOR_TESTING") == "1":
52+
raise RuntimeError("Error during bootstrap for testing")
53+
54+
# forward logs to rust tracing. Defaults to on.
55+
if os.environ.get("MONARCH_PYTHON_LOG_TRACING", "1") == "1":
56+
# we can stream python logs now; no need to forward them to rust processes
57+
pass
58+
# install opentelemetry tracing
59+
60+
try:
61+
with (
62+
importlib.resources.as_file(
63+
importlib.resources.files("monarch") / "py-spy"
64+
) as pyspy,
65+
):
66+
if pyspy.exists():
67+
os.environ["PYSPY_BIN"] = str(pyspy)
68+
# fallback to using local py-spy
69+
except Exception as e:
70+
logging.warning(f"Failed to set up py-spy: {e}")
71+
72+
from monarch._src.actor.debugger.breakpoint import remote_breakpointhook
73+
74+
sys.breakpointhook = remote_breakpointhook
6875
except Exception as e:
69-
logging.warning(f"Failed to set up py-spy: {e}")
70-
71-
from monarch._src.actor.debugger.breakpoint import remote_breakpointhook
72-
73-
sys.breakpointhook = remote_breakpointhook
76+
bootstrap_err = RuntimeError(
77+
f"Failed to bootstrap proc due to: {e}\nMake sure your proc bootstrap command is correct. "
78+
f"Provided command:\n{' '.join([sys.executable, *sys.argv])}\nTo specify your proc bootstrap command, use the "
79+
f"`bootstrap_cmd` kwarg in `monarch.actor.HostMesh.allocate_nonblocking(...)`."
80+
)
81+
raise bootstrap_err from e
7482

7583
# Start an event loop for PythonActors to use.
7684
asyncio.run(main())

0 commit comments

Comments
 (0)