Merge pull request #99 from Point72/tkp/cc

Switch to ccflow hostname resolver, mountoutputs respect hydra output if set

Author: timkpaine

csp_gateway/server/config/base.yaml

@@ -17,4 +17,4 @@ start:
 # https://hydra.cc/docs/configure_hydra/job/
 hydra:
   run:
-    dir: outputs/${oc.env:HOSTNAME,localhost}_${hydra.job.name}/${now:%Y-%m-%d}/${now:%H-%M-%S}
+    dir: outputs/${hostname:}_${hydra.job.name}/${now:%Y-%m-%d}/${now:%H-%M-%S}

csp_gateway/server/config/gateway/demo.yaml

@@ -7,6 +7,8 @@ port: ???
 modules:
   example_module:
     _target_: csp_gateway.server.demo.simple.ExampleModule
+  mount_outputs:
+    _target_: csp_gateway.MountOutputsFolder
   mount_perspective_tables:
     _target_: csp_gateway.MountPerspectiveTables
     update_interval: 00:00:01
@@ -21,6 +23,7 @@ gateway:
     UI: True
   modules:
     - /modules/example_module
+    - /modules/mount_outputs
     - /modules/mount_perspective_tables
     - /modules/mount_rest_routes
   channels:

csp_gateway/server/demo/__main__.py

@@ -4,6 +4,7 @@
 from csp_gateway import (
     Gateway,
     GatewaySettings,
+    MountOutputsFolder,
     MountPerspectiveTables,
     MountRestRoutes,
 )
@@ -19,6 +20,7 @@
     settings=GatewaySettings(),
     modules=[
         ExampleModule(),
+        MountOutputsFolder(),
         MountPerspectiveTables(),
         MountRestRoutes(force_mount_all=True),
     ],

csp_gateway/server/modules/web/outputs.py

@@ -1,8 +1,10 @@
 import os
 import os.path
+from typing import Optional
 
 from fastapi import HTTPException, Request
 from fastapi.responses import HTMLResponse, StreamingResponse
+from hydra.core.hydra_config import HydraConfig
 
 try:
     # conditional on libmagic being installed on the machine
@@ -17,9 +19,14 @@
 
 
 class MountOutputsFolder(GatewayModule):
+    dir: Optional[str] = None
+
     def connect(self, channels: GatewayChannels) -> None:
-        # NO-OP
-        ...
+        if self.dir is None:
+            if HydraConfig.initialized():
+                self.dir = os.path.abspath(os.path.join(HydraConfig.get().runtime.output_dir, "..", "..", ".."))
+            else:
+                self.dir = os.path.abspath(os.path.join(os.getcwd(), "outputs"))
 
     def rest(self, app: GatewayWebApp) -> None:
         app_router = app.get_router("app")
@@ -32,11 +39,10 @@ def browse_logs(full_path: str, request: Request):
             This endpoint is a small webpage for browsing the [hydra](https://github.com/facebookresearch/hydra)
             output logs and configuration settings of the running application.
             """
-            outputs_dir = os.path.join(os.getcwd(), "outputs")
-            file_or_dir = outputs_dir
+            file_or_dir = self.dir
             if full_path:
                 file_or_dir = os.path.join(file_or_dir, full_path)
-            if os.path.abspath(file_or_dir).startswith(outputs_dir) and os.path.exists(file_or_dir):
+            if os.path.abspath(file_or_dir).startswith(self.dir) and os.path.exists(file_or_dir):
                 if os.path.isdir(file_or_dir):
                     files = os.listdir(file_or_dir)
                     files_paths = sorted([f"{request.url._url}/{f}".replace("outputs//", "outputs/") for f in files])

pyproject.toml

@@ -59,7 +59,7 @@ develop = [
 ]
 server = [
     "atomic-counter>=0.1.3",
-    "ccflow>=0.5.2,<1",
+    "ccflow>=0.5.9,<1",
     "colorlog",
     "csp>=0.8.0,<1",
     "deprecation",
@@ -70,7 +70,7 @@ server = [
     "janus",
     "omegaconf",
     "orjson",
-    "perspective-python>=3.3,!=3.4.0,<4",
+    "perspective-python>=3.7,<4",
     "psutil",
     "pyarrow",
     "pydantic>=2",