chore (inspect): improve logs (add uvicorn handler) and add job_id to model (#3044)

* improve logs (add uvicorn handler) and add job_id to model

Signed-off-by: Ma, Xiangxiang <xiangxiang.ma@intel.com>

* Update application/backend/src/core/logging/setup.py

Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>
Signed-off-by: Max Xiang <maxx.rift@gmail.com>

* Update application/backend/src/core/logging/setup.py

Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>
Signed-off-by: Max Xiang <maxx.rift@gmail.com>

* Update application/backend/src/services/training_service.py

Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>
Signed-off-by: Max Xiang <maxx.rift@gmail.com>

* train_job_id not nullable

Signed-off-by: Ma, Xiangxiang <xiangxiang.ma@intel.com>

* address copilot

Signed-off-by: Ma, Xiangxiang <xiangxiang.ma@intel.com>

---------

Signed-off-by: Ma, Xiangxiang <xiangxiang.ma@intel.com>
Signed-off-by: Max Xiang <maxx.rift@gmail.com>
Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>

Author: maxxgx

application/backend/src/core/lifecycle.py

@@ -9,7 +9,7 @@
 from fastapi import FastAPI
 from loguru import logger
 
-from core.logging import setup_logging
+from core.logging import setup_logging, setup_uvicorn_logging
 from core.scheduler import Scheduler
 from db import init_models
 from settings import get_settings
@@ -21,6 +21,7 @@ async def lifespan(app: FastAPI) -> AsyncGenerator[None]:
     """FastAPI lifespan context manager"""
     # Startup
     setup_logging()
+    setup_uvicorn_logging()
 
     settings = get_settings()
     logger.info(f"Starting {settings.app_name} application...")

application/backend/src/core/logging.py

@@ -0,0 +1,7 @@
+# Copyright (C) 2025 Intel Corporation
+# SPDX-License-Identifier: Apache-2.0
+
+from .log_config import LogConfig
+from .setup import global_log_config, setup_logging, setup_uvicorn_logging
+
+__all__ = ["LogConfig", "global_log_config", "setup_logging", "setup_uvicorn_logging"]

application/backend/src/core/logging/__init__.py

@@ -0,0 +1,43 @@
+# Copyright (C) 2025 Intel Corporation
+# SPDX-License-Identifier: Apache-2.0
+import inspect
+import logging
+
+from loguru import logger
+
+
+class InterceptHandler(logging.Handler):
+    """
+    This handler intercepts standard logging calls and forwards them to loguru
+    while preserving the original caller information.
+    """
+
+    def emit(self, record: logging.LogRecord) -> None:
+        # Get corresponding Loguru level if it exists.
+        level: str | int
+        try:
+            level = logger.level(record.levelname).name
+        except ValueError:
+            level = record.levelno
+
+        # Find caller from where originated the logged message.
+        frame, depth = inspect.currentframe(), 0
+        while frame and (depth == 0 or frame.f_code.co_filename == logging.__file__):
+            frame = frame.f_back
+            depth += 1
+
+        logger.opt(depth=depth, exception=record.exc_info).log(level, record.getMessage())
+
+
+class LoggerStdoutWriter:
+    """Wrapper for redirecting stdout to logger"""
+
+    @staticmethod
+    def write(msg: str) -> None:
+        msg = msg.rstrip("\n")
+        if msg:
+            logger.info(msg)
+
+    @staticmethod
+    def flush() -> None:
+        pass

application/backend/src/core/logging/handlers.py

@@ -0,0 +1,32 @@
+# Copyright (C) 2025 Intel Corporation
+# SPDX-License-Identifier: Apache-2.0
+
+import os
+from dataclasses import dataclass
+
+from workers import DispatchingWorker, InferenceWorker, StreamLoader, TrainingWorker
+
+LOG_FOLDER = "logs"
+WORKERS_FOLDER = os.path.join(LOG_FOLDER, "workers")
+JOBS_FOLDER = os.path.join(LOG_FOLDER, "jobs")
+
+
+@dataclass
+class LogConfig:
+    """Configuration for logging behavior."""
+
+    rotation: str = "10 MB"
+    retention: str = "10 days"
+    level: str = "DEBUG"
+    serialize: bool = True
+    log_folder: str = LOG_FOLDER
+    # Mapping of worker classes to their dedicated log files
+    # None key is used for application-level logs that don't belong to any specific worker
+    worker_log_info = {
+        TrainingWorker.__name__: "training.log",
+        InferenceWorker.__name__: "inference.log",
+        DispatchingWorker.__name__: "dispatching.log",
+        StreamLoader.__name__: "stream_loader.log",
+        None: "app.log",
+    }
+    tensorboard_log_path: str = os.path.join(LOG_FOLDER, "tensorboard")