feat(logging): implement loguru + add json logging format

Author: MickaelCa

.pre-commit-config.yaml

@@ -117,6 +117,7 @@ repos:
             click>=8.0.0,
             'fastapi[standard]>=0.109.1',
             httpx,
+            loguru>=0.7.0,
             pathspec>=0.12.1,
             prometheus-client,
             pydantic,
@@ -143,6 +144,7 @@ repos:
             click>=8.0.0,
             'fastapi[standard]>=0.109.1',
             httpx,
+            loguru>=0.7.0,
             pathspec>=0.12.1,
             prometheus-client,
             pydantic,

CONTRIBUTING.md

@@ -65,8 +65,7 @@ If you ever get stuck, reach out on [Discord](https://discord.com/invite/zerRaGK
 9. **Run the local server** to sanity-check:
 
     ```bash
-    cd src
-    uvicorn server.main:app
+    LOG_FORMAT=human python run_server.py
     ```
 
     Open [http://localhost:8000](http://localhost:8000) to confirm everything works.

Dockerfile

@@ -37,11 +37,12 @@ RUN set -eux; \
 
 COPY --from=python-builder --chown=$UID:$GID /usr/local/lib/python3.13/site-packages/ /usr/local/lib/python3.13/site-packages/
 COPY --chown=$UID:$GID src/ ./
+COPY --chown=$UID:$GID run_server.py ./
 
 RUN set -eux; \
     chown -R appuser:appuser /app
 USER appuser
 
 EXPOSE 8000
 EXPOSE 9090
-CMD ["python", "-m", "uvicorn", "server.main:app", "--host", "0.0.0.0", "--port", "8000"]
+CMD ["python", "run_server.py"]

compose.yml

@@ -1,27 +1,45 @@
-# Common base configuration for all services
+x-base-environment: &base-environment
+  # Python Configuration
+  PYTHONUNBUFFERED: "1"
+  PYTHONDONTWRITEBYTECODE: "1"
+  # Host Configuration
+  ALLOWED_HOSTS: ${ALLOWED_HOSTS:-gitingest.com,*.gitingest.com,localhost,127.0.0.1}
+  # Metrics Configuration
+  GITINGEST_METRICS_ENABLED: ${GITINGEST_METRICS_ENABLED:-true}
+  GITINGEST_METRICS_HOST: ${GITINGEST_METRICS_HOST:-0.0.0.0}
+  GITINGEST_METRICS_PORT: ${GITINGEST_METRICS_PORT:-9090}
+  # Sentry Configuration
+  GITINGEST_SENTRY_ENABLED: ${GITINGEST_SENTRY_ENABLED:-false}
+  GITINGEST_SENTRY_DSN: ${GITINGEST_SENTRY_DSN:-}
+  GITINGEST_SENTRY_TRACES_SAMPLE_RATE: ${GITINGEST_SENTRY_TRACES_SAMPLE_RATE:-1.0}
+  GITINGEST_SENTRY_PROFILE_SESSION_SAMPLE_RATE: ${GITINGEST_SENTRY_PROFILE_SESSION_SAMPLE_RATE:-1.0}
+  GITINGEST_SENTRY_PROFILE_LIFECYCLE: ${GITINGEST_SENTRY_PROFILE_LIFECYCLE:-trace}
+  GITINGEST_SENTRY_SEND_DEFAULT_PII: ${GITINGEST_SENTRY_SEND_DEFAULT_PII:-true}
+
+x-prod-environment: &prod-environment
+  GITINGEST_SENTRY_ENVIRONMENT: ${GITINGEST_SENTRY_ENVIRONMENT:-production}
+
+x-dev-environment: &dev-environment
+  DEBUG: "true"
+  LOG_LEVEL: "debug"
+  RELOAD: "true"
+  GITINGEST_SENTRY_ENVIRONMENT: ${GITINGEST_SENTRY_ENVIRONMENT:-development}
+  # S3 Configuration for development
+  S3_ENABLED: "true"
+  S3_ENDPOINT: http://minio:9000
+  S3_ACCESS_KEY: ${S3_ACCESS_KEY:-gitingest}
+  S3_SECRET_KEY: ${S3_SECRET_KEY:-gitingest123}
+  S3_BUCKET_NAME: ${S3_BUCKET_NAME:-gitingest-bucket}
+  S3_REGION: ${S3_REGION:-us-east-1}
+  S3_DIRECTORY_PREFIX: ${S3_DIRECTORY_PREFIX:-dev}
+  S3_ALIAS_HOST: ${S3_ALIAS_HOST:-http://127.0.0.1:9000/${S3_BUCKET_NAME:-gitingest-bucket}}
+
 x-app-base: &app-base
   ports:
     - "${APP_WEB_BIND:-8000}:8000"  # Main application port
     - "${GITINGEST_METRICS_HOST:-127.0.0.1}:${GITINGEST_METRICS_PORT:-9090}:9090"  # Metrics port
-  environment:
-    # Python Configuration
-    - PYTHONUNBUFFERED=1
-    - PYTHONDONTWRITEBYTECODE=1
-    # Host Configuration
-    - ALLOWED_HOSTS=${ALLOWED_HOSTS:-gitingest.com,*.gitingest.com,localhost,127.0.0.1}
-    # Metrics Configuration
-    - GITINGEST_METRICS_ENABLED=${GITINGEST_METRICS_ENABLED:-true}
-    - GITINGEST_METRICS_HOST=${GITINGEST_METRICS_HOST:-127.0.0.1}
-    - GITINGEST_METRICS_PORT=${GITINGEST_METRICS_PORT:-9090}
-    # Sentry Configuration
-    - GITINGEST_SENTRY_ENABLED=${GITINGEST_SENTRY_ENABLED:-false}
-    - GITINGEST_SENTRY_DSN=${GITINGEST_SENTRY_DSN:-}
-    - GITINGEST_SENTRY_TRACES_SAMPLE_RATE=${GITINGEST_SENTRY_TRACES_SAMPLE_RATE:-1.0}
-    - GITINGEST_SENTRY_PROFILE_SESSION_SAMPLE_RATE=${GITINGEST_SENTRY_PROFILE_SESSION_SAMPLE_RATE:-1.0}
-    - GITINGEST_SENTRY_PROFILE_LIFECYCLE=${GITINGEST_SENTRY_PROFILE_LIFECYCLE:-trace}
-    - GITINGEST_SENTRY_SEND_DEFAULT_PII=${GITINGEST_SENTRY_SEND_DEFAULT_PII:-true}
   user: "1000:1000"
-  command: ["python", "-m", "uvicorn", "server.main:app", "--host", "0.0.0.0", "--port", "8000"]
+  command: ["python", "run_server.py"]
 
 services:
   # Production service configuration
@@ -31,7 +49,7 @@ services:
     profiles:
       - prod
     environment:
-      - GITINGEST_SENTRY_ENVIRONMENT=${GITINGEST_SENTRY_ENVIRONMENT:-production}
+      <<: [*base-environment, *prod-environment]
     restart: unless-stopped
 
   # Development service configuration
@@ -43,24 +61,13 @@ services:
     profiles:
       - dev
     environment:
-      - DEBUG=true
-      - GITINGEST_SENTRY_ENVIRONMENT=${GITINGEST_SENTRY_ENVIRONMENT:-development}
-      # S3 Configuration
-      - S3_ENABLED=true
-      - S3_ENDPOINT=http://minio:9000
-      - S3_ACCESS_KEY=${S3_ACCESS_KEY:-gitingest}
-      - S3_SECRET_KEY=${S3_SECRET_KEY:-gitingest123}
-      # Use lowercase bucket name to ensure compatibility with MinIO
-      - S3_BUCKET_NAME=${S3_BUCKET_NAME:-gitingest-bucket}
-      - S3_REGION=${S3_REGION:-us-east-1}
-      - S3_DIRECTORY_PREFIX=${S3_DIRECTORY_PREFIX:-dev}
-      # Public URL for S3 resources
-      - S3_ALIAS_HOST=${S3_ALIAS_HOST:-http://127.0.0.1:9000/${S3_BUCKET_NAME:-gitingest-bucket}}
+      <<: [*base-environment, *dev-environment]
     volumes:
       # Mount source code for live development
       - ./src:/app:ro
+      - ./run_server.py:/app/run_server.py:ro
     # Use --reload flag for hot reloading during development
-    command: ["python", "-m", "uvicorn", "server.main:app", "--host", "0.0.0.0", "--port", "8000", "--reload"]
+    command: ["python", "run_server.py"]
     depends_on:
       minio-setup:
         condition: service_completed_successfully
@@ -73,9 +80,9 @@ services:
     ports:
       - "9000:9000"  # API port
       - "9001:9001"  # Console port
-    environment:
-      - MINIO_ROOT_USER=${MINIO_ROOT_USER:-minioadmin}
-      - MINIO_ROOT_PASSWORD=${MINIO_ROOT_PASSWORD:-minioadmin}
+    environment: &minio-environment
+      MINIO_ROOT_USER: ${MINIO_ROOT_USER:-minioadmin}
+      MINIO_ROOT_PASSWORD: ${MINIO_ROOT_PASSWORD:-minioadmin}
     volumes:
       - minio-data:/data
     command: server /data --console-address ":9001"
@@ -96,11 +103,10 @@ services:
       minio:
         condition: service_healthy
     environment:
-      - MINIO_ROOT_USER=${MINIO_ROOT_USER:-minioadmin}
-      - MINIO_ROOT_PASSWORD=${MINIO_ROOT_PASSWORD:-minioadmin}
-      - S3_ACCESS_KEY=${S3_ACCESS_KEY:-gitingest}
-      - S3_SECRET_KEY=${S3_SECRET_KEY:-gitingest123}
-      - S3_BUCKET_NAME=${S3_BUCKET_NAME:-gitingest-bucket}
+      <<: *minio-environment
+      S3_ACCESS_KEY: ${S3_ACCESS_KEY:-gitingest}
+      S3_SECRET_KEY: ${S3_SECRET_KEY:-gitingest123}
+      S3_BUCKET_NAME: ${S3_BUCKET_NAME:-gitingest-bucket}
     volumes:
       - ./.docker/minio/setup.sh:/setup.sh:ro
     entrypoint: sh

pyproject.toml

@@ -7,6 +7,7 @@ requires-python = ">= 3.8"
 dependencies = [
     "click>=8.0.0",
     "httpx",
+    "loguru>=0.7.0",
     "pathspec>=0.12.1",
     "pydantic",
     "python-dotenv",
@@ -96,7 +97,6 @@ ignore = [  # https://docs.astral.sh/ruff/rules/...
 
     # TODO: fix the following issues:
     "TD003", # missing-todo-link, TODO: add issue links
-    "T201", # print, TODO: replace with logging
     "S108", # hardcoded-temp-file, TODO: replace with tempfile
     "BLE001", # blind-except, TODO: replace with specific exceptions
     "FAST003", # fast-api-unused-path-parameter, TODO: fix

requirements.txt

@@ -2,6 +2,7 @@ boto3>=1.28.0  # AWS SDK for S3 support
 click>=8.0.0
 fastapi[standard]>=0.109.1  # Vulnerable to https://osv.dev/vulnerability/PYSEC-2024-38
 httpx
+loguru>=0.7.0
 pathspec>=0.12.1
 prometheus-client
 pydantic

run_server.py

@@ -0,0 +1,33 @@
+#!/usr/bin/env python3
+"""Custom server startup script to ensure logging is configured before uvicorn starts."""
+
+import os
+
+import uvicorn
+
+# Import logging configuration first to intercept all logging
+from gitingest.utils.logging_config import get_logger
+
+logger = get_logger(__name__)
+
+if __name__ == "__main__":
+    # Get configuration from environment variables
+    host = os.getenv("HOST", "0.0.0.0")  # noqa: S104
+    port = int(os.getenv("PORT", "8000"))
+    reload = os.getenv("RELOAD", "false").lower() == "true"
+
+    logger.info(
+        "Starting Gitingest server",
+        extra={
+            "host": host,
+            "port": port,
+        },
+    )
+
+    uvicorn.run(
+        "server.main:app",
+        host=host,
+        port=port,
+        reload=reload,
+        log_config=None,  # Disable uvicorn's default logging config
+    )

src/gitingest/__main__.py

@@ -12,6 +12,12 @@
 from gitingest.config import MAX_FILE_SIZE, OUTPUT_FILE_NAME
 from gitingest.entrypoint import ingest_async
 
+# Import logging configuration first to intercept all logging
+from gitingest.utils.logging_config import get_logger
+
+# Initialize logger for this module
+logger = get_logger(__name__)
+
 
 class _CLIArgs(TypedDict):
     source: str

src/gitingest/ingestion.py

@@ -9,10 +9,14 @@
 from gitingest.output_formatter import format_node
 from gitingest.schemas import FileSystemNode, FileSystemNodeType, FileSystemStats
 from gitingest.utils.ingestion_utils import _should_exclude, _should_include
+from gitingest.utils.logging_config import get_logger
 
 if TYPE_CHECKING:
     from gitingest.schemas import IngestionQuery
 
+# Initialize logger for this module
+logger = get_logger(__name__)
+
 
 def ingest_query(query: IngestionQuery) -> tuple[str, str, str]:
     """Run the ingestion process for a parsed query.
@@ -111,7 +115,14 @@ def _process_node(node: FileSystemNode, query: IngestionQuery, stats: FileSystem
             _process_symlink(path=sub_path, parent_node=node, stats=stats, local_path=query.local_path)
         elif sub_path.is_file():
             if sub_path.stat().st_size > query.max_file_size:
-                print(f"Skipping file {sub_path}: would exceed max file size limit")
+                logger.warning(
+                    "Skipping file: would exceed max file size limit",
+                    extra={
+                        "file_path": str(sub_path),
+                        "file_size": sub_path.stat().st_size,
+                        "max_file_size": query.max_file_size,
+                    },
+                )
                 continue
             _process_file(path=sub_path, parent_node=node, stats=stats, local_path=query.local_path)
         elif sub_path.is_dir():
@@ -133,7 +144,7 @@ def _process_node(node: FileSystemNode, query: IngestionQuery, stats: FileSystem
             node.file_count += child_directory_node.file_count
             node.dir_count += 1 + child_directory_node.dir_count
         else:
-            print(f"Warning: {sub_path} is an unknown file type, skipping")
+            logger.warning("Unknown file type, skipping", extra={"file_path": str(sub_path)})
 
     node.sort_children()
 
@@ -186,12 +197,27 @@ def _process_file(path: Path, parent_node: FileSystemNode, stats: FileSystemStat
 
     """
     if stats.total_files + 1 > MAX_FILES:
-        print(f"Maximum file limit ({MAX_FILES}) reached")
+        logger.warning(
+            "Maximum file limit reached",
+            extra={
+                "current_files": stats.total_files,
+                "max_files": MAX_FILES,
+                "file_path": str(path),
+            },
+        )
         return
 
     file_size = path.stat().st_size
     if stats.total_size + file_size > MAX_TOTAL_SIZE_BYTES:
-        print(f"Skipping file {path}: would exceed total size limit")
+        logger.warning(
+            "Skipping file: would exceed total size limit",
+            extra={
+                "file_path": str(path),
+                "file_size": file_size,
+                "current_total_size": stats.total_size,
+                "max_total_size": MAX_TOTAL_SIZE_BYTES,
+            },
+        )
         return
 
     stats.total_files += 1
@@ -232,15 +258,33 @@ def limit_exceeded(stats: FileSystemStats, depth: int) -> bool:
 
     """
     if depth > MAX_DIRECTORY_DEPTH:
-        print(f"Maximum depth limit ({MAX_DIRECTORY_DEPTH}) reached")
+        logger.warning(
+            "Maximum directory depth limit reached",
+            extra={
+                "current_depth": depth,
+                "max_depth": MAX_DIRECTORY_DEPTH,
+            },
+        )
         return True
 
     if stats.total_files >= MAX_FILES:
-        print(f"Maximum file limit ({MAX_FILES}) reached")
+        logger.warning(
+            "Maximum file limit reached",
+            extra={
+                "current_files": stats.total_files,
+                "max_files": MAX_FILES,
+            },
+        )
         return True  # TODO: end recursion
 
     if stats.total_size >= MAX_TOTAL_SIZE_BYTES:
-        print(f"Maxumum total size limit ({MAX_TOTAL_SIZE_BYTES / 1024 / 1024:.1f}MB) reached")
+        logger.warning(
+            "Maximum total size limit reached",
+            extra={
+                "current_size_mb": stats.total_size / 1024 / 1024,
+                "max_size_mb": MAX_TOTAL_SIZE_BYTES / 1024 / 1024,
+            },
+        )
         return True  # TODO: end recursion
 
     return False

src/gitingest/output_formatter.py

@@ -8,10 +8,14 @@
 
 from gitingest.schemas import FileSystemNode, FileSystemNodeType
 from gitingest.utils.compat_func import readlink
+from gitingest.utils.logging_config import get_logger
 
 if TYPE_CHECKING:
     from gitingest.schemas import IngestionQuery
 
+# Initialize logger for this module
+logger = get_logger(__name__)
+
 _TOKEN_THRESHOLDS: list[tuple[int, str]] = [
     (1_000_000, "M"),
     (1_000, "k"),
@@ -189,8 +193,8 @@ def _format_token_count(text: str) -> str | None:
     try:
         encoding = tiktoken.get_encoding("o200k_base")  # gpt-4o, gpt-4o-mini
         total_tokens = len(encoding.encode(text, disallowed_special=()))
-    except (ValueError, UnicodeEncodeError) as exc:
-        print(exc)
+    except (ValueError, UnicodeEncodeError):
+        logger.exception("Failed to encode text for token counting")
         return None
 
     for threshold, suffix in _TOKEN_THRESHOLDS: