Add LOG_FORMAT env, add logging for the backend

Author: ix-56h

.env.example

@@ -56,3 +56,4 @@ S3_REGION=us-east-1
 S3_ALIAS_HOST=127.0.0.1:9000/gitingest-bucket
 # Optional prefix for S3 file paths (if set, prefixes all S3 paths with this value)
 # S3_DIRECTORY_PREFIX=my-prefix
+LOG_FORMAT=JSON

src/gitingest/__main__.py

@@ -12,9 +12,9 @@
 
 from gitingest.config import MAX_FILE_SIZE, OUTPUT_FILE_NAME
 from gitingest.entrypoint import ingest_async
-from gitingest.logging_config import setup_json_logging
+from gitingest.logging_config import setup_logging
 
-setup_json_logging()
+setup_logging()
 
 logger = logging.getLogger(__name__)
 
@@ -169,9 +169,9 @@ async def _async_main(
         output_target = output if output is not None else OUTPUT_FILE_NAME
 
         if output_target == "-":
-            click.echo("Analyzing source, preparing output for stdout...", err=True)
+            logger.debug("Analyzing source, preparing output for stdout...")
         else:
-            click.echo(f"Analyzing source, output will be written to '{output_target}'...", err=True)
+            logger.debug("Analyzing source, output will be written to '%s'...", output_target)
 
         summary, _, _ = await ingest_async(
             source,
@@ -186,18 +186,18 @@ async def _async_main(
         )
     except Exception as exc:
         # Convert any exception into Click.Abort so that exit status is non-zero
-        click.echo(f"Error: {exc}", err=True)
+        logger.exception("Ingest failed.", exc_info=exc)
         raise click.Abort from exc
 
     if output_target == "-":  # stdout
-        click.echo("\n--- Summary ---", err=True)
-        click.echo(summary, err=True)
-        click.echo("--- End Summary ---", err=True)
-        click.echo("Analysis complete! Output sent to stdout.", err=True)
+        logger.info("--- Summary ---")
+        logger.info(summary)
+        logger.info("--- End Summary ---")
+        logger.info("Analysis complete! Output sent to stdout.")
     else:  # file
-        click.echo(f"Analysis complete! Output written to: {output_target}")
-        click.echo("\nSummary:")
-        click.echo(summary)
+        logger.info("Analysis complete! Output written to: %s", output_target)
+        logger.info("Summary:")
+        logger.info(summary)
 
 
 if __name__ == "__main__":

src/gitingest/entrypoint.py

@@ -302,22 +302,17 @@ def _handle_remove_readonly(
 
 
 async def _write_output(tree: str, content: str, target: str | None) -> None:
-    """Write combined output to ``target`` (``"-"`` ⇒ stdout).
-
-    Parameters
-    ----------
-    tree : str
-        The tree-like string representation of the file structure.
-    content : str
-        The content of the files in the repository or directory.
-    target : str | None
-        The path to the output file. If ``None``, the results are not written to a file.
-
-    """
+    """Write combined output to ``target`` (``"-"`` ⇒ stdout)."""
     data = f"{tree}\n{content}"
     loop = asyncio.get_running_loop()
-    if target == "-":
-        await loop.run_in_executor(None, sys.stdout.write, data)
-        await loop.run_in_executor(None, sys.stdout.flush)
-    elif target is not None:
-        await loop.run_in_executor(None, Path(target).write_text, data, "utf-8")
+    try:
+        if target == "-":
+            logger.debug("Writing output to stdout.")
+            await loop.run_in_executor(None, sys.stdout.write, data)
+            await loop.run_in_executor(None, sys.stdout.flush)
+        elif target is not None:
+            logger.debug("Writing output to file: %s", target)
+            await loop.run_in_executor(None, Path(target).write_text, data, "utf-8")
+    except Exception as exc:
+        logger.exception("Failed to write output to %s.", target, exc_info=exc)
+        raise

src/gitingest/ingestion.py

@@ -114,7 +114,7 @@ def _process_node(node: FileSystemNode, query: IngestionQuery, stats: FileSystem
             _process_symlink(path=sub_path, parent_node=node, stats=stats, local_path=query.local_path)
         elif sub_path.is_file():
             if sub_path.stat().st_size > query.max_file_size:
-                logger.info("Skipping file %s: would exceed max file size limit", sub_path)
+                logger.debug("Skipping file %s: would exceed max file size limit", sub_path)
                 continue
             _process_file(path=sub_path, parent_node=node, stats=stats, local_path=query.local_path)
         elif sub_path.is_dir():
@@ -194,7 +194,7 @@ def _process_file(path: Path, parent_node: FileSystemNode, stats: FileSystemStat
 
     file_size = path.stat().st_size
     if stats.total_size + file_size > MAX_TOTAL_SIZE_BYTES:
-        logger.info("Skipping file %s: would exceed total size limit", path)
+        logger.debug("Skipping file %s: would exceed total size limit", path)
         return
 
     stats.total_files += 1

src/gitingest/logging_config.py

@@ -1,16 +1,29 @@
 """Global logger configuration."""
 
 import logging
+import os
 from typing import Literal
 
 from pythonjsonlogger import jsonlogger
 
 
-def setup_json_logging(level: Literal = logging.INFO) -> None:
-    """Configure json logger for the whole gitingest module."""
-    logger = logging.getLogger(__name__)
+def setup_logging(level: Literal = logging.INFO) -> None:
+    """Configure logger for the whole gitingest module.
+
+    Selects formatter based on LOG_FORMAT env variable:
+    - 'json': JSON formatter (time/level/msg, then extras)
+    - any other value or unset: default formatter
+    """
+    logger = logging.getLogger()
     logger.setLevel(level)
     log_handler = logging.StreamHandler()
-    formatter = jsonlogger.JsonFormatter("%(asctime)s %(levelname)s %(name)s %(message)s")
+
+    log_format = os.getenv("LOG_FORMAT", "default").lower()
+    if log_format == "json":
+        formatter = jsonlogger.JsonFormatter(
+            "%(asctime)s %(levelname)s %(message)s %(name)s %(module)s %(funcName)s %(lineno)d",
+        )
+    else:
+        formatter = logging.Formatter("%(asctime)s %(levelname)s %(message)s")
     log_handler.setFormatter(formatter)
     logger.handlers = [log_handler]

src/gitingest/output_formatter.py

@@ -193,7 +193,7 @@ def _format_token_count(text: str) -> str | None:
         encoding = tiktoken.get_encoding("o200k_base")  # gpt-4o, gpt-4o-mini
         total_tokens = len(encoding.encode(text, disallowed_special=()))
     except (ValueError, UnicodeEncodeError):
-        logger.exception()
+        logger.exception("Failed to estimate token size.")
         return None
 
     for threshold, suffix in _TOKEN_THRESHOLDS:

src/gitingest/schemas/ingestion.py

@@ -2,6 +2,7 @@
 
 from __future__ import annotations
 
+import logging
 from pathlib import Path  # noqa: TC003 (typing-only-standard-library-import) needed for type checking (pydantic)
 from uuid import UUID  # noqa: TC003 (typing-only-standard-library-import) needed for type checking (pydantic)
 
@@ -10,6 +11,8 @@
 from gitingest.config import MAX_FILE_SIZE
 from gitingest.schemas.cloning import CloneConfig
 
+logger = logging.getLogger(__name__)
+
 
 class IngestionQuery(BaseModel):  # pylint: disable=too-many-instance-attributes
     """Pydantic model to store the parsed details of the repository or file path.
@@ -72,21 +75,18 @@ class IngestionQuery(BaseModel):  # pylint: disable=too-many-instance-attributes
     s3_url: str | None = None
 
     def extract_clone_config(self) -> CloneConfig:
-        """Extract the relevant fields for the CloneConfig object.
-
-        Returns
-        -------
-        CloneConfig
-            A CloneConfig object containing the relevant fields.
-
-        Raises
-        ------
-        ValueError
-            If the ``url`` parameter is not provided.
-
-        """
+        """Extract the relevant fields for the CloneConfig object."""
+        logger.debug(
+            "Extracting CloneConfig for url=%s, local_path=%s, branch=%s, tag=%s, commit=%s",
+            self.url,
+            self.local_path,
+            self.branch,
+            self.tag,
+            self.commit,
+        )
         if not self.url:
             msg = "The 'url' parameter is required."
+            logger.error(msg)
             raise ValueError(msg)
 
         return CloneConfig(

src/gitingest/utils/git_utils.py

@@ -99,10 +99,12 @@ async def ensure_git_installed() -> None:
         try:
             stdout, _ = await run_command("git", "config", "core.longpaths")
             if stdout.decode().strip().lower() != "true":
-                logger.warning("WARN: Git clone may fail on Windows due to long file paths:")
-                logger.warning("To avoid this issue, consider enabling long path support with:")
-                logger.warning("    git config --global core.longpaths true")
-                logger.warning("Note: This command may require administrator privileges.")
+                logger.warning(
+                    """Git clone may fail on Windows due to long file paths:
+To avoid this issue, consider enabling long path support with:
+    git config --global core.longpaths true
+Note: This command may require administrator privileges.""",
+                )
         except RuntimeError:
             # Ignore if checking 'core.longpaths' fails.
             pass

src/gitingest/utils/notebook.py

@@ -41,6 +41,7 @@ def process_notebook(file: Path, *, include_output: bool = True) -> str:
             notebook: dict[str, Any] = json.load(f)
     except json.JSONDecodeError as exc:
         msg = f"Invalid JSON in notebook: {file}"
+        logger.exception(msg)
         raise InvalidNotebookError(msg) from exc
 
     # Check if the notebook contains worksheets
@@ -125,24 +126,7 @@ def _process_cell(cell: dict[str, Any], *, include_output: bool) -> str | None:
 
 
 def _extract_output(output: dict[str, Any]) -> list[str]:
-    """Extract the output from a Jupyter notebook cell.
-
-    Parameters
-    ----------
-    output : dict[str, Any]
-        The output dictionary from a Jupyter notebook cell.
-
-    Returns
-    -------
-    list[str]
-        The output as a list of strings.
-
-    Raises
-    ------
-    ValueError
-        If an unknown output type is encountered.
-
-    """
+    """Extract the output from a Jupyter notebook cell."""
     output_type = output["output_type"]
 
     if output_type == "stream":
@@ -155,4 +139,5 @@ def _extract_output(output: dict[str, Any]) -> list[str]:
         return [f"Error: {output['ename']}: {output['evalue']}"]
 
     msg = f"Unknown output type: {output_type}"
+    logger.error(msg)
     raise ValueError(msg)

src/server/main.py

@@ -14,6 +14,7 @@
 from slowapi.errors import RateLimitExceeded
 from starlette.middleware.trustedhost import TrustedHostMiddleware
 
+from gitingest.logging_config import setup_logging
 from server.metrics_server import start_metrics_server
 from server.routers import dynamic, index, ingest
 from server.server_config import templates
@@ -22,6 +23,9 @@
 # Load environment variables from .env file
 load_dotenv()
 
+# Setup logging based on LOG_FORMAT env variable
+setup_logging()
+
 # Initialize Sentry SDK if enabled
 if os.getenv("GITINGEST_SENTRY_ENABLED") is not None:
     sentry_dsn = os.getenv("GITINGEST_SENTRY_DSN")