refactor: consistent cloning & pattern-handling (#388)

Co-authored-by: ix-56h <n.guintini@protonmail.com>

Author: filipchristiansen

.github/workflows/ci.yml

@@ -51,7 +51,7 @@ jobs:
     - name: Install dependencies
       run: |
         python -m pip install --upgrade pip
-        python -m pip install ".[dev]"
+        python -m pip install ".[dev,server]"
 
     - name: Run tests
       if: ${{ matrix.coverage != true }}

.github/workflows/publish_to_pypi.yml

@@ -2,7 +2,7 @@ name: Publish to PyPI
 
 on:
   release:
-    types: [created] # Run when you click “Publish release”
+    types: [created] # Run when you click "Publish release"
   workflow_dispatch: # ... or run it manually from the Actions tab
 
 permissions:
@@ -38,7 +38,7 @@ jobs:
           name: dist
           path: dist/
 
-# Publish to PyPI (only if “dist/” succeeded)
+# Publish to PyPI (only if "dist/" succeeded)
   pypi-publish:
     needs: release-build
     runs-on: ubuntu-latest

.pre-commit-config.yaml

@@ -122,9 +122,12 @@ repos:
             pytest-asyncio,
             pytest-mock,
             python-dotenv,
+            'sentry-sdk[fastapi]',
             slowapi,
             starlette>=0.40.0,
+            strenum; python_version < '3.11',
             tiktoken>=0.7.0,
+            typing_extensions>= 4.0.0; python_version < '3.10',
             uvicorn>=0.11.7,
           ]
 
@@ -144,9 +147,12 @@ repos:
             pytest-asyncio,
             pytest-mock,
             python-dotenv,
+            'sentry-sdk[fastapi]',
             slowapi,
             starlette>=0.40.0,
+            strenum; python_version < '3.11',
             tiktoken>=0.7.0,
+            typing_extensions>= 4.0.0; python_version < '3.10',
             uvicorn>=0.11.7,
           ]
 

CONTRIBUTING.md

@@ -32,7 +32,7 @@ If you ever get stuck, reach out on [Discord](https://discord.com/invite/zerRaGK
    ```bash
    python -m venv .venv
    source .venv/bin/activate
-   pip install -e ".[dev]"
+   pip install -e ".[dev,server]"
    pre-commit install
    ```
 

README.md

@@ -66,6 +66,14 @@ You can install it using `pip`:
 pip install gitingest
 ```
 
+or
+
+```bash
+pip install gitingest[server]
+```
+
+to include server dependencies for self-hosting.
+
 However, it might be a good idea to use `pipx` to install it.
 You can install `pipx` using your preferred package manager.
 

pyproject.toml

@@ -6,17 +6,14 @@ readme = {file = "README.md", content-type = "text/markdown" }
 requires-python = ">= 3.8"
 dependencies = [
     "click>=8.0.0",
-    "fastapi[standard]>=0.109.1",  # Minimum safe release (https://osv.dev/vulnerability/PYSEC-2024-38)
     "httpx",
     "pathspec>=0.12.1",
     "pydantic",
     "python-dotenv",
-    "slowapi",
     "starlette>=0.40.0",  # Minimum safe release (https://osv.dev/vulnerability/GHSA-f96h-pmfr-66vw)
+    "strenum; python_version < '3.11'",
     "tiktoken>=0.7.0",  # Support for o200k_base encoding
     "typing_extensions>= 4.0.0; python_version < '3.10'",
-    "uvicorn>=0.11.7",  # Minimum safe release (https://osv.dev/vulnerability/PYSEC-2020-150)
-    "prometheus-client",
 ]
 
 license = {file = "LICENSE"}
@@ -46,6 +43,14 @@ dev = [
     "pytest-mock",
 ]
 
+server = [
+    "fastapi[standard]>=0.109.1",  # Minimum safe release (https://osv.dev/vulnerability/PYSEC-2024-38)
+    "prometheus-client",
+    "sentry-sdk[fastapi]",
+    "slowapi",
+    "uvicorn>=0.11.7",  # Minimum safe release (https://osv.dev/vulnerability/PYSEC-2020-150)
+]
+
 [project.scripts]
 gitingest = "gitingest.__main__:main"
 

src/gitingest/__init__.py

@@ -1,8 +1,5 @@
 """Gitingest: A package for ingesting data from Git repositories."""
 
-from gitingest.clone import clone_repo
 from gitingest.entrypoint import ingest, ingest_async
-from gitingest.ingestion import ingest_query
-from gitingest.query_parser import parse_query
 
-__all__ = ["clone_repo", "ingest", "ingest_async", "ingest_query", "parse_query"]
+__all__ = ["ingest", "ingest_async"]

src/gitingest/clone.py

@@ -8,13 +8,15 @@
 from gitingest.config import DEFAULT_TIMEOUT
 from gitingest.utils.git_utils import (
     check_repo_exists,
+    checkout_partial_clone,
     create_git_auth_header,
     create_git_command,
     ensure_git_installed,
     is_github_host,
+    resolve_commit,
     run_command,
 )
-from gitingest.utils.os_utils import ensure_directory
+from gitingest.utils.os_utils import ensure_directory_exists_or_create
 from gitingest.utils.timeout_wrapper import async_timeout
 
 if TYPE_CHECKING:
@@ -45,71 +47,42 @@ async def clone_repo(config: CloneConfig, *, token: str | None = None) -> None:
     # Extract and validate query parameters
     url: str = config.url
     local_path: str = config.local_path
-    commit: str | None = config.commit
-    branch: str | None = config.branch
-    tag: str | None = config.tag
     partial_clone: bool = config.subpath != "/"
 
-    # Create parent directory if it doesn't exist
-    await ensure_directory(Path(local_path).parent)
+    await ensure_git_installed()
+    await ensure_directory_exists_or_create(Path(local_path).parent)
 
-    # Check if the repository exists
     if not await check_repo_exists(url, token=token):
         msg = "Repository not found. Make sure it is public or that you have provided a valid token."
         raise ValueError(msg)
 
+    commit = await resolve_commit(config, token=token)
+
     clone_cmd = ["git"]
     if token and is_github_host(url):
         clone_cmd += ["-c", create_git_auth_header(token, url=url)]
 
-    clone_cmd += ["clone", "--single-branch"]
-
-    if config.include_submodules:
-        clone_cmd += ["--recurse-submodules"]
-
+    clone_cmd += ["clone", "--single-branch", "--no-checkout", "--depth=1"]
     if partial_clone:
         clone_cmd += ["--filter=blob:none", "--sparse"]
 
-    # Shallow clone unless a specific commit is requested
-    if not commit:
-        clone_cmd += ["--depth=1"]
-
-        # Prefer tag over branch when both are provided
-        if tag:
-            clone_cmd += ["--branch", tag]
-        elif branch and branch.lower() not in ("main", "master"):
-            clone_cmd += ["--branch", branch]
-
     clone_cmd += [url, local_path]
 
     # Clone the repository
-    await ensure_git_installed()
     await run_command(*clone_cmd)
 
     # Checkout the subpath if it is a partial clone
     if partial_clone:
-        await _checkout_partial_clone(config, token)
+        await checkout_partial_clone(config, token=token)
 
-    # Checkout the commit if it is provided
-    if commit:
-        checkout_cmd = create_git_command(["git"], local_path, url, token)
-        await run_command(*checkout_cmd, "checkout", commit)
+    git = create_git_command(["git"], local_path, url, token)
 
+    # Ensure the commit is locally available
+    await run_command(*git, "fetch", "--depth=1", "origin", commit)
 
-async def _checkout_partial_clone(config: CloneConfig, token: str | None) -> None:
-    """Configure sparse-checkout for a partially cloned repository.
+    # Write the work-tree at that commit
+    await run_command(*git, "checkout", commit)
 
-    Parameters
-    ----------
-    config : CloneConfig
-        The configuration for cloning the repository, including subpath and blob flag.
-    token : str | None
-        GitHub personal access token (PAT) for accessing private repositories.
-
-    """
-    subpath = config.subpath.lstrip("/")
-    if config.blob:
-        # Remove the file name from the subpath when ingesting from a file url (e.g. blob/branch/path/file.txt)
-        subpath = str(Path(subpath).parent.as_posix())
-    checkout_cmd = create_git_command(["git"], config.local_path, config.url, token)
-    await run_command(*checkout_cmd, "sparse-checkout", "set", subpath)
+    # Update submodules
+    if config.include_submodules:
+        await run_command(*git, "submodule", "update", "--init", "--recursive", "--depth=1")

src/gitingest/entrypoint.py

@@ -3,19 +3,30 @@
 from __future__ import annotations
 
 import asyncio
+import errno
 import shutil
+import stat
 import sys
 import warnings
 from contextlib import asynccontextmanager
 from pathlib import Path
-from typing import AsyncGenerator
+from typing import TYPE_CHECKING, AsyncGenerator, Callable
+from urllib.parse import urlparse
 
 from gitingest.clone import clone_repo
 from gitingest.config import MAX_FILE_SIZE
 from gitingest.ingestion import ingest_query
-from gitingest.query_parser import IngestionQuery, parse_query
+from gitingest.query_parser import parse_local_dir_path, parse_remote_repo
 from gitingest.utils.auth import resolve_token
+from gitingest.utils.compat_func import removesuffix
 from gitingest.utils.ignore_patterns import load_ignore_patterns
+from gitingest.utils.pattern_utils import process_patterns
+from gitingest.utils.query_parser_utils import KNOWN_GIT_HOSTS
+
+if TYPE_CHECKING:
+    from types import TracebackType
+
+    from gitingest.schemas import IngestionQuery
 
 
 async def ingest_async(
@@ -74,13 +85,23 @@ async def ingest_async(
     """
     token = resolve_token(token)
 
-    query: IngestionQuery = await parse_query(
-        source=source,
-        max_file_size=max_file_size,
-        from_web=False,
+    source = removesuffix(source.strip(), ".git")
+
+    # Determine the parsing method based on the source type
+    if urlparse(source).scheme in ("https", "http") or any(h in source for h in KNOWN_GIT_HOSTS):
+        # We either have a full URL or a domain-less slug
+        query = await parse_remote_repo(source, token=token)
+        query.include_submodules = include_submodules
+        _override_branch_and_tag(query, branch=branch, tag=tag)
+
+    else:
+        # Local path scenario
+        query = parse_local_dir_path(source)
+
+    query.max_file_size = max_file_size
+    query.ignore_patterns, query.include_patterns = process_patterns(
+        exclude_patterns=exclude_patterns,
         include_patterns=include_patterns,
-        ignore_patterns=exclude_patterns,
-        token=token,
     )
 
     if query.url:
@@ -235,17 +256,49 @@ async def _clone_repo_if_remote(query: IngestionQuery, *, token: str | None) ->
         GitHub personal access token (PAT) for accessing private repositories.
 
     """
+    kwargs = {}
+    if sys.version_info >= (3, 12):
+        kwargs["onexc"] = _handle_remove_readonly
+    else:
+        kwargs["onerror"] = _handle_remove_readonly
+
     if query.url:
         clone_config = query.extract_clone_config()
         await clone_repo(clone_config, token=token)
         try:
             yield
         finally:
-            shutil.rmtree(query.local_path.parent)
+            shutil.rmtree(query.local_path.parent, **kwargs)
     else:
         yield
 
 
+def _handle_remove_readonly(
+    func: Callable,
+    path: str,
+    exc_info: BaseException | tuple[type[BaseException], BaseException, TracebackType],
+) -> None:
+    """Handle permission errors raised by ``shutil.rmtree()``.
+
+    * Makes the target writable (removes the read-only attribute).
+    * Retries the original operation (``func``) once.
+
+    """
+    # 'onerror' passes a (type, value, tb) tuple; 'onexc' passes the exception
+    if isinstance(exc_info, tuple):  # 'onerror' (Python <3.12)
+        exc: BaseException = exc_info[1]
+    else:  # 'onexc' (Python 3.12+)
+        exc = exc_info
+
+    # Handle only'Permission denied' and 'Operation not permitted'
+    if not isinstance(exc, OSError) or exc.errno not in {errno.EACCES, errno.EPERM}:
+        raise exc
+
+    # Make the target writable
+    Path(path).chmod(stat.S_IWRITE)
+    func(path)
+
+
 async def _write_output(tree: str, content: str, target: str | None) -> None:
     """Write combined output to ``target`` (``"-"`` ⇒ stdout).
 

src/gitingest/ingestion.py

@@ -11,7 +11,7 @@
 from gitingest.utils.ingestion_utils import _should_exclude, _should_include
 
 if TYPE_CHECKING:
-    from gitingest.query_parser import IngestionQuery
+    from gitingest.schemas import IngestionQuery
 
 
 def ingest_query(query: IngestionQuery) -> tuple[str, str, str]: