From 63e97584642636a4e705b7d5aec277e0a61e7923 Mon Sep 17 00:00:00 2001 From: Napuh Date: Tue, 22 Jul 2025 20:34:15 +0200 Subject: [PATCH 1/4] fix: handle network errors gracefully in token count estimation --- src/gitingest/output_formatter.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/gitingest/output_formatter.py b/src/gitingest/output_formatter.py index 94bbee62..3bfe3401 100644 --- a/src/gitingest/output_formatter.py +++ b/src/gitingest/output_formatter.py @@ -2,8 +2,10 @@ from __future__ import annotations +import ssl from typing import TYPE_CHECKING +import requests.exceptions import tiktoken from gitingest.schemas import FileSystemNode, FileSystemNodeType @@ -189,6 +191,9 @@ def _format_token_count(text: str) -> str | None: except (ValueError, UnicodeEncodeError) as exc: print(exc) return None + except (requests.exceptions.RequestException, ssl.SSLError): + # If network errors, silently skip token count estimation instead of erroring out + return None for threshold, suffix in _TOKEN_THRESHOLDS: if total_tokens >= threshold: From 2882cc0b89da4916ad90b5854221f675c821f4ef Mon Sep 17 00:00:00 2001 From: Napuh Date: Sat, 26 Jul 2025 18:27:32 +0200 Subject: [PATCH 2/4] fix: add warning log when network error --- src/gitingest/output_formatter.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/gitingest/output_formatter.py b/src/gitingest/output_formatter.py index ed85a904..9bcd4fa9 100644 --- a/src/gitingest/output_formatter.py +++ b/src/gitingest/output_formatter.py @@ -194,8 +194,9 @@ def _format_token_count(text: str) -> str | None: except (ValueError, UnicodeEncodeError) as exc: print(exc) return None - except (requests.exceptions.RequestException, ssl.SSLError): - # If network errors, silently skip token count estimation instead of erroring out + except (requests.exceptions.RequestException, ssl.SSLError) as exc: + # If network errors, skip token count estimation instead of erroring out + print(f"Failed to download tiktoken model: {exc}") return None for threshold, suffix in _TOKEN_THRESHOLDS: From 2e7a2206ed321cdb5e7b4fa98f6a04d1b21aac09 Mon Sep 17 00:00:00 2001 From: ix-56h Date: Sat, 26 Jul 2025 20:19:38 +0200 Subject: [PATCH 3/4] use warning instead of print when logging tiktoken errors, fix changelog.md markdownlint errors --- CHANGELOG.md | 8 ++------ src/gitingest/output_formatter.py | 5 +++-- 2 files changed, 5 insertions(+), 8 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 009ff527..0efbaa8f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,11 +2,10 @@ ## [0.2.0](https://github.com/coderamp-labs/gitingest/compare/v0.1.5...v0.2.0) (2025-07-26) - ### ⚠ BREAKING CHANGES -* -* +* +* ### Features @@ -20,7 +19,6 @@ * Refactor backend to a rest api ([#346](https://github.com/coderamp-labs/gitingest/issues/346)) ([2b1f228](https://github.com/coderamp-labs/gitingest/commit/2b1f228ae1f6d1f7ee471794d258b13fcac25a96)) * **ui:** add inline PAT info tooltip inside token field ([#348](https://github.com/coderamp-labs/gitingest/issues/348)) ([2592303](https://github.com/coderamp-labs/gitingest/commit/25923037ea6cd2f8ef33a6cf1f0406c2b4f0c9b6)) - ### Bug Fixes * enable metrics if env var is defined instead of being "True" ([#407](https://github.com/coderamp-labs/gitingest/issues/407)) ([fa2e192](https://github.com/coderamp-labs/gitingest/commit/fa2e192c05864c8db90bda877e9efb9b03caf098)) @@ -33,14 +31,12 @@ * **ui:** update layout in PAT section to avoid overlaps & overflows ([#331](https://github.com/coderamp-labs/gitingest/issues/331)) ([b39ef54](https://github.com/coderamp-labs/gitingest/commit/b39ef5416c1f8a7993a8249161d2a898b7387595)) * **windows:** warn if Git long path support is disabled, do not fail ([b8e375f](https://github.com/coderamp-labs/gitingest/commit/b8e375f71cae7d980cf431396c4414a6dbd0588c)) - ### Documentation * add GitHub Issue Form for bug reports ([#403](https://github.com/coderamp-labs/gitingest/issues/403)) ([4546449](https://github.com/coderamp-labs/gitingest/commit/4546449bbc1e4a7ad0950c4b831b8855a98628fd)) * add GitHub Issue Form for feature requests ([#404](https://github.com/coderamp-labs/gitingest/issues/404)) ([9b1fc58](https://github.com/coderamp-labs/gitingest/commit/9b1fc58900ae18a3416fe3cf9b5e301a65a8e9fd)) * Fix CLI help text accuracy ([#332](https://github.com/coderamp-labs/gitingest/issues/332)) ([fdcbc53](https://github.com/coderamp-labs/gitingest/commit/fdcbc53cadde6a5dc3c3626120df1935b63693b2)) - ### Code Refactoring * centralize PAT validation, streamline repo checks & misc cleanup ([#349](https://github.com/coderamp-labs/gitingest/issues/349)) ([cea0edd](https://github.com/coderamp-labs/gitingest/commit/cea0eddce8c6846bc6271cb3a8d15320e103214c)) diff --git a/src/gitingest/output_formatter.py b/src/gitingest/output_formatter.py index 9bcd4fa9..2a9957b2 100644 --- a/src/gitingest/output_formatter.py +++ b/src/gitingest/output_formatter.py @@ -3,6 +3,7 @@ from __future__ import annotations import ssl +import warnings from typing import TYPE_CHECKING import requests.exceptions @@ -192,11 +193,11 @@ def _format_token_count(text: str) -> str | None: encoding = tiktoken.get_encoding("o200k_base") # gpt-4o, gpt-4o-mini total_tokens = len(encoding.encode(text, disallowed_special=())) except (ValueError, UnicodeEncodeError) as exc: - print(exc) + warnings.warn(f"Failed to estimate token size: {exc}", RuntimeWarning, stacklevel=3) return None except (requests.exceptions.RequestException, ssl.SSLError) as exc: # If network errors, skip token count estimation instead of erroring out - print(f"Failed to download tiktoken model: {exc}") + warnings.warn(f"Failed to download tiktoken model: {exc}", RuntimeWarning, stacklevel=3) return None for threshold, suffix in _TOKEN_THRESHOLDS: From c95c9f7b9f6e5e62235dc18324f5d18fb5c336d6 Mon Sep 17 00:00:00 2001 From: ix-56h Date: Sat, 26 Jul 2025 22:08:21 +0200 Subject: [PATCH 4/4] revert changelog.md changes, add CHANGELOG.md to ignored files --- .pre-commit-config.yaml | 3 ++- CHANGELOG.md | 8 ++++++-- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 529d352a..25732f5d 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -26,6 +26,7 @@ repos: - id: trailing-whitespace description: 'Trim trailing whitespace.' + exclude: CHANGELOG.md - id: check-docstring-first description: 'Check a common error of defining a docstring after code.' @@ -89,7 +90,7 @@ repos: hooks: - id: markdownlint description: 'Lint markdown files.' - args: ['--disable=line-length'] + args: ['--disable=line-length', '--ignore=CHANGELOG.md'] - repo: https://github.com/astral-sh/ruff-pre-commit rev: v0.12.2 diff --git a/CHANGELOG.md b/CHANGELOG.md index 0efbaa8f..009ff527 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,10 +2,11 @@ ## [0.2.0](https://github.com/coderamp-labs/gitingest/compare/v0.1.5...v0.2.0) (2025-07-26) + ### ⚠ BREAKING CHANGES -* -* +* +* ### Features @@ -19,6 +20,7 @@ * Refactor backend to a rest api ([#346](https://github.com/coderamp-labs/gitingest/issues/346)) ([2b1f228](https://github.com/coderamp-labs/gitingest/commit/2b1f228ae1f6d1f7ee471794d258b13fcac25a96)) * **ui:** add inline PAT info tooltip inside token field ([#348](https://github.com/coderamp-labs/gitingest/issues/348)) ([2592303](https://github.com/coderamp-labs/gitingest/commit/25923037ea6cd2f8ef33a6cf1f0406c2b4f0c9b6)) + ### Bug Fixes * enable metrics if env var is defined instead of being "True" ([#407](https://github.com/coderamp-labs/gitingest/issues/407)) ([fa2e192](https://github.com/coderamp-labs/gitingest/commit/fa2e192c05864c8db90bda877e9efb9b03caf098)) @@ -31,12 +33,14 @@ * **ui:** update layout in PAT section to avoid overlaps & overflows ([#331](https://github.com/coderamp-labs/gitingest/issues/331)) ([b39ef54](https://github.com/coderamp-labs/gitingest/commit/b39ef5416c1f8a7993a8249161d2a898b7387595)) * **windows:** warn if Git long path support is disabled, do not fail ([b8e375f](https://github.com/coderamp-labs/gitingest/commit/b8e375f71cae7d980cf431396c4414a6dbd0588c)) + ### Documentation * add GitHub Issue Form for bug reports ([#403](https://github.com/coderamp-labs/gitingest/issues/403)) ([4546449](https://github.com/coderamp-labs/gitingest/commit/4546449bbc1e4a7ad0950c4b831b8855a98628fd)) * add GitHub Issue Form for feature requests ([#404](https://github.com/coderamp-labs/gitingest/issues/404)) ([9b1fc58](https://github.com/coderamp-labs/gitingest/commit/9b1fc58900ae18a3416fe3cf9b5e301a65a8e9fd)) * Fix CLI help text accuracy ([#332](https://github.com/coderamp-labs/gitingest/issues/332)) ([fdcbc53](https://github.com/coderamp-labs/gitingest/commit/fdcbc53cadde6a5dc3c3626120df1935b63693b2)) + ### Code Refactoring * centralize PAT validation, streamline repo checks & misc cleanup ([#349](https://github.com/coderamp-labs/gitingest/issues/349)) ([cea0edd](https://github.com/coderamp-labs/gitingest/commit/cea0eddce8c6846bc6271cb3a8d15320e103214c))