Respect the timeout and retry settings for converting docs

Should fix #573

Author: stchris

ingestors/settings.py

@@ -4,8 +4,8 @@
 
 TESTING = False
 
-CONVERT_TIMEOUT = env.to_int("INGESTORS_CONVERT_TIMEOUT", 7200)  # 2 hrs
-CONVERT_RETRIES = env.to_int("INGESTORS_CONVERT_RETRIES", 256)
+CONVERT_TIMEOUT = env.to_int("INGESTORS_CONVERT_TIMEOUT", 300)  # seconds
+CONVERT_RETRIES = env.to_int("INGESTORS_CONVERT_RETRIES", 3)
 
 # Enable (expensive!) Google Cloud API
 OCR_VISION_API = env.to_bool("INGESTORS_OCR_VISION_API", False)

ingestors/support/convert.py

@@ -9,11 +9,10 @@
 from ingestors.support.cache import CacheSupport
 from ingestors.support.temp import TempFileSupport
 from ingestors.exc import ProcessingException
+from ingestors import settings
 
 log = logging.getLogger(__name__)
 
-TIMEOUT = 3600  # seconds
-CONVERT_RETRIES = 5
 
 PDF_CACHE_ACCESSED = Counter(
     "ingestfile_pdf_cache_accessed",
@@ -45,7 +44,9 @@ def document_to_pdf(self, unique_tmpdir, file_path, entity):
             self.tags.set(key, content_hash)
         return pdf_file
 
-    def _document_to_pdf(self, unique_tmpdir, file_path, entity, timeout=TIMEOUT):
+    def _document_to_pdf(
+        self, unique_tmpdir, file_path, entity, timeout=settings.CONVERT_TIMEOUT
+    ):
         """Converts an office document to PDF."""
         file_name = entity_filename(entity)
         log.info("Converting [%s] to PDF", entity)
@@ -72,17 +73,15 @@ def _document_to_pdf(self, unique_tmpdir, file_path, entity, timeout=TIMEOUT):
             file_path,
         ]
         try:
-            for attempt in range(1, CONVERT_RETRIES):
+            for attempt in range(1, settings.CONVERT_RETRIES):
                 log.info(
-                    f"Starting LibreOffice: %s with timeout %s attempt #{attempt}/{CONVERT_RETRIES}",
-                    cmd,
-                    timeout,
+                    f"Starting LibreOffice: {cmd} with timeout {timeout} attempt #{attempt}/{settings.CONVERT_RETRIES}",
                 )
                 try:
                     subprocess.run(cmd, timeout=timeout, check=True)
                 except Exception as e:
                     log.info(
-                        f"Could not be converted to PDF (attempt {attempt}/{CONVERT_RETRIES}): {e}"
+                        f"Could not be converted to PDF (attempt {attempt}/{settings.CONVERT_RETRIES}): {e}"
                     )
                     continue
 
@@ -95,7 +94,7 @@ def _document_to_pdf(self, unique_tmpdir, file_path, entity, timeout=TIMEOUT):
                     log.info(f"Successfully converted {out_file}")
                     return out_file
             raise ProcessingException(
-                f"Could not be converted to PDF (attempt #{attempt}/{CONVERT_RETRIES})"
+                f"Could not be converted to PDF (attempt #{attempt}/{settings.CONVERT_RETRIES})"
             )
         except Exception as e:
             raise ProcessingException("Could not be converted to PDF") from e