Stop waiting if InitializationFailed state is reached (#495)

## Problem

After creating an index, the SDK polls until the index is ready to be
used. But if the index state transitions to `InitializationFailed`, the
client will hang. Sometimes this causes our test suite to get stuck and
run for hours until somebody manually intervenes or a CI timeout is
reached.

## Solution

Raise an exception and stop waiting if an index reaches the
`InitializationFailed` state.

## Type of Change

- [x] Bug fix (non-breaking change which fixes an issue)

Author: jhamon

pinecone/db_control/resources/asyncio/index.py

@@ -110,42 +110,28 @@ async def create_from_backup(
         return await self.__poll_describe_index_until_ready(name, timeout)
 
     async def __poll_describe_index_until_ready(self, name: str, timeout: Optional[int] = None):
-        description = None
-
-        async def is_ready() -> bool:
-            nonlocal description
-            description = await self.describe(name=name)
-            return description.status.ready
-
         total_wait_time = 0
-        if timeout is None:
-            # Wait indefinitely
-            while not await is_ready():
-                logger.debug(
-                    f"Waiting for index {name} to be ready. Total wait time {total_wait_time} seconds."
+        while True:
+            description = await self.describe(name=name)
+            if description.status.state == "InitializationFailed":
+                raise Exception(f"Index {name} failed to initialize.")
+            if description.status.ready:
+                return description
+
+            if timeout is not None and total_wait_time >= timeout:
+                logger.error(
+                    f"Index {name} is not ready after {total_wait_time} seconds. Timeout reached."
                 )
-                total_wait_time += 5
-                await asyncio.sleep(5)
+                link = docslinks["API_DESCRIBE_INDEX"](API_VERSION)
+                timeout_msg = f"Index {name} is not ready after {total_wait_time} seconds. Please call describe_index() to confirm index status. See docs at {link}"
+                raise TimeoutError(timeout_msg)
 
-        else:
-            # Wait for a maximum of timeout seconds
-            while not await is_ready():
-                if timeout < 0:
-                    logger.error(f"Index {name} is not ready. Timeout reached.")
-                    link = docslinks["API_DESCRIBE_INDEX"](API_VERSION)
-                    timeout_msg = (
-                        f"Please call describe_index() to confirm index status. See docs at {link}"
-                    )
-                    raise TimeoutError(timeout_msg)
-
-                logger.debug(
-                    f"Waiting for index {name} to be ready. Total wait time: {total_wait_time}"
-                )
-                total_wait_time += 5
-                await asyncio.sleep(5)
-                timeout -= 5
+            logger.debug(
+                f"Waiting for index {name} to be ready. Total wait time {total_wait_time} seconds."
+            )
 
-        return description
+            total_wait_time += 5
+            await asyncio.sleep(5)
 
     @require_kwargs
     async def delete(self, *, name: str, timeout: Optional[int] = None):

pinecone/db_control/resources/sync/index.py

@@ -145,42 +145,30 @@ def create_from_backup(
         return self.__poll_describe_index_until_ready(name, timeout)
 
     def __poll_describe_index_until_ready(self, name: str, timeout: Optional[int] = None):
-        description = None
-
-        def is_ready() -> bool:
-            nonlocal description
-            description = self.describe(name=name)
-            return description.status.ready
-
         total_wait_time = 0
-        if timeout is None:
-            # Wait indefinitely
-            while not is_ready():
-                logger.debug(
-                    f"Waiting for index {name} to be ready. Total wait time {total_wait_time} seconds."
+        while True:
+            description = self.describe(name=name)
+            if description.status.state == "InitializationFailed":
+                raise Exception(
+                    f"Index {name} failed to initialize. The index status is {description.status.state}."
                 )
-                total_wait_time += 5
-                time.sleep(5)
+            if description.status.ready:
+                return description
 
-        else:
-            # Wait for a maximum of timeout seconds
-            while not is_ready():
-                if timeout < 0:
-                    logger.error(f"Index {name} is not ready. Timeout reached.")
-                    link = docslinks["API_DESCRIBE_INDEX"](API_VERSION)
-                    timeout_msg = (
-                        f"Please call describe_index() to confirm index status. See docs at {link}"
-                    )
-                    raise TimeoutError(timeout_msg)
-
-                logger.debug(
-                    f"Waiting for index {name} to be ready. Total wait time: {total_wait_time}"
+            if timeout is not None and total_wait_time >= timeout:
+                logger.error(
+                    f"Index {name} is not ready after {total_wait_time} seconds. Timeout reached."
                 )
-                total_wait_time += 5
-                time.sleep(5)
-                timeout -= 5
+                link = docslinks["API_DESCRIBE_INDEX"](API_VERSION)
+                timeout_msg = f"Index {name} is not ready after {total_wait_time} seconds. Please call describe_index() to confirm index status. See docs at {link}"
+                raise TimeoutError(timeout_msg)
+
+            logger.debug(
+                f"Waiting for index {name} to be ready. Total wait time {total_wait_time} seconds."
+            )
 
-        return description
+            total_wait_time += 5
+            time.sleep(5)
 
     @require_kwargs
     def delete(self, *, name: str, timeout: Optional[int] = None) -> None: