fix: Implement retry logic for database connection and user fetching with exponential backoff (#623)

Author: Harsh-Microsoft

src/App/backend/services/sqldb_service.py

@@ -8,6 +8,8 @@
 
 from backend.common.config import config
 
+import time
+
 load_dotenv()
 
 driver = config.ODBC_DRIVER
@@ -33,33 +35,47 @@ def dict_cursor(cursor):
 
 
 def get_connection():
-    try:
-        credential = DefaultAzureCredential(managed_identity_client_id=mid_id)
+    max_retries = 5
+    retry_delay = 2
 
-        token_bytes = credential.get_token(
-            "https://database.windows.net/.default"
-        ).token.encode("utf-16-LE")
-        token_struct = struct.pack(
-            f"<I{len(token_bytes)}s", len(token_bytes), token_bytes
-        )
-        SQL_COPT_SS_ACCESS_TOKEN = (
-            1256  # This connection option is defined by Microsoft in msodbcsql.h
-        )
+    for attempt in range(max_retries):
+        try:
+            credential = DefaultAzureCredential(managed_identity_client_id=mid_id)
 
-        # Set up the connection
-        connection_string = f"DRIVER={driver};SERVER={server};DATABASE={database};"
-        conn = pyodbc.connect(
-            connection_string, attrs_before={SQL_COPT_SS_ACCESS_TOKEN: token_struct}
-        )
-        return conn
-    except pyodbc.Error as e:
-        logging.error(f"Failed with Default Credential: {str(e)}")
-        conn = pyodbc.connect(
-            f"DRIVER={driver};SERVER={server};DATABASE={database};UID={username};PWD={password}",
-            timeout=5,
-        )
-        logging.info("Connected using Username & Password")
-        return conn
+            token_bytes = credential.get_token(
+                "https://database.windows.net/.default"
+            ).token.encode("utf-16-LE")
+            token_struct = struct.pack(
+                f"<I{len(token_bytes)}s", len(token_bytes), token_bytes
+            )
+            SQL_COPT_SS_ACCESS_TOKEN = (
+                1256  # This connection option is defined by Microsoft in msodbcsql.h
+            )
+
+            # Set up the connection
+            connection_string = f"DRIVER={driver};SERVER={server};DATABASE={database};"
+            conn = pyodbc.connect(
+                connection_string, attrs_before={SQL_COPT_SS_ACCESS_TOKEN: token_struct}
+            )
+            return conn
+        except pyodbc.Error as e:
+            logging.error(f"Failed with Default Credential: {str(e)}")
+            try:
+                conn = pyodbc.connect(
+                    f"DRIVER={driver};SERVER={server};DATABASE={database};UID={username};PWD={password}",
+                    timeout=5,
+                )
+                logging.info("Connected using Username & Password")
+                return conn
+            except pyodbc.Error as e:
+                logging.error(f"Failed with Username & Password: {str(e)}")
+
+                if attempt < max_retries - 1:
+                    logging.info(f"Retrying in {retry_delay} seconds...")
+                    time.sleep(retry_delay)
+                    retry_delay *= 2  # Exponential backoff
+                else:
+                    raise e
 
 
 def get_client_name_from_db(client_id: str) -> str:

src/App/frontend/src/api/api.ts

@@ -42,20 +42,33 @@ export const getpbi = async (): Promise<string> => {
   return '';
 }
 
+const sleep = (ms: number) => new Promise(resolve => setTimeout(resolve, ms));
+
 export const getUsers = async (): Promise<User[]> => {
-  try {
-    const response = await fetch('/api/users');
-    if (!response.ok) {
-      throw new Error(`Failed to fetch users: ${response.statusText}`);
+  const maxRetries = 1;
+  for (let attempt = 0; attempt <= maxRetries; attempt++) {
+    try {
+      const response = await fetch('/api/users', {
+        signal: AbortSignal.timeout(60000)
+      });
+      if (!response.ok) {
+        throw new Error(`Failed to fetch users: ${response.statusText}`);
+      }
+      const data: User[] = await response.json();
+      console.log('Fetched users:', data);
+      return data;
+    } catch (error) {
+      if (attempt < maxRetries && 
+          error instanceof Error) {
+        console.warn(`Retrying fetch users... (retry ${attempt + 1}/${maxRetries})`);
+        await sleep(5000); // Simple 5 second delay
+      } else {
+        console.error('Error fetching users:', error);
+        return [];
+      }
     }
-    const data: User[] = await response.json();
-    console.log('Fetched users:', data);
-    return data;
-  } catch (error) {
-    console.error('Error fetching users:', error);
-    return [];
-    // throw error;
   }
+  return [];
 };
 
 // export const fetchChatHistoryInit = async (): Promise<Conversation[] | null> => {