Merge pull request #867 from cmu-delphi/krivard/delete_smaller_batches

Split large deletion batches into smaller chunks

Author: krivard

src/acquisition/covidcast/database.py

@@ -333,7 +333,11 @@ def delete_batch(self, cc_deletions):
       if isinstance(cc_deletions, str):
         self._cursor.execute(load_tmp_table_infile_sql)
       elif isinstance(cc_deletions, list):
-        self._cursor.executemany(load_tmp_table_insert_sql, cc_deletions)
+        def split_list(lst, n):
+          for i in range(0, len(lst), n):
+            yield lst[i:(i+n)]
+        for deletions_batch in split_list(cc_deletions, 100000):
+          self._cursor.executemany(load_tmp_table_insert_sql, deletions_batch)
       else:
         raise Exception(f"Bad deletions argument: need a filename or a list of tuples; got a {type(cc_deletions)}")
       self._cursor.execute(add_id_sql)