⚡️ Make 1000 rows chunks for tables

Author: simonwoerpel

ingestors/support/table.py

@@ -18,6 +18,7 @@ class TableSupport(EncodingSupport, TempFileSupport):
     def emit_row_dicts(self, table, rows, headers=None):
         csv_path = self.make_work_file(table.id)
         row_count = 0
+        fragment_rows = []
         with open(csv_path, "w", encoding=self.DEFAULT_ENCODING) as fp:
             csv_writer = csv.writer(fp, dialect="unix")
             for row in rows:
@@ -28,11 +29,15 @@ def emit_row_dicts(self, table, rows, headers=None):
                 if length == 0:
                     continue
                 csv_writer.writerow(values)
-                self.manager.emit_text_fragment(table, values, row_count)
+                fragment_rows.append(",".join([v or "" for v in values]))
                 row_count += 1
                 if row_count > 0 and row_count % 1000 == 0:
                     log.info("Table emit [%s]: %s...", table, row_count)
+                    fragment_rows = []
+                    self.manager.emit_text_fragment(table, fragment_rows, row_count)
         if row_count > 0:
+            if len(fragment_rows):
+                self.manager.emit_text_fragment(table, fragment_rows, row_count)
             csv_hash = self.manager.store(csv_path, mime_type=CSV)
             table.set("csvHash", csv_hash)
         table.set("rowCount", row_count + 1)

tests/test_dbf.py

@@ -1,5 +1,5 @@
 # -*- coding: utf-8 -*-
-from .support import TestCase
+from tests.support import TestCase
 
 
 class DBFIngestorTest(TestCase):
@@ -13,5 +13,5 @@ def test_simple_dbf(self):
         self.assertEqual(entity.schema.name, "Table")
         self.assertTrue(entity.has("csvHash"))
         self.assertEqual(int(entity.first("rowCount")), 9)
-        self.assertIn("Azad Kashmir", table.get("indexText"))
-        self.assertIn("Pakistan", table.get("indexText"))
+        self.assertIn("Azad Kashmir", "".join(table.get("indexText")))
+        self.assertIn("Pakistan", "".join(table.get("indexText")))

tests/test_tabular.py

@@ -1,6 +1,6 @@
 # -*- coding: utf-8 -*-
-from .support import TestCase
 from ingestors.exc import ENCRYPTED_MSG
+from tests.support import TestCase
 
 
 class TabularIngestorTest(TestCase):
@@ -16,7 +16,7 @@ def test_simple_xlsx(self):
         table = [t for t in tables if "1" in t.first("title")][0]
         self.assertTrue(table.has("csvHash"))
         self.assertEqual(int(table.first("rowCount")), 3)
-        self.assertIn("Mihai Viteazul", table.get("indexText"))
+        self.assertIn("Mihai Viteazul", "".join(table.get("indexText")))
 
     def test_unicode_xls(self):
         fixture_path, entity = self.fixture("rom.xls")