option to auto-map foreign keys (see #14)

sheppard · sheppard · commit 9a9422683f3d · 2020-02-19T16:04:24.000+09:00
diff --git a/data_wizard/__init__.py b/data_wizard/__init__.py
@@ -1,5 +1,6 @@
 from django.utils.module_loading import autodiscover_modules
 from .registry import registry
+from . import idmap
 
 
 __all__ = (
@@ -9,6 +10,7 @@
     "register",
     "set_loader",
     "default_app_config",
+    "idmap",
 )
 
 
diff --git a/data_wizard/idmap.py b/data_wizard/idmap.py
@@ -0,0 +1,18 @@
+from rest_framework.serializers import ValidationError
+
+
+def never(value, field):
+    return None
+
+
+def existing(value, field):
+    try:
+        field.to_internal_value(value)
+    except ValidationError:
+        return None
+    else:
+        return value
+
+
+def always(value, field):
+    return value
diff --git a/data_wizard/models.py b/data_wizard/models.py
@@ -4,6 +4,7 @@
 from django.conf import settings
 from . import registry
 from .compat import reverse
+from .settings import import_setting
 
 
 class Run(models.Model):
@@ -86,6 +87,12 @@ def get_serializer_options(self):
         else:
             raise Exception("No serializer specified!")
 
+    def get_idmap(self):
+        idmap = self.get_serializer_options().get('idmap')
+        if not idmap:
+            idmap = import_setting('IDMAP')
+        return idmap
+
     def already_parsed(self):
         return self.range_set.count()
 
diff --git a/data_wizard/settings.py b/data_wizard/settings.py
@@ -5,6 +5,7 @@
 DEFAULTS = {
     'BACKEND': 'data_wizard.backends.threading',
     'LOADER': 'data_wizard.loaders.FileLoader',
+    'IDMAP': 'data_wizard.idmap.never',   # FIXME: Change to 'existing' in 2.0
     'AUTHENTICATION': 'rest_framework.authentication.SessionAuthentication',
     'PERMISSION': 'rest_framework.permissions.IsAdminUser',
 }
diff --git a/data_wizard/tasks.py b/data_wizard/tasks.py
@@ -508,13 +508,15 @@ def read_row_identifiers(run, user=None):
 def parse_row_identifiers(run):
     run.add_event('parse_row_identifiers')
 
+    idmap = run.get_idmap()
     lookup_cols = get_lookup_columns(run)
     lookup_fields = OrderedDict()
     for col in lookup_cols:
         field_name = col['field_name']
         lookup_fields.setdefault(field_name, {
             'cols': [],
             'ids': OrderedDict(),
+            'serializer_field': col['serializer_field'],
             'start_col': 1e10,
             'end_col': -1,
         })
@@ -568,10 +570,13 @@ def parse_row_identifiers(run):
             ).first()
 
             if not ident:
+                value = idmap(name, info['serializer_field'])
                 ident = Identifier.objects.create(
                     serializer=run.serializer,
                     field=field_name,
                     name=name,
+                    value=value,
+                    resolved=bool(value),
                 )
 
             run.range_set.create(
diff --git a/tests/data_app/wizard.py b/tests/data_app/wizard.py
@@ -17,6 +17,27 @@ class Meta:
         }
 
 
+# FKModelSerializer = Data Wizard default
+
+
+class FKMapExistingSerializer(serializers.ModelSerializer):
+    class Meta:
+        model = FKModel
+        fields = "__all__"
+        data_wizard = {
+             'idmap': data_wizard.idmap.existing
+        }
+
+
+class FKMapAlwaysSerializer(serializers.ModelSerializer):
+    class Meta:
+        model = FKModel
+        fields = "__all__"
+        data_wizard = {
+             'idmap': data_wizard.idmap.always
+        }
+
+
 class SlugSerializer(serializers.ModelSerializer):
     type = serializers.SlugRelatedField(
         queryset=Type.objects.all(),
@@ -27,6 +48,12 @@ class Meta:
         model = FKModel
         fields = "__all__"
 
+class SlugMapExistingSerializer(SlugSerializer):
+    class Meta(SlugSerializer.Meta):
+        data_wizard = {
+             'idmap': data_wizard.idmap.existing
+        }
+
 
 class NestedFKSerializer(serializers.ModelSerializer):
     class Meta:
@@ -67,7 +94,12 @@ class Meta:
 data_wizard.register(SimpleModel)
 data_wizard.register('Simple Model - Incomplete', IncompleteSerializer)
 data_wizard.register(FKModel)
+data_wizard.register('FK Model - Use existing FKs', FKMapExistingSerializer)
+data_wizard.register('FK Model - Use FKs always', FKMapAlwaysSerializer)
 data_wizard.register('FK Model By Name', SlugSerializer)
+data_wizard.register(
+    'FK Model By Name - Use existing', SlugMapExistingSerializer,
+)
 data_wizard.register('New Type + FK Model', NestedSerializer)
 data_wizard.register(Address)
 data_wizard.register('Address with Zip Code', AddressSerializer)
diff --git a/tests/test_foreignkey.py b/tests/test_foreignkey.py
@@ -30,21 +30,21 @@ def check_data(self, run, expect_last_record=None, extra_ranges=[]):
 
 class ForeignKeyTestCase(BaseFKTestCase):
     serializer_name = 'data_wizard.registry.FKModelSerializer'
+    row4_error = '{"type": ["Invalid pk \\"100\\" - object does not exist."]}'
 
     def test_manual(self):
         run = self.upload_file('fkid.csv')
-        row4_error = ('{"type": ["Invalid pk \\"100\\"'
-                      ' - object does not exist."]}')
 
         # Start data import process, wait for completion
         self.start_import(run, [{
             'row': 5,
-            'reason': row4_error,
+            'reason': self.row4_error,
         }])
 
         # Verify results
         self.check_data(
-            run, expect_last_record="Failed at row 4: %s" % row4_error
+            run,
+            expect_last_record="Failed at row 4: %s" % self.row4_error,
         )
         self.assert_log(run, [
             'created',
@@ -53,8 +53,27 @@ def test_manual(self):
             'import_complete',
         ])
 
-    def test_auto(self):
-        # Should abort due to unknown type id
+    def test_auto_idmap_never(self):
+        # Should abort due to previously unmapped type ids
+        run = self.upload_file('fkid.csv')
+        self.auto_import(run, expect_input_required=True)
+        self.assert_log(run, [
+            'created',
+            'auto_import',
+            'parse_columns',
+            'parse_row_identifiers',
+        ])
+        self.assert_ranges(run, [
+            "Data Column 'type -> type' at Rows 1-4, Column 0",
+            "Data Column 'notes -> notes' at Rows 1-4, Column 1",
+            "Cell value 'Unresolved: 1' at Rows 1-2, Column 0",
+            "Cell value 'Unresolved: 2' at Row 3, Column 0",
+            "Cell value 'Unresolved: 100' at Row 4, Column 0",
+        ])
+
+    def test_auto_idmap_existing(self):
+        # Should auto-map existing ids 1 & 2, but abort due to unknown id 100
+        self.serializer_name = 'tests.data_app.wizard.FKMapExistingSerializer'
         run = self.upload_file('fkid.csv')
         self.auto_import(run, expect_input_required=True)
         self.assert_log(run, [
@@ -63,6 +82,40 @@ def test_auto(self):
             'parse_columns',
             'parse_row_identifiers',
         ])
+        self.assert_ranges(run, [
+            "Data Column 'type -> type' at Rows 1-4, Column 0",
+            "Data Column 'notes -> notes' at Rows 1-4, Column 1",
+            "Cell value '1 -> type=1' at Rows 1-2, Column 0",
+            "Cell value '2 -> type=2' at Row 3, Column 0",
+            "Cell value 'Unresolved: 100' at Row 4, Column 0",
+        ])
+
+    def test_auto_idmap_always(self):
+        # Map all ids without checking, producing same result as test_manual
+
+        self.serializer_name = 'tests.data_app.wizard.FKMapAlwaysSerializer'
+        run = self.upload_file('fkid.csv')
+
+        self.auto_import(run, expect_input_required=False)
+
+        self.assert_log(run, [
+            'created',
+            'auto_import',
+            'parse_columns',
+            'parse_row_identifiers',
+            'do_import',
+            'import_complete',
+        ])
+
+        self.check_data(
+            run,
+            expect_last_record="Failed at row 4: %s" % self.row4_error,
+            extra_ranges=[
+                "Cell value '1 -> type=1' at Rows 1-2, Column 0",
+                "Cell value '2 -> type=2' at Row 3, Column 0",
+                "Cell value '100 -> type=100' at Row 4, Column 0",
+            ]
+        )
 
 
 class SlugTestCase(BaseFKTestCase):
@@ -90,8 +143,8 @@ def test_manual(self):
             'import_complete',
         ])
 
-    def test_auto(self):
-        # Should abort due to unknown type id
+    def test_auto_idmap_never(self):
+        # Should abort due to previously unmapped type ids
         run = self.upload_file('fkname.csv')
         self.auto_import(run, expect_input_required=True)
         self.assert_log(run, [
@@ -131,7 +184,7 @@ def test_auto(self):
             'import_complete',
         ])
 
-    def test_auto_preset(self):
+    def test_auto_idmap_never_preset(self):
         self.create_identifier('Type #1', 'type', 'Type #1')
         self.create_identifier('Type #2', 'type', 'Type #2')
         self.create_identifier('Type #2 - Alternate Name', 'type', 'Type #2')
@@ -158,6 +211,56 @@ def test_auto_preset(self):
             'import_complete',
         ])
 
+    def test_auto_idmap_existing(self):
+        # Should auto-map 2 existing slugs, but abort due to unknown Alt Name
+        self.serializer_name = (
+            'tests.data_app.wizard.SlugMapExistingSerializer'
+        )
+        run = self.upload_file('fkname.csv')
+        self.auto_import(run, expect_input_required=True)
+        self.assert_log(run, [
+            'created',
+            'auto_import',
+            'parse_columns',
+            'parse_row_identifiers',
+        ])
+        self.assert_ranges(run, [
+            "Data Column 'type -> type' at Rows 1-4, Column 0",
+            "Data Column 'notes -> notes' at Rows 1-4, Column 1",
+            "Cell value 'Type #1 -> type=Type #1' at Rows 1-2, Column 0",
+            "Cell value 'Type #2 -> type=Type #2' at Row 3, Column 0",
+            "Cell value 'Unresolved: Type #2 - Alternate Name'"
+            " at Row 4, Column 0"
+        ])
+
+        # Update identifiers and try again
+        self.update_row_identifiers(run, {
+            'data_app.type': {
+                'Type #2 - Alternate Name': 'Type #2',
+            }
+        })
+        self.auto_import(run, expect_input_required=False)
+        self.check_data(
+            run,
+            expect_last_record="Imported 'Type #2 (Test Note 4)' at row 4",
+            extra_ranges=[
+                "Cell value 'Type #1 -> type=Type #1' at Rows 1-2, Column 0",
+                "Cell value 'Type #2 -> type=Type #2' at Row 3, Column 0",
+                "Cell value 'Type #2 - Alternate Name -> type=Type #2'"
+                " at Row 4, Column 0",
+            ]
+        )
+        self.assert_log(run, [
+            'created',
+            'auto_import',
+            'parse_columns',
+            'parse_row_identifiers',
+            'update_row_identifiers',
+            'auto_import',
+            'do_import',
+            'import_complete',
+        ])
+
 
 class SplitTestCase(BaseFKTestCase):
     serializer_name = 'tests.data_app.wizard.SlugSerializer'

Original file line number	Diff line number	Diff line change
`@@ -5,6 +5,7 @@`
`5`	`5`	`DEFAULTS = {`
`6`	`6`	`'BACKEND': 'data_wizard.backends.threading',`
`7`	`7`	`'LOADER': 'data_wizard.loaders.FileLoader',`
	`8`	`+ 'IDMAP': 'data_wizard.idmap.never', # FIXME: Change to 'existing' in 2.0`
`8`	`9`	`'AUTHENTICATION': 'rest_framework.authentication.SessionAuthentication',`
`9`	`10`	`'PERMISSION': 'rest_framework.permissions.IsAdminUser',`
`10`	`11`	`}`