Add cleanse_data and tests.

Used test_data containing all possible states for ease of testing.

Author: Tomasz-Kluczkowski

data_extractor.py

@@ -28,8 +28,29 @@ def amend_domain_values(self, prefix='www.'):
             amended.append(item)
         return amended
 
+    def cleanse_data(self):
+        """
+        Fix errors in "secure" key values. All urls starting with https should be set to "secure": True, those starting
+        with http "secure": False.
+        :return: amended: list(dict), amended list of web records.
+        """
+        amended = []
+        for item in self.data:
+            url = item.get('url')
+            secure = item.get('secure')
+            # https marked as secure = False
+            if url and url.startswith('https:') and not item.get('secure'):
+                item['secure'] = True
+            # http marked as secure = True
+            elif url and url.startswith('http:') and item.get('secure'):
+                item['secure'] = False
+            amended.append(item)
+        return amended
+
+
 
-data_extractor = DataExtractor(WEBSITES)
+# data_extractor = DataExtractor(WEBSITES)
 # print(data_extractor.amend_domain_values())
-print(data_extractor.find_items(4))
-print(len(data_extractor.find_items(4)))
+# print(data_extractor.find_items(4))
+# print(len(data_extractor.find_items(4)))
+# print(data_extractor.cleanse_data())

tests/test_data_extractor.py

@@ -107,3 +107,68 @@ def test_amend_domain_values_retains_original_if_prefix_matching(self):
         ]
         _data_extractor = DataExtractor(test_data)
         assert _data_extractor.amend_domain_values() == test_data
+
+    def test_cleanse_data(self):
+        test_data = [
+            {
+                'name': 'Google',
+                'url': 'https://www.google.co.uk',
+                'domain': 'google.co.uk',
+                'secure': False,
+                'value': 5
+            },
+            {
+                'name': 'Facebook',
+                'url': 'http://developers.facebook.com/blog/post/2018/10/02/facebook-login-update/',
+                'domain': 'facebook.com',
+                'secure': True,
+                'value': 4
+            },
+            {
+                'name': 'Bing',
+                'url': 'http://www.bing.com/search?q=athlete&qs=n&form=QBLH&sp=-1&pq=athlete&sc=8-7&sk=&cvid=53830DD7FB2E47B7A5D9CF27F106BC9A',
+                'domain': 'bing.com',
+                'secure': False,
+                'value': 3
+            },
+            {
+                'name': 'Duck Duck Go',
+                'url': 'https://duckduckgo.com/?q=plane&t=h_&ia=web',
+                'domain': 'duckduckgo.com',
+                'secure': True,
+                'value': 2
+            },
+        ]
+
+        expected = [
+            {
+                'name': 'Google',
+                'url': 'https://www.google.co.uk',
+                'domain': 'google.co.uk',
+                'secure': True,
+                'value': 5
+            },
+            {
+                'name': 'Facebook',
+                'url': 'http://developers.facebook.com/blog/post/2018/10/02/facebook-login-update/',
+                'domain': 'facebook.com',
+                'secure': False,
+                'value': 4
+            },
+            {
+                'name': 'Bing',
+                'url': 'http://www.bing.com/search?q=athlete&qs=n&form=QBLH&sp=-1&pq=athlete&sc=8-7&sk=&cvid=53830DD7FB2E47B7A5D9CF27F106BC9A',
+                'domain': 'bing.com',
+                'secure': False,
+                'value': 3
+            },
+            {
+                'name': 'Duck Duck Go',
+                'url': 'https://duckduckgo.com/?q=plane&t=h_&ia=web',
+                'domain': 'duckduckgo.com',
+                'secure': True,
+                'value': 2
+            },
+        ]
+        _data_extractor = DataExtractor(test_data)
+        assert _data_extractor.cleanse_data() == expected