Merge pull request #115 from hazelnutsgz/conll2003

[new] Add a loader for conll2003 dataset

Author: xpqiu

fastNLP/io/dataset_loader.py

@@ -417,6 +417,55 @@ def convert(self, data):
         data_set.set_input("seq_len")
         return data_set
 
+    
+class Conll2003Loader(DataSetLoader):
+    """Self-defined loader of conll2003 dataset
+    
+        More information about the given dataset cound be found on 
+        https://sites.google.com/site/ermasoftware/getting-started/ne-tagging-conll2003-data 
+    
+    """
+    
+    def __init__(self):
+        super(Conll2003Loader, self).__init__()
+        
+    def load(self, dataset_path):
+        with open(dataset_path, "r", encoding="utf-8") as f:
+            lines = f.readlines()
+        
+        ##Parse the dataset line by line
+        parsed_data = []
+        sentence = []
+        tokens = []
+        for line in lines:
+            if '-DOCSTART- -X- -X- O' in line or line == '\n':
+                if sentence != []:
+                    parsed_data.append((sentence, tokens))
+                    sentence = []
+                    tokens = []
+                continue
+                
+            temp = line.strip().split(" ")
+            sentence.append(temp[0]) 
+            tokens.append(temp[1:4])
+                           
+        return self.convert(parsed_data)
+            
+    def convert(self, parsed_data):
+        dataset = DataSet()
+        for sample in parsed_data:
+            label0_list = list(map(
+                lambda labels: labels[0], sample[1]))
+            label1_list = list(map(
+                lambda labels: labels[1], sample[1]))
+            label2_list = list(map(
+                lambda labels: labels[2], sample[1]))
+            dataset.append(Instance(token_list=sample[0], 
+                                    label0_list=label0_list, 
+                                    label1_list=label1_list,
+                                    label2_list=label2_list))
+            
+        return dataset
 
 class SNLIDataSetLoader(DataSetLoader):
     """A data set loader for SNLI data set.