Merge pull request #40 from CBIIT/CRDCDH-2043-0012-pgu

Crdcdh 2043

Author: n2iw

src/common/s3util.py

@@ -32,7 +32,23 @@ def set_s3_client(self, bucket, credentials):
             self.bucket = self.s3.Bucket(bucket)
             self.credential = None
 
-       
+    def file_exists_on_s3(self, key):
+        '''
+        Check if file exists in S3, return True only if file exists
+
+        :param key: file path
+        :return: boolean
+        '''
+        try:
+            self.client.head_object(Bucket=self.bucket.name, Key=key)
+            return True
+        except ClientError as e:
+            if e.response['Error']['Code'] in ['404', '412']:
+                return False
+            else:
+                self.log.error('Unknown S3 client error!')
+                self.log.exception(e)
+                return False   
 
     def put_file_obj(self, key, data, md5_base64):
         return self.bucket.put_object(Key=key,

src/file_uploader.py

@@ -26,7 +26,7 @@ def __init__(self, configs, file_list):
         self.prefix = configs[FILE_PREFIX]
         self.bucket_name = configs.get(S3_BUCKET)
         self.credential = configs.get(TEMP_CREDENTIAL)
-        self.pre_manifest = configs.get(PRE_MANIFEST)
+        # self.pre_manifest = configs.get(PRE_MANIFEST)
         self.file_info_list = file_list
         self.copier = None
         self.count = len(file_list)

src/file_validator.py

@@ -4,7 +4,7 @@
 import glob
 from common.constants import UPLOAD_TYPE, TYPE_FILE, TYPE_MATE_DATA, FILE_NAME_DEFAULT, FILE_SIZE_DEFAULT, MD5_DEFAULT, \
     FILE_DIR, FILE_MD5_FIELD, PRE_MANIFEST, FILE_NAME_FIELD, FILE_SIZE_FIELD, FILE_PATH, SUCCEEDED, ERRORS, FILE_ID_DEFAULT,\
-    FILE_ID_FIELD, OMIT_DCF_PREFIX, FROM_S3, TEMP_DOWNLOAD_DIR
+    FILE_ID_FIELD, OMIT_DCF_PREFIX, FROM_S3, TEMP_DOWNLOAD_DIR, S3_START
 from common.utils import clean_up_key_value, clean_up_strs, is_valid_uuid
 from bento.common.utils import get_logger, get_md5
 from common.utils import extract_s3_info_from_url
@@ -56,9 +56,6 @@ def validate(self):
                 self.fileList.append({FILE_NAME_DEFAULT:filename, FILE_PATH: filepath, FILE_SIZE_DEFAULT: size})
 
         elif self.uploadType == TYPE_FILE: #file
-            if not os.path.isfile(self.pre_manifest):
-                self.log.critical(f'manifest file is not valid!')
-                return False 
             try: 
                 return self.validate_size_md5()
             except Exception as e:
@@ -155,6 +152,29 @@ def read_manifest(self):
         files_info = []
         files_dict = {}
         manifest_rows = []
+        is_s3_manifest = self.pre_manifest.startswith(S3_START)
+        if is_s3_manifest:
+            s3_bucket = None
+            bucket_name, key = extract_s3_info_from_url(self.pre_manifest)
+            self.download_file_dir = TEMP_DOWNLOAD_DIR
+            os.makedirs(self.download_file_dir, exist_ok=True)
+            local_manifest = os.path.join(self.download_file_dir, key.split('/')[-1])
+            try:
+                s3_bucket = S3Bucket()
+                s3_bucket.set_s3_client(bucket_name, None)
+                if s3_bucket.file_exists_on_s3(key) == False:
+                    self.log.critical(f"Manifest file {self.pre_manifest} does not exist!")
+                    return None
+                s3_bucket.download_object(key, local_manifest)
+                self.pre_manifest = self.configs[PRE_MANIFEST] = local_manifest
+            except Exception as e:
+                self.log.debug(e)
+                self.log.exception(f"Downloading manifest failed - internal error. Please try again and contact the helpdesk if this error persists.")
+                return None
+            finally:
+                if s3_bucket:
+                    s3_bucket.close()
+
         try:
             with open(self.pre_manifest) as pre_m:
                 reader = csv.DictReader(pre_m, delimiter='\t')

src/upload_config.py

@@ -111,10 +111,11 @@ def validate(self):
                 if manifest is None:
                     self.log.critical(f'Please provide “manifest” in configuration file or command line argument.')
                     return False
-                if not os.path.isfile(manifest): 
-                    self.log.critical(f'Manifest file “{manifest}” is not readable. Please make sure the path is correct and the file is readable.')
-                    return False
-                
+                if not manifest.startswith(S3_START):
+                    if not os.path.isfile(manifest): 
+                        self.log.critical(f'Manifest file “{manifest}” is not readable. Please make sure the path is correct and the file is readable.')
+                        return False
+
                 self.data[PRE_MANIFEST]  = manifest
                 #check header names in manifest file
                 file_name_header= self.data.get(FILE_NAME_FIELD)