Merge pull request #41 from CBIIT/CRDCDH-2050-001-pgu

Crdcdh 2050

Author: n2iw

src/common/s3util.py

@@ -41,14 +41,19 @@ def file_exists_on_s3(self, key):
         '''
         try:
             self.client.head_object(Bucket=self.bucket.name, Key=key)
-            return True
+            return True, None
         except ClientError as e:
+            msg = None
             if e.response['Error']['Code'] in ['404', '412']:
-                return False
+                msg = f'File {key} does not exist in the specified S3 bucket path.'
+                return False, msg
+            if e.response['Error']['Code'] in ['403']:
+                msg = f'Access Denied: Unable to access files in the specified S3 bucket path: {key}'
+                return False, msg
             else:
-                self.log.error('Unknown S3 client error!')
+                msg = f'Unknown S3 client error!'
                 self.log.exception(e)
-                return False   
+                return False, msg  
 
     def put_file_obj(self, key, data, md5_base64):
         return self.bucket.put_object(Key=key,
@@ -62,23 +67,48 @@ def upload_file_obj(self, key, data, config=None, extra_args={'ACL': BUCKET_OWNE
     def get_object_size(self, key):
         try:
             res = self.client.head_object(Bucket=self.bucket_name, Key=key)
-            return res['ContentLength']
+            return res['ContentLength'], None
         except ClientError as e:
-            return None
+            msg = None
+            if e.response['Error']['Code'] in ['404', '412']:
+                msg = f'File {key} does not exist in the specified S3 bucket path.'
+                return None, msg
+            if e.response['Error']['Code'] in ['403']:
+                msg = f'Access Denied: Unable to access files in the specified S3 bucket path: {key}'
+                return None, msg
+            else:
+                msg = f'Unknown S3 client error!'
+                self.log.exception(e)
+                return None, msg  
 
     def same_size_file_exists(self, key, file_size):
-        return file_size == self.get_object_size(key)
+        file_size1, msg = self.get_object_size(key)
+        if msg:
+            self.log.error(msg)
+            return False
+        return file_size == file_size1
 
     def download_object(self, key, local_file_path):
         try:
             self.bucket.download_file( key, local_file_path)
-            return True
+            return True, None
         except ClientError as ce:
-            self.log.error(ce)
-            return False
+            msg = None
+            if e.response['Error']['Code'] in ['404', '412']:
+                msg = f'File {key} does not exist in the specified S3 bucket path.'
+                return False, msg
+            if e.response['Error']['Code'] in ['403']:
+                msg = f'Access Denied: Unable to access files in the specified S3 bucket path: {key}'
+                return False, msg
+            else:
+                msg = f'Unknown S3 client error!'
+                self.log.exception(e)
+                return False, msg  
         except Exception as e:
+            # 
+            msg = f'Unknown error!'
             self.log.error(e)
-            return False
+            return False,
 
     def close(self):
         self.client.close()

src/copier.py

@@ -103,7 +103,7 @@ def copy_file(self, file_info, overwrite, dryrun):
                 file_info[SKIPPED] = False
             #self.log.info(f'Copying from {org_url} to s3://{self.bucket_name}/{key.strip("/")} ...')
             self.log.info(f'Copying from {org_url} to destination folder in S3 bucket ...')
-            dest_size = self._upload_obj(org_url, key, org_size)
+            dest_size, msg = self._upload_obj(org_url, key, org_size)
             if dest_size != org_size:
                 self.log.error(f'Uploading “{file_name}” failed - uploading was not complete. Please try again and contact the helpdesk if this error persists.')
                 return {self.STATUS: False}

src/file_uploader.py

@@ -84,8 +84,9 @@ def upload(self):
                 file_info = job[self.INFO]
                 file_path = file_info[FILE_PATH]
                 if self.from_s3 == True: #download file from s3
-                    if not self.s3_bucket.download_object(os.path.join(self.from_prefix, file_info[FILE_NAME_DEFAULT]), file_path): 
-                        invalid_reason = f"Failed to download {file_info[FILE_NAME_DEFAULT]} from {self.file_dir}!"
+                    result, msg = self.s3_bucket.download_object(os.path.join(self.from_prefix, file_info[FILE_NAME_DEFAULT]), file_path)
+                    if not result: 
+                        invalid_reason = msg
                         file_info[SUCCEEDED] = False
                         file_info[ERRORS] = [invalid_reason]
                         self.invalid_count += 1

src/file_validator.py

@@ -119,9 +119,9 @@ def validate_size_md5(self):
                     self.log.error(invalid_reason)
                     continue
             else: # check file existing and validate file size in s3 bucket
-                s3_file_size = self.s3_bucket.get_object_size(os.path.join(self.from_prefix, info[FILE_NAME_DEFAULT]))
+                s3_file_size, msg = self.s3_bucket.get_object_size(os.path.join(self.from_prefix, info[FILE_NAME_DEFAULT]))
                 if not s3_file_size:
-                    invalid_reason += f"File {info[FILE_NAME_DEFAULT]} does not exist in {self.file_dir}!"
+                    invalid_reason += msg
                     self.fileList.append({FILE_ID_DEFAULT: file_id, FILE_NAME_DEFAULT: info.get(FILE_NAME_DEFAULT), FILE_PATH: file_path, FILE_SIZE_DEFAULT: size_info, MD5_DEFAULT: info[MD5_DEFAULT], SUCCEEDED: False, ERRORS: [invalid_reason]})
                     self.invalid_count += 1
                     self.log.error(invalid_reason)
@@ -162,8 +162,9 @@ def read_manifest(self):
             try:
                 s3_bucket = S3Bucket()
                 s3_bucket.set_s3_client(bucket_name, None)
-                if s3_bucket.file_exists_on_s3(key) == False:
-                    self.log.critical(f"Manifest file {self.pre_manifest} does not exist!")
+                result, msg = s3_bucket.file_exists_on_s3(key)
+                if  result == False:
+                    self.log.critical(msg)
                     return None
                 s3_bucket.download_object(key, local_manifest)
                 self.pre_manifest = self.configs[PRE_MANIFEST] = local_manifest