diff --git a/jupyter_drives/handlers.py b/jupyter_drives/handlers.py index 7a279a7..c4a7f69 100644 --- a/jupyter_drives/handlers.py +++ b/jupyter_drives/handlers.py @@ -75,7 +75,8 @@ async def get(self, drive: str = "", path: str = ""): @tornado.web.authenticated async def post(self, drive: str = "", path: str = ""): - result = await self._manager.new_file(drive, path) + body = self.get_json_body() + result = await self._manager.new_file(drive, path, **body) self.finish(result) @tornado.web.authenticated diff --git a/jupyter_drives/manager.py b/jupyter_drives/manager.py index 76d6a4b..91645f9 100644 --- a/jupyter_drives/manager.py +++ b/jupyter_drives/manager.py @@ -2,6 +2,7 @@ import json import logging from typing import Dict, List, Optional, Tuple, Union, Any +from datetime import datetime import os import tornado @@ -262,28 +263,33 @@ async def get_contents(self, drive_name, path): return response - async def new_file(self, drive_name, path): + async def new_file(self, drive_name, path, is_dir): """Create a new file or directory at the given path. Args: drive_name: name of drive where the new content is created path: path where new content should be created + is_dir: boolean showing whether we are dealing with a directory or a file """ data = {} try: # eliminate leading and trailing backslashes path = path.strip('/') - # TO DO: switch to mode "created", which is not implemented yet - await obs.put_async(self._content_managers[drive_name]["store"], path, b"", mode = "overwrite") - metadata = await obs.head_async(self._content_managers[drive_name]["store"], path) - - data = { - "path": path, - "content": "", - "last_modified": metadata["last_modified"].isoformat(), - "size": metadata["size"] - } + if is_dir == False or self._config.provider != 's3': + # TO DO: switch to mode "created", which is not implemented yet + await obs.put_async(self._content_managers[drive_name]["store"], path, b"", mode = "overwrite") + metadata = await obs.head_async(self._content_managers[drive_name]["store"], path) + data = { + "path": path, + "content": "", + "last_modified": metadata["last_modified"].isoformat(), + "size": metadata["size"] + } + elif is_dir == True and self._config.provider == 's3': + # create an empty directory through boto, as obstore does not allow it + data = self._create_empty_directory(drive_name, path) + except Exception as e: raise tornado.web.HTTPError( status_code= httpx.codes.BAD_REQUEST, @@ -361,18 +367,39 @@ async def rename_file(self, drive_name, path, new_path): new_path: path of new file name """ data = {} + finished = False try: # eliminate leading and trailing backslashes path = path.strip('/') - - await obs.rename_async(self._content_managers[drive_name]["store"], path, new_path) - metadata = await obs.head_async(self._content_managers[drive_name]["store"], new_path) - data = { - "path": new_path, - "last_modified": metadata["last_modified"].isoformat(), - "size": metadata["size"] - } + # get list of contents with given prefix (path) + stream = obs.list(self._content_managers[drive_name]["store"], path, chunk_size=100, return_arrow=True) + async for batch in stream: + contents_list = pyarrow.record_batch(batch).to_pylist() + # rename each object within directory + for object in contents_list: + finished = True + remaining_path = object["path"][len(path)+1:] + old_path = path if remaining_path == '' else os.path.join(path, remaining_path) + formatted_new_path = new_path if remaining_path == '' else os.path.join(new_path, remaining_path) + try: + await obs.rename_async(self._content_managers[drive_name]["store"], old_path, formatted_new_path) + except Exception as e: + # we are dealing with a directory rename in S3 and obstore doesn't find the object + if self._config.provider == 's3': + self._rename_directory(drive_name, old_path, formatted_new_path) + else: + raise tornado.web.HTTPError( + status_code= httpx.codes.BAD_REQUEST, + reason=f"The following error occured when renaming the object: {e}", + ) + + # no extra S3 directories to rename + if data == {} and finished == False: + # rename single file from root(won't be listed above) + await obs.rename_async(self._content_managers[drive_name]["store"], path, new_path) + + data = await self._get_metadata(drive_name, new_path) except Exception as e: raise tornado.web.HTTPError( status_code= httpx.codes.BAD_REQUEST, @@ -394,8 +421,22 @@ async def delete_file(self, drive_name, path): try: # eliminate leading and trailing backslashes path = path.strip('/') - await obs.delete_async(self._content_managers[drive_name]["store"], path) + # get list of contents with given prefix (path) + stream = obs.list(self._content_managers[drive_name]["store"], path, chunk_size=100, return_arrow=True) + async for batch in stream: + contents_list = pyarrow.record_batch(batch).to_pylist() + # delete each object within directory + for object in contents_list: + await obs.delete_async(self._content_managers[drive_name]["store"], object["path"]) + + # delete file + await obs.delete_async(self._content_managers[drive_name]["store"], path) + + # when dealing with S3 directory, use helping function to delete remaining directories + if self._config.provider == 's3': + await self._delete_directories(drive_name, path) + except Exception as e: raise tornado.web.HTTPError( status_code= httpx.codes.BAD_REQUEST, @@ -500,6 +541,122 @@ def _check_object(self, drive_name, path): return isDir + def _create_empty_directory(self, drive_name, path): + """Helping function to create an empty directory, when dealing with S3 buckets. + + Args: + drive_name: name of drive where to create object + path: path of new object + """ + data = {} + try: + location = self._content_managers[drive_name]["location"] + if location not in self._s3_clients: + self._s3_clients[location] = self._s3_session.client('s3', location) + + self._s3_clients[location].put_object(Bucket = drive_name, Key = path + '/') + metadata = self._s3_clients[location].head_object(Bucket = drive_name, Key = path + '/') + + data = { + "path": path, + "content": "", + "last_modified": metadata["LastModified"].isoformat(), + "size": 0 + } + except Exception as e: + raise tornado.web.HTTPError( + status_code= httpx.codes.BAD_REQUEST, + reason=f"The following error occured when creating the directory: {e}", + ) + + return data + + async def _delete_directories(self, drive_name, path): + """Helping function to delete directories, when dealing with S3 buckets. + + Args: + drive_name: name of drive where to create object + path: path of new object + """ + try: + location = self._content_managers[drive_name]["location"] + if location not in self._s3_clients: + self._s3_clients[location] = self._s3_session.client('s3', location) + + # delete remaining sub-directories + stream = obs.list(self._content_managers[drive_name]["store"], path, chunk_size=100, return_arrow=True) + async for batch in stream: + contents_list = pyarrow.record_batch(batch).to_pylist() + for object in contents_list: + self._s3_clients[location].delete_object(Bucket=drive_name, Key=object["path"]+'/') + + # delete main directory + self._s3_clients[location].delete_object(Bucket=drive_name, Key=path+'/') + + except Exception as e: + raise tornado.web.HTTPError( + status_code= httpx.codes.BAD_REQUEST, + reason=f"The following error occured when deleting the directory: {e}", + ) + + return + + def _rename_directory(self, drive_name, path, new_path): + """Helping function to rename directories, when dealing with S3 buckets. + + Args: + drive_name: name of drive where to create object + path: path of object + new_path: new path of object + """ + try: + location = self._content_managers[drive_name]["location"] + if location not in self._s3_clients: + self._s3_clients[location] = self._s3_session.client('s3', location) + + self._s3_clients[location].copy_object(Bucket=drive_name, CopySource=os.path.join(drive_name, path)+'/', Key=new_path + '/') + self._s3_clients[location].delete_object(Bucket=drive_name, Key = path + '/') + except Exception: + # object is not found if we are not dealing with directory + pass + + return + + async def _get_metadata(self, drive_name, path): + """Helping function to get metadata of object. + + Args: + drive_name: name of drive where to create object + path: path of object + """ + try: + metadata = await obs.head_async(self._content_managers[drive_name]["store"], path) + data = { + "path": path, + "last_modified": metadata["last_modified"].isoformat(), + "size": metadata["size"] + } + except Exception: + try: + location = self._content_managers[drive_name]["location"] + if location not in self._s3_clients: + self._s3_clients[location] = self._s3_session.client('s3', location) + + metadata = self._s3_clients[location].head_object(Bucket=drive_name, Key=path + '/') + data = { + "path": path, + "last_modified": metadata["last_modified"].isoformat(), + "size": metadata["size"] + } + except Exception: + data = { + "path": path, + "last_modified": datetime.now().isoformat(), + "size": 0 + } + + return data + async def _call_provider( self, url: str, diff --git a/src/contents.ts b/src/contents.ts index 67f140f..f65e916 100644 --- a/src/contents.ts +++ b/src/contents.ts @@ -300,6 +300,7 @@ export class Drive implements Contents.IDrive { data = await createObject(currentDrive.name, { name: name, path: relativePath, + isDir: options.type === 'directory' ? true : false, registeredFileTypes: this._registeredFileTypes }); } else { diff --git a/src/requests.ts b/src/requests.ts index 242157c..8914a2f 100644 --- a/src/requests.ts +++ b/src/requests.ts @@ -195,6 +195,7 @@ export async function saveObject( * * @param driveName * @param options.path The path of new object. + * @param options.isDir A boolean variable showing if we are dealing with a directory or file. * @param options.registeredFileTypes The list containing all registered file types. * * @returns A promise which resolves with the contents model. @@ -204,6 +205,7 @@ export async function createObject( options: { name: string; path: string; + isDir: boolean; registeredFileTypes: IRegisteredFileTypes; } ) { @@ -212,7 +214,10 @@ export async function createObject( : options.name; const response = await requestAPI( 'drives/' + driveName + '/' + path, - 'POST' + 'POST', + { + is_dir: options.isDir + } ); const [fileType, fileMimeType, fileFormat] = getFileType( @@ -249,26 +254,7 @@ export async function deleteObjects( path: string; } ) { - // get list of contents with given prefix (path) - const response = await requestAPI( - 'drives/' + driveName + '/' + options.path, - 'GET' - ); - - // deleting contents of a directory - if (response.data.length !== undefined && response.data.length !== 0) { - await Promise.all( - response.data.map(async (c: any) => { - return Private.deleteSingleObject(driveName, c.path); - }) - ); - } - try { - // always deleting the object (file or main directory) - return Private.deleteSingleObject(driveName, options.path); - } catch (error) { - // deleting failed if directory didn't exist and was only part of a path - } + await requestAPI('drives/' + driveName + '/' + options.path, 'DELETE'); } /** @@ -300,47 +286,26 @@ export async function renameObjects( options.registeredFileTypes ); - // get list of contents with given prefix (path) const response = await requestAPI( 'drives/' + driveName + '/' + options.path, - 'GET' + 'PATCH', + { + new_path: formattedNewPath + } ); - // renaming contents of a directory - if (response.data.length !== undefined && response.data.length !== 0) { - await Promise.all( - response.data.map(async (c: any) => { - const remainingFilePath = c.path.substring(options.path.length); - Private.renameSingleObject( - driveName, - PathExt.join(options.path, remainingFilePath), - PathExt.join(formattedNewPath, remainingFilePath) - ); - }) - ); - } - // always rename the object (file or main directory) - try { - const renamedObject = await Private.renameSingleObject( - driveName, - options.path, - formattedNewPath - ); - data = { - name: options.newFileName, - path: PathExt.join(driveName, formattedNewPath), - last_modified: renamedObject.data.last_modified, - created: '', - content: PathExt.extname(options.newFileName) !== '' ? null : [], // TODO: add dir check - format: fileFormat as Contents.FileFormat, - mimetype: fileMimeType, - size: renamedObject.data.size, - writable: true, - type: fileType - }; - } catch (error) { - // renaming failed if directory didn't exist and was only part of a path - } + data = { + name: options.newFileName, + path: PathExt.join(driveName, formattedNewPath), + last_modified: response.data.last_modified, + created: '', + content: null, + format: fileFormat as Contents.FileFormat, + mimetype: fileMimeType, + size: response.data.size, + writable: true, + type: fileType + }; return data; } @@ -471,41 +436,6 @@ export const countObjectNameAppearances = async ( }; namespace Private { - /** - * Helping function for deleting files inside - * a directory, in the case of deleting the directory. - * - * @param driveName - * @param objectPath complete path of object to delete - */ - export async function deleteSingleObject( - driveName: string, - objectPath: string - ) { - await requestAPI('drives/' + driveName + '/' + objectPath, 'DELETE'); - } - - /** - * Helping function for renaming files inside - * a directory, in the case of deleting the directory. - * - * @param driveName - * @param objectPath complete path of object to rename - */ - export async function renameSingleObject( - driveName: string, - objectPath: string, - newObjectPath: string - ) { - return await requestAPI( - 'drives/' + driveName + '/' + objectPath, - 'PATCH', - { - new_path: newObjectPath - } - ); - } - /** * Helping function for copying files inside * a directory, in the case of deleting the directory.