diff --git a/api/tests/conftest.py b/api/tests/conftest.py index af5196f..8773bbd 100644 --- a/api/tests/conftest.py +++ b/api/tests/conftest.py @@ -4,17 +4,57 @@ import pytest from pyramid import testing +from managers.article_manager import ArticleManager +from persistence.databases import InMemoryDBManager +from persistence.services import ChangesService +from persistence.seqnum_generator import SeqNumGenerator + @pytest.fixture def dummy_request(): request = testing.DummyRequest() request.db_settings = { - 'host': 'http://localhost', - 'port': '12345', + 'db_host': 'http://localhost', + 'db_port': '12345' } return request +@pytest.fixture +def inmemory_article_manager(request): + db_host = 'http://inmemory' + articles_dbmanager = InMemoryDBManager(database_uri=db_host, + database_name='articles') + files_dbmanager = InMemoryDBManager(database_uri=db_host, + database_name='files') + changes_dbmanager = InMemoryDBManager(database_uri=db_host, + database_name='changes') + changes_seq_dbmanager = InMemoryDBManager(database_uri=db_host, + database_name='changes_seqnum') + + def fin(): + try: + articles_dbmanager.drop_database() + files_dbmanager.drop_database() + changes_dbmanager.drop_database() + changes_seq_dbmanager.drop_database() + except Exception: + pass + + request.addfinalizer(fin) + return ArticleManager( + articles_dbmanager, + files_dbmanager, + ChangesService( + changes_dbmanager, + SeqNumGenerator( + changes_seq_dbmanager, + 'CHANGE' + ) + ) + ) + + @pytest.fixture def test_xml_file(): return """ diff --git a/api/tests/test_article.py b/api/tests/test_article.py index 8d76f85..4243867 100644 --- a/api/tests/test_article.py +++ b/api/tests/test_article.py @@ -364,16 +364,14 @@ def test_http_get_asset_file_succeeded(mocked_get_asset_file, assert response.content_type == expected[0] -@patch.object(managers, 'create_file') @patch.object(managers, 'post_article') def test_post_article_invalid_xml(mocked_post_article, - mocked_create_file, dummy_request, test_xml_file): xml_file = MockCGIFieldStorage("test_xml_file.xml", BytesIO(test_xml_file.encode('utf-8'))) error_msg = 'Invalid XML Content' - mocked_create_file.side_effect = \ + mocked_post_article.side_effect = \ managers.exceptions.ManagerFileError( message=error_msg ) @@ -410,10 +408,13 @@ def test_post_article_internal_error(mocked_post_article, assert excinfo.value.message == error_msg -@patch.object(managers, 'post_article') -def test_post_article_returns_article_version_url(mocked_post_article, +@patch.object(managers, '_get_article_manager') +def test_post_article_returns_article_version_url(mocked__get_article_manager, + inmemory_article_manager, dummy_request, test_xml_file): + mocked__get_article_manager.return_value = inmemory_article_manager + inmemory_db_settings = '/rawfile' xml_file = MockCGIFieldStorage("test_xml_file.xml", BytesIO(test_xml_file.encode('utf-8'))) dummy_request.POST = { @@ -423,10 +424,10 @@ def test_post_article_returns_article_version_url(mocked_post_article, article_api = ArticleAPI(dummy_request) response = article_api.collection_post() - mocked_post_article.assert_called_once() assert response.status_code == 201 assert response.json is not None assert response.json.get('url').endswith(xml_file.filename) + assert response.json.get('url').startswith(inmemory_db_settings) @patch.object(managers, 'get_article_document') diff --git a/api/views/article.py b/api/views/article.py index 6de6e05..856e150 100644 --- a/api/views/article.py +++ b/api/views/article.py @@ -46,18 +46,22 @@ def _get_file_property(self, file_field): raise HTTPBadRequest(detail=e.message) def collection_post(self): + """ + Receive new Article document package which must contain a XML file. + """ try: xml_file_field = self.request.POST.get('xml_file') - xml_file = self._get_file_property(xml_file_field) - managers.post_article( + xml_id = Path(xml_file_field.filename).name + xml_file = xml_file_field.file.read() + article_url = managers.post_article( article_id=self.request.POST['article_id'], + xml_id=xml_id, xml_file=xml_file, **self.request.db_settings ) - body = { - 'url': '/rawfiles/7ca9f9b2687cb/' + xml_file_field.filename - } - return Response(status_code=201, json=body) + return Response(status_code=201, json={'url': article_url}) + except managers.exceptions.ManagerFileError as e: + raise HTTPBadRequest(detail=e.message) except managers.article_manager.ArticleManagerException as e: raise HTTPInternalServerError(detail=e.message) diff --git a/managers/__init__.py b/managers/__init__.py index 1c22b96..764e56d 100644 --- a/managers/__init__.py +++ b/managers/__init__.py @@ -1,6 +1,6 @@ from managers.article_manager import ArticleManager from managers.exceptions import ManagerFileError -from managers.models.article_model import ArticleDocument +from managers.models.article_model import ArticleDocument, InvalidXMLContent from managers.models.file import File from persistence.databases import CouchDBManager from persistence.services import ( @@ -32,12 +32,14 @@ def _get_changes_services(db_settings): def _get_article_manager(**db_settings): - database_config = db_settings - articles_database_config = database_config.copy() + articles_database_config = db_settings.copy() articles_database_config['database_name'] = "articles" + files_database_config = db_settings.copy() + files_database_config['database_name'] = "files" return ArticleManager( CouchDBManager(**articles_database_config), + CouchDBManager(**files_database_config), _get_changes_services(db_settings) ) @@ -57,11 +59,34 @@ def create_file(filename, content): return File(file_name=filename, content=content) -def post_article(xml_file, **db_settings): - """""" +def post_article(article_id, xml_id, xml_file, **db_settings): + """ + Registra novo documento de artigo em banco de dados informado, persistindo + a versão codificada em XML recebida e um manifesto do artigo contendo a + referência para recuperar o arquivo XML. + + :param article_id: ID do Documento do tipo Artigo, para identificação + referencial + :param xml_id: identificação do arquivo + :param xml_file: objeto File-like conteúdo do XML + :param db_settings: dicionário com as configurações do banco de dados. + Deve conter: + - database_uri: URI do banco de dados (host:porta) + - database_username: usuário do banco de dados + - database_password: senha do banco de dados + + :returns: URL pública para recuperar a versão registrada do artigo + codificado em XML + :rtype: str + """ + article_document = ArticleDocument(article_id) article_manager = _get_article_manager(**db_settings) - article_manager.add_document() - return xml_file.get_version() + try: + article_document.add_version(xml_id, xml_file) + except InvalidXMLContent as e: + raise ManagerFileError(message=e.message) + else: + return article_manager.add_document(article_document) def put_article(article_id, xml_file, assets_files=[], **db_settings): diff --git a/managers/article_manager.py b/managers/article_manager.py index 76e68bf..38c4f43 100644 --- a/managers/article_manager.py +++ b/managers/article_manager.py @@ -29,9 +29,12 @@ class ArticleManagerMissingAssetFileException(Exception): class ArticleManager: - def __init__(self, articles_db_manager, changes_services): + def __init__(self, articles_db_manager, files_db_manager, + changes_services): self.article_db_service = DatabaseService( articles_db_manager, changes_services) + self.file_db_service = DatabaseService( + files_db_manager, changes_services) def receive_package(self, id, xml_file, files=None): article = self.receive_xml_file(id, xml_file) @@ -75,7 +78,14 @@ def receive_asset_file(self, article, file): ) def add_document(self, article_document): - pass + added_file_url = self.file_db_service.add_file( + file_id=article_document.xml_file_id, + content=article_document.xml_tree.content + ) + article_document.update_version(added_file_url) + article_record = article_document.get_record() + self.article_db_service.register(article_document.id, article_record) + return "/rawfile/" + article_document.xml_file_id def get_article_data(self, article_id): try: diff --git a/managers/models/article_model.py b/managers/models/article_model.py index 783360d..f177ca8 100644 --- a/managers/models/article_model.py +++ b/managers/models/article_model.py @@ -1,9 +1,19 @@ # coding=utf-8 +from enum import Enum import os from ..xml.article_xml_tree import ArticleXMLTree +class InvalidXMLContent(Exception): + message = "Invalid XML Content" + + +class DocumentType(Enum): + DOCUMENT = 'DOC' + ARTICLE = 'ART' + + class AssetDocument: """Metadados de um documento do tipo Ativo Digital. Um Ativo Digital é um arquivo associado a um documento do tipo Artigo @@ -46,6 +56,7 @@ class ArticleDocument: """ def __init__(self, article_id): self.id = article_id + self.versions = [] self.assets = {} self.unexpected_files_list = [] self._xml_file = None @@ -53,6 +64,27 @@ def __init__(self, article_id): self.xml_name = None self.xml_content = None + def add_version(self, file_id, xml_content): + """Adiciona nova versão de artigo codificado em XML em :attr:`versions` + e cria nova referência para atualizar os dados do manifesto do artigo. + Caso o conteúdo do XML for inválido, a exceção + :class:`InvalidXMLContent` é lançada. + """ + self.xml_tree = ArticleXMLTree(xml_content) + if self.xml_tree.xml_error: + raise InvalidXMLContent + self.xml_file_id = '/'.join([self.xml_tree.checksum[:13], file_id]) + self.versions.append({ + 'data': self.xml_file_id, + 'assets': [] + }) + + def update_version(self, added_file_url): + """ + Atualiza referência do artigo codificado em XML em :attr:`versions`. + """ + self.versions[-1].update({'data': added_file_url}) + @property def xml_file(self): """Acessa ou define o documento Artigo em XML, representado por uma @@ -72,7 +104,9 @@ def xml_file(self): def xml_file(self, xml_file): self._xml_file = xml_file if xml_file is not None: - self.xml_tree = ArticleXMLTree(self._xml_file.content) + self.xml_tree = ArticleXMLTree(xml_file.content) + if self.xml_tree.xml_error: + raise InvalidXMLContent self.assets = { name: AssetDocument(node) for name, node in self.xml_tree.asset_nodes.items() @@ -121,6 +155,18 @@ def get_record_content(self): ] return record_content + def get_record(self): + """Obtém um dicionário que descreve a instância de + :class:`ArticleDocument` da seguinte maneira: chave ``id``, contendo o + ID do artigo e chave ``versions``, contendo uma lista de versões do + artigo, com a URI da respectiva codificação XML e seus assets. + """ + record_content = {} + record_content['document_id'] = self.id + record_content['document_type'] = DocumentType.ARTICLE.value + record_content['versions'] = self.versions + return record_content + @property def missing_files_list(self): """Obtém uma lista com os nomes dos arquivos dos ativos digitais do diff --git a/managers/tests/conftest.py b/managers/tests/conftest.py index 0f82c85..294e97d 100644 --- a/managers/tests/conftest.py +++ b/managers/tests/conftest.py @@ -24,13 +24,11 @@ def test_fixture_dir(): def read_file(fixture_dir, dir_path, filename): - xml_file = File(filename) file_path = os.path.join(fixture_dir, dir_path, filename) if os.path.isfile(file_path): with open(file_path, 'rb') as fb: - xml_file.content = fb.read() - xml_file.size = os.stat(file_path).st_size - return xml_file + xml_file = File(filename, fb.read()) + return xml_file @pytest.fixture(scope="module") @@ -100,9 +98,10 @@ def functional_config(request): @pytest.fixture def change_service(functional_config): + db_host = 'http://inmemory' return ( - InMemoryDBManager(database_name='articles'), - InMemoryDBManager(database_name='changes') + InMemoryDBManager(database_uri=db_host, database_name='articles'), + InMemoryDBManager(database_uri=db_host, database_name='changes') ) @@ -120,8 +119,9 @@ def xml_test(): @pytest.fixture def seqnumber_generator(request): + db_host = 'http://inmemory' s = SeqNumGenerator( - InMemoryDBManager(database_name='test3'), + InMemoryDBManager(database_uri=db_host, database_name='test3'), 'CHANGE' ) @@ -134,16 +134,18 @@ def fin(): @pytest.fixture def changes_service(request, seqnumber_generator): + db_host = 'http://inmemory' return ChangesService( - InMemoryDBManager(database_name='test2'), + InMemoryDBManager(database_uri=db_host, database_name='test2'), seqnumber_generator ) @pytest.fixture def databaseservice_params(functional_config, changes_service): + db_host = 'http://inmemory' return ( - InMemoryDBManager(database_name='test1'), + InMemoryDBManager(database_uri=db_host, database_name='test1'), changes_service ) @@ -161,13 +163,20 @@ def fin(): @pytest.fixture -def inmemory_receive_package(databaseservice_params, test_package_A): - article_manager = ArticleManager( - databaseservice_params[0], +def set_inmemory_article_manager(setup, databaseservice_params): + db_host = 'http://inmemory' + return ArticleManager( + InMemoryDBManager(database_uri=db_host, database_name='articles'), + InMemoryDBManager(database_uri=db_host, database_name='files'), databaseservice_params[1]) - return article_manager.receive_package(id='ID', - xml_file=test_package_A[0], - files=test_package_A[1:]) + + +@pytest.fixture +def inmemory_receive_package(set_inmemory_article_manager, test_package_A): + return set_inmemory_article_manager.receive_package( + id='ID', + xml_file=test_package_A[0], + files=test_package_A[1:]) @pytest.fixture diff --git a/managers/tests/test_article_manager.py b/managers/tests/test_article_manager.py index df1b8ff..13ed6f1 100644 --- a/managers/tests/test_article_manager.py +++ b/managers/tests/test_article_manager.py @@ -2,30 +2,21 @@ import pytest -from persistence.databases import ( - DocumentNotFound, - DBFailed, -) +from persistence.databases import DBFailed, DocumentNotFound, InMemoryDBManager from persistence.services import DatabaseService -from persistence.models import RecordType - -from managers.models.article_model import ( - ArticleDocument, -) from managers.article_manager import ( ArticleManager, ArticleManagerException ) +from managers.models.article_model import ArticleDocument from managers.xml.xml_tree import ( XMLTree ) -def test_receive_xml_file(databaseservice_params, test_package_A, +def test_receive_xml_file(set_inmemory_article_manager, test_package_A, test_packA_filenames): - article_manager = ArticleManager( - databaseservice_params[0], - databaseservice_params[1]) + article_manager = set_inmemory_article_manager expected = { 'attachments': [test_packA_filenames[0]], 'content': { @@ -44,10 +35,8 @@ def test_receive_xml_file(databaseservice_params, test_package_A, assert sorted(got['attachments']) == sorted(expected['attachments']) -def test_receive_package(databaseservice_params, test_package_A): - article_manager = ArticleManager( - databaseservice_params[0], - databaseservice_params[1]) +def test_receive_package(set_inmemory_article_manager, test_package_A): + article_manager = set_inmemory_article_manager unexpected, missing = article_manager.receive_package( id='ID', xml_file=test_package_A[0], @@ -57,18 +46,139 @@ def test_receive_package(databaseservice_params, test_package_A): assert missing == [] +def test_article_manager(databaseservice_params): + db_host = 'http://inmemory' + article_manager = ArticleManager( + InMemoryDBManager(database_uri=db_host, database_name='articles'), + InMemoryDBManager(database_uri=db_host, database_name='files'), + databaseservice_params[1] + ) + assert article_manager.article_db_service is not None + assert isinstance(article_manager.article_db_service, DatabaseService) + assert article_manager.file_db_service is not None + assert isinstance(article_manager.file_db_service, DatabaseService) + + +@patch.object(DatabaseService, 'add_file', side_effect=Exception()) +def test_add_document_add_file_error(mocked_register, + set_inmemory_article_manager, + test_package_A): + article_document = ArticleDocument(test_package_A[0].name) + article_manager = set_inmemory_article_manager + pytest.raises( + Exception, + article_manager.add_document, + article_document + ) + + +@patch.object(DatabaseService, 'register') +def test_add_document_add_file_with_file_name_and_content( + mocked_register, + set_inmemory_article_manager, + test_package_A +): + xml_file = test_package_A[0] + article_document = ArticleDocument(xml_file.name) + article_document.add_version(xml_file.name, xml_file.content) + article_manager = set_inmemory_article_manager + with patch.object(article_manager.file_db_service, 'add_file') \ + as mocked_add_file: + article_manager.add_document(article_document) + mocked_add_file.assert_called_once_with( + file_id=article_document.xml_file_id, + content=article_document.xml_tree.content + ) + + +@patch.object(DatabaseService, 'add_file') +@patch.object(DatabaseService, 'register') +def test_add_document_updates_article_document_xml_file_id( + mocked_register, + mocked_add_file, + set_inmemory_article_manager, + test_package_A +): + xml_file = test_package_A[0] + article_document = ArticleDocument(xml_file.name) + article_document.add_version(xml_file.name, xml_file.content) + added_file_url = '/rawfile/' + article_document.xml_file_id + mocked_add_file.return_value = added_file_url + article_manager = set_inmemory_article_manager + article_manager.add_document(article_document) + assert article_document.versions[-1]['data'] == added_file_url + + +def test_add_document_article_get_record(set_inmemory_article_manager, + test_package_A): + xml_file = test_package_A[0] + article_document = ArticleDocument(xml_file.name) + article_document.add_version(xml_file.name, xml_file.content) + with patch.object(article_document, 'get_record'): + article_manager = set_inmemory_article_manager + article_manager.add_document(article_document) + article_document.get_record.assert_called_once() + + +@patch.object(DatabaseService, 'register', side_effect=Exception()) +def test_add_document_register_to_database_error(mocked_register, + set_inmemory_article_manager, + test_package_A): + xml_file = test_package_A[0] + article_document = ArticleDocument(xml_file.name) + article_document.add_version(xml_file.name, xml_file.content) + article_manager = set_inmemory_article_manager + pytest.raises( + Exception, + article_manager.add_document, + article_document + ) + + +@patch.object(DatabaseService, 'register') +def test_add_document_register_with_article_record( + mocked_register, + set_inmemory_article_manager, + test_package_A +): + xml_file = test_package_A[0] + fake_article_record = { + 'document_id': '1234', + 'content': b'acbdet' + } + article_document = ArticleDocument(xml_file.name) + article_document.add_version(xml_file.name, xml_file.content) + article_manager = set_inmemory_article_manager + with patch.object(article_document, 'get_record'): + article_document.get_record.return_value = fake_article_record + article_manager = set_inmemory_article_manager + article_manager.add_document(article_document) + mocked_register.assert_called_with(xml_file.name, fake_article_record) + + +def test_add_document_register_to_database_ok_returns_article_url( + set_inmemory_article_manager, + test_package_A +): + xml_file = test_package_A[0] + article_document = ArticleDocument('ID') + article_document.add_version(xml_file.name, xml_file.content) + article_manager = set_inmemory_article_manager + article_url = article_manager.add_document(article_document) + assert article_url is not None + assert article_url.endswith('/' + xml_file.name) + + @patch.object(DatabaseService, 'read') def test_get_article_in_database(mocked_dataservices_read, setup, - databaseservice_params, + set_inmemory_article_manager, inmemory_receive_package): article_id = 'ID' mocked_dataservices_read.return_value = {'document_id': article_id} - article_manager = ArticleManager( - databaseservice_params[0], - databaseservice_params[1] - ) + article_manager = set_inmemory_article_manager article_document = article_manager.get_article_document(article_id) + assert article_document is not None assert isinstance(article_document, ArticleDocument) mocked_dataservices_read.assert_called_with(article_id) @@ -76,15 +186,12 @@ def test_get_article_in_database(mocked_dataservices_read, @patch.object(DatabaseService, 'read', side_effect=DocumentNotFound) def test_get_article_in_database_not_found(mocked_dataservices_read, setup, - databaseservice_params, + set_inmemory_article_manager, inmemory_receive_package): article_id = 'ID' mocked_dataservices_read.return_value = {'document_id': article_id} - article_manager = ArticleManager( - databaseservice_params[0], - databaseservice_params[1] - ) + article_manager = set_inmemory_article_manager pytest.raises( ArticleManagerException, article_manager.get_article_document, @@ -95,15 +202,12 @@ def test_get_article_in_database_not_found(mocked_dataservices_read, @patch.object(DatabaseService, 'read', side_effect=DBFailed) def test_get_manifest_db_failed(mocked_dataservices_read, setup, - databaseservice_params, + set_inmemory_article_manager, inmemory_receive_package): article_id = 'ID' mocked_dataservices_read.return_value = {'document_id': article_id} - article_manager = ArticleManager( - databaseservice_params[0], - databaseservice_params[1] - ) + article_manager = set_inmemory_article_manager pytest.raises( DBFailed, article_manager.get_article_document, @@ -112,12 +216,9 @@ def test_get_manifest_db_failed(mocked_dataservices_read, def test_get_article_data(setup, - databaseservice_params, + set_inmemory_article_manager, inmemory_receive_package): - article_manager = ArticleManager( - databaseservice_params[0], - databaseservice_params[1] - ) + article_manager = set_inmemory_article_manager article_id = 'ID' article_document = article_manager.get_article_document(article_id) assert isinstance(article_document, ArticleDocument) @@ -128,16 +229,13 @@ def test_get_article_data(setup, @patch.object(DatabaseService, 'get_attachment') def test_get_article_file_in_database(mocked_get_attachment, setup, - databaseservice_params, + set_inmemory_article_manager, inmemory_receive_package, xml_test, test_packA_filenames): mocked_get_attachment.return_value = xml_test.encode('utf-8') article_id = 'ID' - article_manager = ArticleManager( - databaseservice_params[0], - databaseservice_params[1] - ) + article_manager = set_inmemory_article_manager article_manager.get_article_file(article_id) mocked_get_attachment.assert_called_with( document_id=article_id, @@ -148,12 +246,9 @@ def test_get_article_file_in_database(mocked_get_attachment, @patch.object(DatabaseService, 'get_attachment', side_effect=DocumentNotFound) def test_get_article_file_not_found(mocked_get_attachment, setup, - databaseservice_params, + set_inmemory_article_manager, inmemory_receive_package): - article_manager = ArticleManager( - databaseservice_params[0], - databaseservice_params[1] - ) + article_manager = set_inmemory_article_manager pytest.raises( ArticleManagerException, article_manager.get_article_file, @@ -162,13 +257,10 @@ def test_get_article_file_not_found(mocked_get_attachment, def test_get_article_file(setup, - databaseservice_params, + set_inmemory_article_manager, inmemory_receive_package, test_package_A): - article_manager = ArticleManager( - databaseservice_params[0], - databaseservice_params[1] - ) + article_manager = set_inmemory_article_manager article_document = article_manager.get_article_file('ID') assert article_document is not None xml_tree = XMLTree(test_package_A[0].content) @@ -178,12 +270,9 @@ def test_get_article_file(setup, @patch.object(DatabaseService, 'get_attachment', side_effect=DocumentNotFound) def test_get_asset_file_not_found(mocked_get_attachment, setup, - databaseservice_params, + set_inmemory_article_manager, inmemory_receive_package): - article_manager = ArticleManager( - databaseservice_params[0], - databaseservice_params[1] - ) + article_manager = set_inmemory_article_manager pytest.raises( ArticleManagerException, article_manager.get_asset_file, @@ -192,12 +281,10 @@ def test_get_asset_file_not_found(mocked_get_attachment, ) -def test_get_asset_file(databaseservice_params, +def test_get_asset_file(set_inmemory_article_manager, test_package_A, test_packA_filenames): - article_manager = ArticleManager( - databaseservice_params[0], - databaseservice_params[1]) + article_manager = set_inmemory_article_manager article_manager.receive_package(id='ID', xml_file=test_package_A[0], files=test_package_A[1:]) @@ -207,11 +294,9 @@ def test_get_asset_file(databaseservice_params, assert file.content == content -def test_get_asset_files(databaseservice_params, test_package_A): +def test_get_asset_files(set_inmemory_article_manager, test_package_A): files = test_package_A[1:] - article_manager = ArticleManager( - databaseservice_params[0], - databaseservice_params[1]) + article_manager = set_inmemory_article_manager article_manager.receive_package(id='ID', xml_file=test_package_A[0], files=test_package_A[1:]) diff --git a/managers/tests/test_article_model.py b/managers/tests/test_article_model.py index 93277c4..bc7df10 100644 --- a/managers/tests/test_article_model.py +++ b/managers/tests/test_article_model.py @@ -1,10 +1,61 @@ +import hashlib + +import pytest from managers.models.article_model import ( ArticleDocument, + InvalidXMLContent ) +from managers.xml.article_xml_tree import ArticleXMLTree + + +def test_article_document(test_package_A, test_packA_filenames): + article_document = ArticleDocument('ID') + assert article_document.id == 'ID' + assert article_document.versions == [] + assert article_document.get_record() == { + 'document_id': 'ID', + 'document_type': 'ART', + 'versions': [] + } + +def test_article_invalid_xml(): + article_document = ArticleDocument('ID') + with pytest.raises(InvalidXMLContent): + article_document.add_version( + 'file.xml', + b'
\n\n
' + ) -def article(test_package_A, test_packA_filenames): + +def test_article_document_add_version(test_package_A, test_packA_filenames): + xml_file = test_package_A[0] + checksum = hashlib.sha1( + ArticleXMLTree(xml_file.content).content).hexdigest() + filename = '/'.join([checksum[:13], xml_file.name]) + expected = { + 'document_id': 'ID', + 'document_type': 'ART', + 'versions': [ + { + 'data': filename, + 'assets': [] + } + ] + } + article_document = ArticleDocument('ID') + article_document.add_version(file_id=xml_file.name, + xml_content=xml_file.content) + assert article_document.xml_file_id == filename + assert article_document.xml_tree is not None + assert isinstance(article_document.xml_tree, ArticleXMLTree) + assert article_document.xml_tree.xml_error is None + assert article_document.xml_tree.compare(xml_file.content) + assert article_document.get_record() == expected + + +def test_article_update_asset_files(test_package_A, test_packA_filenames): article = ArticleDocument('ID') article.xml_file = test_package_A[0] article.update_asset_files(test_package_A[1:]) @@ -149,4 +200,3 @@ def test_assets_last_version(): article = ArticleDocument('ID') article.set_data(manifest) assert article.assets_last_version == expected - diff --git a/managers/tests/test_managers.py b/managers/tests/test_managers.py index 3915462..6f1dfb4 100644 --- a/managers/tests/test_managers.py +++ b/managers/tests/test_managers.py @@ -1,7 +1,8 @@ -import hashlib from unittest.mock import patch +import pytest + import managers from persistence.services import DatabaseService from managers.article_manager import ArticleManager @@ -26,15 +27,16 @@ def test_list_changes_return_from_changeservice_list_changes( assert changes == list_changes_expected -@patch.object(ArticleManager, 'add_document') -def test_post_article( - mocked_article_manager_add, - test_package_A, - database_config -): - checksum = hashlib.sha1(test_package_A[0].content).hexdigest() - expected = '/'.join([checksum[:13], test_package_A[0].name]) - db_settings = { +def test_create_file_succeeded(test_package_A): + xml_file = test_package_A[0] + file = managers.create_file(filename=xml_file.name, + content=xml_file.content) + assert file is not None + assert isinstance(file, managers.models.file.File) + + +def test_get_article_manager(database_config): + db_config = { 'database_uri': '{}:{}'.format( database_config['db_host'], database_config['db_port'] @@ -42,10 +44,75 @@ def test_post_article( 'database_username': database_config['username'], 'database_password': database_config['password'], } - result = managers.post_article( - xml_file=test_package_A[0], - **db_settings + article_manager = managers._get_article_manager( + **db_config + ) + assert article_manager is not None + assert isinstance(article_manager, managers.article_manager.ArticleManager) + assert article_manager.article_db_service is not None + assert isinstance(article_manager.article_db_service, DatabaseService) + assert article_manager.file_db_service is not None + assert isinstance(article_manager.file_db_service, DatabaseService) + + +@patch.object(managers, '_get_article_manager') +@patch.object(managers.models.article_model.ArticleDocument, 'add_version') +def test_post_article_file_error(mocked_article_add_version, + mocked__get_article_manager, + test_package_A, + set_inmemory_article_manager): + mocked__get_article_manager.return_value = set_inmemory_article_manager + error_msg = 'Invalid XML Content' + mocked_article_add_version.side_effect = \ + managers.models.article_model.InvalidXMLContent() + with pytest.raises(managers.exceptions.ManagerFileError) as excinfo: + managers.post_article( + article_id='ID-post-article-123', + xml_id=test_package_A[0].name, + xml_file=test_package_A[0].content, + **{} + ) + assert excinfo.value.message == error_msg + + +@patch.object(managers, '_get_article_manager') +@patch.object(ArticleManager, 'add_document') +def test_post_article_insert_file_to_database_error( + mocked_article_manager_add, + mocked__get_article_manager, + test_package_A, + set_inmemory_article_manager +): + mocked__get_article_manager.return_value = set_inmemory_article_manager + error_msg = 'Database Error' + mocked_article_manager_add.side_effect = \ + managers.article_manager.ArticleManagerException(message=error_msg) + + with pytest.raises(managers.article_manager.ArticleManagerException) \ + as excinfo: + managers.post_article( + article_id='ID-post-article-123', + xml_id=test_package_A[0].name, + xml_file=test_package_A[0].content, + **{} + ) + assert excinfo.value.message == error_msg + + +@patch.object(managers, '_get_article_manager') +def test_post_article_insert_file_to_database_ok( + mocked__get_article_manager, + test_package_A, + set_inmemory_article_manager +): + xml_file = test_package_A[0] + mocked__get_article_manager.return_value = set_inmemory_article_manager + + article_url = managers.post_article( + article_id='ID-post-article-123', + xml_id=xml_file.name, + xml_file=xml_file.content, + **{} ) - mocked_article_manager_add.assert_called_once() - assert result is not None - assert result == expected + assert article_url is not None + assert article_url.endswith(xml_file.name) diff --git a/managers/xml/article_xml_tree.py b/managers/xml/article_xml_tree.py index b2c8feb..6684d8c 100644 --- a/managers/xml/article_xml_tree.py +++ b/managers/xml/article_xml_tree.py @@ -1,4 +1,5 @@ # coding=utf-8 +import hashlib from .xml_tree import ( XMLTree, @@ -49,3 +50,7 @@ def nodes_which_has_xlink_href(self): if self.tree is not None: return self.tree.findall( './/*[@{http://www.w3.org/1999/xlink}href]') + + @property + def checksum(self): + return hashlib.sha1(self.content).hexdigest() diff --git a/persistence/databases.py b/persistence/databases.py index eff42e8..cd46188 100644 --- a/persistence/databases.py +++ b/persistence/databases.py @@ -97,11 +97,16 @@ def get_attachment_properties(self, id, file_id): doc = self.read(id) return doc.get(self._attachments_properties_key, {}).get(file_id) + @abc.abstractmethod + def insert_file(self, file_id, content) -> None: + return NotImplemented + class InMemoryDBManager(BaseDBManager): def __init__(self, **kwargs): self._database_name = kwargs['database_name'] + self._database_url = kwargs['database_uri'] self._attachments_key = 'attachments' self._attachments_properties_key = 'attachments_properties' self._database = {} @@ -237,15 +242,22 @@ def list_attachments(self, id): doc = self.read(id) return list(doc.get(self._attachments_key, {}).keys()) + def insert_file(self, file_id, content): + id = '/'.join([self._database_url, file_id]) + file = self.database.get(id) + if not file: + self.database.update({id: content}) + class CouchDBManager(BaseDBManager): def __init__(self, **kwargs): self._database_name = kwargs['database_name'] + self._database_url = kwargs['database_uri'] self._attachments_key = '_attachments' self._attachments_properties_key = 'attachments_properties' self._database = None - self._db_server = couchdb.Server(kwargs['database_uri']) + self._db_server = couchdb.Server(self._database_url) self._db_server.resource.credentials = ( kwargs['database_username'], kwargs['database_password'] @@ -393,6 +405,18 @@ def list_attachments(self, id): doc = self.read(id) return list(doc.get(self._attachments_key, {}).keys()) + def insert_file(self, file_id, content): + id = '/'.join([self._database_url, file_id]) + file = self.database.get(id) + if not file: + self.create(id=id, document={}) + doc = self.database.get(id) + self.database.put_attachment( + doc=doc, + content=content, + filename=id + ) + def sort_results(results, sort): scores = [list() for i in results] diff --git a/persistence/services.py b/persistence/services.py index 1b5a5e5..aceddc3 100644 --- a/persistence/services.py +++ b/persistence/services.py @@ -78,6 +78,20 @@ def register_change(self, ) return change_record['record_id'] + def register(self, record_id, change_type): + sequencial = self.seqnum_generator.new() + change_record = { + 'change_id': sequencial, + 'document_id': record_id, + 'type': change_type.value, + 'created_date': str(datetime.utcnow().timestamp()), + } + self.changes_db_manager.create( + str(sequencial), + change_record + ) + return sequencial + class DatabaseService: """ @@ -253,6 +267,19 @@ def get_attachment_properties(self, document_id, file_id): """ return self.db_manager.get_attachment_properties(document_id, file_id) + @REQUEST_TIME_ATT_UPD.time() + def add_file(self, file_id, content): + """ + Persiste conteúdo de arquivo na base de dados identificado com ID + informado. + + Params: + file_id: ID do arquivo + content: conteúdo do arquivo + """ + self.db_manager.insert_file(file_id=file_id, content=content) + self.changes_service.register(file_id, ChangeType.CREATE) + @REQUEST_TIME_CHANGES_LIST.time() def list_changes(self, last_sequence, limit): """ diff --git a/persistence/tests/conftest.py b/persistence/tests/conftest.py index df1e4bb..be68304 100644 --- a/persistence/tests/conftest.py +++ b/persistence/tests/conftest.py @@ -1,7 +1,6 @@ from datetime import datetime from random import randint -from pyramid import testing import pytest from persistence.databases import ( @@ -18,17 +17,6 @@ from persistence.seqnum_generator import SeqNumGenerator -@pytest.yield_fixture -def persistence_config(request): - yield testing.setUp() - testing.tearDown() - - -@pytest.fixture -def fake_change_list(): - return ['Test1', 'Test2', 'Test3', 'Test4', 'Test5', 'Test6'] - - @pytest.fixture def article_db_settings(): return { @@ -39,6 +27,11 @@ def article_db_settings(): } +@pytest.fixture +def article_dbinmemory_settings(): + return {'database_uri': '/rawfile', 'database_name': 'articles'} + + @pytest.fixture def change_db_settings(): return { @@ -76,47 +69,79 @@ def fin(): return s -@pytest.fixture(params=[ - CouchDBManager, - InMemoryDBManager -]) -def database_service(request, article_db_settings, change_db_settings, - seqnumber_generator): +@pytest.fixture( + params=[ + CouchDBManager, + InMemoryDBManager + ] +) +def database_service(request, + article_db_settings, + change_db_settings, + seqnum_db_settings): DBManager = request.param + s = SeqNumGenerator( + DBManager(**seqnum_db_settings), + 'CHANGE' + ) db_service = DatabaseService( DBManager(**article_db_settings), ChangesService( DBManager(**change_db_settings), - seqnumber_generator + s ) ) def fin(): - db_service.db_manager.drop_database() - db_service.changes_service.changes_db_manager.drop_database() + try: + db_service.db_manager.drop_database() + db_service.changes_service.changes_db_manager.drop_database() + s.db_manager.drop_database() + except Exception: + pass + request.addfinalizer(fin) return db_service @pytest.fixture -def inmemory_db_setup(request, persistence_config, change_db_settings): +def inmemory_db_setup(request): + db_host = '/rawfile' inmemory_db_service = DatabaseService( - InMemoryDBManager(database_name='articles'), + InMemoryDBManager(database_uri=db_host, database_name='articles'), ChangesService( - InMemoryDBManager(database_name='changes'), + InMemoryDBManager(database_uri=db_host, database_name='changes'), SeqNumGenerator( - InMemoryDBManager(database_name='seqnum'), + InMemoryDBManager(database_uri=db_host, + database_name='seqnum'), 'CHANGE' ) ) ) def fin(): - inmemory_db_service.changes_service.changes_db_manager.drop_database() + try: + inmemory_db_service.db_manager.drop_database() + inmemory_db_service.changes_service.changes_db_manager.\ + drop_database() + inmemory_db_service.changes_service.db_manager.drop_database() + except Exception: + pass request.addfinalizer(fin) return inmemory_db_service +@pytest.fixture( + params=[ + (CouchDBManager, article_db_settings), + (InMemoryDBManager, article_dbinmemory_settings) + ] +) +def db_manager_test(request): + db_settings = request.param[1]() + return request.param[0](**db_settings) + + @pytest.fixture def test_changes_records(request): changes_list = [] diff --git a/persistence/tests/test_changes.py b/persistence/tests/test_changes.py index 17b4732..f13fb0e 100644 --- a/persistence/tests/test_changes.py +++ b/persistence/tests/test_changes.py @@ -1,7 +1,8 @@ from datetime import datetime +from random import randint from uuid import uuid4 -from persistence.services import ChangeType +from persistence.services import ChangeType, SortOrder from persistence.models import get_record, RecordType @@ -13,7 +14,7 @@ def get_article_record(content={'Test': 'ChangeRecord'}): created_date=datetime.utcnow()) -def test_register_create_change(database_service): +def test_register_change_create(database_service): article_record = get_article_record() change_id = database_service.changes_service.register_change( article_record, @@ -21,7 +22,8 @@ def test_register_create_change(database_service): ) check_change = dict( - database_service.changes_service.changes_db_manager.database[change_id]) + database_service.changes_service.changes_db_manager.database[change_id] + ) assert check_change is not None assert check_change['document_id'] == article_record['document_id'] assert check_change['document_type'] == article_record['document_type'] @@ -29,6 +31,51 @@ def test_register_create_change(database_service): assert check_change['created_date'] is not None +def test_register_change(database_service): + document_id = 'ID-1234' + change_id = database_service.changes_service.register( + document_id, + ChangeType.CREATE + ) + assert change_id is not None + check_change = dict( + database_service.changes_service.changes_db_manager.database[ + str(change_id) + ] + ) + assert check_change is not None + assert check_change['document_id'] == document_id + assert check_change['type'] == ChangeType.CREATE.value + assert check_change['created_date'] is not None + + +def test_register_change_must_keep_sequential_order(database_service): + changes_service = database_service.changes_service + change_type_list = list(ChangeType) + + for counter in range(1, 11): + changes_service.register( + 'ID-{}'.format(counter), + change_type_list[randint(0, len(change_type_list)-1)] + ) + sort = [{'change_id': SortOrder.ASC.value}] + changes_list = changes_service.changes_db_manager.find(fields=[], + filter={}, + sort=sort) + + assert changes_list is not None + assert len(changes_list) == 10 + assert all([ + isinstance(change_list['change_id'], int) + for change_list in changes_list + ]) + # XXX: Usar sorted(list) ao invés de gerar uma lista ordenada + assert all([ + changes_list[i]['change_id'] < changes_list[i + 1]['change_id'] + for i in range(len(changes_list) - 1) + ]) + + def test_register_update_change(database_service): article_record = get_article_record({'Test': 'ChangeRecord2'}) change_id = database_service.changes_service.register_change( @@ -37,7 +84,8 @@ def test_register_update_change(database_service): ) check_change = dict( - database_service.changes_service.changes_db_manager.database[change_id]) + database_service.changes_service.changes_db_manager.database[change_id] + ) assert check_change is not None assert check_change['document_id'] == article_record['document_id'] assert check_change['document_type'] == article_record['document_type'] @@ -53,7 +101,8 @@ def test_register_delete_change(database_service): ) check_change = dict( - database_service.changes_service.changes_db_manager.database[change_id]) + database_service.changes_service.changes_db_manager.database[change_id] + ) assert check_change is not None assert check_change['document_id'] == article_record['document_id'] assert check_change['document_type'] == article_record['document_type'] @@ -84,7 +133,8 @@ def test_add_attachment_create_change(database_service, xml_test): ) check_change = dict( - database_service.changes_service.changes_db_manager.database[change_id]) + database_service.changes_service.changes_db_manager.database[change_id] + ) assert check_change is not None assert check_change['attachment_id'] == attachment_id assert check_change['document_id'] == article_record['document_id'] @@ -116,7 +166,8 @@ def test_update_attachment_create_change(database_service, xml_test): ) check_change = dict( - database_service.changes_service.changes_db_manager.database[change_id]) + database_service.changes_service.changes_db_manager.database[change_id] + ) assert check_change is not None assert check_change['attachment_id'] == attachment_id assert check_change['document_id'] == article_record['document_id'] diff --git a/persistence/tests/test_databases.py b/persistence/tests/test_databases.py index f3611c7..4d28dd9 100644 --- a/persistence/tests/test_databases.py +++ b/persistence/tests/test_databases.py @@ -503,3 +503,54 @@ def compare_documents(document, expected): assert document[k] == expected[k] else: assert document == expected + + +def test_add_file_insert_file_into_database_ok( + db_manager_test, + xml_test +): + db_manager_test.insert_file( + file_id='href_file1', + content=xml_test.encode('utf-8') + ) + file = db_manager_test.database.get( + '/'.join([db_manager_test._database_url, 'href_file1']) + ) + assert file is not None + + +def test_add_file_call_dbmanager_insert_file(database_service, xml_test): + with patch.object(database_service.db_manager, 'insert_file') \ + as mocked_insert_file: + database_service.add_file( + file_id='href_file1', + content=xml_test.encode('utf-8'), + ) + mocked_insert_file.assert_called_once_with( + file_id='href_file1', + content=xml_test.encode('utf-8') + ) + + +def test_add_file_call_changeservice_register(inmemory_db_setup, xml_test): + with patch.object(inmemory_db_setup.changes_service, 'register') \ + as mocked_change_register: + file_id = '/rawfile/href_file1' + inmemory_db_setup.add_file( + file_id=file_id, + content=xml_test.encode('utf-8'), + ) + mocked_change_register.assert_called_once_with( + file_id, ChangeType.CREATE + ) + + +def test_add_file_dbmanager_insert_into_database(inmemory_db_setup, xml_test): + inmemory_db_setup.add_file( + file_id='href_file1', + content=xml_test.encode('utf-8'), + ) + file = inmemory_db_setup.db_manager.database.get( + '/'.join([inmemory_db_setup.db_manager._database_url, 'href_file1']) + ) + assert file is not None diff --git a/persistence/tests/test_services.py b/persistence/tests/test_services.py index cdd2901..355f753 100644 --- a/persistence/tests/test_services.py +++ b/persistence/tests/test_services.py @@ -1,4 +1,4 @@ -from unittest.mock import Mock +from unittest.mock import patch from persistence.databases import QueryOperator from persistence.services import ChangeType, SortOrder @@ -9,31 +9,31 @@ def test_list_changes_calls_db_manager_find(inmemory_db_setup, xml_test): _changes_db_manager = inmemory_db_setup.changes_service.changes_db_manager - _changes_db_manager.find = Mock() - _changes_db_manager.find.return_value = [] - last_sequence = '123456' - limit = 10 - expected_fields = [ - 'change_id', - 'document_id', - 'document_type', - 'type', - 'created_date' - ] - filter = { - 'change_id': [ - (QueryOperator.GREATER_THAN, last_sequence) + with patch.object(_changes_db_manager, 'find'): + _changes_db_manager.find.return_value = [] + last_sequence = '123456' + limit = 10 + expected_fields = [ + 'change_id', + 'document_id', + 'document_type', + 'type', + 'created_date' ] - } - sort = [{'change_id': SortOrder.ASC.value}] - inmemory_db_setup.list_changes(last_sequence=last_sequence, - limit=limit) - _changes_db_manager.find.assert_called_once_with( - fields=expected_fields, - limit=limit, - filter=filter, - sort=sort - ) + filter = { + 'change_id': [ + (QueryOperator.GREATER_THAN, last_sequence) + ] + } + sort = [{'change_id': SortOrder.ASC.value}] + inmemory_db_setup.list_changes(last_sequence=last_sequence, + limit=limit) + _changes_db_manager.find.assert_called_once_with( + fields=expected_fields, + limit=limit, + filter=filter, + sort=sort + ) def test_list_changes_returns_db_manager_find_all(inmemory_db_setup, diff --git a/requirements.txt b/requirements.txt index 3496e77..7d6c6f4 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,6 +1,7 @@ cornice==3.4.0 cornice-swagger==0.6.0 CouchDB==1.2 +freezegun==0.3.10 gunicorn==19.7.1 lxml==4.2.1 Mako==1.0.7 diff --git a/setup.py b/setup.py index 04a75a7..376ca27 100644 --- a/setup.py +++ b/setup.py @@ -13,7 +13,13 @@ 'prometheus_client>=0.2.0', ] -test_requires = ['webtest', 'pytest', 'pytest-cov', 'pytest-lazy-fixture'] +test_requires = [ + 'webtest', + 'pytest', + 'pytest-cov', + 'pytest-lazy-fixture', + 'freezegun' +] setup_requires = ['pytest-runner'] setup( diff --git a/tests/test_functional.py b/tests/test_functional.py index f326383..9b4abf3 100644 --- a/tests/test_functional.py +++ b/tests/test_functional.py @@ -7,6 +7,46 @@ from lxml import etree +def test_post_article_register_change(testapp, test_package_A): + article_id = 'ID-post-article-123' + url = '/articles' + + # Um documento é registrado no módulo de persistencia. Ex: Artigo + xml_file_path = test_package_A[0] + article_url = os.path.basename(xml_file_path) + changes_expected = { + 'results': [{ + 'change_id': 1, + 'document_id': article_url, + 'type': 'CREATE' + }], + 'latest': 1 + } + params = OrderedDict([ + ("article_id", article_id), + ("xml_file", webtest.forms.Upload(xml_file_path)) + ]) + result = testapp.post(url, + params=params, + content_type='multipart/form-data') + assert result.status_code == 201 + assert result.json is not None + assert result.json.get('url').endswith(article_url) + + # Deve ser possível recuperar o registro de mudança do documento de + # acordo com os parâmetros informados no serviço + last_sequence = '' + limit = 10 + result = testapp.get('/changes?since={}&limit={}'.format(last_sequence, + limit)) + assert result.status_code == 200 + assert result.json is not None + assert len(result.json) > len(changes_expected['results']) + for resp_result, expected in zip(result.json, changes_expected['results']): + assert resp_result['document_id'].endswith(expected['document_id']) + assert resp_result['type'] == expected['type'] + + def test_add_article_register_change(testapp, test_package_A): article_id = 'ID-post-article-123' url = '/articles/{}'.format(article_id)