From 0bd53aed2c231edbc9b978c9cad936dac8082e22 Mon Sep 17 00:00:00 2001 From: Sylvain Zimmer Date: Thu, 24 Jul 2025 02:31:50 +0200 Subject: [PATCH 1/5] =?UTF-8?q?=E2=9C=A8(api)=20add=20API=20route=20to=20f?= =?UTF-8?q?etch=20document=20content?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This allows API users to process document content, enabling the use of Docs as a headless CMS for instance, or any kind of document processing. Fixes #1206. --- CHANGELOG.md | 1 + docker/auth/realm.json | 2 +- src/backend/core/api/serializers.py | 6 +- src/backend/core/api/viewsets.py | 64 ++++++ src/backend/core/models.py | 1 + .../core/services/converter_services.py | 53 ----- .../core/services/yprovider_services.py | 84 +++++++ .../documents/test_api_documents_content.py | 161 ++++++++++++++ .../test_api_documents_create_for_owner.py | 6 +- .../documents/test_api_documents_retrieve.py | 5 + .../documents/test_api_documents_trashbin.py | 1 + .../core/tests/test_models_documents.py | 8 + ...py => test_services_yprovider_services.py} | 82 +++++-- .../y-provider/__tests__/content.test.ts | 207 ++++++++++++++++++ src/frontend/servers/y-provider/package.json | 2 +- .../y-provider/src/handlers/contentHandler.ts | 71 ++++++ .../servers/y-provider/src/handlers/index.ts | 1 + src/frontend/servers/y-provider/src/routes.ts | 1 + .../y-provider/src/servers/appServer.ts | 6 + 19 files changed, 688 insertions(+), 74 deletions(-) delete mode 100644 src/backend/core/services/converter_services.py create mode 100644 src/backend/core/services/yprovider_services.py create mode 100644 src/backend/core/tests/documents/test_api_documents_content.py rename src/backend/core/tests/{test_services_converter_services.py => test_services_yprovider_services.py} (54%) create mode 100644 src/frontend/servers/y-provider/__tests__/content.test.ts create mode 100644 src/frontend/servers/y-provider/src/handlers/contentHandler.ts diff --git a/CHANGELOG.md b/CHANGELOG.md index 8d16acc8a4..c4db3af158 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,7 @@ and this project adheres to ### Added - ✨(frontend) add duplicate action to doc tree #1175 +- ✨(api) add API route to fetch document content #1206 ### Changed diff --git a/docker/auth/realm.json b/docker/auth/realm.json index db5f1be021..06ab6e55e4 100644 --- a/docker/auth/realm.json +++ b/docker/auth/realm.json @@ -26,7 +26,7 @@ "oauth2DeviceCodeLifespan": 600, "oauth2DevicePollingInterval": 5, "enabled": true, - "sslRequired": "external", + "sslRequired": "none", "registrationAllowed": true, "registrationEmailAsUsername": false, "rememberMe": true, diff --git a/src/backend/core/api/serializers.py b/src/backend/core/api/serializers.py index 83afc260d9..572b844b46 100644 --- a/src/backend/core/api/serializers.py +++ b/src/backend/core/api/serializers.py @@ -14,9 +14,9 @@ from core import choices, enums, models, utils from core.services.ai_services import AI_ACTIONS -from core.services.converter_services import ( +from core.services.yprovider_services import ( ConversionError, - YdocConverter, + YProviderAPI, ) @@ -431,7 +431,7 @@ def create(self, validated_data): language = user.language or language try: - document_content = YdocConverter().convert(validated_data["content"]) + document_content = YProviderAPI().convert(validated_data["content"]) except ConversionError as err: raise serializers.ValidationError( {"content": ["Could not convert content"]} diff --git a/src/backend/core/api/viewsets.py b/src/backend/core/api/viewsets.py index 7919033e16..5af4d02676 100644 --- a/src/backend/core/api/viewsets.py +++ b/src/backend/core/api/viewsets.py @@ -37,6 +37,15 @@ from core import authentication, choices, enums, models from core.services.ai_services import AIService from core.services.collaboration_services import CollaborationService +from core.services.yprovider_services import ( + ServiceUnavailableError as YProviderServiceUnavailableError, +) +from core.services.yprovider_services import ( + ValidationError as YProviderValidationError, +) +from core.services.yprovider_services import ( + YProviderAPI, +) from core.tasks.mail import send_ask_for_access_mail from core.utils import extract_attachments, filter_descendants @@ -1443,6 +1452,61 @@ def cors_proxy(self, request, *args, **kwargs): status=status.HTTP_500_INTERNAL_SERVER_ERROR, ) + @drf.decorators.action( + detail=True, + methods=["get"], + url_path="content", + name="Get document content in different formats", + ) + def content(self, request, pk=None): + """ + Retrieve document content in different formats (JSON, Markdown, HTML). + + Query parameters: + - content_format: The desired output format (json, markdown, html) + + Returns: + JSON response with content in the specified format. + """ + + document = self.get_object() + + content_format = request.query_params.get("content_format", "json").lower() + if content_format not in {"json", "markdown", "html"}: + raise drf.exceptions.ValidationError( + "Invalid format. Must be one of: json, markdown, html" + ) + + # Get the base64 content from the document + content = None + base64_content = document.content + if base64_content is not None: + # Convert using the y-provider service + try: + yprovider = YProviderAPI() + result = yprovider.content(base64_content, content_format) + content = result["content"] + except YProviderValidationError as e: + return drf_response.Response( + {"error": str(e)}, status=status.HTTP_400_BAD_REQUEST + ) + except YProviderServiceUnavailableError as e: + logger.error("Error getting content for document %s: %s", pk, e) + return drf_response.Response( + {"error": "Failed to get document content"}, + status=status.HTTP_500_INTERNAL_SERVER_ERROR, + ) + + return drf_response.Response( + { + "id": str(document.id), + "title": document.title, + "content": content, + "created_at": document.created_at, + "updated_at": document.updated_at, + } + ) + class DocumentAccessViewSet( ResourceAccessViewsetMixin, diff --git a/src/backend/core/models.py b/src/backend/core/models.py index d6e79c5100..1098bbe940 100644 --- a/src/backend/core/models.py +++ b/src/backend/core/models.py @@ -786,6 +786,7 @@ def get_abilities(self, user): "children_list": can_get, "children_create": can_update and user.is_authenticated, "collaboration_auth": can_get, + "content": can_get, "cors_proxy": can_get, "descendants": can_get, "destroy": is_owner, diff --git a/src/backend/core/services/converter_services.py b/src/backend/core/services/converter_services.py deleted file mode 100644 index d6a6dbf4b1..0000000000 --- a/src/backend/core/services/converter_services.py +++ /dev/null @@ -1,53 +0,0 @@ -"""Converter services.""" - -from base64 import b64encode - -from django.conf import settings - -import requests - - -class ConversionError(Exception): - """Base exception for conversion-related errors.""" - - -class ValidationError(ConversionError): - """Raised when the input validation fails.""" - - -class ServiceUnavailableError(ConversionError): - """Raised when the conversion service is unavailable.""" - - -class YdocConverter: - """Service class for conversion-related operations.""" - - @property - def auth_header(self): - """Build microservice authentication header.""" - # Note: Yprovider microservice accepts only raw token, which is not recommended - return f"Bearer {settings.Y_PROVIDER_API_KEY}" - - def convert(self, text): - """Convert a Markdown text into our internal format using an external microservice.""" - - if not text: - raise ValidationError("Input text cannot be empty") - - try: - response = requests.post( - f"{settings.Y_PROVIDER_API_BASE_URL}{settings.CONVERSION_API_ENDPOINT}/", - data=text, - headers={ - "Authorization": self.auth_header, - "Content-Type": "text/markdown", - }, - timeout=settings.CONVERSION_API_TIMEOUT, - verify=settings.CONVERSION_API_SECURE, - ) - response.raise_for_status() - return b64encode(response.content).decode("utf-8") - except requests.RequestException as err: - raise ServiceUnavailableError( - "Failed to connect to conversion service", - ) from err diff --git a/src/backend/core/services/yprovider_services.py b/src/backend/core/services/yprovider_services.py new file mode 100644 index 0000000000..16ffed7d96 --- /dev/null +++ b/src/backend/core/services/yprovider_services.py @@ -0,0 +1,84 @@ +"""Y-Provider API services.""" + +import json +import logging +from base64 import b64encode + +from django.conf import settings + +import requests + +logger = logging.getLogger(__name__) + + +class ConversionError(Exception): + """Base exception for conversion-related errors.""" + + +class ValidationError(ConversionError): + """Raised when the input validation fails.""" + + +class ServiceUnavailableError(ConversionError): + """Raised when the conversion service is unavailable.""" + + +class YProviderAPI: + """Service class for Y-Provider API operations.""" + + @property + def auth_header(self): + """Build microservice authentication header.""" + # Note: Yprovider microservice accepts only raw token, which is not recommended + return f"Bearer {settings.Y_PROVIDER_API_KEY}" + + def _request(self, url, data, content_type): + """Make a request to the Y-Provider API.""" + response = requests.post( + url, + data=data, + headers={ + "Authorization": self.auth_header, + "Content-Type": content_type, + }, + timeout=settings.CONVERSION_API_TIMEOUT, + verify=settings.CONVERSION_API_SECURE, + ) + response.raise_for_status() + return response + + def convert(self, text): + """Convert a Markdown text into our internal format using an external microservice.""" + + if not text: + raise ValidationError("Input text cannot be empty") + + try: + response = self._request( + f"{settings.Y_PROVIDER_API_BASE_URL}{settings.CONVERSION_API_ENDPOINT}/", + text, + "text/markdown", + ) + return b64encode(response.content).decode("utf-8") + except requests.RequestException as err: + raise ServiceUnavailableError( + "Failed to connect to backend service", + ) from err + + def content(self, base64_content, format_type): + """Convert base64 Yjs content to different formats using the y-provider service.""" + + if not base64_content: + raise ValidationError("Input content cannot be empty") + + data = json.dumps({"content": base64_content, "format": format_type}) + logger.warning(f"{settings.Y_PROVIDER_API_BASE_URL}api/content/") + try: + response = self._request( + f"{settings.Y_PROVIDER_API_BASE_URL}content/", data, "application/json" + ) + return response.json() + except requests.RequestException as err: + raise ServiceUnavailableError( + "Failed to connect to backend service", + ) from err diff --git a/src/backend/core/tests/documents/test_api_documents_content.py b/src/backend/core/tests/documents/test_api_documents_content.py new file mode 100644 index 0000000000..cabb2fc65e --- /dev/null +++ b/src/backend/core/tests/documents/test_api_documents_content.py @@ -0,0 +1,161 @@ +""" +Tests for Documents API endpoint in impress's core app: content +""" + +from unittest.mock import patch + +import pytest +import requests +from rest_framework import status +from rest_framework.test import APIClient + +from core import choices, factories + +pytestmark = pytest.mark.django_db + + +@pytest.mark.parametrize( + "reach, role", + [ + ("public", "reader"), + ("public", "editor"), + ], +) +@patch("core.services.yprovider_services.YProviderAPI.content") +def test_api_documents_content_public(mock_content, reach, role): + """Anonymous users should be allowed to access content of public documents.""" + document = factories.DocumentFactory(link_reach=reach, link_role=role) + mock_content.return_value = {"content": {"some": "data"}} + + response = APIClient().get(f"/api/v1.0/documents/{document.id!s}/content/") + + assert response.status_code == status.HTTP_200_OK + data = response.json() + assert data["id"] == str(document.id) + assert data["title"] == document.title + assert data["content"] == {"some": "data"} + mock_content.assert_called_once_with(document.content, "json") + + +@pytest.mark.parametrize( + "reach, doc_role, user_role", + [ + ("restricted", "reader", "reader"), + ("restricted", "reader", "editor"), + ("restricted", "reader", "administrator"), + ("restricted", "reader", "owner"), + ("restricted", "editor", "reader"), + ("restricted", "editor", "editor"), + ("restricted", "editor", "administrator"), + ("restricted", "editor", "owner"), + ("authenticated", "reader", None), + ("authenticated", "editor", None), + ], +) +@patch("core.services.yprovider_services.YProviderAPI.content") +def test_api_documents_content_not_public(mock_content, reach, doc_role, user_role): + """Authenticated users need access to get non-public document content.""" + user = factories.UserFactory() + document = factories.DocumentFactory(link_reach=reach, link_role=doc_role) + mock_content.return_value = {"content": {"some": "data"}} + + # First anonymous request should fail + client = APIClient() + response = client.get(f"/api/v1.0/documents/{document.id!s}/content/") + + assert response.status_code == status.HTTP_401_UNAUTHORIZED + mock_content.assert_not_called() + + # Login and try again + client.force_login(user) + response = client.get(f"/api/v1.0/documents/{document.id!s}/content/") + + # If restricted, we still should not have access + if user_role is not None: + assert response.status_code == status.HTTP_403_FORBIDDEN + mock_content.assert_not_called() + + # Create an access as a reader. This should unlock the access. + factories.UserDocumentAccessFactory( + document=document, user=user, role=user_role + ) + + response = client.get(f"/api/v1.0/documents/{document.id!s}/content/") + + assert response.status_code == status.HTTP_200_OK + data = response.json() + assert data["id"] == str(document.id) + assert data["title"] == document.title + assert data["content"] == {"some": "data"} + mock_content.assert_called_once_with(document.content, "json") + + +@pytest.mark.parametrize( + "content_format", + ["markdown", "html", "json"], +) +@patch("core.services.yprovider_services.YProviderAPI.content") +def test_api_documents_content_format(mock_content, content_format): + """Test that the content endpoint returns a specific format.""" + document = factories.DocumentFactory(link_reach="public") + mock_content.return_value = {"content": "whatever"} + + response = APIClient().get( + f"/api/v1.0/documents/{document.id!s}/content/?content_format={content_format}" + ) + + assert response.status_code == status.HTTP_200_OK + data = response.json() + assert data["id"] == str(document.id) + assert data["title"] == document.title + assert data["content"] == "whatever" + mock_content.assert_called_once_with(document.content, content_format) + + +@patch("core.services.yprovider_services.YProviderAPI._request") +def test_api_documents_content_invalid_format(mock_request): + """Test that the content endpoint rejects invalid formats.""" + document = factories.DocumentFactory(link_reach="public") + + response = APIClient().get( + f"/api/v1.0/documents/{document.id!s}/content/?content_format=invalid" + ) + assert response.status_code == status.HTTP_400_BAD_REQUEST + mock_request.assert_not_called() + + +@patch("core.services.yprovider_services.YProviderAPI._request") +def test_api_documents_content_yservice_error(mock_request): + """Test that service errors are handled properly.""" + document = factories.DocumentFactory(link_reach="public") + mock_request.side_effect = requests.RequestException() + + response = APIClient().get(f"/api/v1.0/documents/{document.id!s}/content/") + mock_request.assert_called_once() + assert response.status_code == status.HTTP_500_INTERNAL_SERVER_ERROR + + +@patch("core.services.yprovider_services.YProviderAPI._request") +def test_api_documents_content_nonexistent_document(mock_request): + """Test that accessing a nonexistent document returns 404.""" + client = APIClient() + response = client.get( + "/api/v1.0/documents/00000000-0000-0000-0000-000000000000/content/" + ) + assert response.status_code == status.HTTP_404_NOT_FOUND + mock_request.assert_not_called() + + +@patch("core.services.yprovider_services.YProviderAPI._request") +def test_api_documents_content_empty_document(mock_request): + """Test that accessing an empty document returns empty content.""" + document = factories.DocumentFactory(link_reach="public", content="") + + response = APIClient().get(f"/api/v1.0/documents/{document.id!s}/content/") + + assert response.status_code == status.HTTP_200_OK + data = response.json() + assert data["id"] == str(document.id) + assert data["title"] == document.title + assert data["content"] is None + mock_request.assert_not_called() diff --git a/src/backend/core/tests/documents/test_api_documents_create_for_owner.py b/src/backend/core/tests/documents/test_api_documents_create_for_owner.py index cbe35b997e..a28fbecabd 100644 --- a/src/backend/core/tests/documents/test_api_documents_create_for_owner.py +++ b/src/backend/core/tests/documents/test_api_documents_create_for_owner.py @@ -16,16 +16,16 @@ from core import factories from core.api.serializers import ServerCreateDocumentSerializer from core.models import Document, Invitation, User -from core.services.converter_services import ConversionError, YdocConverter +from core.services.yprovider_services import ConversionError, YProviderAPI pytestmark = pytest.mark.django_db @pytest.fixture def mock_convert_md(): - """Mock YdocConverter.convert to return a converted content.""" + """Mock YProviderAPI.convert to return a converted content.""" with patch.object( - YdocConverter, + YProviderAPI, "convert", return_value="Converted document content", ) as mock: diff --git a/src/backend/core/tests/documents/test_api_documents_retrieve.py b/src/backend/core/tests/documents/test_api_documents_retrieve.py index b229adb5c9..63880f2e69 100644 --- a/src/backend/core/tests/documents/test_api_documents_retrieve.py +++ b/src/backend/core/tests/documents/test_api_documents_retrieve.py @@ -37,6 +37,7 @@ def test_api_documents_retrieve_anonymous_public_standalone(): "children_list": True, "collaboration_auth": True, "cors_proxy": True, + "content": True, "descendants": True, "destroy": False, "duplicate": False, @@ -112,6 +113,7 @@ def test_api_documents_retrieve_anonymous_public_parent(): "collaboration_auth": True, "descendants": True, "cors_proxy": True, + "content": True, "destroy": False, "duplicate": False, # Anonymous user can't favorite a document even with read access @@ -216,6 +218,7 @@ def test_api_documents_retrieve_authenticated_unrelated_public_or_authenticated( "collaboration_auth": True, "descendants": True, "cors_proxy": True, + "content": True, "destroy": False, "duplicate": True, "favorite": True, @@ -297,6 +300,7 @@ def test_api_documents_retrieve_authenticated_public_or_authenticated_parent(rea "collaboration_auth": True, "descendants": True, "cors_proxy": True, + "content": True, "destroy": False, "duplicate": True, "favorite": True, @@ -490,6 +494,7 @@ def test_api_documents_retrieve_authenticated_related_parent(): "collaboration_auth": True, "descendants": True, "cors_proxy": True, + "content": True, "destroy": access.role == "owner", "duplicate": True, "favorite": True, diff --git a/src/backend/core/tests/documents/test_api_documents_trashbin.py b/src/backend/core/tests/documents/test_api_documents_trashbin.py index 9e80539777..66a023fdbe 100644 --- a/src/backend/core/tests/documents/test_api_documents_trashbin.py +++ b/src/backend/core/tests/documents/test_api_documents_trashbin.py @@ -81,6 +81,7 @@ def test_api_documents_trashbin_format(): "collaboration_auth": True, "descendants": True, "cors_proxy": True, + "content": True, "destroy": True, "duplicate": True, "favorite": True, diff --git a/src/backend/core/tests/test_models_documents.py b/src/backend/core/tests/test_models_documents.py index 57427fb9c8..72d9aa574f 100644 --- a/src/backend/core/tests/test_models_documents.py +++ b/src/backend/core/tests/test_models_documents.py @@ -161,6 +161,7 @@ def test_models_documents_get_abilities_forbidden( "collaboration_auth": False, "descendants": False, "cors_proxy": False, + "content": False, "destroy": False, "duplicate": False, "favorite": False, @@ -223,6 +224,7 @@ def test_models_documents_get_abilities_reader( "collaboration_auth": True, "descendants": True, "cors_proxy": True, + "content": True, "destroy": False, "duplicate": is_authenticated, "favorite": is_authenticated, @@ -287,6 +289,7 @@ def test_models_documents_get_abilities_editor( "collaboration_auth": True, "descendants": True, "cors_proxy": True, + "content": True, "destroy": False, "duplicate": is_authenticated, "favorite": is_authenticated, @@ -340,6 +343,7 @@ def test_models_documents_get_abilities_owner(django_assert_num_queries): "collaboration_auth": True, "descendants": True, "cors_proxy": True, + "content": True, "destroy": True, "duplicate": True, "favorite": True, @@ -390,6 +394,7 @@ def test_models_documents_get_abilities_administrator(django_assert_num_queries) "collaboration_auth": True, "descendants": True, "cors_proxy": True, + "content": True, "destroy": False, "duplicate": True, "favorite": True, @@ -443,6 +448,7 @@ def test_models_documents_get_abilities_editor_user(django_assert_num_queries): "collaboration_auth": True, "descendants": True, "cors_proxy": True, + "content": True, "destroy": False, "duplicate": True, "favorite": True, @@ -503,6 +509,7 @@ def test_models_documents_get_abilities_reader_user( "collaboration_auth": True, "descendants": True, "cors_proxy": True, + "content": True, "destroy": False, "duplicate": True, "favorite": True, @@ -561,6 +568,7 @@ def test_models_documents_get_abilities_preset_role(django_assert_num_queries): "collaboration_auth": True, "descendants": True, "cors_proxy": True, + "content": True, "destroy": False, "duplicate": True, "favorite": True, diff --git a/src/backend/core/tests/test_services_converter_services.py b/src/backend/core/tests/test_services_yprovider_services.py similarity index 54% rename from src/backend/core/tests/test_services_converter_services.py rename to src/backend/core/tests/test_services_yprovider_services.py index 01773f8570..603c22072a 100644 --- a/src/backend/core/tests/test_services_converter_services.py +++ b/src/backend/core/tests/test_services_yprovider_services.py @@ -1,28 +1,29 @@ -"""Test converter services.""" +"""Test y-provider services.""" +import json from base64 import b64decode from unittest.mock import MagicMock, patch import pytest import requests -from core.services.converter_services import ( +from core.services.yprovider_services import ( ServiceUnavailableError, ValidationError, - YdocConverter, + YProviderAPI, ) def test_auth_header(settings): """Test authentication header generation.""" settings.Y_PROVIDER_API_KEY = "test-key" - converter = YdocConverter() + converter = YProviderAPI() assert converter.auth_header == "Bearer test-key" def test_convert_empty_text(): """Should raise ValidationError when text is empty.""" - converter = YdocConverter() + converter = YProviderAPI() with pytest.raises(ValidationError, match="Input text cannot be empty"): converter.convert("") @@ -30,13 +31,13 @@ def test_convert_empty_text(): @patch("requests.post") def test_convert_service_unavailable(mock_post): """Should raise ServiceUnavailableError when service is unavailable.""" - converter = YdocConverter() + converter = YProviderAPI() mock_post.side_effect = requests.RequestException("Connection error") with pytest.raises( ServiceUnavailableError, - match="Failed to connect to conversion service", + match="Failed to connect to backend service", ): converter.convert("test text") @@ -44,7 +45,7 @@ def test_convert_service_unavailable(mock_post): @patch("requests.post") def test_convert_http_error(mock_post): """Should raise ServiceUnavailableError when HTTP error occurs.""" - converter = YdocConverter() + converter = YProviderAPI() mock_response = MagicMock() mock_response.raise_for_status.side_effect = requests.HTTPError("HTTP Error") @@ -52,7 +53,7 @@ def test_convert_http_error(mock_post): with pytest.raises( ServiceUnavailableError, - match="Failed to connect to conversion service", + match="Failed to connect to backend service", ): converter.convert("test text") @@ -67,7 +68,7 @@ def test_convert_full_integration(mock_post, settings): settings.CONVERSION_API_TIMEOUT = 5 settings.CONVERSION_API_CONTENT_FIELD = "content" - converter = YdocConverter() + converter = YProviderAPI() expected_content = b"converted content" mock_response = MagicMock() @@ -93,20 +94,75 @@ def test_convert_full_integration(mock_post, settings): @patch("requests.post") def test_convert_timeout(mock_post): """Should raise ServiceUnavailableError when request times out.""" - converter = YdocConverter() + converter = YProviderAPI() mock_post.side_effect = requests.Timeout("Request timed out") with pytest.raises( ServiceUnavailableError, - match="Failed to connect to conversion service", + match="Failed to connect to backend service", ): converter.convert("test text") def test_convert_none_input(): """Should raise ValidationError when input is None.""" - converter = YdocConverter() + converter = YProviderAPI() with pytest.raises(ValidationError, match="Input text cannot be empty"): converter.convert(None) + + +def test_content_empty_content(): + """Should raise ValidationError when content is empty.""" + converter = YProviderAPI() + with pytest.raises(ValidationError, match="Input content cannot be empty"): + converter.content("", "markdown") + + +@patch("requests.post") +def test_content_service_unavailable(mock_post): + """Should raise ServiceUnavailableError when service is unavailable.""" + converter = YProviderAPI() + + mock_post.side_effect = requests.RequestException("Connection error") + + with pytest.raises( + ServiceUnavailableError, + match="Failed to connect to backend service", + ): + converter.content("test_content", "markdown") + + +@patch("requests.post") +def test_content_success(mock_post, settings): + """Test successful content fetch.""" + settings.Y_PROVIDER_API_BASE_URL = "http://test.com/api/" + settings.Y_PROVIDER_API_KEY = "test-key" + settings.CONVERSION_API_TIMEOUT = 5 + settings.CONVERSION_API_SECURE = False + + converter = YProviderAPI() + + expected_response = { + "content": "# Test Document\n\nThis is test content.", + "format": "markdown", + } + mock_response = MagicMock() + mock_response.json.return_value = expected_response + mock_response.raise_for_status.return_value = None + mock_post.return_value = mock_response + + result = converter.content("test_content", "markdown") + + assert result == expected_response + mock_post.assert_called_once_with( + "http://test.com/api/content/", + data=json.dumps({"content": "test_content", "format": "markdown"}), + headers={ + "Authorization": "Bearer test-key", + "Content-Type": "application/json", + }, + timeout=5, + verify=False, + ) diff --git a/src/frontend/servers/y-provider/__tests__/content.test.ts b/src/frontend/servers/y-provider/__tests__/content.test.ts new file mode 100644 index 0000000000..555b453ae4 --- /dev/null +++ b/src/frontend/servers/y-provider/__tests__/content.test.ts @@ -0,0 +1,207 @@ +import { ServerBlockNoteEditor } from '@blocknote/server-util'; +import request from 'supertest'; +import { describe, expect, test, vi } from 'vitest'; +import * as Y from 'yjs'; + +vi.mock('../src/env', async (importOriginal) => { + return { + ...(await importOriginal()), + COLLABORATION_SERVER_ORIGIN: 'http://localhost:3000', + Y_PROVIDER_API_KEY: 'yprovider-api-key', + }; +}); + +import { initApp } from '@/servers'; + +import { + Y_PROVIDER_API_KEY as apiKey, + COLLABORATION_SERVER_ORIGIN as origin, +} from '../src/env'; + +console.error = vi.fn(); + +describe('Content API Tests', () => { + test('POST /api/content with incorrect API key responds with 401', async () => { + const app = initApp(); + + const response = await request(app) + .post('/api/content') + .set('origin', origin) + .set('authorization', 'wrong-api-key') + .set('content-type', 'application/json') + .send({ + content: 'dGVzdA==', // base64 for "test" + format: 'json', + }); + + expect(response.status).toBe(401); + expect(response.body).toStrictEqual({ + error: 'Unauthorized: Invalid API Key', + }); + }); + + test('POST /api/content with incorrect Bearer token responds with 401', async () => { + const app = initApp(); + + const response = await request(app) + .post('/api/content') + .set('origin', origin) + .set('authorization', 'Bearer test-secret-api-key') + .set('content-type', 'application/json') + .send({ + content: 'dGVzdA==', // base64 for "test" + format: 'json', + }); + + expect(response.status).toBe(401); + expect(response.body).toStrictEqual({ + error: 'Unauthorized: Invalid API Key', + }); + }); + + test('POST /api/content with missing content parameter', async () => { + const app = initApp(); + + const response = await request(app) + .post('/api/content') + .set('origin', origin) + .set('authorization', apiKey) + .set('content-type', 'application/json') + .send({ + format: 'json', + }); + + expect(response.status).toBe(400); + expect(response.body).toStrictEqual({ + error: 'Invalid request: missing content', + }); + }); + + test('POST /api/content with empty content', async () => { + const app = initApp(); + + const response = await request(app) + .post('/api/content') + .set('origin', origin) + .set('authorization', apiKey) + .set('content-type', 'application/json') + .send({ + content: '', + format: 'json', + }); + + expect(response.status).toBe(400); + expect(response.body).toStrictEqual({ + error: 'Invalid request: missing content', + }); + }); + + test('POST /api/content with missing format parameter', async () => { + const app = initApp(); + + const response = await request(app) + .post('/api/content') + .set('origin', origin) + .set('authorization', apiKey) + .set('content-type', 'application/json') + .send({ + content: 'dGVzdA==', + }); + + expect(response.status).toBe(400); + expect(response.body).toStrictEqual({ + error: 'Invalid format. Must be one of: json, markdown, html', + }); + }); + + test('POST /api/content with invalid format', async () => { + const app = initApp(); + + const response = await request(app) + .post('/api/content') + .set('origin', origin) + .set('authorization', apiKey) + .set('content-type', 'application/json') + .send({ + content: 'dGVzdA==', + format: 'invalid', + }); + + expect(response.status).toBe(400); + expect(response.body).toStrictEqual({ + error: 'Invalid format. Must be one of: json, markdown, html', + }); + }); + + test.each([ + { authHeader: `Bearer ${apiKey}`, format: 'json' }, + { authHeader: `Bearer ${apiKey}`, format: 'markdown' }, + { authHeader: `Bearer ${apiKey}`, format: 'html' }, + ])( + 'POST /api/content with correct content and format $format with Authorization: $authHeader', + async ({ authHeader, format }) => { + const app = initApp(); + + // Create a simple Yjs document for testing using BlockNote + const editor = ServerBlockNoteEditor.create(); + const markdownContent = '# Test Document\n\nThis is test content.'; + const blocks = await editor.tryParseMarkdownToBlocks(markdownContent); + const yDocument = editor.blocksToYDoc(blocks, 'document-store'); + const yjsUpdate = Y.encodeStateAsUpdate(yDocument); + const base64Content = Buffer.from(yjsUpdate).toString('base64'); + + const response = await request(app) + .post('/api/content') + .set('Origin', origin) + .set('Authorization', authHeader) + .set('content-type', 'application/json') + .send({ + content: base64Content, + format: format, + }); + expect(response.status).toBe(200); + expect(response.body).toHaveProperty('content'); + expect(response.body).toHaveProperty('format', format); + + // Verify the content based on format + if (format === 'json') { + const parsedContent = response.body.content; + expect(Array.isArray(parsedContent)).toBe(true); + expect(parsedContent.length).toBe(2); + expect(parsedContent[0].type).toBe('heading'); + expect(parsedContent[1].type).toBe('paragraph'); + expect(parsedContent[0].content[0].type).toBe('text'); + expect(parsedContent[0].content[0].text).toBe('Test Document'); + expect(parsedContent[1].content[0].type).toBe('text'); + expect(parsedContent[1].content[0].text).toBe('This is test content.'); + } else if (format === 'markdown') { + expect(typeof response.body.content).toBe('string'); + expect(response.body.content.trim()).toBe(markdownContent); + } else if (format === 'html') { + expect(typeof response.body.content).toBe('string'); + expect(response.body.content).toBe( + '

Test Document

This is test content.

', + ); + } + }, + ); + + test('POST /api/content with invalid base64 content returns 500', async () => { + const app = initApp(); + + const response = await request(app) + .post('/api/content') + .set('origin', origin) + .set('authorization', apiKey) + .set('content-type', 'application/json') + .send({ + content: 'invalid-base64-content!@#', + format: 'json', + }); + + expect(response.status).toBe(500); + expect(response.body).toStrictEqual({ + error: 'An error occurred during conversion', + }); + }); +}); diff --git a/src/frontend/servers/y-provider/package.json b/src/frontend/servers/y-provider/package.json index 7364781bfc..6a89231815 100644 --- a/src/frontend/servers/y-provider/package.json +++ b/src/frontend/servers/y-provider/package.json @@ -10,7 +10,7 @@ "dev": "cross-env COLLABORATION_LOGGING=true && nodemon --config nodemon.json", "start": "node ./dist/start-server.js", "lint": "eslint . --ext .ts", - "test": "vitest --run" + "test": "vitest --run --disable-console-intercept" }, "engines": { "node": ">=22" diff --git a/src/frontend/servers/y-provider/src/handlers/contentHandler.ts b/src/frontend/servers/y-provider/src/handlers/contentHandler.ts new file mode 100644 index 0000000000..671ec89d13 --- /dev/null +++ b/src/frontend/servers/y-provider/src/handlers/contentHandler.ts @@ -0,0 +1,71 @@ +import { PartialBlock } from '@blocknote/core'; +import { ServerBlockNoteEditor } from '@blocknote/server-util'; +import { Request, Response } from 'express'; +import * as Y from 'yjs'; + +import { logger } from '@/utils'; + +interface ErrorResponse { + error: string; +} + +interface ContentRequest { + content: string; + format: string; +} + +const editor = ServerBlockNoteEditor.create(); + +export const contentHandler = async ( + req: Request, + res: Response, +) => { + const { content, format } = req.body; + + if (!content) { + res.status(400).json({ error: 'Invalid request: missing content' }); + return; + } + + if (!format || !['json', 'markdown', 'html'].includes(format)) { + res + .status(400) + .json({ error: 'Invalid format. Must be one of: json, markdown, html' }); + return; + } + + try { + // Decode base64 content to Uint8Array + const uint8Array = new Uint8Array(Buffer.from(content, 'base64')); + + // Create Yjs document and apply the update + const yDocument = new Y.Doc(); + Y.applyUpdate(yDocument, uint8Array); + + // Convert to blocks + const blocks = editor.yDocToBlocks(yDocument, 'document-store'); + + let result: string | object | null; + + if (!blocks || blocks.length === 0) { + result = null; + } else if (format === 'json') { + result = blocks; + } else if (format === 'markdown') { + result = await editor.blocksToMarkdownLossy(blocks as PartialBlock[]); + } else if (format === 'html') { + result = await editor.blocksToHTMLLossy(blocks as PartialBlock[]); + } else { + res.status(400).json({ error: 'Unsupported format' }); + return; + } + + res.status(200).json({ + content: result, + format: format, + }); + } catch (e) { + logger('content conversion failed:', e); + res.status(500).json({ error: 'An error occurred during conversion' }); + } +}; diff --git a/src/frontend/servers/y-provider/src/handlers/index.ts b/src/frontend/servers/y-provider/src/handlers/index.ts index 26b0ebedab..196662508c 100644 --- a/src/frontend/servers/y-provider/src/handlers/index.ts +++ b/src/frontend/servers/y-provider/src/handlers/index.ts @@ -1,4 +1,5 @@ export * from './collaborationResetConnectionsHandler'; export * from './collaborationWSHandler'; export * from './convertHandler'; +export * from './contentHandler'; export * from './getDocumentConnectionInfoHandler'; diff --git a/src/frontend/servers/y-provider/src/routes.ts b/src/frontend/servers/y-provider/src/routes.ts index 5bb73365fb..4c5d8ca2e7 100644 --- a/src/frontend/servers/y-provider/src/routes.ts +++ b/src/frontend/servers/y-provider/src/routes.ts @@ -2,5 +2,6 @@ export const routes = { COLLABORATION_WS: '/collaboration/ws/', COLLABORATION_RESET_CONNECTIONS: '/collaboration/api/reset-connections/', CONVERT: '/api/convert/', + CONTENT: '/api/content/', COLLABORATION_GET_CONNECTIONS: '/collaboration/api/get-connections/', }; diff --git a/src/frontend/servers/y-provider/src/servers/appServer.ts b/src/frontend/servers/y-provider/src/servers/appServer.ts index 0c355fee8e..ae008b36c1 100644 --- a/src/frontend/servers/y-provider/src/servers/appServer.ts +++ b/src/frontend/servers/y-provider/src/servers/appServer.ts @@ -8,6 +8,7 @@ import expressWebsockets from 'express-ws'; import { collaborationResetConnectionsHandler, collaborationWSHandler, + contentHandler, convertHandler, getDocumentConnectionInfoHandler, } from '@/handlers'; @@ -61,6 +62,11 @@ export const initApp = () => { convertHandler, ); + /** + * Route to convert base64 Yjs content to different formats + */ + app.post(routes.CONTENT, httpSecurity, express.json(), contentHandler); + Sentry.setupExpressErrorHandler(app); app.get('/ping', (req, res) => { From cdc24114b66770729a1f8b51911bd7e8f68fd818 Mon Sep 17 00:00:00 2001 From: Sylvain Zimmer Date: Thu, 24 Jul 2025 02:42:30 +0200 Subject: [PATCH 2/5] =?UTF-8?q?=F0=9F=9A=A8(lint)=20fix=20lint?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Oops --- src/backend/core/services/yprovider_services.py | 4 ---- .../core/tests/documents/test_api_documents_content.py | 2 +- 2 files changed, 1 insertion(+), 5 deletions(-) diff --git a/src/backend/core/services/yprovider_services.py b/src/backend/core/services/yprovider_services.py index 16ffed7d96..09b80f8c7f 100644 --- a/src/backend/core/services/yprovider_services.py +++ b/src/backend/core/services/yprovider_services.py @@ -1,15 +1,12 @@ """Y-Provider API services.""" import json -import logging from base64 import b64encode from django.conf import settings import requests -logger = logging.getLogger(__name__) - class ConversionError(Exception): """Base exception for conversion-related errors.""" @@ -72,7 +69,6 @@ def content(self, base64_content, format_type): raise ValidationError("Input content cannot be empty") data = json.dumps({"content": base64_content, "format": format_type}) - logger.warning(f"{settings.Y_PROVIDER_API_BASE_URL}api/content/") try: response = self._request( f"{settings.Y_PROVIDER_API_BASE_URL}content/", data, "application/json" diff --git a/src/backend/core/tests/documents/test_api_documents_content.py b/src/backend/core/tests/documents/test_api_documents_content.py index cabb2fc65e..62b215c928 100644 --- a/src/backend/core/tests/documents/test_api_documents_content.py +++ b/src/backend/core/tests/documents/test_api_documents_content.py @@ -9,7 +9,7 @@ from rest_framework import status from rest_framework.test import APIClient -from core import choices, factories +from core import factories pytestmark = pytest.mark.django_db From ab05fa6557b8422e0a0f2a605be31cee14c8ad6b Mon Sep 17 00:00:00 2001 From: Sylvain Zimmer Date: Thu, 24 Jul 2025 14:08:18 +0200 Subject: [PATCH 3/5] =?UTF-8?q?=E2=99=BB=EF=B8=8F(convert)=20reuse=20exist?= =?UTF-8?q?ing=20convert=20yprovider=20endpoint=20for=20content=20API?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docker/auth/realm.json | 2 +- src/backend/core/api/serializers.py | 6 +- src/backend/core/api/viewsets.py | 23 +- ...ider_services.py => converter_services.py} | 42 ++-- .../documents/test_api_documents_content.py | 49 +++-- .../test_api_documents_create_for_owner.py | 6 +- ...py => test_services_converter_services.py} | 112 ++++------ .../y-provider/__tests__/content.test.ts | 207 ------------------ .../y-provider/__tests__/convert.test.ts | 112 ++++++++++ src/frontend/servers/y-provider/package.json | 2 +- .../y-provider/src/handlers/contentHandler.ts | 71 ------ .../y-provider/src/handlers/convertHandler.ts | 70 +++++- .../servers/y-provider/src/handlers/index.ts | 1 - .../y-provider/src/servers/appServer.ts | 8 +- 14 files changed, 293 insertions(+), 418 deletions(-) rename src/backend/core/services/{yprovider_services.py => converter_services.py} (61%) rename src/backend/core/tests/{test_services_yprovider_services.py => test_services_converter_services.py} (65%) delete mode 100644 src/frontend/servers/y-provider/__tests__/content.test.ts delete mode 100644 src/frontend/servers/y-provider/src/handlers/contentHandler.ts diff --git a/docker/auth/realm.json b/docker/auth/realm.json index 06ab6e55e4..db5f1be021 100644 --- a/docker/auth/realm.json +++ b/docker/auth/realm.json @@ -26,7 +26,7 @@ "oauth2DeviceCodeLifespan": 600, "oauth2DevicePollingInterval": 5, "enabled": true, - "sslRequired": "none", + "sslRequired": "external", "registrationAllowed": true, "registrationEmailAsUsername": false, "rememberMe": true, diff --git a/src/backend/core/api/serializers.py b/src/backend/core/api/serializers.py index 572b844b46..83afc260d9 100644 --- a/src/backend/core/api/serializers.py +++ b/src/backend/core/api/serializers.py @@ -14,9 +14,9 @@ from core import choices, enums, models, utils from core.services.ai_services import AI_ACTIONS -from core.services.yprovider_services import ( +from core.services.converter_services import ( ConversionError, - YProviderAPI, + YdocConverter, ) @@ -431,7 +431,7 @@ def create(self, validated_data): language = user.language or language try: - document_content = YProviderAPI().convert(validated_data["content"]) + document_content = YdocConverter().convert(validated_data["content"]) except ConversionError as err: raise serializers.ValidationError( {"content": ["Could not convert content"]} diff --git a/src/backend/core/api/viewsets.py b/src/backend/core/api/viewsets.py index 5af4d02676..bdeba5a23d 100644 --- a/src/backend/core/api/viewsets.py +++ b/src/backend/core/api/viewsets.py @@ -1,6 +1,7 @@ """API endpoints""" # pylint: disable=too-many-lines +import base64 import json import logging import uuid @@ -37,14 +38,14 @@ from core import authentication, choices, enums, models from core.services.ai_services import AIService from core.services.collaboration_services import CollaborationService -from core.services.yprovider_services import ( +from core.services.converter_services import ( ServiceUnavailableError as YProviderServiceUnavailableError, ) -from core.services.yprovider_services import ( +from core.services.converter_services import ( ValidationError as YProviderValidationError, ) -from core.services.yprovider_services import ( - YProviderAPI, +from core.services.converter_services import ( + YdocConverter, ) from core.tasks.mail import send_ask_for_access_mail from core.utils import extract_attachments, filter_descendants @@ -1483,9 +1484,17 @@ def content(self, request, pk=None): if base64_content is not None: # Convert using the y-provider service try: - yprovider = YProviderAPI() - result = yprovider.content(base64_content, content_format) - content = result["content"] + yprovider = YdocConverter() + result = yprovider.convert( + base64.b64decode(base64_content), + "application/vnd.yjs.doc", + { + "markdown": "text/markdown", + "html": "text/html", + "json": "application/json", + }[content_format], + ) + content = result except YProviderValidationError as e: return drf_response.Response( {"error": str(e)}, status=status.HTTP_400_BAD_REQUEST diff --git a/src/backend/core/services/yprovider_services.py b/src/backend/core/services/converter_services.py similarity index 61% rename from src/backend/core/services/yprovider_services.py rename to src/backend/core/services/converter_services.py index 09b80f8c7f..9c79a7192d 100644 --- a/src/backend/core/services/yprovider_services.py +++ b/src/backend/core/services/converter_services.py @@ -1,6 +1,5 @@ """Y-Provider API services.""" -import json from base64 import b64encode from django.conf import settings @@ -20,8 +19,8 @@ class ServiceUnavailableError(ConversionError): """Raised when the conversion service is unavailable.""" -class YProviderAPI: - """Service class for Y-Provider API operations.""" +class YdocConverter: + """Service class for conversion-related operations.""" @property def auth_header(self): @@ -29,7 +28,7 @@ def auth_header(self): # Note: Yprovider microservice accepts only raw token, which is not recommended return f"Bearer {settings.Y_PROVIDER_API_KEY}" - def _request(self, url, data, content_type): + def _request(self, url, data, content_type, accept): """Make a request to the Y-Provider API.""" response = requests.post( url, @@ -37,6 +36,7 @@ def _request(self, url, data, content_type): headers={ "Authorization": self.auth_header, "Content-Type": content_type, + "Accept": accept, }, timeout=settings.CONVERSION_API_TIMEOUT, verify=settings.CONVERSION_API_SECURE, @@ -44,7 +44,9 @@ def _request(self, url, data, content_type): response.raise_for_status() return response - def convert(self, text): + def convert( + self, text, content_type="text/markdown", accept="application/vnd.yjs.doc" + ): """Convert a Markdown text into our internal format using an external microservice.""" if not text: @@ -54,27 +56,17 @@ def convert(self, text): response = self._request( f"{settings.Y_PROVIDER_API_BASE_URL}{settings.CONVERSION_API_ENDPOINT}/", text, - "text/markdown", + content_type, + accept, ) - return b64encode(response.content).decode("utf-8") + if accept == "application/vnd.yjs.doc": + return b64encode(response.content).decode("utf-8") + if accept in {"text/markdown", "text/html"}: + return response.text + if accept == "application/json": + return response.json() + raise ValidationError("Unsupported format") except requests.RequestException as err: raise ServiceUnavailableError( - "Failed to connect to backend service", - ) from err - - def content(self, base64_content, format_type): - """Convert base64 Yjs content to different formats using the y-provider service.""" - - if not base64_content: - raise ValidationError("Input content cannot be empty") - - data = json.dumps({"content": base64_content, "format": format_type}) - try: - response = self._request( - f"{settings.Y_PROVIDER_API_BASE_URL}content/", data, "application/json" - ) - return response.json() - except requests.RequestException as err: - raise ServiceUnavailableError( - "Failed to connect to backend service", + "Failed to connect to conversion service", ) from err diff --git a/src/backend/core/tests/documents/test_api_documents_content.py b/src/backend/core/tests/documents/test_api_documents_content.py index 62b215c928..459bf5a0ce 100644 --- a/src/backend/core/tests/documents/test_api_documents_content.py +++ b/src/backend/core/tests/documents/test_api_documents_content.py @@ -2,6 +2,7 @@ Tests for Documents API endpoint in impress's core app: content """ +import base64 from unittest.mock import patch import pytest @@ -21,11 +22,11 @@ ("public", "editor"), ], ) -@patch("core.services.yprovider_services.YProviderAPI.content") +@patch("core.services.converter_services.YdocConverter.convert") def test_api_documents_content_public(mock_content, reach, role): """Anonymous users should be allowed to access content of public documents.""" document = factories.DocumentFactory(link_reach=reach, link_role=role) - mock_content.return_value = {"content": {"some": "data"}} + mock_content.return_value = {"some": "data"} response = APIClient().get(f"/api/v1.0/documents/{document.id!s}/content/") @@ -34,7 +35,11 @@ def test_api_documents_content_public(mock_content, reach, role): assert data["id"] == str(document.id) assert data["title"] == document.title assert data["content"] == {"some": "data"} - mock_content.assert_called_once_with(document.content, "json") + mock_content.assert_called_once_with( + base64.b64decode(document.content), + "application/vnd.yjs.doc", + "application/json", + ) @pytest.mark.parametrize( @@ -52,12 +57,12 @@ def test_api_documents_content_public(mock_content, reach, role): ("authenticated", "editor", None), ], ) -@patch("core.services.yprovider_services.YProviderAPI.content") +@patch("core.services.converter_services.YdocConverter.convert") def test_api_documents_content_not_public(mock_content, reach, doc_role, user_role): """Authenticated users need access to get non-public document content.""" user = factories.UserFactory() document = factories.DocumentFactory(link_reach=reach, link_role=doc_role) - mock_content.return_value = {"content": {"some": "data"}} + mock_content.return_value = {"some": "data"} # First anonymous request should fail client = APIClient() @@ -87,18 +92,26 @@ def test_api_documents_content_not_public(mock_content, reach, doc_role, user_ro assert data["id"] == str(document.id) assert data["title"] == document.title assert data["content"] == {"some": "data"} - mock_content.assert_called_once_with(document.content, "json") + mock_content.assert_called_once_with( + base64.b64decode(document.content), + "application/vnd.yjs.doc", + "application/json", + ) @pytest.mark.parametrize( - "content_format", - ["markdown", "html", "json"], + "content_format, accept", + [ + ("markdown", "text/markdown"), + ("html", "text/html"), + ("json", "application/json"), + ], ) -@patch("core.services.yprovider_services.YProviderAPI.content") -def test_api_documents_content_format(mock_content, content_format): +@patch("core.services.converter_services.YdocConverter.convert") +def test_api_documents_content_format(mock_content, content_format, accept): """Test that the content endpoint returns a specific format.""" document = factories.DocumentFactory(link_reach="public") - mock_content.return_value = {"content": "whatever"} + mock_content.return_value = {"some": "data"} response = APIClient().get( f"/api/v1.0/documents/{document.id!s}/content/?content_format={content_format}" @@ -108,11 +121,13 @@ def test_api_documents_content_format(mock_content, content_format): data = response.json() assert data["id"] == str(document.id) assert data["title"] == document.title - assert data["content"] == "whatever" - mock_content.assert_called_once_with(document.content, content_format) + assert data["content"] == {"some": "data"} + mock_content.assert_called_once_with( + base64.b64decode(document.content), "application/vnd.yjs.doc", accept + ) -@patch("core.services.yprovider_services.YProviderAPI._request") +@patch("core.services.converter_services.YdocConverter._request") def test_api_documents_content_invalid_format(mock_request): """Test that the content endpoint rejects invalid formats.""" document = factories.DocumentFactory(link_reach="public") @@ -124,7 +139,7 @@ def test_api_documents_content_invalid_format(mock_request): mock_request.assert_not_called() -@patch("core.services.yprovider_services.YProviderAPI._request") +@patch("core.services.converter_services.YdocConverter._request") def test_api_documents_content_yservice_error(mock_request): """Test that service errors are handled properly.""" document = factories.DocumentFactory(link_reach="public") @@ -135,7 +150,7 @@ def test_api_documents_content_yservice_error(mock_request): assert response.status_code == status.HTTP_500_INTERNAL_SERVER_ERROR -@patch("core.services.yprovider_services.YProviderAPI._request") +@patch("core.services.converter_services.YdocConverter._request") def test_api_documents_content_nonexistent_document(mock_request): """Test that accessing a nonexistent document returns 404.""" client = APIClient() @@ -146,7 +161,7 @@ def test_api_documents_content_nonexistent_document(mock_request): mock_request.assert_not_called() -@patch("core.services.yprovider_services.YProviderAPI._request") +@patch("core.services.converter_services.YdocConverter._request") def test_api_documents_content_empty_document(mock_request): """Test that accessing an empty document returns empty content.""" document = factories.DocumentFactory(link_reach="public", content="") diff --git a/src/backend/core/tests/documents/test_api_documents_create_for_owner.py b/src/backend/core/tests/documents/test_api_documents_create_for_owner.py index a28fbecabd..cbe35b997e 100644 --- a/src/backend/core/tests/documents/test_api_documents_create_for_owner.py +++ b/src/backend/core/tests/documents/test_api_documents_create_for_owner.py @@ -16,16 +16,16 @@ from core import factories from core.api.serializers import ServerCreateDocumentSerializer from core.models import Document, Invitation, User -from core.services.yprovider_services import ConversionError, YProviderAPI +from core.services.converter_services import ConversionError, YdocConverter pytestmark = pytest.mark.django_db @pytest.fixture def mock_convert_md(): - """Mock YProviderAPI.convert to return a converted content.""" + """Mock YdocConverter.convert to return a converted content.""" with patch.object( - YProviderAPI, + YdocConverter, "convert", return_value="Converted document content", ) as mock: diff --git a/src/backend/core/tests/test_services_yprovider_services.py b/src/backend/core/tests/test_services_converter_services.py similarity index 65% rename from src/backend/core/tests/test_services_yprovider_services.py rename to src/backend/core/tests/test_services_converter_services.py index 603c22072a..086d132b35 100644 --- a/src/backend/core/tests/test_services_yprovider_services.py +++ b/src/backend/core/tests/test_services_converter_services.py @@ -1,29 +1,28 @@ """Test y-provider services.""" -import json from base64 import b64decode from unittest.mock import MagicMock, patch import pytest import requests -from core.services.yprovider_services import ( +from core.services.converter_services import ( ServiceUnavailableError, ValidationError, - YProviderAPI, + YdocConverter, ) def test_auth_header(settings): """Test authentication header generation.""" settings.Y_PROVIDER_API_KEY = "test-key" - converter = YProviderAPI() + converter = YdocConverter() assert converter.auth_header == "Bearer test-key" def test_convert_empty_text(): """Should raise ValidationError when text is empty.""" - converter = YProviderAPI() + converter = YdocConverter() with pytest.raises(ValidationError, match="Input text cannot be empty"): converter.convert("") @@ -31,13 +30,13 @@ def test_convert_empty_text(): @patch("requests.post") def test_convert_service_unavailable(mock_post): """Should raise ServiceUnavailableError when service is unavailable.""" - converter = YProviderAPI() + converter = YdocConverter() mock_post.side_effect = requests.RequestException("Connection error") with pytest.raises( ServiceUnavailableError, - match="Failed to connect to backend service", + match="Failed to connect to conversion service", ): converter.convert("test text") @@ -45,7 +44,7 @@ def test_convert_service_unavailable(mock_post): @patch("requests.post") def test_convert_http_error(mock_post): """Should raise ServiceUnavailableError when HTTP error occurs.""" - converter = YProviderAPI() + converter = YdocConverter() mock_response = MagicMock() mock_response.raise_for_status.side_effect = requests.HTTPError("HTTP Error") @@ -53,7 +52,7 @@ def test_convert_http_error(mock_post): with pytest.raises( ServiceUnavailableError, - match="Failed to connect to backend service", + match="Failed to connect to conversion service", ): converter.convert("test text") @@ -68,7 +67,7 @@ def test_convert_full_integration(mock_post, settings): settings.CONVERSION_API_TIMEOUT = 5 settings.CONVERSION_API_CONTENT_FIELD = "content" - converter = YProviderAPI() + converter = YdocConverter() expected_content = b"converted content" mock_response = MagicMock() @@ -85,6 +84,7 @@ def test_convert_full_integration(mock_post, settings): headers={ "Authorization": "Bearer test-key", "Content-Type": "text/markdown", + "Accept": "application/vnd.yjs.doc", }, timeout=5, verify=False, @@ -92,77 +92,57 @@ def test_convert_full_integration(mock_post, settings): @patch("requests.post") -def test_convert_timeout(mock_post): - """Should raise ServiceUnavailableError when request times out.""" - converter = YProviderAPI() - - mock_post.side_effect = requests.Timeout("Request timed out") - - with pytest.raises( - ServiceUnavailableError, - match="Failed to connect to backend service", - ): - converter.convert("test text") - - -def test_convert_none_input(): - """Should raise ValidationError when input is None.""" - converter = YProviderAPI() - - with pytest.raises(ValidationError, match="Input text cannot be empty"): - converter.convert(None) - - -def test_content_empty_content(): - """Should raise ValidationError when content is empty.""" - converter = YProviderAPI() - with pytest.raises(ValidationError, match="Input content cannot be empty"): - converter.content("", "markdown") - - -@patch("requests.post") -def test_content_service_unavailable(mock_post): - """Should raise ServiceUnavailableError when service is unavailable.""" - converter = YProviderAPI() - - mock_post.side_effect = requests.RequestException("Connection error") - - with pytest.raises( - ServiceUnavailableError, - match="Failed to connect to backend service", - ): - converter.content("test_content", "markdown") - - -@patch("requests.post") -def test_content_success(mock_post, settings): - """Test successful content fetch.""" - settings.Y_PROVIDER_API_BASE_URL = "http://test.com/api/" +def test_convert_full_integration_with_specific_headers(mock_post, settings): + """Test successful conversion with specific content type and accept headers.""" + settings.Y_PROVIDER_API_BASE_URL = "http://test.com/" settings.Y_PROVIDER_API_KEY = "test-key" + settings.CONVERSION_API_ENDPOINT = "conversion-endpoint" settings.CONVERSION_API_TIMEOUT = 5 settings.CONVERSION_API_SECURE = False - converter = YProviderAPI() + converter = YdocConverter() - expected_response = { - "content": "# Test Document\n\nThis is test content.", - "format": "markdown", - } + expected_response = "# Test Document\n\nThis is test content." mock_response = MagicMock() - mock_response.json.return_value = expected_response + mock_response.text = expected_response mock_response.raise_for_status.return_value = None mock_post.return_value = mock_response - result = converter.content("test_content", "markdown") + result = converter.convert( + b"test_content", "application/vnd.yjs.doc", "text/markdown" + ) assert result == expected_response mock_post.assert_called_once_with( - "http://test.com/api/content/", - data=json.dumps({"content": "test_content", "format": "markdown"}), + "http://test.com/conversion-endpoint/", + data=b"test_content", headers={ "Authorization": "Bearer test-key", - "Content-Type": "application/json", + "Content-Type": "application/vnd.yjs.doc", + "Accept": "text/markdown", }, timeout=5, verify=False, ) + + +@patch("requests.post") +def test_convert_timeout(mock_post): + """Should raise ServiceUnavailableError when request times out.""" + converter = YdocConverter() + + mock_post.side_effect = requests.Timeout("Request timed out") + + with pytest.raises( + ServiceUnavailableError, + match="Failed to connect to conversion service", + ): + converter.convert("test text") + + +def test_convert_none_input(): + """Should raise ValidationError when input is None.""" + converter = YdocConverter() + + with pytest.raises(ValidationError, match="Input text cannot be empty"): + converter.convert(None) diff --git a/src/frontend/servers/y-provider/__tests__/content.test.ts b/src/frontend/servers/y-provider/__tests__/content.test.ts deleted file mode 100644 index 555b453ae4..0000000000 --- a/src/frontend/servers/y-provider/__tests__/content.test.ts +++ /dev/null @@ -1,207 +0,0 @@ -import { ServerBlockNoteEditor } from '@blocknote/server-util'; -import request from 'supertest'; -import { describe, expect, test, vi } from 'vitest'; -import * as Y from 'yjs'; - -vi.mock('../src/env', async (importOriginal) => { - return { - ...(await importOriginal()), - COLLABORATION_SERVER_ORIGIN: 'http://localhost:3000', - Y_PROVIDER_API_KEY: 'yprovider-api-key', - }; -}); - -import { initApp } from '@/servers'; - -import { - Y_PROVIDER_API_KEY as apiKey, - COLLABORATION_SERVER_ORIGIN as origin, -} from '../src/env'; - -console.error = vi.fn(); - -describe('Content API Tests', () => { - test('POST /api/content with incorrect API key responds with 401', async () => { - const app = initApp(); - - const response = await request(app) - .post('/api/content') - .set('origin', origin) - .set('authorization', 'wrong-api-key') - .set('content-type', 'application/json') - .send({ - content: 'dGVzdA==', // base64 for "test" - format: 'json', - }); - - expect(response.status).toBe(401); - expect(response.body).toStrictEqual({ - error: 'Unauthorized: Invalid API Key', - }); - }); - - test('POST /api/content with incorrect Bearer token responds with 401', async () => { - const app = initApp(); - - const response = await request(app) - .post('/api/content') - .set('origin', origin) - .set('authorization', 'Bearer test-secret-api-key') - .set('content-type', 'application/json') - .send({ - content: 'dGVzdA==', // base64 for "test" - format: 'json', - }); - - expect(response.status).toBe(401); - expect(response.body).toStrictEqual({ - error: 'Unauthorized: Invalid API Key', - }); - }); - - test('POST /api/content with missing content parameter', async () => { - const app = initApp(); - - const response = await request(app) - .post('/api/content') - .set('origin', origin) - .set('authorization', apiKey) - .set('content-type', 'application/json') - .send({ - format: 'json', - }); - - expect(response.status).toBe(400); - expect(response.body).toStrictEqual({ - error: 'Invalid request: missing content', - }); - }); - - test('POST /api/content with empty content', async () => { - const app = initApp(); - - const response = await request(app) - .post('/api/content') - .set('origin', origin) - .set('authorization', apiKey) - .set('content-type', 'application/json') - .send({ - content: '', - format: 'json', - }); - - expect(response.status).toBe(400); - expect(response.body).toStrictEqual({ - error: 'Invalid request: missing content', - }); - }); - - test('POST /api/content with missing format parameter', async () => { - const app = initApp(); - - const response = await request(app) - .post('/api/content') - .set('origin', origin) - .set('authorization', apiKey) - .set('content-type', 'application/json') - .send({ - content: 'dGVzdA==', - }); - - expect(response.status).toBe(400); - expect(response.body).toStrictEqual({ - error: 'Invalid format. Must be one of: json, markdown, html', - }); - }); - - test('POST /api/content with invalid format', async () => { - const app = initApp(); - - const response = await request(app) - .post('/api/content') - .set('origin', origin) - .set('authorization', apiKey) - .set('content-type', 'application/json') - .send({ - content: 'dGVzdA==', - format: 'invalid', - }); - - expect(response.status).toBe(400); - expect(response.body).toStrictEqual({ - error: 'Invalid format. Must be one of: json, markdown, html', - }); - }); - - test.each([ - { authHeader: `Bearer ${apiKey}`, format: 'json' }, - { authHeader: `Bearer ${apiKey}`, format: 'markdown' }, - { authHeader: `Bearer ${apiKey}`, format: 'html' }, - ])( - 'POST /api/content with correct content and format $format with Authorization: $authHeader', - async ({ authHeader, format }) => { - const app = initApp(); - - // Create a simple Yjs document for testing using BlockNote - const editor = ServerBlockNoteEditor.create(); - const markdownContent = '# Test Document\n\nThis is test content.'; - const blocks = await editor.tryParseMarkdownToBlocks(markdownContent); - const yDocument = editor.blocksToYDoc(blocks, 'document-store'); - const yjsUpdate = Y.encodeStateAsUpdate(yDocument); - const base64Content = Buffer.from(yjsUpdate).toString('base64'); - - const response = await request(app) - .post('/api/content') - .set('Origin', origin) - .set('Authorization', authHeader) - .set('content-type', 'application/json') - .send({ - content: base64Content, - format: format, - }); - expect(response.status).toBe(200); - expect(response.body).toHaveProperty('content'); - expect(response.body).toHaveProperty('format', format); - - // Verify the content based on format - if (format === 'json') { - const parsedContent = response.body.content; - expect(Array.isArray(parsedContent)).toBe(true); - expect(parsedContent.length).toBe(2); - expect(parsedContent[0].type).toBe('heading'); - expect(parsedContent[1].type).toBe('paragraph'); - expect(parsedContent[0].content[0].type).toBe('text'); - expect(parsedContent[0].content[0].text).toBe('Test Document'); - expect(parsedContent[1].content[0].type).toBe('text'); - expect(parsedContent[1].content[0].text).toBe('This is test content.'); - } else if (format === 'markdown') { - expect(typeof response.body.content).toBe('string'); - expect(response.body.content.trim()).toBe(markdownContent); - } else if (format === 'html') { - expect(typeof response.body.content).toBe('string'); - expect(response.body.content).toBe( - '

Test Document

This is test content.

', - ); - } - }, - ); - - test('POST /api/content with invalid base64 content returns 500', async () => { - const app = initApp(); - - const response = await request(app) - .post('/api/content') - .set('origin', origin) - .set('authorization', apiKey) - .set('content-type', 'application/json') - .send({ - content: 'invalid-base64-content!@#', - format: 'json', - }); - - expect(response.status).toBe(500); - expect(response.body).toStrictEqual({ - error: 'An error occurred during conversion', - }); - }); -}); diff --git a/src/frontend/servers/y-provider/__tests__/convert.test.ts b/src/frontend/servers/y-provider/__tests__/convert.test.ts index 67de07cf1f..e8a81c2c6e 100644 --- a/src/frontend/servers/y-provider/__tests__/convert.test.ts +++ b/src/frontend/servers/y-provider/__tests__/convert.test.ts @@ -121,6 +121,31 @@ describe('Server Tests', () => { }); }); + test('POST /api/convert with unsupported Content-Type returns 415', async () => { + const app = initApp(); + const response = await request(app) + .post('/api/convert') + .set('origin', origin) + .set('authorization', apiKey) + .set('content-type', 'image/png') + .send('randomdata'); + expect(response.status).toBe(415); + expect(response.body).toStrictEqual({ error: 'Unsupported Content-Type' }); + }); + + test('POST /api/convert with unsupported Accept returns 406', async () => { + const app = initApp(); + const response = await request(app) + .post('/api/convert') + .set('origin', origin) + .set('authorization', apiKey) + .set('content-type', 'text/markdown') + .set('accept', 'image/png') + .send('# Header'); + expect(response.status).toBe(406); + expect(response.body).toStrictEqual({ error: 'Unsupported format' }); + }); + test.each([[apiKey], [`Bearer ${apiKey}`]])( 'POST /api/convert with correct content with Authorization: %s', async (authHeader) => { @@ -137,6 +162,8 @@ describe('Server Tests', () => { .post('/api/convert') .set('Origin', origin) .set('Authorization', authHeader) + .set('content-type', 'text/markdown') + .set('accept', 'application/vnd.yjs.doc') .send(document); expect(response.status).toBe(200); @@ -150,4 +177,89 @@ describe('Server Tests', () => { expect(blocks).toStrictEqual(expectedBlocks); }, ); + + test('POST /api/convert Yjs to HTML', async () => { + const app = initApp(); + const editor = ServerBlockNoteEditor.create(); + const markdownContent = '# Test Document\n\nThis is test content.'; + const blocks = await editor.tryParseMarkdownToBlocks(markdownContent); + const yDocument = editor.blocksToYDoc(blocks, 'document-store'); + const yjsUpdate = Y.encodeStateAsUpdate(yDocument); + const response = await request(app) + .post('/api/convert') + .set('origin', origin) + .set('authorization', apiKey) + .set('content-type', 'application/vnd.yjs.doc') + .set('accept', 'text/html') + .send(Buffer.from(yjsUpdate)); + expect(response.status).toBe(200); + expect(response.header['content-type']).toBe('text/html; charset=utf-8'); + expect(typeof response.text).toBe('string'); + expect(response.text).toBe( + '

Test Document

This is test content.

', + ); + }); + + test('POST /api/convert Yjs to Markdown', async () => { + const app = initApp(); + const editor = ServerBlockNoteEditor.create(); + const markdownContent = '# Test Document\n\nThis is test content.'; + const blocks = await editor.tryParseMarkdownToBlocks(markdownContent); + const yDocument = editor.blocksToYDoc(blocks, 'document-store'); + const yjsUpdate = Y.encodeStateAsUpdate(yDocument); + const response = await request(app) + .post('/api/convert') + .set('origin', origin) + .set('authorization', apiKey) + .set('content-type', 'application/vnd.yjs.doc') + .set('accept', 'text/markdown') + .send(Buffer.from(yjsUpdate)); + expect(response.status).toBe(200); + expect(response.header['content-type']).toBe( + 'text/markdown; charset=utf-8', + ); + expect(typeof response.text).toBe('string'); + expect(response.text.trim()).toBe(markdownContent); + }); + + test('POST /api/convert Yjs to JSON', async () => { + const app = initApp(); + const editor = ServerBlockNoteEditor.create(); + const markdownContent = '# Test Document\n\nThis is test content.'; + const blocks = await editor.tryParseMarkdownToBlocks(markdownContent); + const yDocument = editor.blocksToYDoc(blocks, 'document-store'); + const yjsUpdate = Y.encodeStateAsUpdate(yDocument); + const response = await request(app) + .post('/api/convert') + .set('origin', origin) + .set('authorization', apiKey) + .set('content-type', 'application/vnd.yjs.doc') + .set('accept', 'application/json') + .send(Buffer.from(yjsUpdate)); + expect(response.status).toBe(200); + expect(response.header['content-type']).toBe( + 'application/json; charset=utf-8', + ); + expect(Array.isArray(response.body)).toBe(true); + expect(response.body.length).toBe(2); + expect(response.body[0].type).toBe('heading'); + expect(response.body[1].type).toBe('paragraph'); + expect(response.body[0].content[0].type).toBe('text'); + expect(response.body[0].content[0].text).toBe('Test Document'); + expect(response.body[1].content[0].type).toBe('text'); + expect(response.body[1].content[0].text).toBe('This is test content.'); + }); + + test('POST /api/convert with invalid Yjs content returns 400', async () => { + const app = initApp(); + const response = await request(app) + .post('/api/convert') + .set('origin', origin) + .set('authorization', apiKey) + .set('content-type', 'application/vnd.yjs.doc') + .set('accept', 'application/json') + .send(Buffer.from('notvalidyjs')); + expect(response.status).toBe(400); + expect(response.body).toStrictEqual({ error: 'Invalid Yjs content' }); + }); }); diff --git a/src/frontend/servers/y-provider/package.json b/src/frontend/servers/y-provider/package.json index 6a89231815..185ab1cc06 100644 --- a/src/frontend/servers/y-provider/package.json +++ b/src/frontend/servers/y-provider/package.json @@ -9,7 +9,7 @@ "build": "tsc -p tsconfig.build.json && tsc-alias", "dev": "cross-env COLLABORATION_LOGGING=true && nodemon --config nodemon.json", "start": "node ./dist/start-server.js", - "lint": "eslint . --ext .ts", + "lint": "eslint . --ext .ts --fix", "test": "vitest --run --disable-console-intercept" }, "engines": { diff --git a/src/frontend/servers/y-provider/src/handlers/contentHandler.ts b/src/frontend/servers/y-provider/src/handlers/contentHandler.ts deleted file mode 100644 index 671ec89d13..0000000000 --- a/src/frontend/servers/y-provider/src/handlers/contentHandler.ts +++ /dev/null @@ -1,71 +0,0 @@ -import { PartialBlock } from '@blocknote/core'; -import { ServerBlockNoteEditor } from '@blocknote/server-util'; -import { Request, Response } from 'express'; -import * as Y from 'yjs'; - -import { logger } from '@/utils'; - -interface ErrorResponse { - error: string; -} - -interface ContentRequest { - content: string; - format: string; -} - -const editor = ServerBlockNoteEditor.create(); - -export const contentHandler = async ( - req: Request, - res: Response, -) => { - const { content, format } = req.body; - - if (!content) { - res.status(400).json({ error: 'Invalid request: missing content' }); - return; - } - - if (!format || !['json', 'markdown', 'html'].includes(format)) { - res - .status(400) - .json({ error: 'Invalid format. Must be one of: json, markdown, html' }); - return; - } - - try { - // Decode base64 content to Uint8Array - const uint8Array = new Uint8Array(Buffer.from(content, 'base64')); - - // Create Yjs document and apply the update - const yDocument = new Y.Doc(); - Y.applyUpdate(yDocument, uint8Array); - - // Convert to blocks - const blocks = editor.yDocToBlocks(yDocument, 'document-store'); - - let result: string | object | null; - - if (!blocks || blocks.length === 0) { - result = null; - } else if (format === 'json') { - result = blocks; - } else if (format === 'markdown') { - result = await editor.blocksToMarkdownLossy(blocks as PartialBlock[]); - } else if (format === 'html') { - result = await editor.blocksToHTMLLossy(blocks as PartialBlock[]); - } else { - res.status(400).json({ error: 'Unsupported format' }); - return; - } - - res.status(200).json({ - content: result, - format: format, - }); - } catch (e) { - logger('content conversion failed:', e); - res.status(500).json({ error: 'An error occurred during conversion' }); - } -}; diff --git a/src/frontend/servers/y-provider/src/handlers/convertHandler.ts b/src/frontend/servers/y-provider/src/handlers/convertHandler.ts index 15bab784d5..d5e23039bb 100644 --- a/src/frontend/servers/y-provider/src/handlers/convertHandler.ts +++ b/src/frontend/servers/y-provider/src/handlers/convertHandler.ts @@ -1,3 +1,4 @@ +import { PartialBlock } from '@blocknote/core'; import { ServerBlockNoteEditor } from '@blocknote/server-util'; import { Request, Response } from 'express'; import * as Y from 'yjs'; @@ -12,29 +13,80 @@ const editor = ServerBlockNoteEditor.create(); export const convertHandler = async ( req: Request, - res: Response, + res: Response, ) => { if (!req.body || req.body.length === 0) { res.status(400).json({ error: 'Invalid request: missing content' }); return; } + const contentType = (req.header('content-type') || 'text/markdown').split( + ';', + )[0]; + const accept = (req.header('accept') || 'application/vnd.yjs.doc').split( + ';', + )[0]; + + let blocks: PartialBlock[] | null = null; try { - // Perform the conversion from markdown to Blocknote.js blocks - const blocks = await editor.tryParseMarkdownToBlocks(req.body.toString()); + // First, convert from the input format to blocks + // application/x-www-form-urlencoded is interpreted as Markdown for backward compatibility + if ( + contentType === 'text/markdown' || + contentType === 'application/x-www-form-urlencoded' + ) { + blocks = await editor.tryParseMarkdownToBlocks(req.body.toString()); + } else if ( + contentType === 'application/vnd.yjs.doc' || + contentType === 'application/octet-stream' + ) { + try { + const ydoc = new Y.Doc(); + Y.applyUpdate(ydoc, req.body); + blocks = editor.yDocToBlocks(ydoc, 'document-store') as PartialBlock[]; + } catch (e) { + logger('Invalid Yjs content:', e); + res.status(400).json({ error: 'Invalid Yjs content' }); + return; + } + } else { + res.status(415).json({ error: 'Unsupported Content-Type' }); + return; + } if (!blocks || blocks.length === 0) { res.status(500).json({ error: 'No valid blocks were generated' }); return; } - // Create a Yjs Document from blocks - const yDocument = editor.blocksToYDoc(blocks, 'document-store'); + // Then, convert from blocks to the output format + if (accept === 'application/json') { + res.status(200).json(blocks); + } else { + const yDocument = editor.blocksToYDoc(blocks, 'document-store'); - res - .status(200) - .setHeader('content-type', 'application/octet-stream') - .send(Y.encodeStateAsUpdate(yDocument)); + if ( + accept === 'application/vnd.yjs.doc' || + accept === 'application/octet-stream' + ) { + res + .status(200) + .setHeader('content-type', 'application/octet-stream') + .send(Y.encodeStateAsUpdate(yDocument)); + } else if (accept === 'text/markdown') { + res + .status(200) + .setHeader('content-type', 'text/markdown') + .send(await editor.blocksToMarkdownLossy(blocks)); + } else if (accept === 'text/html') { + res + .status(200) + .setHeader('content-type', 'text/html') + .send(await editor.blocksToHTMLLossy(blocks)); + } else { + res.status(406).json({ error: 'Unsupported format' }); + } + } } catch (e) { logger('conversion failed:', e); res.status(500).json({ error: 'An error occurred' }); diff --git a/src/frontend/servers/y-provider/src/handlers/index.ts b/src/frontend/servers/y-provider/src/handlers/index.ts index 196662508c..26b0ebedab 100644 --- a/src/frontend/servers/y-provider/src/handlers/index.ts +++ b/src/frontend/servers/y-provider/src/handlers/index.ts @@ -1,5 +1,4 @@ export * from './collaborationResetConnectionsHandler'; export * from './collaborationWSHandler'; export * from './convertHandler'; -export * from './contentHandler'; export * from './getDocumentConnectionInfoHandler'; diff --git a/src/frontend/servers/y-provider/src/servers/appServer.ts b/src/frontend/servers/y-provider/src/servers/appServer.ts index ae008b36c1..c9807bf783 100644 --- a/src/frontend/servers/y-provider/src/servers/appServer.ts +++ b/src/frontend/servers/y-provider/src/servers/appServer.ts @@ -8,7 +8,6 @@ import expressWebsockets from 'express-ws'; import { collaborationResetConnectionsHandler, collaborationWSHandler, - contentHandler, convertHandler, getDocumentConnectionInfoHandler, } from '@/handlers'; @@ -50,7 +49,7 @@ export const initApp = () => { ); /** - * Route to convert Markdown or BlockNote blocks + * Route to convert Markdown or BlockNote blocks and Yjs content */ app.post( routes.CONVERT, @@ -62,11 +61,6 @@ export const initApp = () => { convertHandler, ); - /** - * Route to convert base64 Yjs content to different formats - */ - app.post(routes.CONTENT, httpSecurity, express.json(), contentHandler); - Sentry.setupExpressErrorHandler(app); app.get('/ping', (req, res) => { From 148890a2956f7e7cb0669f87b9e4fdaa8cb060b5 Mon Sep 17 00:00:00 2001 From: Sylvain Zimmer Date: Thu, 24 Jul 2025 14:49:20 +0200 Subject: [PATCH 4/5] =?UTF-8?q?=E2=9C=85(convert)=20improve=20tests=20with?= =?UTF-8?q?=20stricter=20tests=20and=20less=20ipsum?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../y-provider/__tests__/convert.test.ts | 52 ++++++++++--------- src/frontend/servers/y-provider/package.json | 2 +- .../y-provider/src/handlers/convertHandler.ts | 1 - 3 files changed, 28 insertions(+), 27 deletions(-) diff --git a/src/frontend/servers/y-provider/__tests__/convert.test.ts b/src/frontend/servers/y-provider/__tests__/convert.test.ts index e8a81c2c6e..44c21c2870 100644 --- a/src/frontend/servers/y-provider/__tests__/convert.test.ts +++ b/src/frontend/servers/y-provider/__tests__/convert.test.ts @@ -18,6 +18,9 @@ import { COLLABORATION_SERVER_ORIGIN as origin, } from '../src/env'; +const expectedMarkdown = '# Example document\n\nLorem ipsum dolor sit amet.'; +const expectedHTML = + '

Example document

Lorem ipsum dolor sit amet.

'; const expectedBlocks = [ { children: [], @@ -151,20 +154,13 @@ describe('Server Tests', () => { async (authHeader) => { const app = initApp(); - const document = [ - '# Example document', - '', - 'Lorem ipsum dolor sit amet.', - '', - ].join('\n'); - const response = await request(app) .post('/api/convert') .set('Origin', origin) .set('Authorization', authHeader) .set('content-type', 'text/markdown') .set('accept', 'application/vnd.yjs.doc') - .send(document); + .send(expectedMarkdown); expect(response.status).toBe(200); expect(response.body).toBeInstanceOf(Buffer); @@ -181,8 +177,7 @@ describe('Server Tests', () => { test('POST /api/convert Yjs to HTML', async () => { const app = initApp(); const editor = ServerBlockNoteEditor.create(); - const markdownContent = '# Test Document\n\nThis is test content.'; - const blocks = await editor.tryParseMarkdownToBlocks(markdownContent); + const blocks = await editor.tryParseMarkdownToBlocks(expectedMarkdown); const yDocument = editor.blocksToYDoc(blocks, 'document-store'); const yjsUpdate = Y.encodeStateAsUpdate(yDocument); const response = await request(app) @@ -195,16 +190,13 @@ describe('Server Tests', () => { expect(response.status).toBe(200); expect(response.header['content-type']).toBe('text/html; charset=utf-8'); expect(typeof response.text).toBe('string'); - expect(response.text).toBe( - '

Test Document

This is test content.

', - ); + expect(response.text).toBe(expectedHTML); }); test('POST /api/convert Yjs to Markdown', async () => { const app = initApp(); const editor = ServerBlockNoteEditor.create(); - const markdownContent = '# Test Document\n\nThis is test content.'; - const blocks = await editor.tryParseMarkdownToBlocks(markdownContent); + const blocks = await editor.tryParseMarkdownToBlocks(expectedMarkdown); const yDocument = editor.blocksToYDoc(blocks, 'document-store'); const yjsUpdate = Y.encodeStateAsUpdate(yDocument); const response = await request(app) @@ -219,14 +211,13 @@ describe('Server Tests', () => { 'text/markdown; charset=utf-8', ); expect(typeof response.text).toBe('string'); - expect(response.text.trim()).toBe(markdownContent); + expect(response.text.trim()).toBe(expectedMarkdown); }); test('POST /api/convert Yjs to JSON', async () => { const app = initApp(); const editor = ServerBlockNoteEditor.create(); - const markdownContent = '# Test Document\n\nThis is test content.'; - const blocks = await editor.tryParseMarkdownToBlocks(markdownContent); + const blocks = await editor.tryParseMarkdownToBlocks(expectedMarkdown); const yDocument = editor.blocksToYDoc(blocks, 'document-store'); const yjsUpdate = Y.encodeStateAsUpdate(yDocument); const response = await request(app) @@ -241,13 +232,24 @@ describe('Server Tests', () => { 'application/json; charset=utf-8', ); expect(Array.isArray(response.body)).toBe(true); - expect(response.body.length).toBe(2); - expect(response.body[0].type).toBe('heading'); - expect(response.body[1].type).toBe('paragraph'); - expect(response.body[0].content[0].type).toBe('text'); - expect(response.body[0].content[0].text).toBe('Test Document'); - expect(response.body[1].content[0].type).toBe('text'); - expect(response.body[1].content[0].text).toBe('This is test content.'); + expect(response.body).toStrictEqual(expectedBlocks); + }); + + test('POST /api/convert Markdown to JSON', async () => { + const app = initApp(); + const response = await request(app) + .post('/api/convert') + .set('origin', origin) + .set('authorization', apiKey) + .set('content-type', 'text/markdown') + .set('accept', 'application/json') + .send(expectedMarkdown); + expect(response.status).toBe(200); + expect(response.header['content-type']).toBe( + 'application/json; charset=utf-8', + ); + expect(Array.isArray(response.body)).toBe(true); + expect(response.body).toStrictEqual(expectedBlocks); }); test('POST /api/convert with invalid Yjs content returns 400', async () => { diff --git a/src/frontend/servers/y-provider/package.json b/src/frontend/servers/y-provider/package.json index 185ab1cc06..6a89231815 100644 --- a/src/frontend/servers/y-provider/package.json +++ b/src/frontend/servers/y-provider/package.json @@ -9,7 +9,7 @@ "build": "tsc -p tsconfig.build.json && tsc-alias", "dev": "cross-env COLLABORATION_LOGGING=true && nodemon --config nodemon.json", "start": "node ./dist/start-server.js", - "lint": "eslint . --ext .ts --fix", + "lint": "eslint . --ext .ts", "test": "vitest --run --disable-console-intercept" }, "engines": { diff --git a/src/frontend/servers/y-provider/src/handlers/convertHandler.ts b/src/frontend/servers/y-provider/src/handlers/convertHandler.ts index d5e23039bb..9d8cd237c8 100644 --- a/src/frontend/servers/y-provider/src/handlers/convertHandler.ts +++ b/src/frontend/servers/y-provider/src/handlers/convertHandler.ts @@ -29,7 +29,6 @@ export const convertHandler = async ( let blocks: PartialBlock[] | null = null; try { - // First, convert from the input format to blocks // application/x-www-form-urlencoded is interpreted as Markdown for backward compatibility if ( From b7ffc766d8022d73478a1465ad6b01daa3631317 Mon Sep 17 00:00:00 2001 From: Sylvain Zimmer Date: Thu, 24 Jul 2025 14:52:16 +0200 Subject: [PATCH 5/5] =?UTF-8?q?=F0=9F=97=91=EF=B8=8F(convert)=20cleanup=20?= =?UTF-8?q?old=20content=20route?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/frontend/servers/y-provider/src/routes.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/src/frontend/servers/y-provider/src/routes.ts b/src/frontend/servers/y-provider/src/routes.ts index 4c5d8ca2e7..5bb73365fb 100644 --- a/src/frontend/servers/y-provider/src/routes.ts +++ b/src/frontend/servers/y-provider/src/routes.ts @@ -2,6 +2,5 @@ export const routes = { COLLABORATION_WS: '/collaboration/ws/', COLLABORATION_RESET_CONNECTIONS: '/collaboration/api/reset-connections/', CONVERT: '/api/convert/', - CONTENT: '/api/content/', COLLABORATION_GET_CONNECTIONS: '/collaboration/api/get-connections/', };