From a9f72f2b72f96a5a4414af4940151ec3c109251b Mon Sep 17 00:00:00 2001 From: qqmyers Date: Thu, 11 Aug 2022 14:49:39 -0400 Subject: [PATCH 01/40] HDC 3b 1.5 initial update --- .../harvard/iq/dataverse/MailServiceBean.java | 4 +- .../harvard/iq/dataverse/api/LDNInbox.java | 132 +++++----- .../LDNAnnounceDatasetVersionStep.java | 246 +++++++++--------- 3 files changed, 177 insertions(+), 205 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/MailServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/MailServiceBean.java index fa5216140c2..084a7058e70 100644 --- a/src/main/java/edu/harvard/iq/dataverse/MailServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/MailServiceBean.java @@ -621,9 +621,9 @@ public String getMessageTextBasedOnNotification(UserNotification userNotificatio Object[] paramArrayDatasetMentioned = { userNotification.getUser().getName(), BrandingUtil.getInstallationBrandName(), - citingResource.getString("@type"), + citingResource.getString("@type", "External Resource"), citingResource.getString("@id"), - citingResource.getString("name"), + citingResource.getString("name", citingResource.getString("@id")), citingResource.getString("relationship"), systemConfig.getDataverseSiteUrl(), dataset.getGlobalId().toString(), diff --git a/src/main/java/edu/harvard/iq/dataverse/api/LDNInbox.java b/src/main/java/edu/harvard/iq/dataverse/api/LDNInbox.java index 3912b9102e2..fabc067abf7 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/LDNInbox.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/LDNInbox.java @@ -73,8 +73,6 @@ public Response acceptMessage(String body) { String whitelist = settingsService.get(SettingsServiceBean.Key.LDNMessageHosts.toString(), ""); // Only do something if we listen to this host if (whitelist.equals("*") || whitelist.contains(origin.toString())) { - String citingPID = null; - String citingType = null; boolean sent = false; JsonObject jsonld = null; @@ -106,85 +104,73 @@ public Response acceptMessage(String body) { String objectKey = new JsonLDTerm(activityStreams, "object").getUrl(); if (jsonld.containsKey(objectKey)) { JsonObject msgObject = jsonld.getJsonObject(objectKey); - - citingPID = msgObject.getJsonObject(new JsonLDTerm(ietf, "cite-as").getUrl()).getString("@id"); - logger.fine("Citing PID: " + citingPID); - if (msgObject.containsKey("@type")) { - citingType = msgObject.getString("@type"); - if (citingType.startsWith(JsonLDNamespace.schema.getUrl())) { - citingType = citingType.replace(JsonLDNamespace.schema.getUrl(), ""); - } - if (msgObject.containsKey(JsonLDTerm.schemaOrg("name").getUrl())) { - name = msgObject.getString(JsonLDTerm.schemaOrg("name").getUrl()); - } - logger.fine("Citing Type: " + citingType); - String contextKey = new JsonLDTerm(activityStreams, "context").getUrl(); - - if (jsonld.containsKey(contextKey)) { - JsonObject context = jsonld.getJsonObject(contextKey); - for (Map.Entry entry : context.entrySet()) { - - relationship = entry.getKey().replace("_:", ""); - // Assuming only one for now - should check for array and loop - JsonObject citedResource = (JsonObject) entry.getValue(); - String pid = citedResource.getJsonObject(new JsonLDTerm(ietf, "cite-as").getUrl()) - .getString("@id"); - if (citedResource.getString("@type").equals(JsonLDTerm.schemaOrg("Dataset").getUrl())) { - logger.fine("Raw PID: " + pid); - if (pid.startsWith(GlobalId.DOI_RESOLVER_URL)) { - pid = pid.replace(GlobalId.DOI_RESOLVER_URL, GlobalId.DOI_PROTOCOL + ":"); - } else if (pid.startsWith(GlobalId.HDL_RESOLVER_URL)) { - pid = pid.replace(GlobalId.HDL_RESOLVER_URL, GlobalId.HDL_PROTOCOL + ":"); - } - logger.fine("Protocol PID: " + pid); - Optional id = GlobalId.parse(pid); - Dataset dataset = datasetSvc.findByGlobalId(pid); - if (dataset != null) { - JsonObject citingResource = Json.createObjectBuilder().add("@id", citingPID) - .add("@type", citingType).add("relationship", relationship) - .add("name", name).build(); - StringWriter sw = new StringWriter(128); - try (JsonWriter jw = Json.createWriter(sw)) { - jw.write(citingResource); - } - String jsonstring = sw.toString(); - Set ras = roleService.rolesAssignments(dataset); - - roleService.rolesAssignments(dataset).stream() - .filter(ra -> ra.getRole().permissions() - .contains(Permission.PublishDataset)) - .flatMap( - ra -> roleAssigneeService - .getExplicitUsers(roleAssigneeService - .getRoleAssignee(ra.getAssigneeIdentifier())) - .stream()) - .distinct() // prevent double-send - .forEach(au -> { - - if (au.isSuperuser()) { - userNotificationService.sendNotification(au, - new Timestamp(new Date().getTime()), - UserNotification.Type.DATASETMENTIONED, dataset.getId(), - null, null, true, jsonstring); - - } - }); - sent = true; + if (new JsonLDTerm(activityStreams, "Relationship").getUrl().equals(msgObject.getJsonString("@type"))) { + // We have a relationship message - need to get the subject and object and + // relationship type + String subjectId = msgObject.getJsonObject(new JsonLDTerm(activityStreams, "subject").getUrl()) + .getString("@id"); + String objectId = msgObject.getJsonObject(new JsonLDTerm(activityStreams, "object").getUrl()) + .getString("@id"); + String relationshipId = msgObject + .getJsonObject(new JsonLDTerm(activityStreams, "relationship").getUrl()).getString("@id"); + if (subjectId != null && objectId != null && relationshipId != null) { + // Strip the URL part from a relationship ID/URL assuming a usable label exists + // after a # char. Default is to use the whole URI. + relationship = relationshipId.substring(relationship.indexOf("#") + 1); + // Parse the URI as a PID and see if this Dataverse instance has this dataset + String pid = GlobalId.getInternalFormOfPID(objectId); + Optional id = GlobalId.parse(pid); + if (id.isPresent()) { + Dataset dataset = datasetSvc.findByGlobalId(pid); + if (dataset != null) { + JsonObject citingResource = Json.createObjectBuilder().add("@id", subjectId) + .add("relationship", relationship).build(); + StringWriter sw = new StringWriter(128); + try (JsonWriter jw = Json.createWriter(sw)) { + jw.write(citingResource); } + String jsonstring = sw.toString(); + Set ras = roleService.rolesAssignments(dataset); + + roleService.rolesAssignments(dataset).stream() + .filter(ra -> ra.getRole().permissions().contains(Permission.PublishDataset)) + .flatMap(ra -> roleAssigneeService + .getExplicitUsers( + roleAssigneeService.getRoleAssignee(ra.getAssigneeIdentifier())) + .stream()) + .distinct() // prevent double-send + .forEach(au -> { + + if (au.isSuperuser()) { + userNotificationService.sendNotification(au, + new Timestamp(new Date().getTime()), + UserNotification.Type.DATASETMENTIONED, dataset.getId(), null, + null, true, jsonstring); + + } + }); + sent = true; } + } else { + // We don't have a dataset corresponding to the object of the relationship - do + // nothing } + + } else { + // Can't get subject, relationship, object from message + logger.info("Can't find the subject, relationship or object in the message - ignoring"); + } + } else { } + // This isn't a Relationship announcement message - ignore + logger.info("This is not a relationship announcement - ignoring message of type " + + msgObject.getJsonString("@type")); + } if (!sent) { - if (citingPID == null || citingType == null) { - throw new BadRequestException( - "Could not parse message to find acceptable citation link to a dataset."); - } else { - throw new ServiceUnavailableException( - "Unable to process message. Please contact the administrators."); - } + throw new ServiceUnavailableException("Unable to process message. Please contact the administrators."); } } else { logger.info("Ignoring message from IP address: " + origin.toString()); diff --git a/src/main/java/edu/harvard/iq/dataverse/workflow/internalspi/LDNAnnounceDatasetVersionStep.java b/src/main/java/edu/harvard/iq/dataverse/workflow/internalspi/LDNAnnounceDatasetVersionStep.java index 3478d9398f0..1b373ffefa8 100644 --- a/src/main/java/edu/harvard/iq/dataverse/workflow/internalspi/LDNAnnounceDatasetVersionStep.java +++ b/src/main/java/edu/harvard/iq/dataverse/workflow/internalspi/LDNAnnounceDatasetVersionStep.java @@ -18,8 +18,10 @@ import java.net.URI; import java.net.URISyntaxException; import java.nio.charset.StandardCharsets; -import java.util.Collection; +import java.util.Arrays; import java.util.HashMap; +import java.util.HashSet; +import java.util.Iterator; import java.util.List; import java.util.Map; import java.util.Map.Entry; @@ -32,6 +34,7 @@ import javax.json.JsonArrayBuilder; import javax.json.JsonObject; import javax.json.JsonObjectBuilder; +import javax.json.JsonString; import javax.json.JsonValue; import org.apache.http.client.methods.CloseableHttpResponse; @@ -56,9 +59,7 @@ public class LDNAnnounceDatasetVersionStep implements WorkflowStep { private static final String LDN_TARGET = ":LDNTarget"; private static final String RELATED_PUBLICATION = "publication"; - JsonLDTerm publicationIDType = null; - JsonLDTerm publicationIDNumber = null; - JsonLDTerm publicationURL = null; + public LDNAnnounceDatasetVersionStep(Map paramSet) { new HashMap<>(paramSet); @@ -77,32 +78,49 @@ public WorkflowStepResult run(WorkflowContext context) { HttpPost announcement; try { - announcement = buildAnnouncement(false, context, target); - } catch (URISyntaxException e) { - return new Failure("LDNAnnounceDatasetVersion workflow step failed: unable to parse inbox in :LDNTarget setting."); - } - if(announcement==null) { - logger.info(context.getDataset().getGlobalId().asString() + "does not have metadata required to send LDN message. Nothing sent."); - return OK; - } - // execute - try (CloseableHttpResponse response = client.execute(announcement)) { - int code = response.getStatusLine().getStatusCode(); - if (code >= 200 && code < 300) { - // HTTP OK range - return OK; - } else { - String responseBody = new String(response.getEntity().getContent().readAllBytes(), - StandardCharsets.UTF_8); - ; - return new Failure("Error communicating with " + inboxUrl + ". Server response: " + responseBody - + " (" + response + ")."); + // First check that we have what is required + Dataset d = context.getDataset(); + DatasetVersion dv = d.getReleasedVersion(); + List dvf = dv.getDatasetFields(); + Map fields = new HashMap(); + List reqFields = Arrays.asList(((String) context.getSettings().getOrDefault(REQUIRED_FIELDS, "")).split(",\\s*")); + for (DatasetField df : dvf) { + if(!df.isEmpty() && reqFields.contains(df.getDatasetFieldType().getName())) { + fields.put(df.getDatasetFieldType().getName(), df); + } + } + //Loop through and send a message for each supported relationship + boolean success=false; + for (JsonObject rel : getObjects(context, fields).getValuesAs(JsonObject.class)) { + announcement = buildAnnouncement(d, rel, target); + // execute + try (CloseableHttpResponse response = client.execute(announcement)) { + int code = response.getStatusLine().getStatusCode(); + if (code >= 200 && code < 300) { + // HTTP OK range + success=true; + logger.fine("Successfully sent message for " + rel.toString()); + } else { + String responseBody = new String(response.getEntity().getContent().readAllBytes(), + StandardCharsets.UTF_8); + ; + return new Failure((success ? "Partial failure":"") + "Error communicating with " + inboxUrl + " for relationship " + rel.toString() +". Server response: " + responseBody + + " (" + response + ")."); + } + + } catch (Exception ex) { + logger.log(Level.SEVERE, "Error communicating with remote server: " + ex.getMessage(), ex); + return new Failure((success ? "Partial failure":"") + "Error executing request: " + ex.getLocalizedMessage(), + "Cannot communicate with remote server."); + } + } + //Any failure and we would have returned already. + return OK; - } catch (Exception ex) { - logger.log(Level.SEVERE, "Error communicating with remote server: " + ex.getMessage(), ex); - return new Failure("Error executing request: " + ex.getLocalizedMessage(), - "Cannot communicate with remote server."); + + } catch (URISyntaxException e) { + return new Failure("LDNAnnounceDatasetVersion workflow step failed: unable to parse inbox in :LDNTarget setting."); } } return new Failure("LDNAnnounceDatasetVersion workflow step failed: :LDNTarget setting missing or invalid."); @@ -118,145 +136,113 @@ public void rollback(WorkflowContext context, Failure reason) { throw new UnsupportedOperationException("Not supported yet."); // This class does not need to resume. } - HttpPost buildAnnouncement(boolean qb, WorkflowContext ctxt, JsonObject target) throws URISyntaxException { - - // First check that we have what is required - DatasetVersion dv = ctxt.getDataset().getReleasedVersion(); - List dvf = dv.getDatasetFields(); - Map fields = new HashMap(); - String[] requiredFields = ((String) ctxt.getSettings().getOrDefault(REQUIRED_FIELDS, "")).split(",\\s*"); - for (String field : requiredFields) { - fields.put(field, null); - } - Set reqFields = fields.keySet(); - for (DatasetField df : dvf) { - if(!df.isEmpty() && reqFields.contains(df.getDatasetFieldType().getName())) { - fields.put(df.getDatasetFieldType().getName(), df); - } - } - if (fields.containsValue(null)) { - logger.fine("DatasetVersion doesn't contain metadata required to trigger announcement"); - return null; - } - // We do, so construct the json-ld body and method - + /**Scan through all fields and return an array of relationship JsonObjects with subjectId, relationship, objectId, and @context + * + * @param ctxt + * @param fields + * @return JsonArray of JsonObjects with subjectId, relationship, objectId, and @context + */ + JsonArray getObjects(WorkflowContext ctxt, Map fields) { + JsonArrayBuilder jab = Json.createArrayBuilder(); Map localContext = new HashMap(); - JsonObjectBuilder coarContext = Json.createObjectBuilder(); Map emptyCvocMap = new HashMap(); - boolean includeLocalContext = false; + + Dataset d = ctxt.getDataset(); for (Entry entry : fields.entrySet()) { DatasetField field = entry.getValue(); DatasetFieldType dft = field.getDatasetFieldType(); - String dfTypeName = entry.getKey(); JsonValue jv = OREMap.getJsonLDForField(field, false, emptyCvocMap, localContext); - switch (dfTypeName) { - case RELATED_PUBLICATION: - JsonArrayBuilder relArrayBuilder = Json.createArrayBuilder(); - publicationIDType = null; - publicationIDNumber = null; - publicationURL = null; - Collection childTypes = dft.getChildDatasetFieldTypes(); - for (DatasetFieldType cdft : childTypes) { - switch (cdft.getName()) { - case "publicationURL": - publicationURL = cdft.getJsonLDTerm(); - break; - case "publicationIDType": - publicationIDType = cdft.getJsonLDTerm(); - break; - case "publicationIDNumber": - publicationIDNumber = cdft.getJsonLDTerm(); - break; + //jv is a JsonArray for multi-val fields, so loop + if (jv != null) { + if (jv instanceof JsonArray) { + JsonArray rels = (JsonArray) jv; + Iterator iter = rels.iterator(); + while(iter.hasNext()) { + JsonValue jval =iter.next(); + jab.add(getRelationshipObject(dft, jval, d, localContext)); } - + } else { + jab.add(getRelationshipObject(dft, jv, d, localContext)); } + } + + } + return jab.build(); + } + + private JsonObject getRelationshipObject(DatasetFieldType dft, JsonValue jval, Dataset d, Map localContext) { + String id = getBestId(dft, jval); + return Json.createObjectBuilder().add("object", id).add("relationship", dft.getJsonLDTerm().getUrl()) + .add("subject", d.getGlobalId().toURL().toString()).build(); + } + - if (jv != null) { - if (jv instanceof JsonArray) { - JsonArray rels = (JsonArray) jv; - for (JsonObject jo : rels.getValuesAs(JsonObject.class)) { - String id = getBestPubId(jo); - relArrayBuilder.add(Json.createObjectBuilder().add("id", id).add("ietf:cite-as", id) - .add("type", "sorg:ScholaryArticle").build()); - } - } + HttpPost buildAnnouncement(Dataset d, JsonObject rel, JsonObject target) throws URISyntaxException { - else { // JsonObject - String id = getBestPubId((JsonObject) jv); - relArrayBuilder.add(Json.createObjectBuilder().add("id", id).add("ietf:cite-as", id) - .add("type", "sorg:ScholaryArticle").build()); - } - } - coarContext.add("IsSupplementTo", relArrayBuilder); - break; - default: - if (jv != null) { - includeLocalContext = true; - coarContext.add(dft.getJsonLDTerm().getLabel(), jv); - } - } - } - dvf.get(0).getDatasetFieldType().getName(); JsonObjectBuilder job = Json.createObjectBuilder(); JsonArrayBuilder context = Json.createArrayBuilder().add("https://purl.org/coar/notify") .add("https://www.w3.org/ns/activitystreams"); - if (includeLocalContext && !localContext.isEmpty()) { - JsonObjectBuilder contextBuilder = Json.createObjectBuilder(); - for (Entry e : localContext.entrySet()) { - contextBuilder.add(e.getKey(), e.getValue()); - } - context.add(contextBuilder); - } job.add("@context", context); job.add("id", "urn:uuid:" + UUID.randomUUID().toString()); job.add("actor", Json.createObjectBuilder().add("id", SystemConfig.getDataverseSiteUrlStatic()) .add("name", BrandingUtil.getInstallationBrandName()).add("type", "Service")); - job.add("context", coarContext); - Dataset d = ctxt.getDataset(); - job.add("object", - Json.createObjectBuilder().add("id", d.getLocalURL()) - .add("ietf:cite-as", d.getGlobalId().toURL().toExternalForm()) - .add("sorg:name", d.getDisplayName()).add("type", "sorg:Dataset")); + job.add("object", rel); job.add("origin", Json.createObjectBuilder().add("id", SystemConfig.getDataverseSiteUrlStatic()) .add("inbox", SystemConfig.getDataverseSiteUrlStatic() + "/api/inbox").add("type", "Service")); job.add("target", target); - job.add("type", Json.createArrayBuilder().add("Announce").add("coar-notify:ReleaseAction")); + job.add("type", Json.createArrayBuilder().add("Announce").add("coar-notify:RelationshipAction")); HttpPost annPost = new HttpPost(); annPost.setURI(new URI(target.getString("inbox"))); String body = JsonUtil.prettyPrint(job.build()); - logger.fine("Body: " + body); + logger.info("Body: " + body); annPost.setEntity(new StringEntity(JsonUtil.prettyPrint(body), "utf-8")); annPost.setHeader("Content-Type", "application/ld+json"); return annPost; } - private String getBestPubId(JsonObject jo) { + private String getBestId(DatasetFieldType dft, JsonValue jv) { + //Primitive value + if(jv instanceof JsonString) { + return ((JsonString)jv).getString(); + } + //Compound - apply type specific logic to get best Id String id = null; - if (jo.containsKey(publicationURL.getLabel())) { - id = jo.getString(publicationURL.getLabel()); - } else if (jo.containsKey(publicationIDType.getLabel())) { - if ((jo.containsKey(publicationIDNumber.getLabel()))) { - String number = jo.getString(publicationIDNumber.getLabel()); + switch (dft.getName()) { + case RELATED_PUBLICATION: + JsonLDTerm publicationIDType = null; + JsonLDTerm publicationIDNumber = null; + JsonLDTerm publicationURL = null; + JsonObject jo = jv.asJsonObject(); + if (jo.containsKey(publicationURL.getLabel())) { + id = jo.getString(publicationURL.getLabel()); + } else if (jo.containsKey(publicationIDType.getLabel())) { + if ((jo.containsKey(publicationIDNumber.getLabel()))) { + String number = jo.getString(publicationIDNumber.getLabel()); - switch (jo.getString(publicationIDType.getLabel())) { - case "doi": - if (number.startsWith("https://doi.org/")) { - id = number; - } else if (number.startsWith("doi:")) { - id = "https://doi.org/" + number.substring(4); - } + switch (jo.getString(publicationIDType.getLabel())) { + case "doi": + if (number.startsWith("https://doi.org/")) { + id = number; + } else if (number.startsWith("doi:")) { + id = "https://doi.org/" + number.substring(4); + } - break; - case "DASH-URN": - if (number.startsWith("http")) { - id = number; + break; + case "DASH-URN": + if (number.startsWith("http")) { + id = number; + } + break; } - break; } } + break; + default: + break; } + return id; } From 63aeed1abe9c6155937d7f10076007be0ead2fae Mon Sep 17 00:00:00 2001 From: qqmyers Date: Thu, 11 Aug 2022 15:01:08 -0400 Subject: [PATCH 02/40] add workflow settings to main list per qa --- doc/sphinx-guides/source/installation/config.rst | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/doc/sphinx-guides/source/installation/config.rst b/doc/sphinx-guides/source/installation/config.rst index 2d294980720..3405285fefb 100644 --- a/doc/sphinx-guides/source/installation/config.rst +++ b/doc/sphinx-guides/source/installation/config.rst @@ -943,7 +943,7 @@ Some external tools are also ready to be translated, especially if they are usin Tools for Translators -+++++++++++++++++++++++++++++++++++++++++++++++++++++++ ++++++++++++++++++++++ The list below depicts a set of tools that can be used to ease the amount of work necessary for translating the Dataverse software by facilitating this collaborative effort and enabling the reuse of previous work: @@ -2865,4 +2865,15 @@ For configuration details, see :ref:`mute-notifications`. :LDNMessageHosts ++++++++++++++++ -The comma-separated list of hosts allowed to send Dataverse Linked Data Notification messages. See :doc:`/api/linkeddatanotification` for details. ``*`` allows messages from anywhere (not recommended for production). By default, messages are not accepted from anywhere. +The comma-separated list of hosts allowed to send Dataverse Linked Data Notification messages. See :doc:`/api/linkeddatanotification` for details. ``*`` allows messages from anywhere (not recommended for production). By default, messages are not accepted from anywhere. + + +:LDN_TARGET ++++++++++++ + +The URL of an LDN Inbox to which the LDN Announce workflow step will send messages. See :doc:`/developers/workflows` for details. + +:LDNAnnounceRequiredFields +++++++++++++++++++++++++++ + +The list of parent dataset field names for which the LDN Announce workflow step should send messages. See :doc:`/developers/workflows` for details. From a4106590eece76b651d23d96b655409df96436c8 Mon Sep 17 00:00:00 2001 From: qqmyers Date: Thu, 11 Aug 2022 15:54:59 -0400 Subject: [PATCH 03/40] start adding default bestID --- .../LDNAnnounceDatasetVersionStep.java | 34 +++++++++++++++++-- 1 file changed, 32 insertions(+), 2 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/workflow/internalspi/LDNAnnounceDatasetVersionStep.java b/src/main/java/edu/harvard/iq/dataverse/workflow/internalspi/LDNAnnounceDatasetVersionStep.java index 1b373ffefa8..b2376ff89c3 100644 --- a/src/main/java/edu/harvard/iq/dataverse/workflow/internalspi/LDNAnnounceDatasetVersionStep.java +++ b/src/main/java/edu/harvard/iq/dataverse/workflow/internalspi/LDNAnnounceDatasetVersionStep.java @@ -19,6 +19,7 @@ import java.net.URISyntaxException; import java.nio.charset.StandardCharsets; import java.util.Arrays; +import java.util.Collection; import java.util.HashMap; import java.util.HashSet; import java.util.Iterator; @@ -208,13 +209,30 @@ private String getBestId(DatasetFieldType dft, JsonValue jv) { return ((JsonString)jv).getString(); } //Compound - apply type specific logic to get best Id + JsonObject jo = jv.asJsonObject(); String id = null; switch (dft.getName()) { case RELATED_PUBLICATION: JsonLDTerm publicationIDType = null; JsonLDTerm publicationIDNumber = null; JsonLDTerm publicationURL = null; - JsonObject jo = jv.asJsonObject(); + + Collection childTypes = dft.getChildDatasetFieldTypes(); + for (DatasetFieldType cdft : childTypes) { + switch (cdft.getName()) { + case "publicationURL": + publicationURL = cdft.getJsonLDTerm(); + break; + case "publicationIDType": + publicationIDType = cdft.getJsonLDTerm(); + break; + case "publicationIDNumber": + publicationIDNumber = cdft.getJsonLDTerm(); + break; + } + } + + if (jo.containsKey(publicationURL.getLabel())) { id = jo.getString(publicationURL.getLabel()); } else if (jo.containsKey(publicationIDType.getLabel())) { @@ -227,8 +245,10 @@ private String getBestId(DatasetFieldType dft, JsonValue jv) { id = number; } else if (number.startsWith("doi:")) { id = "https://doi.org/" + number.substring(4); + } else { + //Assume a raw DOI, e.g. 10.5072/FK2ABCDEF + id = "https://doi.org/" + number; } - break; case "DASH-URN": if (number.startsWith("http")) { @@ -240,6 +260,16 @@ private String getBestId(DatasetFieldType dft, JsonValue jv) { } break; default: + Collection childDFTs = dft.getChildDatasetFieldTypes(); + for (DatasetFieldType cdft : childDFTs) { + String fieldname = cdft.getName(); + if(fieldname.contains("URL")) { + if(jo.containsKey(cdft.getJsonLDTerm().getLabel())) { + id=jo.getString(cdft.getJsonLDTerm().getLabel()); + break; + } + } + } break; } From 8b69c0f630ba26147e417339b12bdd99537833fc Mon Sep 17 00:00:00 2001 From: qqmyers Date: Thu, 11 Aug 2022 16:29:55 -0400 Subject: [PATCH 04/40] replace system.out.println call --- .../java/edu/harvard/iq/dataverse/util/json/JSONLDUtil.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/util/json/JSONLDUtil.java b/src/main/java/edu/harvard/iq/dataverse/util/json/JSONLDUtil.java index 127632bf711..f58dd0893c1 100644 --- a/src/main/java/edu/harvard/iq/dataverse/util/json/JSONLDUtil.java +++ b/src/main/java/edu/harvard/iq/dataverse/util/json/JSONLDUtil.java @@ -540,7 +540,7 @@ public static JsonObject decontextualizeJsonLD(String jsonLDString) { logger.fine("Decontextualized object: " + jsonld); return jsonld; } catch (JsonLdError e) { - System.out.println(e.getMessage()); + logger.warning(e.getMessage()); return null; } } From cbd20faf6e6ca8f74e68e48535ff6148a8680dad Mon Sep 17 00:00:00 2001 From: qqmyers Date: Thu, 11 Aug 2022 16:30:43 -0400 Subject: [PATCH 05/40] Improve default compound field handling and some reformatting --- .../LDNAnnounceDatasetVersionStep.java | 105 +++++++++++------- 1 file changed, 66 insertions(+), 39 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/workflow/internalspi/LDNAnnounceDatasetVersionStep.java b/src/main/java/edu/harvard/iq/dataverse/workflow/internalspi/LDNAnnounceDatasetVersionStep.java index b2376ff89c3..ed19bd42a8e 100644 --- a/src/main/java/edu/harvard/iq/dataverse/workflow/internalspi/LDNAnnounceDatasetVersionStep.java +++ b/src/main/java/edu/harvard/iq/dataverse/workflow/internalspi/LDNAnnounceDatasetVersionStep.java @@ -60,8 +60,6 @@ public class LDNAnnounceDatasetVersionStep implements WorkflowStep { private static final String LDN_TARGET = ":LDNTarget"; private static final String RELATED_PUBLICATION = "publication"; - - public LDNAnnounceDatasetVersionStep(Map paramSet) { new HashMap<>(paramSet); } @@ -76,7 +74,7 @@ public WorkflowStepResult run(WorkflowContext context) { CloseableHttpClient client = HttpClients.createDefault(); // build method - + HttpPost announcement; try { // First check that we have what is required @@ -84,14 +82,15 @@ public WorkflowStepResult run(WorkflowContext context) { DatasetVersion dv = d.getReleasedVersion(); List dvf = dv.getDatasetFields(); Map fields = new HashMap(); - List reqFields = Arrays.asList(((String) context.getSettings().getOrDefault(REQUIRED_FIELDS, "")).split(",\\s*")); + List reqFields = Arrays + .asList(((String) context.getSettings().getOrDefault(REQUIRED_FIELDS, "")).split(",\\s*")); for (DatasetField df : dvf) { - if(!df.isEmpty() && reqFields.contains(df.getDatasetFieldType().getName())) { + if (!df.isEmpty() && reqFields.contains(df.getDatasetFieldType().getName())) { fields.put(df.getDatasetFieldType().getName(), df); } } - //Loop through and send a message for each supported relationship - boolean success=false; + // Loop through and send a message for each supported relationship + boolean success = false; for (JsonObject rel : getObjects(context, fields).getValuesAs(JsonObject.class)) { announcement = buildAnnouncement(d, rel, target); // execute @@ -99,29 +98,30 @@ public WorkflowStepResult run(WorkflowContext context) { int code = response.getStatusLine().getStatusCode(); if (code >= 200 && code < 300) { // HTTP OK range - success=true; + success = true; logger.fine("Successfully sent message for " + rel.toString()); } else { String responseBody = new String(response.getEntity().getContent().readAllBytes(), StandardCharsets.UTF_8); ; - return new Failure((success ? "Partial failure":"") + "Error communicating with " + inboxUrl + " for relationship " + rel.toString() +". Server response: " + responseBody - + " (" + response + ")."); + return new Failure((success ? "Partial failure" : "") + "Error communicating with " + + inboxUrl + " for relationship " + rel.toString() + ". Server response: " + + responseBody + " (" + response + ")."); } } catch (Exception ex) { logger.log(Level.SEVERE, "Error communicating with remote server: " + ex.getMessage(), ex); - return new Failure((success ? "Partial failure":"") + "Error executing request: " + ex.getLocalizedMessage(), - "Cannot communicate with remote server."); + return new Failure((success ? "Partial failure" : "") + "Error executing request: " + + ex.getLocalizedMessage(), "Cannot communicate with remote server."); } } - //Any failure and we would have returned already. + // Any failure and we would have returned already. return OK; - } catch (URISyntaxException e) { - return new Failure("LDNAnnounceDatasetVersion workflow step failed: unable to parse inbox in :LDNTarget setting."); + return new Failure( + "LDNAnnounceDatasetVersion workflow step failed: unable to parse inbox in :LDNTarget setting."); } } return new Failure("LDNAnnounceDatasetVersion workflow step failed: :LDNTarget setting missing or invalid."); @@ -137,11 +137,14 @@ public void rollback(WorkflowContext context, Failure reason) { throw new UnsupportedOperationException("Not supported yet."); // This class does not need to resume. } - /**Scan through all fields and return an array of relationship JsonObjects with subjectId, relationship, objectId, and @context + /** + * Scan through all fields and return an array of relationship JsonObjects with + * subjectId, relationship, objectId, and @context * * @param ctxt * @param fields - * @return JsonArray of JsonObjects with subjectId, relationship, objectId, and @context + * @return JsonArray of JsonObjects with subjectId, relationship, objectId, + * and @context */ JsonArray getObjects(WorkflowContext ctxt, Map fields) { JsonArrayBuilder jab = Json.createArrayBuilder(); @@ -153,34 +156,33 @@ JsonArray getObjects(WorkflowContext ctxt, Map fields) { DatasetField field = entry.getValue(); DatasetFieldType dft = field.getDatasetFieldType(); JsonValue jv = OREMap.getJsonLDForField(field, false, emptyCvocMap, localContext); - //jv is a JsonArray for multi-val fields, so loop + // jv is a JsonArray for multi-val fields, so loop if (jv != null) { if (jv instanceof JsonArray) { JsonArray rels = (JsonArray) jv; Iterator iter = rels.iterator(); - while(iter.hasNext()) { - JsonValue jval =iter.next(); - jab.add(getRelationshipObject(dft, jval, d, localContext)); + while (iter.hasNext()) { + JsonValue jval = iter.next(); + jab.add(getRelationshipObject(dft, jval, d, localContext)); } } else { jab.add(getRelationshipObject(dft, jv, d, localContext)); } } - + } return jab.build(); } - - private JsonObject getRelationshipObject(DatasetFieldType dft, JsonValue jval, Dataset d, Map localContext) { + + private JsonObject getRelationshipObject(DatasetFieldType dft, JsonValue jval, Dataset d, + Map localContext) { String id = getBestId(dft, jval); return Json.createObjectBuilder().add("object", id).add("relationship", dft.getJsonLDTerm().getUrl()) .add("subject", d.getGlobalId().toURL().toString()).build(); } - HttpPost buildAnnouncement(Dataset d, JsonObject rel, JsonObject target) throws URISyntaxException { - JsonObjectBuilder job = Json.createObjectBuilder(); JsonArrayBuilder context = Json.createArrayBuilder().add("https://purl.org/coar/notify") .add("https://www.w3.org/ns/activitystreams"); @@ -204,11 +206,11 @@ HttpPost buildAnnouncement(Dataset d, JsonObject rel, JsonObject target) throws } private String getBestId(DatasetFieldType dft, JsonValue jv) { - //Primitive value - if(jv instanceof JsonString) { - return ((JsonString)jv).getString(); + // Primitive value + if (jv instanceof JsonString) { + return ((JsonString) jv).getString(); } - //Compound - apply type specific logic to get best Id + // Compound - apply type specific logic to get best Id JsonObject jo = jv.asJsonObject(); String id = null; switch (dft.getName()) { @@ -216,7 +218,7 @@ private String getBestId(DatasetFieldType dft, JsonValue jv) { JsonLDTerm publicationIDType = null; JsonLDTerm publicationIDNumber = null; JsonLDTerm publicationURL = null; - + Collection childTypes = dft.getChildDatasetFieldTypes(); for (DatasetFieldType cdft : childTypes) { switch (cdft.getName()) { @@ -231,8 +233,6 @@ private String getBestId(DatasetFieldType dft, JsonValue jv) { break; } } - - if (jo.containsKey(publicationURL.getLabel())) { id = jo.getString(publicationURL.getLabel()); } else if (jo.containsKey(publicationIDType.getLabel())) { @@ -246,7 +246,7 @@ private String getBestId(DatasetFieldType dft, JsonValue jv) { } else if (number.startsWith("doi:")) { id = "https://doi.org/" + number.substring(4); } else { - //Assume a raw DOI, e.g. 10.5072/FK2ABCDEF + // Assume a raw DOI, e.g. 10.5072/FK2ABCDEF id = "https://doi.org/" + number; } break; @@ -261,18 +261,45 @@ private String getBestId(DatasetFieldType dft, JsonValue jv) { break; default: Collection childDFTs = dft.getChildDatasetFieldTypes(); + // Loop through child fields and select one + // The order of preference is for a field with URL in the name, followed by one + // with 'ID',then 'Name', and as a last resort, a field. for (DatasetFieldType cdft : childDFTs) { String fieldname = cdft.getName(); - if(fieldname.contains("URL")) { - if(jo.containsKey(cdft.getJsonLDTerm().getLabel())) { - id=jo.getString(cdft.getJsonLDTerm().getLabel()); + if (fieldname.contains("URL")) { + if (jo.containsKey(cdft.getJsonLDTerm().getLabel())) { + id = jo.getString(cdft.getJsonLDTerm().getLabel()); break; } } } - break; - } + if (id == null) { + for (DatasetFieldType cdft : childDFTs) { + String fieldname = cdft.getName(); + if (fieldname.contains("ID") || fieldname.contains("Id")) { + if (jo.containsKey(cdft.getJsonLDTerm().getLabel())) { + id = jo.getString(cdft.getJsonLDTerm().getLabel()); + break; + } + + } + } + } + if (id == null) { + for (DatasetFieldType cdft : childDFTs) { + String fieldname = cdft.getName(); + + if (fieldname.contains("Name")) { + if (jo.containsKey(cdft.getJsonLDTerm().getLabel())) { + id = jo.getString(cdft.getJsonLDTerm().getLabel()); + break; + } + } + } + } + id = jo.getString(jo.keySet().iterator().next()); + } return id; } From 94d40e6c95a7786c671ac4c1abfca1eed21deb33 Mon Sep 17 00:00:00 2001 From: qqmyers Date: Thu, 11 Aug 2022 19:37:46 -0400 Subject: [PATCH 06/40] add template custom instructions info --- .../source/user/dataverse-management.rst | 29 ++++++++++--------- 1 file changed, 15 insertions(+), 14 deletions(-) diff --git a/doc/sphinx-guides/source/user/dataverse-management.rst b/doc/sphinx-guides/source/user/dataverse-management.rst index efe98e8327c..ed90497da8c 100755 --- a/doc/sphinx-guides/source/user/dataverse-management.rst +++ b/doc/sphinx-guides/source/user/dataverse-management.rst @@ -44,7 +44,7 @@ To edit your Dataverse collection, navigate to your Dataverse collection's landi - :ref:`Theme `: upload a logo for your Dataverse collection, add a link to your department or personal website, add a custom footer image, and select colors for your Dataverse collection in order to brand it - :ref:`Widgets `: get code to add to your website to have your Dataverse collection display on it - :ref:`Permissions `: give other users permissions to your Dataverse collection, i.e.-can edit datasets, and see which users already have which permissions for your Dataverse collection -- :ref:`Dataset Templates `: these are useful when you have several datasets that have the same information in multiple metadata fields that you would prefer not to have to keep manually typing in +- :ref:`Dataset Templates `: these are useful when you want to provide custom instructions on how to fill out fields or have several datasets that have the same information in multiple metadata fields that you would prefer not to have to keep manually typing in - :ref:`Dataset Guestbooks `: allows you to collect data about who is downloading the files from your datasets - :ref:`Featured Dataverse collections `: if you have one or more Dataverse collection, you can use this option to show them at the top of your Dataverse collection page to help others easily find interesting or important Dataverse collections - **Delete Dataverse**: you are able to delete your Dataverse collection as long as it is not published and does not have any draft datasets @@ -52,7 +52,7 @@ To edit your Dataverse collection, navigate to your Dataverse collection's landi .. _general-information: General Information ---------------------- +------------------- The General Information page is how you edit the information you filled in while creating your Dataverse collection. If you need to change or add a contact email address, this is the place to do it. Additionally, you can update the metadata elements used for datasets within the Dataverse collection, change which metadata fields are hidden, required, or optional, and update the facets you would like displayed for browsing the Dataverse collection. If you plan on using templates, you need to select the metadata fields on the General Information page. @@ -60,8 +60,8 @@ Tip: The metadata fields you select as required will appear on the Create Datase .. _theme: -Theme ---------- +Theme +----- The Theme features provides you with a way to customize the look of your Dataverse collection. You can: @@ -77,7 +77,7 @@ Supported image types for logo images and footer images are JPEG, TIFF, or PNG a .. _dataverse-widgets: Widgets --------------- +------- The Widgets feature provides you with code for you to put on your personal website to have your Dataverse collection displayed there. There are two types of Widgets for a Dataverse collection, a Dataverse collection Search Box widget and a Dataverse collection Listing widget. Once a Dataverse collection has been published, from the Widgets tab on the Dataverse collection's Theme + Widgets page, it is possible to copy the code snippets for the widget(s) you would like to add to your website. If you need to adjust the height of the widget on your website, you may do so by editing the `heightPx=500` parameter in the code snippet. @@ -94,7 +94,7 @@ The Dataverse Collection Listing Widget provides a listing of all your Dataverse .. _openscholar-dataverse-level: Adding Widgets to an OpenScholar Website -****************************************** +**************************************** #. Log in to your OpenScholar website #. Either build a new page or navigate to the page you would like to use to show the Dataverse collection widgets. #. Click on the Settings Cog and select Layout @@ -102,8 +102,8 @@ Adding Widgets to an OpenScholar Website .. _dataverse-permissions: -Roles & Permissions ---------------------- +Roles & Permissions +------------------- Dataverse installation user accounts can be granted roles that define which actions they are allowed to take on specific Dataverse collections, datasets, and/or files. Each role comes with a set of permissions, which define the specific actions that users may take. Roles and permissions may also be granted to groups. Groups can be defined as a collection of Dataverse installation user accounts, a collection of IP addresses (e.g. all users of a library's computers), or a collection of all users who log in using a particular institutional login (e.g. everyone who logs in with a particular university's account credentials). @@ -127,7 +127,7 @@ When you access a Dataverse collection's permissions page, you will see three se Please note that even on a newly created Dataverse collection, you may see user and groups have already been granted role(s) if your installation has ``:InheritParentRoleAssignments`` set. For more on this setting, see the :doc:`/installation/config` section of the Installation Guide. Setting Access Configurations -******************************* +***************************** Under the Permissions tab, you can click the "Edit Access" button to open a box where you can add to your Dataverse collection and what permissions are granted to those who add to your Dataverse collection. @@ -140,7 +140,7 @@ The second question on this page allows you to choose the role (and thus the per Both of these settings can be changed at any time. Assigning Roles to Users and Groups -************************************* +*********************************** Under the Users/Groups tab, you can add, edit, or remove the roles granted to users and groups on your Dataverse collection. A role is a set of permissions granted to a user or group when they're using your Dataverse collection. For example, giving your research assistant the "Contributor" role would give them the following self-explanatory permissions on your Dataverse collection and all datasets within your Dataverse collection: "ViewUnpublishedDataset", "DownloadFile", "EditDataset", and "DeleteDatasetDraft". They would, however, lack the "PublishDataset" permission, and thus would be unable to publish datasets on your Dataverse collection. If you wanted to give them that permission, you would give them a role with that permission, like the Curator role. Users and groups can hold multiple roles at the same time if needed. Roles can be removed at any time. All roles and their associated permissions are listed under the "Roles" tab of the same page. @@ -155,15 +155,16 @@ Note: If you need to assign a role to ALL user accounts in a Dataverse installat .. _dataset-templates: Dataset Templates -------------------- +----------------- -Templates are useful when you have several datasets that have the same information in multiple metadata fields that you would prefer not to have to keep manually typing in, or if you want to use a custom set of Terms of Use and Access for multiple datasets in a Dataverse collection. In Dataverse Software 4.0+, templates are created at the Dataverse collection level, can be deleted (so it does not show for future datasets), set to default (not required), or can be copied so you do not have to start over when creating a new template with similar metadata from another template. When a template is deleted, it does not impact the datasets that have used the template already. +Templates are useful when you want to provide custom instructions on how to fill out a field, have several datasets that have the same information in multiple metadata fields that you would prefer not to have to keep manually typing in, or if you want to use a custom set of Terms of Use and Access for multiple datasets in a Dataverse collection. In Dataverse Software 4.0+, templates are created at the Dataverse collection level, can be deleted (so it does not show for future datasets), set to default (not required), or can be copied so you do not have to start over when creating a new template with similar metadata from another template. When a template is deleted, it does not impact the datasets that have used the template already. How do you create a template? #. Navigate to your Dataverse collection, click on the Edit Dataverse button and select Dataset Templates. #. Once you have clicked on Dataset Templates, you will be brought to the Dataset Templates page. On this page, you can 1) decide to use the dataset templates from your parent Dataverse collection 2) create a new dataset template or 3) do both. #. Click on the Create Dataset Template to get started. You will see that the template is the same as the create dataset page with an additional field at the top of the page to add a name for the template. +#. To add custom instructions, click on ''(None - click to add)'' and enter the instructions you wish users to see. If you wish to edit existing instructions, click on them to make the text editable. #. After adding information into the metadata fields you have information for and clicking Save and Add Terms, you will be brought to the page where you can add custom Terms of Use and Access. If you do not need custom Terms of Use and Access, click the Save Dataset Template, and only the metadata fields will be saved. #. After clicking Save Dataset Template, you will be brought back to the Manage Dataset Templates page and should see your template listed there now with the make default, edit, view, or delete options. #. A Dataverse collection does not have to have a default template and users can select which template they would like to use while on the Create Dataset page. @@ -174,7 +175,7 @@ How do you create a template? .. _dataset-guestbooks: Dataset Guestbooks ------------------------------ +------------------ Guestbooks allow you to collect data about who is downloading the files from your datasets. You can decide to collect account information (username, given name & last name, affiliation, etc.) as well as create custom questions (e.g., What do you plan to use this data for?). You are also able to download the data collected from the enabled guestbooks as CSV files to store and use outside of the Dataverse installation. @@ -227,7 +228,7 @@ Similarly to dataset linking, Dataverse collection linking allows a Dataverse co If you need to have a Dataverse collection linked to your Dataverse collection, please contact the support team for the Dataverse installation you are using. Publish Your Dataverse Collection -================================================================= +================================= Once your Dataverse collection is ready to go public, go to your Dataverse collection page, click on the "Publish" button on the right hand side of the page. A pop-up will appear to confirm that you are ready to actually Publish, since once a Dataverse collection From aa881820f537c283eb9b8caa999dddc63c8dc89f Mon Sep 17 00:00:00 2001 From: qqmyers Date: Fri, 12 Aug 2022 14:16:59 -0400 Subject: [PATCH 07/40] get blocks from metadataroot --- src/main/java/edu/harvard/iq/dataverse/Template.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/Template.java b/src/main/java/edu/harvard/iq/dataverse/Template.java index ba601c1df87..61f0a78656f 100644 --- a/src/main/java/edu/harvard/iq/dataverse/Template.java +++ b/src/main/java/edu/harvard/iq/dataverse/Template.java @@ -272,7 +272,7 @@ public void setMetadataValueBlocks() { Map instructionsMap = getInstructionsMap(); List viewMDB = new ArrayList<>(); - List editMDB=this.getDataverse().getMetadataBlocks(true); + List editMDB=this.getDataverse().getMetadataBlocks(false); //The metadatablocks in this template include any from the Dataverse it is associated with //plus any others where the template has a displayable field (i.e. from before a block was dropped in the dataverse/collection) From 62fe2143d61ed93d09c160961c315982b61e37f8 Mon Sep 17 00:00:00 2001 From: qqmyers Date: Thu, 1 Sep 2022 14:55:03 -0400 Subject: [PATCH 08/40] fix parsing to match spec/dash msg --- .../java/edu/harvard/iq/dataverse/api/LDNInbox.java | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/api/LDNInbox.java b/src/main/java/edu/harvard/iq/dataverse/api/LDNInbox.java index fabc067abf7..5e434ec6bef 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/LDNInbox.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/LDNInbox.java @@ -104,15 +104,13 @@ public Response acceptMessage(String body) { String objectKey = new JsonLDTerm(activityStreams, "object").getUrl(); if (jsonld.containsKey(objectKey)) { JsonObject msgObject = jsonld.getJsonObject(objectKey); - if (new JsonLDTerm(activityStreams, "Relationship").getUrl().equals(msgObject.getJsonString("@type"))) { + if (new JsonLDTerm(activityStreams, "Relationship").getUrl().equals(msgObject.getJsonString("type"))) { // We have a relationship message - need to get the subject and object and // relationship type - String subjectId = msgObject.getJsonObject(new JsonLDTerm(activityStreams, "subject").getUrl()) - .getString("@id"); - String objectId = msgObject.getJsonObject(new JsonLDTerm(activityStreams, "object").getUrl()) - .getString("@id"); + String subjectId = msgObject.getString((new JsonLDTerm(activityStreams, "subject").getUrl()); + String objectId = msgObject.getString(new JsonLDTerm(activityStreams, "object").getUrl()); String relationshipId = msgObject - .getJsonObject(new JsonLDTerm(activityStreams, "relationship").getUrl()).getString("@id"); + .getString(new JsonLDTerm(activityStreams, "relationship").getUrl()); if (subjectId != null && objectId != null && relationshipId != null) { // Strip the URL part from a relationship ID/URL assuming a usable label exists // after a # char. Default is to use the whole URI. From e5228af91781be53ba69e1ad803d03130383d026 Mon Sep 17 00:00:00 2001 From: qqmyers Date: Thu, 1 Sep 2022 16:04:31 -0400 Subject: [PATCH 09/40] use getString for 'Relationship', revert other changes --- .../java/edu/harvard/iq/dataverse/api/LDNInbox.java | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/api/LDNInbox.java b/src/main/java/edu/harvard/iq/dataverse/api/LDNInbox.java index 5e434ec6bef..96f15dd5c81 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/LDNInbox.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/LDNInbox.java @@ -16,6 +16,7 @@ import edu.harvard.iq.dataverse.util.json.JSONLDUtil; import edu.harvard.iq.dataverse.util.json.JsonLDNamespace; import edu.harvard.iq.dataverse.util.json.JsonLDTerm; +import edu.harvard.iq.dataverse.util.json.JsonUtil; import java.util.Date; import java.util.Map; @@ -96,6 +97,8 @@ public Response acceptMessage(String body) { if (jsonld == null) { throw new BadRequestException("Could not parse message to find acceptable citation link to a dataset."); } + //To Do - lower level for PR + logger.info(JsonUtil.prettyPrint(jsonld)); String relationship = "isRelatedTo"; String name = null; JsonLDNamespace activityStreams = JsonLDNamespace.defineNamespace("as", @@ -104,13 +107,13 @@ public Response acceptMessage(String body) { String objectKey = new JsonLDTerm(activityStreams, "object").getUrl(); if (jsonld.containsKey(objectKey)) { JsonObject msgObject = jsonld.getJsonObject(objectKey); - if (new JsonLDTerm(activityStreams, "Relationship").getUrl().equals(msgObject.getJsonString("type"))) { + if (new JsonLDTerm(activityStreams, "Relationship").getUrl().equals(msgObject.getString("@type"))) { // We have a relationship message - need to get the subject and object and // relationship type - String subjectId = msgObject.getString((new JsonLDTerm(activityStreams, "subject").getUrl()); - String objectId = msgObject.getString(new JsonLDTerm(activityStreams, "object").getUrl()); + String subjectId = msgObject.getJsonObject(new JsonLDTerm(activityStreams, "subject").getUrl()).getString("@id"); + String objectId = msgObject.getJsonObject(new JsonLDTerm(activityStreams, "object").getUrl()).getString("@id"); String relationshipId = msgObject - .getString(new JsonLDTerm(activityStreams, "relationship").getUrl()); + .getJsonObject(new JsonLDTerm(activityStreams, "relationship").getUrl()).getString("@id"); if (subjectId != null && objectId != null && relationshipId != null) { // Strip the URL part from a relationship ID/URL assuming a usable label exists // after a # char. Default is to use the whole URI. From fb56d0413bda84901432310364f2a96093829739 Mon Sep 17 00:00:00 2001 From: qqmyers Date: Thu, 1 Sep 2022 16:04:51 -0400 Subject: [PATCH 10/40] temporarily drop name/type in display --- src/main/webapp/dataverseuser.xhtml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/main/webapp/dataverseuser.xhtml b/src/main/webapp/dataverseuser.xhtml index 9fb6c3bdac0..59b02744a56 100644 --- a/src/main/webapp/dataverseuser.xhtml +++ b/src/main/webapp/dataverseuser.xhtml @@ -406,9 +406,11 @@ - + + + + - #{item.theObject.getDisplayName()} From 8ecb8e46f1a3d0910176a6f3dbcd06eea62cc853 Mon Sep 17 00:00:00 2001 From: qqmyers Date: Thu, 1 Sep 2022 16:13:57 -0400 Subject: [PATCH 11/40] misplaced } --- src/main/java/edu/harvard/iq/dataverse/api/LDNInbox.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/api/LDNInbox.java b/src/main/java/edu/harvard/iq/dataverse/api/LDNInbox.java index 96f15dd5c81..b69a22d5630 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/LDNInbox.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/LDNInbox.java @@ -163,11 +163,11 @@ public Response acceptMessage(String body) { } } else { - } + // This isn't a Relationship announcement message - ignore logger.info("This is not a relationship announcement - ignoring message of type " + msgObject.getJsonString("@type")); - + } } if (!sent) { From 789b3c91211ef1561792bd76846c3bd6242c8b76 Mon Sep 17 00:00:00 2001 From: qqmyers Date: Tue, 6 Sep 2022 15:42:40 -0400 Subject: [PATCH 12/40] Add callback loop and make display tolerant wrt name/type being found --- .../harvard/iq/dataverse/api/LDNInbox.java | 63 +++++++++++++++++-- src/main/java/propertyFiles/Bundle.properties | 1 + src/main/webapp/dataverseuser.xhtml | 6 +- 3 files changed, 62 insertions(+), 8 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/api/LDNInbox.java b/src/main/java/edu/harvard/iq/dataverse/api/LDNInbox.java index b69a22d5630..32bb1d12b36 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/LDNInbox.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/LDNInbox.java @@ -23,12 +23,15 @@ import java.util.Optional; import java.util.Set; import java.io.StringWriter; +import java.net.URI; +import java.net.URL; import java.sql.Timestamp; import java.util.logging.Logger; import javax.ejb.EJB; import javax.json.Json; import javax.json.JsonObject; +import javax.json.JsonObjectBuilder; import javax.json.JsonValue; import javax.json.JsonWriter; import javax.servlet.http.HttpServletRequest; @@ -41,6 +44,13 @@ import javax.ws.rs.core.Context; import javax.ws.rs.core.Response; +import org.apache.commons.httpclient.HttpClient; +import org.apache.http.client.methods.CloseableHttpResponse; +import org.apache.http.client.methods.HttpGet; +import org.apache.http.impl.client.CloseableHttpClient; +import org.apache.http.impl.client.HttpClients; +import org.apache.http.util.EntityUtils; + @Path("inbox") public class LDNInbox extends AbstractApiBean { @@ -99,8 +109,9 @@ public Response acceptMessage(String body) { } //To Do - lower level for PR logger.info(JsonUtil.prettyPrint(jsonld)); - String relationship = "isRelatedTo"; + //String relationship = "isRelatedTo"; String name = null; + String itemType = null; JsonLDNamespace activityStreams = JsonLDNamespace.defineNamespace("as", "https://www.w3.org/ns/activitystreams#"); JsonLDNamespace ietf = JsonLDNamespace.defineNamespace("ietf", "http://www.iana.org/assignments/relation/"); @@ -112,25 +123,69 @@ public Response acceptMessage(String body) { // relationship type String subjectId = msgObject.getJsonObject(new JsonLDTerm(activityStreams, "subject").getUrl()).getString("@id"); String objectId = msgObject.getJsonObject(new JsonLDTerm(activityStreams, "object").getUrl()).getString("@id"); + // Best-effort to get name by following redirects and looing for a 'name' field in the returned json + try (CloseableHttpClient client = HttpClients.createDefault()) { + logger.fine("Getting " + subjectId); + HttpGet get = new HttpGet(new URI(subjectId)); + get.addHeader("Accept", "application/json"); + + int statusCode=0; + do { + CloseableHttpResponse response = client.execute(get); + statusCode = response.getStatusLine().getStatusCode(); + switch (statusCode) { + case 302: + case 303: + String location=response.getFirstHeader("location").getValue(); + logger.fine("Redirecting to: " + location); + get = new HttpGet(location); + get.addHeader("Accept", "application/json"); + + break; + case 200: + String responseString = EntityUtils.toString(response.getEntity(), "UTF-8"); + logger.fine("Received: " + responseString); + JsonObject job = JsonUtil.getJsonObject(responseString); + name = job.getString("name", null); + itemType = job.getString("type", null); + break; + default: + logger.fine("Received " + statusCode + " when accessing " + objectId); + } + } while(statusCode == 302); + } catch (Exception e) { + logger.fine("Unable to get name from " + objectId); + logger.fine(e.getLocalizedMessage()); + } String relationshipId = msgObject .getJsonObject(new JsonLDTerm(activityStreams, "relationship").getUrl()).getString("@id"); if (subjectId != null && objectId != null && relationshipId != null) { // Strip the URL part from a relationship ID/URL assuming a usable label exists // after a # char. Default is to use the whole URI. - relationship = relationshipId.substring(relationship.indexOf("#") + 1); + int index = relationshipId.indexOf("#"); + logger.fine("Found # at " + index + " in " + relationshipId); + String relationship = relationshipId.substring(index + 1); // Parse the URI as a PID and see if this Dataverse instance has this dataset String pid = GlobalId.getInternalFormOfPID(objectId); Optional id = GlobalId.parse(pid); if (id.isPresent()) { Dataset dataset = datasetSvc.findByGlobalId(pid); if (dataset != null) { - JsonObject citingResource = Json.createObjectBuilder().add("@id", subjectId) - .add("relationship", relationship).build(); + JsonObjectBuilder citingResourceBuilder = Json.createObjectBuilder().add("@id", subjectId) + .add("relationship", relationship); + if(name!=null && !name.isBlank()) { + citingResourceBuilder.add("name",name); + } + if(itemType!=null && !itemType.isBlank()) { + citingResourceBuilder.add("@type",itemType.substring(0,1).toUpperCase() + itemType.substring(1)); + } + JsonObject citingResource = citingResourceBuilder.build(); StringWriter sw = new StringWriter(128); try (JsonWriter jw = Json.createWriter(sw)) { jw.write(citingResource); } String jsonstring = sw.toString(); + logger.fine("Storing: " + jsonstring); Set ras = roleService.rolesAssignments(dataset); roleService.rolesAssignments(dataset).stream() diff --git a/src/main/java/propertyFiles/Bundle.properties b/src/main/java/propertyFiles/Bundle.properties index 5b6216aaff1..74d6cbc3d60 100644 --- a/src/main/java/propertyFiles/Bundle.properties +++ b/src/main/java/propertyFiles/Bundle.properties @@ -217,6 +217,7 @@ notification.workflowFailed=An external workflow run on {0} in {1} has failed. C notification.workflowSucceeded=An external workflow run on {0} in {1} has succeeded. Check your email and/or view the Dataset page which may have additional details. notification.statusUpdated=The status of dataset {0} has been updated to {1}. notification.datasetMentioned=Announcement Received: Newly released {0} {2} {3} Dataset {4}. +notification.datasetMentioned.itemType=Resource notification.ingestCompleted=Dataset {1} has one or more tabular files that completed the tabular ingest process and are available in archival formats. notification.ingestCompletedWithErrors=Dataset {1} has one or more tabular files that are available but are not supported for tabular ingest. diff --git a/src/main/webapp/dataverseuser.xhtml b/src/main/webapp/dataverseuser.xhtml index 59b02744a56..5538072b7be 100644 --- a/src/main/webapp/dataverseuser.xhtml +++ b/src/main/webapp/dataverseuser.xhtml @@ -406,11 +406,9 @@ - - - - + + #{item.theObject.getDisplayName()} From 37ebf7dd04066e772b2ef2819abfb961860fd38a Mon Sep 17 00:00:00 2001 From: qqmyers Date: Fri, 23 Sep 2022 10:24:09 -0400 Subject: [PATCH 13/40] add fields missing in https://notify.coar-repositories.org/scenarios/10/ --- .../workflow/internalspi/LDNAnnounceDatasetVersionStep.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/workflow/internalspi/LDNAnnounceDatasetVersionStep.java b/src/main/java/edu/harvard/iq/dataverse/workflow/internalspi/LDNAnnounceDatasetVersionStep.java index ed19bd42a8e..2aea94d08e5 100644 --- a/src/main/java/edu/harvard/iq/dataverse/workflow/internalspi/LDNAnnounceDatasetVersionStep.java +++ b/src/main/java/edu/harvard/iq/dataverse/workflow/internalspi/LDNAnnounceDatasetVersionStep.java @@ -178,7 +178,7 @@ private JsonObject getRelationshipObject(DatasetFieldType dft, JsonValue jval, D Map localContext) { String id = getBestId(dft, jval); return Json.createObjectBuilder().add("object", id).add("relationship", dft.getJsonLDTerm().getUrl()) - .add("subject", d.getGlobalId().toURL().toString()).build(); + .add("subject", d.getGlobalId().toURL().toString()).add("id", "urn:uuid:" + UUID.randomUUID().toString()).add("type","Relationship").build(); } HttpPost buildAnnouncement(Dataset d, JsonObject rel, JsonObject target) throws URISyntaxException { From ceb3ff576efc6da0ba50da43c2c2eaac10a8f172 Mon Sep 17 00:00:00 2001 From: qqmyers Date: Wed, 26 Oct 2022 16:12:31 -0400 Subject: [PATCH 14/40] debugging --- src/main/java/edu/harvard/iq/dataverse/api/LDNInbox.java | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/main/java/edu/harvard/iq/dataverse/api/LDNInbox.java b/src/main/java/edu/harvard/iq/dataverse/api/LDNInbox.java index 32bb1d12b36..c916cff9179 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/LDNInbox.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/LDNInbox.java @@ -80,6 +80,7 @@ public class LDNInbox extends AbstractApiBean { @Path("/") @Consumes("application/ld+json, application/json-ld") public Response acceptMessage(String body) { + try { IpAddress origin = new DataverseRequest(null, httpRequest).getSourceAddress(); String whitelist = settingsService.get(SettingsServiceBean.Key.LDNMessageHosts.toString(), ""); // Only do something if we listen to this host @@ -232,6 +233,12 @@ public Response acceptMessage(String body) { logger.info("Ignoring message from IP address: " + origin.toString()); throw new ForbiddenException("Inbox does not acept messages from this address"); } + } catch (Throwable t) { + logger.severe(t.getLocalizedMessage()); + t.printStackTrace(); + + throw t; + } return ok("Message Received"); } } From 62888e26fdecae70c55e5eab9e0f72683ef3af54 Mon Sep 17 00:00:00 2001 From: qqmyers Date: Thu, 27 Oct 2022 12:09:39 -0400 Subject: [PATCH 15/40] added logging/switched to info for HDC debugging --- .../edu/harvard/iq/dataverse/api/LDNInbox.java | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/api/LDNInbox.java b/src/main/java/edu/harvard/iq/dataverse/api/LDNInbox.java index c916cff9179..4cf272c1d88 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/LDNInbox.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/LDNInbox.java @@ -126,7 +126,7 @@ public Response acceptMessage(String body) { String objectId = msgObject.getJsonObject(new JsonLDTerm(activityStreams, "object").getUrl()).getString("@id"); // Best-effort to get name by following redirects and looing for a 'name' field in the returned json try (CloseableHttpClient client = HttpClients.createDefault()) { - logger.fine("Getting " + subjectId); + logger.info("Getting " + subjectId); HttpGet get = new HttpGet(new URI(subjectId)); get.addHeader("Accept", "application/json"); @@ -138,25 +138,25 @@ public Response acceptMessage(String body) { case 302: case 303: String location=response.getFirstHeader("location").getValue(); - logger.fine("Redirecting to: " + location); + logger.info("Redirecting to: " + location); get = new HttpGet(location); get.addHeader("Accept", "application/json"); break; case 200: String responseString = EntityUtils.toString(response.getEntity(), "UTF-8"); - logger.fine("Received: " + responseString); + logger.info("Received: " + responseString); JsonObject job = JsonUtil.getJsonObject(responseString); name = job.getString("name", null); itemType = job.getString("type", null); break; default: - logger.fine("Received " + statusCode + " when accessing " + objectId); + logger.info("Received " + statusCode + " when accessing " + subjectId); } } while(statusCode == 302); } catch (Exception e) { - logger.fine("Unable to get name from " + objectId); - logger.fine(e.getLocalizedMessage()); + logger.info("Unable to get name from " + subjectId); + logger.info(e.getLocalizedMessage()); } String relationshipId = msgObject .getJsonObject(new JsonLDTerm(activityStreams, "relationship").getUrl()).getString("@id"); @@ -164,7 +164,7 @@ public Response acceptMessage(String body) { // Strip the URL part from a relationship ID/URL assuming a usable label exists // after a # char. Default is to use the whole URI. int index = relationshipId.indexOf("#"); - logger.fine("Found # at " + index + " in " + relationshipId); + logger.info("Found # at " + index + " in " + relationshipId); String relationship = relationshipId.substring(index + 1); // Parse the URI as a PID and see if this Dataverse instance has this dataset String pid = GlobalId.getInternalFormOfPID(objectId); @@ -186,7 +186,7 @@ public Response acceptMessage(String body) { jw.write(citingResource); } String jsonstring = sw.toString(); - logger.fine("Storing: " + jsonstring); + logger.info("Storing: " + jsonstring); Set ras = roleService.rolesAssignments(dataset); roleService.rolesAssignments(dataset).stream() @@ -209,6 +209,7 @@ public Response acceptMessage(String body) { sent = true; } } else { + logger.info("Didn't find dataset"); // We don't have a dataset corresponding to the object of the relationship - do // nothing } From d714a1a2b68ff77eaa01f07422bcd64c95595c4b Mon Sep 17 00:00:00 2001 From: qqmyers Date: Tue, 23 Apr 2024 17:15:29 -0400 Subject: [PATCH 16/40] merge issues --- .../workflow/internalspi/LDNAnnounceDatasetVersionStep.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/workflow/internalspi/LDNAnnounceDatasetVersionStep.java b/src/main/java/edu/harvard/iq/dataverse/workflow/internalspi/LDNAnnounceDatasetVersionStep.java index 9d29cba42ff..abe9f26f98d 100644 --- a/src/main/java/edu/harvard/iq/dataverse/workflow/internalspi/LDNAnnounceDatasetVersionStep.java +++ b/src/main/java/edu/harvard/iq/dataverse/workflow/internalspi/LDNAnnounceDatasetVersionStep.java @@ -35,6 +35,7 @@ import jakarta.json.JsonArrayBuilder; import jakarta.json.JsonObject; import jakarta.json.JsonObjectBuilder; +import jakarta.json.JsonString; import jakarta.json.JsonValue; import org.apache.http.client.methods.CloseableHttpResponse; @@ -177,7 +178,7 @@ private JsonObject getRelationshipObject(DatasetFieldType dft, JsonValue jval, D Map localContext) { String id = getBestId(dft, jval); return Json.createObjectBuilder().add("object", id).add("relationship", dft.getJsonLDTerm().getUrl()) - .add("subject", d.getGlobalId().toURL().toString()).add("id", "urn:uuid:" + UUID.randomUUID().toString()).add("type","Relationship").build(); + .add("subject", d.getGlobalId().asURL().toString()).add("id", "urn:uuid:" + UUID.randomUUID().toString()).add("type","Relationship").build(); } HttpPost buildAnnouncement(Dataset d, JsonObject rel, JsonObject target) throws URISyntaxException { From 9cdb4f6396612fbf58bdb9451f3c8ad6f3e6a0f7 Mon Sep 17 00:00:00 2001 From: qqmyers Date: Fri, 3 Oct 2025 10:57:37 -0400 Subject: [PATCH 17/40] updates to send context, use as namespace, use relationType, DASH-NRS --- .../LDNAnnounceDatasetVersionStep.java | 33 ++++++++++++++----- 1 file changed, 24 insertions(+), 9 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/workflow/internalspi/LDNAnnounceDatasetVersionStep.java b/src/main/java/edu/harvard/iq/dataverse/workflow/internalspi/LDNAnnounceDatasetVersionStep.java index abe9f26f98d..deb7ccfbb9f 100644 --- a/src/main/java/edu/harvard/iq/dataverse/workflow/internalspi/LDNAnnounceDatasetVersionStep.java +++ b/src/main/java/edu/harvard/iq/dataverse/workflow/internalspi/LDNAnnounceDatasetVersionStep.java @@ -176,20 +176,26 @@ JsonArray getObjects(WorkflowContext ctxt, Map fields) { private JsonObject getRelationshipObject(DatasetFieldType dft, JsonValue jval, Dataset d, Map localContext) { - String id = getBestId(dft, jval); - return Json.createObjectBuilder().add("object", id).add("relationship", dft.getJsonLDTerm().getUrl()) - .add("subject", d.getGlobalId().asURL().toString()).add("id", "urn:uuid:" + UUID.randomUUID().toString()).add("type","Relationship").build(); + String[] answers = getBestIdAndType(dft, jval); + String id = answers[0]; + String type = answers[1]; + return Json.createObjectBuilder().add("as:object", id).add("as:relationship", type) + .add("as:subject", d.getGlobalId().asURL().toString()).add("id", "urn:uuid:" + UUID.randomUUID().toString()).add("type","Relationship").build(); } HttpPost buildAnnouncement(Dataset d, JsonObject rel, JsonObject target) throws URISyntaxException { JsonObjectBuilder job = Json.createObjectBuilder(); - JsonArrayBuilder context = Json.createArrayBuilder().add("https://purl.org/coar/notify") - .add("https://www.w3.org/ns/activitystreams"); + JsonArrayBuilder context = Json.createArrayBuilder() + .add("https://www.w3.org/ns/activitystreams") + .add("https://coar-notify.net"); job.add("@context", context); job.add("id", "urn:uuid:" + UUID.randomUUID().toString()); job.add("actor", Json.createObjectBuilder().add("id", SystemConfig.getDataverseSiteUrlStatic()) .add("name", BrandingUtil.getInstallationBrandName()).add("type", "Service")); + JsonObjectBuilder coarContextBuilder = Json.createObjectBuilder(); + coarContextBuilder.add("id", rel.getString("as:object")); + job.add("context", coarContextBuilder.build()); job.add("object", rel); job.add("origin", Json.createObjectBuilder().add("id", SystemConfig.getDataverseSiteUrlStatic()) .add("inbox", SystemConfig.getDataverseSiteUrlStatic() + "/api/inbox").add("type", "Service")); @@ -205,10 +211,12 @@ HttpPost buildAnnouncement(Dataset d, JsonObject rel, JsonObject target) throws return annPost; } - private String getBestId(DatasetFieldType dft, JsonValue jv) { + private String[] getBestIdAndType(DatasetFieldType dft, JsonValue jv) { + + String type = "https://purl.org/datacite/ontology#isSupplementTo"; // Primitive value if (jv instanceof JsonString) { - return ((JsonString) jv).getString(); + return new String[] { ((JsonString) jv).getString(), type }; } // Compound - apply type specific logic to get best Id JsonObject jo = jv.asJsonObject(); @@ -218,6 +226,7 @@ private String getBestId(DatasetFieldType dft, JsonValue jv) { JsonLDTerm publicationIDType = null; JsonLDTerm publicationIDNumber = null; JsonLDTerm publicationURL = null; + JsonLDTerm publicationRelationType = null; Collection childTypes = dft.getChildDatasetFieldTypes(); for (DatasetFieldType cdft : childTypes) { @@ -231,6 +240,8 @@ private String getBestId(DatasetFieldType dft, JsonValue jv) { case "publicationIDNumber": publicationIDNumber = cdft.getJsonLDTerm(); break; + case "publicationRelationType": + publicationRelationType = cdft.getJsonLDTerm(); } } if (jo.containsKey(publicationURL.getLabel())) { @@ -250,7 +261,7 @@ private String getBestId(DatasetFieldType dft, JsonValue jv) { id = "https://doi.org/" + number; } break; - case "DASH-URN": + case "DASH-NRS": if (number.startsWith("http")) { id = number; } @@ -258,6 +269,10 @@ private String getBestId(DatasetFieldType dft, JsonValue jv) { } } } + if(jo.containsKey(publicationRelationType.getLabel())) { + type = jo.getString(publicationRelationType.getLabel()); + type = "https://purl.org/datacite/ontology#" + type.substring(0,1).toLowerCase() + type.substring(1); + } break; default: Collection childDFTs = dft.getChildDatasetFieldTypes(); @@ -300,7 +315,7 @@ private String getBestId(DatasetFieldType dft, JsonValue jv) { } id = jo.getString(jo.keySet().iterator().next()); } - return id; + return new String[] {id, type}; } String process(String template, Map values) { From 0ffbb8e9aba9d99ba55f87fc565433e4a8c1459a Mon Sep 17 00:00:00 2001 From: qqmyers Date: Mon, 6 Oct 2025 13:59:26 -0400 Subject: [PATCH 18/40] doc updates --- doc/release-notes/10490-COAR-Notify.md | 19 +++++++++++++++++++ .../source/developers/workflows.rst | 5 +++-- 2 files changed, 22 insertions(+), 2 deletions(-) create mode 100644 doc/release-notes/10490-COAR-Notify.md diff --git a/doc/release-notes/10490-COAR-Notify.md b/doc/release-notes/10490-COAR-Notify.md new file mode 100644 index 00000000000..934288d4d4c --- /dev/null +++ b/doc/release-notes/10490-COAR-Notify.md @@ -0,0 +1,19 @@ +### Support for COAR Notify Relationship Announcement + +Dataverse now supports sending and recieving [Linked Data Notification ](https://www.w3.org/TR/ldn/) messages involved in the +[COAR Notify Relationship Announcement Workflow](https://coar-notify.net/catalogue/workflows/repository-relationship-repository/). + +Dataverse can send messages to configured repositories announcing that a dataset has a related publication (as defined in the dataset metadata). This may be done automatically upon publication or triggered manually by a superuser. The receiving repository may do anything with the message, with the default expectation being that the repository will create a backlink from the publication to the dataset (assuming the publication exists in the repository, admins agree the link makes sense, etc.) + +Conversely, Dataverse can recieve notices from other configured repositories announcing relationships between their publications and datasets. If the referenced dataset exists in the Dataverse instance, a notification will be sent to users who can publish the dataset. They can then decide whether to create a backlink to the publication in the dataset metadata. + +(Earlier releases of Dataverse had experimental support in this area that was based on message formats defined prior to finalization of the COAR Notify specification for relationship announcements.) + +Configuration for sending messages involves specifying the +:LDNTarget and :LDNAnnounceRequiredFields + +Configuration to receive messages involves specifying the +:LDNMessageHosts + +(FWIW: These settings are not new) + diff --git a/doc/sphinx-guides/source/developers/workflows.rst b/doc/sphinx-guides/source/developers/workflows.rst index 38ca6f4e141..a7f6c8a044c 100644 --- a/doc/sphinx-guides/source/developers/workflows.rst +++ b/doc/sphinx-guides/source/developers/workflows.rst @@ -205,13 +205,13 @@ Note - the example step includes two settings required for any archiver, three ( ldnannounce +++++++++++ -An experimental step that sends a Linked Data Notification (LDN) message to a specific LDN Inbox announcing the publication/availability of a dataset meeting certain criteria. +A step that sends a `Linked Data Notification (LDN)`_ message to a specific LDN Inbox announcing a relationship between an newly published/available dataset with a relationship to an external resource (e.g. one managed by the recipient). The two parameters are * ``:LDNAnnounceRequiredFields`` - a list of metadata fields that must exist to trigger the message. Currently, the message also includes the values for these fields but future versions may only send the dataset's persistent identifier (making the receiver responsible for making a call-back to get any metadata). * ``:LDNTarget`` - a JSON object containing an ``inbox`` key whose value is the URL of the target LDN inbox to which messages should be sent, e.g. ``{"id": "https://dashv7-dev.lib.harvard.edu","inbox": "https://dashv7-api-dev.lib.harvard.edu/server/ldn/inbox","type": "Service"}`` ). -The supported message format is desribed by `our preliminary specification `_. The format is expected to change in the near future to match the standard for relationship announcements being developed as part of `the COAR Notify Project `_. +The message format is defined by the `COAR Notify Relationship Announcement `_ standard. .. code:: json @@ -224,6 +224,7 @@ The supported message format is desribed by `our preliminary specification Date: Mon, 6 Oct 2025 13:59:35 -0400 Subject: [PATCH 19/40] note some ToDos --- .../internalspi/LDNAnnounceDatasetVersionStep.java | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/workflow/internalspi/LDNAnnounceDatasetVersionStep.java b/src/main/java/edu/harvard/iq/dataverse/workflow/internalspi/LDNAnnounceDatasetVersionStep.java index deb7ccfbb9f..7d2aa166461 100644 --- a/src/main/java/edu/harvard/iq/dataverse/workflow/internalspi/LDNAnnounceDatasetVersionStep.java +++ b/src/main/java/edu/harvard/iq/dataverse/workflow/internalspi/LDNAnnounceDatasetVersionStep.java @@ -56,6 +56,8 @@ public class LDNAnnounceDatasetVersionStep implements WorkflowStep { private static final Logger logger = Logger.getLogger(LDNAnnounceDatasetVersionStep.class.getName()); + //ToDo - not required fields at this point - each results in a message, so a) change to LDNAnnounceFields, and b) consider settings + // connecting field and targets (only DB settings are supported in workflows at present) private static final String REQUIRED_FIELDS = ":LDNAnnounceRequiredFields"; private static final String LDN_TARGET = ":LDNTarget"; private static final String RELATED_PUBLICATION = "publication"; @@ -274,7 +276,10 @@ private String[] getBestIdAndType(DatasetFieldType dft, JsonValue jv) { type = "https://purl.org/datacite/ontology#" + type.substring(0,1).toLowerCase() + type.substring(1); } break; - default: + default: + //ToDo - handle primary field + //ToDo - handle "Identifier" vs "IdentifierType" + //ToDo - check for URL form Collection childDFTs = dft.getChildDatasetFieldTypes(); // Loop through child fields and select one // The order of preference is for a field with URL in the name, followed by one From 26e0ee45fc1f6c3f29d077d79201d3075660942a Mon Sep 17 00:00:00 2001 From: qqmyers Date: Tue, 7 Oct 2025 13:11:41 -0400 Subject: [PATCH 20/40] New format has strings, not objects with @id --- .../java/edu/harvard/iq/dataverse/api/LDNInbox.java | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/api/LDNInbox.java b/src/main/java/edu/harvard/iq/dataverse/api/LDNInbox.java index b9bddb02cfe..6125027f41f 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/LDNInbox.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/LDNInbox.java @@ -119,8 +119,8 @@ public Response acceptMessage(String body) { if (new JsonLDTerm(activityStreams, "Relationship").getUrl().equals(msgObject.getString("@type"))) { // We have a relationship message - need to get the subject and object and // relationship type - String subjectId = msgObject.getJsonObject(new JsonLDTerm(activityStreams, "subject").getUrl()).getString("@id"); - String objectId = msgObject.getJsonObject(new JsonLDTerm(activityStreams, "object").getUrl()).getString("@id"); + String subjectId = msgObject.getString(new JsonLDTerm(activityStreams, "subject").getUrl()); + String objectId = msgObject.getString(new JsonLDTerm(activityStreams, "object").getUrl()); // Best-effort to get name by following redirects and looing for a 'name' field in the returned json try (CloseableHttpClient client = HttpClients.createDefault()) { logger.info("Getting " + subjectId); @@ -156,7 +156,7 @@ public Response acceptMessage(String body) { logger.info(e.getLocalizedMessage()); } String relationshipId = msgObject - .getJsonObject(new JsonLDTerm(activityStreams, "relationship").getUrl()).getString("@id"); + .getString(new JsonLDTerm(activityStreams, "relationship").getUrl()); if (subjectId != null && objectId != null && relationshipId != null) { // Strip the URL part from a relationship ID/URL assuming a usable label exists // after a # char. Default is to use the whole URI. @@ -178,11 +178,7 @@ public Response acceptMessage(String body) { citingResourceBuilder.add("@type",itemType.substring(0,1).toUpperCase() + itemType.substring(1)); } JsonObject citingResource = citingResourceBuilder.build(); - StringWriter sw = new StringWriter(128); - try (JsonWriter jw = Json.createWriter(sw)) { - jw.write(citingResource); - } - String jsonstring = sw.toString(); + String jsonstring = JsonUtil.prettyPrint(citingResource); logger.info("Storing: " + jsonstring); //Set ras = roleService.rolesAssignments(dataset); From f4a4ed504e6f3b8c3f8a5c5dc1a6df623ec6f9db Mon Sep 17 00:00:00 2001 From: qqmyers Date: Fri, 24 Oct 2025 16:50:42 -0400 Subject: [PATCH 21/40] add space to fix link --- doc/sphinx-guides/source/developers/workflows.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/sphinx-guides/source/developers/workflows.rst b/doc/sphinx-guides/source/developers/workflows.rst index a7f6c8a044c..8248144d4d8 100644 --- a/doc/sphinx-guides/source/developers/workflows.rst +++ b/doc/sphinx-guides/source/developers/workflows.rst @@ -205,7 +205,7 @@ Note - the example step includes two settings required for any archiver, three ( ldnannounce +++++++++++ -A step that sends a `Linked Data Notification (LDN)`_ message to a specific LDN Inbox announcing a relationship between an newly published/available dataset with a relationship to an external resource (e.g. one managed by the recipient). +A step that sends a `Linked Data Notification (LDN) `_ message to a specific LDN Inbox announcing a relationship between an newly published/available dataset with a relationship to an external resource (e.g. one managed by the recipient). The two parameters are * ``:LDNAnnounceRequiredFields`` - a list of metadata fields that must exist to trigger the message. Currently, the message also includes the values for these fields but future versions may only send the dataset's persistent identifier (making the receiver responsible for making a call-back to get any metadata). From 76304a4e0b98372e999c3a240e42705911c3973b Mon Sep 17 00:00:00 2001 From: qqmyers Date: Mon, 27 Oct 2025 17:22:42 -0400 Subject: [PATCH 22/40] more doc tweaks --- .../source/api/linkeddatanotification.rst | 55 +++++++++++-------- .../source/developers/workflows.rst | 4 +- .../internal-ldnannounce-workflow.json | 3 +- 3 files changed, 37 insertions(+), 25 deletions(-) diff --git a/doc/sphinx-guides/source/api/linkeddatanotification.rst b/doc/sphinx-guides/source/api/linkeddatanotification.rst index d55dc4da084..0bf65b48f99 100644 --- a/doc/sphinx-guides/source/api/linkeddatanotification.rst +++ b/doc/sphinx-guides/source/api/linkeddatanotification.rst @@ -1,7 +1,10 @@ Linked Data Notification API ============================ -Dataverse has a limited, experimental API implementing a Linked Data Notification inbox allowing it to receive messages indicating a link between an external resource and a Dataverse dataset. +Dataverse has an API implementing a Linked Data Notification (LDN) inbox allowing it to receive messages implementing the `COAR Notify Relationship Announcement `_ indicating a link between an external resource and a Dataverse dataset. + +Dataverse has a related capability to send COAR Notify Relationship Announcement messages, automatically upon publication or manually. See the :doc:`/developers/workflows` section of the Guides. + The motivating use case is to support a use case where Dataverse administrators may wish to create back-links to the remote resource (e.g. as a Related Publication, Related Material, etc.). Upon receipt of a relevant message, Dataverse will create Announcement Received notifications for superusers, who can edit the dataset involved. (In the motivating use case, these users may then add an appropriate relationship and use the Update Curent Version publishing option to add it to the most recently published version of the dataset.) @@ -15,10 +18,13 @@ Messages can be sent via POST, using the application/ld+json ContentType: export SERVER_URL=https://demo.dataverse.org curl -X POST -H 'ContentType:application/ld+json' $SERVER_URL/api/inbox --upload-file message.jsonld + +https://coar-notify.net/catalogue/workflows/repository-relationship-repository/2/ -The supported message format is described by `our preliminary specification `_. The format is expected to change in the near future to match the standard for relationship announcements being developed as part of `the COAR Notify Project `_. +The supported message format is described by `the COAR Notify Relationship Announcement specification `_. -An example message is shown below. It indicates that a resource with the name "An Interesting Title" exists and "IsSupplementedBy" the dataset with DOI https://doi.org/10.5072/FK2/GGCCDL. If this dataset is managed in the receiving Dataverse, a notification will be sent to user with the relevant permissions (as described above). +An example message is shown below. It indicates that a resource in the "Harvard DASH" test server has, as a "supplement", the dataset with DOI doi:10.5074/FKNOAHNQ. +If this dataset is managed in the receiving Dataverse, a notification will be sent to user with the relevant permissions (as described above). .. code:: json @@ -27,39 +33,44 @@ An example message is shown below. It indicates that a resource with the name "A "https://www.w3.org/ns/activitystreams", "https://purl.org/coar/notify" ], - "id": "urn:uuid:94ecae35-dcfd-4182-8550-22c7164fe23f", "actor": { - "id": "https://research-organisation.org/dspace", - "name": "DSpace Repository", + "id": "https://harvard-dash.staging.4science.cloud", + "name": "Harvard DASH", "type": "Service" }, "context": { - "IsSupplementedBy": - { - "id": "http://dev-hdc3b.lib.harvard.edu/dataset.xhtml?persistentId=doi:10.5072/FK2/GGCCDL", - "ietf:cite-as": "https://doi.org/10.5072/FK2/GGCCDL", - "type": "sorg:Dataset" - } + "id": "https://harvard-dash.staging.4science.cloud/handle/1/42718322", + "ietf:cite-as": "https://harvard-dash.staging.4science.cloud/handle/1/42718322", + "ietf:item": { + "id": "https://harvard-dash.staging.4science.cloud/bitstreams/e2ae80a1-35e5-411b-9ef1-9175f6cccf23/download", + "mediaType": "application/pdf", + "type": [ + "Article", + "sorg:ScholarlyArticle" + ] + }, + "type": "sorg:AboutPage" }, + "id": "urn:uuid:3c933c09-c246-473d-bea4-674db168cfee", "object": { - "id": "https://research-organisation.org/dspace/item/35759679-5df3-4633-b7e5-4cf24b4d0614", - "ietf:cite-as": "https://research-organisation.org/authority/resolve/35759679-5df3-4633-b7e5-4cf24b4d0614", - "sorg:name": "An Interesting Title", - "type": "sorg:ScholarlyArticle" + "as:object": "doi: 10.5074/FKNOAHNQ", + "as:relationship": "http://purl.org/vocab/frbr/core#supplement", + "as:subject": "https://harvard-dash.staging.4science.cloud/handle/1/42718322", + "id": "urn:uuid:0851f805-c52f-4d0b-81ac-a07e99c33e20", + "type": "Relationship" }, "origin": { - "id": "https://research-organisation.org/dspace", - "inbox": "https://research-organisation.org/dspace/inbox/", + "id": "https://harvard-dash.staging.4science.cloud", + "inbox": "https://harvard-dash.staging.4science.cloud/server/ldn/inbox", "type": "Service" }, "target": { - "id": "https://research-organisation.org/dataverse", - "inbox": "https://research-organisation.org/dataverse/inbox/", + "id": "http://ec2-3-238-245-253.compute-1.amazonaws.com/", + "inbox": "http://ec2-3-238-245-253.compute-1.amazonaws.com/api/inbox", "type": "Service" }, "type": [ "Announce", - "coar-notify:ReleaseAction" + "coar-notify:RelationshipAction" ] } - diff --git a/doc/sphinx-guides/source/developers/workflows.rst b/doc/sphinx-guides/source/developers/workflows.rst index 8248144d4d8..0d7104e4631 100644 --- a/doc/sphinx-guides/source/developers/workflows.rst +++ b/doc/sphinx-guides/source/developers/workflows.rst @@ -205,7 +205,7 @@ Note - the example step includes two settings required for any archiver, three ( ldnannounce +++++++++++ -A step that sends a `Linked Data Notification (LDN) `_ message to a specific LDN Inbox announcing a relationship between an newly published/available dataset with a relationship to an external resource (e.g. one managed by the recipient). +A step that sends a `Linked Data Notification (LDN) `_ message to a specific LDN Inbox announcing a relationship between a newly published/available dataset and an external resource (e.g. one managed by the recipient). The two parameters are * ``:LDNAnnounceRequiredFields`` - a list of metadata fields that must exist to trigger the message. Currently, the message also includes the values for these fields but future versions may only send the dataset's persistent identifier (making the receiver responsible for making a call-back to get any metadata). @@ -221,7 +221,7 @@ The message format is defined by the `COAR Notify Relationship Announcement Date: Wed, 29 Oct 2025 14:41:50 -0400 Subject: [PATCH 23/40] refactor, use JVMSettings, add tests, add superusers only flag --- .../harvard/iq/dataverse/api/LDNInbox.java | 264 +++++--------- .../COARNotifyRelationshipAnnouncement.java | 305 ++++++++++++++++ .../iq/dataverse/settings/JvmSettings.java | 7 + .../harvard/iq/dataverse/api/LDNInboxIT.java | 334 ++++++++++++++++++ .../edu/harvard/iq/dataverse/api/UtilIT.java | 9 + 5 files changed, 750 insertions(+), 169 deletions(-) create mode 100644 src/main/java/edu/harvard/iq/dataverse/api/ldn/COARNotifyRelationshipAnnouncement.java create mode 100644 src/test/java/edu/harvard/iq/dataverse/api/LDNInboxIT.java diff --git a/src/main/java/edu/harvard/iq/dataverse/api/LDNInbox.java b/src/main/java/edu/harvard/iq/dataverse/api/LDNInbox.java index 6125027f41f..798c0095fb9 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/LDNInbox.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/LDNInbox.java @@ -1,40 +1,31 @@ + package edu.harvard.iq.dataverse.api; -import edu.harvard.iq.dataverse.Dataset; import edu.harvard.iq.dataverse.DatasetServiceBean; import edu.harvard.iq.dataverse.DataverseRoleServiceBean; -import edu.harvard.iq.dataverse.GlobalId; import edu.harvard.iq.dataverse.MailServiceBean; import edu.harvard.iq.dataverse.RoleAssigneeServiceBean; -import edu.harvard.iq.dataverse.RoleAssignment; -import edu.harvard.iq.dataverse.UserNotification; import edu.harvard.iq.dataverse.UserNotificationServiceBean; -import edu.harvard.iq.dataverse.authorization.Permission; +import edu.harvard.iq.dataverse.api.ldn.COARNotifyRelationshipAnnouncement; import edu.harvard.iq.dataverse.authorization.groups.impl.ipaddress.ip.IpAddress; import edu.harvard.iq.dataverse.engine.command.DataverseRequest; -import edu.harvard.iq.dataverse.pidproviders.PidProvider; +import edu.harvard.iq.dataverse.settings.JvmSettings; import edu.harvard.iq.dataverse.settings.SettingsServiceBean; import edu.harvard.iq.dataverse.util.json.JSONLDUtil; import edu.harvard.iq.dataverse.util.json.JsonLDNamespace; import edu.harvard.iq.dataverse.util.json.JsonLDTerm; import edu.harvard.iq.dataverse.util.json.JsonUtil; -import java.util.Date; -import java.util.Optional; -import java.util.Set; -import java.io.StringWriter; -import java.net.URI; -import java.sql.Timestamp; +import java.util.logging.Level; import java.util.logging.Logger; import jakarta.ejb.EJB; -import jakarta.json.Json; import jakarta.json.JsonObject; -import jakarta.json.JsonObjectBuilder; -import jakarta.json.JsonWriter; import jakarta.servlet.http.HttpServletRequest; import jakarta.ws.rs.BadRequestException; +import jakarta.ws.rs.ClientErrorException; import jakarta.ws.rs.ServiceUnavailableException; +import jakarta.ws.rs.WebApplicationException; import jakarta.ws.rs.Consumes; import jakarta.ws.rs.ForbiddenException; import jakarta.ws.rs.POST; @@ -42,12 +33,6 @@ import jakarta.ws.rs.core.Context; import jakarta.ws.rs.core.Response; -import org.apache.http.client.methods.CloseableHttpResponse; -import org.apache.http.client.methods.HttpGet; -import org.apache.http.impl.client.CloseableHttpClient; -import org.apache.http.impl.client.HttpClients; -import org.apache.http.util.EntityUtils; - @Path("inbox") public class LDNInbox extends AbstractApiBean { @@ -70,169 +55,110 @@ public class LDNInbox extends AbstractApiBean { @EJB RoleAssigneeServiceBean roleAssigneeService; + @Context protected HttpServletRequest httpRequest; + public static final JsonLDNamespace activityStreams = JsonLDNamespace.defineNamespace("as", + "https://www.w3.org/ns/activitystreams#"); + public static final String objectKey = new JsonLDTerm(activityStreams, "object").getUrl(); + @POST @Path("/") @Consumes("application/ld+json, application/json-ld") public Response acceptMessage(String body) { try { - IpAddress origin = new DataverseRequest(null, httpRequest).getSourceAddress(); - String whitelist = settingsService.get(SettingsServiceBean.Key.LDNMessageHosts.toString(), ""); - // Only do something if we listen to this host - if (whitelist.equals("*") || whitelist.contains(origin.toString())) { - boolean sent = false; - - JsonObject jsonld = null; - jsonld = JSONLDUtil.decontextualizeJsonLD(body); - if (jsonld == null) { - // Kludge - something about the coar notify URL causes a - // LOADING_REMOTE_CONTEXT_FAILED error in the titanium library - so replace it - // and try with a local copy - body = body.replace("\"https://purl.org/coar/notify\"", - "{\n" + " \"@vocab\": \"http://purl.org/coar/notify_vocabulary/\",\n" - + " \"ietf\": \"http://www.iana.org/assignments/relation/\",\n" - + " \"coar-notify\": \"http://purl.org/coar/notify_vocabulary/\",\n" - + " \"sorg\": \"http://schema.org/\",\n" - + " \"ReviewAction\": \"coar-notify:ReviewAction\",\n" - + " \"EndorsementAction\": \"coar-notify:EndorsementAction\",\n" - + " \"IngestAction\": \"coar-notify:IngestAction\",\n" - + " \"ietf:cite-as\": {\n" + " \"@type\": \"@id\"\n" - + " }}"); - jsonld = JSONLDUtil.decontextualizeJsonLD(body); + IpAddress origin = new DataverseRequest(null, httpRequest).getSourceAddress(); + String allowedIPs = JvmSettings.LINKEDDATANOTIFICATION_ALLOWED_HOSTS.lookupOptional().orElse(""); + + // Only process messages from whitelisted hosts + if (!allowedIPs.equals("*") && !allowedIPs.contains(origin.toString())) { + logger.info("Ignoring message from IP address: " + origin.toString()); + throw new ForbiddenException("The LDN Inbox does not accept messages from this address"); } + + // Parse JSON-LD message + JsonObject jsonld = parseJsonLD(body); if (jsonld == null) { - throw new BadRequestException("Could not parse message to find acceptable citation link to a dataset."); + throw new BadRequestException("Could not parse JSON message."); } - //To Do - lower level for PR + logger.info(JsonUtil.prettyPrint(jsonld)); - //String relationship = "isRelatedTo"; - String name = null; - String itemType = null; - JsonLDNamespace activityStreams = JsonLDNamespace.defineNamespace("as", - "https://www.w3.org/ns/activitystreams#"); - //JsonLDNamespace ietf = JsonLDNamespace.defineNamespace("ietf", "http://www.iana.org/assignments/relation/"); - String objectKey = new JsonLDTerm(activityStreams, "object").getUrl(); - if (jsonld.containsKey(objectKey)) { - JsonObject msgObject = jsonld.getJsonObject(objectKey); - if (new JsonLDTerm(activityStreams, "Relationship").getUrl().equals(msgObject.getString("@type"))) { - // We have a relationship message - need to get the subject and object and - // relationship type - String subjectId = msgObject.getString(new JsonLDTerm(activityStreams, "subject").getUrl()); - String objectId = msgObject.getString(new JsonLDTerm(activityStreams, "object").getUrl()); - // Best-effort to get name by following redirects and looing for a 'name' field in the returned json - try (CloseableHttpClient client = HttpClients.createDefault()) { - logger.info("Getting " + subjectId); - HttpGet get = new HttpGet(new URI(subjectId)); - get.addHeader("Accept", "application/json"); - - int statusCode=0; - do { - CloseableHttpResponse response = client.execute(get); - statusCode = response.getStatusLine().getStatusCode(); - switch (statusCode) { - case 302: - case 303: - String location=response.getFirstHeader("location").getValue(); - logger.info("Redirecting to: " + location); - get = new HttpGet(location); - get.addHeader("Accept", "application/json"); - - break; - case 200: - String responseString = EntityUtils.toString(response.getEntity(), "UTF-8"); - logger.info("Received: " + responseString); - JsonObject job = JsonUtil.getJsonObject(responseString); - name = job.getString("name", null); - itemType = job.getString("type", null); - break; - default: - logger.info("Received " + statusCode + " when accessing " + subjectId); - } - } while(statusCode == 302); - } catch (Exception e) { - logger.info("Unable to get name from " + subjectId); - logger.info(e.getLocalizedMessage()); - } - String relationshipId = msgObject - .getString(new JsonLDTerm(activityStreams, "relationship").getUrl()); - if (subjectId != null && objectId != null && relationshipId != null) { - // Strip the URL part from a relationship ID/URL assuming a usable label exists - // after a # char. Default is to use the whole URI. - int index = relationshipId.indexOf("#"); - logger.info("Found # at " + index + " in " + relationshipId); - String relationship = relationshipId.substring(index + 1); - // Parse the URI as a PID and see if this Dataverse instance has this dataset - Optional id = PidProvider.parse(objectId); - if (id.isPresent()) { - //ToDo - avoid reparsing GlobalId by making a findByGlobalId(GlobalId) method? - Dataset dataset = datasetSvc.findByGlobalId(objectId); - if (dataset != null) { - JsonObjectBuilder citingResourceBuilder = Json.createObjectBuilder().add("@id", subjectId) - .add("relationship", relationship); - if(name!=null && !name.isBlank()) { - citingResourceBuilder.add("name",name); - } - if(itemType!=null && !itemType.isBlank()) { - citingResourceBuilder.add("@type",itemType.substring(0,1).toUpperCase() + itemType.substring(1)); - } - JsonObject citingResource = citingResourceBuilder.build(); - String jsonstring = JsonUtil.prettyPrint(citingResource); - logger.info("Storing: " + jsonstring); - //Set ras = roleService.rolesAssignments(dataset); - - roleService.rolesAssignments(dataset).stream() - .filter(ra -> ra.getRole().permissions().contains(Permission.PublishDataset)) - .flatMap(ra -> roleAssigneeService - .getExplicitUsers( - roleAssigneeService.getRoleAssignee(ra.getAssigneeIdentifier())) - .stream()) - .distinct() // prevent double-send - .forEach(au -> { - - if (au.isSuperuser()) { - userNotificationService.sendNotification(au, - new Timestamp(new Date().getTime()), - UserNotification.Type.DATASETMENTIONED, dataset.getId(), null, - null, true, jsonstring); - - } - }); - sent = true; - } - } else { - logger.info("Didn't find dataset"); - // We don't have a dataset corresponding to the object of the relationship - do - // nothing - } - - } else { - // Can't get subject, relationship, object from message - logger.info("Can't find the subject, relationship or object in the message - ignoring"); - - } - } else { - - // This isn't a Relationship announcement message - ignore - logger.info("This is not a relationship announcement - ignoring message of type " - + msgObject.getJsonString("@type")); - } - } - if (!sent) { - throw new ServiceUnavailableException("Unable to process message. Please contact the administrators."); + // Process message based on type + processMessage(jsonld); + + return ok("Message Received"); + + } catch (Throwable t) { + logger.warning(t.getLocalizedMessage()); + if(logger.isLoggable(Level.FINE)) { + t.printStackTrace(); } - } else { - logger.info("Ignoring message from IP address: " + origin.toString()); - throw new ForbiddenException("Inbox does not acept messages from this address"); + throw t; } - } catch (Throwable t) { - logger.severe(t.getLocalizedMessage()); - t.printStackTrace(); + } + + /** + * Parse JSON-LD message with fallback for COAR Notify context issues. + */ + private JsonObject parseJsonLD(String body) { + JsonObject jsonld = JSONLDUtil.decontextualizeJsonLD(body); + + if (jsonld == null) { + // The COAR Notify URL has many redirects which cause a + // LOADING_REMOTE_CONTEXT_FAILED error in the titanium library - so replace it + // with the contents of the final redirect (current as of 10/29/2025) + // and try again + body = body.replace("\"https://purl.org/coar/notify\"", + "{\n" + " \"@vocab\": \"http://purl.org/coar/notify_vocabulary/\",\n" + + " \"ietf\": \"http://www.iana.org/assignments/relation/\",\n" + + " \"coar-notify\": \"http://purl.org/coar/notify_vocabulary/\",\n" + + " \"sorg\": \"http://schema.org/\",\n" + + " \"ReviewAction\": \"coar-notify:ReviewAction\",\n" + + " \"EndorsementAction\": \"coar-notify:EndorsementAction\",\n" + + " \"IngestAction\": \"coar-notify:IngestAction\",\n" + + " \"ietf:cite-as\": {\n" + " \"@type\": \"@id\"\n" + + " }}"); + jsonld = JSONLDUtil.decontextualizeJsonLD(body); + } + + return jsonld; + } + + /** + * Process the message based on its type. + * Returns true if the message was successfully processed. + */ + private void processMessage(JsonObject jsonld) throws WebApplicationException { + if (!jsonld.containsKey(objectKey)) { + throw new BadRequestException("Message does not contain an 'object' key - ignoring"); + } + + JsonObject msgObject = jsonld.getJsonObject(objectKey); + String messageType = msgObject.getString("@type", ""); + + switch (messageType) { + case "https://www.w3.org/ns/activitystreams#Relationship": + handleRelationshipAnnouncement(msgObject); + break; - throw t; + default: + throw new ServiceUnavailableException("Unsupported message type: " + messageType + " - ignoring"); } - return ok("Message Received"); } -} + + /** + * Handle COAR Notify Relationship Announcement messages. + */ + private void handleRelationshipAnnouncement(JsonObject msgObject) { + COARNotifyRelationshipAnnouncement handler = new COARNotifyRelationshipAnnouncement( + datasetService, + userNotificationService, + roleService, + roleAssigneeService + ); + + handler.processMessage(msgObject); + } +} \ No newline at end of file diff --git a/src/main/java/edu/harvard/iq/dataverse/api/ldn/COARNotifyRelationshipAnnouncement.java b/src/main/java/edu/harvard/iq/dataverse/api/ldn/COARNotifyRelationshipAnnouncement.java new file mode 100644 index 00000000000..c9189374343 --- /dev/null +++ b/src/main/java/edu/harvard/iq/dataverse/api/ldn/COARNotifyRelationshipAnnouncement.java @@ -0,0 +1,305 @@ +package edu.harvard.iq.dataverse.api.ldn; + +import edu.harvard.iq.dataverse.Dataset; +import edu.harvard.iq.dataverse.DatasetServiceBean; +import edu.harvard.iq.dataverse.DataverseRoleServiceBean; +import edu.harvard.iq.dataverse.GlobalId; +import edu.harvard.iq.dataverse.RoleAssigneeServiceBean; +import edu.harvard.iq.dataverse.UserNotification; +import edu.harvard.iq.dataverse.UserNotificationServiceBean; +import edu.harvard.iq.dataverse.authorization.Permission; +import edu.harvard.iq.dataverse.pidproviders.PidProvider; +import edu.harvard.iq.dataverse.settings.JvmSettings; +import edu.harvard.iq.dataverse.util.json.JsonLDNamespace; +import edu.harvard.iq.dataverse.util.json.JsonLDTerm; +import edu.harvard.iq.dataverse.util.json.JsonUtil; + +import static edu.harvard.iq.dataverse.api.LDNInbox.activityStreams; +import static edu.harvard.iq.dataverse.api.LDNInbox.objectKey; + +import jakarta.json.Json; +import jakarta.json.JsonObject; +import jakarta.json.JsonObjectBuilder; +import jakarta.ws.rs.BadRequestException; + +import org.apache.http.client.methods.CloseableHttpResponse; +import org.apache.http.client.methods.HttpGet; +import org.apache.http.client.methods.HttpHead; +import org.apache.http.impl.client.CloseableHttpClient; +import org.apache.http.impl.client.HttpClients; +import org.apache.http.util.EntityUtils; + +import java.net.URI; +import java.sql.Timestamp; +import java.util.Date; +import java.util.Optional; +import java.util.logging.Logger; + +/** + * Handles COAR Notify Relationship Announcement messages. + * Processes relationship announcements between scholarly resources and datasets. + */ +public class COARNotifyRelationshipAnnouncement { + + private static final Logger logger = Logger.getLogger(COARNotifyRelationshipAnnouncement.class.getName()); + + private static final String subjectKey = new JsonLDTerm(activityStreams, "subject").getUrl(); + private static final String relationshipKey = new JsonLDTerm(activityStreams, "relationship").getUrl(); + + private static final boolean notifySuperusersOnly = JvmSettings.COARNOTIFY_RELATIONSHIP_ANNOUNCEMENT_NOTIFY_SUPERUSERS_ONLY + .lookupOptional(Boolean.class).orElse(false); + + private final DatasetServiceBean datasetService; + private final UserNotificationServiceBean userNotificationService; + private final DataverseRoleServiceBean roleService; + private final RoleAssigneeServiceBean roleAssigneeService; + + public COARNotifyRelationshipAnnouncement( + DatasetServiceBean datasetService, + UserNotificationServiceBean userNotificationService, + DataverseRoleServiceBean roleService, + RoleAssigneeServiceBean roleAssigneeService) { + this.datasetService = datasetService; + this.userNotificationService = userNotificationService; + this.roleService = roleService; + this.roleAssigneeService = roleAssigneeService; + } + + /** + * Process a COAR Notify Relationship Announcement message. + * + * @param msgObject The JSON-LD message object + * @return true if the message was successfully processed, false otherwise + */ + public void processMessage(JsonObject msgObject) { + // Extract subject, object, and relationship from the message + String subjectId = extractField(msgObject, subjectKey); + String objectId = extractField(msgObject, objectKey); + String relationshipId = extractField(msgObject, relationshipKey); + + if (subjectId == null || objectId == null || relationshipId == null) { + throw new BadRequestException("Can't find the subject, relationship or object in the message - ignoring"); + } + + // Get metadata about the citing resource + ResourceMetadata metadata = retrieveResourceMetadata(subjectId); + + // Extract relationship label from URI + String relationship = extractRelationshipLabel(relationshipId); + + // Find the dataset being cited + Optional id = PidProvider.parse(objectId); + if (!id.isPresent()) { + throw new BadRequestException("Unable to parse relationship object ID as a PID: " + objectId); + } + + Dataset dataset = datasetService.findByGlobalId(objectId); + if (dataset == null) { + logger.info("Didn't find dataset for object ID: " + objectId + " - ignoring"); + } + + // Create the citing resource JSON + JsonObject citingResource = buildCitingResourceJson(subjectId, relationship, metadata); + String jsonString = JsonUtil.prettyPrint(citingResource); + logger.info("Citing resource: " + jsonString); + + // Send notifications to users with publish permissions + sendNotifications(dataset, jsonString); + } + + /** + * Extract a field value from the message object. + */ + private String extractField(JsonObject msgObject, String key) { + return msgObject.containsKey(key) ? msgObject.getString(key) : null; + } + + /** + * Retrieve metadata about the citing resource using Signposting and DataCite. + */ + private ResourceMetadata retrieveResourceMetadata(String subjectId) { + ResourceMetadata metadata = new ResourceMetadata(); + + try (CloseableHttpClient client = HttpClients.createDefault()) { + logger.info("Getting " + subjectId); + + // Step 1: Initial GET request expecting a 30x redirect + HttpGet initialGet = new HttpGet(new URI(subjectId)); + initialGet.addHeader("Accept", "application/json"); + + CloseableHttpResponse initialResponse = client.execute(initialGet); + int statusCode = initialResponse.getStatusLine().getStatusCode(); + + if (statusCode == 302 || statusCode == 303) { + String location = initialResponse.getFirstHeader("Location").getValue(); + logger.info("Redirecting to: " + location); + initialResponse.close(); + + // Step 2: HEAD request to get Signposting links + HttpHead headRequest = new HttpHead(location); + CloseableHttpResponse headResponse = client.execute(headRequest); + + if (headResponse.getStatusLine().getStatusCode() == 200) { + String dataciteXmlUrl = extractDataCiteXmlUrl(headResponse); + headResponse.close(); + + // Step 3: Retrieve and parse DataCite XML + if (dataciteXmlUrl != null) { + parseDataCiteXml(dataciteXmlUrl, client, metadata); + } else { + logger.info("No DataCite XML URL found in Signposting links"); + } + } else { + logger.info("HEAD request failed with status: " + headResponse.getStatusLine().getStatusCode()); + headResponse.close(); + } + } else { + logger.info("Expected 302/303 redirect but received status: " + statusCode); + initialResponse.close(); + } + + } catch (Exception e) { + logger.info("Unable to get metadata from " + subjectId); + logger.info(e.getLocalizedMessage()); + } + + return metadata; + } + + /** + * Extract DataCite XML URL from Signposting Link headers. + */ + private String extractDataCiteXmlUrl(CloseableHttpResponse headResponse) { + org.apache.http.Header[] linkHeaders = headResponse.getHeaders("Link"); + + for (org.apache.http.Header linkHeader : linkHeaders) { + String linkValue = linkHeader.getValue(); + if (linkValue.contains("application/vnd.datacite.datacite+xml")) { + int urlStart = linkValue.indexOf('<'); + int urlEnd = linkValue.indexOf('>'); + if (urlStart != -1 && urlEnd != -1 && urlEnd > urlStart) { + String dataciteXmlUrl = linkValue.substring(urlStart + 1, urlEnd); + logger.info("Found DataCite XML URL: " + dataciteXmlUrl); + return dataciteXmlUrl; + } + } + } + + return null; + } + + /** + * Parse DataCite XML to extract title and resource type. + */ + private void parseDataCiteXml(String dataciteXmlUrl, CloseableHttpClient client, ResourceMetadata metadata) { + try { + HttpGet xmlGet = new HttpGet(new URI(dataciteXmlUrl)); + xmlGet.addHeader("Accept", "application/vnd.datacite.datacite+xml, application/xml"); + + CloseableHttpResponse xmlResponse = client.execute(xmlGet); + + if (xmlResponse.getStatusLine().getStatusCode() == 200) { + String xmlContent = EntityUtils.toString(xmlResponse.getEntity(), "UTF-8"); + logger.info("Retrieved DataCite XML"); + + javax.xml.parsers.DocumentBuilderFactory factory = javax.xml.parsers.DocumentBuilderFactory.newInstance(); + factory.setNamespaceAware(true); + javax.xml.parsers.DocumentBuilder builder = factory.newDocumentBuilder(); + org.w3c.dom.Document doc = builder.parse( + new java.io.ByteArrayInputStream(xmlContent.getBytes("UTF-8"))); + + // Extract title + org.w3c.dom.NodeList titleNodes = doc.getElementsByTagNameNS("*", "title"); + if (titleNodes.getLength() == 0) { + titleNodes = doc.getElementsByTagName("title"); + } + + if (titleNodes.getLength() > 0) { + metadata.name = titleNodes.item(0).getTextContent(); + logger.info("Extracted title from DataCite XML: " + metadata.name); + } else { + logger.info("No title element found in DataCite XML"); + } + + // Extract resource type + org.w3c.dom.NodeList resourceTypeNodes = doc.getElementsByTagNameNS("*", "resourceType"); + if (resourceTypeNodes.getLength() == 0) { + resourceTypeNodes = doc.getElementsByTagName("resourceType"); + } + + if (resourceTypeNodes.getLength() > 0) { + org.w3c.dom.Element resourceTypeElement = (org.w3c.dom.Element) resourceTypeNodes.item(0); + String resourceTypeGeneral = resourceTypeElement.getAttribute("resourceTypeGeneral"); + if (resourceTypeGeneral != null && !resourceTypeGeneral.isEmpty()) { + metadata.itemType = resourceTypeGeneral.toLowerCase(); + logger.info("Extracted resource type: " + metadata.itemType); + } + } + + } else { + logger.info("Failed to retrieve DataCite XML. Status: " + xmlResponse.getStatusLine().getStatusCode()); + } + + xmlResponse.close(); + + } catch (Exception parseException) { + logger.warning("Failed to parse DataCite XML: " + parseException.getMessage()); + } + } + + /** + * Extract relationship label from relationship URI. + * Assumes the label exists after a # character. + */ + private String extractRelationshipLabel(String relationshipId) { + int index = relationshipId.indexOf("#"); + logger.info("Found # at " + index + " in " + relationshipId); + return relationshipId.substring(index + 1); + } + + /** + * Build the JSON object representing the citing resource. + */ + private JsonObject buildCitingResourceJson(String subjectId, String relationship, ResourceMetadata metadata) { + JsonObjectBuilder citingResourceBuilder = Json.createObjectBuilder() + .add("@id", subjectId) + .add("relationship", relationship); + + if (metadata.name != null && !metadata.name.isBlank()) { + citingResourceBuilder.add("name", metadata.name); + } + + if (metadata.itemType != null && !metadata.itemType.isBlank()) { + citingResourceBuilder.add("@type", metadata.itemType); + } + + return citingResourceBuilder.build(); + } + + /** + * Send notifications to users with publish permissions on the dataset. + */ + private void sendNotifications(Dataset dataset, String jsonString) { + roleService.rolesAssignments(dataset).stream() + .filter(assignment -> assignment.getRole().permissions().contains(Permission.PublishDataset)) + .flatMap(ra -> roleAssigneeService + .getExplicitUsers(roleAssigneeService.getRoleAssignee(ra.getAssigneeIdentifier())).stream()) + .distinct() // prevent double-send + .forEach(au -> { + + if (!notifySuperusersOnly || au.isSuperuser()) { + userNotificationService.sendNotification(au, new Timestamp(new Date().getTime()), + UserNotification.Type.DATASETMENTIONED, dataset.getId(), null, null, true, jsonString); + + } + }); + } + + /** + * Inner class to hold resource metadata. + */ + private static class ResourceMetadata { + String name; + String itemType; + } +} diff --git a/src/main/java/edu/harvard/iq/dataverse/settings/JvmSettings.java b/src/main/java/edu/harvard/iq/dataverse/settings/JvmSettings.java index 87123801a3e..f5d8375df11 100644 --- a/src/main/java/edu/harvard/iq/dataverse/settings/JvmSettings.java +++ b/src/main/java/edu/harvard/iq/dataverse/settings/JvmSettings.java @@ -301,6 +301,13 @@ public enum JvmSettings { SCOPE_LOCALCONTEXTS(PREFIX, "localcontexts"), LOCALCONTEXTS_URL(SCOPE_LOCALCONTEXTS, "url"), LOCALCONTEXTS_API_KEY(SCOPE_LOCALCONTEXTS, "api-key"), + + // LinkedDataNotification + SCOPE_LINKEDDATANOTIFICATION(PREFIX, "ldn"), + LINKEDDATANOTIFICATION_ALLOWED_HOSTS(SCOPE_LINKEDDATANOTIFICATION, "allowed-hosts"), + SCOPE_COARNOTIFY(SCOPE_LINKEDDATANOTIFICATION, "coar-notify"), + SCOPE_COARNOTIFY_RELATIONSHIP_ANNOUNCEMENT(SCOPE_COARNOTIFY, "relationship-announcement"), + COARNOTIFY_RELATIONSHIP_ANNOUNCEMENT_NOTIFY_SUPERUSERS_ONLY(SCOPE_COARNOTIFY_RELATIONSHIP_ANNOUNCEMENT, "notify-superusers-only"), ; private static final String SCOPE_SEPARATOR = "."; diff --git a/src/test/java/edu/harvard/iq/dataverse/api/LDNInboxIT.java b/src/test/java/edu/harvard/iq/dataverse/api/LDNInboxIT.java new file mode 100644 index 00000000000..b75f8136698 --- /dev/null +++ b/src/test/java/edu/harvard/iq/dataverse/api/LDNInboxIT.java @@ -0,0 +1,334 @@ + +package edu.harvard.iq.dataverse.api; + +import jakarta.json.Json; +import jakarta.json.JsonObject; +import jakarta.json.JsonObjectBuilder; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; + +import edu.harvard.iq.dataverse.settings.JvmSettings; +import edu.harvard.iq.dataverse.settings.SettingsServiceBean; +import edu.harvard.iq.dataverse.util.testing.JvmSetting; +import edu.harvard.iq.dataverse.util.testing.LocalJvmSettings; +import io.restassured.RestAssured; +import io.restassured.response.Response; + +import java.util.UUID; + + +import static jakarta.ws.rs.core.Response.Status.FORBIDDEN; +import static jakarta.ws.rs.core.Response.Status.OK; +import static jakarta.ws.rs.core.Response.Status.SERVICE_UNAVAILABLE; +import static org.hamcrest.CoreMatchers.equalTo; +import static org.hamcrest.CoreMatchers.notNullValue; +import static org.junit.jupiter.api.Assertions.assertTrue; + +@LocalJvmSettings +@JvmSetting(key = JvmSettings.LINKEDDATANOTIFICATION_ALLOWED_HOSTS, value = "*") +public class LDNInboxIT { + + private static String apiToken; + private static String username; + private static String dataverseAlias; + private static String datasetPid; + private static Integer datasetId; + private static String superuserApiToken; + private static String superusername; + + @BeforeAll + public static void setUpClass() { + RestAssured.baseURI = UtilIT.getRestAssuredBaseUri(); + + // Create test user and get API token + Response createUser = UtilIT.createRandomUser(); + apiToken = UtilIT.getApiTokenFromResponse(createUser); + username = UtilIT.getUsernameFromResponse(createUser); + + // Create superuser for receiving notifications + Response createSuperuser = UtilIT.createRandomUser(); + superuserApiToken = UtilIT.getApiTokenFromResponse(createSuperuser); + superusername = UtilIT.getUsernameFromResponse(createSuperuser); + Response makeSuperuser = UtilIT.setSuperuserStatus(superusername, true); + makeSuperuser.then().assertThat().statusCode(OK.getStatusCode()); + + // Create dataverse + Response createDataverse = UtilIT.createRandomDataverse(apiToken); + createDataverse.then().assertThat().statusCode(201); + dataverseAlias = UtilIT.getAliasFromResponse(createDataverse); + + // Create and publish a dataset + Response createDataset = UtilIT.createRandomDatasetViaNativeApi(dataverseAlias, apiToken); + createDataset.then().assertThat().statusCode(201); + datasetId = UtilIT.getDatasetIdFromResponse(createDataset); + datasetPid = UtilIT.getDatasetPersistentIdFromResponse(createDataset); + + // Publish the dataset + Response publishDataverse = UtilIT.publishDataverseViaNativeApi(dataverseAlias, apiToken); + publishDataverse.then().assertThat().statusCode(OK.getStatusCode()); + + Response publishDataset = UtilIT.publishDatasetViaNativeApi(datasetPid, "major", apiToken); + publishDataset.then().assertThat().statusCode(OK.getStatusCode()); + + } + + @AfterAll + public static void afterClass() { + // Clean up: remove test dataset and dataverse + if (datasetId != null) { + Response destroyDataset = UtilIT.destroyDataset(datasetId, apiToken); + destroyDataset.then().assertThat().statusCode(OK.getStatusCode()); + } + + if (dataverseAlias != null) { + Response deleteDataverse = UtilIT.deleteDataverse(dataverseAlias, apiToken); + deleteDataverse.then().assertThat().statusCode(OK.getStatusCode()); + } + + // Delete test users + Response deleteUser = UtilIT.deleteUser(username); + deleteUser.then().assertThat().statusCode(OK.getStatusCode()); + Response deleteSuperuser = UtilIT.deleteUser(superusername); + deleteSuperuser.then().assertThat().statusCode(OK.getStatusCode()); + } + + @Test + public void testAcceptRelationshipAnnouncementMessage() { + // Create a COAR Notify Relationship Announcement message + String citingResourceId = "https://doi.org/10.1234/example-publication"; + String citingResourceName = "An Example Publication Citing the Dataset"; + String relationship = "Cites"; + + JsonObject message = createRelationshipAnnouncementMessage( + citingResourceId, + citingResourceName, + datasetPid, + relationship + ); + + // Send the message to the LDN inbox + Response response = UtilIT.sendMessageToLDNInbox(message.toString()); + + // Verify the response + response.then().assertThat() + .statusCode(OK.getStatusCode()) + .body("data.message", equalTo("Message Received")); + + // Wait a moment for notification to be created + try { + Thread.sleep(1000); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } + + // Verify that a notification was created for the superuser + Response notifications = UtilIT.getNotifications(superuserApiToken); + + notifications.then().assertThat() + .statusCode(OK.getStatusCode()) + .body("data", notNullValue()); + + // Check that at least one notification exists with type DATASETMENTIONED + String notificationsJson = notifications.asString(); + assertTrue(notificationsJson.contains("DATASETMENTIONED"), + "Expected to find DATASETMENTIONED notification"); + assertTrue(notificationsJson.contains(citingResourceId), + "Expected notification to contain citing resource ID"); + } + + @Test + @JvmSetting(key = JvmSettings.LINKEDDATANOTIFICATION_ALLOWED_HOSTS, value = "192.0.2.1") + public void testRejectMessageFromNonWhitelistedHost() { + + JsonObject message = createRelationshipAnnouncementMessage( + "https://doi.org/10.1234/test", + "Test Publication", + datasetPid, + "Cites" + ); + + // Send the message - should be rejected + Response response = UtilIT.sendMessageToLDNInbox(message.toString()); + + response.then().assertThat() + .statusCode(FORBIDDEN.getStatusCode()); + + // Restore whitelist + Response restoreWhitelist = UtilIT.setSetting( + SettingsServiceBean.Key.LDNMessageHosts, + "*" + ); + restoreWhitelist.then().assertThat().statusCode(OK.getStatusCode()); + } + + @Test + public void testRejectInvalidJsonLD() { + String invalidJson = "{ this is not valid json }"; + + Response response = UtilIT.sendMessageToLDNInbox(invalidJson); + + response.then().assertThat() + .statusCode(400); // Bad Request + } + + @Test + public void testRejectMessageForNonExistentDataset() { + String nonExistentPid = "doi:10.5072/FK2/NONEXISTENT"; + + JsonObject message = createRelationshipAnnouncementMessage( + "https://doi.org/10.1234/test", + "Test Publication", + nonExistentPid, + "Cites" + ); + + Response response = UtilIT.sendMessageToLDNInbox(message.toString()); + + response.then().assertThat() + .statusCode(SERVICE_UNAVAILABLE.getStatusCode()); + } + + @Test + public void testAcceptMessageWithMultipleRelationshipTypes() { + // Test different relationship types + String[] relationships = { + "Cites", + "IsSupplementTo", + "IsReferencedBy", + "IsRelatedTo" + }; + + for (String relationship : relationships) { + JsonObject message = createRelationshipAnnouncementMessage( + "https://doi.org/10.1234/test-" + relationship.toLowerCase(), + "Test Publication for " + relationship, + datasetPid, + relationship + ); + + Response response = UtilIT.sendMessageToLDNInbox(message.toString()); + + response.then().assertThat() + .statusCode(OK.getStatusCode()) + .body("data.message", equalTo("Message Received")); + } + } + + @Test + public void testAcceptMessageWithDifferentResourceTypes() { + String[] resourceTypes = { + "ScholarlyArticle", + "Dataset", + "Software", + "Preprint" + }; + + for (String resourceType : resourceTypes) { + JsonObject message = createRelationshipAnnouncementMessageWithType( + "https://doi.org/10.1234/test-" + resourceType.toLowerCase(), + "Test Resource of type " + resourceType, + datasetPid, + "Cites", + resourceType + ); + + Response response = UtilIT.sendMessageToLDNInbox(message.toString()); + + response.then().assertThat() + .statusCode(OK.getStatusCode()) + .body("data.message", equalTo("Message Received")); + } + } + + /** + * Helper method to create a COAR Notify Relationship Announcement message + */ + private JsonObject createRelationshipAnnouncementMessage( + String citingResourceId, + String citingResourceName, + String citedDatasetPid, + String relationship) { + return createRelationshipAnnouncementMessageWithType( + citingResourceId, + citingResourceName, + citedDatasetPid, + relationship, + "ScholarlyArticle" + ); + } + + /** + * Helper method to create a COAR Notify Relationship Announcement message with specific resource type + */ + private JsonObject createRelationshipAnnouncementMessageWithType( + String citingResourceId, + String citingResourceName, + String citedDatasetPid, + String relationship, + String resourceType) { + + String messageId = "urn:uuid:" + UUID.randomUUID().toString(); + String relationshipId = "urn:uuid:" + UUID.randomUUID().toString(); + + // Convert PID to URL format if needed + String citedDatasetUrl = citedDatasetPid; + if (citedDatasetPid.startsWith("doi:")) { + citedDatasetUrl = "https://doi.org/" + citedDatasetPid.substring(4); + } else if (citedDatasetPid.startsWith("hdl:")) { + citedDatasetUrl = "https://hdl.handle.net/" + + + citedDatasetPid.substring(4); + } + + // Build the COAR Notify message following the specification + JsonObjectBuilder messageBuilder = Json.createObjectBuilder(); + + // Add @context + messageBuilder.add("@context", Json.createArrayBuilder() + .add("https://www.w3.org/ns/activitystreams") + .add("https://purl.org/coar/notify")); + + // Add message id and type + messageBuilder.add("id", messageId); + messageBuilder.add("type", Json.createArrayBuilder() + .add("Announce") + .add("coar-notify:RelationshipAction")); + + // Add actor (the system sending the notification) + messageBuilder.add("actor", Json.createObjectBuilder() + .add("id", "https://example.org/repository") + .add("name", "Example Repository") + .add("type", "Service")); + + // Add origin (inbox of the sender) + messageBuilder.add("origin", Json.createObjectBuilder() + .add("id", "https://example.org/repository") + .add("inbox", "https://example.org/inbox") + .add("type", "Service")); + + // Add target (inbox of the receiver - this Dataverse instance) + messageBuilder.add("target", Json.createObjectBuilder() + .add("id", RestAssured.baseURI) + .add("inbox", RestAssured.baseURI + "/api/inbox") + .add("type", "Service")); + + // Add object (the relationship being announced) + messageBuilder.add("object", Json.createObjectBuilder() + .add("id", relationshipId) + .add("type", "Relationship") + .add("as:subject", citingResourceId) + .add("as:relationship", "https://purl.org/datacite/ontology#" + relationship) + .add("as:object", citedDatasetUrl)); + + // Add context (the citing resource details) + messageBuilder.add("context", Json.createObjectBuilder() + .add("id", citingResourceId) + .add("ietf:cite-as", citingResourceId) + .add("type", "sorg:" + resourceType) + .add("sorg:name", citingResourceName)); + + return messageBuilder.build(); + } +} + diff --git a/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java b/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java index 9724efb2d32..f8931719f81 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java @@ -5143,4 +5143,13 @@ public static Response getDatasetFilesRoleAssignmentHistory(Integer datasetId, b return requestSpecification .get("/api/v1/datasets/" + datasetId + "/files/assignments/history"); } + + public static Response sendMessageToLDNInbox(String message) { + // Send the message to the LDN inbox + return given() + .contentType("application/ld+json") + .body(message) + .when() + .post("/api/inbox/"); + } } From 6e9030ded40aa9dc0eec10c3275d0a3f2ee45561 Mon Sep 17 00:00:00 2001 From: qqmyers Date: Wed, 29 Oct 2025 17:27:16 -0400 Subject: [PATCH 24/40] refactor wf step - name change, multiple targets, gen. improvements prioritize publicationIDNumber field over url, support all IDTypes if they provide URIs, check DOIs and Handles for any form, including raw, fix logic for other fields, check for URI form in all cases, handle null results, ... --- .../source/api/linkeddatanotification.rst | 1 - .../source/developers/workflows.rst | 21 +- ...y-relationship-announcement-workflow.json} | 7 +- ...ARNotifyRelationshipAnnouncementStep.java} | 249 +++++++++++------- .../internalspi/InternalWorkflowStepSP.java | 4 +- 5 files changed, 173 insertions(+), 109 deletions(-) rename scripts/api/data/workflows/{internal-ldnannounce-workflow.json => internal-coar-notify-relationship-announcement-workflow.json} (52%) rename src/main/java/edu/harvard/iq/dataverse/workflow/internalspi/{LDNAnnounceDatasetVersionStep.java => COARNotifyRelationshipAnnouncementStep.java} (53%) diff --git a/doc/sphinx-guides/source/api/linkeddatanotification.rst b/doc/sphinx-guides/source/api/linkeddatanotification.rst index 0bf65b48f99..b241065e6b7 100644 --- a/doc/sphinx-guides/source/api/linkeddatanotification.rst +++ b/doc/sphinx-guides/source/api/linkeddatanotification.rst @@ -19,7 +19,6 @@ Messages can be sent via POST, using the application/ld+json ContentType: curl -X POST -H 'ContentType:application/ld+json' $SERVER_URL/api/inbox --upload-file message.jsonld -https://coar-notify.net/catalogue/workflows/repository-relationship-repository/2/ The supported message format is described by `the COAR Notify Relationship Announcement specification `_. diff --git a/doc/sphinx-guides/source/developers/workflows.rst b/doc/sphinx-guides/source/developers/workflows.rst index 0d7104e4631..3f271990069 100644 --- a/doc/sphinx-guides/source/developers/workflows.rst +++ b/doc/sphinx-guides/source/developers/workflows.rst @@ -202,16 +202,16 @@ Note - the example step includes two settings required for any archiver, three ( } -ldnannounce -+++++++++++ +coarNotifyRelationshipAnnouncement +++++++++++++++++++++++++++++++++++ -A step that sends a `Linked Data Notification (LDN) `_ message to a specific LDN Inbox announcing a relationship between a newly published/available dataset and an external resource (e.g. one managed by the recipient). +A step that sends a `COAR Notify Relationship Announcement `_ message, using the `Linked Data Notification (LDN) `_ message standard, +to a specific set of LDN Inboxes announcing a relationship between a newly published/available dataset and an external resource (e.g. one managed by the recipient). The two parameters are -* ``:LDNAnnounceRequiredFields`` - a list of metadata fields that must exist to trigger the message. Currently, the message also includes the values for these fields but future versions may only send the dataset's persistent identifier (making the receiver responsible for making a call-back to get any metadata). -* ``:LDNTarget`` - a JSON object containing an ``inbox`` key whose value is the URL of the target LDN inbox to which messages should be sent, e.g. ``{"id": "https://dashv7-dev.lib.harvard.edu","inbox": "https://dashv7-api-dev.lib.harvard.edu/server/ldn/inbox","type": "Service"}`` ). - -The message format is defined by the `COAR Notify Relationship Announcement `_ standard. +* ``:COARNotifyRelationshipAnnouncementTriggerFields`` - a list of metadata field types that can trigger messages. Separate messages will be sent for each field (whether of the same type or not). +* ``:COARNotifyRelationshipAnnouncementTargets`` - a JSON Array of JSON objects containing ``id``, ``inbox``, and ``type`` fields as required by the `COAR Notify Relationship Announcement specification `_ . +The ``inbox`` value should be the full URL of the target LDN inbox to which messages should be sent, e.g. ``{"id": "https://dashv7-dev.lib.harvard.edu","inbox": "https://dashv7-api-dev.lib.harvard.edu/server/ldn/inbox","type": "Service"}`` ). .. code:: json @@ -219,14 +219,13 @@ The message format is defined by the `COAR Notify Relationship Announcement paramSet) { + public COARNotifyRelationshipAnnouncementStep(Map paramSet) { new HashMap<>(paramSet); } @Override public WorkflowStepResult run(WorkflowContext context) { - JsonObject target = JsonUtil.getJsonObject((String) context.getSettings().get(LDN_TARGET)); - if (target != null) { - String inboxUrl = target.getString("inbox"); - + JsonArray targets = JsonUtil.getJsonArray((String) context.getSettings().get(CN_RA_TARGETS)); + if (targets != null && !targets.isEmpty()) { CloseableHttpClient client = HttpClients.createDefault(); - // build method - - HttpPost announcement; try { // First check that we have what is required Dataset d = context.getDataset(); @@ -91,42 +94,67 @@ public WorkflowStepResult run(WorkflowContext context) { fields.put(df.getDatasetFieldType().getName(), df); } } - // Loop through and send a message for each supported relationship - boolean success = false; - for (JsonObject rel : getObjects(context, fields).getValuesAs(JsonObject.class)) { - announcement = buildAnnouncement(d, rel, target); - // execute - try (CloseableHttpResponse response = client.execute(announcement)) { - int code = response.getStatusLine().getStatusCode(); - if (code >= 200 && code < 300) { - // HTTP OK range - success = true; - logger.fine("Successfully sent message for " + rel.toString()); - } else { - String responseBody = new String(response.getEntity().getContent().readAllBytes(), - StandardCharsets.UTF_8); - ; - return new Failure((success ? "Partial failure" : "") + "Error communicating with " - + inboxUrl + " for relationship " + rel.toString() + ". Server response: " - + responseBody + " (" + response + ")."); - } - } catch (Exception ex) { - logger.log(Level.SEVERE, "Error communicating with remote server: " + ex.getMessage(), ex); - return new Failure((success ? "Partial failure" : "") + "Error executing request: " - + ex.getLocalizedMessage(), "Cannot communicate with remote server."); + // Get all relationship objects once + JsonArray relationships = getObjects(context, fields); + + if (relationships.isEmpty()) { + logger.fine("No valid relationships found to announce"); + return OK; + } + + // Track overall success + boolean anySuccess = false; + StringBuilder errors = new StringBuilder(); + + // Loop through each target + for (JsonObject target : targets.getValuesAs(JsonObject.class)) { + String inboxUrl = target.getString("inbox"); + + // Send a message for each relationship to this target + for (JsonObject rel : relationships.getValuesAs(JsonObject.class)) { + HttpPost announcement = buildAnnouncement(d, rel, target); + + try (CloseableHttpResponse response = client.execute(announcement)) { + int code = response.getStatusLine().getStatusCode(); + if (code >= 200 && code < 300) { + // HTTP OK range + anySuccess = true; + logger.fine("Successfully sent message for " + rel.toString() + " to " + inboxUrl); + } else { + String responseBody = new String(response.getEntity().getContent().readAllBytes(), + StandardCharsets.UTF_8); + String errorMsg = "Error communicating with " + inboxUrl + " for relationship " + + rel.toString() + ". Server response: " + responseBody + " (" + response + + ")."; + logger.warning(errorMsg); + errors.append(errorMsg).append("\n"); + } + } catch (Exception ex) { + logger.log(Level.SEVERE, "Error communicating with " + inboxUrl + ": " + ex.getMessage(), + ex); + String errorMsg = "Error executing request to " + inboxUrl + ": " + + ex.getLocalizedMessage(); + errors.append(errorMsg).append("\n"); + } } + } + // If we had any errors but also some successes, report partial failure + if (errors.length() > 0) { + return new Failure((anySuccess ? "Partial failure: " : "") + errors.toString()); } - // Any failure and we would have returned already. + + // All succeeded return OK; } catch (URISyntaxException e) { return new Failure( - "LDNAnnounceDatasetVersion workflow step failed: unable to parse inbox in :LDNTarget setting."); + "COARNotifyRelationshipAnnouncementStep workflow step failed: unable to parse inbox in target setting."); } } - return new Failure("LDNAnnounceDatasetVersion workflow step failed: :LDNTarget setting missing or invalid."); + return new Failure("COARNotifyRelationshipAnnouncementStep workflow step failed: " + CN_RA_TARGETS + + " setting missing or invalid."); } @Override @@ -165,10 +193,16 @@ JsonArray getObjects(WorkflowContext ctxt, Map fields) { Iterator iter = rels.iterator(); while (iter.hasNext()) { JsonValue jval = iter.next(); - jab.add(getRelationshipObject(dft, jval, d, localContext)); + JsonObject relObject = getRelationshipObject(dft, jval, d, localContext); + if (relObject != null) { + jab.add(relObject); + } } } else { - jab.add(getRelationshipObject(dft, jv, d, localContext)); + JsonObject relObject = getRelationshipObject(dft, jv, d, localContext); + if (relObject != null) { + jab.add(relObject); + } } } @@ -181,15 +215,19 @@ private JsonObject getRelationshipObject(DatasetFieldType dft, JsonValue jval, D String[] answers = getBestIdAndType(dft, jval); String id = answers[0]; String type = answers[1]; + // Skip if we couldn't determine a valid ID + if (id == null || type == null) { + return null; + } return Json.createObjectBuilder().add("as:object", id).add("as:relationship", type) - .add("as:subject", d.getGlobalId().asURL().toString()).add("id", "urn:uuid:" + UUID.randomUUID().toString()).add("type","Relationship").build(); + .add("as:subject", d.getGlobalId().asURL().toString()) + .add("id", "urn:uuid:" + UUID.randomUUID().toString()).add("type", "Relationship").build(); } HttpPost buildAnnouncement(Dataset d, JsonObject rel, JsonObject target) throws URISyntaxException { JsonObjectBuilder job = Json.createObjectBuilder(); - JsonArrayBuilder context = Json.createArrayBuilder() - .add("https://www.w3.org/ns/activitystreams") + JsonArrayBuilder context = Json.createArrayBuilder().add("https://www.w3.org/ns/activitystreams") .add("https://coar-notify.net"); job.add("@context", context); job.add("id", "urn:uuid:" + UUID.randomUUID().toString()); @@ -214,11 +252,16 @@ HttpPost buildAnnouncement(Dataset d, JsonObject rel, JsonObject target) throws } private String[] getBestIdAndType(DatasetFieldType dft, JsonValue jv) { - + String type = "https://purl.org/datacite/ontology#isSupplementTo"; // Primitive value if (jv instanceof JsonString) { - return new String[] { ((JsonString) jv).getString(), type }; + String value = ((JsonString) jv).getString(); + if (isURI(value)) { + return new String[] { ((JsonString) jv).getString(), type }; + } else { + return new String[] { null, null }; + } } // Compound - apply type specific logic to get best Id JsonObject jo = jv.asJsonObject(); @@ -246,40 +289,59 @@ private String[] getBestIdAndType(DatasetFieldType dft, JsonValue jv) { publicationRelationType = cdft.getJsonLDTerm(); } } - if (jo.containsKey(publicationURL.getLabel())) { - id = jo.getString(publicationURL.getLabel()); - } else if (jo.containsKey(publicationIDType.getLabel())) { + if (jo.containsKey(publicationIDType.getLabel())) { if ((jo.containsKey(publicationIDNumber.getLabel()))) { String number = jo.getString(publicationIDNumber.getLabel()); switch (jo.getString(publicationIDType.getLabel())) { - case "doi": - if (number.startsWith("https://doi.org/")) { - id = number; - } else if (number.startsWith("doi:")) { - id = "https://doi.org/" + number.substring(4); - } else { - // Assume a raw DOI, e.g. 10.5072/FK2ABCDEF - id = "https://doi.org/" + number; + case AbstractDOIProvider.DOI_PROTOCOL: + if (number.startsWith("10")) { + number = AbstractDOIProvider.DOI_PROTOCOL + number; + } + // Validate using GlobalId + try { + GlobalId pid = PidUtil.parseAsGlobalID(number); + id = pid.asURL(); + } catch (IllegalArgumentException e) { + // Ignore + } + break; + case HandlePidProvider.HDL_PROTOCOL: + if (!number.startsWith(HandlePidProvider.HDL_PROTOCOL) + && !number.startsWith(HandlePidProvider.HDL_RESOLVER_URL) + && !number.startsWith(HandlePidProvider.HTTP_HDL_RESOLVER_URL)) { + number = "hdl:" + number; + } + // Validate using GlobalId + try { + GlobalId pid = PidUtil.parseAsGlobalID(number); + id = pid.asURL(); + } catch (IllegalArgumentException e) { + // Ignore } break; - case "DASH-NRS": - if (number.startsWith("http")) { + default: + // Check if the number can be interpreted as a valid URI of some sort + if (isURI(number)) { id = number; } + break; } } + } else if (jo.containsKey(publicationURL.getLabel())) { + + String value = jo.getString(publicationURL.getLabel()); + if (isURI(value)) { + id = value; + } } - if(jo.containsKey(publicationRelationType.getLabel())) { + if (jo.containsKey(publicationRelationType.getLabel())) { type = jo.getString(publicationRelationType.getLabel()); - type = "https://purl.org/datacite/ontology#" + type.substring(0,1).toLowerCase() + type.substring(1); + type = "https://purl.org/datacite/ontology#" + type.substring(0, 1).toLowerCase() + type.substring(1); } break; - default: - //ToDo - handle primary field - //ToDo - handle "Identifier" vs "IdentifierType" - //ToDo - check for URL form + default: Collection childDFTs = dft.getChildDatasetFieldTypes(); // Loop through child fields and select one // The order of preference is for a field with URL in the name, followed by one @@ -288,7 +350,10 @@ private String[] getBestIdAndType(DatasetFieldType dft, JsonValue jv) { String fieldname = cdft.getName(); if (fieldname.contains("URL")) { if (jo.containsKey(cdft.getJsonLDTerm().getLabel())) { - id = jo.getString(cdft.getJsonLDTerm().getLabel()); + String value = jo.getString(cdft.getJsonLDTerm().getLabel()); + if (isURI(value)) { + id = value; + } break; } } @@ -297,9 +362,12 @@ private String[] getBestIdAndType(DatasetFieldType dft, JsonValue jv) { for (DatasetFieldType cdft : childDFTs) { String fieldname = cdft.getName(); - if (fieldname.contains("ID") || fieldname.contains("Id")) { + if ((fieldname.contains("ID") || fieldname.contains("Id")) && !fieldname.contains("Type")) { if (jo.containsKey(cdft.getJsonLDTerm().getLabel())) { - id = jo.getString(cdft.getJsonLDTerm().getLabel()); + String value = jo.getString(cdft.getJsonLDTerm().getLabel()); + if (isURI(value)) { + id = value; + } break; } @@ -312,31 +380,30 @@ private String[] getBestIdAndType(DatasetFieldType dft, JsonValue jv) { if (fieldname.contains("Name")) { if (jo.containsKey(cdft.getJsonLDTerm().getLabel())) { - id = jo.getString(cdft.getJsonLDTerm().getLabel()); + String value = jo.getString(cdft.getJsonLDTerm().getLabel()); + if (isURI(value)) { + id = value; + } break; } } } } - id = jo.getString(jo.keySet().iterator().next()); } - return new String[] {id, type}; + return new String[] { id, type }; } - String process(String template, Map values) { - String curValue = template; - for (Map.Entry ent : values.entrySet()) { - String val = ent.getValue(); - if (val == null) { - val = ""; - } - String varRef = "${" + ent.getKey() + "}"; - while (curValue.contains(varRef)) { - curValue = curValue.replace(varRef, val); + private boolean isURI(String number) { + try { + URI uri = new URI(number); + if (uri.isAbsolute()) { + return true; } + } catch (URISyntaxException e) { + // Not a valid URI, skip + logger.fine("Value is not a valid URI: " + number); } - - return curValue; + return false; } } diff --git a/src/main/java/edu/harvard/iq/dataverse/workflow/internalspi/InternalWorkflowStepSP.java b/src/main/java/edu/harvard/iq/dataverse/workflow/internalspi/InternalWorkflowStepSP.java index d99e0901d3c..aee5c11f39e 100644 --- a/src/main/java/edu/harvard/iq/dataverse/workflow/internalspi/InternalWorkflowStepSP.java +++ b/src/main/java/edu/harvard/iq/dataverse/workflow/internalspi/InternalWorkflowStepSP.java @@ -25,8 +25,8 @@ public WorkflowStep getStep(String stepType, Map stepParameters) return new AuthorizedExternalStep(stepParameters); case "archiver": return new ArchivalSubmissionWorkflowStep(stepParameters); - case "ldnannounce": - return new LDNAnnounceDatasetVersionStep(stepParameters); + case "coarNotifyRelationshipAnnouncement": + return new COARNotifyRelationshipAnnouncementStep(stepParameters); default: throw new IllegalArgumentException("Unsupported step type: '" + stepType + "'."); } From 486a498f86fcab52ac18ed67be6445a97fb952c1 Mon Sep 17 00:00:00 2001 From: qqmyers Date: Wed, 29 Oct 2025 17:27:40 -0400 Subject: [PATCH 25/40] drop lower case start to DataCite relation types, use constants --- ...OARNotifyRelationshipAnnouncementStep.java | 27 +++++++++++-------- 1 file changed, 16 insertions(+), 11 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/workflow/internalspi/COARNotifyRelationshipAnnouncementStep.java b/src/main/java/edu/harvard/iq/dataverse/workflow/internalspi/COARNotifyRelationshipAnnouncementStep.java index fcbf63b35da..fb18b2918f9 100644 --- a/src/main/java/edu/harvard/iq/dataverse/workflow/internalspi/COARNotifyRelationshipAnnouncementStep.java +++ b/src/main/java/edu/harvard/iq/dataverse/workflow/internalspi/COARNotifyRelationshipAnnouncementStep.java @@ -69,6 +69,7 @@ public class COARNotifyRelationshipAnnouncementStep implements WorkflowStep { private static final String REQUIRED_FIELDS = ":COARNotifyRelationshipAnnouncementTriggerFields"; private static final String CN_RA_TARGETS = ":COARNotifyRelationshipAnnouncementTargets"; private static final String RELATED_PUBLICATION = "publication"; + public static final String DATACITE_URI_PREFIX = "https://purl.org/datacite/ontology#"; public COARNotifyRelationshipAnnouncementStep(Map paramSet) { new HashMap<>(paramSet); @@ -113,7 +114,7 @@ public WorkflowStepResult run(WorkflowContext context) { // Send a message for each relationship to this target for (JsonObject rel : relationships.getValuesAs(JsonObject.class)) { - HttpPost announcement = buildAnnouncement(d, rel, target); + HttpPost announcement = buildAnnouncementPost(rel, target); try (CloseableHttpResponse response = client.execute(announcement)) { int code = response.getStatusLine().getStatusCode(); @@ -224,8 +225,18 @@ private JsonObject getRelationshipObject(DatasetFieldType dft, JsonValue jval, D .add("id", "urn:uuid:" + UUID.randomUUID().toString()).add("type", "Relationship").build(); } - HttpPost buildAnnouncement(Dataset d, JsonObject rel, JsonObject target) throws URISyntaxException { + HttpPost buildAnnouncementPost(JsonObject rel, JsonObject target) throws URISyntaxException { + String body = buildAnnouncement(rel, target); + logger.info("Body: " + body); + + HttpPost annPost = new HttpPost(); + annPost.setURI(new URI(target.getString("inbox"))); + annPost.setEntity(new StringEntity(JsonUtil.prettyPrint(body), "utf-8")); + annPost.setHeader("Content-Type", "application/ld+json"); + return annPost; + } + public static String buildAnnouncement(JsonObject rel, JsonObject target) { JsonObjectBuilder job = Json.createObjectBuilder(); JsonArrayBuilder context = Json.createArrayBuilder().add("https://www.w3.org/ns/activitystreams") .add("https://coar-notify.net"); @@ -242,18 +253,12 @@ HttpPost buildAnnouncement(Dataset d, JsonObject rel, JsonObject target) throws job.add("target", target); job.add("type", Json.createArrayBuilder().add("Announce").add("coar-notify:RelationshipAction")); - HttpPost annPost = new HttpPost(); - annPost.setURI(new URI(target.getString("inbox"))); - String body = JsonUtil.prettyPrint(job.build()); - logger.info("Body: " + body); - annPost.setEntity(new StringEntity(JsonUtil.prettyPrint(body), "utf-8")); - annPost.setHeader("Content-Type", "application/ld+json"); - return annPost; + return JsonUtil.prettyPrint(job.build()); } private String[] getBestIdAndType(DatasetFieldType dft, JsonValue jv) { - String type = "https://purl.org/datacite/ontology#isSupplementTo"; + String type = DATACITE_URI_PREFIX + "IsSupplementTo"; // Primitive value if (jv instanceof JsonString) { String value = ((JsonString) jv).getString(); @@ -338,7 +343,7 @@ private String[] getBestIdAndType(DatasetFieldType dft, JsonValue jv) { } if (jo.containsKey(publicationRelationType.getLabel())) { type = jo.getString(publicationRelationType.getLabel()); - type = "https://purl.org/datacite/ontology#" + type.substring(0, 1).toLowerCase() + type.substring(1); + type = DATACITE_URI_PREFIX + type; } break; default: From da6e5a4e0abc11c3e2b7402dd4bbdb7db5c0c6a2 Mon Sep 17 00:00:00 2001 From: qqmyers Date: Wed, 29 Oct 2025 17:29:39 -0400 Subject: [PATCH 26/40] test fixes, use COAR step announcement builder --- .../harvard/iq/dataverse/api/LDNInboxIT.java | 267 ++++++------------ 1 file changed, 92 insertions(+), 175 deletions(-) diff --git a/src/test/java/edu/harvard/iq/dataverse/api/LDNInboxIT.java b/src/test/java/edu/harvard/iq/dataverse/api/LDNInboxIT.java index b75f8136698..a55e40c89e0 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/LDNInboxIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/LDNInboxIT.java @@ -7,17 +7,22 @@ import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Test; +import org.mockito.Mockito; +import edu.harvard.iq.dataverse.DataverseServiceBean; +import edu.harvard.iq.dataverse.branding.BrandingUtil; import edu.harvard.iq.dataverse.settings.JvmSettings; import edu.harvard.iq.dataverse.settings.SettingsServiceBean; import edu.harvard.iq.dataverse.util.testing.JvmSetting; import edu.harvard.iq.dataverse.util.testing.LocalJvmSettings; +import edu.harvard.iq.dataverse.workflow.internalspi.COARNotifyRelationshipAnnouncementStep; +import static edu.harvard.iq.dataverse.workflow.internalspi.COARNotifyRelationshipAnnouncementStep.DATACITE_URI_PREFIX; import io.restassured.RestAssured; import io.restassured.response.Response; import java.util.UUID; - +import static jakarta.ws.rs.core.Response.Status.BAD_REQUEST; import static jakarta.ws.rs.core.Response.Status.FORBIDDEN; import static jakarta.ws.rs.core.Response.Status.OK; import static jakarta.ws.rs.core.Response.Status.SERVICE_UNAVAILABLE; @@ -37,6 +42,11 @@ public class LDNInboxIT { private static String superuserApiToken; private static String superusername; + private static final String FRBR_SUPPLEMENT = "http://purl.org/vocab/frbr/core#supplement"; + + static SettingsServiceBean settingsServiceBean = Mockito.mock(SettingsServiceBean.class); + static DataverseServiceBean dataverseServiceBean = Mockito.mock(DataverseServiceBean.class); + @BeforeAll public static void setUpClass() { RestAssured.baseURI = UtilIT.getRestAssuredBaseUri(); @@ -71,6 +81,10 @@ public static void setUpClass() { Response publishDataset = UtilIT.publishDatasetViaNativeApi(datasetPid, "major", apiToken); publishDataset.then().assertThat().statusCode(OK.getStatusCode()); + // Setup mocks behavior, inject as deps - needed to build announcements + Mockito.when(settingsServiceBean.getValueForKey(SettingsServiceBean.Key.InstallationName)) + .thenReturn("LDN IT Tester"); + BrandingUtil.injectServices(dataverseServiceBean, settingsServiceBean); } @AfterAll @@ -97,23 +111,15 @@ public static void afterClass() { public void testAcceptRelationshipAnnouncementMessage() { // Create a COAR Notify Relationship Announcement message String citingResourceId = "https://doi.org/10.1234/example-publication"; - String citingResourceName = "An Example Publication Citing the Dataset"; - String relationship = "Cites"; + String relationship = DATACITE_URI_PREFIX + "Cites"; - JsonObject message = createRelationshipAnnouncementMessage( - citingResourceId, - citingResourceName, - datasetPid, - relationship - ); + String message = createRelationshipAnnouncementMessage(citingResourceId, datasetPid, relationship); // Send the message to the LDN inbox - Response response = UtilIT.sendMessageToLDNInbox(message.toString()); + Response response = UtilIT.sendMessageToLDNInbox(message); // Verify the response - response.then().assertThat() - .statusCode(OK.getStatusCode()) - .body("data.message", equalTo("Message Received")); + response.then().assertThat().statusCode(OK.getStatusCode()).body("data.message", equalTo("Message Received")); // Wait a moment for notification to be created try { @@ -125,41 +131,24 @@ public void testAcceptRelationshipAnnouncementMessage() { // Verify that a notification was created for the superuser Response notifications = UtilIT.getNotifications(superuserApiToken); - notifications.then().assertThat() - .statusCode(OK.getStatusCode()) - .body("data", notNullValue()); + notifications.then().assertThat().statusCode(OK.getStatusCode()).body("data", notNullValue()); // Check that at least one notification exists with type DATASETMENTIONED String notificationsJson = notifications.asString(); - assertTrue(notificationsJson.contains("DATASETMENTIONED"), - "Expected to find DATASETMENTIONED notification"); - assertTrue(notificationsJson.contains(citingResourceId), - "Expected notification to contain citing resource ID"); + assertTrue(notificationsJson.contains("DATASETMENTIONED"), "Expected to find DATASETMENTIONED notification"); + assertTrue(notificationsJson.contains(citingResourceId), "Expected notification to contain citing resource ID"); } @Test @JvmSetting(key = JvmSettings.LINKEDDATANOTIFICATION_ALLOWED_HOSTS, value = "192.0.2.1") public void testRejectMessageFromNonWhitelistedHost() { - - JsonObject message = createRelationshipAnnouncementMessage( - "https://doi.org/10.1234/test", - "Test Publication", - datasetPid, - "Cites" - ); + String message = createRelationshipAnnouncementMessage("https://doi.org/10.1234/test", datasetPid, + DATACITE_URI_PREFIX + "Cites"); // Send the message - should be rejected - Response response = UtilIT.sendMessageToLDNInbox(message.toString()); + Response response = UtilIT.sendMessageToLDNInbox(message); - response.then().assertThat() - .statusCode(FORBIDDEN.getStatusCode()); - - // Restore whitelist - Response restoreWhitelist = UtilIT.setSetting( - SettingsServiceBean.Key.LDNMessageHosts, - "*" - ); - restoreWhitelist.then().assertThat().statusCode(OK.getStatusCode()); + response.then().assertThat().statusCode(FORBIDDEN.getStatusCode()); } @Test @@ -168,167 +157,95 @@ public void testRejectInvalidJsonLD() { Response response = UtilIT.sendMessageToLDNInbox(invalidJson); - response.then().assertThat() - .statusCode(400); // Bad Request + response.then().assertThat().statusCode(BAD_REQUEST.getStatusCode()); } @Test public void testRejectMessageForNonExistentDataset() { String nonExistentPid = "doi:10.5072/FK2/NONEXISTENT"; - JsonObject message = createRelationshipAnnouncementMessage( - "https://doi.org/10.1234/test", - "Test Publication", - nonExistentPid, - "Cites" - ); + String message = createRelationshipAnnouncementMessage("https://doi.org/10.1234/test", nonExistentPid, + DATACITE_URI_PREFIX + "Cites"); - Response response = UtilIT.sendMessageToLDNInbox(message.toString()); + Response response = UtilIT.sendMessageToLDNInbox(message); - response.then().assertThat() - .statusCode(SERVICE_UNAVAILABLE.getStatusCode()); + response.then().assertThat().statusCode(SERVICE_UNAVAILABLE.getStatusCode()); } @Test public void testAcceptMessageWithMultipleRelationshipTypes() { - // Test different relationship types - String[] relationships = { - "Cites", - "IsSupplementTo", - "IsReferencedBy", - "IsRelatedTo" - }; + // Test different relationship types - the ones supported by Dataverse and the + // default from DSpace + String[] relationships = { DATACITE_URI_PREFIX + "Cites", DATACITE_URI_PREFIX + "IsSupplementTo", + DATACITE_URI_PREFIX + "IsReferencedBy", DATACITE_URI_PREFIX + "IsCitedBy", + DATACITE_URI_PREFIX + "IsSupplementedBy", DATACITE_URI_PREFIX + "References", FRBR_SUPPLEMENT }; for (String relationship : relationships) { - JsonObject message = createRelationshipAnnouncementMessage( - "https://doi.org/10.1234/test-" + relationship.toLowerCase(), - "Test Publication for " + relationship, - datasetPid, - relationship - ); - - Response response = UtilIT.sendMessageToLDNInbox(message.toString()); - - response.then().assertThat() - .statusCode(OK.getStatusCode()) - .body("data.message", equalTo("Message Received")); + String message = createRelationshipAnnouncementMessage( + "https://doi.org/10.1234/test-" + relationship.toLowerCase(), datasetPid, relationship); + + Response response = UtilIT.sendMessageToLDNInbox(message); + + response.then().assertThat().statusCode(OK.getStatusCode()).body("data.message", + equalTo("Message Received")); } } @Test - public void testAcceptMessageWithDifferentResourceTypes() { - String[] resourceTypes = { - "ScholarlyArticle", - "Dataset", - "Software", - "Preprint" - }; - - for (String resourceType : resourceTypes) { - JsonObject message = createRelationshipAnnouncementMessageWithType( - "https://doi.org/10.1234/test-" + resourceType.toLowerCase(), - "Test Resource of type " + resourceType, - datasetPid, - "Cites", - resourceType - ); - - Response response = UtilIT.sendMessageToLDNInbox(message.toString()); - - response.then().assertThat() - .statusCode(OK.getStatusCode()) - .body("data.message", equalTo("Message Received")); - } + public void testAcceptMessageWithUrn() { + // Test with URN format identifier + String urnPid = "urn:nbn:de:0000-12345"; + + // Create a dataset with URN (in real scenario, this would be configured) + // For this test, we'll use the existing dataset but reference it with a URN in + // the message + String message = createRelationshipAnnouncementMessage("https://doi.org/10.1234/test-urn", urnPid, + DATACITE_URI_PREFIX + "Cites"); + + Response response = UtilIT.sendMessageToLDNInbox(message); + + response.then().assertThat().statusCode(OK.getStatusCode()); } - /** - * Helper method to create a COAR Notify Relationship Announcement message - */ - private JsonObject createRelationshipAnnouncementMessage( - String citingResourceId, - String citingResourceName, - String citedDatasetPid, - String relationship) { - return createRelationshipAnnouncementMessageWithType( - citingResourceId, - citingResourceName, - citedDatasetPid, - relationship, - "ScholarlyArticle" - ); + @Test + public void testAcceptMessageWithHandle() { + // Test with Handle format identifier + String handlePid = "hdl:1234.5/67890"; + + String message = createRelationshipAnnouncementMessage("https://doi.org/10.1234/test-handle", handlePid, + DATACITE_URI_PREFIX + "Cites"); + + Response response = UtilIT.sendMessageToLDNInbox(message); + + response.then().assertThat().statusCode(OK.getStatusCode()); + } + + @Test + public void testRejectMessageWithNonUriRelationship() { + // Test with an unsupported relationship type + String message = createRelationshipAnnouncementMessage("https://doi.org/10.1234/test-unsupported", datasetPid, + "UnsupportedRelationType"); + + Response response = UtilIT.sendMessageToLDNInbox(message); + + // Should be rejected + response.then().assertThat().statusCode(BAD_REQUEST.getStatusCode()); } /** - * Helper method to create a COAR Notify Relationship Announcement message with specific resource type + * Helper method to create a COAR Notify Relationship Announcement message with + * specific resource type */ - private JsonObject createRelationshipAnnouncementMessageWithType( - String citingResourceId, - String citingResourceName, - String citedDatasetPid, - String relationship, - String resourceType) { - - String messageId = "urn:uuid:" + UUID.randomUUID().toString(); - String relationshipId = "urn:uuid:" + UUID.randomUUID().toString(); - - // Convert PID to URL format if needed - String citedDatasetUrl = citedDatasetPid; - if (citedDatasetPid.startsWith("doi:")) { - citedDatasetUrl = "https://doi.org/" + citedDatasetPid.substring(4); - } else if (citedDatasetPid.startsWith("hdl:")) { - citedDatasetUrl = "https://hdl.handle.net/" + - - citedDatasetPid.substring(4); - } + private String createRelationshipAnnouncementMessage(String citingResourceId, String targetDatasetPid, + String relationship) { - // Build the COAR Notify message following the specification - JsonObjectBuilder messageBuilder = Json.createObjectBuilder(); - - // Add @context - messageBuilder.add("@context", Json.createArrayBuilder() - .add("https://www.w3.org/ns/activitystreams") - .add("https://purl.org/coar/notify")); - - // Add message id and type - messageBuilder.add("id", messageId); - messageBuilder.add("type", Json.createArrayBuilder() - .add("Announce") - .add("coar-notify:RelationshipAction")); - - // Add actor (the system sending the notification) - messageBuilder.add("actor", Json.createObjectBuilder() - .add("id", "https://example.org/repository") - .add("name", "Example Repository") - .add("type", "Service")); - - // Add origin (inbox of the sender) - messageBuilder.add("origin", Json.createObjectBuilder() - .add("id", "https://example.org/repository") - .add("inbox", "https://example.org/inbox") - .add("type", "Service")); - - // Add target (inbox of the receiver - this Dataverse instance) - messageBuilder.add("target", Json.createObjectBuilder() - .add("id", RestAssured.baseURI) - .add("inbox", RestAssured.baseURI + "/api/inbox") - .add("type", "Service")); - - // Add object (the relationship being announced) - messageBuilder.add("object", Json.createObjectBuilder() - .add("id", relationshipId) - .add("type", "Relationship") - .add("as:subject", citingResourceId) - .add("as:relationship", "https://purl.org/datacite/ontology#" + relationship) - .add("as:object", citedDatasetUrl)); - - // Add context (the citing resource details) - messageBuilder.add("context", Json.createObjectBuilder() - .add("id", citingResourceId) - .add("ietf:cite-as", citingResourceId) - .add("type", "sorg:" + resourceType) - .add("sorg:name", citingResourceName)); - - return messageBuilder.build(); - } -} + JsonObjectBuilder targetBuilder = Json.createObjectBuilder().add("id", RestAssured.baseURI) + .add("inbox", RestAssured.baseURI + "/api/inbox").add("type", "Service"); + + JsonObject rel = Json.createObjectBuilder().add("as:object", targetDatasetPid) + .add("as:relationship", relationship).add("as:subject", citingResourceId) + .add("id", "urn:uuid:" + UUID.randomUUID().toString()).add("type", "Relationship").build(); + return COARNotifyRelationshipAnnouncementStep.buildAnnouncement(rel, targetBuilder.build()); + } +} \ No newline at end of file From e9dd212bc6359ed91efb8039491589b5965706c4 Mon Sep 17 00:00:00 2001 From: qqmyers Date: Thu, 30 Oct 2025 16:07:13 -0400 Subject: [PATCH 27/40] don't autofollow redirect, handle more redirect codes --- .../dataverse/api/ldn/COARNotifyRelationshipAnnouncement.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/api/ldn/COARNotifyRelationshipAnnouncement.java b/src/main/java/edu/harvard/iq/dataverse/api/ldn/COARNotifyRelationshipAnnouncement.java index c9189374343..f49d9cd9154 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/ldn/COARNotifyRelationshipAnnouncement.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/ldn/COARNotifyRelationshipAnnouncement.java @@ -120,7 +120,7 @@ private String extractField(JsonObject msgObject, String key) { private ResourceMetadata retrieveResourceMetadata(String subjectId) { ResourceMetadata metadata = new ResourceMetadata(); - try (CloseableHttpClient client = HttpClients.createDefault()) { + try (CloseableHttpClient client = HttpClients.custom().disableRedirectHandling().build()) { logger.info("Getting " + subjectId); // Step 1: Initial GET request expecting a 30x redirect @@ -130,7 +130,7 @@ private ResourceMetadata retrieveResourceMetadata(String subjectId) { CloseableHttpResponse initialResponse = client.execute(initialGet); int statusCode = initialResponse.getStatusLine().getStatusCode(); - if (statusCode == 302 || statusCode == 303) { + if (statusCode == 301 || statusCode == 302 || statusCode == 303 || statusCode == 307 || statusCode == 308) { String location = initialResponse.getFirstHeader("Location").getValue(); logger.info("Redirecting to: " + location); initialResponse.close(); From 21682b53f385532f359688f15e6535bd030e68f7 Mon Sep 17 00:00:00 2001 From: qqmyers Date: Thu, 30 Oct 2025 16:07:48 -0400 Subject: [PATCH 28/40] get absolute redirect url, fix signposting parsing --- .../COARNotifyRelationshipAnnouncement.java | 32 +++++++++++++------ 1 file changed, 23 insertions(+), 9 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/api/ldn/COARNotifyRelationshipAnnouncement.java b/src/main/java/edu/harvard/iq/dataverse/api/ldn/COARNotifyRelationshipAnnouncement.java index f49d9cd9154..efb21d39a2b 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/ldn/COARNotifyRelationshipAnnouncement.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/ldn/COARNotifyRelationshipAnnouncement.java @@ -134,9 +134,14 @@ private ResourceMetadata retrieveResourceMetadata(String subjectId) { String location = initialResponse.getFirstHeader("Location").getValue(); logger.info("Redirecting to: " + location); initialResponse.close(); - + // Assure the location is an absolute URI by converting any relative redirect + // location to absolute URI using the original request URI as base + // An absolute location will not be changed by this call + URI locationUri = new URI(subjectId).resolve(new URI(location)); + logger.info("Absolute redirect: " + locationUri.toString()); + // Step 2: HEAD request to get Signposting links - HttpHead headRequest = new HttpHead(location); + HttpHead headRequest = new HttpHead(locationUri); CloseableHttpResponse headResponse = client.execute(headRequest); if (headResponse.getStatusLine().getStatusCode() == 200) { @@ -174,13 +179,21 @@ private String extractDataCiteXmlUrl(CloseableHttpResponse headResponse) { for (org.apache.http.Header linkHeader : linkHeaders) { String linkValue = linkHeader.getValue(); - if (linkValue.contains("application/vnd.datacite.datacite+xml")) { - int urlStart = linkValue.indexOf('<'); - int urlEnd = linkValue.indexOf('>'); - if (urlStart != -1 && urlEnd != -1 && urlEnd > urlStart) { - String dataciteXmlUrl = linkValue.substring(urlStart + 1, urlEnd); - logger.info("Found DataCite XML URL: " + dataciteXmlUrl); - return dataciteXmlUrl; + // Split by comma to handle multiple links in a single header + String[] links = linkValue.split(","); + + for (String link : links) { + link = link.trim(); + // Check if this link has the DataCite XML type + if (link.contains("type=\"application/vnd.datacite.datacite+xml\"") || + link.contains("type='application/vnd.datacite.datacite+xml'")) { + int urlStart = link.indexOf('<'); + int urlEnd = link.indexOf('>'); + if (urlStart != -1 && urlEnd != -1 && urlEnd > urlStart) { + String dataciteXmlUrl = link.substring(urlStart + 1, urlEnd); + logger.info("Found DataCite XML URL: " + dataciteXmlUrl); + return dataciteXmlUrl; + } } } } @@ -294,6 +307,7 @@ private void sendNotifications(Dataset dataset, String jsonString) { } }); } + /** * Inner class to hold resource metadata. From 4276798104035e2cf00fa62b24684e73e5035a7a Mon Sep 17 00:00:00 2001 From: qqmyers Date: Thu, 30 Oct 2025 16:56:28 -0400 Subject: [PATCH 29/40] remove overloaded method (failing in xhtml) --- .../java/edu/harvard/iq/dataverse/api/MakeDataCountApi.java | 2 +- .../dataverse/engine/command/impl/GetProvJsonCommand.java | 2 +- .../java/edu/harvard/iq/dataverse/util/json/JsonUtil.java | 6 +++--- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/api/MakeDataCountApi.java b/src/main/java/edu/harvard/iq/dataverse/api/MakeDataCountApi.java index ca4f55da822..ab95976ecdc 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/MakeDataCountApi.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/MakeDataCountApi.java @@ -269,7 +269,7 @@ private boolean processCitationUpdate(Dataset dataset, GlobalId pid, PidProvider JsonObject report; try (InputStream inStream = connection.getInputStream()) { - report = JsonUtil.getJsonObject(inStream); + report = JsonUtil.getJsonObjectFromInputStream(inStream); } finally { connection.disconnect(); } diff --git a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/GetProvJsonCommand.java b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/GetProvJsonCommand.java index b98cd70a4da..c1410d1cd27 100644 --- a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/GetProvJsonCommand.java +++ b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/GetProvJsonCommand.java @@ -38,7 +38,7 @@ public JsonObject execute(CommandContext ctxt) throws CommandException { try (InputStream inputStream = dataAccess.getAuxFileAsInputStream(provJsonExtension)) { JsonObject jsonObject = null; if (null != inputStream) { - jsonObject = JsonUtil.getJsonObject(inputStream); + jsonObject = JsonUtil.getJsonObjectFromInputStream(inputStream); } return jsonObject; } diff --git a/src/main/java/edu/harvard/iq/dataverse/util/json/JsonUtil.java b/src/main/java/edu/harvard/iq/dataverse/util/json/JsonUtil.java index 72a1cd2e1eb..3f320e3f715 100644 --- a/src/main/java/edu/harvard/iq/dataverse/util/json/JsonUtil.java +++ b/src/main/java/edu/harvard/iq/dataverse/util/json/JsonUtil.java @@ -69,7 +69,7 @@ public static String prettyPrint(JsonObject jsonObject) { * not catch any exceptions. * @param serializedJson the JSON object serialized as a {@code String} * @throws JsonException when parsing fails. - * @see #getJsonObject(InputStream) + * @see #getJsonObjectFromInputStream(InputStream) * @see #getJsonObjectFromFile(String) * @see #getJsonArray(String) */ @@ -92,7 +92,7 @@ public static JsonObject getJsonObject(String serializedJson) { * @see #getJsonObject(String) * @see #getJsonObjectFromFile(String) */ - public static JsonObject getJsonObject(InputStream stream) { + public static JsonObject getJsonObjectFromInputStream(InputStream stream) { try (JsonReader jsonReader = Json.createReader(stream)) { return jsonReader.readObject(); } @@ -106,7 +106,7 @@ public static JsonObject getJsonObject(InputStream stream) { * @throws FileNotFoundException when the file cannot be opened for reading * @throws JsonException when parsing fails. * @see #getJsonObject(String) - * @see #getJsonObject(InputStream) + * @see #getJsonObjectFromInputStream(InputStream) */ public static JsonObject getJsonObjectFromFile(String fileName) throws IOException { try (FileReader rdr = new FileReader(fileName)) { From 140d9649394f33a66a8de4a753db2b315805e240 Mon Sep 17 00:00:00 2001 From: Jim Myers Date: Fri, 31 Oct 2025 12:17:22 -0400 Subject: [PATCH 30/40] cleanup, fine logging --- .../COARNotifyRelationshipAnnouncement.java | 85 +++++++++++-------- 1 file changed, 49 insertions(+), 36 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/api/ldn/COARNotifyRelationshipAnnouncement.java b/src/main/java/edu/harvard/iq/dataverse/api/ldn/COARNotifyRelationshipAnnouncement.java index efb21d39a2b..238ba2d74b8 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/ldn/COARNotifyRelationshipAnnouncement.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/ldn/COARNotifyRelationshipAnnouncement.java @@ -95,13 +95,13 @@ public void processMessage(JsonObject msgObject) { Dataset dataset = datasetService.findByGlobalId(objectId); if (dataset == null) { - logger.info("Didn't find dataset for object ID: " + objectId + " - ignoring"); + logger.fine("Didn't find dataset for object ID: " + objectId + " - ignoring"); } // Create the citing resource JSON JsonObject citingResource = buildCitingResourceJson(subjectId, relationship, metadata); String jsonString = JsonUtil.prettyPrint(citingResource); - logger.info("Citing resource: " + jsonString); + logger.fine("Citing resource: " + jsonString); // Send notifications to users with publish permissions sendNotifications(dataset, jsonString); @@ -121,7 +121,7 @@ private ResourceMetadata retrieveResourceMetadata(String subjectId) { ResourceMetadata metadata = new ResourceMetadata(); try (CloseableHttpClient client = HttpClients.custom().disableRedirectHandling().build()) { - logger.info("Getting " + subjectId); + logger.fine("Getting " + subjectId); // Step 1: Initial GET request expecting a 30x redirect HttpGet initialGet = new HttpGet(new URI(subjectId)); @@ -132,13 +132,13 @@ private ResourceMetadata retrieveResourceMetadata(String subjectId) { if (statusCode == 301 || statusCode == 302 || statusCode == 303 || statusCode == 307 || statusCode == 308) { String location = initialResponse.getFirstHeader("Location").getValue(); - logger.info("Redirecting to: " + location); + logger.fine("Redirecting to: " + location); initialResponse.close(); // Assure the location is an absolute URI by converting any relative redirect // location to absolute URI using the original request URI as base // An absolute location will not be changed by this call URI locationUri = new URI(subjectId).resolve(new URI(location)); - logger.info("Absolute redirect: " + locationUri.toString()); + logger.fine("Absolute redirect: " + locationUri.toString()); // Step 2: HEAD request to get Signposting links HttpHead headRequest = new HttpHead(locationUri); @@ -152,20 +152,20 @@ private ResourceMetadata retrieveResourceMetadata(String subjectId) { if (dataciteXmlUrl != null) { parseDataCiteXml(dataciteXmlUrl, client, metadata); } else { - logger.info("No DataCite XML URL found in Signposting links"); + logger.fine("No DataCite XML URL found in Signposting links"); } } else { - logger.info("HEAD request failed with status: " + headResponse.getStatusLine().getStatusCode()); + logger.fine("HEAD request failed with status: " + headResponse.getStatusLine().getStatusCode()); headResponse.close(); } } else { - logger.info("Expected 302/303 redirect but received status: " + statusCode); + logger.fine("Expected 302/303 redirect but received status: " + statusCode); initialResponse.close(); } } catch (Exception e) { - logger.info("Unable to get metadata from " + subjectId); - logger.info(e.getLocalizedMessage()); + logger.fine("Unable to get metadata from " + subjectId); + logger.fine(e.getLocalizedMessage()); } return metadata; @@ -174,33 +174,46 @@ private ResourceMetadata retrieveResourceMetadata(String subjectId) { /** * Extract DataCite XML URL from Signposting Link headers. */ - private String extractDataCiteXmlUrl(CloseableHttpResponse headResponse) { - org.apache.http.Header[] linkHeaders = headResponse.getHeaders("Link"); - for (org.apache.http.Header linkHeader : linkHeaders) { - String linkValue = linkHeader.getValue(); - // Split by comma to handle multiple links in a single header - String[] links = linkValue.split(","); +/** + * Extract DataCite XML URL from Signposting Link headers. + */ +private String extractDataCiteXmlUrl(CloseableHttpResponse headResponse) { + org.apache.http.Header[] linkHeaders = headResponse.getHeaders("Link"); + + for (org.apache.http.Header linkHeader : linkHeaders) { + String linkValue = linkHeader.getValue(); + logger.fine("Full Link header: " + linkValue); + + // Split by comma to handle multiple links in a single header + String[] links = linkValue.split(","); + + for (String link : links) { + link = link.trim(); + logger.fine("Checking link segment: [" + link + "]"); - for (String link : links) { - link = link.trim(); - // Check if this link has the DataCite XML type - if (link.contains("type=\"application/vnd.datacite.datacite+xml\"") || - link.contains("type='application/vnd.datacite.datacite+xml'")) { - int urlStart = link.indexOf('<'); - int urlEnd = link.indexOf('>'); - if (urlStart != -1 && urlEnd != -1 && urlEnd > urlStart) { - String dataciteXmlUrl = link.substring(urlStart + 1, urlEnd); - logger.info("Found DataCite XML URL: " + dataciteXmlUrl); - return dataciteXmlUrl; - } + // Check if this link has the DataCite XML type + boolean hasDataCiteType = link.contains("type=\"application/vnd.datacite.datacite+xml\"") || + link.contains("type='application/vnd.datacite.datacite+xml'"); + + logger.fine("Contains DataCite type: " + hasDataCiteType); + + if (hasDataCiteType) { + int urlStart = link.indexOf('<'); + int urlEnd = link.indexOf('>'); + if (urlStart != -1 && urlEnd != -1 && urlEnd > urlStart) { + String dataciteXmlUrl = link.substring(urlStart + 1, urlEnd); + logger.fine("Found DataCite XML URL: " + dataciteXmlUrl); + return dataciteXmlUrl; } } } - - return null; } + logger.fine("No DataCite XML URL found in Link headers"); + return null; +} + /** * Parse DataCite XML to extract title and resource type. */ @@ -213,7 +226,7 @@ private void parseDataCiteXml(String dataciteXmlUrl, CloseableHttpClient client, if (xmlResponse.getStatusLine().getStatusCode() == 200) { String xmlContent = EntityUtils.toString(xmlResponse.getEntity(), "UTF-8"); - logger.info("Retrieved DataCite XML"); + logger.fine("Retrieved DataCite XML"); javax.xml.parsers.DocumentBuilderFactory factory = javax.xml.parsers.DocumentBuilderFactory.newInstance(); factory.setNamespaceAware(true); @@ -229,9 +242,9 @@ private void parseDataCiteXml(String dataciteXmlUrl, CloseableHttpClient client, if (titleNodes.getLength() > 0) { metadata.name = titleNodes.item(0).getTextContent(); - logger.info("Extracted title from DataCite XML: " + metadata.name); + logger.fine("Extracted title from DataCite XML: " + metadata.name); } else { - logger.info("No title element found in DataCite XML"); + logger.fine("No title element found in DataCite XML"); } // Extract resource type @@ -245,12 +258,12 @@ private void parseDataCiteXml(String dataciteXmlUrl, CloseableHttpClient client, String resourceTypeGeneral = resourceTypeElement.getAttribute("resourceTypeGeneral"); if (resourceTypeGeneral != null && !resourceTypeGeneral.isEmpty()) { metadata.itemType = resourceTypeGeneral.toLowerCase(); - logger.info("Extracted resource type: " + metadata.itemType); + logger.fine("Extracted resource type: " + metadata.itemType); } } } else { - logger.info("Failed to retrieve DataCite XML. Status: " + xmlResponse.getStatusLine().getStatusCode()); + logger.fine("Failed to retrieve DataCite XML. Status: " + xmlResponse.getStatusLine().getStatusCode()); } xmlResponse.close(); @@ -266,7 +279,7 @@ private void parseDataCiteXml(String dataciteXmlUrl, CloseableHttpClient client, */ private String extractRelationshipLabel(String relationshipId) { int index = relationshipId.indexOf("#"); - logger.info("Found # at " + index + " in " + relationshipId); + logger.fine("Found # at " + index + " in " + relationshipId); return relationshipId.substring(index + 1); } From e2804f1780618ff75f7aaa66078f9d91700e526b Mon Sep 17 00:00:00 2001 From: qqmyers Date: Fri, 31 Oct 2025 12:49:53 -0400 Subject: [PATCH 31/40] info to fine --- src/main/java/edu/harvard/iq/dataverse/api/LDNInbox.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/api/LDNInbox.java b/src/main/java/edu/harvard/iq/dataverse/api/LDNInbox.java index 798c0095fb9..4749e258668 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/LDNInbox.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/LDNInbox.java @@ -73,7 +73,7 @@ public Response acceptMessage(String body) { // Only process messages from whitelisted hosts if (!allowedIPs.equals("*") && !allowedIPs.contains(origin.toString())) { - logger.info("Ignoring message from IP address: " + origin.toString()); + logger.fine("Ignoring message from IP address: " + origin.toString()); throw new ForbiddenException("The LDN Inbox does not accept messages from this address"); } @@ -83,7 +83,7 @@ public Response acceptMessage(String body) { throw new BadRequestException("Could not parse JSON message."); } - logger.info(JsonUtil.prettyPrint(jsonld)); + logger.fine(JsonUtil.prettyPrint(jsonld)); // Process message based on type processMessage(jsonld); From b8288f8e011c40b09d45c32708116ba33ba34ec1 Mon Sep 17 00:00:00 2001 From: qqmyers Date: Fri, 31 Oct 2025 12:50:05 -0400 Subject: [PATCH 32/40] doc updates --- doc/release-notes/10490-COAR-Notify.md | 14 +++++++++----- .../source/api/linkeddatanotification.rst | 6 ++++-- .../source/developers/workflows.rst | 2 +- .../source/installation/config.rst | 19 ++++++++++++++++++- 4 files changed, 32 insertions(+), 9 deletions(-) diff --git a/doc/release-notes/10490-COAR-Notify.md b/doc/release-notes/10490-COAR-Notify.md index 934288d4d4c..06fe6774fcb 100644 --- a/doc/release-notes/10490-COAR-Notify.md +++ b/doc/release-notes/10490-COAR-Notify.md @@ -5,15 +5,19 @@ Dataverse now supports sending and recieving [Linked Data Notification ](https:/ Dataverse can send messages to configured repositories announcing that a dataset has a related publication (as defined in the dataset metadata). This may be done automatically upon publication or triggered manually by a superuser. The receiving repository may do anything with the message, with the default expectation being that the repository will create a backlink from the publication to the dataset (assuming the publication exists in the repository, admins agree the link makes sense, etc.) -Conversely, Dataverse can recieve notices from other configured repositories announcing relationships between their publications and datasets. If the referenced dataset exists in the Dataverse instance, a notification will be sent to users who can publish the dataset. They can then decide whether to create a backlink to the publication in the dataset metadata. +Conversely, Dataverse can recieve notices from other configured repositories announcing relationships between their publications and datasets. If the referenced dataset exists in the Dataverse instance, a notification will be sent to users who can publish the dataset, or, optionally only superusers who can publish the dataset. They can then decide whether to create a backlink to the publication in the dataset metadata. (Earlier releases of Dataverse had experimental support in this area that was based on message formats defined prior to finalization of the COAR Notify specification for relationship announcements.) +#### New Settings/JVM Options + Configuration for sending messages involves specifying the -:LDNTarget and :LDNAnnounceRequiredFields +:COARNotifyRelationshipAnnpuncementTargets and :COARNotifyRelationshipAnnpuncementTriggerFields -Configuration to receive messages involves specifying the -:LDNMessageHosts +Configuration to receive messages involves specifying +DATAVERSE_LDN_ALLOWED_HOSTS (dataverse.ldn.allowed-hosts) -(FWIW: These settings are not new) +Notifications are sent by default to users who can publish a dataset. The option below can be used to restrict notifications to superusers who can publish the dataset. + +DATAVERSE_COAR_NOTOIFY_RELATIONSHIP_ANNOUNCEMENT_NOTIFY_SUPERSUSERS_ONLY diff --git a/doc/sphinx-guides/source/api/linkeddatanotification.rst b/doc/sphinx-guides/source/api/linkeddatanotification.rst index b241065e6b7..683f9c78e34 100644 --- a/doc/sphinx-guides/source/api/linkeddatanotification.rst +++ b/doc/sphinx-guides/source/api/linkeddatanotification.rst @@ -7,9 +7,11 @@ Dataverse has a related capability to send COAR Notify Relationship Announcement The motivating use case is to support a use case where Dataverse administrators may wish to create back-links to the remote resource (e.g. as a Related Publication, Related Material, etc.). -Upon receipt of a relevant message, Dataverse will create Announcement Received notifications for superusers, who can edit the dataset involved. (In the motivating use case, these users may then add an appropriate relationship and use the Update Curent Version publishing option to add it to the most recently published version of the dataset.) +Upon receipt of a relevant message, Dataverse will create Announcement Received notifications for users who can edit the dataset involved. Notifications can be restricted to superusers who can publish the dataset as described below. (In the motivating use case, these superusers may then add an appropriate relationship and use the Update Curent Version publishing option to add it to the most recently published version of the dataset.) -The ``:LDNMessageHosts`` setting is a comma-separated whitelist of hosts from which Dataverse will accept and process messages. By default, no hosts are allowed. ``*`` can be used in testing to indicate all hosts are allowed. +The ``dataverse.ldn.allowed-hosts`` JVM option is a comma-separated whitelist of hosts from which Dataverse will accept and process messages. By default, no hosts are allowed. ``*`` can be used in testing to indicate all hosts are allowed. + +The ``dataverse.ldn.coar-noptify.relationship-announcement.notify-superusers-only`` JVM option can be set to ``true`` to restrict notifications to superusers only (those who can publish the dataset). The default is to notify all users who can publish the dataset. Messages can be sent via POST, using the application/ld+json ContentType: diff --git a/doc/sphinx-guides/source/developers/workflows.rst b/doc/sphinx-guides/source/developers/workflows.rst index 3f271990069..bc894a7dfeb 100644 --- a/doc/sphinx-guides/source/developers/workflows.rst +++ b/doc/sphinx-guides/source/developers/workflows.rst @@ -211,7 +211,7 @@ to a specific set of LDN Inboxes announcing a relationship between a newly publi The two parameters are * ``:COARNotifyRelationshipAnnouncementTriggerFields`` - a list of metadata field types that can trigger messages. Separate messages will be sent for each field (whether of the same type or not). * ``:COARNotifyRelationshipAnnouncementTargets`` - a JSON Array of JSON objects containing ``id``, ``inbox``, and ``type`` fields as required by the `COAR Notify Relationship Announcement specification `_ . -The ``inbox`` value should be the full URL of the target LDN inbox to which messages should be sent, e.g. ``{"id": "https://dashv7-dev.lib.harvard.edu","inbox": "https://dashv7-api-dev.lib.harvard.edu/server/ldn/inbox","type": "Service"}`` ). +The ``inbox`` value should be the full URL of the target LDN inbox to which messages should be sent, e.g. ``{"id": "https://dashv7-dev.lib.harvard.edu","inbox": "https://dashv7-api-dev.lib.harvard.edu/server/ldn/inbox","type": ["Service"]}`` ). .. code:: json diff --git a/doc/sphinx-guides/source/installation/config.rst b/doc/sphinx-guides/source/installation/config.rst index bbf0a0d2449..85df28024cd 100644 --- a/doc/sphinx-guides/source/installation/config.rst +++ b/doc/sphinx-guides/source/installation/config.rst @@ -3669,7 +3669,7 @@ Experimental. See :doc:`/developers/search-services`. .. _dataverse.cors: CORS Settings -------------- ++++++++++++++ The following settings control Cross-Origin Resource Sharing (CORS) for your Dataverse installation. @@ -3747,6 +3747,23 @@ Example: ``dataverse.api.mdc.min-delay-ms=100`` (enforces a minimum 100ms delay Can also be set via any `supported MicroProfile Config API source`_, e.g. the environment variable ``DATAVERSE_API_MDC_MIN_DELAY_MS``. +.. dataverse.ldn + +Linked Data Notifications (LDN) Allowed Hosts ++++++++++++++++++++++++++++++++++++++++++++++ + +Dataverse supports receiving LDN notifications via the /api/inbox endpoint. The datavers.ldn.allowed-hosts allows you to specify the list of host IP addresses from which LDN notifications can be received, or ``*`` to receive messages from anywhere. + +Example: ``dataverse.ldn.allowed-hosts=*`` + +COAR Notify Relationship Announcement Notify Superusers Only +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ + +When Dataverse receives an LDN message conforming to the COAR Notify Relationship Announcement format and the message is about a Dataset hosted in the installation, Dataverse will send an notification to users who have permission to publish the dataset. +This can instead be restricted to only superusers who can publish the dataset using this option. + +Example: ``dataverse.coar-notify.relationship-announcement.notify-superusers-only=true`` + .. _feature-flags: Feature Flags From d127ab5f9c98f707f9789bb405df6b79712bf07f Mon Sep 17 00:00:00 2001 From: qqmyers Date: Thu, 13 Nov 2025 09:42:55 -0500 Subject: [PATCH 33/40] Changes per review --- .../source/api/linkeddatanotification.rst | 2 +- .../harvard/iq/dataverse/api/LDNInbox.java | 22 +++++++++++-------- 2 files changed, 14 insertions(+), 10 deletions(-) diff --git a/doc/sphinx-guides/source/api/linkeddatanotification.rst b/doc/sphinx-guides/source/api/linkeddatanotification.rst index 683f9c78e34..e82b9216b6a 100644 --- a/doc/sphinx-guides/source/api/linkeddatanotification.rst +++ b/doc/sphinx-guides/source/api/linkeddatanotification.rst @@ -9,7 +9,7 @@ The motivating use case is to support a use case where Dataverse administrators Upon receipt of a relevant message, Dataverse will create Announcement Received notifications for users who can edit the dataset involved. Notifications can be restricted to superusers who can publish the dataset as described below. (In the motivating use case, these superusers may then add an appropriate relationship and use the Update Curent Version publishing option to add it to the most recently published version of the dataset.) -The ``dataverse.ldn.allowed-hosts`` JVM option is a comma-separated whitelist of hosts from which Dataverse will accept and process messages. By default, no hosts are allowed. ``*`` can be used in testing to indicate all hosts are allowed. +The ``dataverse.ldn.allowed-hosts`` JVM option is a comma-separated list of hosts from which Dataverse will accept and process messages. By default, no hosts are allowed. ``*`` can be used in testing to indicate all hosts are allowed. The ``dataverse.ldn.coar-noptify.relationship-announcement.notify-superusers-only`` JVM option can be set to ``true`` to restrict notifications to superusers only (those who can publish the dataset). The default is to notify all users who can publish the dataset. diff --git a/src/main/java/edu/harvard/iq/dataverse/api/LDNInbox.java b/src/main/java/edu/harvard/iq/dataverse/api/LDNInbox.java index 4749e258668..071450a5f98 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/LDNInbox.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/LDNInbox.java @@ -111,15 +111,19 @@ private JsonObject parseJsonLD(String body) { // with the contents of the final redirect (current as of 10/29/2025) // and try again body = body.replace("\"https://purl.org/coar/notify\"", - "{\n" + " \"@vocab\": \"http://purl.org/coar/notify_vocabulary/\",\n" - + " \"ietf\": \"http://www.iana.org/assignments/relation/\",\n" - + " \"coar-notify\": \"http://purl.org/coar/notify_vocabulary/\",\n" - + " \"sorg\": \"http://schema.org/\",\n" - + " \"ReviewAction\": \"coar-notify:ReviewAction\",\n" - + " \"EndorsementAction\": \"coar-notify:EndorsementAction\",\n" - + " \"IngestAction\": \"coar-notify:IngestAction\",\n" - + " \"ietf:cite-as\": {\n" + " \"@type\": \"@id\"\n" - + " }}"); + """ + { + "@vocab": "http://purl.org/coar/notify_vocabulary/", + "ietf": "http://www.iana.org/assignments/relation/", + "coar-notify": "http://purl.org/coar/notify_vocabulary/", + "sorg": "http://schema.org/", + "ReviewAction": "coar-notify:ReviewAction", + "EndorsementAction": "coar-notify:EndorsementAction", + "IngestAction": "coar-notify:IngestAction", + "ietf:cite-as": { + "@type": "@id" + } + }"""); jsonld = JSONLDUtil.decontextualizeJsonLD(body); } From 8c96c774a775a725f9d0d087b40c72a46d6042f8 Mon Sep 17 00:00:00 2001 From: qqmyers Date: Thu, 13 Nov 2025 14:29:00 -0500 Subject: [PATCH 34/40] only send new relationships --- .../edu/harvard/iq/dataverse/Dataset.java | 14 + ...OARNotifyRelationshipAnnouncementStep.java | 170 +++++++- ...otifyRelationshipAnnouncementStepTest.java | 380 ++++++++++++++++++ 3 files changed, 563 insertions(+), 1 deletion(-) create mode 100644 src/test/java/edu/harvard/iq/dataverse/workflow/internalspi/COARNotifyRelationshipAnnouncementStepTest.java diff --git a/src/main/java/edu/harvard/iq/dataverse/Dataset.java b/src/main/java/edu/harvard/iq/dataverse/Dataset.java index e7e5903482c..2acb92f690e 100644 --- a/src/main/java/edu/harvard/iq/dataverse/Dataset.java +++ b/src/main/java/edu/harvard/iq/dataverse/Dataset.java @@ -534,6 +534,20 @@ public DatasetVersion getReleasedVersion() { return null; } + public DatasetVersion getPriorReleasedVersion() { + boolean foundReleasedVersion = false; + for (DatasetVersion version : this.getVersions()) { + if (version.isReleased()) { + if(foundReleasedVersion) { + return version; + } else { + foundReleasedVersion = true; + } + } + } + return null; + } + public DatasetVersion getVersionFromId(Long datasetVersionId) { for (DatasetVersion version : this.getVersions()) { if (datasetVersionId == version.getId().longValue()) { diff --git a/src/main/java/edu/harvard/iq/dataverse/workflow/internalspi/COARNotifyRelationshipAnnouncementStep.java b/src/main/java/edu/harvard/iq/dataverse/workflow/internalspi/COARNotifyRelationshipAnnouncementStep.java index fb18b2918f9..451a00efcd2 100644 --- a/src/main/java/edu/harvard/iq/dataverse/workflow/internalspi/COARNotifyRelationshipAnnouncementStep.java +++ b/src/main/java/edu/harvard/iq/dataverse/workflow/internalspi/COARNotifyRelationshipAnnouncementStep.java @@ -2,7 +2,9 @@ import edu.harvard.iq.dataverse.Dataset; import edu.harvard.iq.dataverse.DatasetField; +import edu.harvard.iq.dataverse.DatasetFieldCompoundValue; import edu.harvard.iq.dataverse.DatasetFieldType; +import edu.harvard.iq.dataverse.DatasetFieldValue; import edu.harvard.iq.dataverse.DatasetVersion; import edu.harvard.iq.dataverse.GlobalId; import edu.harvard.iq.dataverse.branding.BrandingUtil; @@ -22,6 +24,7 @@ import java.net.URI; import java.net.URISyntaxException; import java.nio.charset.StandardCharsets; +import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; import java.util.HashMap; @@ -40,6 +43,7 @@ import jakarta.json.JsonString; import jakarta.json.JsonValue; +import org.apache.commons.lang3.Strings; import org.apache.http.client.methods.CloseableHttpResponse; import org.apache.http.client.methods.HttpPost; import org.apache.http.entity.StringEntity; @@ -86,13 +90,35 @@ public WorkflowStepResult run(WorkflowContext context) { // First check that we have what is required Dataset d = context.getDataset(); DatasetVersion dv = d.getReleasedVersion(); + DatasetVersion priorVersion = d.getPriorReleasedVersion(); List dvf = dv.getDatasetFields(); Map fields = new HashMap(); List reqFields = Arrays .asList(((String) context.getSettings().getOrDefault(REQUIRED_FIELDS, "")).split(",\\s*")); + + Map priorFields = new HashMap(); + if (priorVersion != null) { + for (DatasetField pdf : priorVersion.getDatasetFields()) { + if (!pdf.isEmpty() && reqFields.contains(pdf.getDatasetFieldType().getName())) { + priorFields.put(pdf.getDatasetFieldType().getName(), pdf); + } + } + } + for (DatasetField df : dvf) { if (!df.isEmpty() && reqFields.contains(df.getDatasetFieldType().getName())) { - fields.put(df.getDatasetFieldType().getName(), df); + DatasetField priorField = priorFields.get(df.getDatasetFieldType().getName()); + + if (priorVersion == null || priorField == null) { + // No prior version, include all values + fields.put(df.getDatasetFieldType().getName(), df); + } else { + // Create a filtered field with only new values + DatasetField filteredField = filterNewValues(df, priorField); + if (!filteredField.isEmpty()) { + fields.put(df.getDatasetFieldType().getName(), filteredField); + } + } } } @@ -411,4 +437,146 @@ private boolean isURI(String number) { return false; } + /** + * Create a new DatasetField containing only values that are new compared to the prior field. + * This creates a detached copy to avoid modifying the managed entity. + * + * @param currentField The field from the current version + * @param priorField The field from the prior version + * @return A new DatasetField with only new values + */ + private DatasetField filterNewValues(DatasetField currentField, DatasetField priorField) { + DatasetField filteredField = new DatasetField(); + filteredField.setDatasetFieldType(currentField.getDatasetFieldType()); + + if (currentField.getDatasetFieldType().isCompound()) { + // Handle compound values + List newCompoundValues = new ArrayList<>(); + + for (DatasetFieldCompoundValue currentCompoundValue : currentField.getDatasetFieldCompoundValues()) { + boolean isNew = true; + + // Check if this compound value exists in prior field + if (priorField != null && priorField.getDatasetFieldCompoundValues() != null) { + for (DatasetFieldCompoundValue priorCompoundValue : priorField.getDatasetFieldCompoundValues()) { + if (compoundValuesEqual(currentCompoundValue, priorCompoundValue)) { + isNew = false; + break; + } + } + } + + if (isNew) { + // Create a copy of the compound value + DatasetFieldCompoundValue newCompoundValue = copyCompoundValue(currentCompoundValue, filteredField); + newCompoundValues.add(newCompoundValue); + } + } + + filteredField.setDatasetFieldCompoundValues(newCompoundValues); + + } else if (currentField.getDatasetFieldType().isAllowMultiples()) { + // Handle multiple simple values + List newValues = new ArrayList<>(); + + for (DatasetFieldValue currentValue : currentField.getDatasetFieldValues()) { + boolean isNew = true; + + if (priorField != null && priorField.getDatasetFieldValues() != null) { + for (DatasetFieldValue priorValue : priorField.getDatasetFieldValues()) { + if (valuesEqual(currentValue, priorValue)) { + isNew = false; + break; + } + } + } + + if (isNew) { + DatasetFieldValue newValue = new DatasetFieldValue(); + newValue.setValue(currentValue.getValue()); + newValue.setDatasetField(filteredField); + newValues.add(newValue); + } + } + + filteredField.setDatasetFieldValues(newValues); + + } else { + // Handle single value + if (priorField == null || !valuesEqual(currentField.getSingleValue(), priorField.getSingleValue())) { + filteredField.setSingleValue(currentField.getValue()); + } + } + + return filteredField; + } + + /** + * Check if two compound values are equal by comparing all their child fields. + * Since child fields are ordered, we can do a simpler comparison. + */ + private boolean compoundValuesEqual(DatasetFieldCompoundValue cv1, DatasetFieldCompoundValue cv2) { + if (cv1 == null && cv2 == null) { + return true; + } + if (cv1 == null || cv2 == null) { + return false; + } + + List children1 = cv1.getChildDatasetFields(); + List children2 = cv2.getChildDatasetFields(); + + if (children1.size() != children2.size()) { + return false; + } + + // Since fields are ordered, we can compare them directly by position + for (int i = 0; i < children1.size(); i++) { + DatasetField child1 = children1.get(i); + DatasetField child2 = children2.get(i); + + // Compare field types + if (!child1.getDatasetFieldType().equals(child2.getDatasetFieldType())) { + return false; + } + + // Compare values using Apache Commons StringUtils + if (!Strings.CS.equals(child1.getValue(), child2.getValue())) { + return false; + } + } + + return true; + } + + /** + * Create a deep copy of a compound value + */ + private DatasetFieldCompoundValue copyCompoundValue(DatasetFieldCompoundValue source, DatasetField parentField) { + DatasetFieldCompoundValue copy = new DatasetFieldCompoundValue(); + copy.setParentDatasetField(parentField); + copy.setDisplayOrder(source.getDisplayOrder()); + + List childFieldsCopy = new ArrayList<>(); + for (DatasetField sourceChild : source.getChildDatasetFields()) { + DatasetField childCopy = new DatasetField(); + childCopy.setDatasetFieldType(sourceChild.getDatasetFieldType()); + childCopy.setParentDatasetFieldCompoundValue(copy); + childCopy.setSingleValue(sourceChild.getValue()); + childFieldsCopy.add(childCopy); + } + + copy.setChildDatasetFields(childFieldsCopy); + return copy; + } + + private boolean valuesEqual(DatasetFieldValue v1, DatasetFieldValue v2) { + if (v1 == null && v2 == null) { + return true; + } + if (v1 == null || v2 == null) { + return false; + } + return Strings.CS.equals(v1.getValue(), v2.getValue()); + } } diff --git a/src/test/java/edu/harvard/iq/dataverse/workflow/internalspi/COARNotifyRelationshipAnnouncementStepTest.java b/src/test/java/edu/harvard/iq/dataverse/workflow/internalspi/COARNotifyRelationshipAnnouncementStepTest.java new file mode 100644 index 00000000000..49d75491211 --- /dev/null +++ b/src/test/java/edu/harvard/iq/dataverse/workflow/internalspi/COARNotifyRelationshipAnnouncementStepTest.java @@ -0,0 +1,380 @@ +package edu.harvard.iq.dataverse.workflow.internalspi; + +import edu.harvard.iq.dataverse.DatasetField; +import edu.harvard.iq.dataverse.DatasetFieldCompoundValue; +import edu.harvard.iq.dataverse.DatasetFieldType; +import edu.harvard.iq.dataverse.DatasetFieldValue; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import static org.junit.jupiter.api.Assertions.*; + +class COARNotifyRelationshipAnnouncementStepTest { + + private COARNotifyRelationshipAnnouncementStep step; + private DatasetFieldType multiValueFieldType; + private DatasetFieldType singleValueFieldType; + private DatasetFieldType compoundFieldType; + private DatasetFieldType childFieldType1; + private DatasetFieldType childFieldType2; + + +@BeforeEach +void setUp() { + Map params = new HashMap<>(); + step = new COARNotifyRelationshipAnnouncementStep(params); + + // Setup field types + multiValueFieldType = new DatasetFieldType(); + multiValueFieldType.setName("testMultiField"); + multiValueFieldType.setAllowMultiples(true); + multiValueFieldType.setFieldType(DatasetFieldType.FieldType.TEXT); + multiValueFieldType.setChildDatasetFieldTypes(new ArrayList<>()); + + singleValueFieldType = new DatasetFieldType(); + singleValueFieldType.setName("testSingleField"); + singleValueFieldType.setAllowMultiples(false); + singleValueFieldType.setFieldType(DatasetFieldType.FieldType.TEXT); + singleValueFieldType.setChildDatasetFieldTypes(new ArrayList<>()); + + // Setup compound field type with child fields + compoundFieldType = new DatasetFieldType(); + compoundFieldType.setName("testCompoundField"); + compoundFieldType.setAllowMultiples(true); + compoundFieldType.setFieldType(DatasetFieldType.FieldType.NONE); + + childFieldType1 = new DatasetFieldType(); + childFieldType1.setName("authorName"); + childFieldType1.setFieldType(DatasetFieldType.FieldType.TEXT); + childFieldType1.setParentDatasetFieldType(compoundFieldType); + childFieldType1.setChildDatasetFieldTypes(new ArrayList<>()); + + childFieldType2 = new DatasetFieldType(); + childFieldType2.setName("authorAffiliation"); + childFieldType2.setFieldType(DatasetFieldType.FieldType.TEXT); + childFieldType2.setParentDatasetFieldType(compoundFieldType); + childFieldType2.setChildDatasetFieldTypes(new ArrayList<>()); + + compoundFieldType.setChildDatasetFieldTypes(List.of(childFieldType1, childFieldType2)); +} + @Test + void testFilterNewValues_MultiValue_AllNew() throws Exception { + // Create current field with 3 values + DatasetField currentField = createMultiValueField(multiValueFieldType, "value1", "value2", "value3"); + + // Create prior field with no values + DatasetField priorField = createMultiValueField(multiValueFieldType); + + // Filter + DatasetField filtered = invokeFilterNewValues(currentField, priorField); + + // All values should be included + assertEquals(3, filtered.getDatasetFieldValues().size()); + assertTrue(containsValue(filtered, "value1")); + assertTrue(containsValue(filtered, "value2")); + assertTrue(containsValue(filtered, "value3")); + } + + @Test + void testFilterNewValues_MultiValue_SomeNew() throws Exception { + // Create current field with 3 values + DatasetField currentField = createMultiValueField(multiValueFieldType, "value1", "value2", "value3"); + + // Create prior field with 2 existing values + DatasetField priorField = createMultiValueField(multiValueFieldType, "value1", "value2"); + + // Filter + DatasetField filtered = invokeFilterNewValues(currentField, priorField); + + // Only new value should be included + assertEquals(1, filtered.getDatasetFieldValues().size()); + assertFalse(containsValue(filtered, "value1")); + assertFalse(containsValue(filtered, "value2")); + assertTrue(containsValue(filtered, "value3")); + } + + @Test + void testFilterNewValues_MultiValue_NoneNew() throws Exception { + // Create current field with 2 values + DatasetField currentField = createMultiValueField(multiValueFieldType, "value1", "value2"); + + // Create prior field with same values + DatasetField priorField = createMultiValueField(multiValueFieldType, "value1", "value2"); + + // Filter + DatasetField filtered = invokeFilterNewValues(currentField, priorField); + + // No values should be included + assertEquals(0, filtered.getDatasetFieldValues().size()); + assertTrue(filtered.isEmpty()); + } + + @Test + void testFilterNewValues_SingleValue_Changed() throws Exception { + // Create current field with new value + DatasetField currentField = createSingleValueField(singleValueFieldType, "newValue"); + + // Create prior field with old value + DatasetField priorField = createSingleValueField(singleValueFieldType, "oldValue"); + + // Filter + DatasetField filtered = invokeFilterNewValues(currentField, priorField); + + // New value should be included + assertFalse(filtered.isEmpty()); + assertEquals("newValue", filtered.getValue()); + } + + @Test + void testFilterNewValues_SingleValue_Unchanged() throws Exception { + // Create current field with same value + DatasetField currentField = createSingleValueField(singleValueFieldType, "sameValue"); + + // Create prior field with same value + DatasetField priorField = createSingleValueField(singleValueFieldType, "sameValue"); + + // Filter + DatasetField filtered = invokeFilterNewValues(currentField, priorField); + + // No value should be included + assertTrue(filtered.isEmpty()); + } + + @Test + void testFilterNewValues_DoesNotModifyOriginal() throws Exception { + // Create current field with 3 values + DatasetField currentField = createMultiValueField(multiValueFieldType, "value1", "value2", "value3"); + int originalSize = currentField.getDatasetFieldValues().size(); + + // Create prior field with 2 existing values + DatasetField priorField = createMultiValueField(multiValueFieldType, "value1", "value2"); + + // Filter + DatasetField filtered = invokeFilterNewValues(currentField, priorField); + + // Original field should be unchanged + assertEquals(originalSize, currentField.getDatasetFieldValues().size()); + assertTrue(containsValue(currentField, "value1")); + assertTrue(containsValue(currentField, "value2")); + assertTrue(containsValue(currentField, "value3")); + + // Filtered field should only have new value + assertEquals(1, filtered.getDatasetFieldValues().size()); + assertTrue(containsValue(filtered, "value3")); + } + + + @Test + void testFilterNewValues_CompoundValue_AllNew() throws Exception { + // Create current field with 2 compound values + DatasetField currentField = createCompoundField(compoundFieldType, + new String[]{"Author1", "Affiliation1"}, + new String[]{"Author2", "Affiliation2"}); + + // Create prior field with no values + DatasetField priorField = createCompoundField(compoundFieldType); + + // Filter + DatasetField filtered = invokeFilterNewValues(currentField, priorField); + + // All compound values should be included + assertEquals(2, filtered.getDatasetFieldCompoundValues().size()); + assertTrue(containsCompoundValue(filtered, "Author1", "Affiliation1")); + assertTrue(containsCompoundValue(filtered, "Author2", "Affiliation2")); + } + + @Test + void testFilterNewValues_CompoundValue_SomeNew() throws Exception { + // Create current field with 3 compound values + DatasetField currentField = createCompoundField(compoundFieldType, + new String[]{"Author1", "Affiliation1"}, + new String[]{"Author2", "Affiliation2"}, + new String[]{"Author3", "Affiliation3"}); + + // Create prior field with 2 existing compound values + DatasetField priorField = createCompoundField(compoundFieldType, + new String[]{"Author1", "Affiliation1"}, + new String[]{"Author2", "Affiliation2"}); + + // Filter + DatasetField filtered = invokeFilterNewValues(currentField, priorField); + + // Only new compound value should be included + assertEquals(1, filtered.getDatasetFieldCompoundValues().size()); + assertFalse(containsCompoundValue(filtered, "Author1", "Affiliation1")); + assertFalse(containsCompoundValue(filtered, "Author2", "Affiliation2")); + assertTrue(containsCompoundValue(filtered, "Author3", "Affiliation3")); + } + + @Test + void testFilterNewValues_CompoundValue_NoneNew() throws Exception { + // Create current field with 2 compound values + DatasetField currentField = createCompoundField(compoundFieldType, + new String[]{"Author1", "Affiliation1"}, + new String[]{"Author2", "Affiliation2"}); + + // Create prior field with same compound values + DatasetField priorField = createCompoundField(compoundFieldType, + new String[]{"Author1", "Affiliation1"}, + new String[]{"Author2", "Affiliation2"}); + + // Filter + DatasetField filtered = invokeFilterNewValues(currentField, priorField); + + // No compound values should be included + assertEquals(0, filtered.getDatasetFieldCompoundValues().size()); + assertTrue(filtered.isEmpty()); + } + + @Test + void testFilterNewValues_CompoundValue_PartialMatch() throws Exception { + // Create current field with compound value where one child field changed + DatasetField currentField = createCompoundField(compoundFieldType, + new String[]{"Author1", "NewAffiliation"}); + + // Create prior field with same author but different affiliation + DatasetField priorField = createCompoundField(compoundFieldType, + new String[]{"Author1", "OldAffiliation"}); + + // Filter + DatasetField filtered = invokeFilterNewValues(currentField, priorField); + + // Should be treated as a new compound value since child field changed + assertEquals(1, filtered.getDatasetFieldCompoundValues().size()); + assertTrue(containsCompoundValue(filtered, "Author1", "NewAffiliation")); + } + + @Test + void testFilterNewValues_CompoundValue_DoesNotModifyOriginal() throws Exception { + // Create current field with 3 compound values + DatasetField currentField = createCompoundField(compoundFieldType, + new String[]{"Author1", "Affiliation1"}, + new String[]{"Author2", "Affiliation2"}, + new String[]{"Author3", "Affiliation3"}); + int originalSize = currentField.getDatasetFieldCompoundValues().size(); + + // Create prior field with 2 existing compound values + DatasetField priorField = createCompoundField(compoundFieldType, + new String[]{"Author1", "Affiliation1"}, + new String[]{"Author2", "Affiliation2"}); + + // Filter + DatasetField filtered = invokeFilterNewValues(currentField, priorField); + + // Original field should be unchanged + assertEquals(originalSize, currentField.getDatasetFieldCompoundValues().size()); + assertTrue(containsCompoundValue(currentField, "Author1", "Affiliation1")); + assertTrue(containsCompoundValue(currentField, "Author2", "Affiliation2")); + assertTrue(containsCompoundValue(currentField, "Author3", "Affiliation3")); + + // Filtered field should only have new compound value + assertEquals(1, filtered.getDatasetFieldCompoundValues().size()); + assertTrue(containsCompoundValue(filtered, "Author3", "Affiliation3")); + } + + // Helper methods + + private DatasetField createMultiValueField(DatasetFieldType fieldType, String... values) { + DatasetField field = new DatasetField(); + field.setDatasetFieldType(fieldType); + + List fieldValues = new ArrayList<>(); + for (String value : values) { + DatasetFieldValue dfv = new DatasetFieldValue(); + dfv.setValue(value); + dfv.setDatasetField(field); + fieldValues.add(dfv); + } + field.setDatasetFieldValues(fieldValues); + + return field; + } + + private DatasetField createSingleValueField(DatasetFieldType fieldType, String value) { + DatasetField field = new DatasetField(); + field.setDatasetFieldType(fieldType); + field.setSingleValue(value); + return field; + } + + private boolean containsValue(DatasetField field, String value) { + for (DatasetFieldValue dfv : field.getDatasetFieldValues()) { + if (value.equals(dfv.getDisplayValue())) { + return true; + } + } + return false; + } + + private DatasetField createCompoundField(DatasetFieldType fieldType, String[]... compoundValues) { + DatasetField field = new DatasetField(); + field.setDatasetFieldType(fieldType); + + List compoundValueList = new ArrayList<>(); + for (String[] values : compoundValues) { + DatasetFieldCompoundValue compoundValue = new DatasetFieldCompoundValue(); + compoundValue.setParentDatasetField(field); + + List childFields = new ArrayList<>(); + + // First child field (e.g., author name) + DatasetField childField1 = new DatasetField(); + childField1.setDatasetFieldType(childFieldType1); + childField1.setParentDatasetFieldCompoundValue(compoundValue); + childField1.setSingleValue(values[0]); + childFields.add(childField1); + + // Second child field (e.g., affiliation) + if (values.length > 1) { + DatasetField childField2 = new DatasetField(); + childField2.setDatasetFieldType(childFieldType2); + childField2.setParentDatasetFieldCompoundValue(compoundValue); + childField2.setSingleValue(values[1]); + childFields.add(childField2); + } + + compoundValue.setChildDatasetFields(childFields); + compoundValueList.add(compoundValue); + } + + field.setDatasetFieldCompoundValues(compoundValueList); + return field; + } + + private boolean containsCompoundValue(DatasetField field, String childValue1, String childValue2) { + for (DatasetFieldCompoundValue cv : field.getDatasetFieldCompoundValues()) { + boolean hasValue1 = false; + boolean hasValue2 = false; + + for (DatasetField childField : cv.getChildDatasetFields()) { + String displayValue = childField.getDisplayValue(); + if (childValue1.equals(displayValue)) { + hasValue1 = true; + } + if (childValue2.equals(displayValue)) { + hasValue2 = true; + } + } + + if (hasValue1 && hasValue2) { + return true; + } + } + return false; + } + + /** + * Use reflection to invoke the private filterNewValues method + */ + private DatasetField invokeFilterNewValues(DatasetField currentField, DatasetField priorField) throws Exception { + var method = COARNotifyRelationshipAnnouncementStep.class.getDeclaredMethod( + "filterNewValues", DatasetField.class, DatasetField.class); + method.setAccessible(true); + return (DatasetField) method.invoke(step, currentField, priorField); + } +} \ No newline at end of file From 7641bc4858a39f3875e7d31559039cb3d8b0fa21 Mon Sep 17 00:00:00 2001 From: qqmyers Date: Mon, 17 Nov 2025 10:54:20 -0500 Subject: [PATCH 35/40] refactor/cleanup --- .../dataverse/DatasetFieldCompoundValue.java | 42 +++++++++ .../iq/dataverse/DatasetFieldValue.java | 19 +++- ...OARNotifyRelationshipAnnouncementStep.java | 93 ++----------------- 3 files changed, 66 insertions(+), 88 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/DatasetFieldCompoundValue.java b/src/main/java/edu/harvard/iq/dataverse/DatasetFieldCompoundValue.java index c03baec73af..3c460320686 100644 --- a/src/main/java/edu/harvard/iq/dataverse/DatasetFieldCompoundValue.java +++ b/src/main/java/edu/harvard/iq/dataverse/DatasetFieldCompoundValue.java @@ -27,6 +27,7 @@ import jakarta.persistence.Transient; import org.apache.commons.lang3.StringUtils; +import org.apache.commons.lang3.Strings; import org.apache.commons.lang3.tuple.ImmutablePair; import org.apache.commons.lang3.tuple.Pair; @@ -254,4 +255,45 @@ private Map removeLastComma(Map mapI return mapIn; } + + /** + * Compares this DatasetFieldCompoundValue with another for equality based on their child fields. + * Two compound values are considered equal if they have the same child fields with the same values + * in the same order. + * + * @param other The DatasetFieldCompoundValue to compare with + * @return true if both compound values have equal child fields, false otherwise + */ + public boolean valuesEqual(DatasetFieldCompoundValue other) { + if (this == other) { + return true; + } + if (other == null) { + return false; + } + + List children1 = this.getChildDatasetFields(); + List children2 = other.getChildDatasetFields(); + + if (children1.size() != children2.size()) { + return false; + } + + // Since fields are ordered, we can compare them directly by position + for (int i = 0; i < children1.size(); i++) { + DatasetField child1 = children1.get(i); + DatasetField child2 = children2.get(i); + + // Compare field types + if (!child1.getDatasetFieldType().equals(child2.getDatasetFieldType())) { + return false; + } + + // Compare values using Apache Commons StringUtils + if (!Strings.CS.equals(child1.getValue(), child2.getValue())) { + return false; + } + } + return true; + } } diff --git a/src/main/java/edu/harvard/iq/dataverse/DatasetFieldValue.java b/src/main/java/edu/harvard/iq/dataverse/DatasetFieldValue.java index 1064187ccd6..fb56636bef6 100644 --- a/src/main/java/edu/harvard/iq/dataverse/DatasetFieldValue.java +++ b/src/main/java/edu/harvard/iq/dataverse/DatasetFieldValue.java @@ -22,6 +22,7 @@ import jakarta.persistence.Table; import jakarta.persistence.Transient; import org.apache.commons.lang3.StringUtils; +import org.apache.commons.lang3.Strings; /** * @@ -193,6 +194,22 @@ public DatasetFieldValue copy(DatasetField dsf) { dsfv.setValue(value); return dsfv; - } + } + /** + * Compares this DatasetFieldValue with another for equality based on their values. + * + * @param other The DatasetFieldValue to compare with + * @return true if both values are equal (case-sensitive), false otherwise + */ + public boolean valuesEqual(DatasetFieldValue other) { + if (this == other) { + return true; + } + if (other == null) { + return false; + } + return Strings.CS.equals(this.getValue(), other.getValue()); + } + } diff --git a/src/main/java/edu/harvard/iq/dataverse/workflow/internalspi/COARNotifyRelationshipAnnouncementStep.java b/src/main/java/edu/harvard/iq/dataverse/workflow/internalspi/COARNotifyRelationshipAnnouncementStep.java index 451a00efcd2..335ef855789 100644 --- a/src/main/java/edu/harvard/iq/dataverse/workflow/internalspi/COARNotifyRelationshipAnnouncementStep.java +++ b/src/main/java/edu/harvard/iq/dataverse/workflow/internalspi/COARNotifyRelationshipAnnouncementStep.java @@ -455,128 +455,47 @@ private DatasetField filterNewValues(DatasetField currentField, DatasetField pri for (DatasetFieldCompoundValue currentCompoundValue : currentField.getDatasetFieldCompoundValues()) { boolean isNew = true; - // Check if this compound value exists in prior field if (priorField != null && priorField.getDatasetFieldCompoundValues() != null) { for (DatasetFieldCompoundValue priorCompoundValue : priorField.getDatasetFieldCompoundValues()) { - if (compoundValuesEqual(currentCompoundValue, priorCompoundValue)) { + if (currentCompoundValue.valuesEqual(priorCompoundValue)) { isNew = false; break; } } } - if (isNew) { - // Create a copy of the compound value - DatasetFieldCompoundValue newCompoundValue = copyCompoundValue(currentCompoundValue, filteredField); + // Use the existing copy method from DatasetFieldCompoundValue + DatasetFieldCompoundValue newCompoundValue = currentCompoundValue.copy(filteredField); newCompoundValues.add(newCompoundValue); } } - filteredField.setDatasetFieldCompoundValues(newCompoundValues); - } else if (currentField.getDatasetFieldType().isAllowMultiples()) { // Handle multiple simple values List newValues = new ArrayList<>(); - for (DatasetFieldValue currentValue : currentField.getDatasetFieldValues()) { boolean isNew = true; - if (priorField != null && priorField.getDatasetFieldValues() != null) { for (DatasetFieldValue priorValue : priorField.getDatasetFieldValues()) { - if (valuesEqual(currentValue, priorValue)) { + if (currentValue.valuesEqual(priorValue)) { isNew = false; break; } } } - if (isNew) { - DatasetFieldValue newValue = new DatasetFieldValue(); - newValue.setValue(currentValue.getValue()); - newValue.setDatasetField(filteredField); + DatasetFieldValue newValue = currentValue.copy(filteredField); newValues.add(newValue); } } - filteredField.setDatasetFieldValues(newValues); - } else { // Handle single value - if (priorField == null || !valuesEqual(currentField.getSingleValue(), priorField.getSingleValue())) { + if (priorField == null || !(currentField.getSingleValue().valuesEqual(priorField.getSingleValue()))) { filteredField.setSingleValue(currentField.getValue()); } } - return filteredField; } - - /** - * Check if two compound values are equal by comparing all their child fields. - * Since child fields are ordered, we can do a simpler comparison. - */ - private boolean compoundValuesEqual(DatasetFieldCompoundValue cv1, DatasetFieldCompoundValue cv2) { - if (cv1 == null && cv2 == null) { - return true; - } - if (cv1 == null || cv2 == null) { - return false; - } - - List children1 = cv1.getChildDatasetFields(); - List children2 = cv2.getChildDatasetFields(); - - if (children1.size() != children2.size()) { - return false; - } - - // Since fields are ordered, we can compare them directly by position - for (int i = 0; i < children1.size(); i++) { - DatasetField child1 = children1.get(i); - DatasetField child2 = children2.get(i); - - // Compare field types - if (!child1.getDatasetFieldType().equals(child2.getDatasetFieldType())) { - return false; - } - - // Compare values using Apache Commons StringUtils - if (!Strings.CS.equals(child1.getValue(), child2.getValue())) { - return false; - } - } - - return true; - } - - /** - * Create a deep copy of a compound value - */ - private DatasetFieldCompoundValue copyCompoundValue(DatasetFieldCompoundValue source, DatasetField parentField) { - DatasetFieldCompoundValue copy = new DatasetFieldCompoundValue(); - copy.setParentDatasetField(parentField); - copy.setDisplayOrder(source.getDisplayOrder()); - - List childFieldsCopy = new ArrayList<>(); - for (DatasetField sourceChild : source.getChildDatasetFields()) { - DatasetField childCopy = new DatasetField(); - childCopy.setDatasetFieldType(sourceChild.getDatasetFieldType()); - childCopy.setParentDatasetFieldCompoundValue(copy); - childCopy.setSingleValue(sourceChild.getValue()); - childFieldsCopy.add(childCopy); - } - - copy.setChildDatasetFields(childFieldsCopy); - return copy; - } - - private boolean valuesEqual(DatasetFieldValue v1, DatasetFieldValue v2) { - if (v1 == null && v2 == null) { - return true; - } - if (v1 == null || v2 == null) { - return false; - } - return Strings.CS.equals(v1.getValue(), v2.getValue()); - } } From 156377cadada23575486402623c92df3d01624e2 Mon Sep 17 00:00:00 2001 From: qqmyers Date: Mon, 17 Nov 2025 11:18:00 -0500 Subject: [PATCH 36/40] release note --- doc/release-notes/11983-COAR-Notify2.md | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 doc/release-notes/11983-COAR-Notify2.md diff --git a/doc/release-notes/11983-COAR-Notify2.md b/doc/release-notes/11983-COAR-Notify2.md new file mode 100644 index 00000000000..1c3451a81aa --- /dev/null +++ b/doc/release-notes/11983-COAR-Notify2.md @@ -0,0 +1,3 @@ +### Improved COAR Notify Relationship Announcement Support + +Dataverse no longer sends duplicate [COAR Notify Relationship Announcement Workflow](https://coar-notify.net/catalogue/workflows/repository-relationship-repository/) messages when new dataset versions are published (and the relationship metadata has not been changed). From 776325765679dbddde7482ae4c191de6ce8153bd Mon Sep 17 00:00:00 2001 From: qqmyers Date: Tue, 18 Nov 2025 16:58:36 -0500 Subject: [PATCH 37/40] fix #11990 --- src/main/java/edu/harvard/iq/dataverse/DatasetField.java | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/DatasetField.java b/src/main/java/edu/harvard/iq/dataverse/DatasetField.java index a735ae7470c..0f47caf256b 100644 --- a/src/main/java/edu/harvard/iq/dataverse/DatasetField.java +++ b/src/main/java/edu/harvard/iq/dataverse/DatasetField.java @@ -197,6 +197,7 @@ public void setDatasetFieldValues(List datasetFieldValues) { @ManyToMany(cascade = {CascadeType.MERGE}) @JoinTable(indexes = {@Index(columnList="datasetfield_id"),@Index(columnList="controlledvocabularyvalues_id")}) + @OrderBy("displayOrder ASC") private List controlledVocabularyValues = new ArrayList<>(); public List getControlledVocabularyValues() { @@ -604,14 +605,15 @@ private DatasetField copy(Object versionOrTemplate, DatasetFieldCompoundValue pa if (versionOrTemplate != null) { if (versionOrTemplate instanceof DatasetVersion) { - dsf.setDatasetVersion((DatasetVersion) versionOrTemplate); + dsf.setDatasetVersion((DatasetVersion) versionOrTemplate); } else { dsf.setTemplate((Template) versionOrTemplate); } } dsf.setParentDatasetFieldCompoundValue(parent); - dsf.setControlledVocabularyValues(controlledVocabularyValues); + + dsf.getControlledVocabularyValues().addAll(controlledVocabularyValues); for (DatasetFieldValue dsfv : datasetFieldValues) { dsf.getDatasetFieldValues().add(dsfv.copy(dsf)); From 10d2f24d5939a5a384592a52bee4d796610bce1d Mon Sep 17 00:00:00 2001 From: qqmyers Date: Tue, 18 Nov 2025 17:06:35 -0500 Subject: [PATCH 38/40] diff and test additions/fixes --- .../edu/harvard/iq/dataverse/Dataset.java | 8 +- .../dataverse/DatasetFieldCompoundValue.java | 64 +- ...OARNotifyRelationshipAnnouncementStep.java | 107 ++- ...otifyRelationshipAnnouncementStepTest.java | 819 +++++++++++++++++- 4 files changed, 917 insertions(+), 81 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/Dataset.java b/src/main/java/edu/harvard/iq/dataverse/Dataset.java index 2acb92f690e..eac5e4959ea 100644 --- a/src/main/java/edu/harvard/iq/dataverse/Dataset.java +++ b/src/main/java/edu/harvard/iq/dataverse/Dataset.java @@ -533,7 +533,13 @@ public DatasetVersion getReleasedVersion() { } return null; } - + + /** + * Returns the second-most-recent released version of this dataset. + * Assumes versions are ordered from most recent to oldest. + * + * @return The prior released version, or null if there is only one or no released versions + */ public DatasetVersion getPriorReleasedVersion() { boolean foundReleasedVersion = false; for (DatasetVersion version : this.getVersions()) { diff --git a/src/main/java/edu/harvard/iq/dataverse/DatasetFieldCompoundValue.java b/src/main/java/edu/harvard/iq/dataverse/DatasetFieldCompoundValue.java index 3c460320686..a48e110fb45 100644 --- a/src/main/java/edu/harvard/iq/dataverse/DatasetFieldCompoundValue.java +++ b/src/main/java/edu/harvard/iq/dataverse/DatasetFieldCompoundValue.java @@ -256,10 +256,11 @@ private Map removeLastComma(Map mapI return mapIn; } + /** - * Compares this DatasetFieldCompoundValue with another for equality based on their child fields. - * Two compound values are considered equal if they have the same child fields with the same values - * in the same order. + * Compares this DatasetFieldCompoundValue with another for equality based on + * their child fields. Two compound values are considered equal if they have the + * same child fields with the same values in the same order. * * @param other The DatasetFieldCompoundValue to compare with * @return true if both compound values have equal child fields, false otherwise @@ -278,20 +279,57 @@ public boolean valuesEqual(DatasetFieldCompoundValue other) { if (children1.size() != children2.size()) { return false; } + + // Compare each child field + for (DatasetField child1 : children1) { - // Since fields are ordered, we can compare them directly by position - for (int i = 0; i < children1.size(); i++) { - DatasetField child1 = children1.get(i); - DatasetField child2 = children2.get(i); + DatasetField child2 = children2.stream() + .filter(c -> c.getDatasetFieldType().equals(child1.getDatasetFieldType())).findFirst().orElse(null); - // Compare field types - if (!child1.getDatasetFieldType().equals(child2.getDatasetFieldType())) { + if (child2 == null) { return false; } - - // Compare values using Apache Commons StringUtils - if (!Strings.CS.equals(child1.getValue(), child2.getValue())) { - return false; + + // Compare values based on field type + if (child1.getDatasetFieldType().isControlledVocabulary()) { + + List cvs1 = child1.getControlledVocabularyValues(); + List cvs2 = child2.getControlledVocabularyValues(); + + if (cvs1.size() != cvs2.size()) { + + return false; + } + + for (ControlledVocabularyValue cv1Val : cvs1) { + boolean found = cvs2.stream().anyMatch(cv2Val -> cv1Val.getStrValue().equals(cv2Val.getStrValue())); + if (!found) { + return false; + } + } + } else { + // Handle regular field values (including multiple values) + List dfvs1 = child1.getDatasetFieldValues(); + List dfvs2 = child2.getDatasetFieldValues(); + + if (dfvs1.size() != dfvs2.size()) { + return false; + } + + for (DatasetFieldValue dfv1 : dfvs1) { + String value1 = dfv1.getValue(); + boolean found = dfvs2.stream() + .anyMatch(dfv2 -> { + String value2 = dfv2.getValue(); + if (value1 == null && value2 == null) { + return true; + } + return value1 != null && value1.equals(value2); + }); + if (!found) { + return false; + } + } } } return true; diff --git a/src/main/java/edu/harvard/iq/dataverse/workflow/internalspi/COARNotifyRelationshipAnnouncementStep.java b/src/main/java/edu/harvard/iq/dataverse/workflow/internalspi/COARNotifyRelationshipAnnouncementStep.java index 335ef855789..c0d5c82a23b 100644 --- a/src/main/java/edu/harvard/iq/dataverse/workflow/internalspi/COARNotifyRelationshipAnnouncementStep.java +++ b/src/main/java/edu/harvard/iq/dataverse/workflow/internalspi/COARNotifyRelationshipAnnouncementStep.java @@ -1,5 +1,6 @@ package edu.harvard.iq.dataverse.workflow.internalspi; +import edu.harvard.iq.dataverse.ControlledVocabularyValue; import edu.harvard.iq.dataverse.Dataset; import edu.harvard.iq.dataverse.DatasetField; import edu.harvard.iq.dataverse.DatasetFieldCompoundValue; @@ -43,7 +44,6 @@ import jakarta.json.JsonString; import jakarta.json.JsonValue; -import org.apache.commons.lang3.Strings; import org.apache.http.client.methods.CloseableHttpResponse; import org.apache.http.client.methods.HttpPost; import org.apache.http.entity.StringEntity; @@ -438,64 +438,89 @@ private boolean isURI(String number) { } /** - * Create a new DatasetField containing only values that are new compared to the prior field. - * This creates a detached copy to avoid modifying the managed entity. + * Create a new DatasetField containing only values that are new compared to the + * prior field. This creates a detached copy to avoid modifying the managed + * entity. * * @param currentField The field from the current version - * @param priorField The field from the prior version + * @param priorField The field from the prior version * @return A new DatasetField with only new values */ private DatasetField filterNewValues(DatasetField currentField, DatasetField priorField) { - DatasetField filteredField = new DatasetField(); - filteredField.setDatasetFieldType(currentField.getDatasetFieldType()); - - if (currentField.getDatasetFieldType().isCompound()) { - // Handle compound values - List newCompoundValues = new ArrayList<>(); - - for (DatasetFieldCompoundValue currentCompoundValue : currentField.getDatasetFieldCompoundValues()) { - boolean isNew = true; - // Check if this compound value exists in prior field - if (priorField != null && priorField.getDatasetFieldCompoundValues() != null) { - for (DatasetFieldCompoundValue priorCompoundValue : priorField.getDatasetFieldCompoundValues()) { - if (currentCompoundValue.valuesEqual(priorCompoundValue)) { + DatasetField filtered = new DatasetField(); + DatasetFieldType fieldType = currentField.getDatasetFieldType(); + filtered.setDatasetFieldType(fieldType); + + // Handle primitive fields + if (fieldType.isPrimitive()) { + if (fieldType.isControlledVocabulary()) { + // Handle controlled vocabulary fields + List currentCVs = currentField.getControlledVocabularyValues(); + List priorCVs = priorField != null ? priorField.getControlledVocabularyValues() : new ArrayList<>(); + + List newCVs = new ArrayList<>(); + for (ControlledVocabularyValue currentCV : currentCVs) { + boolean isNew = true; + for (ControlledVocabularyValue priorCV : priorCVs) { + if (currentCV.getStrValue().equals(priorCV.getStrValue())) { isNew = false; break; } } + if (isNew) { + newCVs.add(currentCV); + } } - if (isNew) { - // Use the existing copy method from DatasetFieldCompoundValue - DatasetFieldCompoundValue newCompoundValue = currentCompoundValue.copy(filteredField); - newCompoundValues.add(newCompoundValue); - } - } - filteredField.setDatasetFieldCompoundValues(newCompoundValues); - } else if (currentField.getDatasetFieldType().isAllowMultiples()) { - // Handle multiple simple values - List newValues = new ArrayList<>(); - for (DatasetFieldValue currentValue : currentField.getDatasetFieldValues()) { - boolean isNew = true; - if (priorField != null && priorField.getDatasetFieldValues() != null) { - for (DatasetFieldValue priorValue : priorField.getDatasetFieldValues()) { - if (currentValue.valuesEqual(priorValue)) { + filtered.setControlledVocabularyValues(newCVs); + } else { + // Handle regular fields + List currentDFVs = currentField.getDatasetFieldValues(); + List priorDFVs = priorField != null ? priorField.getDatasetFieldValues() : new ArrayList<>(); + + List newDFVs = new ArrayList<>(); + for (DatasetFieldValue currentDFV : currentDFVs) { + boolean isNew = true; + for (DatasetFieldValue priorDFV : priorDFVs) { + if (currentDFV.valuesEqual(priorDFV)) { isNew = false; break; } } + if (isNew) { + newDFVs.add(currentDFV); + } } - if (isNew) { - DatasetFieldValue newValue = currentValue.copy(filteredField); - newValues.add(newValue); - } + filtered.setDatasetFieldValues(newDFVs); } - filteredField.setDatasetFieldValues(newValues); } else { - // Handle single value - if (priorField == null || !(currentField.getSingleValue().valuesEqual(priorField.getSingleValue()))) { - filteredField.setSingleValue(currentField.getValue()); + // Handle compound fields + List currentCompounds = currentField.getDatasetFieldCompoundValues(); + List priorCompounds = priorField != null ? priorField.getDatasetFieldCompoundValues() : new ArrayList<>(); + + List newCompounds = new ArrayList<>(); + + for (DatasetFieldCompoundValue currentCompound : currentCompounds) { + boolean isNew = true; + + for (DatasetFieldCompoundValue priorCompound : priorCompounds) { + + if (currentCompound.valuesEqual(priorCompound)) { + isNew = false; + break; + } + } + + if (isNew) { + // Create a copy of the compound value with all its children + DatasetFieldCompoundValue newCompound = currentCompound.copy(filtered); + newCompound.setParentDatasetField(filtered); + newCompounds.add(newCompound); + } } + + filtered.setDatasetFieldCompoundValues(newCompounds); } - return filteredField; + + return filtered; } } diff --git a/src/test/java/edu/harvard/iq/dataverse/workflow/internalspi/COARNotifyRelationshipAnnouncementStepTest.java b/src/test/java/edu/harvard/iq/dataverse/workflow/internalspi/COARNotifyRelationshipAnnouncementStepTest.java index 49d75491211..063e3d58dda 100644 --- a/src/test/java/edu/harvard/iq/dataverse/workflow/internalspi/COARNotifyRelationshipAnnouncementStepTest.java +++ b/src/test/java/edu/harvard/iq/dataverse/workflow/internalspi/COARNotifyRelationshipAnnouncementStepTest.java @@ -1,12 +1,12 @@ package edu.harvard.iq.dataverse.workflow.internalspi; +import edu.harvard.iq.dataverse.ControlledVocabularyValue; import edu.harvard.iq.dataverse.DatasetField; import edu.harvard.iq.dataverse.DatasetFieldCompoundValue; import edu.harvard.iq.dataverse.DatasetFieldType; import edu.harvard.iq.dataverse.DatasetFieldValue; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; - import java.util.ArrayList; import java.util.HashMap; import java.util.List; @@ -31,12 +31,14 @@ void setUp() { // Setup field types multiValueFieldType = new DatasetFieldType(); + multiValueFieldType.setId(1L); multiValueFieldType.setName("testMultiField"); multiValueFieldType.setAllowMultiples(true); multiValueFieldType.setFieldType(DatasetFieldType.FieldType.TEXT); multiValueFieldType.setChildDatasetFieldTypes(new ArrayList<>()); singleValueFieldType = new DatasetFieldType(); + singleValueFieldType.setId(2L); singleValueFieldType.setName("testSingleField"); singleValueFieldType.setAllowMultiples(false); singleValueFieldType.setFieldType(DatasetFieldType.FieldType.TEXT); @@ -44,17 +46,20 @@ void setUp() { // Setup compound field type with child fields compoundFieldType = new DatasetFieldType(); + compoundFieldType.setId(3L); compoundFieldType.setName("testCompoundField"); compoundFieldType.setAllowMultiples(true); compoundFieldType.setFieldType(DatasetFieldType.FieldType.NONE); childFieldType1 = new DatasetFieldType(); + childFieldType1.setId(4L); childFieldType1.setName("authorName"); childFieldType1.setFieldType(DatasetFieldType.FieldType.TEXT); childFieldType1.setParentDatasetFieldType(compoundFieldType); childFieldType1.setChildDatasetFieldTypes(new ArrayList<>()); childFieldType2 = new DatasetFieldType(); + childFieldType2.setId(5L); childFieldType2.setName("authorAffiliation"); childFieldType2.setFieldType(DatasetFieldType.FieldType.TEXT); childFieldType2.setParentDatasetFieldType(compoundFieldType); @@ -276,6 +281,759 @@ void testFilterNewValues_CompoundValue_DoesNotModifyOriginal() throws Exception assertEquals(1, filtered.getDatasetFieldCompoundValues().size()); assertTrue(containsCompoundValue(filtered, "Author3", "Affiliation3")); } + + @Test + void testFilterNewValues_ControlledVocab_AllNew() throws Exception { + // Setup controlled vocabulary field type + DatasetFieldType cvFieldType = new DatasetFieldType(); + cvFieldType.setName("testCVField"); + cvFieldType.setAllowMultiples(true); + cvFieldType.setFieldType(DatasetFieldType.FieldType.TEXT); + cvFieldType.setAllowControlledVocabulary(true); + cvFieldType.setChildDatasetFieldTypes(new ArrayList<>()); + + // Create controlled vocabulary values + ControlledVocabularyValue cvv1 = new ControlledVocabularyValue(); + cvv1.setStrValue("Medicine, Health and Life Sciences"); + cvv1.setDatasetFieldType(cvFieldType); + + ControlledVocabularyValue cvv2 = new ControlledVocabularyValue(); + cvv2.setStrValue("Social Sciences"); + cvv2.setDatasetFieldType(cvFieldType); + + ControlledVocabularyValue cvv3 = new ControlledVocabularyValue(); + cvv3.setStrValue("Engineering"); + cvv3.setDatasetFieldType(cvFieldType); + + // Create current field with 3 CV values + DatasetField currentField = new DatasetField(); + currentField.setDatasetFieldType(cvFieldType); + currentField.setControlledVocabularyValues(List.of(cvv1, cvv2, cvv3)); + + // Create prior field with no values + DatasetField priorField = new DatasetField(); + priorField.setDatasetFieldType(cvFieldType); + priorField.setControlledVocabularyValues(new ArrayList<>()); + + // Filter + DatasetField filtered = invokeFilterNewValues(currentField, priorField); + + // All CV values should be included + assertEquals(3, filtered.getControlledVocabularyValues().size()); + assertTrue(containsControlledVocabValue(filtered, "Medicine, Health and Life Sciences")); + assertTrue(containsControlledVocabValue(filtered, "Social Sciences")); + assertTrue(containsControlledVocabValue(filtered, "Engineering")); + } + + @Test + void testFilterNewValues_ControlledVocab_SomeNew() throws Exception { + // Setup controlled vocabulary field type + DatasetFieldType cvFieldType = new DatasetFieldType(); + cvFieldType.setName("testCVField"); + cvFieldType.setAllowMultiples(true); + cvFieldType.setFieldType(DatasetFieldType.FieldType.TEXT); + cvFieldType.setAllowControlledVocabulary(true); + cvFieldType.setChildDatasetFieldTypes(new ArrayList<>()); + + // Create controlled vocabulary values + ControlledVocabularyValue cvv1 = new ControlledVocabularyValue(); + cvv1.setStrValue("Medicine, Health and Life Sciences"); + cvv1.setDatasetFieldType(cvFieldType); + + ControlledVocabularyValue cvv2 = new ControlledVocabularyValue(); + cvv2.setStrValue("Social Sciences"); + cvv2.setDatasetFieldType(cvFieldType); + + ControlledVocabularyValue cvv3 = new ControlledVocabularyValue(); + cvv3.setStrValue("Engineering"); + cvv3.setDatasetFieldType(cvFieldType); + + // Create current field with 3 CV values + DatasetField currentField = new DatasetField(); + currentField.setDatasetFieldType(cvFieldType); + currentField.setControlledVocabularyValues(List.of(cvv1, cvv2, cvv3)); + + // Create prior field with 2 existing CV values + DatasetField priorField = new DatasetField(); + priorField.setDatasetFieldType(cvFieldType); + priorField.setControlledVocabularyValues(List.of(cvv1, cvv2)); + + // Filter + DatasetField filtered = invokeFilterNewValues(currentField, priorField); + + // Only new CV value should be included + assertEquals(1, filtered.getControlledVocabularyValues().size()); + assertFalse(containsControlledVocabValue(filtered, "Medicine, Health and Life Sciences")); + assertFalse(containsControlledVocabValue(filtered, "Social Sciences")); + assertTrue(containsControlledVocabValue(filtered, "Engineering")); + } + + @Test + void testFilterNewValues_ControlledVocab_NoneNew() throws Exception { + // Setup controlled vocabulary field type + DatasetFieldType cvFieldType = new DatasetFieldType(); + cvFieldType.setName("testCVField"); + cvFieldType.setAllowMultiples(true); + cvFieldType.setFieldType(DatasetFieldType.FieldType.TEXT); + cvFieldType.setAllowControlledVocabulary(true); + cvFieldType.setChildDatasetFieldTypes(new ArrayList<>()); + + // Create controlled vocabulary values + ControlledVocabularyValue cvv1 = new ControlledVocabularyValue(); + cvv1.setStrValue("Medicine, Health and Life Sciences"); + cvv1.setDatasetFieldType(cvFieldType); + + ControlledVocabularyValue cvv2 = new ControlledVocabularyValue(); + cvv2.setStrValue("Social Sciences"); + cvv2.setDatasetFieldType(cvFieldType); + + // Create current field with 2 CV values + DatasetField currentField = new DatasetField(); + currentField.setDatasetFieldType(cvFieldType); + currentField.setControlledVocabularyValues(List.of(cvv1, cvv2)); + + // Create prior field with same CV values + DatasetField priorField = new DatasetField(); + priorField.setDatasetFieldType(cvFieldType); + priorField.setControlledVocabularyValues(List.of(cvv1, cvv2)); + + // Filter + DatasetField filtered = invokeFilterNewValues(currentField, priorField); + + // No CV values should be included + assertEquals(0, filtered.getControlledVocabularyValues().size()); + assertTrue(filtered.isEmpty()); + } + + @Test + void testFilterNewValues_ControlledVocab_SingleValue_Changed() throws Exception { + // Setup controlled vocabulary field type (non-multiple) + DatasetFieldType cvFieldType = new DatasetFieldType(); + cvFieldType.setName("testCVField"); + cvFieldType.setAllowMultiples(false); + cvFieldType.setFieldType(DatasetFieldType.FieldType.TEXT); + cvFieldType.setAllowControlledVocabulary(true); + cvFieldType.setChildDatasetFieldTypes(new ArrayList<>()); + + // Create controlled vocabulary values + ControlledVocabularyValue cvvOld = new ControlledVocabularyValue(); + cvvOld.setStrValue("Medicine, Health and Life Sciences"); + cvvOld.setDatasetFieldType(cvFieldType); + + ControlledVocabularyValue cvvNew = new ControlledVocabularyValue(); + cvvNew.setStrValue("Social Sciences"); + cvvNew.setDatasetFieldType(cvFieldType); + + // Create current field with new CV value + DatasetField currentField = new DatasetField(); + currentField.setDatasetFieldType(cvFieldType); + currentField.setControlledVocabularyValues(List.of(cvvNew)); + + // Create prior field with old CV value + DatasetField priorField = new DatasetField(); + priorField.setDatasetFieldType(cvFieldType); + priorField.setControlledVocabularyValues(List.of(cvvOld)); + + // Filter + DatasetField filtered = invokeFilterNewValues(currentField, priorField); + + // New CV value should be included + assertFalse(filtered.isEmpty()); + assertEquals(1, filtered.getControlledVocabularyValues().size()); + assertTrue(containsControlledVocabValue(filtered, "Social Sciences")); + } + + @Test + void testFilterNewValues_CompoundWithControlledVocabChild_AllNew() throws Exception { + // Setup compound field type with CV child field + DatasetFieldType compoundType = new DatasetFieldType(); + compoundType.setName("testCompoundWithCV"); + compoundType.setAllowMultiples(true); + compoundType.setFieldType(DatasetFieldType.FieldType.NONE); + + DatasetFieldType childTextType = new DatasetFieldType(); + childTextType.setName("childText"); + childTextType.setFieldType(DatasetFieldType.FieldType.TEXT); + childTextType.setParentDatasetFieldType(compoundType); + childTextType.setChildDatasetFieldTypes(new ArrayList<>()); + + DatasetFieldType childCVType = new DatasetFieldType(); + childCVType.setName("childCV"); + childCVType.setFieldType(DatasetFieldType.FieldType.TEXT); + childCVType.setAllowControlledVocabulary(true); + childCVType.setParentDatasetFieldType(compoundType); + childCVType.setChildDatasetFieldTypes(new ArrayList<>()); + + compoundType.setChildDatasetFieldTypes(List.of(childTextType, childCVType)); + + // Create controlled vocabulary values + ControlledVocabularyValue cvv1 = new ControlledVocabularyValue(); + cvv1.setStrValue("ark"); + cvv1.setDatasetFieldType(childCVType); + + ControlledVocabularyValue cvv2 = new ControlledVocabularyValue(); + cvv2.setStrValue("doi"); + cvv2.setDatasetFieldType(childCVType); + + // Create current field with 2 compound values containing CV child fields + DatasetField currentField = new DatasetField(); + currentField.setDatasetFieldType(compoundType); + + List compoundValues = new ArrayList<>(); + + // First compound value + DatasetFieldCompoundValue cv1 = new DatasetFieldCompoundValue(); + cv1.setParentDatasetField(currentField); + + DatasetField child1Text = new DatasetField(); + child1Text.setDatasetFieldType(childTextType); + child1Text.setParentDatasetFieldCompoundValue(cv1); + child1Text.setSingleValue("Value1"); + + DatasetField child1CV = new DatasetField(); + child1CV.setDatasetFieldType(childCVType); + child1CV.setParentDatasetFieldCompoundValue(cv1); + child1CV.setControlledVocabularyValues(List.of(cvv1)); + + cv1.setChildDatasetFields(List.of(child1Text, child1CV)); + compoundValues.add(cv1); + + // Second compound value + DatasetFieldCompoundValue cv2 = new DatasetFieldCompoundValue(); + cv2.setParentDatasetField(currentField); + + DatasetField child2Text = new DatasetField(); + child2Text.setDatasetFieldType(childTextType); + child2Text.setParentDatasetFieldCompoundValue(cv2); + child2Text.setSingleValue("Value2"); + + DatasetField child2CV = new DatasetField(); + child2CV.setDatasetFieldType(childCVType); + child2CV.setParentDatasetFieldCompoundValue(cv2); + child2CV.setControlledVocabularyValues(List.of(cvv2)); + + cv2.setChildDatasetFields(List.of(child2Text, child2CV)); + compoundValues.add(cv2); + + currentField.setDatasetFieldCompoundValues(compoundValues); + + // Create prior field with no values + DatasetField priorField = new DatasetField(); + priorField.setDatasetFieldType(compoundType); + priorField.setDatasetFieldCompoundValues(new ArrayList<>()); + + // Filter + DatasetField filtered = invokeFilterNewValues(currentField, priorField); + + // All compound values should be included + assertEquals(2, filtered.getDatasetFieldCompoundValues().size()); + } + + +@Test +void testFilterNewValues_CompoundWithControlledVocabChild_CVChanged() throws Exception { + // Setup compound field type with CV child field + DatasetFieldType compoundType = new DatasetFieldType(); + compoundType.setId(1L); + compoundType.setName("testCompoundWithCV"); + compoundType.setAllowMultiples(true); + compoundType.setFieldType(DatasetFieldType.FieldType.NONE); + + DatasetFieldType childTextType = new DatasetFieldType(); + childTextType.setId(2L); + childTextType.setName("childText"); + childTextType.setFieldType(DatasetFieldType.FieldType.TEXT); + childTextType.setParentDatasetFieldType(compoundType); + childTextType.setChildDatasetFieldTypes(new ArrayList<>()); + + DatasetFieldType childCVType = new DatasetFieldType(); + childCVType.setId(3L); + childCVType.setName("childCV"); + childCVType.setFieldType(DatasetFieldType.FieldType.TEXT); + childCVType.setAllowControlledVocabulary(true); + childCVType.setParentDatasetFieldType(compoundType); + childCVType.setChildDatasetFieldTypes(new ArrayList<>()); + + compoundType.setChildDatasetFieldTypes(List.of(childTextType, childCVType)); + + // Create controlled vocabulary values + ControlledVocabularyValue cvvOld = new ControlledVocabularyValue(); + cvvOld.setStrValue("ark"); + cvvOld.setDatasetFieldType(childCVType); + + ControlledVocabularyValue cvvNew = new ControlledVocabularyValue(); + cvvNew.setStrValue("doi"); + cvvNew.setDatasetFieldType(childCVType); + + // Create current field with compound value containing new CV + DatasetField currentField = new DatasetField(); + currentField.setDatasetFieldType(compoundType); + + DatasetFieldCompoundValue currentCV = new DatasetFieldCompoundValue(); + currentCV.setParentDatasetField(currentField); + + DatasetField currentChildText = new DatasetField(); + currentChildText.setDatasetFieldType(childTextType); + currentChildText.setParentDatasetFieldCompoundValue(currentCV); + currentChildText.setSingleValue("SameValue"); + + DatasetField currentChildCV = new DatasetField(); + currentChildCV.setDatasetFieldType(childCVType); + currentChildCV.setParentDatasetFieldCompoundValue(currentCV); + currentChildCV.setControlledVocabularyValues(List.of(cvvNew)); + + currentCV.setChildDatasetFields(List.of(currentChildText, currentChildCV)); + currentField.setDatasetFieldCompoundValues(List.of(currentCV)); + + // Create prior field with compound value containing old CV + DatasetField priorField = new DatasetField(); + priorField.setDatasetFieldType(compoundType); + + DatasetFieldCompoundValue priorCV = new DatasetFieldCompoundValue(); + priorCV.setParentDatasetField(priorField); + + DatasetField priorChildText = new DatasetField(); + priorChildText.setDatasetFieldType(childTextType); + priorChildText.setParentDatasetFieldCompoundValue(priorCV); + priorChildText.setSingleValue("SameValue"); + + DatasetField priorChildCV = new DatasetField(); + priorChildCV.setDatasetFieldType(childCVType); + priorChildCV.setParentDatasetFieldCompoundValue(priorCV); + priorChildCV.setControlledVocabularyValues(List.of(cvvOld)); + + priorCV.setChildDatasetFields(List.of(priorChildText, priorChildCV)); + priorField.setDatasetFieldCompoundValues(List.of(priorCV)); + + // Filter + DatasetField filtered = invokeFilterNewValues(currentField, priorField); + + // Should be treated as new compound value since CV child changed + assertEquals(1, filtered.getDatasetFieldCompoundValues().size()); + + // Verify the CV value in the filtered compound + DatasetFieldCompoundValue filteredCV = filtered.getDatasetFieldCompoundValues().get(0); + DatasetField filteredChildCV = filteredCV.getChildDatasetFields().stream() + .filter(f -> f.getDatasetFieldType().equals(childCVType)) + .findFirst() + .orElse(null); + + assertNotNull(filteredChildCV); + assertEquals(1, filteredChildCV.getControlledVocabularyValues().size()); + assertEquals("doi", filteredChildCV.getControlledVocabularyValues().get(0).getStrValue()); +} + + +@Test +void testFilterNewValues_CompoundWithControlledVocabChild_CVUnchanged() throws Exception { + // Setup compound field type with CV child field + DatasetFieldType compoundType = new DatasetFieldType(); + compoundType.setId(1L); + compoundType.setName("testCompoundWithCV"); + compoundType.setAllowMultiples(true); + compoundType.setFieldType(DatasetFieldType.FieldType.NONE); + + DatasetFieldType childTextType = new DatasetFieldType(); + childTextType.setId(2L); + childTextType.setName("childText"); + childTextType.setFieldType(DatasetFieldType.FieldType.TEXT); + childTextType.setParentDatasetFieldType(compoundType); + childTextType.setChildDatasetFieldTypes(new ArrayList<>()); + + DatasetFieldType childCVType = new DatasetFieldType(); + childCVType.setId(3L); + childCVType.setName("childCV"); + childCVType.setFieldType(DatasetFieldType.FieldType.TEXT); + childCVType.setAllowControlledVocabulary(true); + childCVType.setParentDatasetFieldType(compoundType); + childCVType.setChildDatasetFieldTypes(new ArrayList<>()); + + compoundType.setChildDatasetFieldTypes(List.of(childTextType, childCVType)); + + // Create controlled vocabulary value + ControlledVocabularyValue cvv = new ControlledVocabularyValue(); + cvv.setStrValue("ark"); + cvv.setDatasetFieldType(childCVType); + + // Create current field with compound value + DatasetField currentField = new DatasetField(); + currentField.setDatasetFieldType(compoundType); + + DatasetFieldCompoundValue currentCV = new DatasetFieldCompoundValue(); + currentCV.setParentDatasetField(currentField); + + DatasetField currentChildText = new DatasetField(); + currentChildText.setDatasetFieldType(childTextType); + currentChildText.setParentDatasetFieldCompoundValue(currentCV); + currentChildText.setSingleValue("SameValue"); + + DatasetField currentChildCV = new DatasetField(); + currentChildCV.setDatasetFieldType(childCVType); + currentChildCV.setParentDatasetFieldCompoundValue(currentCV); + currentChildCV.setControlledVocabularyValues(List.of(cvv)); + + currentCV.setChildDatasetFields(List.of(currentChildText, currentChildCV)); + currentField.setDatasetFieldCompoundValues(List.of(currentCV)); + + // Create prior field with same compound value + DatasetField priorField = new DatasetField(); + priorField.setDatasetFieldType(compoundType); + + DatasetFieldCompoundValue priorCV = new DatasetFieldCompoundValue(); + priorCV.setParentDatasetField(priorField); + + DatasetField priorChildText = new DatasetField(); + priorChildText.setDatasetFieldType(childTextType); + priorChildText.setParentDatasetFieldCompoundValue(priorCV); + priorChildText.setSingleValue("SameValue"); + + DatasetField priorChildCV = new DatasetField(); + priorChildCV.setDatasetFieldType(childCVType); + priorChildCV.setParentDatasetFieldCompoundValue(priorCV); + priorChildCV.setControlledVocabularyValues(List.of(cvv)); + + priorCV.setChildDatasetFields(List.of(priorChildText, priorChildCV)); + priorField.setDatasetFieldCompoundValues(List.of(priorCV)); + + // Filter + DatasetField filtered = invokeFilterNewValues(currentField, priorField); + + // No compound values should be included since nothing changed + assertEquals(0, filtered.getDatasetFieldCompoundValues().size()); + assertTrue(filtered.isEmpty()); +} + + @Test + void testFilterNewValues_CompoundWithPrimitiveChild_AllNew() throws Exception { + // Setup compound field type with primitive child fields + DatasetFieldType compoundType = new DatasetFieldType(); + compoundType.setId(1L); + compoundType.setName("testCompoundWithPrimitive"); + compoundType.setAllowMultiples(true); + compoundType.setFieldType(DatasetFieldType.FieldType.NONE); + + DatasetFieldType childTextField = new DatasetFieldType(); + childTextField.setId(2L); + childTextField.setName("childText"); + childTextField.setFieldType(DatasetFieldType.FieldType.TEXT); + childTextField.setParentDatasetFieldType(compoundType); + childTextField.setChildDatasetFieldTypes(new ArrayList<>()); + + DatasetFieldType childIntField = new DatasetFieldType(); + childIntField.setId(3L); + childIntField.setName("childInt"); + childIntField.setFieldType(DatasetFieldType.FieldType.INT); + childIntField.setParentDatasetFieldType(compoundType); + childIntField.setChildDatasetFieldTypes(new ArrayList<>()); + + compoundType.setChildDatasetFieldTypes(List.of(childTextField, childIntField)); + + // Create current field with 2 compound values + DatasetField currentField = new DatasetField(); + currentField.setDatasetFieldType(compoundType); + + List compoundValues = new ArrayList<>(); + + // First compound value + DatasetFieldCompoundValue cv1 = new DatasetFieldCompoundValue(); + cv1.setParentDatasetField(currentField); + + DatasetField child1Text = new DatasetField(); + child1Text.setDatasetFieldType(childTextField); + child1Text.setParentDatasetFieldCompoundValue(cv1); + child1Text.setSingleValue("Text1"); + + DatasetField child1Int = new DatasetField(); + child1Int.setDatasetFieldType(childIntField); + child1Int.setParentDatasetFieldCompoundValue(cv1); + child1Int.setSingleValue("123"); + + cv1.setChildDatasetFields(List.of(child1Text, child1Int)); + compoundValues.add(cv1); + + // Second compound value + DatasetFieldCompoundValue cv2 = new DatasetFieldCompoundValue(); + cv2.setParentDatasetField(currentField); + + DatasetField child2Text = new DatasetField(); + child2Text.setDatasetFieldType(childTextField); + child2Text.setParentDatasetFieldCompoundValue(cv2); + child2Text.setSingleValue("Text2"); + + DatasetField child2Int = new DatasetField(); + child2Int.setDatasetFieldType(childIntField); + child2Int.setParentDatasetFieldCompoundValue(cv2); + child2Int.setSingleValue("456"); + + cv2.setChildDatasetFields(List.of(child2Text, child2Int)); + compoundValues.add(cv2); + + currentField.setDatasetFieldCompoundValues(compoundValues); + + // Create prior field with no values + DatasetField priorField = new DatasetField(); + priorField.setDatasetFieldType(compoundType); + priorField.setDatasetFieldCompoundValues(new ArrayList<>()); + + // Filter + DatasetField filtered = invokeFilterNewValues(currentField, priorField); + + // All compound values should be included + assertEquals(2, filtered.getDatasetFieldCompoundValues().size()); + } + + @Test + void testFilterNewValues_CompoundWithPrimitiveChild_SomeNew() throws Exception { + // Setup compound field type with primitive child fields + DatasetFieldType compoundType = new DatasetFieldType(); + compoundType.setId(1L); + compoundType.setName("testCompoundWithPrimitive"); + compoundType.setAllowMultiples(true); + compoundType.setFieldType(DatasetFieldType.FieldType.NONE); + + DatasetFieldType childTextField = new DatasetFieldType(); + childTextField.setId(2L); + childTextField.setName("childText"); + childTextField.setFieldType(DatasetFieldType.FieldType.TEXT); + childTextField.setParentDatasetFieldType(compoundType); + childTextField.setChildDatasetFieldTypes(new ArrayList<>()); + + DatasetFieldType childIntField = new DatasetFieldType(); + childIntField.setId(3L); + childIntField.setName("childInt"); + childIntField.setFieldType(DatasetFieldType.FieldType.INT); + childIntField.setParentDatasetFieldType(compoundType); + childIntField.setChildDatasetFieldTypes(new ArrayList<>()); + + compoundType.setChildDatasetFieldTypes(List.of(childTextField, childIntField)); + + // Create current field with 3 compound values + DatasetField currentField = new DatasetField(); + currentField.setDatasetFieldType(compoundType); + + List currentCompoundValues = new ArrayList<>(); + + // First compound value (existing) + DatasetFieldCompoundValue cv1 = new DatasetFieldCompoundValue(); + cv1.setParentDatasetField(currentField); + + DatasetField child1Text = new DatasetField(); + child1Text.setDatasetFieldType(childTextField); + child1Text.setParentDatasetFieldCompoundValue(cv1); + child1Text.setSingleValue("Text1"); + + DatasetField child1Int = new DatasetField(); + child1Int.setDatasetFieldType(childIntField); + child1Int.setParentDatasetFieldCompoundValue(cv1); + child1Int.setSingleValue("123"); + + cv1.setChildDatasetFields(List.of(child1Text, child1Int)); + currentCompoundValues.add(cv1); + + // Second compound value (new) + DatasetFieldCompoundValue cv2 = new DatasetFieldCompoundValue(); + cv2.setParentDatasetField(currentField); + + DatasetField child2Text = new DatasetField(); + child2Text.setDatasetFieldType(childTextField); + child2Text.setParentDatasetFieldCompoundValue(cv2); + child2Text.setSingleValue("Text2"); + + DatasetField child2Int = new DatasetField(); + child2Int.setDatasetFieldType(childIntField); + child2Int.setParentDatasetFieldCompoundValue(cv2); + child2Int.setSingleValue("456"); + + cv2.setChildDatasetFields(List.of(child2Text, child2Int)); + currentCompoundValues.add(cv2); + + // Third compound value (new) + DatasetFieldCompoundValue cv3 = new DatasetFieldCompoundValue(); + cv3.setParentDatasetField(currentField); + + DatasetField child3Text = new DatasetField(); + child3Text.setDatasetFieldType(childTextField); + child3Text.setParentDatasetFieldCompoundValue(cv3); + child3Text.setSingleValue("Text3"); + + DatasetField child3Int = new DatasetField(); + child3Int.setDatasetFieldType(childIntField); + child3Int.setParentDatasetFieldCompoundValue(cv3); + child3Int.setSingleValue("789"); + + cv3.setChildDatasetFields(List.of(child3Text, child3Int)); + currentCompoundValues.add(cv3); + + currentField.setDatasetFieldCompoundValues(currentCompoundValues); + + // Create prior field with 1 existing compound value + DatasetField priorField = new DatasetField(); + priorField.setDatasetFieldType(compoundType); + + List priorCompoundValues = new ArrayList<>(); + + DatasetFieldCompoundValue priorCv1 = new DatasetFieldCompoundValue(); + priorCv1.setParentDatasetField(priorField); + + DatasetField priorChild1Text = new DatasetField(); + priorChild1Text.setDatasetFieldType(childTextField); + priorChild1Text.setParentDatasetFieldCompoundValue(priorCv1); + priorChild1Text.setSingleValue("Text1"); + + DatasetField priorChild1Int = new DatasetField(); + priorChild1Int.setDatasetFieldType(childIntField); + priorChild1Int.setParentDatasetFieldCompoundValue(priorCv1); + priorChild1Int.setSingleValue("123"); + + priorCv1.setChildDatasetFields(List.of(priorChild1Text, priorChild1Int)); + priorCompoundValues.add(priorCv1); + + priorField.setDatasetFieldCompoundValues(priorCompoundValues); + + // Filter + DatasetField filtered = invokeFilterNewValues(currentField, priorField); + + // Only 2 new compound values should be included + assertEquals(2, filtered.getDatasetFieldCompoundValues().size()); + + // Verify the new values are present + boolean hasText2 = false; + boolean hasText3 = false; + + for (DatasetFieldCompoundValue cv : filtered.getDatasetFieldCompoundValues()) { + for (DatasetField childField : cv.getChildDatasetFields()) { + if (childField.getDatasetFieldType().equals(childTextField)) { + String value = childField.getDisplayValue(); + if ("Text2".equals(value)) { + hasText2 = true; + } else if ("Text3".equals(value)) { + hasText3 = true; + } + } + } + } + + assertTrue(hasText2); + assertTrue(hasText3); + } + + @Test + void testFilterNewValues_CompoundWithPrimitiveChild_NoneNew() throws Exception { + // Setup compound field type with primitive child fields + DatasetFieldType compoundType = new DatasetFieldType(); + compoundType.setId(1L); + compoundType.setName("testCompoundWithPrimitive"); + compoundType.setAllowMultiples(true); + compoundType.setFieldType(DatasetFieldType.FieldType.NONE); + + DatasetFieldType childTextField = new DatasetFieldType(); + childTextField.setId(2L); + childTextField.setName("childText"); + childTextField.setFieldType(DatasetFieldType.FieldType.TEXT); + childTextField.setParentDatasetFieldType(compoundType); + childTextField.setChildDatasetFieldTypes(new ArrayList<>()); + + DatasetFieldType childIntField = new DatasetFieldType(); + childIntField.setId(3L); + childIntField.setName("childInt"); + childIntField.setFieldType(DatasetFieldType.FieldType.INT); + childIntField.setParentDatasetFieldType(compoundType); + childIntField.setChildDatasetFieldTypes(new ArrayList<>()); + + compoundType.setChildDatasetFieldTypes(List.of(childTextField, childIntField)); + + // Create current field with 2 compound values + DatasetField currentField = new DatasetField(); + currentField.setDatasetFieldType(compoundType); + + List currentCompoundValues = new ArrayList<>(); + + // First compound value + DatasetFieldCompoundValue cv1 = new DatasetFieldCompoundValue(); + cv1.setParentDatasetField(currentField); + + DatasetField child1Text = new DatasetField(); + child1Text.setDatasetFieldType(childTextField); + child1Text.setParentDatasetFieldCompoundValue(cv1); + child1Text.setSingleValue("Text1"); + + DatasetField child1Int = new DatasetField(); + child1Int.setDatasetFieldType(childIntField); + child1Int.setParentDatasetFieldCompoundValue(cv1); + child1Int.setSingleValue("123"); + + cv1.setChildDatasetFields(List.of(child1Text, child1Int)); + currentCompoundValues.add(cv1); + + // Second compound value + DatasetFieldCompoundValue cv2 = new DatasetFieldCompoundValue(); + cv2.setParentDatasetField(currentField); + + DatasetField child2Text = new DatasetField(); + child2Text.setDatasetFieldType(childTextField); + child2Text.setParentDatasetFieldCompoundValue(cv2); + child2Text.setSingleValue("Text2"); + + DatasetField child2Int = new DatasetField(); + child2Int.setDatasetFieldType(childIntField); + child2Int.setParentDatasetFieldCompoundValue(cv2); + child2Int.setSingleValue("456"); + + cv2.setChildDatasetFields(List.of(child2Text, child2Int)); + currentCompoundValues.add(cv2); + + currentField.setDatasetFieldCompoundValues(currentCompoundValues); + + // Create prior field with same compound values + DatasetField priorField = new DatasetField(); + priorField.setDatasetFieldType(compoundType); + + List priorCompoundValues = new ArrayList<>(); + + // First compound value (same as current) + DatasetFieldCompoundValue priorCv1 = new DatasetFieldCompoundValue(); + priorCv1.setParentDatasetField(priorField); + + DatasetField priorChild1Text = new DatasetField(); + priorChild1Text.setDatasetFieldType(childTextField); + priorChild1Text.setParentDatasetFieldCompoundValue(priorCv1); + priorChild1Text.setSingleValue("Text1"); + + DatasetField priorChild1Int = new DatasetField(); + priorChild1Int.setDatasetFieldType(childIntField); + priorChild1Int.setParentDatasetFieldCompoundValue(priorCv1); + priorChild1Int.setSingleValue("123"); + + priorCv1.setChildDatasetFields(List.of(priorChild1Text, priorChild1Int)); + priorCompoundValues.add(priorCv1); + + // Second compound value (same as current) + DatasetFieldCompoundValue priorCv2 = new DatasetFieldCompoundValue(); + priorCv2.setParentDatasetField(priorField); + + DatasetField priorChild2Text = new DatasetField(); + priorChild2Text.setDatasetFieldType(childTextField); + priorChild2Text.setParentDatasetFieldCompoundValue(priorCv2); + priorChild2Text.setSingleValue("Text2"); + + DatasetField priorChild2Int = new DatasetField(); + priorChild2Int.setDatasetFieldType(childIntField); + priorChild2Int.setParentDatasetFieldCompoundValue(priorCv2); + priorChild2Int.setSingleValue("456"); + + priorCv2.setChildDatasetFields(List.of(priorChild2Text, priorChild2Int)); + priorCompoundValues.add(priorCv2); + + priorField.setDatasetFieldCompoundValues(priorCompoundValues); + + // Filter + DatasetField filtered = invokeFilterNewValues(currentField, priorField); + + + // No compound values should be included + assertEquals(0, filtered.getDatasetFieldCompoundValues().size()); + assertTrue(filtered.isEmpty()); + } // Helper methods @@ -321,21 +1079,15 @@ private DatasetField createCompoundField(DatasetFieldType fieldType, String[]... compoundValue.setParentDatasetField(field); List childFields = new ArrayList<>(); + List childTypes = new ArrayList<>(fieldType.getChildDatasetFieldTypes()); - // First child field (e.g., author name) - DatasetField childField1 = new DatasetField(); - childField1.setDatasetFieldType(childFieldType1); - childField1.setParentDatasetFieldCompoundValue(compoundValue); - childField1.setSingleValue(values[0]); - childFields.add(childField1); - - // Second child field (e.g., affiliation) - if (values.length > 1) { - DatasetField childField2 = new DatasetField(); - childField2.setDatasetFieldType(childFieldType2); - childField2.setParentDatasetFieldCompoundValue(compoundValue); - childField2.setSingleValue(values[1]); - childFields.add(childField2); + // Create child fields based on the parent's child types and provided values + for (int i = 0; i < Math.min(values.length, childTypes.size()); i++) { + DatasetField childField = new DatasetField(); + childField.setDatasetFieldType(childTypes.get(i)); + childField.setParentDatasetFieldCompoundValue(compoundValue); + childField.setSingleValue(values[i]); + childFields.add(childField); } compoundValue.setChildDatasetFields(childFields); @@ -346,28 +1098,43 @@ private DatasetField createCompoundField(DatasetFieldType fieldType, String[]... return field; } - private boolean containsCompoundValue(DatasetField field, String childValue1, String childValue2) { + private boolean containsCompoundValue(DatasetField field, String... childValues) { for (DatasetFieldCompoundValue cv : field.getDatasetFieldCompoundValues()) { - boolean hasValue1 = false; - boolean hasValue2 = false; + List cvValues = new ArrayList<>(); for (DatasetField childField : cv.getChildDatasetFields()) { - String displayValue = childField.getDisplayValue(); - if (childValue1.equals(displayValue)) { - hasValue1 = true; - } - if (childValue2.equals(displayValue)) { - hasValue2 = true; + cvValues.add(childField.getDisplayValue()); + } + + // Check if all provided values are present in this compound value + boolean allMatch = true; + for (String value : childValues) { + if (!cvValues.contains(value)) { + allMatch = false; + break; } } - if (hasValue1 && hasValue2) { + if (allMatch && cvValues.size() == childValues.length) { return true; } } return false; } - + + private boolean containsControlledVocabValue(DatasetField field, String strValue) { + if (field.getControlledVocabularyValues() == null) { + return false; + } + + for (ControlledVocabularyValue cvv : field.getControlledVocabularyValues()) { + if (cvv.getStrValue().equals(strValue)) { + return true; + } + } + return false; + } + /** * Use reflection to invoke the private filterNewValues method */ From ad37d932749f7f906a967d180d1f9a2192b316e0 Mon Sep 17 00:00:00 2001 From: qqmyers Date: Thu, 20 Nov 2025 12:30:46 -0500 Subject: [PATCH 39/40] cleanup --- ...otifyRelationshipAnnouncementStepTest.java | 797 +++++++++--------- 1 file changed, 399 insertions(+), 398 deletions(-) diff --git a/src/test/java/edu/harvard/iq/dataverse/workflow/internalspi/COARNotifyRelationshipAnnouncementStepTest.java b/src/test/java/edu/harvard/iq/dataverse/workflow/internalspi/COARNotifyRelationshipAnnouncementStepTest.java index 063e3d58dda..ce128c452b4 100644 --- a/src/test/java/edu/harvard/iq/dataverse/workflow/internalspi/COARNotifyRelationshipAnnouncementStepTest.java +++ b/src/test/java/edu/harvard/iq/dataverse/workflow/internalspi/COARNotifyRelationshipAnnouncementStepTest.java @@ -5,6 +5,8 @@ import edu.harvard.iq.dataverse.DatasetFieldCompoundValue; import edu.harvard.iq.dataverse.DatasetFieldType; import edu.harvard.iq.dataverse.DatasetFieldValue; + +import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import java.util.ArrayList; @@ -17,67 +19,70 @@ class COARNotifyRelationshipAnnouncementStepTest { private COARNotifyRelationshipAnnouncementStep step; - private DatasetFieldType multiValueFieldType; - private DatasetFieldType singleValueFieldType; - private DatasetFieldType compoundFieldType; - private DatasetFieldType childFieldType1; - private DatasetFieldType childFieldType2; - - -@BeforeEach -void setUp() { - Map params = new HashMap<>(); - step = new COARNotifyRelationshipAnnouncementStep(params); - - // Setup field types - multiValueFieldType = new DatasetFieldType(); - multiValueFieldType.setId(1L); - multiValueFieldType.setName("testMultiField"); - multiValueFieldType.setAllowMultiples(true); - multiValueFieldType.setFieldType(DatasetFieldType.FieldType.TEXT); - multiValueFieldType.setChildDatasetFieldTypes(new ArrayList<>()); - - singleValueFieldType = new DatasetFieldType(); - singleValueFieldType.setId(2L); - singleValueFieldType.setName("testSingleField"); - singleValueFieldType.setAllowMultiples(false); - singleValueFieldType.setFieldType(DatasetFieldType.FieldType.TEXT); - singleValueFieldType.setChildDatasetFieldTypes(new ArrayList<>()); - - // Setup compound field type with child fields - compoundFieldType = new DatasetFieldType(); - compoundFieldType.setId(3L); - compoundFieldType.setName("testCompoundField"); - compoundFieldType.setAllowMultiples(true); - compoundFieldType.setFieldType(DatasetFieldType.FieldType.NONE); - - childFieldType1 = new DatasetFieldType(); - childFieldType1.setId(4L); - childFieldType1.setName("authorName"); - childFieldType1.setFieldType(DatasetFieldType.FieldType.TEXT); - childFieldType1.setParentDatasetFieldType(compoundFieldType); - childFieldType1.setChildDatasetFieldTypes(new ArrayList<>()); - - childFieldType2 = new DatasetFieldType(); - childFieldType2.setId(5L); - childFieldType2.setName("authorAffiliation"); - childFieldType2.setFieldType(DatasetFieldType.FieldType.TEXT); - childFieldType2.setParentDatasetFieldType(compoundFieldType); - childFieldType2.setChildDatasetFieldTypes(new ArrayList<>()); - - compoundFieldType.setChildDatasetFieldTypes(List.of(childFieldType1, childFieldType2)); -} + private static DatasetFieldType multiValueFieldType; + private static DatasetFieldType singleValueFieldType; + private static DatasetFieldType compoundFieldType; + private static DatasetFieldType childFieldType1; + private static DatasetFieldType childFieldType2; + + @BeforeAll + static void setUpFieldTypes() { + // Setup field types + multiValueFieldType = new DatasetFieldType(); + multiValueFieldType.setId(1L); + multiValueFieldType.setName("testMultiField"); + multiValueFieldType.setAllowMultiples(true); + multiValueFieldType.setFieldType(DatasetFieldType.FieldType.TEXT); + multiValueFieldType.setChildDatasetFieldTypes(new ArrayList<>()); + + singleValueFieldType = new DatasetFieldType(); + singleValueFieldType.setId(2L); + singleValueFieldType.setName("testSingleField"); + singleValueFieldType.setAllowMultiples(false); + singleValueFieldType.setFieldType(DatasetFieldType.FieldType.TEXT); + singleValueFieldType.setChildDatasetFieldTypes(new ArrayList<>()); + + // Setup compound field type with child fields + compoundFieldType = new DatasetFieldType(); + compoundFieldType.setId(3L); + compoundFieldType.setName("testCompoundField"); + compoundFieldType.setAllowMultiples(true); + compoundFieldType.setFieldType(DatasetFieldType.FieldType.NONE); + + childFieldType1 = new DatasetFieldType(); + childFieldType1.setId(4L); + childFieldType1.setName("authorName"); + childFieldType1.setFieldType(DatasetFieldType.FieldType.TEXT); + childFieldType1.setParentDatasetFieldType(compoundFieldType); + childFieldType1.setChildDatasetFieldTypes(new ArrayList<>()); + + childFieldType2 = new DatasetFieldType(); + childFieldType2.setId(5L); + childFieldType2.setName("authorAffiliation"); + childFieldType2.setFieldType(DatasetFieldType.FieldType.TEXT); + childFieldType2.setParentDatasetFieldType(compoundFieldType); + childFieldType2.setChildDatasetFieldTypes(new ArrayList<>()); + + compoundFieldType.setChildDatasetFieldTypes(List.of(childFieldType1, childFieldType2)); + } + + @BeforeEach + void setUp() { + Map params = new HashMap<>(); + step = new COARNotifyRelationshipAnnouncementStep(params); + } + @Test void testFilterNewValues_MultiValue_AllNew() throws Exception { // Create current field with 3 values DatasetField currentField = createMultiValueField(multiValueFieldType, "value1", "value2", "value3"); - + // Create prior field with no values DatasetField priorField = createMultiValueField(multiValueFieldType); - + // Filter DatasetField filtered = invokeFilterNewValues(currentField, priorField); - + // All values should be included assertEquals(3, filtered.getDatasetFieldValues().size()); assertTrue(containsValue(filtered, "value1")); @@ -89,13 +94,13 @@ void testFilterNewValues_MultiValue_AllNew() throws Exception { void testFilterNewValues_MultiValue_SomeNew() throws Exception { // Create current field with 3 values DatasetField currentField = createMultiValueField(multiValueFieldType, "value1", "value2", "value3"); - + // Create prior field with 2 existing values DatasetField priorField = createMultiValueField(multiValueFieldType, "value1", "value2"); - + // Filter DatasetField filtered = invokeFilterNewValues(currentField, priorField); - + // Only new value should be included assertEquals(1, filtered.getDatasetFieldValues().size()); assertFalse(containsValue(filtered, "value1")); @@ -107,13 +112,13 @@ void testFilterNewValues_MultiValue_SomeNew() throws Exception { void testFilterNewValues_MultiValue_NoneNew() throws Exception { // Create current field with 2 values DatasetField currentField = createMultiValueField(multiValueFieldType, "value1", "value2"); - + // Create prior field with same values DatasetField priorField = createMultiValueField(multiValueFieldType, "value1", "value2"); - + // Filter DatasetField filtered = invokeFilterNewValues(currentField, priorField); - + // No values should be included assertEquals(0, filtered.getDatasetFieldValues().size()); assertTrue(filtered.isEmpty()); @@ -123,13 +128,13 @@ void testFilterNewValues_MultiValue_NoneNew() throws Exception { void testFilterNewValues_SingleValue_Changed() throws Exception { // Create current field with new value DatasetField currentField = createSingleValueField(singleValueFieldType, "newValue"); - + // Create prior field with old value DatasetField priorField = createSingleValueField(singleValueFieldType, "oldValue"); - + // Filter DatasetField filtered = invokeFilterNewValues(currentField, priorField); - + // New value should be included assertFalse(filtered.isEmpty()); assertEquals("newValue", filtered.getValue()); @@ -139,13 +144,13 @@ void testFilterNewValues_SingleValue_Changed() throws Exception { void testFilterNewValues_SingleValue_Unchanged() throws Exception { // Create current field with same value DatasetField currentField = createSingleValueField(singleValueFieldType, "sameValue"); - + // Create prior field with same value DatasetField priorField = createSingleValueField(singleValueFieldType, "sameValue"); - + // Filter DatasetField filtered = invokeFilterNewValues(currentField, priorField); - + // No value should be included assertTrue(filtered.isEmpty()); } @@ -155,38 +160,37 @@ void testFilterNewValues_DoesNotModifyOriginal() throws Exception { // Create current field with 3 values DatasetField currentField = createMultiValueField(multiValueFieldType, "value1", "value2", "value3"); int originalSize = currentField.getDatasetFieldValues().size(); - + // Create prior field with 2 existing values DatasetField priorField = createMultiValueField(multiValueFieldType, "value1", "value2"); - + // Filter DatasetField filtered = invokeFilterNewValues(currentField, priorField); - + // Original field should be unchanged assertEquals(originalSize, currentField.getDatasetFieldValues().size()); assertTrue(containsValue(currentField, "value1")); assertTrue(containsValue(currentField, "value2")); assertTrue(containsValue(currentField, "value3")); - + // Filtered field should only have new value assertEquals(1, filtered.getDatasetFieldValues().size()); assertTrue(containsValue(filtered, "value3")); } - @Test void testFilterNewValues_CompoundValue_AllNew() throws Exception { // Create current field with 2 compound values DatasetField currentField = createCompoundField(compoundFieldType, new String[]{"Author1", "Affiliation1"}, - new String[]{"Author2", "Affiliation2"}); - + new String[] { "Author2", "Affiliation2" }); + // Create prior field with no values DatasetField priorField = createCompoundField(compoundFieldType); - + // Filter DatasetField filtered = invokeFilterNewValues(currentField, priorField); - + // All compound values should be included assertEquals(2, filtered.getDatasetFieldCompoundValues().size()); assertTrue(containsCompoundValue(filtered, "Author1", "Affiliation1")); @@ -200,15 +204,15 @@ void testFilterNewValues_CompoundValue_SomeNew() throws Exception { new String[]{"Author1", "Affiliation1"}, new String[]{"Author2", "Affiliation2"}, new String[]{"Author3", "Affiliation3"}); - + // Create prior field with 2 existing compound values DatasetField priorField = createCompoundField(compoundFieldType, new String[]{"Author1", "Affiliation1"}, - new String[]{"Author2", "Affiliation2"}); - + new String[] { "Author2", "Affiliation2" }); + // Filter DatasetField filtered = invokeFilterNewValues(currentField, priorField); - + // Only new compound value should be included assertEquals(1, filtered.getDatasetFieldCompoundValues().size()); assertFalse(containsCompoundValue(filtered, "Author1", "Affiliation1")); @@ -221,16 +225,16 @@ void testFilterNewValues_CompoundValue_NoneNew() throws Exception { // Create current field with 2 compound values DatasetField currentField = createCompoundField(compoundFieldType, new String[]{"Author1", "Affiliation1"}, - new String[]{"Author2", "Affiliation2"}); - + new String[] { "Author2", "Affiliation2" }); + // Create prior field with same compound values DatasetField priorField = createCompoundField(compoundFieldType, new String[]{"Author1", "Affiliation1"}, - new String[]{"Author2", "Affiliation2"}); - + new String[] { "Author2", "Affiliation2" }); + // Filter DatasetField filtered = invokeFilterNewValues(currentField, priorField); - + // No compound values should be included assertEquals(0, filtered.getDatasetFieldCompoundValues().size()); assertTrue(filtered.isEmpty()); @@ -240,15 +244,15 @@ void testFilterNewValues_CompoundValue_NoneNew() throws Exception { void testFilterNewValues_CompoundValue_PartialMatch() throws Exception { // Create current field with compound value where one child field changed DatasetField currentField = createCompoundField(compoundFieldType, - new String[]{"Author1", "NewAffiliation"}); - + new String[] { "Author1", "NewAffiliation" }); + // Create prior field with same author but different affiliation DatasetField priorField = createCompoundField(compoundFieldType, new String[]{"Author1", "OldAffiliation"}); - + // Filter DatasetField filtered = invokeFilterNewValues(currentField, priorField); - + // Should be treated as a new compound value since child field changed assertEquals(1, filtered.getDatasetFieldCompoundValues().size()); assertTrue(containsCompoundValue(filtered, "Author1", "NewAffiliation")); @@ -262,26 +266,26 @@ void testFilterNewValues_CompoundValue_DoesNotModifyOriginal() throws Exception new String[]{"Author2", "Affiliation2"}, new String[]{"Author3", "Affiliation3"}); int originalSize = currentField.getDatasetFieldCompoundValues().size(); - + // Create prior field with 2 existing compound values DatasetField priorField = createCompoundField(compoundFieldType, new String[]{"Author1", "Affiliation1"}, - new String[]{"Author2", "Affiliation2"}); - + new String[] { "Author2", "Affiliation2" }); + // Filter DatasetField filtered = invokeFilterNewValues(currentField, priorField); - + // Original field should be unchanged assertEquals(originalSize, currentField.getDatasetFieldCompoundValues().size()); assertTrue(containsCompoundValue(currentField, "Author1", "Affiliation1")); assertTrue(containsCompoundValue(currentField, "Author2", "Affiliation2")); assertTrue(containsCompoundValue(currentField, "Author3", "Affiliation3")); - + // Filtered field should only have new compound value assertEquals(1, filtered.getDatasetFieldCompoundValues().size()); assertTrue(containsCompoundValue(filtered, "Author3", "Affiliation3")); } - + @Test void testFilterNewValues_ControlledVocab_AllNew() throws Exception { // Setup controlled vocabulary field type @@ -291,33 +295,33 @@ void testFilterNewValues_ControlledVocab_AllNew() throws Exception { cvFieldType.setFieldType(DatasetFieldType.FieldType.TEXT); cvFieldType.setAllowControlledVocabulary(true); cvFieldType.setChildDatasetFieldTypes(new ArrayList<>()); - + // Create controlled vocabulary values ControlledVocabularyValue cvv1 = new ControlledVocabularyValue(); cvv1.setStrValue("Medicine, Health and Life Sciences"); cvv1.setDatasetFieldType(cvFieldType); - + ControlledVocabularyValue cvv2 = new ControlledVocabularyValue(); cvv2.setStrValue("Social Sciences"); cvv2.setDatasetFieldType(cvFieldType); - + ControlledVocabularyValue cvv3 = new ControlledVocabularyValue(); cvv3.setStrValue("Engineering"); cvv3.setDatasetFieldType(cvFieldType); - + // Create current field with 3 CV values DatasetField currentField = new DatasetField(); currentField.setDatasetFieldType(cvFieldType); currentField.setControlledVocabularyValues(List.of(cvv1, cvv2, cvv3)); - + // Create prior field with no values DatasetField priorField = new DatasetField(); priorField.setDatasetFieldType(cvFieldType); priorField.setControlledVocabularyValues(new ArrayList<>()); - + // Filter DatasetField filtered = invokeFilterNewValues(currentField, priorField); - + // All CV values should be included assertEquals(3, filtered.getControlledVocabularyValues().size()); assertTrue(containsControlledVocabValue(filtered, "Medicine, Health and Life Sciences")); @@ -334,33 +338,33 @@ void testFilterNewValues_ControlledVocab_SomeNew() throws Exception { cvFieldType.setFieldType(DatasetFieldType.FieldType.TEXT); cvFieldType.setAllowControlledVocabulary(true); cvFieldType.setChildDatasetFieldTypes(new ArrayList<>()); - + // Create controlled vocabulary values ControlledVocabularyValue cvv1 = new ControlledVocabularyValue(); cvv1.setStrValue("Medicine, Health and Life Sciences"); cvv1.setDatasetFieldType(cvFieldType); - + ControlledVocabularyValue cvv2 = new ControlledVocabularyValue(); cvv2.setStrValue("Social Sciences"); cvv2.setDatasetFieldType(cvFieldType); - + ControlledVocabularyValue cvv3 = new ControlledVocabularyValue(); cvv3.setStrValue("Engineering"); cvv3.setDatasetFieldType(cvFieldType); - + // Create current field with 3 CV values DatasetField currentField = new DatasetField(); currentField.setDatasetFieldType(cvFieldType); currentField.setControlledVocabularyValues(List.of(cvv1, cvv2, cvv3)); - + // Create prior field with 2 existing CV values DatasetField priorField = new DatasetField(); priorField.setDatasetFieldType(cvFieldType); priorField.setControlledVocabularyValues(List.of(cvv1, cvv2)); - + // Filter DatasetField filtered = invokeFilterNewValues(currentField, priorField); - + // Only new CV value should be included assertEquals(1, filtered.getControlledVocabularyValues().size()); assertFalse(containsControlledVocabValue(filtered, "Medicine, Health and Life Sciences")); @@ -377,29 +381,29 @@ void testFilterNewValues_ControlledVocab_NoneNew() throws Exception { cvFieldType.setFieldType(DatasetFieldType.FieldType.TEXT); cvFieldType.setAllowControlledVocabulary(true); cvFieldType.setChildDatasetFieldTypes(new ArrayList<>()); - + // Create controlled vocabulary values ControlledVocabularyValue cvv1 = new ControlledVocabularyValue(); cvv1.setStrValue("Medicine, Health and Life Sciences"); cvv1.setDatasetFieldType(cvFieldType); - + ControlledVocabularyValue cvv2 = new ControlledVocabularyValue(); cvv2.setStrValue("Social Sciences"); cvv2.setDatasetFieldType(cvFieldType); - + // Create current field with 2 CV values DatasetField currentField = new DatasetField(); currentField.setDatasetFieldType(cvFieldType); currentField.setControlledVocabularyValues(List.of(cvv1, cvv2)); - + // Create prior field with same CV values DatasetField priorField = new DatasetField(); priorField.setDatasetFieldType(cvFieldType); priorField.setControlledVocabularyValues(List.of(cvv1, cvv2)); - + // Filter DatasetField filtered = invokeFilterNewValues(currentField, priorField); - + // No CV values should be included assertEquals(0, filtered.getControlledVocabularyValues().size()); assertTrue(filtered.isEmpty()); @@ -414,29 +418,29 @@ void testFilterNewValues_ControlledVocab_SingleValue_Changed() throws Exception cvFieldType.setFieldType(DatasetFieldType.FieldType.TEXT); cvFieldType.setAllowControlledVocabulary(true); cvFieldType.setChildDatasetFieldTypes(new ArrayList<>()); - + // Create controlled vocabulary values ControlledVocabularyValue cvvOld = new ControlledVocabularyValue(); cvvOld.setStrValue("Medicine, Health and Life Sciences"); cvvOld.setDatasetFieldType(cvFieldType); - + ControlledVocabularyValue cvvNew = new ControlledVocabularyValue(); cvvNew.setStrValue("Social Sciences"); cvvNew.setDatasetFieldType(cvFieldType); - + // Create current field with new CV value DatasetField currentField = new DatasetField(); currentField.setDatasetFieldType(cvFieldType); currentField.setControlledVocabularyValues(List.of(cvvNew)); - + // Create prior field with old CV value DatasetField priorField = new DatasetField(); priorField.setDatasetFieldType(cvFieldType); priorField.setControlledVocabularyValues(List.of(cvvOld)); - + // Filter DatasetField filtered = invokeFilterNewValues(currentField, priorField); - + // New CV value should be included assertFalse(filtered.isEmpty()); assertEquals(1, filtered.getControlledVocabularyValues().size()); @@ -450,258 +454,256 @@ void testFilterNewValues_CompoundWithControlledVocabChild_AllNew() throws Except compoundType.setName("testCompoundWithCV"); compoundType.setAllowMultiples(true); compoundType.setFieldType(DatasetFieldType.FieldType.NONE); - + DatasetFieldType childTextType = new DatasetFieldType(); childTextType.setName("childText"); childTextType.setFieldType(DatasetFieldType.FieldType.TEXT); childTextType.setParentDatasetFieldType(compoundType); childTextType.setChildDatasetFieldTypes(new ArrayList<>()); - + DatasetFieldType childCVType = new DatasetFieldType(); childCVType.setName("childCV"); childCVType.setFieldType(DatasetFieldType.FieldType.TEXT); childCVType.setAllowControlledVocabulary(true); childCVType.setParentDatasetFieldType(compoundType); childCVType.setChildDatasetFieldTypes(new ArrayList<>()); - + compoundType.setChildDatasetFieldTypes(List.of(childTextType, childCVType)); - + // Create controlled vocabulary values ControlledVocabularyValue cvv1 = new ControlledVocabularyValue(); cvv1.setStrValue("ark"); cvv1.setDatasetFieldType(childCVType); - + ControlledVocabularyValue cvv2 = new ControlledVocabularyValue(); cvv2.setStrValue("doi"); cvv2.setDatasetFieldType(childCVType); - + // Create current field with 2 compound values containing CV child fields DatasetField currentField = new DatasetField(); currentField.setDatasetFieldType(compoundType); - + List compoundValues = new ArrayList<>(); - + // First compound value DatasetFieldCompoundValue cv1 = new DatasetFieldCompoundValue(); cv1.setParentDatasetField(currentField); - + DatasetField child1Text = new DatasetField(); child1Text.setDatasetFieldType(childTextType); child1Text.setParentDatasetFieldCompoundValue(cv1); child1Text.setSingleValue("Value1"); - + DatasetField child1CV = new DatasetField(); child1CV.setDatasetFieldType(childCVType); child1CV.setParentDatasetFieldCompoundValue(cv1); child1CV.setControlledVocabularyValues(List.of(cvv1)); - + cv1.setChildDatasetFields(List.of(child1Text, child1CV)); compoundValues.add(cv1); - + // Second compound value DatasetFieldCompoundValue cv2 = new DatasetFieldCompoundValue(); cv2.setParentDatasetField(currentField); - + DatasetField child2Text = new DatasetField(); child2Text.setDatasetFieldType(childTextType); child2Text.setParentDatasetFieldCompoundValue(cv2); child2Text.setSingleValue("Value2"); - + DatasetField child2CV = new DatasetField(); child2CV.setDatasetFieldType(childCVType); child2CV.setParentDatasetFieldCompoundValue(cv2); child2CV.setControlledVocabularyValues(List.of(cvv2)); - + cv2.setChildDatasetFields(List.of(child2Text, child2CV)); compoundValues.add(cv2); - + currentField.setDatasetFieldCompoundValues(compoundValues); - + // Create prior field with no values DatasetField priorField = new DatasetField(); priorField.setDatasetFieldType(compoundType); priorField.setDatasetFieldCompoundValues(new ArrayList<>()); - + // Filter DatasetField filtered = invokeFilterNewValues(currentField, priorField); - + // All compound values should be included assertEquals(2, filtered.getDatasetFieldCompoundValues().size()); } + @Test + void testFilterNewValues_CompoundWithControlledVocabChild_CVChanged() throws Exception { + // Setup compound field type with CV child field + DatasetFieldType compoundType = new DatasetFieldType(); + compoundType.setId(1L); + compoundType.setName("testCompoundWithCV"); + compoundType.setAllowMultiples(true); + compoundType.setFieldType(DatasetFieldType.FieldType.NONE); + + DatasetFieldType childTextType = new DatasetFieldType(); + childTextType.setId(2L); + childTextType.setName("childText"); + childTextType.setFieldType(DatasetFieldType.FieldType.TEXT); + childTextType.setParentDatasetFieldType(compoundType); + childTextType.setChildDatasetFieldTypes(new ArrayList<>()); + + DatasetFieldType childCVType = new DatasetFieldType(); + childCVType.setId(3L); + childCVType.setName("childCV"); + childCVType.setFieldType(DatasetFieldType.FieldType.TEXT); + childCVType.setAllowControlledVocabulary(true); + childCVType.setParentDatasetFieldType(compoundType); + childCVType.setChildDatasetFieldTypes(new ArrayList<>()); + + compoundType.setChildDatasetFieldTypes(List.of(childTextType, childCVType)); + + // Create controlled vocabulary values + ControlledVocabularyValue cvvOld = new ControlledVocabularyValue(); + cvvOld.setStrValue("ark"); + cvvOld.setDatasetFieldType(childCVType); + + ControlledVocabularyValue cvvNew = new ControlledVocabularyValue(); + cvvNew.setStrValue("doi"); + cvvNew.setDatasetFieldType(childCVType); + + // Create current field with compound value containing new CV + DatasetField currentField = new DatasetField(); + currentField.setDatasetFieldType(compoundType); + + DatasetFieldCompoundValue currentCV = new DatasetFieldCompoundValue(); + currentCV.setParentDatasetField(currentField); + + DatasetField currentChildText = new DatasetField(); + currentChildText.setDatasetFieldType(childTextType); + currentChildText.setParentDatasetFieldCompoundValue(currentCV); + currentChildText.setSingleValue("SameValue"); + + DatasetField currentChildCV = new DatasetField(); + currentChildCV.setDatasetFieldType(childCVType); + currentChildCV.setParentDatasetFieldCompoundValue(currentCV); + currentChildCV.setControlledVocabularyValues(List.of(cvvNew)); + + currentCV.setChildDatasetFields(List.of(currentChildText, currentChildCV)); + currentField.setDatasetFieldCompoundValues(List.of(currentCV)); + + // Create prior field with compound value containing old CV + DatasetField priorField = new DatasetField(); + priorField.setDatasetFieldType(compoundType); + + DatasetFieldCompoundValue priorCV = new DatasetFieldCompoundValue(); + priorCV.setParentDatasetField(priorField); + + DatasetField priorChildText = new DatasetField(); + priorChildText.setDatasetFieldType(childTextType); + priorChildText.setParentDatasetFieldCompoundValue(priorCV); + priorChildText.setSingleValue("SameValue"); + + DatasetField priorChildCV = new DatasetField(); + priorChildCV.setDatasetFieldType(childCVType); + priorChildCV.setParentDatasetFieldCompoundValue(priorCV); + priorChildCV.setControlledVocabularyValues(List.of(cvvOld)); + + priorCV.setChildDatasetFields(List.of(priorChildText, priorChildCV)); + priorField.setDatasetFieldCompoundValues(List.of(priorCV)); + + // Filter + DatasetField filtered = invokeFilterNewValues(currentField, priorField); + + // Should be treated as new compound value since CV child changed + assertEquals(1, filtered.getDatasetFieldCompoundValues().size()); + + // Verify the CV value in the filtered compound + DatasetFieldCompoundValue filteredCV = filtered.getDatasetFieldCompoundValues().get(0); + DatasetField filteredChildCV = filteredCV.getChildDatasetFields().stream() + .filter(f -> f.getDatasetFieldType().equals(childCVType)) + .findFirst() + .orElse(null); + + assertNotNull(filteredChildCV); + assertEquals(1, filteredChildCV.getControlledVocabularyValues().size()); + assertEquals("doi", filteredChildCV.getControlledVocabularyValues().get(0).getStrValue()); + } + + @Test + void testFilterNewValues_CompoundWithControlledVocabChild_CVUnchanged() throws Exception { + // Setup compound field type with CV child field + DatasetFieldType compoundType = new DatasetFieldType(); + compoundType.setId(1L); + compoundType.setName("testCompoundWithCV"); + compoundType.setAllowMultiples(true); + compoundType.setFieldType(DatasetFieldType.FieldType.NONE); + + DatasetFieldType childTextType = new DatasetFieldType(); + childTextType.setId(2L); + childTextType.setName("childText"); + childTextType.setFieldType(DatasetFieldType.FieldType.TEXT); + childTextType.setParentDatasetFieldType(compoundType); + childTextType.setChildDatasetFieldTypes(new ArrayList<>()); + + DatasetFieldType childCVType = new DatasetFieldType(); + childCVType.setId(3L); + childCVType.setName("childCV"); + childCVType.setFieldType(DatasetFieldType.FieldType.TEXT); + childCVType.setAllowControlledVocabulary(true); + childCVType.setParentDatasetFieldType(compoundType); + childCVType.setChildDatasetFieldTypes(new ArrayList<>()); + + compoundType.setChildDatasetFieldTypes(List.of(childTextType, childCVType)); + + // Create controlled vocabulary value + ControlledVocabularyValue cvv = new ControlledVocabularyValue(); + cvv.setStrValue("ark"); + cvv.setDatasetFieldType(childCVType); + + // Create current field with compound value + DatasetField currentField = new DatasetField(); + currentField.setDatasetFieldType(compoundType); + + DatasetFieldCompoundValue currentCV = new DatasetFieldCompoundValue(); + currentCV.setParentDatasetField(currentField); + + DatasetField currentChildText = new DatasetField(); + currentChildText.setDatasetFieldType(childTextType); + currentChildText.setParentDatasetFieldCompoundValue(currentCV); + currentChildText.setSingleValue("SameValue"); -@Test -void testFilterNewValues_CompoundWithControlledVocabChild_CVChanged() throws Exception { - // Setup compound field type with CV child field - DatasetFieldType compoundType = new DatasetFieldType(); - compoundType.setId(1L); - compoundType.setName("testCompoundWithCV"); - compoundType.setAllowMultiples(true); - compoundType.setFieldType(DatasetFieldType.FieldType.NONE); - - DatasetFieldType childTextType = new DatasetFieldType(); - childTextType.setId(2L); - childTextType.setName("childText"); - childTextType.setFieldType(DatasetFieldType.FieldType.TEXT); - childTextType.setParentDatasetFieldType(compoundType); - childTextType.setChildDatasetFieldTypes(new ArrayList<>()); - - DatasetFieldType childCVType = new DatasetFieldType(); - childCVType.setId(3L); - childCVType.setName("childCV"); - childCVType.setFieldType(DatasetFieldType.FieldType.TEXT); - childCVType.setAllowControlledVocabulary(true); - childCVType.setParentDatasetFieldType(compoundType); - childCVType.setChildDatasetFieldTypes(new ArrayList<>()); - - compoundType.setChildDatasetFieldTypes(List.of(childTextType, childCVType)); - - // Create controlled vocabulary values - ControlledVocabularyValue cvvOld = new ControlledVocabularyValue(); - cvvOld.setStrValue("ark"); - cvvOld.setDatasetFieldType(childCVType); - - ControlledVocabularyValue cvvNew = new ControlledVocabularyValue(); - cvvNew.setStrValue("doi"); - cvvNew.setDatasetFieldType(childCVType); - - // Create current field with compound value containing new CV - DatasetField currentField = new DatasetField(); - currentField.setDatasetFieldType(compoundType); - - DatasetFieldCompoundValue currentCV = new DatasetFieldCompoundValue(); - currentCV.setParentDatasetField(currentField); - - DatasetField currentChildText = new DatasetField(); - currentChildText.setDatasetFieldType(childTextType); - currentChildText.setParentDatasetFieldCompoundValue(currentCV); - currentChildText.setSingleValue("SameValue"); - - DatasetField currentChildCV = new DatasetField(); - currentChildCV.setDatasetFieldType(childCVType); - currentChildCV.setParentDatasetFieldCompoundValue(currentCV); - currentChildCV.setControlledVocabularyValues(List.of(cvvNew)); - - currentCV.setChildDatasetFields(List.of(currentChildText, currentChildCV)); - currentField.setDatasetFieldCompoundValues(List.of(currentCV)); - - // Create prior field with compound value containing old CV - DatasetField priorField = new DatasetField(); - priorField.setDatasetFieldType(compoundType); - - DatasetFieldCompoundValue priorCV = new DatasetFieldCompoundValue(); - priorCV.setParentDatasetField(priorField); - - DatasetField priorChildText = new DatasetField(); - priorChildText.setDatasetFieldType(childTextType); - priorChildText.setParentDatasetFieldCompoundValue(priorCV); - priorChildText.setSingleValue("SameValue"); - - DatasetField priorChildCV = new DatasetField(); - priorChildCV.setDatasetFieldType(childCVType); - priorChildCV.setParentDatasetFieldCompoundValue(priorCV); - priorChildCV.setControlledVocabularyValues(List.of(cvvOld)); - - priorCV.setChildDatasetFields(List.of(priorChildText, priorChildCV)); - priorField.setDatasetFieldCompoundValues(List.of(priorCV)); - - // Filter - DatasetField filtered = invokeFilterNewValues(currentField, priorField); - - // Should be treated as new compound value since CV child changed - assertEquals(1, filtered.getDatasetFieldCompoundValues().size()); - - // Verify the CV value in the filtered compound - DatasetFieldCompoundValue filteredCV = filtered.getDatasetFieldCompoundValues().get(0); - DatasetField filteredChildCV = filteredCV.getChildDatasetFields().stream() - .filter(f -> f.getDatasetFieldType().equals(childCVType)) - .findFirst() - .orElse(null); - - assertNotNull(filteredChildCV); - assertEquals(1, filteredChildCV.getControlledVocabularyValues().size()); - assertEquals("doi", filteredChildCV.getControlledVocabularyValues().get(0).getStrValue()); -} - - -@Test -void testFilterNewValues_CompoundWithControlledVocabChild_CVUnchanged() throws Exception { - // Setup compound field type with CV child field - DatasetFieldType compoundType = new DatasetFieldType(); - compoundType.setId(1L); - compoundType.setName("testCompoundWithCV"); - compoundType.setAllowMultiples(true); - compoundType.setFieldType(DatasetFieldType.FieldType.NONE); - - DatasetFieldType childTextType = new DatasetFieldType(); - childTextType.setId(2L); - childTextType.setName("childText"); - childTextType.setFieldType(DatasetFieldType.FieldType.TEXT); - childTextType.setParentDatasetFieldType(compoundType); - childTextType.setChildDatasetFieldTypes(new ArrayList<>()); - - DatasetFieldType childCVType = new DatasetFieldType(); - childCVType.setId(3L); - childCVType.setName("childCV"); - childCVType.setFieldType(DatasetFieldType.FieldType.TEXT); - childCVType.setAllowControlledVocabulary(true); - childCVType.setParentDatasetFieldType(compoundType); - childCVType.setChildDatasetFieldTypes(new ArrayList<>()); - - compoundType.setChildDatasetFieldTypes(List.of(childTextType, childCVType)); - - // Create controlled vocabulary value - ControlledVocabularyValue cvv = new ControlledVocabularyValue(); - cvv.setStrValue("ark"); - cvv.setDatasetFieldType(childCVType); - - // Create current field with compound value - DatasetField currentField = new DatasetField(); - currentField.setDatasetFieldType(compoundType); - - DatasetFieldCompoundValue currentCV = new DatasetFieldCompoundValue(); - currentCV.setParentDatasetField(currentField); - - DatasetField currentChildText = new DatasetField(); - currentChildText.setDatasetFieldType(childTextType); - currentChildText.setParentDatasetFieldCompoundValue(currentCV); - currentChildText.setSingleValue("SameValue"); - - DatasetField currentChildCV = new DatasetField(); - currentChildCV.setDatasetFieldType(childCVType); - currentChildCV.setParentDatasetFieldCompoundValue(currentCV); - currentChildCV.setControlledVocabularyValues(List.of(cvv)); - - currentCV.setChildDatasetFields(List.of(currentChildText, currentChildCV)); - currentField.setDatasetFieldCompoundValues(List.of(currentCV)); - - // Create prior field with same compound value - DatasetField priorField = new DatasetField(); - priorField.setDatasetFieldType(compoundType); - - DatasetFieldCompoundValue priorCV = new DatasetFieldCompoundValue(); - priorCV.setParentDatasetField(priorField); - - DatasetField priorChildText = new DatasetField(); - priorChildText.setDatasetFieldType(childTextType); - priorChildText.setParentDatasetFieldCompoundValue(priorCV); - priorChildText.setSingleValue("SameValue"); - - DatasetField priorChildCV = new DatasetField(); - priorChildCV.setDatasetFieldType(childCVType); - priorChildCV.setParentDatasetFieldCompoundValue(priorCV); - priorChildCV.setControlledVocabularyValues(List.of(cvv)); - - priorCV.setChildDatasetFields(List.of(priorChildText, priorChildCV)); - priorField.setDatasetFieldCompoundValues(List.of(priorCV)); - - // Filter - DatasetField filtered = invokeFilterNewValues(currentField, priorField); - - // No compound values should be included since nothing changed - assertEquals(0, filtered.getDatasetFieldCompoundValues().size()); - assertTrue(filtered.isEmpty()); -} + DatasetField currentChildCV = new DatasetField(); + currentChildCV.setDatasetFieldType(childCVType); + currentChildCV.setParentDatasetFieldCompoundValue(currentCV); + currentChildCV.setControlledVocabularyValues(List.of(cvv)); + + currentCV.setChildDatasetFields(List.of(currentChildText, currentChildCV)); + currentField.setDatasetFieldCompoundValues(List.of(currentCV)); + + // Create prior field with same compound value + DatasetField priorField = new DatasetField(); + priorField.setDatasetFieldType(compoundType); + + DatasetFieldCompoundValue priorCV = new DatasetFieldCompoundValue(); + priorCV.setParentDatasetField(priorField); + + DatasetField priorChildText = new DatasetField(); + priorChildText.setDatasetFieldType(childTextType); + priorChildText.setParentDatasetFieldCompoundValue(priorCV); + priorChildText.setSingleValue("SameValue"); + + DatasetField priorChildCV = new DatasetField(); + priorChildCV.setDatasetFieldType(childCVType); + priorChildCV.setParentDatasetFieldCompoundValue(priorCV); + priorChildCV.setControlledVocabularyValues(List.of(cvv)); + + priorCV.setChildDatasetFields(List.of(priorChildText, priorChildCV)); + priorField.setDatasetFieldCompoundValues(List.of(priorCV)); + + // Filter + DatasetField filtered = invokeFilterNewValues(currentField, priorField); + + // No compound values should be included since nothing changed + assertEquals(0, filtered.getDatasetFieldCompoundValues().size()); + assertTrue(filtered.isEmpty()); + } @Test void testFilterNewValues_CompoundWithPrimitiveChild_AllNew() throws Exception { @@ -711,73 +713,73 @@ void testFilterNewValues_CompoundWithPrimitiveChild_AllNew() throws Exception { compoundType.setName("testCompoundWithPrimitive"); compoundType.setAllowMultiples(true); compoundType.setFieldType(DatasetFieldType.FieldType.NONE); - + DatasetFieldType childTextField = new DatasetFieldType(); childTextField.setId(2L); childTextField.setName("childText"); childTextField.setFieldType(DatasetFieldType.FieldType.TEXT); childTextField.setParentDatasetFieldType(compoundType); childTextField.setChildDatasetFieldTypes(new ArrayList<>()); - + DatasetFieldType childIntField = new DatasetFieldType(); childIntField.setId(3L); childIntField.setName("childInt"); childIntField.setFieldType(DatasetFieldType.FieldType.INT); childIntField.setParentDatasetFieldType(compoundType); childIntField.setChildDatasetFieldTypes(new ArrayList<>()); - + compoundType.setChildDatasetFieldTypes(List.of(childTextField, childIntField)); - + // Create current field with 2 compound values DatasetField currentField = new DatasetField(); currentField.setDatasetFieldType(compoundType); - + List compoundValues = new ArrayList<>(); - + // First compound value DatasetFieldCompoundValue cv1 = new DatasetFieldCompoundValue(); cv1.setParentDatasetField(currentField); - + DatasetField child1Text = new DatasetField(); child1Text.setDatasetFieldType(childTextField); child1Text.setParentDatasetFieldCompoundValue(cv1); child1Text.setSingleValue("Text1"); - + DatasetField child1Int = new DatasetField(); child1Int.setDatasetFieldType(childIntField); child1Int.setParentDatasetFieldCompoundValue(cv1); child1Int.setSingleValue("123"); - + cv1.setChildDatasetFields(List.of(child1Text, child1Int)); compoundValues.add(cv1); - + // Second compound value DatasetFieldCompoundValue cv2 = new DatasetFieldCompoundValue(); cv2.setParentDatasetField(currentField); - + DatasetField child2Text = new DatasetField(); child2Text.setDatasetFieldType(childTextField); child2Text.setParentDatasetFieldCompoundValue(cv2); child2Text.setSingleValue("Text2"); - + DatasetField child2Int = new DatasetField(); child2Int.setDatasetFieldType(childIntField); child2Int.setParentDatasetFieldCompoundValue(cv2); child2Int.setSingleValue("456"); - + cv2.setChildDatasetFields(List.of(child2Text, child2Int)); compoundValues.add(cv2); - + currentField.setDatasetFieldCompoundValues(compoundValues); - + // Create prior field with no values DatasetField priorField = new DatasetField(); priorField.setDatasetFieldType(compoundType); priorField.setDatasetFieldCompoundValues(new ArrayList<>()); - + // Filter DatasetField filtered = invokeFilterNewValues(currentField, priorField); - + // All compound values should be included assertEquals(2, filtered.getDatasetFieldCompoundValues().size()); } @@ -790,116 +792,116 @@ void testFilterNewValues_CompoundWithPrimitiveChild_SomeNew() throws Exception { compoundType.setName("testCompoundWithPrimitive"); compoundType.setAllowMultiples(true); compoundType.setFieldType(DatasetFieldType.FieldType.NONE); - + DatasetFieldType childTextField = new DatasetFieldType(); childTextField.setId(2L); childTextField.setName("childText"); childTextField.setFieldType(DatasetFieldType.FieldType.TEXT); childTextField.setParentDatasetFieldType(compoundType); childTextField.setChildDatasetFieldTypes(new ArrayList<>()); - + DatasetFieldType childIntField = new DatasetFieldType(); childIntField.setId(3L); childIntField.setName("childInt"); childIntField.setFieldType(DatasetFieldType.FieldType.INT); childIntField.setParentDatasetFieldType(compoundType); childIntField.setChildDatasetFieldTypes(new ArrayList<>()); - + compoundType.setChildDatasetFieldTypes(List.of(childTextField, childIntField)); - + // Create current field with 3 compound values DatasetField currentField = new DatasetField(); currentField.setDatasetFieldType(compoundType); - + List currentCompoundValues = new ArrayList<>(); - + // First compound value (existing) DatasetFieldCompoundValue cv1 = new DatasetFieldCompoundValue(); cv1.setParentDatasetField(currentField); - + DatasetField child1Text = new DatasetField(); child1Text.setDatasetFieldType(childTextField); child1Text.setParentDatasetFieldCompoundValue(cv1); child1Text.setSingleValue("Text1"); - + DatasetField child1Int = new DatasetField(); child1Int.setDatasetFieldType(childIntField); child1Int.setParentDatasetFieldCompoundValue(cv1); child1Int.setSingleValue("123"); - + cv1.setChildDatasetFields(List.of(child1Text, child1Int)); currentCompoundValues.add(cv1); - + // Second compound value (new) DatasetFieldCompoundValue cv2 = new DatasetFieldCompoundValue(); cv2.setParentDatasetField(currentField); - + DatasetField child2Text = new DatasetField(); child2Text.setDatasetFieldType(childTextField); child2Text.setParentDatasetFieldCompoundValue(cv2); child2Text.setSingleValue("Text2"); - + DatasetField child2Int = new DatasetField(); child2Int.setDatasetFieldType(childIntField); child2Int.setParentDatasetFieldCompoundValue(cv2); child2Int.setSingleValue("456"); - + cv2.setChildDatasetFields(List.of(child2Text, child2Int)); currentCompoundValues.add(cv2); - + // Third compound value (new) DatasetFieldCompoundValue cv3 = new DatasetFieldCompoundValue(); cv3.setParentDatasetField(currentField); - + DatasetField child3Text = new DatasetField(); child3Text.setDatasetFieldType(childTextField); child3Text.setParentDatasetFieldCompoundValue(cv3); child3Text.setSingleValue("Text3"); - + DatasetField child3Int = new DatasetField(); child3Int.setDatasetFieldType(childIntField); child3Int.setParentDatasetFieldCompoundValue(cv3); child3Int.setSingleValue("789"); - + cv3.setChildDatasetFields(List.of(child3Text, child3Int)); currentCompoundValues.add(cv3); - + currentField.setDatasetFieldCompoundValues(currentCompoundValues); - + // Create prior field with 1 existing compound value DatasetField priorField = new DatasetField(); priorField.setDatasetFieldType(compoundType); - + List priorCompoundValues = new ArrayList<>(); - + DatasetFieldCompoundValue priorCv1 = new DatasetFieldCompoundValue(); priorCv1.setParentDatasetField(priorField); - + DatasetField priorChild1Text = new DatasetField(); priorChild1Text.setDatasetFieldType(childTextField); priorChild1Text.setParentDatasetFieldCompoundValue(priorCv1); priorChild1Text.setSingleValue("Text1"); - + DatasetField priorChild1Int = new DatasetField(); priorChild1Int.setDatasetFieldType(childIntField); priorChild1Int.setParentDatasetFieldCompoundValue(priorCv1); priorChild1Int.setSingleValue("123"); - + priorCv1.setChildDatasetFields(List.of(priorChild1Text, priorChild1Int)); priorCompoundValues.add(priorCv1); - + priorField.setDatasetFieldCompoundValues(priorCompoundValues); - + // Filter DatasetField filtered = invokeFilterNewValues(currentField, priorField); - + // Only 2 new compound values should be included assertEquals(2, filtered.getDatasetFieldCompoundValues().size()); - + // Verify the new values are present boolean hasText2 = false; boolean hasText3 = false; - + for (DatasetFieldCompoundValue cv : filtered.getDatasetFieldCompoundValues()) { for (DatasetField childField : cv.getChildDatasetFields()) { if (childField.getDatasetFieldType().equals(childTextField)) { @@ -912,7 +914,7 @@ void testFilterNewValues_CompoundWithPrimitiveChild_SomeNew() throws Exception { } } } - + assertTrue(hasText2); assertTrue(hasText3); } @@ -925,110 +927,109 @@ void testFilterNewValues_CompoundWithPrimitiveChild_NoneNew() throws Exception { compoundType.setName("testCompoundWithPrimitive"); compoundType.setAllowMultiples(true); compoundType.setFieldType(DatasetFieldType.FieldType.NONE); - + DatasetFieldType childTextField = new DatasetFieldType(); childTextField.setId(2L); childTextField.setName("childText"); childTextField.setFieldType(DatasetFieldType.FieldType.TEXT); childTextField.setParentDatasetFieldType(compoundType); childTextField.setChildDatasetFieldTypes(new ArrayList<>()); - + DatasetFieldType childIntField = new DatasetFieldType(); childIntField.setId(3L); childIntField.setName("childInt"); childIntField.setFieldType(DatasetFieldType.FieldType.INT); childIntField.setParentDatasetFieldType(compoundType); childIntField.setChildDatasetFieldTypes(new ArrayList<>()); - + compoundType.setChildDatasetFieldTypes(List.of(childTextField, childIntField)); - + // Create current field with 2 compound values DatasetField currentField = new DatasetField(); currentField.setDatasetFieldType(compoundType); - + List currentCompoundValues = new ArrayList<>(); - + // First compound value DatasetFieldCompoundValue cv1 = new DatasetFieldCompoundValue(); cv1.setParentDatasetField(currentField); - + DatasetField child1Text = new DatasetField(); child1Text.setDatasetFieldType(childTextField); child1Text.setParentDatasetFieldCompoundValue(cv1); child1Text.setSingleValue("Text1"); - + DatasetField child1Int = new DatasetField(); child1Int.setDatasetFieldType(childIntField); child1Int.setParentDatasetFieldCompoundValue(cv1); child1Int.setSingleValue("123"); - + cv1.setChildDatasetFields(List.of(child1Text, child1Int)); currentCompoundValues.add(cv1); - + // Second compound value DatasetFieldCompoundValue cv2 = new DatasetFieldCompoundValue(); cv2.setParentDatasetField(currentField); - + DatasetField child2Text = new DatasetField(); child2Text.setDatasetFieldType(childTextField); child2Text.setParentDatasetFieldCompoundValue(cv2); child2Text.setSingleValue("Text2"); - + DatasetField child2Int = new DatasetField(); child2Int.setDatasetFieldType(childIntField); child2Int.setParentDatasetFieldCompoundValue(cv2); child2Int.setSingleValue("456"); - + cv2.setChildDatasetFields(List.of(child2Text, child2Int)); currentCompoundValues.add(cv2); - + currentField.setDatasetFieldCompoundValues(currentCompoundValues); - + // Create prior field with same compound values DatasetField priorField = new DatasetField(); priorField.setDatasetFieldType(compoundType); - + List priorCompoundValues = new ArrayList<>(); - + // First compound value (same as current) DatasetFieldCompoundValue priorCv1 = new DatasetFieldCompoundValue(); priorCv1.setParentDatasetField(priorField); - + DatasetField priorChild1Text = new DatasetField(); priorChild1Text.setDatasetFieldType(childTextField); priorChild1Text.setParentDatasetFieldCompoundValue(priorCv1); priorChild1Text.setSingleValue("Text1"); - + DatasetField priorChild1Int = new DatasetField(); priorChild1Int.setDatasetFieldType(childIntField); priorChild1Int.setParentDatasetFieldCompoundValue(priorCv1); priorChild1Int.setSingleValue("123"); - + priorCv1.setChildDatasetFields(List.of(priorChild1Text, priorChild1Int)); priorCompoundValues.add(priorCv1); - + // Second compound value (same as current) DatasetFieldCompoundValue priorCv2 = new DatasetFieldCompoundValue(); priorCv2.setParentDatasetField(priorField); - + DatasetField priorChild2Text = new DatasetField(); priorChild2Text.setDatasetFieldType(childTextField); priorChild2Text.setParentDatasetFieldCompoundValue(priorCv2); priorChild2Text.setSingleValue("Text2"); - + DatasetField priorChild2Int = new DatasetField(); priorChild2Int.setDatasetFieldType(childIntField); priorChild2Int.setParentDatasetFieldCompoundValue(priorCv2); priorChild2Int.setSingleValue("456"); - + priorCv2.setChildDatasetFields(List.of(priorChild2Text, priorChild2Int)); priorCompoundValues.add(priorCv2); - + priorField.setDatasetFieldCompoundValues(priorCompoundValues); - + // Filter DatasetField filtered = invokeFilterNewValues(currentField, priorField); - // No compound values should be included assertEquals(0, filtered.getDatasetFieldCompoundValues().size()); @@ -1040,7 +1041,7 @@ void testFilterNewValues_CompoundWithPrimitiveChild_NoneNew() throws Exception { private DatasetField createMultiValueField(DatasetFieldType fieldType, String... values) { DatasetField field = new DatasetField(); field.setDatasetFieldType(fieldType); - + List fieldValues = new ArrayList<>(); for (String value : values) { DatasetFieldValue dfv = new DatasetFieldValue(); @@ -1049,7 +1050,7 @@ private DatasetField createMultiValueField(DatasetFieldType fieldType, String... fieldValues.add(dfv); } field.setDatasetFieldValues(fieldValues); - + return field; } @@ -1068,19 +1069,19 @@ private boolean containsValue(DatasetField field, String value) { } return false; } - + private DatasetField createCompoundField(DatasetFieldType fieldType, String[]... compoundValues) { DatasetField field = new DatasetField(); field.setDatasetFieldType(fieldType); - + List compoundValueList = new ArrayList<>(); for (String[] values : compoundValues) { DatasetFieldCompoundValue compoundValue = new DatasetFieldCompoundValue(); compoundValue.setParentDatasetField(field); - + List childFields = new ArrayList<>(); List childTypes = new ArrayList<>(fieldType.getChildDatasetFieldTypes()); - + // Create child fields based on the parent's child types and provided values for (int i = 0; i < Math.min(values.length, childTypes.size()); i++) { DatasetField childField = new DatasetField(); @@ -1089,11 +1090,11 @@ private DatasetField createCompoundField(DatasetFieldType fieldType, String[]... childField.setSingleValue(values[i]); childFields.add(childField); } - + compoundValue.setChildDatasetFields(childFields); compoundValueList.add(compoundValue); } - + field.setDatasetFieldCompoundValues(compoundValueList); return field; } @@ -1101,11 +1102,11 @@ private DatasetField createCompoundField(DatasetFieldType fieldType, String[]... private boolean containsCompoundValue(DatasetField field, String... childValues) { for (DatasetFieldCompoundValue cv : field.getDatasetFieldCompoundValues()) { List cvValues = new ArrayList<>(); - + for (DatasetField childField : cv.getChildDatasetFields()) { cvValues.add(childField.getDisplayValue()); } - + // Check if all provided values are present in this compound value boolean allMatch = true; for (String value : childValues) { @@ -1114,19 +1115,19 @@ private boolean containsCompoundValue(DatasetField field, String... childValues) break; } } - + if (allMatch && cvValues.size() == childValues.length) { return true; } } return false; } - + private boolean containsControlledVocabValue(DatasetField field, String strValue) { if (field.getControlledVocabularyValues() == null) { return false; } - + for (ControlledVocabularyValue cvv : field.getControlledVocabularyValues()) { if (cvv.getStrValue().equals(strValue)) { return true; @@ -1134,7 +1135,7 @@ private boolean containsControlledVocabValue(DatasetField field, String strValue } return false; } - + /** * Use reflection to invoke the private filterNewValues method */ From 28c728c0ca9cd98a775aec22bba26c7314805a5f Mon Sep 17 00:00:00 2001 From: qqmyers Date: Thu, 20 Nov 2025 13:11:29 -0500 Subject: [PATCH 40/40] typo --- doc/release-notes/10490-COAR-Notify.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/release-notes/10490-COAR-Notify.md b/doc/release-notes/10490-COAR-Notify.md index 06fe6774fcb..fcc461370b1 100644 --- a/doc/release-notes/10490-COAR-Notify.md +++ b/doc/release-notes/10490-COAR-Notify.md @@ -19,5 +19,5 @@ DATAVERSE_LDN_ALLOWED_HOSTS (dataverse.ldn.allowed-hosts) Notifications are sent by default to users who can publish a dataset. The option below can be used to restrict notifications to superusers who can publish the dataset. -DATAVERSE_COAR_NOTOIFY_RELATIONSHIP_ANNOUNCEMENT_NOTIFY_SUPERSUSERS_ONLY +DATAVERSE_COAR_NOTIFY_RELATIONSHIP_ANNOUNCEMENT_NOTIFY_SUPERSUSERS_ONLY