workflow-curation
@@ -341,6 +341,14 @@
org.apache.logging.log4j
log4j-api
+
+ org.apache.logging.log4j
+ log4j-core
+
+
+ org.apache.logging.log4j
+ log4j-slf4j2-impl
+
org.hibernate.orm
hibernate-core
@@ -388,6 +396,13 @@
org.springframework
spring-orm
+
+
+
+ org.springframework
+ spring-jcl
+
+
@@ -406,6 +421,16 @@
org.mortbay.jasper
apache-jsp
+
+
+ org.bouncycastle
+ bcpkix-jdk15on
+
+
+ org.bouncycastle
+ bcprov-jdk15on
+
@@ -475,10 +500,6 @@
jakarta.annotation
jakarta.annotation-api
-
- jakarta.el
- jakarta.el-api
-
jaxen
jaxen
@@ -623,7 +644,7 @@
dnsjava
dnsjava
- 2.1.9
+ 3.6.3
@@ -667,28 +688,6 @@
${flyway.version}
-
-
- com.google.apis
- google-api-services-analytics
-
-
- com.google.api-client
- google-api-client
-
-
- com.google.http-client
- google-http-client
-
-
- com.google.http-client
- google-http-client-jackson2
-
-
- com.google.oauth-client
- google-oauth-client
-
-
com.google.code.findbugs
@@ -702,7 +701,6 @@
jakarta.inject
jakarta.inject-api
- 2.0.1
@@ -733,7 +731,7 @@
com.amazonaws
aws-java-sdk-s3
- 1.12.261
+ 1.12.785
+
+ org.yaml
+ snakeyaml
+
@@ -769,25 +772,27 @@
com.opencsv
opencsv
- 5.9
+ 5.11.1
org.apache.velocity
velocity-engine-core
+ 2.4.1
org.xmlunit
xmlunit-core
+ 2.10.2
test
org.apache.bcel
bcel
- 6.7.0
+ 6.10.0
test
@@ -814,7 +819,7 @@
org.mock-server
mockserver-junit-rule
- 5.11.2
+ 5.15.0
test
@@ -855,76 +860,11 @@
-
-
-
-
-
-
- io.netty
- netty-buffer
- 4.1.106.Final
-
-
- io.netty
- netty-transport
- 4.1.106.Final
-
-
- io.netty
- netty-transport-native-unix-common
- 4.1.106.Final
-
-
- io.netty
- netty-common
- 4.1.106.Final
-
-
- io.netty
- netty-handler
- 4.1.106.Final
-
-
- io.netty
- netty-codec
- 4.1.106.Final
-
-
- org.apache.velocity
- velocity-engine-core
- 2.3
-
-
- org.xmlunit
- xmlunit-core
- 2.10.0
- test
-
-
- com.github.java-json-tools
- json-schema-validator
- 2.2.14
-
-
- jakarta.validation
- jakarta.validation-api
- 3.0.2
-
-
- io.swagger
- swagger-core
- 1.6.2
-
-
- org.scala-lang
- scala-library
- 2.13.11
+
+ com.squareup.okhttp3
+ mockwebserver
test
-
-
-
-
+
+
diff --git a/dspace-api/src/main/java/org/dspace/access/status/DefaultAccessStatusHelper.java b/dspace-api/src/main/java/org/dspace/access/status/DefaultAccessStatusHelper.java
index 5f0e6d8b259b..52cdec3517bc 100644
--- a/dspace-api/src/main/java/org/dspace/access/status/DefaultAccessStatusHelper.java
+++ b/dspace-api/src/main/java/org/dspace/access/status/DefaultAccessStatusHelper.java
@@ -8,6 +8,7 @@
package org.dspace.access.status;
import java.sql.SQLException;
+import java.time.Instant;
import java.util.Date;
import java.util.List;
import java.util.Objects;
@@ -26,7 +27,6 @@
import org.dspace.core.Constants;
import org.dspace.core.Context;
import org.dspace.eperson.Group;
-import org.joda.time.LocalDate;
/**
* Default plugin implementation of the access status helper.
@@ -230,7 +230,7 @@ private Date retrieveShortestEmbargo(Context context, Bitstream bitstream) throw
// If the policy is not valid there is an active embargo
Date startDate = policy.getStartDate();
- if (startDate != null && !startDate.before(LocalDate.now().toDate())) {
+ if (startDate != null && !startDate.before(Date.from(Instant.now()))) {
// There is an active embargo: aim to take the shortest embargo (account for rare cases where
// more than one resource policy exists)
if (embargoDate == null) {
diff --git a/dspace-api/src/main/java/org/dspace/administer/RegistryImporter.java b/dspace-api/src/main/java/org/dspace/administer/RegistryImporter.java
index 27a653421312..c74e56bce890 100644
--- a/dspace-api/src/main/java/org/dspace/administer/RegistryImporter.java
+++ b/dspace-api/src/main/java/org/dspace/administer/RegistryImporter.java
@@ -10,7 +10,6 @@
import java.io.File;
import java.io.IOException;
import javax.xml.parsers.DocumentBuilder;
-import javax.xml.parsers.DocumentBuilderFactory;
import javax.xml.parsers.ParserConfigurationException;
import javax.xml.transform.TransformerException;
import javax.xml.xpath.XPath;
@@ -18,6 +17,7 @@
import javax.xml.xpath.XPathExpressionException;
import javax.xml.xpath.XPathFactory;
+import org.dspace.app.util.XMLUtils;
import org.w3c.dom.Document;
import org.w3c.dom.Node;
import org.w3c.dom.NodeList;
@@ -49,8 +49,9 @@ private RegistryImporter() { }
*/
public static Document loadXML(String filename)
throws IOException, ParserConfigurationException, SAXException {
- DocumentBuilder builder = DocumentBuilderFactory.newInstance()
- .newDocumentBuilder();
+ // This XML builder will *not* disable external entities as XML
+ // registries are considered trusted content
+ DocumentBuilder builder = XMLUtils.getTrustedDocumentBuilder();
Document document = builder.parse(new File(filename));
diff --git a/dspace-api/src/main/java/org/dspace/administer/RegistryLoader.java b/dspace-api/src/main/java/org/dspace/administer/RegistryLoader.java
index bbf320a0d5e5..8bb72e18521e 100644
--- a/dspace-api/src/main/java/org/dspace/administer/RegistryLoader.java
+++ b/dspace-api/src/main/java/org/dspace/administer/RegistryLoader.java
@@ -13,7 +13,6 @@
import java.util.ArrayList;
import java.util.Arrays;
import javax.xml.parsers.DocumentBuilder;
-import javax.xml.parsers.DocumentBuilderFactory;
import javax.xml.parsers.ParserConfigurationException;
import javax.xml.transform.TransformerException;
import javax.xml.xpath.XPath;
@@ -21,7 +20,15 @@
import javax.xml.xpath.XPathExpressionException;
import javax.xml.xpath.XPathFactory;
+import org.apache.commons.cli.CommandLine;
+import org.apache.commons.cli.CommandLineParser;
+import org.apache.commons.cli.DefaultParser;
+import org.apache.commons.cli.HelpFormatter;
+import org.apache.commons.cli.Options;
+import org.apache.commons.cli.ParseException;
+import org.apache.commons.lang3.StringUtils;
import org.apache.logging.log4j.Logger;
+import org.dspace.app.util.XMLUtils;
import org.dspace.authorize.AuthorizeException;
import org.dspace.content.BitstreamFormat;
import org.dspace.content.factory.ContentServiceFactory;
@@ -41,7 +48,7 @@
*
* RegistryLoader -bitstream bitstream-formats.xml
*
- * RegistryLoader -dc dc-types.xml
+ * RegistryLoader -metadata dc-types.xml
*
* @author Robert Tansley
* @version $Revision$
@@ -50,7 +57,7 @@ public class RegistryLoader {
/**
* log4j category
*/
- private static Logger log = org.apache.logging.log4j.LogManager.getLogger(RegistryLoader.class);
+ private static final Logger log = org.apache.logging.log4j.LogManager.getLogger(RegistryLoader.class);
protected static BitstreamFormatService bitstreamFormatService = ContentServiceFactory.getInstance()
.getBitstreamFormatService();
@@ -67,48 +74,97 @@ private RegistryLoader() { }
* @throws Exception if error
*/
public static void main(String[] argv) throws Exception {
- String usage = "Usage: " + RegistryLoader.class.getName()
- + " (-bitstream | -metadata) registry-file.xml";
-
- Context context = null;
+ // Set up command-line options and parse arguments
+ CommandLineParser parser = new DefaultParser();
+ Options options = createCommandLineOptions();
try {
- context = new Context();
+ CommandLine line = parser.parse(options, argv);
+
+ // Check if help option was entered or no options provided
+ if (line.hasOption('h') || line.getOptions().length == 0) {
+ printHelp(options);
+ System.exit(0);
+ }
+
+ Context context = new Context();
// Can't update registries anonymously, so we need to turn off
// authorisation
context.turnOffAuthorisationSystem();
- // Work out what we're loading
- if (argv[0].equalsIgnoreCase("-bitstream")) {
- RegistryLoader.loadBitstreamFormats(context, argv[1]);
- } else if (argv[0].equalsIgnoreCase("-metadata")) {
- // Call MetadataImporter, as it handles Metadata schema updates
- MetadataImporter.loadRegistry(argv[1], true);
- } else {
- System.err.println(usage);
+ try {
+ // Work out what we're loading
+ if (line.hasOption('b')) {
+ String filename = line.getOptionValue('b');
+ if (StringUtils.isEmpty(filename)) {
+ System.err.println("No file path provided for bitstream format registry");
+ printHelp(options);
+ System.exit(1);
+ }
+ RegistryLoader.loadBitstreamFormats(context, filename);
+ } else if (line.hasOption('m')) {
+ String filename = line.getOptionValue('m');
+ if (StringUtils.isEmpty(filename)) {
+ System.err.println("No file path provided for metadata registry");
+ printHelp(options);
+ System.exit(1);
+ }
+ // Call MetadataImporter, as it handles Metadata schema updates
+ MetadataImporter.loadRegistry(filename, true);
+ } else {
+ System.err.println("No registry type specified");
+ printHelp(options);
+ System.exit(1);
+ }
+
+ // Commit changes and close Context
+ context.complete();
+ System.exit(0);
+ } catch (Exception e) {
+ log.fatal(LogHelper.getHeader(context, "error_loading_registries", ""), e);
+ System.err.println("Error: \n - " + e.getMessage());
+ System.exit(1);
+ } finally {
+ // Clean up our context, if it still exists & it was never completed
+ if (context != null && context.isValid()) {
+ context.abort();
+ }
}
+ } catch (ParseException e) {
+ System.err.println("Error parsing command-line arguments: " + e.getMessage());
+ printHelp(options);
+ System.exit(1);
+ }
+ }
- // Commit changes and close Context
- context.complete();
+ /**
+ * Create the command-line options
+ * @return the command-line options
+ */
+ private static Options createCommandLineOptions() {
+ Options options = new Options();
- System.exit(0);
- } catch (ArrayIndexOutOfBoundsException ae) {
- System.err.println(usage);
+ options.addOption("b", "bitstream", true, "load bitstream format registry from specified file");
+ options.addOption("m", "metadata", true, "load metadata registry from specified file");
+ options.addOption("h", "help", false, "print this help message");
- System.exit(1);
- } catch (Exception e) {
- log.fatal(LogHelper.getHeader(context, "error_loading_registries",
- ""), e);
+ return options;
+ }
- System.err.println("Error: \n - " + e.getMessage());
- System.exit(1);
- } finally {
- // Clean up our context, if it still exists & it was never completed
- if (context != null && context.isValid()) {
- context.abort();
- }
- }
+ /**
+ * Print the help message
+ * @param options the command-line options
+ */
+ private static void printHelp(Options options) {
+ HelpFormatter formatter = new HelpFormatter();
+ formatter.printHelp("RegistryLoader",
+ "Load bitstream format or metadata registries into the database\n",
+ options,
+ "\nExamples:\n" +
+ " RegistryLoader -b bitstream-formats.xml\n" +
+ " RegistryLoader -m dc-types.xml",
+ true);
}
/**
@@ -210,8 +266,9 @@ private static void loadFormat(Context context, Node node)
*/
private static Document loadXML(String filename) throws IOException,
ParserConfigurationException, SAXException {
- DocumentBuilder builder = DocumentBuilderFactory.newInstance()
- .newDocumentBuilder();
+ // This XML builder will *not* disable external entities as XML
+ // registries are considered trusted content
+ DocumentBuilder builder = XMLUtils.getTrustedDocumentBuilder();
return builder.parse(new File(filename));
}
@@ -221,7 +278,7 @@ private static Document loadXML(String filename) throws IOException,
* contains:
*
*
- * <foo><mimetype>application/pdf</mimetype></foo>
+ * application/pdf
*
* passing this the foo node and mimetype will
* return application/pdf.
@@ -262,10 +319,10 @@ private static String getElementData(Node parentElement, String childName)
* document contains:
*
*
- * <foo>
- * <bar>val1</bar>
- * <bar>val2</bar>
- * </foo>
+ *
+ * val1
+ * val2
+ *
*
* passing this the foo node and bar will
* return val1 and val2.
diff --git a/dspace-api/src/main/java/org/dspace/administer/StructBuilder.java b/dspace-api/src/main/java/org/dspace/administer/StructBuilder.java
index 13a1b3b5bbf8..f2577a37b176 100644
--- a/dspace-api/src/main/java/org/dspace/administer/StructBuilder.java
+++ b/dspace-api/src/main/java/org/dspace/administer/StructBuilder.java
@@ -27,7 +27,6 @@
import java.util.List;
import java.util.Map;
import javax.xml.parsers.DocumentBuilder;
-import javax.xml.parsers.DocumentBuilderFactory;
import javax.xml.parsers.ParserConfigurationException;
import javax.xml.transform.TransformerException;
import javax.xml.xpath.XPath;
@@ -43,6 +42,7 @@
import org.apache.commons.cli.Options;
import org.apache.commons.cli.ParseException;
import org.apache.commons.lang3.StringUtils;
+import org.dspace.app.util.XMLUtils;
import org.dspace.authorize.AuthorizeException;
import org.dspace.content.Collection;
import org.dspace.content.Community;
@@ -613,8 +613,8 @@ private static String validateCollections(NodeList collections, int level)
*/
private static org.w3c.dom.Document loadXML(InputStream input)
throws IOException, ParserConfigurationException, SAXException {
- DocumentBuilder builder = DocumentBuilderFactory.newInstance()
- .newDocumentBuilder();
+ // This builder factory does not disable external DTD, entities, etc.
+ DocumentBuilder builder = XMLUtils.getTrustedDocumentBuilder();
org.w3c.dom.Document document = builder.parse(input);
@@ -802,7 +802,7 @@ private static Element[] handleCollections(Context context,
// default the short description to the empty string
collectionService.setMetadataSingleValue(context, collection,
- MD_SHORT_DESCRIPTION, Item.ANY, " ");
+ MD_SHORT_DESCRIPTION, null, " ");
// import the rest of the metadata
for (Map.Entry entry : collectionMap.entrySet()) {
diff --git a/dspace-api/src/main/java/org/dspace/app/bulkaccesscontrol/BulkAccessControl.java b/dspace-api/src/main/java/org/dspace/app/bulkaccesscontrol/BulkAccessControl.java
index 7bef232f0450..30f68efaf3cb 100644
--- a/dspace-api/src/main/java/org/dspace/app/bulkaccesscontrol/BulkAccessControl.java
+++ b/dspace-api/src/main/java/org/dspace/app/bulkaccesscontrol/BulkAccessControl.java
@@ -416,7 +416,7 @@ private DiscoverQuery buildDiscoveryQuery(String query, int start, int limit) {
discoverQuery.setQuery(query);
discoverQuery.setStart(start);
discoverQuery.setMaxResults(limit);
-
+ discoverQuery.setSortField("search.resourceid", DiscoverQuery.SORT_ORDER.asc);
return discoverQuery;
}
diff --git a/dspace-api/src/main/java/org/dspace/app/bulkedit/DSpaceCSV.java b/dspace-api/src/main/java/org/dspace/app/bulkedit/DSpaceCSV.java
index cbc052b5573f..3533a2397b3d 100644
--- a/dspace-api/src/main/java/org/dspace/app/bulkedit/DSpaceCSV.java
+++ b/dspace-api/src/main/java/org/dspace/app/bulkedit/DSpaceCSV.java
@@ -188,6 +188,15 @@ public DSpaceCSV(InputStream inputStream, Context c) throws Exception {
// Verify that the heading is valid in the metadata registry
String[] clean = element.split("\\[");
String[] parts = clean[0].split("\\.");
+ // Check language if present, if it's ANY then throw an exception
+ if (clean.length > 1 && clean[1].equals(Item.ANY + "]")) {
+ throw new MetadataImportInvalidHeadingException("Language ANY (*) was found in the heading " +
+ "of the metadata value to import, " +
+ "this should never be the case",
+ MetadataImportInvalidHeadingException.ENTRY,
+ columnCounter);
+
+ }
if (parts.length < 2) {
throw new MetadataImportInvalidHeadingException(element,
@@ -223,6 +232,15 @@ public DSpaceCSV(InputStream inputStream, Context c) throws Exception {
}
}
+ // Verify there isn’t already a header that is the same; if it already exists,
+ // throw MetadataImportInvalidHeadingException
+ String header = authorityPrefix + element;
+ if (headings.contains(header)) {
+ throw new MetadataImportInvalidHeadingException("Duplicate heading found: " + header,
+ MetadataImportInvalidHeadingException.ENTRY,
+ columnCounter);
+ }
+
// Store the heading
headings.add(authorityPrefix + element);
}
diff --git a/dspace-api/src/main/java/org/dspace/app/bulkedit/MetadataExportSearch.java b/dspace-api/src/main/java/org/dspace/app/bulkedit/MetadataExportSearch.java
index 027ad116a7e2..e4bbe335d63e 100644
--- a/dspace-api/src/main/java/org/dspace/app/bulkedit/MetadataExportSearch.java
+++ b/dspace-api/src/main/java/org/dspace/app/bulkedit/MetadataExportSearch.java
@@ -143,7 +143,7 @@ public void internalRun() throws Exception {
Iterator- itemIterator = searchService.iteratorSearch(context, dso, discoverQuery);
handler.logDebug("creating dspacecsv");
- DSpaceCSV dSpaceCSV = metadataDSpaceCsvExportService.export(context, itemIterator, true);
+ DSpaceCSV dSpaceCSV = metadataDSpaceCsvExportService.export(context, itemIterator, true, handler);
handler.logDebug("writing to file " + getFileNameOrExportFile());
handler.writeFilestream(context, getFileNameOrExportFile(), dSpaceCSV.getInputStream(), EXPORT_CSV);
context.restoreAuthSystemState();
diff --git a/dspace-api/src/main/java/org/dspace/app/client/DSpaceHttpClientFactory.java b/dspace-api/src/main/java/org/dspace/app/client/DSpaceHttpClientFactory.java
new file mode 100644
index 000000000000..59c8172f722c
--- /dev/null
+++ b/dspace-api/src/main/java/org/dspace/app/client/DSpaceHttpClientFactory.java
@@ -0,0 +1,152 @@
+/**
+ * The contents of this file are subject to the license and copyright
+ * detailed in the LICENSE and NOTICE files at the root of the source
+ * tree and available online at
+ *
+ * http://www.dspace.org/license/
+ */
+package org.dspace.app.client;
+
+import static org.apache.commons.collections4.ListUtils.emptyIfNull;
+
+import java.util.List;
+
+import org.apache.http.HttpRequestInterceptor;
+import org.apache.http.HttpResponseInterceptor;
+import org.apache.http.client.HttpClient;
+import org.apache.http.client.config.RequestConfig;
+import org.apache.http.impl.client.CloseableHttpClient;
+import org.apache.http.impl.client.HttpClientBuilder;
+import org.dspace.services.ConfigurationService;
+import org.dspace.utils.DSpace;
+import org.springframework.beans.factory.annotation.Autowired;
+
+/**
+ * Factory of {@link HttpClient} with common configurations.
+ *
+ * @author Luca Giamminonni (luca.giamminonni at 4science.it)
+ *
+ */
+public class DSpaceHttpClientFactory {
+
+ @Autowired
+ private ConfigurationService configurationService;
+
+ @Autowired
+ private DSpaceProxyRoutePlanner proxyRoutePlanner;
+
+ @Autowired(required = false)
+ private List requestInterceptors;
+
+ @Autowired(required = false)
+ private List responseInterceptors;
+
+ /**
+ * Get an instance of {@link DSpaceHttpClientFactory} from the Spring context.
+ * @return the bean instance
+ */
+ public static DSpaceHttpClientFactory getInstance() {
+ return new DSpace().getSingletonService(DSpaceHttpClientFactory.class);
+ }
+
+ /**
+ * Build an instance of {@link HttpClient} setting the proxy if configured.
+ *
+ * @return the client
+ */
+ public CloseableHttpClient build() {
+ return build(HttpClientBuilder.create(), true);
+ }
+
+ /**
+ * return a Builder if an instance of {@link HttpClient} pre-setting the proxy if configured.
+ *
+ * @return the client
+ */
+ public HttpClientBuilder builder(boolean setProxy) {
+ HttpClientBuilder clientBuilder = HttpClientBuilder.create();
+ if (setProxy) {
+ clientBuilder.setRoutePlanner(proxyRoutePlanner);
+ }
+ getRequestInterceptors().forEach(clientBuilder::addInterceptorLast);
+ getResponseInterceptors().forEach(clientBuilder::addInterceptorLast);
+ return clientBuilder;
+ }
+
+ /**
+ * Build an instance of {@link HttpClient} without setting the proxy, even if
+ * configured.
+ *
+ * @return the client
+ */
+ public CloseableHttpClient buildWithoutProxy() {
+ return build(HttpClientBuilder.create(), false);
+ }
+
+ /**
+ * Build an instance of {@link HttpClient} setting the proxy if configured,
+ * disabling automatic retries and setting the maximum total connection.
+ *
+ * @param maxConnTotal the maximum total connection value
+ * @return the client
+ */
+ public CloseableHttpClient buildWithoutAutomaticRetries(int maxConnTotal) {
+ HttpClientBuilder clientBuilder = HttpClientBuilder.create()
+ .disableAutomaticRetries()
+ .setMaxConnTotal(maxConnTotal);
+ return build(clientBuilder, true);
+ }
+
+ /**
+ * Build an instance of {@link HttpClient} setting the proxy if configured with
+ * the given request configuration.
+ * @param requestConfig the request configuration
+ * @return the client
+ */
+ public CloseableHttpClient buildWithRequestConfig(RequestConfig requestConfig) {
+ HttpClientBuilder httpClientBuilder = HttpClientBuilder.create()
+ .setDefaultRequestConfig(requestConfig);
+ return build(httpClientBuilder, true);
+ }
+
+ private CloseableHttpClient build(HttpClientBuilder clientBuilder, boolean setProxy) {
+ if (setProxy) {
+ clientBuilder.setRoutePlanner(proxyRoutePlanner);
+ }
+ getRequestInterceptors().forEach(clientBuilder::addInterceptorLast);
+ getResponseInterceptors().forEach(clientBuilder::addInterceptorLast);
+ return clientBuilder.build();
+ }
+
+ public ConfigurationService getConfigurationService() {
+ return configurationService;
+ }
+
+ public void setConfigurationService(ConfigurationService configurationService) {
+ this.configurationService = configurationService;
+ }
+
+ public List getRequestInterceptors() {
+ return emptyIfNull(requestInterceptors);
+ }
+
+ public void setRequestInterceptors(List requestInterceptors) {
+ this.requestInterceptors = requestInterceptors;
+ }
+
+ public List getResponseInterceptors() {
+ return emptyIfNull(responseInterceptors);
+ }
+
+ public void setResponseInterceptors(List responseInterceptors) {
+ this.responseInterceptors = responseInterceptors;
+ }
+
+ public DSpaceProxyRoutePlanner getProxyRoutePlanner() {
+ return proxyRoutePlanner;
+ }
+
+ public void setProxyRoutePlanner(DSpaceProxyRoutePlanner proxyRoutePlanner) {
+ this.proxyRoutePlanner = proxyRoutePlanner;
+ }
+}
\ No newline at end of file
diff --git a/dspace-api/src/main/java/org/dspace/app/client/DSpaceProxyRoutePlanner.java b/dspace-api/src/main/java/org/dspace/app/client/DSpaceProxyRoutePlanner.java
new file mode 100644
index 000000000000..1df7fa4a2985
--- /dev/null
+++ b/dspace-api/src/main/java/org/dspace/app/client/DSpaceProxyRoutePlanner.java
@@ -0,0 +1,73 @@
+/**
+ * The contents of this file are subject to the license and copyright
+ * detailed in the LICENSE and NOTICE files at the root of the source
+ * tree and available online at
+ *
+ * http://www.dspace.org/license/
+ */
+package org.dspace.app.client;
+
+import java.util.Arrays;
+
+import org.apache.commons.lang3.ArrayUtils;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.http.HttpException;
+import org.apache.http.HttpHost;
+import org.apache.http.HttpRequest;
+import org.apache.http.impl.conn.DefaultRoutePlanner;
+import org.apache.http.protocol.HttpContext;
+import org.dspace.services.ConfigurationService;
+
+/**
+ * Extension of {@link DefaultRoutePlanner} that determine the proxy based on
+ * the configuration service, ignoring configured hosts.
+ *
+ * @author Luca Giamminonni (luca.giamminonni at 4science.it)
+ *
+ */
+public class DSpaceProxyRoutePlanner extends DefaultRoutePlanner {
+
+ private ConfigurationService configurationService;
+
+ public DSpaceProxyRoutePlanner(ConfigurationService configurationService) {
+ super(null);
+ this.configurationService = configurationService;
+ }
+
+ @Override
+ protected HttpHost determineProxy(HttpHost target, HttpRequest request, HttpContext context) throws HttpException {
+ if (isTargetHostConfiguredToBeIgnored(target)) {
+ return null;
+ }
+ String proxyHost = configurationService.getProperty("http.proxy.host");
+ String proxyPort = configurationService.getProperty("http.proxy.port");
+ if (StringUtils.isAnyBlank(proxyHost, proxyPort)) {
+ return null;
+ }
+ try {
+ return new HttpHost(proxyHost, Integer.parseInt(proxyPort), "http");
+ } catch (NumberFormatException e) {
+ throw new RuntimeException("Invalid proxy port configuration: " + proxyPort);
+ }
+ }
+
+ private boolean isTargetHostConfiguredToBeIgnored(HttpHost target) {
+ String[] hostsToIgnore = configurationService.getArrayProperty("http.proxy.hosts-to-ignore");
+ if (ArrayUtils.isEmpty(hostsToIgnore)) {
+ return false;
+ }
+ return Arrays.stream(hostsToIgnore)
+ .anyMatch(host -> matchesHost(host, target.getHostName()));
+ }
+
+ private boolean matchesHost(String hostPattern, String hostName) {
+ if (hostName.equals(hostPattern)) {
+ return true;
+ } else if (hostPattern.startsWith("*")) {
+ return hostName.endsWith(StringUtils.removeStart(hostPattern, "*"));
+ } else if (hostPattern.endsWith("*")) {
+ return hostName.startsWith(StringUtils.removeEnd(hostPattern, "*"));
+ }
+ return false;
+ }
+}
\ No newline at end of file
diff --git a/dspace-api/src/main/java/org/dspace/app/itemimport/ItemImportServiceImpl.java b/dspace-api/src/main/java/org/dspace/app/itemimport/ItemImportServiceImpl.java
index 087a33026151..7685ffc8f323 100644
--- a/dspace-api/src/main/java/org/dspace/app/itemimport/ItemImportServiceImpl.java
+++ b/dspace-api/src/main/java/org/dspace/app/itemimport/ItemImportServiceImpl.java
@@ -29,6 +29,7 @@
import java.io.OutputStream;
import java.io.PrintWriter;
import java.net.URL;
+import java.nio.file.Path;
import java.sql.SQLException;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
@@ -47,7 +48,6 @@
import java.util.zip.ZipEntry;
import java.util.zip.ZipFile;
import javax.xml.parsers.DocumentBuilder;
-import javax.xml.parsers.DocumentBuilderFactory;
import javax.xml.parsers.ParserConfigurationException;
import javax.xml.transform.TransformerException;
import javax.xml.xpath.XPath;
@@ -67,6 +67,7 @@
import org.dspace.app.itemimport.service.ItemImportService;
import org.dspace.app.util.LocalSchemaFilenameFilter;
import org.dspace.app.util.RelationshipUtils;
+import org.dspace.app.util.XMLUtils;
import org.dspace.authorize.AuthorizeException;
import org.dspace.authorize.ResourcePolicy;
import org.dspace.authorize.service.AuthorizeService;
@@ -179,6 +180,8 @@ public class ItemImportServiceImpl implements ItemImportService, InitializingBea
@Autowired(required = true)
protected MetadataValueService metadataValueService;
+ protected DocumentBuilder builder;
+
protected String tempWorkDir;
protected boolean isTest = false;
@@ -742,15 +745,22 @@ protected Item addItem(Context c, List mycollections, String path,
myitem = wi.getItem();
}
+ // normalize and validate path to make sure itemname doesn't contain path traversal
+ Path itemPath = new File(path + File.separatorChar + itemname + File.separatorChar)
+ .toPath().normalize();
+ if (!itemPath.startsWith(path)) {
+ throw new IOException("Illegal item metadata path: '" + itemPath);
+ }
+ // Normalization chops off the last separator, and we need to put it back
+ String itemPathDir = itemPath.toString() + File.separatorChar;
+
// now fill out dublin core for item
- loadMetadata(c, myitem, path + File.separatorChar + itemname
- + File.separatorChar);
+ loadMetadata(c, myitem, itemPathDir);
// and the bitstreams from the contents file
// process contents file, add bistreams and bundles, return any
// non-standard permissions
- List options = processContentsFile(c, myitem, path
- + File.separatorChar + itemname, "contents");
+ List options = processContentsFile(c, myitem, itemPathDir, "contents");
if (useWorkflow) {
// don't process handle file
@@ -768,8 +778,7 @@ protected Item addItem(Context c, List mycollections, String path,
}
} else {
// only process handle file if not using workflow system
- String myhandle = processHandleFile(c, myitem, path
- + File.separatorChar + itemname, "handle");
+ String myhandle = processHandleFile(c, myitem, itemPathDir, "handle");
// put item in system
if (!isTest) {
@@ -1001,6 +1010,34 @@ protected void addDCValue(Context c, Item i, String schema, Node n)
}
}
+ /**
+ * Ensures a file path does not attempt to access files outside the designated parent directory.
+ *
+ * @param parentDir The absolute path to the parent directory that should contain the file
+ * @param fileName The name or path of the file to validate
+ * @throws IOException If an error occurs while resolving canonical paths, or the file path attempts
+ * to access a location outside the parent directory
+ */
+ private void validateFilePath(String parentDir, String fileName) throws IOException {
+ File parent = new File(parentDir);
+ File file = new File(fileName);
+
+ // If the fileName is not an absolute path, we resolve it against the parentDir
+ if (!file.isAbsolute()) {
+ file = new File(parent, fileName);
+ }
+
+ String parentCanonicalPath = parent.getCanonicalPath();
+ String fileCanonicalPath = file.getCanonicalPath();
+
+ if (!fileCanonicalPath.startsWith(parentCanonicalPath)) {
+ log.error("File path outside of canonical root requested: fileCanonicalPath={} does not begin " +
+ "with parentCanonicalPath={}", fileCanonicalPath, parentCanonicalPath);
+ throw new IOException("Illegal file path '" + fileName + "' encountered. This references a path " +
+ "outside of the import package. Please see the system logs for more details.");
+ }
+ }
+
/**
* Read the collections file inside the item directory. If there
* is one and it is not empty return a list of collections in
@@ -1201,6 +1238,7 @@ protected List processContentsFile(Context c, Item i, String path,
sDescription = sDescription.replaceFirst("description:", "");
}
+ validateFilePath(path, sFilePath);
registerBitstream(c, i, iAssetstore, sFilePath, sBundle, sDescription);
logInfo("\tRegistering Bitstream: " + sFilePath
+ "\tAssetstore: " + iAssetstore
@@ -1414,6 +1452,7 @@ protected void processContentFileEntry(Context c, Item i, String path,
return;
}
+ validateFilePath(path, fileName);
String fullpath = path + File.separatorChar + fileName;
// get an input stream
@@ -1888,9 +1927,7 @@ protected String getStringValue(Node node) {
*/
protected Document loadXML(String filename) throws IOException,
ParserConfigurationException, SAXException {
- DocumentBuilder builder = DocumentBuilderFactory.newInstance()
- .newDocumentBuilder();
-
+ DocumentBuilder builder = XMLUtils.getDocumentBuilder();
return builder.parse(new File(filename));
}
diff --git a/dspace-api/src/main/java/org/dspace/app/itemupdate/ItemArchive.java b/dspace-api/src/main/java/org/dspace/app/itemupdate/ItemArchive.java
index 26de45caf77e..7dda65a0a75b 100644
--- a/dspace-api/src/main/java/org/dspace/app/itemupdate/ItemArchive.java
+++ b/dspace-api/src/main/java/org/dspace/app/itemupdate/ItemArchive.java
@@ -23,8 +23,6 @@
import java.util.Iterator;
import java.util.List;
import java.util.UUID;
-import javax.xml.parsers.DocumentBuilder;
-import javax.xml.parsers.DocumentBuilderFactory;
import javax.xml.parsers.ParserConfigurationException;
import javax.xml.transform.Transformer;
import javax.xml.transform.TransformerConfigurationException;
@@ -33,6 +31,7 @@
import org.apache.logging.log4j.Logger;
import org.dspace.app.util.LocalSchemaFilenameFilter;
+import org.dspace.app.util.XMLUtils;
import org.dspace.authorize.AuthorizeException;
import org.dspace.content.DSpaceObject;
import org.dspace.content.Item;
@@ -52,7 +51,6 @@ public class ItemArchive {
public static final String DUBLIN_CORE_XML = "dublin_core.xml";
- protected static DocumentBuilder builder = null;
protected Transformer transformer = null;
protected List dtomList = null;
@@ -95,14 +93,14 @@ public static ItemArchive create(Context context, File dir, String itemField)
InputStream is = null;
try {
is = new FileInputStream(new File(dir, DUBLIN_CORE_XML));
- itarch.dtomList = MetadataUtilities.loadDublinCore(getDocumentBuilder(), is);
+ itarch.dtomList = MetadataUtilities.loadDublinCore(XMLUtils.getDocumentBuilder(), is);
//The code to search for local schema files was copied from org.dspace.app.itemimport
// .ItemImportServiceImpl.java
File file[] = dir.listFiles(new LocalSchemaFilenameFilter());
for (int i = 0; i < file.length; i++) {
is = new FileInputStream(file[i]);
- itarch.dtomList.addAll(MetadataUtilities.loadDublinCore(getDocumentBuilder(), is));
+ itarch.dtomList.addAll(MetadataUtilities.loadDublinCore(XMLUtils.getDocumentBuilder(), is));
}
} finally {
if (is != null) {
@@ -126,14 +124,6 @@ public static ItemArchive create(Context context, File dir, String itemField)
return itarch;
}
- protected static DocumentBuilder getDocumentBuilder()
- throws ParserConfigurationException {
- if (builder == null) {
- builder = DocumentBuilderFactory.newInstance().newDocumentBuilder();
- }
- return builder;
- }
-
/**
* Getter for Transformer
*
@@ -318,7 +308,7 @@ public void writeUndo(File undoDir)
try {
out = new FileOutputStream(new File(dir, "dublin_core.xml"));
- Document doc = MetadataUtilities.writeDublinCore(getDocumentBuilder(), undoDtomList);
+ Document doc = MetadataUtilities.writeDublinCore(XMLUtils.getDocumentBuilder(), undoDtomList);
MetadataUtilities.writeDocument(doc, getTransformer(), out);
// if undo has delete bitstream
diff --git a/dspace-api/src/main/java/org/dspace/app/launcher/ScriptLauncher.java b/dspace-api/src/main/java/org/dspace/app/launcher/ScriptLauncher.java
index 89a416bfa883..ab8807c2cae3 100644
--- a/dspace-api/src/main/java/org/dspace/app/launcher/ScriptLauncher.java
+++ b/dspace-api/src/main/java/org/dspace/app/launcher/ScriptLauncher.java
@@ -19,6 +19,7 @@
import org.apache.commons.cli.ParseException;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
+import org.dspace.app.util.XMLUtils;
import org.dspace.core.Context;
import org.dspace.scripts.DSpaceRunnable;
import org.dspace.scripts.DSpaceRunnable.StepResult;
@@ -314,7 +315,7 @@ public static Document getConfig(DSpaceKernelImpl kernelImpl) {
String config = kernelImpl.getConfigurationService().getProperty("dspace.dir") +
System.getProperty("file.separator") + "config" +
System.getProperty("file.separator") + "launcher.xml";
- SAXBuilder saxBuilder = new SAXBuilder();
+ SAXBuilder saxBuilder = XMLUtils.getSAXBuilder();
Document doc = null;
try {
doc = saxBuilder.build(config);
diff --git a/dspace-api/src/main/java/org/dspace/app/ldn/LDNMessageConsumer.java b/dspace-api/src/main/java/org/dspace/app/ldn/LDNMessageConsumer.java
index 210aaa6c9c97..8698559a5f89 100644
--- a/dspace-api/src/main/java/org/dspace/app/ldn/LDNMessageConsumer.java
+++ b/dspace-api/src/main/java/org/dspace/app/ldn/LDNMessageConsumer.java
@@ -21,7 +21,6 @@
import java.util.UUID;
import com.fasterxml.jackson.core.JsonProcessingException;
-import com.fasterxml.jackson.databind.JsonMappingException;
import com.fasterxml.jackson.databind.ObjectMapper;
import org.apache.commons.collections4.CollectionUtils;
import org.apache.commons.lang3.StringUtils;
@@ -48,6 +47,10 @@
import org.dspace.services.ConfigurationService;
import org.dspace.services.factory.DSpaceServicesFactory;
import org.dspace.utils.DSpace;
+import org.dspace.versioning.Version;
+import org.dspace.versioning.VersionHistory;
+import org.dspace.versioning.factory.VersionServiceFactory;
+import org.dspace.versioning.service.VersionHistoryService;
import org.dspace.web.ContextUtil;
/**
@@ -63,6 +66,9 @@ public class LDNMessageConsumer implements Consumer {
private ConfigurationService configurationService;
private ItemService itemService;
private BitstreamService bitstreamService;
+ private final String RESUBMISSION_SUFFIX = "-resubmission";
+ private final String ENDORSEMENT_PATTERN = "request-endorsement";
+ private final String REVIEW_PATTERN = "request-review";
@Override
public void initialize() throws Exception {
@@ -83,6 +89,9 @@ public void consume(Context context, Event event) throws Exception {
}
Item item = (Item) event.getSubject(context);
+ if (item == null) {
+ return;
+ }
createManualLDNMessages(context, item);
createAutomaticLDNMessages(context, item);
}
@@ -90,10 +99,24 @@ public void consume(Context context, Event event) throws Exception {
private void createManualLDNMessages(Context context, Item item) throws SQLException, JsonProcessingException {
List patternsToTrigger =
notifyPatternToTriggerService.findByItem(context, item);
+ // Note that multiple patterns can be submitted and not all support resubmission
+ // 1. Extract all patterns that accept resubmissions, i.e. endorsement and review
+ List patternsSupportingResubmission = patternsToTrigger.stream()
+ .filter(p -> p.getPattern().equals(REVIEW_PATTERN) || p.getPattern().equals(ENDORSEMENT_PATTERN))
+ .map(NotifyPatternToTrigger::getID).toList();
+
+ String resubmissionReplyToID = null;
for (NotifyPatternToTrigger patternToTrigger : patternsToTrigger) {
+ // Only try to fetch resubmission ID if the pattern support resubmission
+ if (patternsSupportingResubmission.contains(patternToTrigger.getID())) {
+ resubmissionReplyToID = findResubmissionReplyToUUID(context, item, patternToTrigger.getNotifyService());
+ }
+
createLDNMessage(context,patternToTrigger.getItem(),
- patternToTrigger.getNotifyService(), patternToTrigger.getPattern());
+ patternToTrigger.getNotifyService(),
+ patternToTrigger.getPattern(),
+ resubmissionReplyToID);
}
}
@@ -104,9 +127,31 @@ private void createAutomaticLDNMessages(Context context, Item item) throws SQLEx
for (NotifyServiceInboundPattern inboundPattern : inboundPatterns) {
if (StringUtils.isEmpty(inboundPattern.getConstraint()) ||
evaluateFilter(context, item, inboundPattern.getConstraint())) {
- createLDNMessage(context, item, inboundPattern.getNotifyService(), inboundPattern.getPattern());
+ createLDNMessage(context, item, inboundPattern.getNotifyService(),
+ inboundPattern.getPattern(), null);
+ }
+ }
+ }
+
+ private String findResubmissionReplyToUUID(Context context, Item item, NotifyServiceEntity service)
+ throws SQLException {
+ // 1.1 Check whether this is a new version submission
+ VersionHistoryService versionHistoryService = VersionServiceFactory.getInstance()
+ .getVersionHistoryService();
+ VersionHistory versionHistory = versionHistoryService.findByItem(context, item);
+
+ if (versionHistory != null) {
+ Version currentVersion = versionHistoryService.getVersion(context, versionHistory, item);
+ Version previousVersion = versionHistoryService.getPrevious(context, versionHistory, currentVersion);
+ if (previousVersion != null) {
+ // 1.2 and a TentativeReject notification, matching the current pattern's service, was received for the
+ // previous item version
+ return ldnMessageService.findEndorsementOrReviewResubmissionIdByItem(context,
+ previousVersion.getItem(), service);
}
}
+ // New submission (new item, or previous version with a tentativeReject notification not found)
+ return null;
}
private boolean evaluateFilter(Context context, Item item, String constraint) {
@@ -116,19 +161,40 @@ private boolean evaluateFilter(Context context, Item item, String constraint) {
return filter != null && filter.getResult(context, item);
}
- private void createLDNMessage(Context context, Item item, NotifyServiceEntity service, String pattern)
- throws SQLException, JsonMappingException, JsonProcessingException {
-
- LDN ldn = getLDNMessage(pattern);
+ private void createLDNMessage(Context context, Item item, NotifyServiceEntity service, String pattern,
+ String resubmissionID)
+ throws SQLException, JsonProcessingException {
+ // Amend current pattern name to trigger
+ // Endorsement or Review offer resubmissions: append '-resubmission' to pattern name to choose the correct
+ // LDN message template: e.g. request-endorsement-resubmission or request-review-resubmission
+ LDN ldn = (resubmissionID != null)
+ ? getLDNMessage(pattern + RESUBMISSION_SUFFIX) : getLDNMessage(pattern);
LDNMessageEntity ldnMessage =
- ldnMessageService.create(context, format("urn:uuid:%s", UUID.randomUUID()));
+ ldnMessageService.create(context, format("urn:uuid:%s", UUID.randomUUID()));
ldnMessage.setObject(item);
ldnMessage.setTarget(service);
ldnMessage.setQueueStatus(LDNMessageEntity.QUEUE_STATUS_QUEUED);
ldnMessage.setQueueTimeout(new Date());
- appendGeneratedMessage(ldn, ldnMessage, pattern);
+ String actorID = null;
+ boolean serviceUsesActorEmailId =
+ configurationService.getBooleanProperty(
+ String.format("ldn.notification.supportsActorEmailId.%d", service.getID()), false);
+ if (serviceUsesActorEmailId) {
+ // If the service has been configured to use actorEmailId, we use the submitter's email and name
+ if (item.getSubmitter() != null) {
+ actorID = item.getSubmitter().getEmail();
+ } else {
+ // Use configured fallback email (defaults to mail.admin property)
+ actorID = configurationService.getProperty("ldn.notification.email.submitter.fallback");
+ }
+ }
+ appendGeneratedMessage(ldn,
+ ldnMessage,
+ actorID,
+ (actorID != null && item.getSubmitter() != null) ? item.getSubmitter().getFullName() : null,
+ resubmissionID);
ObjectMapper mapper = new ObjectMapper();
Notification notification = mapper.readValue(ldnMessage.getMessage(), Notification.class);
@@ -139,6 +205,10 @@ private void createLDNMessage(Context context, Item item, NotifyServiceEntity se
Collections.sort(notificationTypeArrayList);
ldnMessage.setActivityStreamType(notificationTypeArrayList.get(0));
ldnMessage.setCoarNotifyType(notificationTypeArrayList.get(1));
+ // If a resubmission, set inReplyTo
+ if (resubmissionID != null) {
+ ldnMessage.setInReplyTo(ldnMessageService.find(context, resubmissionID));
+ }
ldnMessageService.update(context, ldnMessage);
}
@@ -151,11 +221,16 @@ private LDN getLDNMessage(String pattern) {
}
}
- private void appendGeneratedMessage(LDN ldn, LDNMessageEntity ldnMessage, String pattern) {
+ private void appendGeneratedMessage(LDN ldn, LDNMessageEntity ldnMessage, String actorID, String actorName,
+ String resubmissionId) {
Item item = (Item) ldnMessage.getObject();
- ldn.addArgument(getUiUrl());
+ if (actorID != null) {
+ ldn.addArgument("mailto:" + actorID);
+ } else {
+ ldn.addArgument(getUiUrl());
+ }
ldn.addArgument(configurationService.getProperty("ldn.notify.inbox"));
- ldn.addArgument(configurationService.getProperty("dspace.name"));
+ ldn.addArgument(actorName != null ? actorName : configurationService.getProperty("dspace.name"));
ldn.addArgument(Objects.requireNonNullElse(ldnMessage.getTarget().getUrl(), ""));
ldn.addArgument(Objects.requireNonNullElse(ldnMessage.getTarget().getLdnUrl(), ""));
ldn.addArgument(getUiUrl() + "/handle/" + ldnMessage.getObject().getHandle());
@@ -166,6 +241,17 @@ private void appendGeneratedMessage(LDN ldn, LDNMessageEntity ldnMessage, String
ldn.addArgument(getRelationUri(item));
ldn.addArgument("http://purl.org/vocab/frbr/core#supplement");
ldn.addArgument(format("urn:uuid:%s", UUID.randomUUID()));
+ if (actorID != null) {
+ ldn.addArgument("Person");
+ } else {
+ ldn.addArgument("Service");
+ }
+ // Param 14: UI URL, LDN message origin
+ ldn.addArgument(getUiUrl());
+ // Param 15: inReplyTo ID, used in endorsement resubmission notifications
+ if (resubmissionId != null) {
+ ldn.addArgument(String.format("\"inReplyTo\": \"%s\",", resubmissionId));
+ }
ldnMessage.setMessage(ldn.generateLDNMessage());
}
diff --git a/dspace-api/src/main/java/org/dspace/app/ldn/LDNMessageEntity.java b/dspace-api/src/main/java/org/dspace/app/ldn/LDNMessageEntity.java
index 27257455e0ce..d57567ff1374 100644
--- a/dspace-api/src/main/java/org/dspace/app/ldn/LDNMessageEntity.java
+++ b/dspace-api/src/main/java/org/dspace/app/ldn/LDNMessageEntity.java
@@ -20,6 +20,8 @@
import jakarta.persistence.TemporalType;
import org.dspace.content.DSpaceObject;
import org.dspace.core.ReloadableEntity;
+import org.dspace.services.ConfigurationService;
+import org.dspace.services.factory.DSpaceServicesFactory;
/**
* Class representing ldnMessages stored in the DSpace system and, when locally resolvable,
@@ -289,7 +291,11 @@ public String toString() {
}
public static String getNotificationType(LDNMessageEntity ldnMessage) {
- if (ldnMessage.getInReplyTo() != null || ldnMessage.getOrigin() != null) {
+ // Resubmission outgoing notifications have the inReplyTo, therefore it cannot be used to determine
+ // whether a notification is incoming
+ ConfigurationService configurationService = DSpaceServicesFactory.getInstance().getConfigurationService();
+ if (ldnMessage.getOrigin() != null && !ldnMessage.getOrigin().getLdnUrl()
+ .contains(configurationService.getProperty("dspace.ui.url"))) {
return TYPE_INCOMING;
}
return TYPE_OUTGOING;
diff --git a/dspace-api/src/main/java/org/dspace/app/ldn/action/LDNEmailAction.java b/dspace-api/src/main/java/org/dspace/app/ldn/action/LDNEmailAction.java
index b87001f81500..d1eddb205b7a 100644
--- a/dspace-api/src/main/java/org/dspace/app/ldn/action/LDNEmailAction.java
+++ b/dspace-api/src/main/java/org/dspace/app/ldn/action/LDNEmailAction.java
@@ -27,7 +27,7 @@
import org.springframework.beans.factory.annotation.Autowired;
/**
- * Action to send email to receipients provided in actionSendFilter. The email
+ * Action to send email to recipients provided in actionSendFilter. The email
* body will be result of templating actionSendFilter.
*/
public class LDNEmailAction implements LDNAction {
@@ -139,7 +139,13 @@ private List retrieveRecipientsEmail(Item item) {
List recipients = new LinkedList();
if (actionSendFilter.startsWith("SUBMITTER")) {
- recipients.add(item.getSubmitter().getEmail());
+ if (item.getSubmitter() != null) {
+ recipients.add(item.getSubmitter().getEmail());
+ } else {
+ // Fallback configured option
+ recipients.add(configurationService.getProperty("ldn.notification.email.submitter.fallback"));
+ }
+
} else if (actionSendFilter.startsWith("GROUP:")) {
String groupName = actionSendFilter.replace("GROUP:", "");
String property = format("email.%s.list", groupName);
diff --git a/dspace-api/src/main/java/org/dspace/app/ldn/action/SendLDNMessageAction.java b/dspace-api/src/main/java/org/dspace/app/ldn/action/SendLDNMessageAction.java
index c0ecf04304b8..c5a60144e63c 100644
--- a/dspace-api/src/main/java/org/dspace/app/ldn/action/SendLDNMessageAction.java
+++ b/dspace-api/src/main/java/org/dspace/app/ldn/action/SendLDNMessageAction.java
@@ -18,9 +18,9 @@
import org.apache.http.client.methods.HttpPost;
import org.apache.http.entity.StringEntity;
import org.apache.http.impl.client.CloseableHttpClient;
-import org.apache.http.impl.client.HttpClientBuilder;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
+import org.dspace.app.client.DSpaceHttpClientFactory;
import org.dspace.app.ldn.model.Notification;
import org.dspace.content.Item;
import org.dspace.core.Context;
@@ -34,21 +34,13 @@ public class SendLDNMessageAction implements LDNAction {
private static final Logger log = LogManager.getLogger(SendLDNMessageAction.class);
- private CloseableHttpClient client = null;
+ private CloseableHttpClient client;
public SendLDNMessageAction() {
- HttpClientBuilder builder = HttpClientBuilder.create();
- client = builder
- .disableAutomaticRetries()
- .setMaxConnTotal(5)
- .build();
}
public SendLDNMessageAction(CloseableHttpClient client) {
- this();
- if (client != null) {
- this.client = client;
- }
+ this.client = client;
}
@Override
@@ -66,9 +58,10 @@ public LDNActionStatus execute(Context context, Notification notification, Item
// NOTE: Github believes there is a "Potential server-side request forgery due to a user-provided value"
// This is a false positive because the LDN Service URL is configured by the user from DSpace.
// See the frontend configuration at [dspace.ui.url]/admin/ldn/services
- try (
- CloseableHttpResponse response = client.execute(httpPost);
- ) {
+ if (client == null) {
+ client = DSpaceHttpClientFactory.getInstance().buildWithoutAutomaticRetries(5);
+ }
+ try (CloseableHttpResponse response = client.execute(httpPost)) {
if (isSuccessful(response.getStatusLine().getStatusCode())) {
result = LDNActionStatus.CONTINUE;
} else if (isRedirect(response.getStatusLine().getStatusCode())) {
@@ -77,6 +70,7 @@ public LDNActionStatus execute(Context context, Notification notification, Item
} catch (Exception e) {
log.error(e);
}
+ client.close();
return result;
}
@@ -91,9 +85,9 @@ private boolean isRedirect(int statusCode) {
statusCode == HttpStatus.SC_TEMPORARY_REDIRECT;
}
- private LDNActionStatus handleRedirect(CloseableHttpResponse oldresponse,
+ private LDNActionStatus handleRedirect(CloseableHttpResponse oldResponse,
HttpPost request) throws HttpException {
- Header[] urls = oldresponse.getHeaders(HttpHeaders.LOCATION);
+ Header[] urls = oldResponse.getHeaders(HttpHeaders.LOCATION);
String url = urls.length > 0 && urls[0] != null ? urls[0].getValue() : null;
if (url == null) {
throw new HttpException("Error following redirect, unable to reach"
@@ -102,17 +96,14 @@ private LDNActionStatus handleRedirect(CloseableHttpResponse oldresponse,
LDNActionStatus result = LDNActionStatus.ABORT;
try {
request.setURI(new URI(url));
- try (
- CloseableHttpResponse response = client.execute(request);
- ) {
+ try (CloseableHttpResponse response = client.execute(request)) {
if (isSuccessful(response.getStatusLine().getStatusCode())) {
- return LDNActionStatus.CONTINUE;
+ result = LDNActionStatus.CONTINUE;
}
}
} catch (Exception e) {
log.error("Error following redirect:", e);
}
-
- return LDNActionStatus.ABORT;
+ return result;
}
}
\ No newline at end of file
diff --git a/dspace-api/src/main/java/org/dspace/app/ldn/dao/impl/LDNMessageDaoImpl.java b/dspace-api/src/main/java/org/dspace/app/ldn/dao/impl/LDNMessageDaoImpl.java
index d811f6d39f34..4c935caf0d33 100644
--- a/dspace-api/src/main/java/org/dspace/app/ldn/dao/impl/LDNMessageDaoImpl.java
+++ b/dspace-api/src/main/java/org/dspace/app/ldn/dao/impl/LDNMessageDaoImpl.java
@@ -149,8 +149,11 @@ public List findAllMessagesByItem(
Predicate activityPredicate = null;
andPredicates.add(
criteriaBuilder.equal(root.get(LDNMessageEntity_.queueStatus), LDNMessageEntity.QUEUE_STATUS_PROCESSED));
+ // Added OR with object or context - inbound notifications make use of the context item to provide information
+ // about the repository item the notification refers to
andPredicates.add(
- criteriaBuilder.equal(root.get(LDNMessageEntity_.object), item));
+ criteriaBuilder.or(criteriaBuilder.equal(root.get(LDNMessageEntity_.object), item),
+ criteriaBuilder.equal(root.get(LDNMessageEntity_.context), item)));
if (activities != null && activities.length > 0) {
activityPredicate = root.get(LDNMessageEntity_.activityStreamType).in(activities);
andPredicates.add(activityPredicate);
diff --git a/dspace-api/src/main/java/org/dspace/app/ldn/model/NotifyRequestStatus.java b/dspace-api/src/main/java/org/dspace/app/ldn/model/NotifyRequestStatus.java
index 0302b528aa8d..8333eae91024 100644
--- a/dspace-api/src/main/java/org/dspace/app/ldn/model/NotifyRequestStatus.java
+++ b/dspace-api/src/main/java/org/dspace/app/ldn/model/NotifyRequestStatus.java
@@ -25,7 +25,7 @@
* "Offer", "coar-notify:IngestAction"
* "Offer", "coar-notify:ReviewAction"
*
- * and their acknownledgements - if any
+ * and their acknowledgements - if any
*
* @author Francesco Bacchelli (francesco.bacchelli at 4science dot it)
*/
diff --git a/dspace-api/src/main/java/org/dspace/app/ldn/model/NotifyRequestStatusEnum.java b/dspace-api/src/main/java/org/dspace/app/ldn/model/NotifyRequestStatusEnum.java
index 437c624f84d8..04265255711e 100644
--- a/dspace-api/src/main/java/org/dspace/app/ldn/model/NotifyRequestStatusEnum.java
+++ b/dspace-api/src/main/java/org/dspace/app/ldn/model/NotifyRequestStatusEnum.java
@@ -8,11 +8,12 @@
package org.dspace.app.ldn.model;
/**
* REQUESTED means acknowledgements not received yet
- * ACCEPTED means acknowledgements of "Accept" type received
- * REJECTED means ack of "TentativeReject" type received
+ * ACCEPTED means acknowledgements of "Accept" or "TentativeAccept" type received
+ * REJECTED means ack of "Reject" type received
+ * TENTATIVE_REJECT means ack of "TentativeReject" type received
*
* @author Francesco Bacchelli (francesco.bacchelli at 4science.com)
*/
public enum NotifyRequestStatusEnum {
- REJECTED, ACCEPTED, REQUESTED
+ REJECTED, ACCEPTED, REQUESTED, TENTATIVE_REJECT
}
diff --git a/dspace-api/src/main/java/org/dspace/app/ldn/model/RequestStatus.java b/dspace-api/src/main/java/org/dspace/app/ldn/model/RequestStatus.java
index d19369830787..e33bc3eeb7d5 100644
--- a/dspace-api/src/main/java/org/dspace/app/ldn/model/RequestStatus.java
+++ b/dspace-api/src/main/java/org/dspace/app/ldn/model/RequestStatus.java
@@ -8,7 +8,7 @@
package org.dspace.app.ldn.model;
/**
- * Informations about the Offer and Acknowledgements targeting a specified Item
+ * Information about the Offer and Acknowledgements targeting a specified Item
*
* @author Francesco Bacchelli (francesco.bacchelli at 4science.com)
*/
diff --git a/dspace-api/src/main/java/org/dspace/app/ldn/processor/LDNMetadataProcessor.java b/dspace-api/src/main/java/org/dspace/app/ldn/processor/LDNMetadataProcessor.java
index 43c50173ee61..c0bc89ab213f 100644
--- a/dspace-api/src/main/java/org/dspace/app/ldn/processor/LDNMetadataProcessor.java
+++ b/dspace-api/src/main/java/org/dspace/app/ldn/processor/LDNMetadataProcessor.java
@@ -20,6 +20,7 @@
import org.apache.http.client.HttpResponseException;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
+import org.dspace.app.ldn.LDNMessageEntity;
import org.dspace.app.ldn.action.LDNAction;
import org.dspace.app.ldn.action.LDNActionStatus;
import org.dspace.app.ldn.model.Notification;
@@ -59,6 +60,8 @@ public class LDNMetadataProcessor implements LDNProcessor {
"Announce",
"TentativeReject",
"Accept",
+ "TentativeAccept",
+ "Reject",
"coar-notify:ReviewAction",
"coar-notify:IngestAction",
"coar-notify:EndorsementAction");
@@ -152,7 +155,7 @@ public void setActions(List actions) {
}
/**
- * Lookup associated item to the notification context. If UUID in URL, lookup bu
+ * Lookup associated item to the notification context. If UUID in URL, lookup by
* UUID, else lookup by handle.
*
* @param context current context
@@ -168,7 +171,22 @@ private Item lookupItem(Context context, Notification notification) throws SQLEx
String url = null;
if (CONTEXT_ID_ITEM_TYPES.containsAll(notification.getType())) {
- url = notification.getContext().getId();
+ if (notification.getContext() != null) {
+ url = notification.getContext().getId();
+ } else if (notification.getInReplyTo() != null) {
+ // Find context information (the item this notification relates to) via the inReplyTo notification ID
+ LDNMessageEntity inReplyToldnMessageEntity =
+ ldnMessageService.find(context, notification.getInReplyTo());
+ if (inReplyToldnMessageEntity != null) {
+ String dspaceUrl = configurationService.getProperty("dspace.ui.url")
+ + "/handle/";
+ url = dspaceUrl + inReplyToldnMessageEntity.getObject().getHandle();
+ // Set context based on the inReplyTo and update in DB
+ LDNMessageEntity ldnMessageEntity = ldnMessageService.find(context, notification.getId());
+ ldnMessageEntity.setContext(inReplyToldnMessageEntity.getObject());
+ ldnMessageService.update(context, ldnMessageEntity);
+ }
+ }
} else if (OBJECT_ID_ITEM_TYPES.containsAll(notification.getType())) {
url = notification.getObject().getId();
} else if (OBJECT_SUBJECT_ITEM_TYPES.containsAll(notification.getType())) {
diff --git a/dspace-api/src/main/java/org/dspace/app/ldn/service/LDNMessageService.java b/dspace-api/src/main/java/org/dspace/app/ldn/service/LDNMessageService.java
index eb18c6a69a70..01cb20626418 100644
--- a/dspace-api/src/main/java/org/dspace/app/ldn/service/LDNMessageService.java
+++ b/dspace-api/src/main/java/org/dspace/app/ldn/service/LDNMessageService.java
@@ -130,6 +130,17 @@ public interface LDNMessageService {
*/
public NotifyRequestStatus findRequestsByItem(Context context, Item item) throws SQLException;
+ /**
+ * find the UUID of a previous tentativeReject notification associated with a new resubmission (Endorsement or
+ * Review patterns only)
+ *
+ * @param context the context
+ * @param item the previousVersion item associated with a potential resubmission
+ * @return the UUID of a previous tentativeReject notification associated with a potential resubmission if found
+ */
+ public String findEndorsementOrReviewResubmissionIdByItem(Context context, Item item, NotifyServiceEntity service)
+ throws SQLException;
+
/**
* delete the provided ldn message
*
diff --git a/dspace-api/src/main/java/org/dspace/app/ldn/service/impl/LDNMessageServiceImpl.java b/dspace-api/src/main/java/org/dspace/app/ldn/service/impl/LDNMessageServiceImpl.java
index 15f07a556112..1c101d512134 100644
--- a/dspace-api/src/main/java/org/dspace/app/ldn/service/impl/LDNMessageServiceImpl.java
+++ b/dspace-api/src/main/java/org/dspace/app/ldn/service/impl/LDNMessageServiceImpl.java
@@ -145,6 +145,12 @@ public LDNMessageEntity create(Context context, Notification notification, Strin
ldnMessage.setActivityStreamType(notificationTypeArrayList.get(0));
if (notificationTypeArrayList.size() > 1) {
ldnMessage.setCoarNotifyType(notificationTypeArrayList.get(1));
+ } else {
+ // The Notification's Type array does not include the CoarNotifyType information, e.g. ack notifications
+ // Attempt to find it via the inReplyTo if present
+ if (ldnMessage.getInReplyTo() != null) {
+ ldnMessage.setCoarNotifyType(ldnMessage.getInReplyTo().getCoarNotifyType());
+ }
}
ldnMessage.setQueueStatus(LDNMessageEntity.QUEUE_STATUS_QUEUED);
ldnMessage.setSourceIp(sourceIp);
@@ -368,18 +374,23 @@ public NotifyRequestStatus findRequestsByItem(Context context, Item item) throws
offer.setServiceUrl(nse == null ? "" : nse.getUrl());
offer.setOfferType(LDNUtils.getNotifyType(msg.getCoarNotifyType()));
List acks = ldnMessageDao.findAllRelatedMessagesByItem(
- context, msg, item, "Accept", "TentativeReject", "TentativeAccept", "Announce");
+ context, msg, item, "Accept", "Reject", "TentativeReject", "TentativeAccept",
+ "Announce");
if (acks == null || acks.isEmpty()) {
offer.setStatus(NotifyRequestStatusEnum.REQUESTED);
+ } else if (acks.stream()
+ .filter(c -> (c.getActivityStreamType().equalsIgnoreCase("TentativeReject")))
+ .findAny().isPresent()) {
+ offer.setStatus(NotifyRequestStatusEnum.TENTATIVE_REJECT);
+ } else if (acks.stream()
+ .filter(c -> (c.getActivityStreamType().equalsIgnoreCase("Reject")))
+ .findAny().isPresent()) {
+ offer.setStatus(NotifyRequestStatusEnum.REJECTED);
} else if (acks.stream()
.filter(c -> (c.getActivityStreamType().equalsIgnoreCase("TentativeAccept") ||
c.getActivityStreamType().equalsIgnoreCase("Accept")))
.findAny().isPresent()) {
offer.setStatus(NotifyRequestStatusEnum.ACCEPTED);
- } else if (acks.stream()
- .filter(c -> c.getActivityStreamType().equalsIgnoreCase("TentativeReject"))
- .findAny().isPresent()) {
- offer.setStatus(NotifyRequestStatusEnum.REJECTED);
}
if (acks.stream().filter(
c -> c.getActivityStreamType().equalsIgnoreCase("Announce"))
@@ -391,6 +402,32 @@ public NotifyRequestStatus findRequestsByItem(Context context, Item item) throws
return result;
}
+ @Override
+ public String findEndorsementOrReviewResubmissionIdByItem(Context context, Item item, NotifyServiceEntity service)
+ throws SQLException {
+ List msgs = ldnMessageDao.findAllMessagesByItem(
+ context, item, "TentativeReject");
+
+ if (msgs != null && !msgs.isEmpty()) {
+ for (LDNMessageEntity msg : msgs) {
+ // Review and Endorsement are the only patterns supporting resubmissions at present
+ if (msg.getCoarNotifyType().contains("EndorsementAction")
+ || msg.getCoarNotifyType().contains("ReviewAction")) {
+ // Only provide the resubmissionReplyTo UUID if the pattern supports resubmission
+ // Add an extra check to ensure that this is a resubmission: current notification service
+ // matches the service associated with a previous tentativeReject response. This is to avoid a
+ // case where a previous version of the item received a tentativeReject from one service
+ // and the author decides to submit the version to a different service, instead of a resubmission
+ if (msg.getOrigin() != null && msg.getOrigin().getID().equals(service.getID())) {
+ // Return the first ID found that will be used in the inReplyTo for a resubmission notification
+ return msg.getID();
+ }
+ }
+ }
+ }
+ return null;
+ }
+
public void delete(Context context, LDNMessageEntity ldnMessage) throws SQLException {
ldnMessageDao.delete(context, ldnMessage);
}
diff --git a/dspace-api/src/main/java/org/dspace/app/mediafilter/MediaFilterScript.java b/dspace-api/src/main/java/org/dspace/app/mediafilter/MediaFilterScript.java
index 5fbbebbb28cc..7f022f38b318 100644
--- a/dspace-api/src/main/java/org/dspace/app/mediafilter/MediaFilterScript.java
+++ b/dspace-api/src/main/java/org/dspace/app/mediafilter/MediaFilterScript.java
@@ -7,6 +7,7 @@
*/
package org.dspace.app.mediafilter;
+import java.time.LocalDate;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashMap;
@@ -37,8 +38,9 @@
* MFM: -v verbose outputs all extracted text to STDOUT; -f force forces all
* bitstreams to be processed, even if they have been before; -n noindex does not
* recreate index after processing bitstreams; -i [identifier] limits processing
- * scope to a community, collection or item; and -m [max] limits processing to a
- * maximum number of items.
+ * scope to a community, collection or item; -m [max] limits processing to a
+ * maximum number of items; -fd [fromdate] takes only items starting from this date,
+ * filtering by last_modified in the item table.
*/
public class MediaFilterScript extends DSpaceRunnable {
@@ -60,6 +62,7 @@ public class MediaFilterScript extends DSpaceRunnable> filterFormats = new HashMap<>();
+ private LocalDate fromDate = null;
public MediaFilterScriptConfiguration getScriptConfiguration() {
return new DSpace().getServiceManager()
@@ -112,6 +115,10 @@ public void setup() throws ParseException {
skipIds = commandLine.getOptionValues('s');
}
+ if (commandLine.hasOption('d')) {
+ fromDate = LocalDate.parse(commandLine.getOptionValue('d'));
+ }
+
}
@@ -215,6 +222,10 @@ public void internalRun() throws Exception {
mediaFilterService.setSkipList(Arrays.asList(skipIds));
}
+ if (fromDate != null) {
+ mediaFilterService.setFromDate(fromDate);
+ }
+
Context c = null;
try {
diff --git a/dspace-api/src/main/java/org/dspace/app/mediafilter/MediaFilterScriptConfiguration.java b/dspace-api/src/main/java/org/dspace/app/mediafilter/MediaFilterScriptConfiguration.java
index 7465fa6e1279..c9f61292d617 100644
--- a/dspace-api/src/main/java/org/dspace/app/mediafilter/MediaFilterScriptConfiguration.java
+++ b/dspace-api/src/main/java/org/dspace/app/mediafilter/MediaFilterScriptConfiguration.java
@@ -52,6 +52,8 @@ public Options getOptions() {
.build();
options.addOption(pluginOption);
+ options.addOption("d", "fromdate", true, "Process only item from specified last modified date");
+
Option skipOption = Option.builder("s")
.longOpt("skip")
.hasArg()
diff --git a/dspace-api/src/main/java/org/dspace/app/mediafilter/MediaFilterServiceImpl.java b/dspace-api/src/main/java/org/dspace/app/mediafilter/MediaFilterServiceImpl.java
index a6ba9fde88d9..512b8f803b9b 100644
--- a/dspace-api/src/main/java/org/dspace/app/mediafilter/MediaFilterServiceImpl.java
+++ b/dspace-api/src/main/java/org/dspace/app/mediafilter/MediaFilterServiceImpl.java
@@ -9,8 +9,11 @@
import java.io.InputStream;
import java.sql.SQLException;
+import java.time.LocalDate;
+import java.time.ZoneId;
import java.util.ArrayList;
import java.util.Collections;
+import java.util.Date;
import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
@@ -93,6 +96,7 @@ public class MediaFilterServiceImpl implements MediaFilterService, InitializingB
protected boolean isVerbose = false;
protected boolean isQuiet = false;
protected boolean isForce = false; // default to not forced
+ protected LocalDate fromDate = null;
protected MediaFilterServiceImpl() {
@@ -120,6 +124,15 @@ public void applyFiltersAllItems(Context context) throws Exception {
for (Community topLevelCommunity : topLevelCommunities) {
applyFiltersCommunity(context, topLevelCommunity);
}
+ } else if (fromDate != null) {
+ Iterator
- itemIterator =
+ itemService.findByLastModifiedSince(
+ context,
+ Date.from(fromDate.atStartOfDay(ZoneId.systemDefault()).toInstant())
+ );
+ while (itemIterator.hasNext() && processed < max2Process) {
+ applyFiltersItem(context, itemIterator.next());
+ }
} else {
//otherwise, just find every item and process
Iterator
- itemIterator = itemService.findAll(context);
@@ -588,4 +601,9 @@ public void setFilterFormats(Map> filterFormats) {
public void setLogHandler(DSpaceRunnableHandler handler) {
this.handler = handler;
}
+
+ @Override
+ public void setFromDate(LocalDate fromDate) {
+ this.fromDate = fromDate;
+ }
}
diff --git a/dspace-api/src/main/java/org/dspace/app/mediafilter/TikaTextExtractionFilter.java b/dspace-api/src/main/java/org/dspace/app/mediafilter/TikaTextExtractionFilter.java
index e83bf706ed02..5728f4f42f48 100644
--- a/dspace-api/src/main/java/org/dspace/app/mediafilter/TikaTextExtractionFilter.java
+++ b/dspace-api/src/main/java/org/dspace/app/mediafilter/TikaTextExtractionFilter.java
@@ -18,6 +18,7 @@
import org.apache.commons.lang.StringUtils;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
+import org.apache.poi.util.IOUtils;
import org.apache.tika.Tika;
import org.apache.tika.exception.TikaException;
import org.apache.tika.metadata.Metadata;
@@ -37,6 +38,8 @@
public class TikaTextExtractionFilter
extends MediaFilter {
private final static Logger log = LogManager.getLogger();
+ private static final int DEFAULT_MAX_CHARS = 100_000;
+ private static final int DEFAULT_MAX_ARRAY = 100_000_000;
@Override
public String getFilteredName(String oldFilename) {
@@ -70,9 +73,12 @@ public InputStream getDestinationStream(Item currentItem, InputStream source, bo
}
// Not using temporary file. We'll use Tika's default in-memory parsing.
- // Get maximum characters to extract. Default is 100,000 chars, which is also Tika's default setting.
String extractedText;
- int maxChars = configurationService.getIntProperty("textextractor.max-chars", 100000);
+ // Get maximum characters to extract. Default is 100,000 chars, which is also Tika's default setting.
+ int maxChars = configurationService.getIntProperty("textextractor.max-chars", DEFAULT_MAX_CHARS);
+ // Get maximum size of structure that Tika will try to buffer.
+ int maxArray = configurationService.getIntProperty("textextractor.max-array", DEFAULT_MAX_ARRAY);
+ IOUtils.setByteArrayMaxOverride(maxArray);
try {
// Use Tika to extract text from input. Tika will automatically detect the file type.
Tika tika = new Tika();
@@ -80,13 +86,13 @@ public InputStream getDestinationStream(Item currentItem, InputStream source, bo
extractedText = tika.parseToString(source);
} catch (IOException e) {
System.err.format("Unable to extract text from bitstream in Item %s%n", currentItem.getID().toString());
- e.printStackTrace();
+ e.printStackTrace(System.err);
log.error("Unable to extract text from bitstream in Item {}", currentItem.getID().toString(), e);
throw e;
} catch (OutOfMemoryError oe) {
System.err.format("OutOfMemoryError occurred when extracting text from bitstream in Item %s. " +
"You may wish to enable 'textextractor.use-temp-file'.%n", currentItem.getID().toString());
- oe.printStackTrace();
+ oe.printStackTrace(System.err);
log.error("OutOfMemoryError occurred when extracting text from bitstream in Item {}. " +
"You may wish to enable 'textextractor.use-temp-file'.", currentItem.getID().toString(), oe);
throw oe;
@@ -138,7 +144,7 @@ private InputStream extractUsingTempFile(InputStream source, boolean verbose)
@Override
public void characters(char[] ch, int start, int length) throws SAXException {
try {
- writer.append(new String(ch), start, length);
+ writer.append(new String(ch, start, length));
} catch (IOException e) {
String errorMsg = String.format("Could not append to temporary file at %s " +
"when performing text extraction",
@@ -156,7 +162,7 @@ public void characters(char[] ch, int start, int length) throws SAXException {
@Override
public void ignorableWhitespace(char[] ch, int start, int length) throws SAXException {
try {
- writer.append(new String(ch), start, length);
+ writer.append(new String(ch, start, length));
} catch (IOException e) {
String errorMsg = String.format("Could not append to temporary file at %s " +
"when performing text extraction",
@@ -167,6 +173,10 @@ public void ignorableWhitespace(char[] ch, int start, int length) throws SAXExce
}
});
+ ConfigurationService configurationService = DSpaceServicesFactory.getInstance().getConfigurationService();
+ int maxArray = configurationService.getIntProperty("textextractor.max-array", DEFAULT_MAX_ARRAY);
+ IOUtils.setByteArrayMaxOverride(maxArray);
+
AutoDetectParser parser = new AutoDetectParser();
Metadata metadata = new Metadata();
// parse our source InputStream using the above custom handler
diff --git a/dspace-api/src/main/java/org/dspace/app/mediafilter/service/MediaFilterService.java b/dspace-api/src/main/java/org/dspace/app/mediafilter/service/MediaFilterService.java
index bc92ff521098..30e6dba42f08 100644
--- a/dspace-api/src/main/java/org/dspace/app/mediafilter/service/MediaFilterService.java
+++ b/dspace-api/src/main/java/org/dspace/app/mediafilter/service/MediaFilterService.java
@@ -8,6 +8,7 @@
package org.dspace.app.mediafilter.service;
import java.sql.SQLException;
+import java.time.LocalDate;
import java.util.List;
import java.util.Map;
@@ -149,4 +150,6 @@ public void updatePoliciesOfDerivativeBitstreams(Context context, Item item, Bit
* @param handler
*/
public void setLogHandler(DSpaceRunnableHandler handler);
+
+ public void setFromDate(LocalDate fromDate);
}
diff --git a/dspace-api/src/main/java/org/dspace/app/sfx/SFXFileReaderServiceImpl.java b/dspace-api/src/main/java/org/dspace/app/sfx/SFXFileReaderServiceImpl.java
index 184f00a53e59..d3b447374a2c 100644
--- a/dspace-api/src/main/java/org/dspace/app/sfx/SFXFileReaderServiceImpl.java
+++ b/dspace-api/src/main/java/org/dspace/app/sfx/SFXFileReaderServiceImpl.java
@@ -18,6 +18,7 @@
import org.apache.commons.lang3.StringUtils;
import org.apache.logging.log4j.Logger;
import org.dspace.app.sfx.service.SFXFileReaderService;
+import org.dspace.app.util.XMLUtils;
import org.dspace.content.DCPersonName;
import org.dspace.content.Item;
import org.dspace.content.MetadataValue;
@@ -79,9 +80,9 @@ public Document parseFile(String fileName) {
log.info("Parsing XML file... " + fileName);
DocumentBuilder docBuilder;
Document doc = null;
- DocumentBuilderFactory docBuilderFactory = DocumentBuilderFactory.newInstance();
- docBuilderFactory.setIgnoringElementContentWhitespace(true);
try {
+ DocumentBuilderFactory docBuilderFactory = XMLUtils.getDocumentBuilderFactory();
+ docBuilderFactory.setIgnoringElementContentWhitespace(true);
docBuilder = docBuilderFactory.newDocumentBuilder();
} catch (ParserConfigurationException e) {
log.error("Wrong parser configuration: " + e.getMessage());
diff --git a/dspace-api/src/main/java/org/dspace/app/sherpa/SHERPAService.java b/dspace-api/src/main/java/org/dspace/app/sherpa/SHERPAService.java
index 9ee5ca55cc6d..1fec20f51fee 100644
--- a/dspace-api/src/main/java/org/dspace/app/sherpa/SHERPAService.java
+++ b/dspace-api/src/main/java/org/dspace/app/sherpa/SHERPAService.java
@@ -17,15 +17,15 @@
import org.apache.commons.io.IOUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.http.HttpEntity;
-import org.apache.http.HttpResponse;
import org.apache.http.HttpStatus;
import org.apache.http.client.config.RequestConfig;
+import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.client.utils.URIBuilder;
import org.apache.http.impl.client.CloseableHttpClient;
-import org.apache.http.impl.client.HttpClientBuilder;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
+import org.dspace.app.client.DSpaceHttpClientFactory;
import org.dspace.app.sherpa.v2.SHERPAPublisherResponse;
import org.dspace.app.sherpa.v2.SHERPAResponse;
import org.dspace.app.sherpa.v2.SHERPAUtils;
@@ -45,8 +45,6 @@
*/
public class SHERPAService {
- private CloseableHttpClient client = null;
-
private int maxNumberOfTries;
private long sleepBetweenTimeouts;
private int timeout = 5000;
@@ -59,26 +57,13 @@ public class SHERPAService {
@Autowired
ConfigurationService configurationService;
- /**
- * Create a new HTTP builder with sensible defaults in constructor
- */
- public SHERPAService() {
- HttpClientBuilder builder = HttpClientBuilder.create();
- // httpclient 4.3+ doesn't appear to have any sensible defaults any more. Setting conservative defaults as
- // not to hammer the SHERPA service too much.
- client = builder
- .disableAutomaticRetries()
- .setMaxConnTotal(5)
- .build();
- }
-
/**
* Complete initialization of the Bean.
*/
@SuppressWarnings("unused")
@PostConstruct
private void init() {
- // Get endoint and API key from configuration
+ // Get endpoint and API key from configuration
endpoint = configurationService.getProperty("sherpa.romeo.url",
"https://v2.sherpa.ac.uk/cgi/retrieve");
apiKey = configurationService.getProperty("sherpa.romeo.apikey");
@@ -132,46 +117,47 @@ public SHERPAPublisherResponse performPublisherRequest(String type, String field
timeout,
sleepBetweenTimeouts));
- try {
+ try (CloseableHttpClient client = DSpaceHttpClientFactory.getInstance().buildWithoutAutomaticRetries(5)) {
Thread.sleep(sleepBetweenTimeouts);
// Construct a default HTTP method (first result)
method = constructHttpGet(type, field, predicate, value, start, limit);
// Execute the method
- HttpResponse response = client.execute(method);
- int statusCode = response.getStatusLine().getStatusCode();
+ try (CloseableHttpResponse response = client.execute(method)) {
+ int statusCode = response.getStatusLine().getStatusCode();
- log.debug(response.getStatusLine().getStatusCode() + ": "
- + response.getStatusLine().getReasonPhrase());
+ log.debug(response.getStatusLine().getStatusCode() + ": "
+ + response.getStatusLine().getReasonPhrase());
- if (statusCode != HttpStatus.SC_OK) {
- sherpaResponse = new SHERPAPublisherResponse("SHERPA/RoMEO return not OK status: "
- + statusCode);
- String errorBody = IOUtils.toString(response.getEntity().getContent(), StandardCharsets.UTF_8);
- log.error("Error from SHERPA HTTP request: " + errorBody);
- }
+ if (statusCode != HttpStatus.SC_OK) {
+ sherpaResponse = new SHERPAPublisherResponse("SHERPA/RoMEO return not OK status: "
+ + statusCode);
+ String errorBody = IOUtils.toString(response.getEntity().getContent(), StandardCharsets.UTF_8);
+ log.error("Error from SHERPA HTTP request: " + errorBody);
+ }
- HttpEntity responseBody = response.getEntity();
-
- // If the response body is valid, pass to SHERPAResponse for parsing as JSON
- if (null != responseBody) {
- log.debug("Non-null SHERPA resonse received for query of " + value);
- InputStream content = null;
- try {
- content = responseBody.getContent();
- sherpaResponse =
- new SHERPAPublisherResponse(content, SHERPAPublisherResponse.SHERPAFormat.JSON);
- } catch (IOException e) {
- log.error("Encountered exception while contacting SHERPA/RoMEO: " + e.getMessage(), e);
- } finally {
- if (content != null) {
- content.close();
+ HttpEntity responseBody = response.getEntity();
+
+ // If the response body is valid, pass to SHERPAResponse for parsing as JSON
+ if (null != responseBody) {
+ log.debug("Non-null SHERPA response received for query of " + value);
+ InputStream content = null;
+ try {
+ content = responseBody.getContent();
+ sherpaResponse =
+ new SHERPAPublisherResponse(content, SHERPAPublisherResponse.SHERPAFormat.JSON);
+ } catch (IOException e) {
+ log.error("Encountered exception while contacting SHERPA/RoMEO: " + e.getMessage(), e);
+ } finally {
+ if (content != null) {
+ content.close();
+ }
}
+ } else {
+ log.debug("Empty SHERPA response body for query on " + value);
+ sherpaResponse = new SHERPAPublisherResponse("SHERPA/RoMEO returned no response");
}
- } else {
- log.debug("Empty SHERPA response body for query on " + value);
- sherpaResponse = new SHERPAPublisherResponse("SHERPA/RoMEO returned no response");
}
} catch (URISyntaxException e) {
String errorMessage = "Error building SHERPA v2 API URI: " + e.getMessage();
@@ -235,45 +221,46 @@ public SHERPAResponse performRequest(String type, String field, String predicate
timeout,
sleepBetweenTimeouts));
- try {
+ try (CloseableHttpClient client = DSpaceHttpClientFactory.getInstance().buildWithoutAutomaticRetries(5)) {
Thread.sleep(sleepBetweenTimeouts);
// Construct a default HTTP method (first result)
method = constructHttpGet(type, field, predicate, value, start, limit);
// Execute the method
- HttpResponse response = client.execute(method);
- int statusCode = response.getStatusLine().getStatusCode();
+ try (CloseableHttpResponse response = client.execute(method)) {
+ int statusCode = response.getStatusLine().getStatusCode();
- log.debug(response.getStatusLine().getStatusCode() + ": "
- + response.getStatusLine().getReasonPhrase());
+ log.debug(response.getStatusLine().getStatusCode() + ": "
+ + response.getStatusLine().getReasonPhrase());
- if (statusCode != HttpStatus.SC_OK) {
- sherpaResponse = new SHERPAResponse("SHERPA/RoMEO return not OK status: "
- + statusCode);
- String errorBody = IOUtils.toString(response.getEntity().getContent(), StandardCharsets.UTF_8);
- log.error("Error from SHERPA HTTP request: " + errorBody);
- }
+ if (statusCode != HttpStatus.SC_OK) {
+ sherpaResponse = new SHERPAResponse("SHERPA/RoMEO return not OK status: "
+ + statusCode);
+ String errorBody = IOUtils.toString(response.getEntity().getContent(), StandardCharsets.UTF_8);
+ log.error("Error from SHERPA HTTP request: " + errorBody);
+ }
- HttpEntity responseBody = response.getEntity();
-
- // If the response body is valid, pass to SHERPAResponse for parsing as JSON
- if (null != responseBody) {
- log.debug("Non-null SHERPA resonse received for query of " + value);
- InputStream content = null;
- try {
- content = responseBody.getContent();
- sherpaResponse = new SHERPAResponse(content, SHERPAResponse.SHERPAFormat.JSON);
- } catch (IOException e) {
- log.error("Encountered exception while contacting SHERPA/RoMEO: " + e.getMessage(), e);
- } finally {
- if (content != null) {
- content.close();
+ HttpEntity responseBody = response.getEntity();
+
+ // If the response body is valid, pass to SHERPAResponse for parsing as JSON
+ if (null != responseBody) {
+ log.debug("Non-null SHERPA response received for query of " + value);
+ InputStream content = null;
+ try {
+ content = responseBody.getContent();
+ sherpaResponse = new SHERPAResponse(content, SHERPAResponse.SHERPAFormat.JSON);
+ } catch (IOException e) {
+ log.error("Encountered exception while contacting SHERPA/RoMEO: " + e.getMessage(), e);
+ } finally {
+ if (content != null) {
+ content.close();
+ }
}
+ } else {
+ log.debug("Empty SHERPA response body for query on " + value);
+ sherpaResponse = new SHERPAResponse("SHERPA/RoMEO returned no response");
}
- } else {
- log.debug("Empty SHERPA response body for query on " + value);
- sherpaResponse = new SHERPAResponse("SHERPA/RoMEO returned no response");
}
} catch (URISyntaxException e) {
String errorMessage = "Error building SHERPA v2 API URI: " + e.getMessage();
@@ -283,7 +270,7 @@ public SHERPAResponse performRequest(String type, String field, String predicate
String errorMessage = "Encountered exception while contacting SHERPA/RoMEO: " + e.getMessage();
log.error(errorMessage, e);
sherpaResponse = new SHERPAResponse(errorMessage);
- } catch (InterruptedException e) {
+ } catch (InterruptedException e) {
String errorMessage = "Encountered exception while sleeping thread: " + e.getMessage();
log.error(errorMessage, e);
sherpaResponse = new SHERPAResponse(errorMessage);
diff --git a/dspace-api/src/main/java/org/dspace/app/sitemap/GenerateSitemaps.java b/dspace-api/src/main/java/org/dspace/app/sitemap/GenerateSitemaps.java
index 90962d12aa75..41b0b0f6b3dd 100644
--- a/dspace-api/src/main/java/org/dspace/app/sitemap/GenerateSitemaps.java
+++ b/dspace-api/src/main/java/org/dspace/app/sitemap/GenerateSitemaps.java
@@ -7,6 +7,8 @@
*/
package org.dspace.app.sitemap;
+import static org.dspace.discovery.SearchUtils.RESOURCE_TYPE_FIELD;
+
import java.io.File;
import java.io.IOException;
import java.sql.SQLException;
@@ -189,7 +191,8 @@ public static void generateSitemaps(boolean makeHTMLMap, boolean makeSitemapOrg)
try {
DiscoverQuery discoveryQuery = new DiscoverQuery();
discoveryQuery.setMaxResults(PAGE_SIZE);
- discoveryQuery.setQuery("search.resourcetype:Community");
+ discoveryQuery.setQuery("*:*");
+ discoveryQuery.addFilterQueries(RESOURCE_TYPE_FIELD + ":Community");
do {
discoveryQuery.setStart(offset);
DiscoverResult discoverResult = searchService.search(c, discoveryQuery);
@@ -213,7 +216,8 @@ public static void generateSitemaps(boolean makeHTMLMap, boolean makeSitemapOrg)
offset = 0;
discoveryQuery = new DiscoverQuery();
discoveryQuery.setMaxResults(PAGE_SIZE);
- discoveryQuery.setQuery("search.resourcetype:Collection");
+ discoveryQuery.setQuery("*:*");
+ discoveryQuery.addFilterQueries(RESOURCE_TYPE_FIELD + ":Collection");
do {
discoveryQuery.setStart(offset);
DiscoverResult discoverResult = searchService.search(c, discoveryQuery);
@@ -237,7 +241,8 @@ public static void generateSitemaps(boolean makeHTMLMap, boolean makeSitemapOrg)
offset = 0;
discoveryQuery = new DiscoverQuery();
discoveryQuery.setMaxResults(PAGE_SIZE);
- discoveryQuery.setQuery("search.resourcetype:Item");
+ discoveryQuery.setQuery("*:*");
+ discoveryQuery.addFilterQueries(RESOURCE_TYPE_FIELD + ":Item");
discoveryQuery.addSearchField("search.entitytype");
do {
diff --git a/dspace-api/src/main/java/org/dspace/app/solrdatabaseresync/SolrDatabaseResyncCli.java b/dspace-api/src/main/java/org/dspace/app/solrdatabaseresync/SolrDatabaseResyncCli.java
index aac42ce1acf9..0ce6b1a9ef3d 100644
--- a/dspace-api/src/main/java/org/dspace/app/solrdatabaseresync/SolrDatabaseResyncCli.java
+++ b/dspace-api/src/main/java/org/dspace/app/solrdatabaseresync/SolrDatabaseResyncCli.java
@@ -98,7 +98,8 @@ public void internalRun() throws Exception {
private void performStatusUpdate(Context context) throws SearchServiceException, SolrServerException, IOException {
SolrQuery solrQuery = new SolrQuery();
- solrQuery.setQuery(STATUS_FIELD + ":" + STATUS_FIELD_PREDB);
+ solrQuery.setQuery("*:*");
+ solrQuery.addFilterQuery(STATUS_FIELD + ":" + STATUS_FIELD_PREDB);
solrQuery.addFilterQuery(SearchUtils.RESOURCE_TYPE_FIELD + ":" + IndexableItem.TYPE);
String dateRangeFilter = SearchUtils.LAST_INDEXED_FIELD + ":[* TO " + maxTime + "]";
logDebugAndOut("Date range filter used; " + dateRangeFilter);
diff --git a/dspace-api/src/main/java/org/dspace/app/statistics/LogAnalyser.java b/dspace-api/src/main/java/org/dspace/app/statistics/LogAnalyser.java
index 2e4ed69b268e..c787261419f8 100644
--- a/dspace-api/src/main/java/org/dspace/app/statistics/LogAnalyser.java
+++ b/dspace-api/src/main/java/org/dspace/app/statistics/LogAnalyser.java
@@ -281,10 +281,14 @@ public class LogAnalyser {
*/
private static String fileTemplate = "dspace\\.log.*";
+ private static final ConfigurationService configurationService =
+ DSpaceServicesFactory.getInstance().getConfigurationService();
+
/**
* the configuration file from which to configure the analyser
*/
- private static String configFile;
+ private static String configFile = configurationService.getProperty("dspace.dir")
+ + File.separator + "config" + File.separator + "dstat.cfg";
/**
* the output file to which to write aggregation data
@@ -616,8 +620,6 @@ public static String processLogs(Context context, String myLogDir,
}
// now do the host name and url lookup
- ConfigurationService configurationService
- = DSpaceServicesFactory.getInstance().getConfigurationService();
hostName = Utils.getHostName(configurationService.getProperty("dspace.ui.url"));
name = configurationService.getProperty("dspace.name").trim();
url = configurationService.getProperty("dspace.ui.url").trim();
@@ -658,8 +660,6 @@ public static void setParameters(String myLogDir, String myFileTemplate,
String myConfigFile, String myOutFile,
Date myStartDate, Date myEndDate,
boolean myLookUp) {
- ConfigurationService configurationService
- = DSpaceServicesFactory.getInstance().getConfigurationService();
if (myLogDir != null) {
logDir = myLogDir;
@@ -673,9 +673,6 @@ public static void setParameters(String myLogDir, String myFileTemplate,
if (myConfigFile != null) {
configFile = myConfigFile;
- } else {
- configFile = configurationService.getProperty("dspace.dir")
- + File.separator + "config" + File.separator + "dstat.cfg";
}
if (myStartDate != null) {
diff --git a/dspace-api/src/main/java/org/dspace/app/statistics/package.html b/dspace-api/src/main/java/org/dspace/app/statistics/package.html
index a6d8d8699cf7..931a7039080d 100644
--- a/dspace-api/src/main/java/org/dspace/app/statistics/package.html
+++ b/dspace-api/src/main/java/org/dspace/app/statistics/package.html
@@ -46,8 +46,6 @@
writes event records to the Java logger.
{@link org.dspace.statistics.SolrLoggerUsageEventListener SolrLoggerUsageEventListener}
writes event records to Solr.
- {@link org.dspace.google.GoogleRecorderEventListener GoogleRecorderEventListener}<.dt>
- writes event records to Google Analytics.