From 0615e648f2fbc614c4041b1ed31cc111f0050803 Mon Sep 17 00:00:00 2001 From: "Gantner, Florian Klaus" Date: Fri, 3 Feb 2023 09:29:19 +0100 Subject: [PATCH 01/19] name enhancer for person entity Enhance one dc.title from a list of other metadatafields. Language and Security are not considered. Some optional default value can be specified. --- .../content/enhancer/impl/NameEnhancer.java | 153 ++++++++++++++++++ .../config/spring/api/metadata-enhancers.xml | 13 ++ 2 files changed, 166 insertions(+) create mode 100644 dspace-api/src/main/java/org/dspace/content/enhancer/impl/NameEnhancer.java diff --git a/dspace-api/src/main/java/org/dspace/content/enhancer/impl/NameEnhancer.java b/dspace-api/src/main/java/org/dspace/content/enhancer/impl/NameEnhancer.java new file mode 100644 index 000000000000..60cc672995ec --- /dev/null +++ b/dspace-api/src/main/java/org/dspace/content/enhancer/impl/NameEnhancer.java @@ -0,0 +1,153 @@ +/** + * The contents of this file are subject to the license and copyright + * detailed in the LICENSE and NOTICE files at the root of the source + * tree and available online at + * + * http://www.dspace.org/license/ + */ +package org.dspace.content.enhancer.impl; + +import java.sql.SQLException; +import java.util.List; +import java.util.Objects; + +import org.apache.commons.lang.StringUtils; +import org.dspace.content.Item; +import org.dspace.content.MetadataField; +import org.dspace.content.MetadataValue; +import org.dspace.content.enhancer.AbstractItemEnhancer; +import org.dspace.content.enhancer.ItemEnhancer; +import org.dspace.content.service.ItemService; +import org.dspace.content.service.MetadataFieldService; +import org.dspace.core.Context; +import org.dspace.core.exception.SQLRuntimeException; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; + +/** + * Implementation of {@link ItemEnhancer} that add metadata values on the given + * item from a list of other metadatavalues . + * The first value of the list that is not null matches and overwrites the existing value. + * Some default value can be specified if no fields were found + * + * @author Florian Gantner (florian.gantner@uni-bamberg.de) + * + */ +public class NameEnhancer extends AbstractItemEnhancer { + + private static final Logger LOGGER = LoggerFactory.getLogger(NameEnhancer.class); + + @Autowired + private ItemService itemService; + + @Autowired + private MetadataFieldService metadatafieldService; + + private String sourceEntityType; + + private String targetItemMetadataField; + + private List relatedItemMetadataFields; + + private String defaultValue; + + @Override + public boolean canEnhance(Context context, Item item) { + return sourceEntityType == null || sourceEntityType.equals(itemService.getEntityType(item)); + } + + @Override + public void enhance(Context context, Item item) { + try { + if (Objects.isNull(relatedItemMetadataFields) || relatedItemMetadataFields.isEmpty() || StringUtils.isBlank(targetItemMetadataField)) return; + checkNames(context, item); + } catch (SQLException e) { + LOGGER.error("An error occurs enhancing item with id {}: {}", item.getID(), e.getMessage(), e); + throw new SQLRuntimeException(e); + } + } + + private void checkNames(Context context, Item item) throws SQLException { + // ignore languages of Metadata here. Assume main title is not repeated + // Could by more simplified + List currentnames = itemService.getMetadataByMetadataString(item, targetItemMetadataField); + + if (!currentnames.isEmpty()) { + // some name assigned yet + for (MetadataValue currentname : currentnames) { + String val = currentname.getValue(); + fields: for (String field : relatedItemMetadataFields) { + List fieldnames = itemService.getMetadataByMetadataString(item, field); + if (fieldnames.isEmpty()) continue fields; //No Values, try next loop + for (MetadataValue fieldname : fieldnames) { + if (StringUtils.isNotBlank(fieldname.getValue()) + && fieldname.getValue().contentEquals(val)) { + //Values are the same. No Update necessary + return; + } else { + //values differ. We must update the value + updateTargetMetadata(context, item, fieldname.getValue(), true); + return; + } + } + } + } + if (StringUtils.isNotBlank(defaultValue) + && !currentnames.get(0).getValue().contentEquals(defaultValue)) { + // None of the names above matches. Set Default-Value, if exist. Otherwise do nothing + updateTargetMetadata(context, item, defaultValue, true); + } + + } else { + // No Name assigned yet + // Check existing names + fields: for (String field : relatedItemMetadataFields) { + List fieldnames = itemService.getMetadataByMetadataString(item, field); + if (fieldnames.isEmpty()) continue fields; //No Values, try next loop + for (MetadataValue fieldname : fieldnames) { + if (StringUtils.isNotBlank(fieldname.getValue())) { + //Got some value + updateTargetMetadata(context, item, fieldname.getValue(), false); + return; + } + } + } + // If no name exist, set defaultvalue + if (StringUtils.isNotBlank(defaultValue)) { + updateTargetMetadata(context, item, defaultValue, false); + } + // otherwise do not assign any value + } + } + + private void updateTargetMetadata(Context context, Item item, String value, boolean clear) throws SQLException { + MetadataField targetmd = metadatafieldService.findByString(context, targetItemMetadataField, '.'); + if (targetmd != null){ + if (clear) { + itemService.clearMetadata(context, item, targetmd.getMetadataSchema().getName(), targetmd.getElement(), targetmd.getQualifier(), Item.ANY); + } + itemService.addMetadata(context, item, targetmd, null, value); + } else { + LOGGER.error("No valid metadatavalue to enhance specified"); + } + } + + public void setSourceEntityType(String sourceEntityType) { + this.sourceEntityType = sourceEntityType; + } + + public void setTargetItemMetadataField(String targetItemMetadataField) { + this.targetItemMetadataField = targetItemMetadataField; + } + + public void setRelatedItemMetadataFields(List relatedItemMetadataFields) { + this.relatedItemMetadataFields = relatedItemMetadataFields; + } + + public void setDefaultValue(String defaultvalue) { + this.defaultValue = defaultvalue; + } + +} + diff --git a/dspace/config/spring/api/metadata-enhancers.xml b/dspace/config/spring/api/metadata-enhancers.xml index e36727959c25..26e656cf7ac2 100644 --- a/dspace/config/spring/api/metadata-enhancers.xml +++ b/dspace/config/spring/api/metadata-enhancers.xml @@ -57,6 +57,19 @@ + + + + + + + + crisrp.preferredName + crisrp.fullName + + + + From 4b8e3e3b0e9e3bcbfb469251734e8958ef6e3d7b Mon Sep 17 00:00:00 2001 From: "Gantner, Florian Klaus" Date: Fri, 3 Feb 2023 09:36:27 +0100 Subject: [PATCH 02/19] item enhancer for collections and entity type extended item enhancer. Items can be limit to entity type or collection. Some pagination (max entries) can be defined for testing and some limit after which some commit to the database is made. --- .../script/ItemEnhancerEntityTypeScript.java | 304 ++++++++++++++++++ ...EnhancerEntityTypeScriptConfiguration.java | 67 ++++ dspace/config/spring/api/scripts.xml | 5 + 3 files changed, 376 insertions(+) create mode 100644 dspace-api/src/main/java/org/dspace/content/enhancer/script/ItemEnhancerEntityTypeScript.java create mode 100644 dspace-api/src/main/java/org/dspace/content/enhancer/script/ItemEnhancerEntityTypeScriptConfiguration.java diff --git a/dspace-api/src/main/java/org/dspace/content/enhancer/script/ItemEnhancerEntityTypeScript.java b/dspace-api/src/main/java/org/dspace/content/enhancer/script/ItemEnhancerEntityTypeScript.java new file mode 100644 index 000000000000..bc7c0a1e5422 --- /dev/null +++ b/dspace-api/src/main/java/org/dspace/content/enhancer/script/ItemEnhancerEntityTypeScript.java @@ -0,0 +1,304 @@ +/** + * The contents of this file are subject to the license and copyright + * detailed in the LICENSE and NOTICE files at the root of the source + * tree and available online at + * + * http://www.dspace.org/license/ + */ +package org.dspace.content.enhancer.script; + +import java.sql.SQLException; +import java.util.Iterator; +import java.util.Objects; +import java.util.UUID; + +import org.apache.commons.cli.ParseException; +import org.dspace.content.Collection; +import org.dspace.content.Item; +import org.dspace.content.enhancer.service.ItemEnhancerService; +import org.dspace.content.factory.ContentServiceFactory; +import org.dspace.content.service.CollectionService; +import org.dspace.content.service.EntityTypeService; +import org.dspace.content.service.ItemService; +import org.dspace.core.Context; +import org.dspace.core.exception.SQLRuntimeException; +import org.dspace.eperson.EPerson; +import org.dspace.eperson.factory.EPersonServiceFactory; +import org.dspace.scripts.DSpaceRunnable; +import org.dspace.util.UUIDUtils; +import org.dspace.utils.DSpace; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Script that allows to enhance items, also forcing the updating of the + * calculated metadata with the enhancement. + * + * @author Luca Giamminonni (luca.giamminonni at 4science.it) + * + * Extended to limit the item set to collection/entity wide use to speed up process and recalculation of certain entities. + * Extended to use some pagination option with some limit where some commit to the database is made + * + * @author florian.gantner@uni-bamberg.de + * + */ +public class ItemEnhancerEntityTypeScript extends DSpaceRunnable> { + + private ItemService itemService; + + private CollectionService collectionService; + + private ItemEnhancerService itemEnhancerService; + + private boolean force; + + private UUID collection; + + private String entitytype; + + private Context context; + + private int limit; + + private int max; + + private int offset; + + private EntityTypeService entityTypeService; + + private static final Logger log = LoggerFactory.getLogger(ItemEnhancerEntityTypeScript.class); + + @Override + public void setup() throws ParseException { + + this.itemService = ContentServiceFactory.getInstance().getItemService(); + this.collectionService = ContentServiceFactory.getInstance().getCollectionService(); + this.entityTypeService = ContentServiceFactory.getInstance().getEntityTypeService(); + itemEnhancerService = new DSpace().getSingletonService(ItemEnhancerService.class); + + this.force = commandLine.hasOption('f'); + if (commandLine.hasOption('c')) { + this.collection = UUIDUtils.fromString(commandLine.getOptionValue('c')); + } + if (commandLine.hasOption('e')) { + this.entitytype = commandLine.getOptionValue('e'); + } + if (commandLine.hasOption('l')) { + try{ + this.limit = Integer.parseInt(commandLine.getOptionValue('l')); + }catch (Exception e){ + // + } + } + if (commandLine.hasOption('m')) { + try{ + this.max = Integer.parseInt(commandLine.getOptionValue('m')); + }catch (Exception e){ + // + } + } + if (commandLine.hasOption('o')) { + try{ + this.offset = Integer.parseInt(commandLine.getOptionValue('o')); + }catch (Exception e){ + // + } + } + } + + @Override + public void internalRun() throws Exception { + context = new Context(); + assignCurrentUserInContext(); + assignSpecialGroupsInContext(); + if (commandLine.hasOption('e') && Objects.isNull(entityTypeService.findByEntityType(context, entitytype))) { + throw new Exception("unknown entity " + entitytype); + } + if (commandLine.hasOption('c') && (Objects.isNull(collection) || Objects.isNull(this.collectionService.find(context, collection)))) { + throw new Exception("specified collection does not exist"); + } + context.turnOffAuthorisationSystem(); + try { + enhanceItems(); + context.complete(); + handler.logInfo("Enhancement completed with success"); + } catch (Exception e) { + handler.handleException("An error occurs during enhancement. The process is aborted", e); + context.abort(); + } finally { + context.restoreAuthSystemState(); + } + } + + private void enhanceItems() { + + if (this.limit > 0) { + //use limit and max parameters for pagination. + findItemsToEnhanceLimitMax(); + } else { + //common way + if (Objects.nonNull(entitytype)) { + findItemsToEnhance().forEachRemaining(this::enhanceItemEntityCheck); + } else { + findItemsToEnhance().forEachRemaining(this::enhanceItem); + } + } + } + + private void findItemsToEnhanceLimitMax(){ + Collection coll; + int total, counter = 0; + if (Objects.nonNull(collection)) { + //Paginate through items in one collection + try { + coll = collectionService.find(context, collection); + total = itemService.countItems(context, coll); + } catch (SQLException e){ + handler.logError(e.getMessage()); + return; + } + if ( this.max > 0) total = this.max; + if (this.offset > 0) { + //offset is being added to counter and offset + total += offset; + counter += offset; + handler.logInfo("offset " + offset + " added. Range: [" + counter + " to " + total + "] in " + limit + " steps"); + log.info("offset " + offset + " added. Range: [" + counter + " to " + total + "] in " + limit + " steps"); + } else { + handler.logInfo("Range: [" + counter + " to " + total + "] in " + limit + " steps"); + log.info("Range: [" + counter + " to " + total + "] in " + limit + " steps"); + } + while (counter < total) { + try { + itemService.findAllByCollection(context, coll, limit, counter).forEachRemaining(this::enhanceItem); + counter += limit; + context.commit(); + handler.logInfo("processed " + counter + " out of total " + total + " items"); + log.info("processed " + counter + " out of total " + total + " items"); + } catch (SQLException e){ + handler.logError(e.getMessage()); + counter += limit; + handler.logInfo("processed " + counter + " out of total " + total + " items"); + log.info("processed " + counter + " out of total " + total + " items"); + } + } + } else { + //loop through all! + try { + total = itemService.countTotal(context); + } catch (SQLException e){ + handler.logError(e.getMessage()); + return; + } + if (this.max > 0) total = this.max; + if (this.offset > 0) { + //offset is being added to counter and offset + total += offset; + counter += offset; + handler.logInfo("offset" + offset + " added. Range: [" + counter + " to " + total + "] in " + limit + " steps"); + log.info("offset" + offset + " added. Range: [" + counter + " to " + total + "] in " + limit + " steps"); + } else { + handler.logInfo("Range: [" + counter + " to " + total + "] in " + limit + " steps"); + log.info("Range: [" + counter + " to " + total + "] in " + limit + " steps"); + } + while (counter < total) { + try { + //No Entity check here! + if (Objects.nonNull(this.entitytype)) { + itemService.findAll(context, limit, counter).forEachRemaining(this::enhanceItemEntityCheck); + } else { + itemService.findAll(context, limit, counter).forEachRemaining(this::enhanceItem); + } + counter += limit; + context.commit(); + handler.logInfo("processed " + counter + " out of total " + total + " items"); + log.info("processed " + counter + " out of total " + total + " items"); + } catch (SQLException e){ + handler.logError(e.getMessage()); + counter += limit; + handler.logInfo("processed " + counter + " out of total " + total + " items"); + log.info("processed " + counter + " out of total " + total + " items"); + } + } + } + + } + + private Iterator findItemsToEnhance() { + try { + Iterator result = null; + if (Objects.nonNull(collection)) { + //Check, if uuid exist + Collection coll = collectionService.find(context, collection); + if(coll != null) { + result = itemService.findAllByCollection(context, coll); + } + } else { + result = itemService.findAll(context); + } + return result; + } catch (SQLException e) { + throw new SQLRuntimeException(e); + } + } + + private void enhanceItem(Item item) { + if (force) { + itemEnhancerService.forceEnhancement(context, item); + } else { + itemEnhancerService.enhance(context, item); + } + + uncacheItem(item); + + } + + /** + * Additional Entity Check. + */ + private void enhanceItemEntityCheck(Item item) { + if (Objects.nonNull(entitytype)) { + if (!entitytype.contentEquals(itemService.getEntityType(item))) { + uncacheItem(item); + } + } else { + if (force) { + itemEnhancerService.forceEnhancement(context, item); + } else { + itemEnhancerService.enhance(context, item); + } + + uncacheItem(item); + } + } + + private void uncacheItem(Item item) { + try { + context.uncacheEntity(item); + } catch (SQLException e) { + throw new SQLRuntimeException(e); + } + } + + private void assignCurrentUserInContext() throws SQLException { + UUID uuid = getEpersonIdentifier(); + if (uuid != null) { + EPerson ePerson = EPersonServiceFactory.getInstance().getEPersonService().find(context, uuid); + context.setCurrentUser(ePerson); + } + } + + private void assignSpecialGroupsInContext() throws SQLException { + for (UUID uuid : handler.getSpecialGroups()) { + context.setSpecialGroup(uuid); + } + } + + @Override + @SuppressWarnings("unchecked") + public ItemEnhancerEntityTypeScriptConfiguration getScriptConfiguration() { + return new DSpace().getServiceManager().getServiceByName("item-enhancer-type", + ItemEnhancerEntityTypeScriptConfiguration.class); + } + +} diff --git a/dspace-api/src/main/java/org/dspace/content/enhancer/script/ItemEnhancerEntityTypeScriptConfiguration.java b/dspace-api/src/main/java/org/dspace/content/enhancer/script/ItemEnhancerEntityTypeScriptConfiguration.java new file mode 100644 index 000000000000..b45f5e983a17 --- /dev/null +++ b/dspace-api/src/main/java/org/dspace/content/enhancer/script/ItemEnhancerEntityTypeScriptConfiguration.java @@ -0,0 +1,67 @@ +/** + * The contents of this file are subject to the license and copyright + * detailed in the LICENSE and NOTICE files at the root of the source + * tree and available online at + * + * http://www.dspace.org/license/ + */ +package org.dspace.content.enhancer.script; + +import java.sql.SQLException; + +import org.apache.commons.cli.Options; +import org.dspace.authorize.service.AuthorizeService; +import org.dspace.core.Context; +import org.dspace.scripts.configuration.ScriptConfiguration; +import org.springframework.beans.factory.annotation.Autowired; + +/** + * Script configuration of {@link ItemEnhancerEntityTypeScript}. + * + * @author Luca Giamminonni (luca.giamminonni at 4science.it) + * @author Florian Gantner (florian.gantner@uni-bamberg.de) + */ +public class ItemEnhancerEntityTypeScriptConfiguration extends ScriptConfiguration { + + @Autowired + private AuthorizeService authorizeService; + + private Class dspaceRunnableClass; + + @Override + public boolean isAllowedToExecute(Context context) { + try { + return authorizeService.isAdmin(context); + } catch (SQLException e) { + throw new RuntimeException("SQLException occurred when checking if the current user is an admin", e); + } + } + + @Override + public Options getOptions() { + if (options == null) { + Options options = new Options(); + + options.addOption("f", "force", false, "force the recalculation of all the virtual fields"); + options.addOption("c", "collection", true, "uuid of the collection. If the collection does not exist the script aborts."); + options.addOption("e", "entity", true, "Entity type of the items. Cannot be combined with limit/max option"); + options.addOption("l", "limit", true, "size for iterator --limit items and commit after --limit items"); + options.addOption("m", "max", true, "process max --max items"); + options.addOption("o", "offset", true, "offset of items to start --offset items from the start"); + + super.options = options; + } + return options; + } + + @Override + public Class getDspaceRunnableClass() { + return dspaceRunnableClass; + } + + @Override + public void setDspaceRunnableClass(Class dspaceRunnableClass) { + this.dspaceRunnableClass = dspaceRunnableClass; + } + +} diff --git a/dspace/config/spring/api/scripts.xml b/dspace/config/spring/api/scripts.xml index de14c342c479..0b7fdca833f1 100644 --- a/dspace/config/spring/api/scripts.xml +++ b/dspace/config/spring/api/scripts.xml @@ -134,6 +134,11 @@ + + + + + From 13677d80738608f57c6f43aec7978f0494adb4ff Mon Sep 17 00:00:00 2001 From: "Gantner, Florian Klaus" Date: Fri, 3 Feb 2023 09:39:53 +0100 Subject: [PATCH 03/19] item enhancer for collections and entity type and modifiedDate using solr extended item enhancer based on the solr search. Items can be limit to entity type or collection. Some pagination (max entries) can be defined for testing and some limit after which some commit to the database is made. Additional some solr-query can be defined (e.g. enhance all items of some author/matching the query result) and some solr-based date range to only enhance items lastModified between two dates. --- .../script/ItemEnhancerByDateScript.java | 277 ++++++++++++++++++ ...ItemEnhancerByDateScriptConfiguration.java | 69 +++++ dspace/config/spring/api/scripts.xml | 5 + 3 files changed, 351 insertions(+) create mode 100644 dspace-api/src/main/java/org/dspace/content/enhancer/script/ItemEnhancerByDateScript.java create mode 100644 dspace-api/src/main/java/org/dspace/content/enhancer/script/ItemEnhancerByDateScriptConfiguration.java diff --git a/dspace-api/src/main/java/org/dspace/content/enhancer/script/ItemEnhancerByDateScript.java b/dspace-api/src/main/java/org/dspace/content/enhancer/script/ItemEnhancerByDateScript.java new file mode 100644 index 000000000000..c5f77a5a66b5 --- /dev/null +++ b/dspace-api/src/main/java/org/dspace/content/enhancer/script/ItemEnhancerByDateScript.java @@ -0,0 +1,277 @@ +/** + * The contents of this file are subject to the license and copyright + * detailed in the LICENSE and NOTICE files at the root of the source + * tree and available online at + * + * http://www.dspace.org/license/ + */ +package org.dspace.content.enhancer.script; + +import org.apache.commons.cli.ParseException; +import org.apache.solr.client.solrj.SolrQuery; +import org.apache.solr.client.solrj.SolrServerException; +import org.apache.solr.client.solrj.response.QueryResponse; +import org.apache.solr.client.solrj.response.SolrPingResponse; +import org.apache.solr.common.SolrDocument; +import org.apache.solr.common.SolrDocumentList; +import org.dspace.content.Item; +import org.dspace.content.enhancer.service.ItemEnhancerService; +import org.dspace.content.factory.ContentServiceFactory; +import org.dspace.content.service.CollectionService; +import org.dspace.content.service.EntityTypeService; +import org.dspace.content.service.ItemService; +import org.dspace.core.Context; +import org.dspace.core.exception.SQLRuntimeException; +import org.dspace.discovery.SearchUtils; +import org.dspace.discovery.SolrSearchCore; +import org.dspace.eperson.EPerson; +import org.dspace.eperson.factory.EPersonServiceFactory; +import org.dspace.scripts.DSpaceRunnable; +import org.dspace.util.UUIDUtils; +import org.dspace.utils.DSpace; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; +import java.sql.SQLException; +import java.util.*; + +/** + * Script that allows to enhance items, also forcing the updating of the + * calculated metadata with the enhancement. + * + * @author Luca Giamminonni (luca.giamminonni at 4science.it) + * + * Extended to solr search to discover not-enhanced entities. + * Some Query can be delivered, + * additional some daterange filterquery on the lastModified field + * and additional collection/entitytype queries as filterfacets. + * + * @author florian.gantner@uni-bamberg.de + * + */ +public class ItemEnhancerByDateScript extends DSpaceRunnable> { + + private ItemService itemService; + + private CollectionService collectionService; + + private ItemEnhancerService itemEnhancerService; + + protected SolrSearchCore solrSearchCore; + + private boolean force; + + private UUID collection; + + private String entitytype; + + private String query; + + private String dateupper; + + private String datelower; + + private Context context; + + private int max; + + private int limit; + + private EntityTypeService entityTypeService; + + private static final Logger log = LoggerFactory.getLogger(ItemEnhancerByDateScript.class); + + @Override + public void setup() throws ParseException { + + this.itemService = ContentServiceFactory.getInstance().getItemService(); + this.collectionService = ContentServiceFactory.getInstance().getCollectionService(); + this.entityTypeService = ContentServiceFactory.getInstance().getEntityTypeService(); + itemEnhancerService = new DSpace().getSingletonService(ItemEnhancerService.class); + this.solrSearchCore = new DSpace().getSingletonService(SolrSearchCore.class); + + this.force = commandLine.hasOption('f'); + if (commandLine.hasOption('c')) { + this.collection = UUIDUtils.fromString(commandLine.getOptionValue('c')); + } + if (commandLine.hasOption('e')) { + this.entitytype = commandLine.getOptionValue('e'); + } + if (commandLine.hasOption('q')) { + this.query = commandLine.getOptionValue('q'); + } + if (commandLine.hasOption('d')) { + this.dateupper = commandLine.getOptionValue('d'); + } + if (commandLine.hasOption('s')) { + this.datelower = commandLine.getOptionValue('s'); + } + if (commandLine.hasOption('m')) { + try{ + this.max = Integer.parseInt(commandLine.getOptionValue('m')); + }catch (Exception e){ + // + } + } + if (commandLine.hasOption('l')) { + try{ + this.limit = Integer.parseInt(commandLine.getOptionValue('l')); + }catch (Exception e){ + // + } + } + } + + @Override + public void internalRun() throws Exception { + context = new Context(); + assignCurrentUserInContext(); + assignSpecialGroupsInContext(); + if (commandLine.hasOption('e') && Objects.isNull(entityTypeService.findByEntityType(context, entitytype))) { + throw new Exception("unknown entity " + entitytype); + } + if (commandLine.hasOption('c') && (Objects.isNull(collection) || Objects.isNull(this.collectionService.find(context, collection)))) { + throw new Exception("specified collection does not exist"); + } + SolrPingResponse ping = solrSearchCore.getSolr().ping(); + if(ping.getStatus() > 299) throw new Exception("Solr seems not to be available. Status" + ping.getStatus()); + + context.turnOffAuthorisationSystem(); + try { + searchItems(); + context.complete(); + handler.logInfo("Enhancement completed with success"); + } catch (Exception e) { + handler.handleException("An error occurs during enhancement. The process is aborted", e); + context.abort(); + } finally { + context.restoreAuthSystemState(); + } + } + + + private void searchItems() { + + List items = new ArrayList<>(); + try { + SolrDocumentList results = searchItemsInSolr(this.query, this.dateupper, this.datelower); + for (SolrDocument doc : results) { + String resourceid = (String) doc.getFieldValue(SearchUtils.RESOURCE_ID_FIELD); + if (Objects.nonNull(resourceid) && Objects.nonNull(UUIDUtils.fromString(resourceid))) items.add(resourceid); + } + } catch (SolrServerException | IOException e) { + handler.logError(e.getMessage(), e); + log.error(e.getMessage()); + } + int total = items.size(); + if (total == 0){ + handler.logInfo("No results in solr-Query"); + log.info("No results in solr-Query"); + return; + } + + if (this.limit > 0) { + // Split through every list + int counter = 0, start, end; + while (counter < total){ + start = counter; + end = counter + limit; + if(end > (total -1)) end = total - 1; + try { + this.itemService.findByIds(context, items.subList(start, end)).forEachRemaining(this::enhanceItem); + context.commit(); + counter += limit; + } catch (SQLException e) { + handler.logError(e.getMessage()); + counter += limit; + } + + } + handler.logInfo("enhanced " + total + " items"); + log.info("enhanced " + total + " items"); + + } else { + try { + this.itemService.findByIds(context, items).forEachRemaining(this::enhanceItem); + } catch (SQLException e) { + handler.logError(e.getMessage()); + } + handler.logInfo("enhanced " + total + " items"); + log.info("enhanced " + total + " items"); + } + } + + private SolrDocumentList searchItemsInSolr(String query, String datequeryupper, String datequerylower) throws SolrServerException, IOException { + SolrQuery sQuery; + if (Objects.nonNull(query)) { + sQuery = new SolrQuery(query); + } else { + sQuery = new SolrQuery("*"); + } + if(Objects.nonNull(datequeryupper) && Objects.nonNull(datequerylower)){ + sQuery.addFilterQuery("lastModified:["+datequerylower+" TO " + datequeryupper +"]"); + } else if (Objects.nonNull(datequeryupper) ){ + sQuery.addFilterQuery("lastModified:[* TO " + datequeryupper +"]"); + } else if (Objects.nonNull(datequerylower) ){ + sQuery.addFilterQuery("lastModified:[" + datequerylower + " TO *]"); + } + if (Objects.nonNull(entitytype)) { + sQuery.addFilterQuery("search.entitytype:"+entitytype); + } + sQuery.addFilterQuery(SearchUtils.RESOURCE_TYPE_FIELD+":Item"); + if (Objects.nonNull(collection)) { + sQuery.addFilterQuery("location.coll:"+UUIDUtils.toString(collection)); + } + sQuery.addField(SearchUtils.RESOURCE_ID_FIELD); + if (max > 0) { + sQuery.setRows(this.max); + } else { + sQuery.setRows(Integer.MAX_VALUE); + } + sQuery.setSort("lastModified_dt",SolrQuery.ORDER.asc); + handler.logInfo("Query Params:" + sQuery.toString()); + QueryResponse qResp = solrSearchCore.getSolr().query(sQuery); + return qResp.getResults(); + } + + private void enhanceItem(Item item) { + if (force) { + itemEnhancerService.forceEnhancement(context, item); + } else { + itemEnhancerService.enhance(context, item); + } + + uncacheItem(item); + } + + private void uncacheItem(Item item) { + try { + context.uncacheEntity(item); + } catch (SQLException e) { + throw new SQLRuntimeException(e); + } + } + + private void assignCurrentUserInContext() throws SQLException { + UUID uuid = getEpersonIdentifier(); + if (uuid != null) { + EPerson ePerson = EPersonServiceFactory.getInstance().getEPersonService().find(context, uuid); + context.setCurrentUser(ePerson); + } + } + + private void assignSpecialGroupsInContext() throws SQLException { + for (UUID uuid : handler.getSpecialGroups()) { + context.setSpecialGroup(uuid); + } + } + + @Override + @SuppressWarnings("unchecked") + public ItemEnhancerByDateScriptConfiguration getScriptConfiguration() { + return new DSpace().getServiceManager().getServiceByName("item-enhancer-date", + ItemEnhancerByDateScriptConfiguration.class); + } + +} diff --git a/dspace-api/src/main/java/org/dspace/content/enhancer/script/ItemEnhancerByDateScriptConfiguration.java b/dspace-api/src/main/java/org/dspace/content/enhancer/script/ItemEnhancerByDateScriptConfiguration.java new file mode 100644 index 000000000000..c3a2437ca9e6 --- /dev/null +++ b/dspace-api/src/main/java/org/dspace/content/enhancer/script/ItemEnhancerByDateScriptConfiguration.java @@ -0,0 +1,69 @@ +/** + * The contents of this file are subject to the license and copyright + * detailed in the LICENSE and NOTICE files at the root of the source + * tree and available online at + * + * http://www.dspace.org/license/ + */ +package org.dspace.content.enhancer.script; + +import org.apache.commons.cli.Options; +import org.dspace.authorize.service.AuthorizeService; +import org.dspace.core.Context; +import org.dspace.scripts.configuration.ScriptConfiguration; +import org.springframework.beans.factory.annotation.Autowired; + +import java.sql.SQLException; + +/** + * Script configuration of {@link ItemEnhancerEntityTypeScript}. + * + * @author Luca Giamminonni (luca.giamminonni at 4science.it) + * @author Florian Gantner (florian.gantner@uni-bamberg.de) + */ +public class ItemEnhancerByDateScriptConfiguration extends ScriptConfiguration { + + @Autowired + private AuthorizeService authorizeService; + + private Class dspaceRunnableClass; + + @Override + public boolean isAllowedToExecute(Context context) { + try { + return authorizeService.isAdmin(context); + } catch (SQLException e) { + throw new RuntimeException("SQLException occurred when checking if the current user is an admin", e); + } + } + + @Override + public Options getOptions() { + if (options == null) { + Options options = new Options(); + + options.addOption("f", "force", false, "force the recalculation of all the virtual fields"); + options.addOption("c", "collection", true, "uuid of the collection. If the collection does not exist the script aborts."); + options.addOption("e", "entity", true, "Entity type of the items. Cannot be combined with limit/max option"); + options.addOption("d", "dateupper", true, "iso date as upper range of date query. e.g. 2022-10-27T12:12:17.369Z "); + options.addOption("s", "datelower", true, "iso date as lower range of date query "); + options.addOption("m", "max", true, "--max results/rows from solr"); + options.addOption("l", "limit", true, "commit after --limit entities processed"); + options.addOption("q", "query", true, "additional filterquery for the entities. this can for example be the exclusion of some already enhanced metadata."); + + super.options = options; + } + return options; + } + + @Override + public Class getDspaceRunnableClass() { + return dspaceRunnableClass; + } + + @Override + public void setDspaceRunnableClass(Class dspaceRunnableClass) { + this.dspaceRunnableClass = dspaceRunnableClass; + } + +} diff --git a/dspace/config/spring/api/scripts.xml b/dspace/config/spring/api/scripts.xml index 0b7fdca833f1..c0a2c13d2003 100644 --- a/dspace/config/spring/api/scripts.xml +++ b/dspace/config/spring/api/scripts.xml @@ -139,6 +139,11 @@ + + + + + From 05cd10ff886f7690e0bbd7a40ebb29cbe3020319 Mon Sep 17 00:00:00 2001 From: "Gantner, Florian Klaus" Date: Fri, 3 Feb 2023 10:13:56 +0100 Subject: [PATCH 04/19] enhancer using mapconverter result enhancer using the result value of some mapConverter file, --- .../impl/MapConverterValueEnhancer.java | 129 ++++++++++++++++++ .../config/spring/api/metadata-enhancers.xml | 15 ++ 2 files changed, 144 insertions(+) create mode 100644 dspace-api/src/main/java/org/dspace/content/enhancer/impl/MapConverterValueEnhancer.java diff --git a/dspace-api/src/main/java/org/dspace/content/enhancer/impl/MapConverterValueEnhancer.java b/dspace-api/src/main/java/org/dspace/content/enhancer/impl/MapConverterValueEnhancer.java new file mode 100644 index 000000000000..a0d289c06bdf --- /dev/null +++ b/dspace-api/src/main/java/org/dspace/content/enhancer/impl/MapConverterValueEnhancer.java @@ -0,0 +1,129 @@ +/** + * The contents of this file are subject to the license and copyright + * detailed in the LICENSE and NOTICE files at the root of the source + * tree and available online at + * + * http://www.dspace.org/license/ + */ +package org.dspace.content.enhancer.impl; + +import java.sql.SQLException; +import java.util.Objects; + +import org.apache.commons.lang.StringUtils; +import org.dspace.content.Item; +import org.dspace.content.MetadataField; +import org.dspace.content.enhancer.AbstractItemEnhancer; +import org.dspace.content.enhancer.ItemEnhancer; +import org.dspace.content.service.ItemService; +import org.dspace.content.service.MetadataFieldService; +import org.dspace.core.Context; +import org.dspace.core.exception.SQLRuntimeException; +import org.dspace.services.ConfigurationService; +import org.dspace.util.SimpleMapConverter; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; + +/** + * Implementation of {@link ItemEnhancer} that add metadata values on the given + * item from the result of some mapConverter + * Only consider 1 to 1 value enhancement and no additional security/authority/language settings form the origin value. + * e.g. dc.type (dspace) -> dc.type (coar) + * + * @author Florian Gantner (florian.gantner@uni-bamberg.de) + * + */ +public class MapConverterValueEnhancer extends AbstractItemEnhancer { + + private static final Logger LOGGER = LoggerFactory.getLogger(MapConverterValueEnhancer.class); + + @Autowired + private ItemService itemService; + + @Autowired + private MetadataFieldService metadatafieldService; + + @Autowired + private ConfigurationService configurationService; + + private String sourceEntityType; + + private String sourceItemMetadataField; + + private String targetItemMetadataField; + + private boolean useDefaultLanguage; + + private SimpleMapConverter converter; + + @Override + public boolean canEnhance(Context context, Item item) { + return sourceEntityType == null || sourceEntityType.equals(itemService.getEntityType(item)); + } + + @Override + public void enhance(Context context, Item item) { + try { + if (StringUtils.isBlank(sourceItemMetadataField) || Objects.isNull(converter) || StringUtils.isBlank(targetItemMetadataField)) return; + String sourceval, targetval, calculatedval; + sourceval = itemService.getMetadata(item, sourceItemMetadataField); + targetval = itemService.getMetadata(item, targetItemMetadataField); + if (StringUtils.isNotBlank(sourceval)) { + calculatedval = converter.getValue(sourceval); + if (StringUtils.isNotBlank(targetval) && !targetval.contentEquals(calculatedval)) { + // replace mdv if it's different + removeTargetMetadata(context, item); + addTargetMetadata(context, item, calculatedval); + } else if (StringUtils.isBlank(targetval)) { + // set new value + addTargetMetadata(context, item, calculatedval); + } + } else if (StringUtils.isBlank(sourceval) && StringUtils.isNotBlank(targetval)) { + // remove value + removeTargetMetadata(context, item); + } + } catch (SQLException e) { + LOGGER.error("An error occurs enhancing item with id {}: {}", item.getID(), e.getMessage(), e); + throw new SQLRuntimeException(e); + } + } + private void addTargetMetadata(Context context, Item item, String value) throws SQLException { + MetadataField targetmd = metadatafieldService.findByString(context, targetItemMetadataField, '.'); + if (targetmd != null) { + String lang = (this.useDefaultLanguage) ? this.configurationService.getProperty("default.language") : null; + itemService.addMetadata(context, item, targetmd, lang, value); + } else { + LOGGER.error("No valid metadatavalue to enhance specified"); + } + } + + private void removeTargetMetadata(Context context, Item item) throws SQLException { + MetadataField targetmd = metadatafieldService.findByString(context, targetItemMetadataField, '.'); + if (targetmd != null) { + itemService.clearMetadata(context, item, targetmd.getMetadataSchema().getName(), targetmd.getElement(), targetmd.getQualifier(), Item.ANY); + } else { + LOGGER.error("No valid metadatavalue to enhance specified"); + } + } + + public void setSourceEntityType(String sourceEntityType) { + this.sourceEntityType = sourceEntityType; + } + + public void setTargetItemMetadataField(String targetItemMetadataField) { + this.targetItemMetadataField = targetItemMetadataField; + } + + public void setSourceItemMetadataField(String sourceItemMetadataField) { + this.sourceItemMetadataField = sourceItemMetadataField; + } + + public void setConverter(SimpleMapConverter converter) { + this.converter = converter; + } + + public void setUseDefaultLanguage(boolean useDefaultLanguage) { + this.useDefaultLanguage = useDefaultLanguage; + } +} diff --git a/dspace/config/spring/api/metadata-enhancers.xml b/dspace/config/spring/api/metadata-enhancers.xml index 26e656cf7ac2..36973c0b2a17 100644 --- a/dspace/config/spring/api/metadata-enhancers.xml +++ b/dspace/config/spring/api/metadata-enhancers.xml @@ -70,6 +70,21 @@ + + + + + + + + + + + + + + + From c42703b02d31ee72210571f9f13b6e6128c8b35f Mon Sep 17 00:00:00 2001 From: "Gantner, Florian Klaus" Date: Mon, 3 Jul 2023 17:19:42 +0200 Subject: [PATCH 05/19] clean up code and some fixes --- .../impl/MapConverterValueEnhancer.java | 14 +++---- .../content/enhancer/impl/NameEnhancer.java | 12 +++--- .../script/ItemEnhancerByDateScript.java | 38 +++++++++---------- ...ItemEnhancerByDateScriptConfiguration.java | 6 +-- .../script/ItemEnhancerEntityTypeScript.java | 24 +++++------- ...EnhancerEntityTypeScriptConfiguration.java | 2 +- .../config/spring/api/metadata-enhancers.xml | 7 +++- 7 files changed, 47 insertions(+), 56 deletions(-) diff --git a/dspace-api/src/main/java/org/dspace/content/enhancer/impl/MapConverterValueEnhancer.java b/dspace-api/src/main/java/org/dspace/content/enhancer/impl/MapConverterValueEnhancer.java index a0d289c06bdf..cc1fe52cbae6 100644 --- a/dspace-api/src/main/java/org/dspace/content/enhancer/impl/MapConverterValueEnhancer.java +++ b/dspace-api/src/main/java/org/dspace/content/enhancer/impl/MapConverterValueEnhancer.java @@ -2,7 +2,6 @@ * The contents of this file are subject to the license and copyright * detailed in the LICENSE and NOTICE files at the root of the source * tree and available online at - * * http://www.dspace.org/license/ */ package org.dspace.content.enhancer.impl; @@ -18,7 +17,6 @@ import org.dspace.content.service.ItemService; import org.dspace.content.service.MetadataFieldService; import org.dspace.core.Context; -import org.dspace.core.exception.SQLRuntimeException; import org.dspace.services.ConfigurationService; import org.dspace.util.SimpleMapConverter; import org.slf4j.Logger; @@ -66,9 +64,9 @@ public boolean canEnhance(Context context, Item item) { public void enhance(Context context, Item item) { try { if (StringUtils.isBlank(sourceItemMetadataField) || Objects.isNull(converter) || StringUtils.isBlank(targetItemMetadataField)) return; - String sourceval, targetval, calculatedval; - sourceval = itemService.getMetadata(item, sourceItemMetadataField); - targetval = itemService.getMetadata(item, targetItemMetadataField); + String sourceval, targetval, calculatedval; + sourceval = itemService.getMetadata(item, sourceItemMetadataField); + targetval = itemService.getMetadata(item, targetItemMetadataField); if (StringUtils.isNotBlank(sourceval)) { calculatedval = converter.getValue(sourceval); if (StringUtils.isNotBlank(targetval) && !targetval.contentEquals(calculatedval)) { @@ -83,12 +81,12 @@ public void enhance(Context context, Item item) { // remove value removeTargetMetadata(context, item); } - } catch (SQLException e) { + } catch (Exception e) { LOGGER.error("An error occurs enhancing item with id {}: {}", item.getID(), e.getMessage(), e); - throw new SQLRuntimeException(e); + //throw new SQLRuntimeException(e); } } - private void addTargetMetadata(Context context, Item item, String value) throws SQLException { + private void addTargetMetadata(Context context, Item item, String value) throws Exception { MetadataField targetmd = metadatafieldService.findByString(context, targetItemMetadataField, '.'); if (targetmd != null) { String lang = (this.useDefaultLanguage) ? this.configurationService.getProperty("default.language") : null; diff --git a/dspace-api/src/main/java/org/dspace/content/enhancer/impl/NameEnhancer.java b/dspace-api/src/main/java/org/dspace/content/enhancer/impl/NameEnhancer.java index 60cc672995ec..1f122a422575 100644 --- a/dspace-api/src/main/java/org/dspace/content/enhancer/impl/NameEnhancer.java +++ b/dspace-api/src/main/java/org/dspace/content/enhancer/impl/NameEnhancer.java @@ -20,7 +20,6 @@ import org.dspace.content.service.ItemService; import org.dspace.content.service.MetadataFieldService; import org.dspace.core.Context; -import org.dspace.core.exception.SQLRuntimeException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; @@ -62,15 +61,16 @@ public void enhance(Context context, Item item) { try { if (Objects.isNull(relatedItemMetadataFields) || relatedItemMetadataFields.isEmpty() || StringUtils.isBlank(targetItemMetadataField)) return; checkNames(context, item); - } catch (SQLException e) { + } catch (Exception e) { LOGGER.error("An error occurs enhancing item with id {}: {}", item.getID(), e.getMessage(), e); - throw new SQLRuntimeException(e); + //Exception handling not supported by ItemEnhancerService. Thus just log to continue other enhancers + //throw new SQLRuntimeException(e); } } - private void checkNames(Context context, Item item) throws SQLException { + private void checkNames(Context context, Item item) throws Exception { // ignore languages of Metadata here. Assume main title is not repeated - // Could by more simplified + // Could be more simplified List currentnames = itemService.getMetadataByMetadataString(item, targetItemMetadataField); if (!currentnames.isEmpty()) { @@ -123,7 +123,7 @@ private void checkNames(Context context, Item item) throws SQLException { private void updateTargetMetadata(Context context, Item item, String value, boolean clear) throws SQLException { MetadataField targetmd = metadatafieldService.findByString(context, targetItemMetadataField, '.'); - if (targetmd != null){ + if (targetmd != null) { if (clear) { itemService.clearMetadata(context, item, targetmd.getMetadataSchema().getName(), targetmd.getElement(), targetmd.getQualifier(), Item.ANY); } diff --git a/dspace-api/src/main/java/org/dspace/content/enhancer/script/ItemEnhancerByDateScript.java b/dspace-api/src/main/java/org/dspace/content/enhancer/script/ItemEnhancerByDateScript.java index c5f77a5a66b5..84a76b1d1678 100644 --- a/dspace-api/src/main/java/org/dspace/content/enhancer/script/ItemEnhancerByDateScript.java +++ b/dspace-api/src/main/java/org/dspace/content/enhancer/script/ItemEnhancerByDateScript.java @@ -2,8 +2,7 @@ * The contents of this file are subject to the license and copyright * detailed in the LICENSE and NOTICE files at the root of the source * tree and available online at - * - * http://www.dspace.org/license/ + * http://www.dspace.org/license/ */ package org.dspace.content.enhancer.script; @@ -41,7 +40,7 @@ * calculated metadata with the enhancement. * * @author Luca Giamminonni (luca.giamminonni at 4science.it) - * + * Extended to solr search to discover not-enhanced entities. * Some Query can be delivered, * additional some daterange filterquery on the lastModified field @@ -108,16 +107,16 @@ public void setup() throws ParseException { this.datelower = commandLine.getOptionValue('s'); } if (commandLine.hasOption('m')) { - try{ + try { this.max = Integer.parseInt(commandLine.getOptionValue('m')); - }catch (Exception e){ + } catch (Exception e) { // } } if (commandLine.hasOption('l')) { - try{ + try { this.limit = Integer.parseInt(commandLine.getOptionValue('l')); - }catch (Exception e){ + } catch (Exception e) { // } } @@ -135,7 +134,7 @@ public void internalRun() throws Exception { throw new Exception("specified collection does not exist"); } SolrPingResponse ping = solrSearchCore.getSolr().ping(); - if(ping.getStatus() > 299) throw new Exception("Solr seems not to be available. Status" + ping.getStatus()); + if (ping.getStatus() > 299) throw new Exception("Solr seems not to be available. Status" + ping.getStatus()); context.turnOffAuthorisationSystem(); try { @@ -165,7 +164,7 @@ private void searchItems() { log.error(e.getMessage()); } int total = items.size(); - if (total == 0){ + if (total == 0) { handler.logInfo("No results in solr-Query"); log.info("No results in solr-Query"); return; @@ -174,22 +173,19 @@ private void searchItems() { if (this.limit > 0) { // Split through every list int counter = 0, start, end; - while (counter < total){ + while (counter < total) { start = counter; end = counter + limit; - if(end > (total -1)) end = total - 1; + if (end > (total -1)) end = total - 1; try { this.itemService.findByIds(context, items.subList(start, end)).forEachRemaining(this::enhanceItem); context.commit(); counter += limit; - } catch (SQLException e) { + } catch (Exception e) { handler.logError(e.getMessage()); counter += limit; } - } - handler.logInfo("enhanced " + total + " items"); - log.info("enhanced " + total + " items"); } else { try { @@ -197,9 +193,9 @@ private void searchItems() { } catch (SQLException e) { handler.logError(e.getMessage()); } - handler.logInfo("enhanced " + total + " items"); - log.info("enhanced " + total + " items"); } + handler.logInfo("enhanced " + total + " items"); + log.info("enhanced " + total + " items"); } private SolrDocumentList searchItemsInSolr(String query, String datequeryupper, String datequerylower) throws SolrServerException, IOException { @@ -209,11 +205,11 @@ private SolrDocumentList searchItemsInSolr(String query, String datequeryupper, } else { sQuery = new SolrQuery("*"); } - if(Objects.nonNull(datequeryupper) && Objects.nonNull(datequerylower)){ + if (Objects.nonNull(datequeryupper) && Objects.nonNull(datequerylower)) { sQuery.addFilterQuery("lastModified:["+datequerylower+" TO " + datequeryupper +"]"); - } else if (Objects.nonNull(datequeryupper) ){ + } else if (Objects.nonNull(datequeryupper)) { sQuery.addFilterQuery("lastModified:[* TO " + datequeryupper +"]"); - } else if (Objects.nonNull(datequerylower) ){ + } else if (Objects.nonNull(datequerylower)) { sQuery.addFilterQuery("lastModified:[" + datequerylower + " TO *]"); } if (Objects.nonNull(entitytype)) { @@ -261,7 +257,7 @@ private void assignCurrentUserInContext() throws SQLException { } } - private void assignSpecialGroupsInContext() throws SQLException { + private void assignSpecialGroupsInContext() { for (UUID uuid : handler.getSpecialGroups()) { context.setSpecialGroup(uuid); } diff --git a/dspace-api/src/main/java/org/dspace/content/enhancer/script/ItemEnhancerByDateScriptConfiguration.java b/dspace-api/src/main/java/org/dspace/content/enhancer/script/ItemEnhancerByDateScriptConfiguration.java index c3a2437ca9e6..abd0158b771f 100644 --- a/dspace-api/src/main/java/org/dspace/content/enhancer/script/ItemEnhancerByDateScriptConfiguration.java +++ b/dspace-api/src/main/java/org/dspace/content/enhancer/script/ItemEnhancerByDateScriptConfiguration.java @@ -44,9 +44,9 @@ public Options getOptions() { options.addOption("f", "force", false, "force the recalculation of all the virtual fields"); options.addOption("c", "collection", true, "uuid of the collection. If the collection does not exist the script aborts."); - options.addOption("e", "entity", true, "Entity type of the items. Cannot be combined with limit/max option"); - options.addOption("d", "dateupper", true, "iso date as upper range of date query. e.g. 2022-10-27T12:12:17.369Z "); - options.addOption("s", "datelower", true, "iso date as lower range of date query "); + options.addOption("e", "entity", true, "Entity type of the items"); + options.addOption("d", "dateupper", true, "iso date as upper range of date query for lastModified. e.g. 2022-10-27T12:12:17.369Z "); + options.addOption("s", "datelower", true, "iso date as lower range of date query for lastModified "); options.addOption("m", "max", true, "--max results/rows from solr"); options.addOption("l", "limit", true, "commit after --limit entities processed"); options.addOption("q", "query", true, "additional filterquery for the entities. this can for example be the exclusion of some already enhanced metadata."); diff --git a/dspace-api/src/main/java/org/dspace/content/enhancer/script/ItemEnhancerEntityTypeScript.java b/dspace-api/src/main/java/org/dspace/content/enhancer/script/ItemEnhancerEntityTypeScript.java index bc7c0a1e5422..a0f2851aeba1 100644 --- a/dspace-api/src/main/java/org/dspace/content/enhancer/script/ItemEnhancerEntityTypeScript.java +++ b/dspace-api/src/main/java/org/dspace/content/enhancer/script/ItemEnhancerEntityTypeScript.java @@ -2,7 +2,6 @@ * The contents of this file are subject to the license and copyright * detailed in the LICENSE and NOTICE files at the root of the source * tree and available online at - * * http://www.dspace.org/license/ */ package org.dspace.content.enhancer.script; @@ -35,7 +34,6 @@ * calculated metadata with the enhancement. * * @author Luca Giamminonni (luca.giamminonni at 4science.it) - * * Extended to limit the item set to collection/entity wide use to speed up process and recalculation of certain entities. * Extended to use some pagination option with some limit where some commit to the database is made * @@ -86,21 +84,21 @@ public void setup() throws ParseException { if (commandLine.hasOption('l')) { try{ this.limit = Integer.parseInt(commandLine.getOptionValue('l')); - }catch (Exception e){ + } catch (Exception e) { // } } if (commandLine.hasOption('m')) { try{ this.max = Integer.parseInt(commandLine.getOptionValue('m')); - }catch (Exception e){ + } catch (Exception e) { // } } if (commandLine.hasOption('o')) { try{ this.offset = Integer.parseInt(commandLine.getOptionValue('o')); - }catch (Exception e){ + } catch (Exception e) { // } } @@ -145,7 +143,7 @@ private void enhanceItems() { } } - private void findItemsToEnhanceLimitMax(){ + private void findItemsToEnhanceLimitMax() { Collection coll; int total, counter = 0; if (Objects.nonNull(collection)) { @@ -153,7 +151,7 @@ private void findItemsToEnhanceLimitMax(){ try { coll = collectionService.find(context, collection); total = itemService.countItems(context, coll); - } catch (SQLException e){ + } catch (SQLException e) { handler.logError(e.getMessage()); return; } @@ -186,7 +184,7 @@ private void findItemsToEnhanceLimitMax(){ //loop through all! try { total = itemService.countTotal(context); - } catch (SQLException e){ + } catch (SQLException e) { handler.logError(e.getMessage()); return; } @@ -213,7 +211,7 @@ private void findItemsToEnhanceLimitMax(){ context.commit(); handler.logInfo("processed " + counter + " out of total " + total + " items"); log.info("processed " + counter + " out of total " + total + " items"); - } catch (SQLException e){ + } catch (SQLException e) { handler.logError(e.getMessage()); counter += limit; handler.logInfo("processed " + counter + " out of total " + total + " items"); @@ -221,7 +219,6 @@ private void findItemsToEnhanceLimitMax(){ } } } - } private Iterator findItemsToEnhance() { @@ -230,7 +227,7 @@ private Iterator findItemsToEnhance() { if (Objects.nonNull(collection)) { //Check, if uuid exist Collection coll = collectionService.find(context, collection); - if(coll != null) { + if (coll != null) { result = itemService.findAllByCollection(context, coll); } } else { @@ -248,9 +245,7 @@ private void enhanceItem(Item item) { } else { itemEnhancerService.enhance(context, item); } - uncacheItem(item); - } /** @@ -267,7 +262,6 @@ private void enhanceItemEntityCheck(Item item) { } else { itemEnhancerService.enhance(context, item); } - uncacheItem(item); } } @@ -288,7 +282,7 @@ private void assignCurrentUserInContext() throws SQLException { } } - private void assignSpecialGroupsInContext() throws SQLException { + private void assignSpecialGroupsInContext() { for (UUID uuid : handler.getSpecialGroups()) { context.setSpecialGroup(uuid); } diff --git a/dspace-api/src/main/java/org/dspace/content/enhancer/script/ItemEnhancerEntityTypeScriptConfiguration.java b/dspace-api/src/main/java/org/dspace/content/enhancer/script/ItemEnhancerEntityTypeScriptConfiguration.java index b45f5e983a17..835c27ba7b53 100644 --- a/dspace-api/src/main/java/org/dspace/content/enhancer/script/ItemEnhancerEntityTypeScriptConfiguration.java +++ b/dspace-api/src/main/java/org/dspace/content/enhancer/script/ItemEnhancerEntityTypeScriptConfiguration.java @@ -44,7 +44,7 @@ public Options getOptions() { options.addOption("f", "force", false, "force the recalculation of all the virtual fields"); options.addOption("c", "collection", true, "uuid of the collection. If the collection does not exist the script aborts."); - options.addOption("e", "entity", true, "Entity type of the items. Cannot be combined with limit/max option"); + options.addOption("e", "entity", true, "Entity type of the items. Not recommended, because all items are loaded and then filtered by their entity types."); options.addOption("l", "limit", true, "size for iterator --limit items and commit after --limit items"); options.addOption("m", "max", true, "process max --max items"); options.addOption("o", "offset", true, "offset of items to start --offset items from the start"); diff --git a/dspace/config/spring/api/metadata-enhancers.xml b/dspace/config/spring/api/metadata-enhancers.xml index 36973c0b2a17..58ad367c5193 100644 --- a/dspace/config/spring/api/metadata-enhancers.xml +++ b/dspace/config/spring/api/metadata-enhancers.xml @@ -68,20 +68,23 @@ crisrp.fullName - + + + - + + From 4b51093431bd015972fa985113fe3f91d5433536 Mon Sep 17 00:00:00 2001 From: "Gantner, Florian Klaus" Date: Tue, 11 Jul 2023 12:04:31 +0200 Subject: [PATCH 06/19] fix missing license header --- .../content/enhancer/impl/MapConverterValueEnhancer.java | 1 + .../content/enhancer/script/ItemEnhancerByDateScript.java | 3 ++- .../content/enhancer/script/ItemEnhancerEntityTypeScript.java | 1 + 3 files changed, 4 insertions(+), 1 deletion(-) diff --git a/dspace-api/src/main/java/org/dspace/content/enhancer/impl/MapConverterValueEnhancer.java b/dspace-api/src/main/java/org/dspace/content/enhancer/impl/MapConverterValueEnhancer.java index cc1fe52cbae6..5856cc0962ec 100644 --- a/dspace-api/src/main/java/org/dspace/content/enhancer/impl/MapConverterValueEnhancer.java +++ b/dspace-api/src/main/java/org/dspace/content/enhancer/impl/MapConverterValueEnhancer.java @@ -2,6 +2,7 @@ * The contents of this file are subject to the license and copyright * detailed in the LICENSE and NOTICE files at the root of the source * tree and available online at + * * http://www.dspace.org/license/ */ package org.dspace.content.enhancer.impl; diff --git a/dspace-api/src/main/java/org/dspace/content/enhancer/script/ItemEnhancerByDateScript.java b/dspace-api/src/main/java/org/dspace/content/enhancer/script/ItemEnhancerByDateScript.java index 84a76b1d1678..c64aec7171fb 100644 --- a/dspace-api/src/main/java/org/dspace/content/enhancer/script/ItemEnhancerByDateScript.java +++ b/dspace-api/src/main/java/org/dspace/content/enhancer/script/ItemEnhancerByDateScript.java @@ -2,7 +2,8 @@ * The contents of this file are subject to the license and copyright * detailed in the LICENSE and NOTICE files at the root of the source * tree and available online at - * http://www.dspace.org/license/ + * + * http://www.dspace.org/license/ */ package org.dspace.content.enhancer.script; diff --git a/dspace-api/src/main/java/org/dspace/content/enhancer/script/ItemEnhancerEntityTypeScript.java b/dspace-api/src/main/java/org/dspace/content/enhancer/script/ItemEnhancerEntityTypeScript.java index a0f2851aeba1..a9c6f584c567 100644 --- a/dspace-api/src/main/java/org/dspace/content/enhancer/script/ItemEnhancerEntityTypeScript.java +++ b/dspace-api/src/main/java/org/dspace/content/enhancer/script/ItemEnhancerEntityTypeScript.java @@ -2,6 +2,7 @@ * The contents of this file are subject to the license and copyright * detailed in the LICENSE and NOTICE files at the root of the source * tree and available online at + * * http://www.dspace.org/license/ */ package org.dspace.content.enhancer.script; From ee913993e976372d415b1b7be14f490352909e94 Mon Sep 17 00:00:00 2001 From: "Gantner, Florian Klaus" Date: Tue, 8 Aug 2023 16:31:11 +0200 Subject: [PATCH 07/19] fix a bunch of dspace checkstyle issues --- .../impl/MapConverterValueEnhancer.java | 103 ++++++------ .../content/enhancer/impl/NameEnhancer.java | 146 +++++++++--------- .../script/ItemEnhancerByDateScript.java | 49 +++--- ...ItemEnhancerByDateScriptConfiguration.java | 13 +- .../script/ItemEnhancerEntityTypeScript.java | 74 +++++---- ...EnhancerEntityTypeScriptConfiguration.java | 9 +- 6 files changed, 216 insertions(+), 178 deletions(-) diff --git a/dspace-api/src/main/java/org/dspace/content/enhancer/impl/MapConverterValueEnhancer.java b/dspace-api/src/main/java/org/dspace/content/enhancer/impl/MapConverterValueEnhancer.java index 5856cc0962ec..b27a5e772fe0 100644 --- a/dspace-api/src/main/java/org/dspace/content/enhancer/impl/MapConverterValueEnhancer.java +++ b/dspace-api/src/main/java/org/dspace/content/enhancer/impl/MapConverterValueEnhancer.java @@ -40,21 +40,20 @@ public class MapConverterValueEnhancer extends AbstractItemEnhancer { @Autowired private ItemService itemService; - @Autowired - private MetadataFieldService metadatafieldService; + @Autowired + private MetadataFieldService metadatafieldService; - @Autowired - private ConfigurationService configurationService; + @Autowired + private ConfigurationService configurationService; private String sourceEntityType; private String sourceItemMetadataField; - private String targetItemMetadataField; - private boolean useDefaultLanguage; + private boolean useDefaultLanguage; - private SimpleMapConverter converter; + private SimpleMapConverter converter; @Override public boolean canEnhance(Context context, Item item) { @@ -64,47 +63,53 @@ public boolean canEnhance(Context context, Item item) { @Override public void enhance(Context context, Item item) { try { - if (StringUtils.isBlank(sourceItemMetadataField) || Objects.isNull(converter) || StringUtils.isBlank(targetItemMetadataField)) return; - String sourceval, targetval, calculatedval; - sourceval = itemService.getMetadata(item, sourceItemMetadataField); - targetval = itemService.getMetadata(item, targetItemMetadataField); - if (StringUtils.isNotBlank(sourceval)) { - calculatedval = converter.getValue(sourceval); - if (StringUtils.isNotBlank(targetval) && !targetval.contentEquals(calculatedval)) { - // replace mdv if it's different - removeTargetMetadata(context, item); - addTargetMetadata(context, item, calculatedval); - } else if (StringUtils.isBlank(targetval)) { - // set new value - addTargetMetadata(context, item, calculatedval); - } - } else if (StringUtils.isBlank(sourceval) && StringUtils.isNotBlank(targetval)) { - // remove value - removeTargetMetadata(context, item); - } + if (StringUtils.isBlank(sourceItemMetadataField) || Objects.isNull(converter) || + StringUtils.isBlank(targetItemMetadataField)) { + return; + } + String sourceval; + String targetval; + String calculatedval; + sourceval = itemService.getMetadata(item, sourceItemMetadataField); + targetval = itemService.getMetadata(item, targetItemMetadataField); + if (StringUtils.isNotBlank(sourceval)) { + calculatedval = converter.getValue(sourceval); + if (StringUtils.isNotBlank(targetval) && !targetval.contentEquals(calculatedval)) { + // replace mdv if it's different + removeTargetMetadata(context, item); + addTargetMetadata(context, item, calculatedval); + } else if (StringUtils.isBlank(targetval)) { + // set new value + addTargetMetadata(context, item, calculatedval); + } + } else if (StringUtils.isBlank(sourceval) && StringUtils.isNotBlank(targetval)) { + // remove value + removeTargetMetadata(context, item); + } } catch (Exception e) { LOGGER.error("An error occurs enhancing item with id {}: {}", item.getID(), e.getMessage(), e); //throw new SQLRuntimeException(e); } } - private void addTargetMetadata(Context context, Item item, String value) throws Exception { - MetadataField targetmd = metadatafieldService.findByString(context, targetItemMetadataField, '.'); - if (targetmd != null) { - String lang = (this.useDefaultLanguage) ? this.configurationService.getProperty("default.language") : null; - itemService.addMetadata(context, item, targetmd, lang, value); - } else { - LOGGER.error("No valid metadatavalue to enhance specified"); - } - } - - private void removeTargetMetadata(Context context, Item item) throws SQLException { - MetadataField targetmd = metadatafieldService.findByString(context, targetItemMetadataField, '.'); - if (targetmd != null) { - itemService.clearMetadata(context, item, targetmd.getMetadataSchema().getName(), targetmd.getElement(), targetmd.getQualifier(), Item.ANY); - } else { - LOGGER.error("No valid metadatavalue to enhance specified"); - } - } + private void addTargetMetadata(Context context, Item item, String value) throws Exception { + MetadataField targetmd = metadatafieldService.findByString(context, targetItemMetadataField, '.'); + if (targetmd != null) { + String lang = (this.useDefaultLanguage) ? this.configurationService.getProperty("default.language") : null; + itemService.addMetadata(context, item, targetmd, lang, value); + } else { + LOGGER.error("No valid metadatavalue to enhance specified"); + } + } + + private void removeTargetMetadata(Context context, Item item) throws SQLException { + MetadataField targetmd = metadatafieldService.findByString(context, targetItemMetadataField, '.'); + if (targetmd != null) { + itemService.clearMetadata(context, item, targetmd.getMetadataSchema().getName(), + targetmd.getElement(), targetmd.getQualifier(), Item.ANY); + } else { + LOGGER.error("No valid metadatavalue to enhance specified"); + } + } public void setSourceEntityType(String sourceEntityType) { this.sourceEntityType = sourceEntityType; @@ -118,11 +123,11 @@ public void setSourceItemMetadataField(String sourceItemMetadataField) { this.sourceItemMetadataField = sourceItemMetadataField; } - public void setConverter(SimpleMapConverter converter) { - this.converter = converter; - } + public void setConverter(SimpleMapConverter converter) { + this.converter = converter; + } - public void setUseDefaultLanguage(boolean useDefaultLanguage) { - this.useDefaultLanguage = useDefaultLanguage; - } + public void setUseDefaultLanguage(boolean useDefaultLanguage) { + this.useDefaultLanguage = useDefaultLanguage; + } } diff --git a/dspace-api/src/main/java/org/dspace/content/enhancer/impl/NameEnhancer.java b/dspace-api/src/main/java/org/dspace/content/enhancer/impl/NameEnhancer.java index 1f122a422575..2a50d9fdb75e 100644 --- a/dspace-api/src/main/java/org/dspace/content/enhancer/impl/NameEnhancer.java +++ b/dspace-api/src/main/java/org/dspace/content/enhancer/impl/NameEnhancer.java @@ -26,7 +26,7 @@ /** * Implementation of {@link ItemEnhancer} that add metadata values on the given - * item from a list of other metadatavalues . + * item from a list of other metadatavalues . * The first value of the list that is not null matches and overwrites the existing value. * Some default value can be specified if no fields were found * @@ -40,15 +40,14 @@ public class NameEnhancer extends AbstractItemEnhancer { @Autowired private ItemService itemService; - @Autowired - private MetadataFieldService metadatafieldService; + @Autowired + private MetadataFieldService metadatafieldService; private String sourceEntityType; private String targetItemMetadataField; private List relatedItemMetadataFields; - private String defaultValue; @Override @@ -59,79 +58,88 @@ public boolean canEnhance(Context context, Item item) { @Override public void enhance(Context context, Item item) { try { - if (Objects.isNull(relatedItemMetadataFields) || relatedItemMetadataFields.isEmpty() || StringUtils.isBlank(targetItemMetadataField)) return; - checkNames(context, item); + if (Objects.isNull(relatedItemMetadataFields) || relatedItemMetadataFields.isEmpty() || + StringUtils.isBlank(targetItemMetadataField)) { + return; + } + checkNames(context, item); } catch (Exception e) { LOGGER.error("An error occurs enhancing item with id {}: {}", item.getID(), e.getMessage(), e); - //Exception handling not supported by ItemEnhancerService. Thus just log to continue other enhancers + //Exception handling not supported by ItemEnhancerService. Thus just log to continue other enhancers //throw new SQLRuntimeException(e); } } private void checkNames(Context context, Item item) throws Exception { - // ignore languages of Metadata here. Assume main title is not repeated - // Could be more simplified - List currentnames = itemService.getMetadataByMetadataString(item, targetItemMetadataField); - - if (!currentnames.isEmpty()) { - // some name assigned yet - for (MetadataValue currentname : currentnames) { - String val = currentname.getValue(); - fields: for (String field : relatedItemMetadataFields) { - List fieldnames = itemService.getMetadataByMetadataString(item, field); - if (fieldnames.isEmpty()) continue fields; //No Values, try next loop - for (MetadataValue fieldname : fieldnames) { - if (StringUtils.isNotBlank(fieldname.getValue()) - && fieldname.getValue().contentEquals(val)) { - //Values are the same. No Update necessary - return; - } else { - //values differ. We must update the value - updateTargetMetadata(context, item, fieldname.getValue(), true); - return; - } - } - } - } - if (StringUtils.isNotBlank(defaultValue) - && !currentnames.get(0).getValue().contentEquals(defaultValue)) { - // None of the names above matches. Set Default-Value, if exist. Otherwise do nothing - updateTargetMetadata(context, item, defaultValue, true); - } - - } else { - // No Name assigned yet - // Check existing names - fields: for (String field : relatedItemMetadataFields) { - List fieldnames = itemService.getMetadataByMetadataString(item, field); - if (fieldnames.isEmpty()) continue fields; //No Values, try next loop - for (MetadataValue fieldname : fieldnames) { - if (StringUtils.isNotBlank(fieldname.getValue())) { - //Got some value - updateTargetMetadata(context, item, fieldname.getValue(), false); - return; - } - } - } - // If no name exist, set defaultvalue - if (StringUtils.isNotBlank(defaultValue)) { - updateTargetMetadata(context, item, defaultValue, false); - } - // otherwise do not assign any value - } + // ignore languages of Metadata here. Assume main title is not repeated + // Could be more simplified + List currentnames = itemService.getMetadataByMetadataString(item, targetItemMetadataField); + + if (!currentnames.isEmpty()) { + // some name assigned yet + for (MetadataValue currentname : currentnames) { + String val = currentname.getValue(); + for (String field : relatedItemMetadataFields) { + List fieldnames = + itemService.getMetadataByMetadataString(item, field); + if (fieldnames.isEmpty()) { + continue ; //No Values, try next loop + } + for (MetadataValue fieldname : fieldnames) { + if (StringUtils.isNotBlank(fieldname.getValue()) + && fieldname.getValue().contentEquals(val)) { + //Values are the same. No Update necessary + return; + } else { + //values differ. We must update the value + updateTargetMetadata(context, item, fieldname.getValue(), true); + return; + } + } + } + } + if (StringUtils.isNotBlank(defaultValue) + && !currentnames.get(0).getValue().contentEquals(defaultValue)) { + // None of the names above matches. Set Default-Value, if exist. Otherwise do nothing + updateTargetMetadata(context, item, defaultValue, true); + } + + } else { + // No Name assigned yet + // Check existing names + for (String field : relatedItemMetadataFields) { + List fieldnames = itemService.getMetadataByMetadataString(item, field); + if (fieldnames.isEmpty()) { + continue; //No Values, try next loop + } + for (MetadataValue fieldname : fieldnames) { + if (StringUtils.isNotBlank(fieldname.getValue())) { + //Got some value + updateTargetMetadata(context, item, fieldname.getValue(), false); + return; + } + } + } + // If no name exist, set defaultvalue + if (StringUtils.isNotBlank(defaultValue)) { + updateTargetMetadata(context, item, defaultValue, false); + } + // otherwise do not assign any value + } } - private void updateTargetMetadata(Context context, Item item, String value, boolean clear) throws SQLException { - MetadataField targetmd = metadatafieldService.findByString(context, targetItemMetadataField, '.'); - if (targetmd != null) { - if (clear) { - itemService.clearMetadata(context, item, targetmd.getMetadataSchema().getName(), targetmd.getElement(), targetmd.getQualifier(), Item.ANY); - } - itemService.addMetadata(context, item, targetmd, null, value); - } else { - LOGGER.error("No valid metadatavalue to enhance specified"); - } - } + private void updateTargetMetadata(Context context, Item item, String value, boolean clear) throws SQLException { + MetadataField targetmd = metadatafieldService.findByString(context, targetItemMetadataField, '.'); + if (targetmd != null) { + if (clear) { + itemService.clearMetadata(context, item, targetmd.getMetadataSchema().getName(), targetmd.getElement(), + targetmd.getQualifier(), Item.ANY); + } + itemService.addMetadata(context, item, targetmd, null, value); + } else { + LOGGER.error("No valid metadatavalue to enhance specified"); + } + } public void setSourceEntityType(String sourceEntityType) { this.sourceEntityType = sourceEntityType; @@ -144,7 +152,7 @@ public void setTargetItemMetadataField(String targetItemMetadataField) { public void setRelatedItemMetadataFields(List relatedItemMetadataFields) { this.relatedItemMetadataFields = relatedItemMetadataFields; } - + public void setDefaultValue(String defaultvalue) { this.defaultValue = defaultvalue; } diff --git a/dspace-api/src/main/java/org/dspace/content/enhancer/script/ItemEnhancerByDateScript.java b/dspace-api/src/main/java/org/dspace/content/enhancer/script/ItemEnhancerByDateScript.java index c64aec7171fb..54d4f9564b7e 100644 --- a/dspace-api/src/main/java/org/dspace/content/enhancer/script/ItemEnhancerByDateScript.java +++ b/dspace-api/src/main/java/org/dspace/content/enhancer/script/ItemEnhancerByDateScript.java @@ -7,6 +7,13 @@ */ package org.dspace.content.enhancer.script; +import java.io.IOException; +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.List; +import java.util.Objects; +import java.util.UUID; + import org.apache.commons.cli.ParseException; import org.apache.solr.client.solrj.SolrQuery; import org.apache.solr.client.solrj.SolrServerException; @@ -32,10 +39,6 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.io.IOException; -import java.sql.SQLException; -import java.util.*; - /** * Script that allows to enhance items, also forcing the updating of the * calculated metadata with the enhancement. @@ -50,10 +53,10 @@ * @author florian.gantner@uni-bamberg.de * */ -public class ItemEnhancerByDateScript extends DSpaceRunnable> { +public class ItemEnhancerByDateScript + extends DSpaceRunnable> { private ItemService itemService; - private CollectionService collectionService; private ItemEnhancerService itemEnhancerService; @@ -61,9 +64,7 @@ public class ItemEnhancerByDateScript extends DSpaceRunnable 299) throw new Exception("Solr seems not to be available. Status" + ping.getStatus()); + if (ping.getStatus() > 299) { + throw new Exception("Solr seems not to be available. Status" + ping.getStatus()); + } context.turnOffAuthorisationSystem(); try { @@ -158,7 +162,9 @@ private void searchItems() { SolrDocumentList results = searchItemsInSolr(this.query, this.dateupper, this.datelower); for (SolrDocument doc : results) { String resourceid = (String) doc.getFieldValue(SearchUtils.RESOURCE_ID_FIELD); - if (Objects.nonNull(resourceid) && Objects.nonNull(UUIDUtils.fromString(resourceid))) items.add(resourceid); + if (Objects.nonNull(resourceid) && Objects.nonNull(UUIDUtils.fromString(resourceid))) { + items.add(resourceid); + } } } catch (SolrServerException | IOException e) { handler.logError(e.getMessage(), e); @@ -173,11 +179,15 @@ private void searchItems() { if (this.limit > 0) { // Split through every list - int counter = 0, start, end; + int counter = 0; + int start = 0; + int end = 0; while (counter < total) { start = counter; end = counter + limit; - if (end > (total -1)) end = total - 1; + if (end > (total - 1)) { + end = total - 1; + } try { this.itemService.findByIds(context, items.subList(start, end)).forEachRemaining(this::enhanceItem); context.commit(); @@ -199,7 +209,8 @@ private void searchItems() { log.info("enhanced " + total + " items"); } - private SolrDocumentList searchItemsInSolr(String query, String datequeryupper, String datequerylower) throws SolrServerException, IOException { + private SolrDocumentList searchItemsInSolr(String query, String datequeryupper, String datequerylower) + throws SolrServerException, IOException { SolrQuery sQuery; if (Objects.nonNull(query)) { sQuery = new SolrQuery(query); @@ -207,18 +218,18 @@ private SolrDocumentList searchItemsInSolr(String query, String datequeryupper, sQuery = new SolrQuery("*"); } if (Objects.nonNull(datequeryupper) && Objects.nonNull(datequerylower)) { - sQuery.addFilterQuery("lastModified:["+datequerylower+" TO " + datequeryupper +"]"); + sQuery.addFilterQuery("lastModified:[" + datequerylower + " TO " + datequeryupper + "]"); } else if (Objects.nonNull(datequeryupper)) { - sQuery.addFilterQuery("lastModified:[* TO " + datequeryupper +"]"); + sQuery.addFilterQuery("lastModified:[* TO " + datequeryupper + "]"); } else if (Objects.nonNull(datequerylower)) { sQuery.addFilterQuery("lastModified:[" + datequerylower + " TO *]"); } if (Objects.nonNull(entitytype)) { - sQuery.addFilterQuery("search.entitytype:"+entitytype); + sQuery.addFilterQuery("search.entitytype:" + entitytype); } - sQuery.addFilterQuery(SearchUtils.RESOURCE_TYPE_FIELD+":Item"); + sQuery.addFilterQuery(SearchUtils.RESOURCE_TYPE_FIELD + ":Item"); if (Objects.nonNull(collection)) { - sQuery.addFilterQuery("location.coll:"+UUIDUtils.toString(collection)); + sQuery.addFilterQuery("location.coll:" + UUIDUtils.toString(collection)); } sQuery.addField(SearchUtils.RESOURCE_ID_FIELD); if (max > 0) { diff --git a/dspace-api/src/main/java/org/dspace/content/enhancer/script/ItemEnhancerByDateScriptConfiguration.java b/dspace-api/src/main/java/org/dspace/content/enhancer/script/ItemEnhancerByDateScriptConfiguration.java index abd0158b771f..f5d513000f67 100644 --- a/dspace-api/src/main/java/org/dspace/content/enhancer/script/ItemEnhancerByDateScriptConfiguration.java +++ b/dspace-api/src/main/java/org/dspace/content/enhancer/script/ItemEnhancerByDateScriptConfiguration.java @@ -7,14 +7,14 @@ */ package org.dspace.content.enhancer.script; +import java.sql.SQLException; + import org.apache.commons.cli.Options; import org.dspace.authorize.service.AuthorizeService; import org.dspace.core.Context; import org.dspace.scripts.configuration.ScriptConfiguration; import org.springframework.beans.factory.annotation.Autowired; -import java.sql.SQLException; - /** * Script configuration of {@link ItemEnhancerEntityTypeScript}. * @@ -43,13 +43,16 @@ public Options getOptions() { Options options = new Options(); options.addOption("f", "force", false, "force the recalculation of all the virtual fields"); - options.addOption("c", "collection", true, "uuid of the collection. If the collection does not exist the script aborts."); + options.addOption("c", "collection", true, + "uuid of the collection. If the collection does not exist the script aborts."); options.addOption("e", "entity", true, "Entity type of the items"); - options.addOption("d", "dateupper", true, "iso date as upper range of date query for lastModified. e.g. 2022-10-27T12:12:17.369Z "); + options.addOption("d", "dateupper", true, + "iso date as upper range of date query for lastModified. e.g. 2022-10-27T12:12:17.369Z "); options.addOption("s", "datelower", true, "iso date as lower range of date query for lastModified "); options.addOption("m", "max", true, "--max results/rows from solr"); options.addOption("l", "limit", true, "commit after --limit entities processed"); - options.addOption("q", "query", true, "additional filterquery for the entities. this can for example be the exclusion of some already enhanced metadata."); + options.addOption("q", "query", true, + "additional filterquery for the entities. this can f.e. be the exclusion of already enhanced metadata"); super.options = options; } diff --git a/dspace-api/src/main/java/org/dspace/content/enhancer/script/ItemEnhancerEntityTypeScript.java b/dspace-api/src/main/java/org/dspace/content/enhancer/script/ItemEnhancerEntityTypeScript.java index a9c6f584c567..960d075dc79e 100644 --- a/dspace-api/src/main/java/org/dspace/content/enhancer/script/ItemEnhancerEntityTypeScript.java +++ b/dspace-api/src/main/java/org/dspace/content/enhancer/script/ItemEnhancerEntityTypeScript.java @@ -35,24 +35,21 @@ * calculated metadata with the enhancement. * * @author Luca Giamminonni (luca.giamminonni at 4science.it) - * Extended to limit the item set to collection/entity wide use to speed up process and recalculation of certain entities. + * Extended to limit the item set to collection/entity wide use to speed up process + * and recalculation of certain entities. * Extended to use some pagination option with some limit where some commit to the database is made * * @author florian.gantner@uni-bamberg.de * */ -public class ItemEnhancerEntityTypeScript extends DSpaceRunnable> { +public class ItemEnhancerEntityTypeScript + extends DSpaceRunnable> { private ItemService itemService; - private CollectionService collectionService; - private ItemEnhancerService itemEnhancerService; - private boolean force; - private UUID collection; - private String entitytype; private Context context; @@ -83,21 +80,21 @@ public void setup() throws ParseException { this.entitytype = commandLine.getOptionValue('e'); } if (commandLine.hasOption('l')) { - try{ + try { this.limit = Integer.parseInt(commandLine.getOptionValue('l')); } catch (Exception e) { // } } if (commandLine.hasOption('m')) { - try{ + try { this.max = Integer.parseInt(commandLine.getOptionValue('m')); } catch (Exception e) { // } } if (commandLine.hasOption('o')) { - try{ + try { this.offset = Integer.parseInt(commandLine.getOptionValue('o')); } catch (Exception e) { // @@ -113,7 +110,8 @@ public void internalRun() throws Exception { if (commandLine.hasOption('e') && Objects.isNull(entityTypeService.findByEntityType(context, entitytype))) { throw new Exception("unknown entity " + entitytype); } - if (commandLine.hasOption('c') && (Objects.isNull(collection) || Objects.isNull(this.collectionService.find(context, collection)))) { + if (commandLine.hasOption('c') && (Objects.isNull(collection) + || Objects.isNull(this.collectionService.find(context, collection)))) { throw new Exception("specified collection does not exist"); } context.turnOffAuthorisationSystem(); @@ -146,25 +144,31 @@ private void enhanceItems() { private void findItemsToEnhanceLimitMax() { Collection coll; - int total, counter = 0; + int total = 0; + int counter = 0; if (Objects.nonNull(collection)) { //Paginate through items in one collection try { - coll = collectionService.find(context, collection); - total = itemService.countItems(context, coll); + coll = collectionService.find(context, collection); + total = itemService.countItems(context, coll); } catch (SQLException e) { handler.logError(e.getMessage()); return; } - if ( this.max > 0) total = this.max; + if ( this.max > 0) { + total = this.max; + } if (this.offset > 0) { //offset is being added to counter and offset total += offset; counter += offset; - handler.logInfo("offset " + offset + " added. Range: [" + counter + " to " + total + "] in " + limit + " steps"); - log.info("offset " + offset + " added. Range: [" + counter + " to " + total + "] in " + limit + " steps"); + handler.logInfo("offset " + offset + " added. Range: [" + + counter + " to " + total + "] in " + limit + " steps"); + log.info("offset " + offset + " added. Range: [" + + counter + " to " + total + "] in " + limit + " steps"); } else { - handler.logInfo("Range: [" + counter + " to " + total + "] in " + limit + " steps"); + handler.logInfo("Range: [" + counter + " to " + + total + "] in " + limit + " steps"); log.info("Range: [" + counter + " to " + total + "] in " + limit + " steps"); } while (counter < total) { @@ -174,7 +178,7 @@ private void findItemsToEnhanceLimitMax() { context.commit(); handler.logInfo("processed " + counter + " out of total " + total + " items"); log.info("processed " + counter + " out of total " + total + " items"); - } catch (SQLException e){ + } catch (SQLException e) { handler.logError(e.getMessage()); counter += limit; handler.logInfo("processed " + counter + " out of total " + total + " items"); @@ -189,13 +193,17 @@ private void findItemsToEnhanceLimitMax() { handler.logError(e.getMessage()); return; } - if (this.max > 0) total = this.max; + if (this.max > 0) { + total = this.max; + } if (this.offset > 0) { //offset is being added to counter and offset total += offset; counter += offset; - handler.logInfo("offset" + offset + " added. Range: [" + counter + " to " + total + "] in " + limit + " steps"); - log.info("offset" + offset + " added. Range: [" + counter + " to " + total + "] in " + limit + " steps"); + handler.logInfo("offset" + offset + " added. Range: [" + + counter + " to " + total + "] in " + limit + " steps"); + log.info("offset" + offset + " added. Range: [" + + counter + " to " + total + "] in " + limit + " steps"); } else { handler.logInfo("Range: [" + counter + " to " + total + "] in " + limit + " steps"); log.info("Range: [" + counter + " to " + total + "] in " + limit + " steps"); @@ -224,17 +232,17 @@ private void findItemsToEnhanceLimitMax() { private Iterator findItemsToEnhance() { try { - Iterator result = null; - if (Objects.nonNull(collection)) { - //Check, if uuid exist - Collection coll = collectionService.find(context, collection); - if (coll != null) { - result = itemService.findAllByCollection(context, coll); - } - } else { - result = itemService.findAll(context); - } - return result; + Iterator result = null; + if (Objects.nonNull(collection)) { + //Check, if uuid exist + Collection coll = collectionService.find(context, collection); + if (coll != null) { + result = itemService.findAllByCollection(context, coll); + } + } else { + result = itemService.findAll(context); + } + return result; } catch (SQLException e) { throw new SQLRuntimeException(e); } diff --git a/dspace-api/src/main/java/org/dspace/content/enhancer/script/ItemEnhancerEntityTypeScriptConfiguration.java b/dspace-api/src/main/java/org/dspace/content/enhancer/script/ItemEnhancerEntityTypeScriptConfiguration.java index 835c27ba7b53..3a48fa66d27d 100644 --- a/dspace-api/src/main/java/org/dspace/content/enhancer/script/ItemEnhancerEntityTypeScriptConfiguration.java +++ b/dspace-api/src/main/java/org/dspace/content/enhancer/script/ItemEnhancerEntityTypeScriptConfiguration.java @@ -21,7 +21,8 @@ * @author Luca Giamminonni (luca.giamminonni at 4science.it) * @author Florian Gantner (florian.gantner@uni-bamberg.de) */ -public class ItemEnhancerEntityTypeScriptConfiguration extends ScriptConfiguration { +public class ItemEnhancerEntityTypeScriptConfiguration + extends ScriptConfiguration { @Autowired private AuthorizeService authorizeService; @@ -43,8 +44,10 @@ public Options getOptions() { Options options = new Options(); options.addOption("f", "force", false, "force the recalculation of all the virtual fields"); - options.addOption("c", "collection", true, "uuid of the collection. If the collection does not exist the script aborts."); - options.addOption("e", "entity", true, "Entity type of the items. Not recommended, because all items are loaded and then filtered by their entity types."); + options.addOption("c", "collection", true, + "uuid of the collection. If the collection does not exist the script aborts."); + options.addOption("e", "entity", true, + "Entity type of the items. Not recommended, because all items are loaded and then filtered."); options.addOption("l", "limit", true, "size for iterator --limit items and commit after --limit items"); options.addOption("m", "max", true, "process max --max items"); options.addOption("o", "offset", true, "offset of items to start --offset items from the start"); From a4e7728fddff95cd6a9c541af5400eafffc94b42 Mon Sep 17 00:00:00 2001 From: "Gantner, Florian Klaus" Date: Tue, 8 Aug 2023 18:43:37 +0200 Subject: [PATCH 08/19] fix another dspace checkstyle issues --- .../content/enhancer/script/ItemEnhancerEntityTypeScript.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dspace-api/src/main/java/org/dspace/content/enhancer/script/ItemEnhancerEntityTypeScript.java b/dspace-api/src/main/java/org/dspace/content/enhancer/script/ItemEnhancerEntityTypeScript.java index 960d075dc79e..e73f13811da1 100644 --- a/dspace-api/src/main/java/org/dspace/content/enhancer/script/ItemEnhancerEntityTypeScript.java +++ b/dspace-api/src/main/java/org/dspace/content/enhancer/script/ItemEnhancerEntityTypeScript.java @@ -301,7 +301,7 @@ private void assignSpecialGroupsInContext() { @SuppressWarnings("unchecked") public ItemEnhancerEntityTypeScriptConfiguration getScriptConfiguration() { return new DSpace().getServiceManager().getServiceByName("item-enhancer-type", - ItemEnhancerEntityTypeScriptConfiguration.class); + ItemEnhancerEntityTypeScriptConfiguration.class); } } From fd7922266bf55905dbec1d5969aedaefad6d8582 Mon Sep 17 00:00:00 2001 From: "Gantner, Florian Klaus" Date: Tue, 22 Aug 2023 10:44:41 +0200 Subject: [PATCH 09/19] common itemenhancer integration tests call the script without further options; copy of standard tests from ItemEnhancerScriptIT --- .../config/spring/api/scripts.xml | 10 + .../script/ItemEnhancerByDateScriptIT.java | 484 ++++++++++++++++++ .../ItemEnhancerEntityTypeScriptIT.java | 484 ++++++++++++++++++ dspace/config/spring/api/scripts.xml | 4 +- 4 files changed, 980 insertions(+), 2 deletions(-) create mode 100644 dspace-api/src/test/java/org/dspace/content/enhancer/script/ItemEnhancerByDateScriptIT.java create mode 100644 dspace-api/src/test/java/org/dspace/content/enhancer/script/ItemEnhancerEntityTypeScriptIT.java diff --git a/dspace-api/src/test/data/dspaceFolder/config/spring/api/scripts.xml b/dspace-api/src/test/data/dspaceFolder/config/spring/api/scripts.xml index 6a3830f616d6..d66eb103e468 100644 --- a/dspace-api/src/test/data/dspaceFolder/config/spring/api/scripts.xml +++ b/dspace-api/src/test/data/dspaceFolder/config/spring/api/scripts.xml @@ -135,6 +135,16 @@ + + + + + + + + + + diff --git a/dspace-api/src/test/java/org/dspace/content/enhancer/script/ItemEnhancerByDateScriptIT.java b/dspace-api/src/test/java/org/dspace/content/enhancer/script/ItemEnhancerByDateScriptIT.java new file mode 100644 index 000000000000..0aacc1b8d63f --- /dev/null +++ b/dspace-api/src/test/java/org/dspace/content/enhancer/script/ItemEnhancerByDateScriptIT.java @@ -0,0 +1,484 @@ +/** + * The contents of this file are subject to the license and copyright + * detailed in the LICENSE and NOTICE files at the root of the source + * tree and available online at + * + * http://www.dspace.org/license/ + */ +package org.dspace.content.enhancer.script; + +import static org.dspace.app.matcher.MetadataValueMatcher.with; +import static org.dspace.content.Item.ANY; +import static org.dspace.core.CrisConstants.PLACEHOLDER_PARENT_METADATA_VALUE; +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.contains; +import static org.hamcrest.Matchers.empty; +import static org.hamcrest.Matchers.hasItem; +import static org.hamcrest.Matchers.hasSize; + +import java.sql.SQLException; +import java.util.Arrays; +import java.util.HashSet; +import java.util.List; +import java.util.Set; + +import org.dspace.AbstractIntegrationTestWithDatabase; +import org.dspace.app.launcher.ScriptLauncher; +import org.dspace.app.scripts.handler.impl.TestDSpaceRunnableHandler; +import org.dspace.authorize.AuthorizeException; +import org.dspace.builder.CollectionBuilder; +import org.dspace.builder.CommunityBuilder; +import org.dspace.builder.ItemBuilder; +import org.dspace.builder.WorkspaceItemBuilder; +import org.dspace.content.Collection; +import org.dspace.content.Item; +import org.dspace.content.MetadataValue; +import org.dspace.content.WorkspaceItem; +import org.dspace.content.factory.ContentServiceFactory; +import org.dspace.content.service.ItemService; +import org.dspace.core.ReloadableEntity; +import org.dspace.event.factory.EventServiceFactory; +import org.dspace.event.service.EventService; +import org.dspace.services.ConfigurationService; +import org.dspace.services.factory.DSpaceServicesFactory; +import org.junit.AfterClass; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Test; + +public class ItemEnhancerByDateScriptIT extends AbstractIntegrationTestWithDatabase { + + private static String[] consumers; + + private ItemService itemService; + + private Collection collection; + + /** + * This method will be run before the first test as per @BeforeClass. It will + * configure the event.dispatcher.default.consumers property to remove the + * ItemEnhancerConsumer. + */ + @BeforeClass + public static void initConsumers() { + ConfigurationService configService = DSpaceServicesFactory.getInstance().getConfigurationService(); + consumers = configService.getArrayProperty("event.dispatcher.default.consumers"); + Set consumersSet = new HashSet(Arrays.asList(consumers)); + consumersSet.remove("itemenhancer"); + configService.setProperty("event.dispatcher.default.consumers", consumersSet.toArray()); + EventService eventService = EventServiceFactory.getInstance().getEventService(); + eventService.reloadConfiguration(); + } + + /** + * Reset the event.dispatcher.default.consumers property value. + */ + @AfterClass + public static void resetDefaultConsumers() { + ConfigurationService configService = DSpaceServicesFactory.getInstance().getConfigurationService(); + configService.setProperty("event.dispatcher.default.consumers", consumers); + EventService eventService = EventServiceFactory.getInstance().getEventService(); + eventService.reloadConfiguration(); + } + + @Before + public void setup() { + + itemService = ContentServiceFactory.getInstance().getItemService(); + + context.turnOffAuthorisationSystem(); + parentCommunity = CommunityBuilder.createCommunity(context) + .withName("Parent Community") + .build(); + + collection = CollectionBuilder.createCollection(context, parentCommunity) + .withName("Collection") + .build(); + context.restoreAuthSystemState(); + + } + + @Test + public void testItemsEnhancement() throws Exception { + + context.turnOffAuthorisationSystem(); + + Item firstAuthor = ItemBuilder.createItem(context, collection) + .withTitle("Walter White") + .withPersonMainAffiliation("4Science") + .build(); + + String firstAuthorId = firstAuthor.getID().toString(); + + Item secondAuthor = ItemBuilder.createItem(context, collection) + .withTitle("Jesse Pinkman") + .withPersonMainAffiliation("Company") + .build(); + + String secondAuthorId = secondAuthor.getID().toString(); + + Item firstPublication = ItemBuilder.createItem(context, collection) + .withTitle("Test publication") + .withEntityType("Publication") + .withAuthor("Walter White", firstAuthorId) + .build(); + + Item secondPublication = ItemBuilder.createItem(context, collection) + .withTitle("Test publication 2") + .withEntityType("Publication") + .withAuthor("Walter White", firstAuthorId) + .withAuthor("Jesse Pinkman", secondAuthorId) + .build(); + + WorkspaceItem thirdPublication = WorkspaceItemBuilder.createWorkspaceItem(context, collection) + .withTitle("Test publication 3") + .withEntityType("Publication") + .withAuthor("Jesse Pinkman", secondAuthorId) + .build(); + + context.commit(); + + firstPublication = reload(firstPublication); + secondPublication = reload(secondPublication); + thirdPublication = reload(thirdPublication); + + assertThat(getMetadataValues(firstPublication, "cris.virtual.department"), empty()); + assertThat(getMetadataValues(firstPublication, "cris.virtualsource.department"), empty()); + assertThat(getMetadataValues(secondPublication, "cris.virtual.department"), empty()); + assertThat(getMetadataValues(secondPublication, "cris.virtualsource.department"), empty()); + assertThat(getMetadataValues(thirdPublication, "cris.virtual.department"), empty()); + assertThat(getMetadataValues(thirdPublication, "cris.virtualsource.department"), empty()); + + TestDSpaceRunnableHandler runnableHandler = runScript(false); + + assertThat(runnableHandler.getErrorMessages(), empty()); + assertThat(runnableHandler.getInfoMessages(), contains("Enhancement completed with success")); + + firstPublication = reload(firstPublication); + secondPublication = reload(secondPublication); + + assertThat(getMetadataValues(firstPublication, "cris.virtual.department"), hasSize(1)); + assertThat(getMetadataValues(firstPublication, "cris.virtualsource.department"), hasSize(1)); + + assertThat(getMetadataValues(secondPublication, "cris.virtual.department"), hasSize(2)); + assertThat(getMetadataValues(secondPublication, "cris.virtualsource.department"), hasSize(2)); + + assertThat(firstPublication.getMetadata(), hasItem(with("cris.virtual.department", "4Science"))); + assertThat(firstPublication.getMetadata(), hasItem(with("cris.virtualsource.department", firstAuthorId))); + assertThat(secondPublication.getMetadata(), hasItem(with("cris.virtual.department", "4Science"))); + assertThat(secondPublication.getMetadata(), hasItem(with("cris.virtualsource.department", firstAuthorId))); + assertThat(secondPublication.getMetadata(), hasItem(with("cris.virtual.department", "Company", 1))); + assertThat(secondPublication.getMetadata(), hasItem(with("cris.virtualsource.department", secondAuthorId, 1))); + + assertThat(getMetadataValues(thirdPublication, "cris.virtual.department"), empty()); + assertThat(getMetadataValues(thirdPublication, "cris.virtualsource.department"), empty()); + + } + + @Test + public void testItemEnhancementWithoutForce() throws Exception { + + context.turnOffAuthorisationSystem(); + + Item firstAuthor = ItemBuilder.createItem(context, collection) + .withTitle("Walter White") + .withPersonMainAffiliation("4Science") + .build(); + + String firstAuthorId = firstAuthor.getID().toString(); + + Item secondAuthor = ItemBuilder.createItem(context, collection) + .withTitle("Jesse Pinkman") + .withPersonMainAffiliation("Company") + .build(); + + String secondAuthorId = secondAuthor.getID().toString(); + + Item publication = ItemBuilder.createItem(context, collection) + .withTitle("Test publication 2 ") + .withEntityType("Publication") + .withAuthor("Walter White", firstAuthorId) + .withAuthor("Jesse Pinkman", secondAuthorId) + .build(); + + context.commit(); + publication = reload(publication); + + assertThat(getMetadataValues(publication, "cris.virtual.department"), empty()); + assertThat(getMetadataValues(publication, "cris.virtualsource.department"), empty()); + + TestDSpaceRunnableHandler runnableHandler = runScript(false); + + assertThat(runnableHandler.getErrorMessages(), empty()); + assertThat(runnableHandler.getInfoMessages(), contains("Enhancement completed with success")); + + publication = reload(publication); + + assertThat(getMetadataValues(publication, "cris.virtual.department"), hasSize(2)); + assertThat(getMetadataValues(publication, "cris.virtualsource.department"), hasSize(2)); + + assertThat(publication.getMetadata(), hasItem(with("cris.virtual.department", "4Science"))); + assertThat(publication.getMetadata(), hasItem(with("cris.virtualsource.department", firstAuthorId))); + assertThat(publication.getMetadata(), hasItem(with("cris.virtual.department", "Company", 1))); + assertThat(publication.getMetadata(), hasItem(with("cris.virtualsource.department", secondAuthorId, 1))); + + context.turnOffAuthorisationSystem(); + + MetadataValue authorToRemove = getMetadataValues(publication, "dc.contributor.author").get(1); + itemService.removeMetadataValues(context, publication, List.of(authorToRemove)); + + replaceMetadata(firstAuthor, "person", "affiliation", "name", "University"); + + context.restoreAuthSystemState(); + + runnableHandler = runScript(false); + assertThat(runnableHandler.getErrorMessages(), empty()); + assertThat(runnableHandler.getInfoMessages(), contains("Enhancement completed with success")); + + publication = reload(publication); + + assertThat(getMetadataValues(publication, "cris.virtual.department"), hasSize(1)); + assertThat(getMetadataValues(publication, "cris.virtualsource.department"), hasSize(1)); + + assertThat(publication.getMetadata(), hasItem(with("cris.virtual.department", "4Science"))); + assertThat(publication.getMetadata(), hasItem(with("cris.virtualsource.department", firstAuthorId))); + + } + + @Test + public void testItemEnhancementWithForce() throws Exception { + + context.turnOffAuthorisationSystem(); + + Item firstAuthor = ItemBuilder.createItem(context, collection) + .withTitle("Walter White") + .withPersonMainAffiliation("4Science") + .build(); + + String firstAuthorId = firstAuthor.getID().toString(); + + Item secondAuthor = ItemBuilder.createItem(context, collection) + .withTitle("Jesse Pinkman") + .withPersonMainAffiliation("Company") + .build(); + + String secondAuthorId = secondAuthor.getID().toString(); + + Item publication = ItemBuilder.createItem(context, collection) + .withTitle("Test publication 2 ") + .withEntityType("Publication") + .withAuthor("Walter White", firstAuthorId) + .withAuthor("Jesse Pinkman", secondAuthorId) + .build(); + + context.commit(); + publication = reload(publication); + + assertThat(getMetadataValues(publication, "cris.virtual.department"), empty()); + assertThat(getMetadataValues(publication, "cris.virtualsource.department"), empty()); + + TestDSpaceRunnableHandler runnableHandler = runScript(false); + + assertThat(runnableHandler.getErrorMessages(), empty()); + assertThat(runnableHandler.getInfoMessages(), contains("Enhancement completed with success")); + + publication = reload(publication); + + assertThat(getMetadataValues(publication, "cris.virtual.department"), hasSize(2)); + assertThat(getMetadataValues(publication, "cris.virtualsource.department"), hasSize(2)); + + assertThat(publication.getMetadata(), hasItem(with("cris.virtual.department", "4Science"))); + assertThat(publication.getMetadata(), hasItem(with("cris.virtualsource.department", firstAuthorId))); + assertThat(publication.getMetadata(), hasItem(with("cris.virtual.department", "Company", 1))); + assertThat(publication.getMetadata(), hasItem(with("cris.virtualsource.department", secondAuthorId, 1))); + + context.turnOffAuthorisationSystem(); + + MetadataValue authorToRemove = getMetadataValues(publication, "dc.contributor.author").get(1); + itemService.removeMetadataValues(context, publication, List.of(authorToRemove)); + + replaceMetadata(firstAuthor, "person", "affiliation", "name", "University"); + + context.restoreAuthSystemState(); + + runnableHandler = runScript(true); + assertThat(runnableHandler.getErrorMessages(), empty()); + assertThat(runnableHandler.getInfoMessages(), contains("Enhancement completed with success")); + + publication = reload(publication); + + assertThat(getMetadataValues(publication, "cris.virtual.department"), hasSize(1)); + assertThat(getMetadataValues(publication, "cris.virtualsource.department"), hasSize(1)); + + assertThat(publication.getMetadata(), hasItem(with("cris.virtual.department", "University"))); + assertThat(publication.getMetadata(), hasItem(with("cris.virtualsource.department", firstAuthorId))); + + } + + @Test + public void testItemEnhancementMetadataPositions() throws Exception { + + context.turnOffAuthorisationSystem(); + + Item firstAuthor = ItemBuilder.createItem(context, collection) + .withTitle("John Doe") + .build(); + String firstAuthorId = firstAuthor.getID().toString(); + + Item secondAuthor = ItemBuilder.createItem(context, collection) + .withTitle("Walter White") + .withPersonMainAffiliation("4Science") + .build(); + String secondAuthorId = secondAuthor.getID().toString(); + + Item thirdAuthor = ItemBuilder.createItem(context, collection) + .withTitle("Jesse Pinkman") + .withPersonMainAffiliation("Company") + .build(); + + String thirdAuthorId = thirdAuthor.getID().toString(); + + Item fourthAuthor = ItemBuilder.createItem(context, collection) + .withTitle("Jesse Smith") + .build(); + + String fourthAuthorId = fourthAuthor.getID().toString(); + + Item publication = ItemBuilder.createItem(context, collection) + .withTitle("Test publication 2 ") + .withEntityType("Publication") + .withAuthor("John Doe", firstAuthorId) + .withAuthor("Walter White", secondAuthorId) + .withAuthor("Jesse Pinkman", thirdAuthorId) + .withAuthor("Jesse Smith", fourthAuthorId) + .build(); + + context.commit(); + publication = reload(publication); + + assertThat(getMetadataValues(publication, "cris.virtual.department"), empty()); + assertThat(getMetadataValues(publication, "cris.virtualsource.department"), empty()); + + TestDSpaceRunnableHandler runnableHandler = runScript(false); + + assertThat(runnableHandler.getErrorMessages(), empty()); + assertThat(runnableHandler.getInfoMessages(), contains("Enhancement completed with success")); + + publication = reload(publication); + + assertThat(getMetadataValues(publication, "cris.virtual.department"), hasSize(4)); + assertThat(getMetadataValues(publication, "cris.virtualsource.department"), hasSize(4)); + + assertThat(publication.getMetadata(), hasItem(with("cris.virtual.department", + PLACEHOLDER_PARENT_METADATA_VALUE, 0))); + assertThat(publication.getMetadata(), hasItem(with("cris.virtualsource.department", firstAuthorId,0))); + assertThat(publication.getMetadata(), hasItem(with("cris.virtual.department", "4Science", 1))); + assertThat(publication.getMetadata(), hasItem(with("cris.virtualsource.department", secondAuthorId,1))); + assertThat(publication.getMetadata(), hasItem(with("cris.virtual.department", "Company", 2))); + assertThat(publication.getMetadata(), hasItem(with("cris.virtualsource.department", thirdAuthorId, 2))); + assertThat(publication.getMetadata(), hasItem(with("cris.virtual.department", + PLACEHOLDER_PARENT_METADATA_VALUE, 3))); + assertThat(publication.getMetadata(), hasItem(with("cris.virtualsource.department", fourthAuthorId,3))); + + } + + @Test + public void testItemEnhancementSourceWithoutAuthority() throws Exception { + + context.turnOffAuthorisationSystem(); + + Item secondAuthor = ItemBuilder.createItem(context, collection) + .withTitle("Jesse Smith") + .withPersonMainAffiliation("4Science") + .build(); + + String secondAuthorId = secondAuthor.getID().toString(); + + Item publication = ItemBuilder.createItem(context, collection) + .withTitle("Test publication 2 ") + .withEntityType("Publication") + .withAuthor("Jesse Pinkman") + .withAuthor("Jesse Smith", secondAuthorId) + .build(); + + context.commit(); + publication = reload(publication); + + assertThat(getMetadataValues(publication, "cris.virtual.department"), empty()); + assertThat(getMetadataValues(publication, "cris.virtualsource.department"), empty()); + + TestDSpaceRunnableHandler runnableHandler = runScript(false); + + assertThat(runnableHandler.getErrorMessages(), empty()); + assertThat(runnableHandler.getInfoMessages(), contains("Enhancement completed with success")); + + publication = reload(publication); + + assertThat(getMetadataValues(publication, "cris.virtual.department"), hasSize(2)); + assertThat(getMetadataValues(publication, "cris.virtualsource.department"), hasSize(2)); + + assertThat(publication.getMetadata(), hasItem(with("cris.virtual.department", + PLACEHOLDER_PARENT_METADATA_VALUE, 0))); + assertThat(publication.getMetadata(), hasItem(with("cris.virtualsource.department", + PLACEHOLDER_PARENT_METADATA_VALUE,0))); + assertThat(publication.getMetadata(), hasItem(with("cris.virtual.department", "4Science", 1))); + assertThat(publication.getMetadata(), hasItem(with("cris.virtualsource.department", secondAuthorId,1))); + + } + + @Test + public void testItemEnhancementWithoutAuthorities() throws Exception { + + context.turnOffAuthorisationSystem(); + + Item publication = ItemBuilder.createItem(context, collection) + .withTitle("Test publication 2 ") + .withEntityType("Publication") + .withAuthor("Jesse Pinkman") + .withAuthor("Jesse Smith") + .build(); + + context.commit(); + publication = reload(publication); + + assertThat(getMetadataValues(publication, "cris.virtual.department"), empty()); + assertThat(getMetadataValues(publication, "cris.virtualsource.department"), empty()); + + TestDSpaceRunnableHandler runnableHandler = runScript(false); + + assertThat(runnableHandler.getErrorMessages(), empty()); + assertThat(runnableHandler.getInfoMessages(), contains("Enhancement completed with success")); + + publication = reload(publication); + + assertThat(getMetadataValues(publication, "cris.virtual.department"), empty()); + assertThat(getMetadataValues(publication, "cris.virtualsource.department"), empty()); + + } + + private TestDSpaceRunnableHandler runScript(boolean force) throws InstantiationException, IllegalAccessException { + TestDSpaceRunnableHandler runnableHandler = new TestDSpaceRunnableHandler(); + String[] args = force ? new String[] { "item-enhancer-date", "-f" } : new String[] { "item-enhancer-date" }; + ScriptLauncher.handleScript(args, ScriptLauncher.getConfig(kernelImpl), runnableHandler, kernelImpl); + return runnableHandler; + } + + @SuppressWarnings("rawtypes") + private T reload(T entity) throws SQLException, AuthorizeException { + return context.reloadEntity(entity); + } + + private void replaceMetadata(Item item, String schema, String element, String qualifier, String newValue) + throws SQLException, AuthorizeException { + itemService.replaceMetadata(context, reload(item), schema, element, qualifier, ANY, newValue, null, -1, 0); + } + + private List getMetadataValues(Item item, String metadataField) { + return itemService.getMetadataByMetadataString(item, metadataField); + } + + private List getMetadataValues(WorkspaceItem item, String metadataField) { + return itemService.getMetadataByMetadataString(item.getItem(), metadataField); + } + +} diff --git a/dspace-api/src/test/java/org/dspace/content/enhancer/script/ItemEnhancerEntityTypeScriptIT.java b/dspace-api/src/test/java/org/dspace/content/enhancer/script/ItemEnhancerEntityTypeScriptIT.java new file mode 100644 index 000000000000..805e9abb6138 --- /dev/null +++ b/dspace-api/src/test/java/org/dspace/content/enhancer/script/ItemEnhancerEntityTypeScriptIT.java @@ -0,0 +1,484 @@ +/** + * The contents of this file are subject to the license and copyright + * detailed in the LICENSE and NOTICE files at the root of the source + * tree and available online at + * + * http://www.dspace.org/license/ + */ +package org.dspace.content.enhancer.script; + +import static org.dspace.app.matcher.MetadataValueMatcher.with; +import static org.dspace.content.Item.ANY; +import static org.dspace.core.CrisConstants.PLACEHOLDER_PARENT_METADATA_VALUE; +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.contains; +import static org.hamcrest.Matchers.empty; +import static org.hamcrest.Matchers.hasItem; +import static org.hamcrest.Matchers.hasSize; + +import java.sql.SQLException; +import java.util.Arrays; +import java.util.HashSet; +import java.util.List; +import java.util.Set; + +import org.dspace.AbstractIntegrationTestWithDatabase; +import org.dspace.app.launcher.ScriptLauncher; +import org.dspace.app.scripts.handler.impl.TestDSpaceRunnableHandler; +import org.dspace.authorize.AuthorizeException; +import org.dspace.builder.CollectionBuilder; +import org.dspace.builder.CommunityBuilder; +import org.dspace.builder.ItemBuilder; +import org.dspace.builder.WorkspaceItemBuilder; +import org.dspace.content.Collection; +import org.dspace.content.Item; +import org.dspace.content.MetadataValue; +import org.dspace.content.WorkspaceItem; +import org.dspace.content.factory.ContentServiceFactory; +import org.dspace.content.service.ItemService; +import org.dspace.core.ReloadableEntity; +import org.dspace.event.factory.EventServiceFactory; +import org.dspace.event.service.EventService; +import org.dspace.services.ConfigurationService; +import org.dspace.services.factory.DSpaceServicesFactory; +import org.junit.AfterClass; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Test; + +public class ItemEnhancerEntityTypeScriptIT extends AbstractIntegrationTestWithDatabase { + + private static String[] consumers; + + private ItemService itemService; + + private Collection collection; + + /** + * This method will be run before the first test as per @BeforeClass. It will + * configure the event.dispatcher.default.consumers property to remove the + * ItemEnhancerConsumer. + */ + @BeforeClass + public static void initConsumers() { + ConfigurationService configService = DSpaceServicesFactory.getInstance().getConfigurationService(); + consumers = configService.getArrayProperty("event.dispatcher.default.consumers"); + Set consumersSet = new HashSet(Arrays.asList(consumers)); + consumersSet.remove("itemenhancer"); + configService.setProperty("event.dispatcher.default.consumers", consumersSet.toArray()); + EventService eventService = EventServiceFactory.getInstance().getEventService(); + eventService.reloadConfiguration(); + } + + /** + * Reset the event.dispatcher.default.consumers property value. + */ + @AfterClass + public static void resetDefaultConsumers() { + ConfigurationService configService = DSpaceServicesFactory.getInstance().getConfigurationService(); + configService.setProperty("event.dispatcher.default.consumers", consumers); + EventService eventService = EventServiceFactory.getInstance().getEventService(); + eventService.reloadConfiguration(); + } + + @Before + public void setup() { + + itemService = ContentServiceFactory.getInstance().getItemService(); + + context.turnOffAuthorisationSystem(); + parentCommunity = CommunityBuilder.createCommunity(context) + .withName("Parent Community") + .build(); + + collection = CollectionBuilder.createCollection(context, parentCommunity) + .withName("Collection") + .build(); + context.restoreAuthSystemState(); + + } + + @Test + public void testItemsEnhancement() throws Exception { + + context.turnOffAuthorisationSystem(); + + Item firstAuthor = ItemBuilder.createItem(context, collection) + .withTitle("Walter White") + .withPersonMainAffiliation("4Science") + .build(); + + String firstAuthorId = firstAuthor.getID().toString(); + + Item secondAuthor = ItemBuilder.createItem(context, collection) + .withTitle("Jesse Pinkman") + .withPersonMainAffiliation("Company") + .build(); + + String secondAuthorId = secondAuthor.getID().toString(); + + Item firstPublication = ItemBuilder.createItem(context, collection) + .withTitle("Test publication") + .withEntityType("Publication") + .withAuthor("Walter White", firstAuthorId) + .build(); + + Item secondPublication = ItemBuilder.createItem(context, collection) + .withTitle("Test publication 2") + .withEntityType("Publication") + .withAuthor("Walter White", firstAuthorId) + .withAuthor("Jesse Pinkman", secondAuthorId) + .build(); + + WorkspaceItem thirdPublication = WorkspaceItemBuilder.createWorkspaceItem(context, collection) + .withTitle("Test publication 3") + .withEntityType("Publication") + .withAuthor("Jesse Pinkman", secondAuthorId) + .build(); + + context.commit(); + + firstPublication = reload(firstPublication); + secondPublication = reload(secondPublication); + thirdPublication = reload(thirdPublication); + + assertThat(getMetadataValues(firstPublication, "cris.virtual.department"), empty()); + assertThat(getMetadataValues(firstPublication, "cris.virtualsource.department"), empty()); + assertThat(getMetadataValues(secondPublication, "cris.virtual.department"), empty()); + assertThat(getMetadataValues(secondPublication, "cris.virtualsource.department"), empty()); + assertThat(getMetadataValues(thirdPublication, "cris.virtual.department"), empty()); + assertThat(getMetadataValues(thirdPublication, "cris.virtualsource.department"), empty()); + + TestDSpaceRunnableHandler runnableHandler = runScript(false); + + assertThat(runnableHandler.getErrorMessages(), empty()); + assertThat(runnableHandler.getInfoMessages(), contains("Enhancement completed with success")); + + firstPublication = reload(firstPublication); + secondPublication = reload(secondPublication); + + assertThat(getMetadataValues(firstPublication, "cris.virtual.department"), hasSize(1)); + assertThat(getMetadataValues(firstPublication, "cris.virtualsource.department"), hasSize(1)); + + assertThat(getMetadataValues(secondPublication, "cris.virtual.department"), hasSize(2)); + assertThat(getMetadataValues(secondPublication, "cris.virtualsource.department"), hasSize(2)); + + assertThat(firstPublication.getMetadata(), hasItem(with("cris.virtual.department", "4Science"))); + assertThat(firstPublication.getMetadata(), hasItem(with("cris.virtualsource.department", firstAuthorId))); + assertThat(secondPublication.getMetadata(), hasItem(with("cris.virtual.department", "4Science"))); + assertThat(secondPublication.getMetadata(), hasItem(with("cris.virtualsource.department", firstAuthorId))); + assertThat(secondPublication.getMetadata(), hasItem(with("cris.virtual.department", "Company", 1))); + assertThat(secondPublication.getMetadata(), hasItem(with("cris.virtualsource.department", secondAuthorId, 1))); + + assertThat(getMetadataValues(thirdPublication, "cris.virtual.department"), empty()); + assertThat(getMetadataValues(thirdPublication, "cris.virtualsource.department"), empty()); + + } + + @Test + public void testItemEnhancementWithoutForce() throws Exception { + + context.turnOffAuthorisationSystem(); + + Item firstAuthor = ItemBuilder.createItem(context, collection) + .withTitle("Walter White") + .withPersonMainAffiliation("4Science") + .build(); + + String firstAuthorId = firstAuthor.getID().toString(); + + Item secondAuthor = ItemBuilder.createItem(context, collection) + .withTitle("Jesse Pinkman") + .withPersonMainAffiliation("Company") + .build(); + + String secondAuthorId = secondAuthor.getID().toString(); + + Item publication = ItemBuilder.createItem(context, collection) + .withTitle("Test publication 2 ") + .withEntityType("Publication") + .withAuthor("Walter White", firstAuthorId) + .withAuthor("Jesse Pinkman", secondAuthorId) + .build(); + + context.commit(); + publication = reload(publication); + + assertThat(getMetadataValues(publication, "cris.virtual.department"), empty()); + assertThat(getMetadataValues(publication, "cris.virtualsource.department"), empty()); + + TestDSpaceRunnableHandler runnableHandler = runScript(false); + + assertThat(runnableHandler.getErrorMessages(), empty()); + assertThat(runnableHandler.getInfoMessages(), contains("Enhancement completed with success")); + + publication = reload(publication); + + assertThat(getMetadataValues(publication, "cris.virtual.department"), hasSize(2)); + assertThat(getMetadataValues(publication, "cris.virtualsource.department"), hasSize(2)); + + assertThat(publication.getMetadata(), hasItem(with("cris.virtual.department", "4Science"))); + assertThat(publication.getMetadata(), hasItem(with("cris.virtualsource.department", firstAuthorId))); + assertThat(publication.getMetadata(), hasItem(with("cris.virtual.department", "Company", 1))); + assertThat(publication.getMetadata(), hasItem(with("cris.virtualsource.department", secondAuthorId, 1))); + + context.turnOffAuthorisationSystem(); + + MetadataValue authorToRemove = getMetadataValues(publication, "dc.contributor.author").get(1); + itemService.removeMetadataValues(context, publication, List.of(authorToRemove)); + + replaceMetadata(firstAuthor, "person", "affiliation", "name", "University"); + + context.restoreAuthSystemState(); + + runnableHandler = runScript(false); + assertThat(runnableHandler.getErrorMessages(), empty()); + assertThat(runnableHandler.getInfoMessages(), contains("Enhancement completed with success")); + + publication = reload(publication); + + assertThat(getMetadataValues(publication, "cris.virtual.department"), hasSize(1)); + assertThat(getMetadataValues(publication, "cris.virtualsource.department"), hasSize(1)); + + assertThat(publication.getMetadata(), hasItem(with("cris.virtual.department", "4Science"))); + assertThat(publication.getMetadata(), hasItem(with("cris.virtualsource.department", firstAuthorId))); + + } + + @Test + public void testItemEnhancementWithForce() throws Exception { + + context.turnOffAuthorisationSystem(); + + Item firstAuthor = ItemBuilder.createItem(context, collection) + .withTitle("Walter White") + .withPersonMainAffiliation("4Science") + .build(); + + String firstAuthorId = firstAuthor.getID().toString(); + + Item secondAuthor = ItemBuilder.createItem(context, collection) + .withTitle("Jesse Pinkman") + .withPersonMainAffiliation("Company") + .build(); + + String secondAuthorId = secondAuthor.getID().toString(); + + Item publication = ItemBuilder.createItem(context, collection) + .withTitle("Test publication 2 ") + .withEntityType("Publication") + .withAuthor("Walter White", firstAuthorId) + .withAuthor("Jesse Pinkman", secondAuthorId) + .build(); + + context.commit(); + publication = reload(publication); + + assertThat(getMetadataValues(publication, "cris.virtual.department"), empty()); + assertThat(getMetadataValues(publication, "cris.virtualsource.department"), empty()); + + TestDSpaceRunnableHandler runnableHandler = runScript(false); + + assertThat(runnableHandler.getErrorMessages(), empty()); + assertThat(runnableHandler.getInfoMessages(), contains("Enhancement completed with success")); + + publication = reload(publication); + + assertThat(getMetadataValues(publication, "cris.virtual.department"), hasSize(2)); + assertThat(getMetadataValues(publication, "cris.virtualsource.department"), hasSize(2)); + + assertThat(publication.getMetadata(), hasItem(with("cris.virtual.department", "4Science"))); + assertThat(publication.getMetadata(), hasItem(with("cris.virtualsource.department", firstAuthorId))); + assertThat(publication.getMetadata(), hasItem(with("cris.virtual.department", "Company", 1))); + assertThat(publication.getMetadata(), hasItem(with("cris.virtualsource.department", secondAuthorId, 1))); + + context.turnOffAuthorisationSystem(); + + MetadataValue authorToRemove = getMetadataValues(publication, "dc.contributor.author").get(1); + itemService.removeMetadataValues(context, publication, List.of(authorToRemove)); + + replaceMetadata(firstAuthor, "person", "affiliation", "name", "University"); + + context.restoreAuthSystemState(); + + runnableHandler = runScript(true); + assertThat(runnableHandler.getErrorMessages(), empty()); + assertThat(runnableHandler.getInfoMessages(), contains("Enhancement completed with success")); + + publication = reload(publication); + + assertThat(getMetadataValues(publication, "cris.virtual.department"), hasSize(1)); + assertThat(getMetadataValues(publication, "cris.virtualsource.department"), hasSize(1)); + + assertThat(publication.getMetadata(), hasItem(with("cris.virtual.department", "University"))); + assertThat(publication.getMetadata(), hasItem(with("cris.virtualsource.department", firstAuthorId))); + + } + + @Test + public void testItemEnhancementMetadataPositions() throws Exception { + + context.turnOffAuthorisationSystem(); + + Item firstAuthor = ItemBuilder.createItem(context, collection) + .withTitle("John Doe") + .build(); + String firstAuthorId = firstAuthor.getID().toString(); + + Item secondAuthor = ItemBuilder.createItem(context, collection) + .withTitle("Walter White") + .withPersonMainAffiliation("4Science") + .build(); + String secondAuthorId = secondAuthor.getID().toString(); + + Item thirdAuthor = ItemBuilder.createItem(context, collection) + .withTitle("Jesse Pinkman") + .withPersonMainAffiliation("Company") + .build(); + + String thirdAuthorId = thirdAuthor.getID().toString(); + + Item fourthAuthor = ItemBuilder.createItem(context, collection) + .withTitle("Jesse Smith") + .build(); + + String fourthAuthorId = fourthAuthor.getID().toString(); + + Item publication = ItemBuilder.createItem(context, collection) + .withTitle("Test publication 2 ") + .withEntityType("Publication") + .withAuthor("John Doe", firstAuthorId) + .withAuthor("Walter White", secondAuthorId) + .withAuthor("Jesse Pinkman", thirdAuthorId) + .withAuthor("Jesse Smith", fourthAuthorId) + .build(); + + context.commit(); + publication = reload(publication); + + assertThat(getMetadataValues(publication, "cris.virtual.department"), empty()); + assertThat(getMetadataValues(publication, "cris.virtualsource.department"), empty()); + + TestDSpaceRunnableHandler runnableHandler = runScript(false); + + assertThat(runnableHandler.getErrorMessages(), empty()); + assertThat(runnableHandler.getInfoMessages(), contains("Enhancement completed with success")); + + publication = reload(publication); + + assertThat(getMetadataValues(publication, "cris.virtual.department"), hasSize(4)); + assertThat(getMetadataValues(publication, "cris.virtualsource.department"), hasSize(4)); + + assertThat(publication.getMetadata(), hasItem(with("cris.virtual.department", + PLACEHOLDER_PARENT_METADATA_VALUE, 0))); + assertThat(publication.getMetadata(), hasItem(with("cris.virtualsource.department", firstAuthorId,0))); + assertThat(publication.getMetadata(), hasItem(with("cris.virtual.department", "4Science", 1))); + assertThat(publication.getMetadata(), hasItem(with("cris.virtualsource.department", secondAuthorId,1))); + assertThat(publication.getMetadata(), hasItem(with("cris.virtual.department", "Company", 2))); + assertThat(publication.getMetadata(), hasItem(with("cris.virtualsource.department", thirdAuthorId, 2))); + assertThat(publication.getMetadata(), hasItem(with("cris.virtual.department", + PLACEHOLDER_PARENT_METADATA_VALUE, 3))); + assertThat(publication.getMetadata(), hasItem(with("cris.virtualsource.department", fourthAuthorId,3))); + + } + + @Test + public void testItemEnhancementSourceWithoutAuthority() throws Exception { + + context.turnOffAuthorisationSystem(); + + Item secondAuthor = ItemBuilder.createItem(context, collection) + .withTitle("Jesse Smith") + .withPersonMainAffiliation("4Science") + .build(); + + String secondAuthorId = secondAuthor.getID().toString(); + + Item publication = ItemBuilder.createItem(context, collection) + .withTitle("Test publication 2 ") + .withEntityType("Publication") + .withAuthor("Jesse Pinkman") + .withAuthor("Jesse Smith", secondAuthorId) + .build(); + + context.commit(); + publication = reload(publication); + + assertThat(getMetadataValues(publication, "cris.virtual.department"), empty()); + assertThat(getMetadataValues(publication, "cris.virtualsource.department"), empty()); + + TestDSpaceRunnableHandler runnableHandler = runScript(false); + + assertThat(runnableHandler.getErrorMessages(), empty()); + assertThat(runnableHandler.getInfoMessages(), contains("Enhancement completed with success")); + + publication = reload(publication); + + assertThat(getMetadataValues(publication, "cris.virtual.department"), hasSize(2)); + assertThat(getMetadataValues(publication, "cris.virtualsource.department"), hasSize(2)); + + assertThat(publication.getMetadata(), hasItem(with("cris.virtual.department", + PLACEHOLDER_PARENT_METADATA_VALUE, 0))); + assertThat(publication.getMetadata(), hasItem(with("cris.virtualsource.department", + PLACEHOLDER_PARENT_METADATA_VALUE,0))); + assertThat(publication.getMetadata(), hasItem(with("cris.virtual.department", "4Science", 1))); + assertThat(publication.getMetadata(), hasItem(with("cris.virtualsource.department", secondAuthorId,1))); + + } + + @Test + public void testItemEnhancementWithoutAuthorities() throws Exception { + + context.turnOffAuthorisationSystem(); + + Item publication = ItemBuilder.createItem(context, collection) + .withTitle("Test publication 2 ") + .withEntityType("Publication") + .withAuthor("Jesse Pinkman") + .withAuthor("Jesse Smith") + .build(); + + context.commit(); + publication = reload(publication); + + assertThat(getMetadataValues(publication, "cris.virtual.department"), empty()); + assertThat(getMetadataValues(publication, "cris.virtualsource.department"), empty()); + + TestDSpaceRunnableHandler runnableHandler = runScript(false); + + assertThat(runnableHandler.getErrorMessages(), empty()); + assertThat(runnableHandler.getInfoMessages(), contains("Enhancement completed with success")); + + publication = reload(publication); + + assertThat(getMetadataValues(publication, "cris.virtual.department"), empty()); + assertThat(getMetadataValues(publication, "cris.virtualsource.department"), empty()); + + } + + private TestDSpaceRunnableHandler runScript(boolean force) throws InstantiationException, IllegalAccessException { + TestDSpaceRunnableHandler runnableHandler = new TestDSpaceRunnableHandler(); + String[] args = force ? new String[] { "item-enhancer-type", "-f" } : new String[] { "item-enhancer-type" }; + ScriptLauncher.handleScript(args, ScriptLauncher.getConfig(kernelImpl), runnableHandler, kernelImpl); + return runnableHandler; + } + + @SuppressWarnings("rawtypes") + private T reload(T entity) throws SQLException, AuthorizeException { + return context.reloadEntity(entity); + } + + private void replaceMetadata(Item item, String schema, String element, String qualifier, String newValue) + throws SQLException, AuthorizeException { + itemService.replaceMetadata(context, reload(item), schema, element, qualifier, ANY, newValue, null, -1, 0); + } + + private List getMetadataValues(Item item, String metadataField) { + return itemService.getMetadataByMetadataString(item, metadataField); + } + + private List getMetadataValues(WorkspaceItem item, String metadataField) { + return itemService.getMetadataByMetadataString(item.getItem(), metadataField); + } + +} diff --git a/dspace/config/spring/api/scripts.xml b/dspace/config/spring/api/scripts.xml index ec19026d0d79..4e4d7d8bb149 100644 --- a/dspace/config/spring/api/scripts.xml +++ b/dspace/config/spring/api/scripts.xml @@ -135,12 +135,12 @@ - + - + From fa1c3f0ea61acf359cb67a97c7ded376b2f1fe9a Mon Sep 17 00:00:00 2001 From: "Gantner, Florian Klaus" Date: Mon, 28 Aug 2023 10:42:26 +0200 Subject: [PATCH 10/19] fix wrong config property breaking tests --- dspace/config/local.cfg.EXAMPLE | 2 +- dspace/config/modules/rest.cfg | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/dspace/config/local.cfg.EXAMPLE b/dspace/config/local.cfg.EXAMPLE index 78f5bad26770..e0a1bed1fbb2 100644 --- a/dspace/config/local.cfg.EXAMPLE +++ b/dspace/config/local.cfg.EXAMPLE @@ -243,7 +243,7 @@ db.schema = public # avoid trouble for such browsers (i.e. rest.cors.allowed-origins = ${dspace.ui.url}, https://samltest.id ) #rest.cors.allowed-origins = ${dspace.ui.url} -#rest.cors.bitstream-allowed-origins = ${dspace.ui.url} +#rest.cors.bitstream-allow-origins = ${dspace.ui.url} ################################################# # SPRING BOOT SETTINGS (Used by Server Webapp) # diff --git a/dspace/config/modules/rest.cfg b/dspace/config/modules/rest.cfg index 9d2eb77be2cc..76abcb619528 100644 --- a/dspace/config/modules/rest.cfg +++ b/dspace/config/modules/rest.cfg @@ -134,7 +134,7 @@ rest.regex-clause = text_value ~ ? ##### Customize the REST origins allowed to retrieve the bitstreams ##### ##### default is set to pattern * - use this configuration to restrict/modify this behavior ##### This configuration doens't support the wildcard -bitstream.cors.allowed-origins = +rest.cors.bitstream-allow-origins = ##### Configure REST Report Filters ##### From 742ab6ab131f88232d6911669cf42a474923f0c2 Mon Sep 17 00:00:00 2001 From: "Gantner, Florian Klaus" Date: Wed, 30 Aug 2023 13:00:12 +0200 Subject: [PATCH 11/19] refactor item enhancement entity type script changes: do not iterate over all entities when some entity type is given, but over all collections with this type which improves performance. More checks on specified collection and entity types. --- .../script/ItemEnhancerEntityTypeScript.java | 231 +++++++++++------- ...EnhancerEntityTypeScriptConfiguration.java | 10 +- .../ItemEnhancerEntityTypeScriptIT.java | 17 +- 3 files changed, 161 insertions(+), 97 deletions(-) diff --git a/dspace-api/src/main/java/org/dspace/content/enhancer/script/ItemEnhancerEntityTypeScript.java b/dspace-api/src/main/java/org/dspace/content/enhancer/script/ItemEnhancerEntityTypeScript.java index e73f13811da1..05300a4cb03c 100644 --- a/dspace-api/src/main/java/org/dspace/content/enhancer/script/ItemEnhancerEntityTypeScript.java +++ b/dspace-api/src/main/java/org/dspace/content/enhancer/script/ItemEnhancerEntityTypeScript.java @@ -8,11 +8,12 @@ package org.dspace.content.enhancer.script; import java.sql.SQLException; -import java.util.Iterator; import java.util.Objects; import java.util.UUID; +import java.util.stream.Collectors; import org.apache.commons.cli.ParseException; +import org.apache.commons.lang3.StringUtils; import org.dspace.content.Collection; import org.dspace.content.Item; import org.dspace.content.enhancer.service.ItemEnhancerService; @@ -35,9 +36,13 @@ * calculated metadata with the enhancement. * * @author Luca Giamminonni (luca.giamminonni at 4science.it) - * Extended to limit the item set to collection/entity wide use to speed up process - * and recalculation of certain entities. - * Extended to use some pagination option with some limit where some commit to the database is made + * + * Extended to limit the item set to collection/entitytype to speed up process + * Extended to use pagination option with max/offset/limit options + * - max for testing purposes (process max x items per collection) + * - offset for pagination (start with item 0+offset per collection) + * - limit to make some intermediary commit between x items (recommended 100 steps) + * to make the process more error prone * * @author florian.gantner@uni-bamberg.de * @@ -107,13 +112,25 @@ public void internalRun() throws Exception { context = new Context(); assignCurrentUserInContext(); assignSpecialGroupsInContext(); - if (commandLine.hasOption('e') && Objects.isNull(entityTypeService.findByEntityType(context, entitytype))) { - throw new Exception("unknown entity " + entitytype); + if (Objects.nonNull(entitytype) && Objects.isNull(entityTypeService.findByEntityType(context, entitytype))) { + throw new Exception("unknown EntityType " + entitytype); + } + if (Objects.nonNull(entitytype) && StringUtils.isNotBlank(entitytype) && + this.collectionService.findAll(context).stream() + .noneMatch(col -> col.getEntityType().contentEquals(entitytype))) { + throw new Exception("no Collections with EntityType " + entitytype); + } + if (commandLine.hasOption('c') && Objects.isNull(collection)) { + throw new Exception("invalid uuid in the specified Collection"); + } + if (Objects.nonNull(collection) && (Objects.isNull(this.collectionService.find(context, collection)))) { + throw new Exception("specified Collection does not exist"); } - if (commandLine.hasOption('c') && (Objects.isNull(collection) - || Objects.isNull(this.collectionService.find(context, collection)))) { - throw new Exception("specified collection does not exist"); + if (Objects.nonNull(collection) && (Objects.nonNull(entitytype)) && + !this.collectionService.find(context, collection).getEntityType().contentEquals(entitytype)) { + throw new Exception("the specified Collection does not match with the specified EntityType"); } + context.turnOffAuthorisationSystem(); try { enhanceItems(); @@ -127,29 +144,28 @@ public void internalRun() throws Exception { } } - private void enhanceItems() { - - if (this.limit > 0) { - //use limit and max parameters for pagination. - findItemsToEnhanceLimitMax(); - } else { - //common way - if (Objects.nonNull(entitytype)) { - findItemsToEnhance().forEachRemaining(this::enhanceItemEntityCheck); - } else { - findItemsToEnhance().forEachRemaining(this::enhanceItem); + private void enhanceItems() throws SQLException { + if (Objects.nonNull(collection)) { + Collection coll = this.collectionService.find(context, collection); + findItemsToEnhance(coll); + } else if (Objects.nonNull(entitytype)) { + //for each colelction with entity type + for (Collection coll : collectionService.findAll(context).stream() + .filter(collection1 -> collection1.getEntityType().contentEquals(entitytype)).collect( + Collectors.toList())) { + findItemsToEnhance(coll); } + } else { + findItemsToEnhance(null); } } - private void findItemsToEnhanceLimitMax() { - Collection coll; + private void findItemsToEnhance(Collection coll) { int total = 0; int counter = 0; - if (Objects.nonNull(collection)) { - //Paginate through items in one collection + if (Objects.nonNull(coll)) { + //Paginate through items in one (given) collection try { - coll = collectionService.find(context, collection); total = itemService.countItems(context, coll); } catch (SQLException e) { handler.logError(e.getMessage()); @@ -162,31 +178,62 @@ private void findItemsToEnhanceLimitMax() { //offset is being added to counter and offset total += offset; counter += offset; - handler.logInfo("offset " + offset + " added. Range: [" - + counter + " to " + total + "] in " + limit + " steps"); - log.info("offset " + offset + " added. Range: [" - + counter + " to " + total + "] in " + limit + " steps"); + if (limit > 0) { + handler.logInfo("offset " + offset + " added. Range: [" + + counter + " to " + total + "] in " + limit + " steps"); + log.info("offset " + offset + " added. Range: [" + + counter + " to " + total + "] in " + limit + " steps"); + } else { + handler.logInfo("offset " + offset + " added. Range: [" + + counter + " to " + total + "]"); + log.info("offset " + offset + " added. Range: [" + + counter + " to " + total + "]"); + } } else { - handler.logInfo("Range: [" + counter + " to " - + total + "] in " + limit + " steps"); - log.info("Range: [" + counter + " to " + total + "] in " + limit + " steps"); + if (limit > 0) { + handler.logInfo("Range: [" + counter + " to " + + total + "] in " + limit + " steps"); + log.info("Range: [" + counter + " to " + total + "] in " + limit + " steps"); + } else { + handler.logInfo("Range: [" + counter + " to " + + total + "]"); + log.info("Range: [" + counter + " to " + total + "]"); + } } while (counter < total) { - try { - itemService.findAllByCollection(context, coll, limit, counter).forEachRemaining(this::enhanceItem); - counter += limit; - context.commit(); - handler.logInfo("processed " + counter + " out of total " + total + " items"); - log.info("processed " + counter + " out of total " + total + " items"); - } catch (SQLException e) { - handler.logError(e.getMessage()); - counter += limit; - handler.logInfo("processed " + counter + " out of total " + total + " items"); - log.info("processed " + counter + " out of total " + total + " items"); + if (limit > 0) { + try { + itemService.findAllByCollection(context, coll, limit, counter) + .forEachRemaining(this::enhanceItem); + counter += limit; + context.commit(); + handler.logInfo("processed " + counter + " out of total " + total + " items"); + log.info("processed " + counter + " out of total " + total + " items"); + } catch (SQLException e) { + handler.logError(e.getMessage()); + counter += limit; + handler.logInfo("processed " + counter + " out of total " + total + " items"); + log.info("processed " + counter + " out of total " + total + " items"); + } + } else { + try { + // no limit, so process all items in one commit + itemService.findAllByCollection(context, coll, total, counter) + .forEachRemaining(this::enhanceItem); + counter += total; + context.commit(); + handler.logInfo("processed " + counter + " out of total " + total + " items"); + log.info("processed " + counter + " out of total " + total + " items"); + } catch (SQLException e) { + handler.logError(e.getMessage()); + counter += limit; + handler.logInfo("processed " + counter + " out of total " + total + " items"); + log.info("processed " + counter + " out of total " + total + " items"); + } } } } else { - //loop through all! + // operate over all items try { total = itemService.countTotal(context); } catch (SQLException e) { @@ -200,51 +247,67 @@ private void findItemsToEnhanceLimitMax() { //offset is being added to counter and offset total += offset; counter += offset; - handler.logInfo("offset" + offset + " added. Range: [" - + counter + " to " + total + "] in " + limit + " steps"); - log.info("offset" + offset + " added. Range: [" - + counter + " to " + total + "] in " + limit + " steps"); + if (limit > 0) { + handler.logInfo("offset " + offset + " added. Range: [" + + counter + " to " + total + "] in " + limit + " steps"); + log.info("offset " + offset + " added. Range: [" + + counter + " to " + total + "] in " + limit + " steps"); + } else { + handler.logInfo("offset " + offset + " added. Range: [" + + counter + " to " + total + "]"); + log.info("offset " + offset + " added. Range: [" + + counter + " to " + total + "]"); + } } else { - handler.logInfo("Range: [" + counter + " to " + total + "] in " + limit + " steps"); - log.info("Range: [" + counter + " to " + total + "] in " + limit + " steps"); + if (limit > 0) { + handler.logInfo("Range: [" + counter + " to " + + total + "] in " + limit + " steps"); + log.info("Range: [" + counter + " to " + total + "] in " + limit + " steps"); + } else { + handler.logInfo("Range: [" + counter + " to " + + total + "]"); + log.info("Range: [" + counter + " to " + total + "]"); + } } while (counter < total) { - try { - //No Entity check here! - if (Objects.nonNull(this.entitytype)) { - itemService.findAll(context, limit, counter).forEachRemaining(this::enhanceItemEntityCheck); - } else { - itemService.findAll(context, limit, counter).forEachRemaining(this::enhanceItem); + if (limit > 0) { + try { + // Check for entity type in enhanceItem method + if (Objects.nonNull(this.entitytype)) { + itemService.findAll(context, limit, counter).forEachRemaining(this::enhanceItemEntityCheck); + } else { + itemService.findAll(context, limit, counter).forEachRemaining(this::enhanceItem); + } + counter += limit; + context.commit(); + handler.logInfo("processed " + counter + " out of total " + total + " items"); + log.info("processed " + counter + " out of total " + total + " items"); + } catch (SQLException e) { + handler.logError(e.getMessage()); + counter += limit; + handler.logInfo("processed " + counter + " out of total " + total + " items"); + log.info("processed " + counter + " out of total " + total + " items"); + } + } else { + try { + // Check for entity type in enhanceItem method + if (Objects.nonNull(this.entitytype)) { + itemService.findAll(context, total, counter).forEachRemaining(this::enhanceItemEntityCheck); + } else { + itemService.findAll(context, total, counter).forEachRemaining(this::enhanceItem); + } + counter += total; + context.commit(); + handler.logInfo("processed " + counter + " out of total " + total + " items"); + log.info("processed " + counter + " out of total " + total + " items"); + } catch (SQLException e) { + handler.logError(e.getMessage()); + counter += total; + handler.logInfo("processed " + counter + " out of total " + total + " items"); + log.info("processed " + counter + " out of total " + total + " items"); } - counter += limit; - context.commit(); - handler.logInfo("processed " + counter + " out of total " + total + " items"); - log.info("processed " + counter + " out of total " + total + " items"); - } catch (SQLException e) { - handler.logError(e.getMessage()); - counter += limit; - handler.logInfo("processed " + counter + " out of total " + total + " items"); - log.info("processed " + counter + " out of total " + total + " items"); - } - } - } - } - - private Iterator findItemsToEnhance() { - try { - Iterator result = null; - if (Objects.nonNull(collection)) { - //Check, if uuid exist - Collection coll = collectionService.find(context, collection); - if (coll != null) { - result = itemService.findAllByCollection(context, coll); } - } else { - result = itemService.findAll(context); } - return result; - } catch (SQLException e) { - throw new SQLRuntimeException(e); } } @@ -258,7 +321,7 @@ private void enhanceItem(Item item) { } /** - * Additional Entity Check. + * Additional Entity Check. Only applicable when operating over all entities */ private void enhanceItemEntityCheck(Item item) { if (Objects.nonNull(entitytype)) { diff --git a/dspace-api/src/main/java/org/dspace/content/enhancer/script/ItemEnhancerEntityTypeScriptConfiguration.java b/dspace-api/src/main/java/org/dspace/content/enhancer/script/ItemEnhancerEntityTypeScriptConfiguration.java index 3a48fa66d27d..0302a71d4e08 100644 --- a/dspace-api/src/main/java/org/dspace/content/enhancer/script/ItemEnhancerEntityTypeScriptConfiguration.java +++ b/dspace-api/src/main/java/org/dspace/content/enhancer/script/ItemEnhancerEntityTypeScriptConfiguration.java @@ -47,10 +47,12 @@ public Options getOptions() { options.addOption("c", "collection", true, "uuid of the collection. If the collection does not exist the script aborts."); options.addOption("e", "entity", true, - "Entity type of the items. Not recommended, because all items are loaded and then filtered."); - options.addOption("l", "limit", true, "size for iterator --limit items and commit after --limit items"); - options.addOption("m", "max", true, "process max --max items"); - options.addOption("o", "offset", true, "offset of items to start --offset items from the start"); + "Entity type of the items. Processes all collections with the specific entity type "); + options.addOption("l", "limit", true, + "size for iterator --limit items and commit after --limit items"); + options.addOption("m", "max", true, "process max --max items (per collection)"); + options.addOption("o", "offset", true, + "offset of items to start --offset items from the start (per collection)"); super.options = options; } diff --git a/dspace-api/src/test/java/org/dspace/content/enhancer/script/ItemEnhancerEntityTypeScriptIT.java b/dspace-api/src/test/java/org/dspace/content/enhancer/script/ItemEnhancerEntityTypeScriptIT.java index 805e9abb6138..b7659f8672fe 100644 --- a/dspace-api/src/test/java/org/dspace/content/enhancer/script/ItemEnhancerEntityTypeScriptIT.java +++ b/dspace-api/src/test/java/org/dspace/content/enhancer/script/ItemEnhancerEntityTypeScriptIT.java @@ -11,7 +11,6 @@ import static org.dspace.content.Item.ANY; import static org.dspace.core.CrisConstants.PLACEHOLDER_PARENT_METADATA_VALUE; import static org.hamcrest.MatcherAssert.assertThat; -import static org.hamcrest.Matchers.contains; import static org.hamcrest.Matchers.empty; import static org.hamcrest.Matchers.hasItem; import static org.hamcrest.Matchers.hasSize; @@ -152,7 +151,7 @@ public void testItemsEnhancement() throws Exception { TestDSpaceRunnableHandler runnableHandler = runScript(false); assertThat(runnableHandler.getErrorMessages(), empty()); - assertThat(runnableHandler.getInfoMessages(), contains("Enhancement completed with success")); + assertThat(runnableHandler.getInfoMessages(), hasItem("Enhancement completed with success")); firstPublication = reload(firstPublication); secondPublication = reload(secondPublication); @@ -210,7 +209,7 @@ public void testItemEnhancementWithoutForce() throws Exception { TestDSpaceRunnableHandler runnableHandler = runScript(false); assertThat(runnableHandler.getErrorMessages(), empty()); - assertThat(runnableHandler.getInfoMessages(), contains("Enhancement completed with success")); + assertThat(runnableHandler.getInfoMessages(), hasItem("Enhancement completed with success")); publication = reload(publication); @@ -233,7 +232,7 @@ public void testItemEnhancementWithoutForce() throws Exception { runnableHandler = runScript(false); assertThat(runnableHandler.getErrorMessages(), empty()); - assertThat(runnableHandler.getInfoMessages(), contains("Enhancement completed with success")); + assertThat(runnableHandler.getInfoMessages(), hasItem("Enhancement completed with success")); publication = reload(publication); @@ -280,7 +279,7 @@ public void testItemEnhancementWithForce() throws Exception { TestDSpaceRunnableHandler runnableHandler = runScript(false); assertThat(runnableHandler.getErrorMessages(), empty()); - assertThat(runnableHandler.getInfoMessages(), contains("Enhancement completed with success")); + assertThat(runnableHandler.getInfoMessages(), hasItem("Enhancement completed with success")); publication = reload(publication); @@ -303,7 +302,7 @@ public void testItemEnhancementWithForce() throws Exception { runnableHandler = runScript(true); assertThat(runnableHandler.getErrorMessages(), empty()); - assertThat(runnableHandler.getInfoMessages(), contains("Enhancement completed with success")); + assertThat(runnableHandler.getInfoMessages(), hasItem("Enhancement completed with success")); publication = reload(publication); @@ -362,7 +361,7 @@ public void testItemEnhancementMetadataPositions() throws Exception { TestDSpaceRunnableHandler runnableHandler = runScript(false); assertThat(runnableHandler.getErrorMessages(), empty()); - assertThat(runnableHandler.getInfoMessages(), contains("Enhancement completed with success")); + assertThat(runnableHandler.getInfoMessages(), hasItem("Enhancement completed with success")); publication = reload(publication); @@ -410,7 +409,7 @@ public void testItemEnhancementSourceWithoutAuthority() throws Exception { TestDSpaceRunnableHandler runnableHandler = runScript(false); assertThat(runnableHandler.getErrorMessages(), empty()); - assertThat(runnableHandler.getInfoMessages(), contains("Enhancement completed with success")); + assertThat(runnableHandler.getInfoMessages(), hasItem("Enhancement completed with success")); publication = reload(publication); @@ -447,7 +446,7 @@ public void testItemEnhancementWithoutAuthorities() throws Exception { TestDSpaceRunnableHandler runnableHandler = runScript(false); assertThat(runnableHandler.getErrorMessages(), empty()); - assertThat(runnableHandler.getInfoMessages(), contains("Enhancement completed with success")); + assertThat(runnableHandler.getInfoMessages(), hasItem("Enhancement completed with success")); publication = reload(publication); From cad9330dad83e8a54ea4a450e5946f3a979938f7 Mon Sep 17 00:00:00 2001 From: "Gantner, Florian Klaus" Date: Wed, 30 Aug 2023 13:01:06 +0200 Subject: [PATCH 12/19] refactor item enhancement by date script changes: fix some labels, comment and SearchCore Instantiation for proper tests --- .../script/ItemEnhancerByDateScript.java | 21 +++++++++++++------ .../script/ItemEnhancerByDateScriptIT.java | 17 +++++++-------- 2 files changed, 23 insertions(+), 15 deletions(-) diff --git a/dspace-api/src/main/java/org/dspace/content/enhancer/script/ItemEnhancerByDateScript.java b/dspace-api/src/main/java/org/dspace/content/enhancer/script/ItemEnhancerByDateScript.java index 54d4f9564b7e..dc88497ae140 100644 --- a/dspace-api/src/main/java/org/dspace/content/enhancer/script/ItemEnhancerByDateScript.java +++ b/dspace-api/src/main/java/org/dspace/content/enhancer/script/ItemEnhancerByDateScript.java @@ -34,6 +34,7 @@ import org.dspace.eperson.EPerson; import org.dspace.eperson.factory.EPersonServiceFactory; import org.dspace.scripts.DSpaceRunnable; +import org.dspace.services.factory.DSpaceServicesFactory; import org.dspace.util.UUIDUtils; import org.dspace.utils.DSpace; import org.slf4j.Logger; @@ -45,9 +46,15 @@ * * @author Luca Giamminonni (luca.giamminonni at 4science.it) - * Extended to solr search to discover not-enhanced entities. - * Some Query can be delivered, - * additional some daterange filterquery on the lastModified field + * This Script uses the solr search to discover the subset of entities being processed. + * This offers extended functionalities, e.g. enhance only items modified since or between + * timestamps etc... which cannot be expressed by the database on some easy way. + * - dateupper/datelower: filterquery for items between dates on the lastModified Date + * - entity: filterquery for entitytype (search.resourcetype) + * - collection: filterquery for collection (location.coll) + * - query: free hand search query, e.g. -cris.virtual.author:* . Best to use some criteria on already enhanced items + * - max: perform max items. Best for testing the entries. + * - limit: split result in smaller lists containint limit entries to avoid one big commit in the database * and additional collection/entitytype queries as filterfacets. * * @author florian.gantner@uni-bamberg.de @@ -90,7 +97,9 @@ public void setup() throws ParseException { this.collectionService = ContentServiceFactory.getInstance().getCollectionService(); this.entityTypeService = ContentServiceFactory.getInstance().getEntityTypeService(); itemEnhancerService = new DSpace().getSingletonService(ItemEnhancerService.class); - this.solrSearchCore = new DSpace().getSingletonService(SolrSearchCore.class); + this.solrSearchCore = + DSpaceServicesFactory.getInstance().getServiceManager().getServicesByType(SolrSearchCore.class).get(0); + //this.solrSearchCore = new DSpace().getSingletonService(SolrSearchCore.class); this.force = commandLine.hasOption('f'); if (commandLine.hasOption('c')) { @@ -130,11 +139,11 @@ public void internalRun() throws Exception { assignCurrentUserInContext(); assignSpecialGroupsInContext(); if (commandLine.hasOption('e') && Objects.isNull(entityTypeService.findByEntityType(context, entitytype))) { - throw new Exception("unknown entity " + entitytype); + throw new Exception("unknown EntityType " + entitytype); } if (commandLine.hasOption('c') && (Objects.isNull(collection) || Objects.isNull(this.collectionService.find(context, collection)))) { - throw new Exception("specified collection does not exist"); + throw new Exception("specified Collection does not exist"); } SolrPingResponse ping = solrSearchCore.getSolr().ping(); if (ping.getStatus() > 299) { diff --git a/dspace-api/src/test/java/org/dspace/content/enhancer/script/ItemEnhancerByDateScriptIT.java b/dspace-api/src/test/java/org/dspace/content/enhancer/script/ItemEnhancerByDateScriptIT.java index 0aacc1b8d63f..56f59115197d 100644 --- a/dspace-api/src/test/java/org/dspace/content/enhancer/script/ItemEnhancerByDateScriptIT.java +++ b/dspace-api/src/test/java/org/dspace/content/enhancer/script/ItemEnhancerByDateScriptIT.java @@ -11,7 +11,6 @@ import static org.dspace.content.Item.ANY; import static org.dspace.core.CrisConstants.PLACEHOLDER_PARENT_METADATA_VALUE; import static org.hamcrest.MatcherAssert.assertThat; -import static org.hamcrest.Matchers.contains; import static org.hamcrest.Matchers.empty; import static org.hamcrest.Matchers.hasItem; import static org.hamcrest.Matchers.hasSize; @@ -152,7 +151,7 @@ public void testItemsEnhancement() throws Exception { TestDSpaceRunnableHandler runnableHandler = runScript(false); assertThat(runnableHandler.getErrorMessages(), empty()); - assertThat(runnableHandler.getInfoMessages(), contains("Enhancement completed with success")); + assertThat(runnableHandler.getInfoMessages(), hasItem("Enhancement completed with success")); firstPublication = reload(firstPublication); secondPublication = reload(secondPublication); @@ -210,7 +209,7 @@ public void testItemEnhancementWithoutForce() throws Exception { TestDSpaceRunnableHandler runnableHandler = runScript(false); assertThat(runnableHandler.getErrorMessages(), empty()); - assertThat(runnableHandler.getInfoMessages(), contains("Enhancement completed with success")); + assertThat(runnableHandler.getInfoMessages(), hasItem("Enhancement completed with success")); publication = reload(publication); @@ -233,7 +232,7 @@ public void testItemEnhancementWithoutForce() throws Exception { runnableHandler = runScript(false); assertThat(runnableHandler.getErrorMessages(), empty()); - assertThat(runnableHandler.getInfoMessages(), contains("Enhancement completed with success")); + assertThat(runnableHandler.getInfoMessages(), hasItem("Enhancement completed with success")); publication = reload(publication); @@ -280,7 +279,7 @@ public void testItemEnhancementWithForce() throws Exception { TestDSpaceRunnableHandler runnableHandler = runScript(false); assertThat(runnableHandler.getErrorMessages(), empty()); - assertThat(runnableHandler.getInfoMessages(), contains("Enhancement completed with success")); + assertThat(runnableHandler.getInfoMessages(), hasItem("Enhancement completed with success")); publication = reload(publication); @@ -303,7 +302,7 @@ public void testItemEnhancementWithForce() throws Exception { runnableHandler = runScript(true); assertThat(runnableHandler.getErrorMessages(), empty()); - assertThat(runnableHandler.getInfoMessages(), contains("Enhancement completed with success")); + assertThat(runnableHandler.getInfoMessages(), hasItem("Enhancement completed with success")); publication = reload(publication); @@ -362,7 +361,7 @@ public void testItemEnhancementMetadataPositions() throws Exception { TestDSpaceRunnableHandler runnableHandler = runScript(false); assertThat(runnableHandler.getErrorMessages(), empty()); - assertThat(runnableHandler.getInfoMessages(), contains("Enhancement completed with success")); + assertThat(runnableHandler.getInfoMessages(), hasItem("Enhancement completed with success")); publication = reload(publication); @@ -410,7 +409,7 @@ public void testItemEnhancementSourceWithoutAuthority() throws Exception { TestDSpaceRunnableHandler runnableHandler = runScript(false); assertThat(runnableHandler.getErrorMessages(), empty()); - assertThat(runnableHandler.getInfoMessages(), contains("Enhancement completed with success")); + assertThat(runnableHandler.getInfoMessages(), hasItem("Enhancement completed with success")); publication = reload(publication); @@ -447,7 +446,7 @@ public void testItemEnhancementWithoutAuthorities() throws Exception { TestDSpaceRunnableHandler runnableHandler = runScript(false); assertThat(runnableHandler.getErrorMessages(), empty()); - assertThat(runnableHandler.getInfoMessages(), contains("Enhancement completed with success")); + assertThat(runnableHandler.getInfoMessages(), hasItem("Enhancement completed with success")); publication = reload(publication); From 81035478f3a7eaca3157b50ab4ef9b879b567356 Mon Sep 17 00:00:00 2001 From: Giuseppe Digilio Date: Fri, 9 Feb 2024 17:51:16 +0100 Subject: [PATCH 13/19] [maven-release-plugin] prepare release dspace-cris-2023.02.02 --- dspace-api/pom.xml | 2 +- dspace-iiif/pom.xml | 2 +- dspace-oai/pom.xml | 2 +- dspace-rdf/pom.xml | 2 +- dspace-rest/pom.xml | 4 ++-- dspace-server-webapp/pom.xml | 2 +- dspace-services/pom.xml | 2 +- dspace-sword/pom.xml | 2 +- dspace-swordv2/pom.xml | 2 +- dspace/modules/additions/pom.xml | 2 +- dspace/modules/pom.xml | 2 +- dspace/modules/rest/pom.xml | 2 +- dspace/modules/server-boot/pom.xml | 2 +- dspace/modules/server/pom.xml | 2 +- dspace/pom.xml | 2 +- pom.xml | 30 +++++++++++++++--------------- 16 files changed, 31 insertions(+), 31 deletions(-) diff --git a/dspace-api/pom.xml b/dspace-api/pom.xml index 1c6879a5d701..a7d1fa104edd 100644 --- a/dspace-api/pom.xml +++ b/dspace-api/pom.xml @@ -12,7 +12,7 @@ org.dspace dspace-parent - cris-2023.02.02-SNAPSHOT + cris-2023.02.02 .. diff --git a/dspace-iiif/pom.xml b/dspace-iiif/pom.xml index 3b56ba53e832..f6846b955fbb 100644 --- a/dspace-iiif/pom.xml +++ b/dspace-iiif/pom.xml @@ -15,7 +15,7 @@ org.dspace dspace-parent - cris-2023.02.02-SNAPSHOT + cris-2023.02.02 .. diff --git a/dspace-oai/pom.xml b/dspace-oai/pom.xml index 76718f44ba3c..a3ab33551c0d 100644 --- a/dspace-oai/pom.xml +++ b/dspace-oai/pom.xml @@ -8,7 +8,7 @@ dspace-parent org.dspace - cris-2023.02.02-SNAPSHOT + cris-2023.02.02 .. diff --git a/dspace-rdf/pom.xml b/dspace-rdf/pom.xml index eb63a67e4579..83af00bc343f 100644 --- a/dspace-rdf/pom.xml +++ b/dspace-rdf/pom.xml @@ -9,7 +9,7 @@ org.dspace dspace-parent - cris-2023.02.02-SNAPSHOT + cris-2023.02.02 .. diff --git a/dspace-rest/pom.xml b/dspace-rest/pom.xml index 257d0b3a91f8..a77c960283e7 100644 --- a/dspace-rest/pom.xml +++ b/dspace-rest/pom.xml @@ -3,7 +3,7 @@ org.dspace dspace-rest war - cris-2023.02.02-SNAPSHOT + cris-2023.02.02 DSpace (Deprecated) REST Webapp DSpace RESTful Web Services API. NOTE: this REST API is DEPRECATED. Please consider using the REST API in the dspace-server-webapp instead! @@ -12,7 +12,7 @@ org.dspace dspace-parent - cris-2023.02.02-SNAPSHOT + cris-2023.02.02 .. diff --git a/dspace-server-webapp/pom.xml b/dspace-server-webapp/pom.xml index c83aa538698f..a26174341efe 100644 --- a/dspace-server-webapp/pom.xml +++ b/dspace-server-webapp/pom.xml @@ -14,7 +14,7 @@ org.dspace dspace-parent - cris-2023.02.02-SNAPSHOT + cris-2023.02.02 .. diff --git a/dspace-services/pom.xml b/dspace-services/pom.xml index 39e2ccf36f38..8b67a90a8ea1 100644 --- a/dspace-services/pom.xml +++ b/dspace-services/pom.xml @@ -9,7 +9,7 @@ org.dspace dspace-parent - cris-2023.02.02-SNAPSHOT + cris-2023.02.02 .. diff --git a/dspace-sword/pom.xml b/dspace-sword/pom.xml index 10a44f3615f6..803eba859119 100644 --- a/dspace-sword/pom.xml +++ b/dspace-sword/pom.xml @@ -15,7 +15,7 @@ org.dspace dspace-parent - cris-2023.02.02-SNAPSHOT + cris-2023.02.02 .. diff --git a/dspace-swordv2/pom.xml b/dspace-swordv2/pom.xml index e76bfca65b9f..1c8103016e5d 100644 --- a/dspace-swordv2/pom.xml +++ b/dspace-swordv2/pom.xml @@ -13,7 +13,7 @@ org.dspace dspace-parent - cris-2023.02.02-SNAPSHOT + cris-2023.02.02 .. diff --git a/dspace/modules/additions/pom.xml b/dspace/modules/additions/pom.xml index f5ae804f8b8a..db2f30166a7d 100644 --- a/dspace/modules/additions/pom.xml +++ b/dspace/modules/additions/pom.xml @@ -17,7 +17,7 @@ org.dspace modules - cris-2023.02.02-SNAPSHOT + cris-2023.02.02 .. diff --git a/dspace/modules/pom.xml b/dspace/modules/pom.xml index 00f691235bc4..5a4426dddfd0 100644 --- a/dspace/modules/pom.xml +++ b/dspace/modules/pom.xml @@ -11,7 +11,7 @@ org.dspace dspace-parent - cris-2023.02.02-SNAPSHOT + cris-2023.02.02 ../../pom.xml diff --git a/dspace/modules/rest/pom.xml b/dspace/modules/rest/pom.xml index 4dfa2939bf90..8b80b4021668 100644 --- a/dspace/modules/rest/pom.xml +++ b/dspace/modules/rest/pom.xml @@ -13,7 +13,7 @@ org.dspace modules - cris-2023.02.02-SNAPSHOT + cris-2023.02.02 .. diff --git a/dspace/modules/server-boot/pom.xml b/dspace/modules/server-boot/pom.xml index ee23c8ee7bc7..90ceff73554a 100644 --- a/dspace/modules/server-boot/pom.xml +++ b/dspace/modules/server-boot/pom.xml @@ -11,7 +11,7 @@ modules org.dspace - cris-2023.02.02-SNAPSHOT + cris-2023.02.02 .. diff --git a/dspace/modules/server/pom.xml b/dspace/modules/server/pom.xml index 3797e809dca5..0d26cdb5fa11 100644 --- a/dspace/modules/server/pom.xml +++ b/dspace/modules/server/pom.xml @@ -7,7 +7,7 @@ modules org.dspace - cris-2023.02.02-SNAPSHOT + cris-2023.02.02 .. diff --git a/dspace/pom.xml b/dspace/pom.xml index 85b98dbb10d5..b8cd8ad6c379 100644 --- a/dspace/pom.xml +++ b/dspace/pom.xml @@ -16,7 +16,7 @@ org.dspace dspace-parent - cris-2023.02.02-SNAPSHOT + cris-2023.02.02 ../pom.xml diff --git a/pom.xml b/pom.xml index 8ec60b3a99df..c7b81d3eeb35 100644 --- a/pom.xml +++ b/pom.xml @@ -4,7 +4,7 @@ org.dspace dspace-parent pom - cris-2023.02.02-SNAPSHOT + cris-2023.02.02 DSpace Parent Project DSpace-CRIS is an open source extension of DSpace (http://www.dspace.org) providing out of box support for the CRIS / RIMS and moder Institution Repository use cases with advanced features and optimized configurations @@ -979,14 +979,14 @@ org.dspace dspace-rest - cris-2023.02.02-SNAPSHOT + cris-2023.02.02 jar classes org.dspace dspace-rest - cris-2023.02.02-SNAPSHOT + cris-2023.02.02 war @@ -1137,62 +1137,62 @@ org.dspace dspace-api - cris-2023.02.02-SNAPSHOT + cris-2023.02.02 org.dspace dspace-api test-jar - cris-2023.02.02-SNAPSHOT + cris-2023.02.02 test org.dspace.modules additions - cris-2023.02.02-SNAPSHOT + cris-2023.02.02 org.dspace dspace-sword - cris-2023.02.02-SNAPSHOT + cris-2023.02.02 org.dspace dspace-swordv2 - cris-2023.02.02-SNAPSHOT + cris-2023.02.02 org.dspace dspace-oai - cris-2023.02.02-SNAPSHOT + cris-2023.02.02 org.dspace dspace-services - cris-2023.02.02-SNAPSHOT + cris-2023.02.02 org.dspace dspace-server-webapp test-jar - cris-2023.02.02-SNAPSHOT + cris-2023.02.02 test org.dspace dspace-rdf - cris-2023.02.02-SNAPSHOT + cris-2023.02.02 org.dspace dspace-iiif - cris-2023.02.02-SNAPSHOT + cris-2023.02.02 org.dspace dspace-server-webapp - cris-2023.02.02-SNAPSHOT + cris-2023.02.02 @@ -2037,7 +2037,7 @@ scm:git:git@github.com:4Science/DSpace.git scm:git:git@github.com:4Science/DSpace.git git@github.com:4Science/DSpace.git - dspace-cris-2023.02.02-SNAPSHOT + dspace-cris-2023.02.02 From 2c100420a8c1c50e913ccb45880f311f977a9555 Mon Sep 17 00:00:00 2001 From: "Gantner, Florian Klaus" Date: Tue, 20 Feb 2024 15:19:05 +0100 Subject: [PATCH 14/19] align enhancer scripts to new deepMode force enhancement --- .../enhancer/script/ItemEnhancerByDateScript.java | 7 +------ .../ItemEnhancerByDateScriptConfiguration.java | 3 ++- .../script/ItemEnhancerEntityTypeScript.java | 12 ++---------- .../ItemEnhancerEntityTypeScriptConfiguration.java | 3 ++- 4 files changed, 7 insertions(+), 18 deletions(-) diff --git a/dspace-api/src/main/java/org/dspace/content/enhancer/script/ItemEnhancerByDateScript.java b/dspace-api/src/main/java/org/dspace/content/enhancer/script/ItemEnhancerByDateScript.java index dc88497ae140..7a44a10a577f 100644 --- a/dspace-api/src/main/java/org/dspace/content/enhancer/script/ItemEnhancerByDateScript.java +++ b/dspace-api/src/main/java/org/dspace/content/enhancer/script/ItemEnhancerByDateScript.java @@ -253,12 +253,7 @@ private SolrDocumentList searchItemsInSolr(String query, String datequeryupper, } private void enhanceItem(Item item) { - if (force) { - itemEnhancerService.forceEnhancement(context, item); - } else { - itemEnhancerService.enhance(context, item); - } - + itemEnhancerService.enhance(context, item, force); uncacheItem(item); } diff --git a/dspace-api/src/main/java/org/dspace/content/enhancer/script/ItemEnhancerByDateScriptConfiguration.java b/dspace-api/src/main/java/org/dspace/content/enhancer/script/ItemEnhancerByDateScriptConfiguration.java index f5d513000f67..e64814c4b16b 100644 --- a/dspace-api/src/main/java/org/dspace/content/enhancer/script/ItemEnhancerByDateScriptConfiguration.java +++ b/dspace-api/src/main/java/org/dspace/content/enhancer/script/ItemEnhancerByDateScriptConfiguration.java @@ -42,7 +42,8 @@ public Options getOptions() { if (options == null) { Options options = new Options(); - options.addOption("f", "force", false, "force the recalculation of all the virtual fields"); + options.addOption("f", "force", false, "force the usage of the deep mode" + + " (always compute the enhanced metadata to verify if the item need an update)"); options.addOption("c", "collection", true, "uuid of the collection. If the collection does not exist the script aborts."); options.addOption("e", "entity", true, "Entity type of the items"); diff --git a/dspace-api/src/main/java/org/dspace/content/enhancer/script/ItemEnhancerEntityTypeScript.java b/dspace-api/src/main/java/org/dspace/content/enhancer/script/ItemEnhancerEntityTypeScript.java index 05300a4cb03c..1a021fe3c154 100644 --- a/dspace-api/src/main/java/org/dspace/content/enhancer/script/ItemEnhancerEntityTypeScript.java +++ b/dspace-api/src/main/java/org/dspace/content/enhancer/script/ItemEnhancerEntityTypeScript.java @@ -312,11 +312,7 @@ private void findItemsToEnhance(Collection coll) { } private void enhanceItem(Item item) { - if (force) { - itemEnhancerService.forceEnhancement(context, item); - } else { - itemEnhancerService.enhance(context, item); - } + itemEnhancerService.enhance(context, item, force); uncacheItem(item); } @@ -329,11 +325,7 @@ private void enhanceItemEntityCheck(Item item) { uncacheItem(item); } } else { - if (force) { - itemEnhancerService.forceEnhancement(context, item); - } else { - itemEnhancerService.enhance(context, item); - } + itemEnhancerService.enhance(context, item, force); uncacheItem(item); } } diff --git a/dspace-api/src/main/java/org/dspace/content/enhancer/script/ItemEnhancerEntityTypeScriptConfiguration.java b/dspace-api/src/main/java/org/dspace/content/enhancer/script/ItemEnhancerEntityTypeScriptConfiguration.java index 0302a71d4e08..04451a08e9dc 100644 --- a/dspace-api/src/main/java/org/dspace/content/enhancer/script/ItemEnhancerEntityTypeScriptConfiguration.java +++ b/dspace-api/src/main/java/org/dspace/content/enhancer/script/ItemEnhancerEntityTypeScriptConfiguration.java @@ -43,7 +43,8 @@ public Options getOptions() { if (options == null) { Options options = new Options(); - options.addOption("f", "force", false, "force the recalculation of all the virtual fields"); + options.addOption("f", "force", false, "force the usage of the deep mode" + + " (always compute the enhanced metadata to verify if the item need an update)"); options.addOption("c", "collection", true, "uuid of the collection. If the collection does not exist the script aborts."); options.addOption("e", "entity", true, From 9c46991ac1d27f462743a669133ca4894f0a1fa7 Mon Sep 17 00:00:00 2001 From: "Gantner, Florian Klaus" Date: Tue, 20 Feb 2024 15:48:34 +0100 Subject: [PATCH 15/19] implement deepMode in NameEnhancer --- .../content/enhancer/impl/NameEnhancer.java | 56 ++++++++++++++----- 1 file changed, 41 insertions(+), 15 deletions(-) diff --git a/dspace-api/src/main/java/org/dspace/content/enhancer/impl/NameEnhancer.java b/dspace-api/src/main/java/org/dspace/content/enhancer/impl/NameEnhancer.java index 2a50d9fdb75e..3678c661a06b 100644 --- a/dspace-api/src/main/java/org/dspace/content/enhancer/impl/NameEnhancer.java +++ b/dspace-api/src/main/java/org/dspace/content/enhancer/impl/NameEnhancer.java @@ -56,27 +56,39 @@ public boolean canEnhance(Context context, Item item) { } @Override - public void enhance(Context context, Item item) { + public boolean enhance(Context context, Item item, boolean deepMode) { try { if (Objects.isNull(relatedItemMetadataFields) || relatedItemMetadataFields.isEmpty() || StringUtils.isBlank(targetItemMetadataField)) { - return; + return false; } - checkNames(context, item); + return checkNames(context, item, deepMode); } catch (Exception e) { LOGGER.error("An error occurs enhancing item with id {}: {}", item.getID(), e.getMessage(), e); - //Exception handling not supported by ItemEnhancerService. Thus just log to continue other enhancers + //Exception handling not supported by ItemEnhancerService. Thus, just log to continue other enhancers //throw new SQLRuntimeException(e); } + return false; } - private void checkNames(Context context, Item item) throws Exception { + /** + * Check the names/values from the specified metadatafield and compare the first value found + * with the target metadatafield. Updates the target metadatafield when the value is different + * or when deepMode is set. + * When the target metadatafield has not been set before, then set this value. + * @param context current Context + * @param item current item + * @param deepMode boolean + * @return boolean value if some change/update has happened + * @throws Exception when some error occurs + */ + private boolean checkNames(Context context, Item item, boolean deepMode) throws Exception { // ignore languages of Metadata here. Assume main title is not repeated // Could be more simplified List currentnames = itemService.getMetadataByMetadataString(item, targetItemMetadataField); if (!currentnames.isEmpty()) { - // some name assigned yet + // some name is assigned yet for (MetadataValue currentname : currentnames) { String val = currentname.getValue(); for (String field : relatedItemMetadataFields) { @@ -89,19 +101,22 @@ private void checkNames(Context context, Item item) throws Exception { if (StringUtils.isNotBlank(fieldname.getValue()) && fieldname.getValue().contentEquals(val)) { //Values are the same. No Update necessary - return; + if (deepMode) { + // value is recalculated in deepMode + return updateTargetMetadata(context, item, fieldname.getValue(), true); + } + return false; } else { //values differ. We must update the value - updateTargetMetadata(context, item, fieldname.getValue(), true); - return; + return updateTargetMetadata(context, item, fieldname.getValue(), true); } } } } if (StringUtils.isNotBlank(defaultValue) && !currentnames.get(0).getValue().contentEquals(defaultValue)) { - // None of the names above matches. Set Default-Value, if exist. Otherwise do nothing - updateTargetMetadata(context, item, defaultValue, true); + // None of the names above matches. Set Default-Value, if it exists. Otherwise, do nothing + return updateTargetMetadata(context, item, defaultValue, true); } } else { @@ -115,20 +130,29 @@ private void checkNames(Context context, Item item) throws Exception { for (MetadataValue fieldname : fieldnames) { if (StringUtils.isNotBlank(fieldname.getValue())) { //Got some value - updateTargetMetadata(context, item, fieldname.getValue(), false); - return; + return updateTargetMetadata(context, item, fieldname.getValue(), false); } } } // If no name exist, set defaultvalue if (StringUtils.isNotBlank(defaultValue)) { - updateTargetMetadata(context, item, defaultValue, false); + return updateTargetMetadata(context, item, defaultValue, false); } // otherwise do not assign any value } + return false; } - private void updateTargetMetadata(Context context, Item item, String value, boolean clear) throws SQLException { + /** + * Update/Set the target metadata with option to clear/delete previous metadatavalues + * @param context current Context + * @param item item to set metadatavalue + * @param value value to set + * @param clear clear/delete existing values byfore + * @return boolean value if the value has been updated successfully + * @throws SQLException when some error occurs + */ + private boolean updateTargetMetadata(Context context, Item item, String value, boolean clear) throws SQLException { MetadataField targetmd = metadatafieldService.findByString(context, targetItemMetadataField, '.'); if (targetmd != null) { if (clear) { @@ -136,9 +160,11 @@ private void updateTargetMetadata(Context context, Item item, String value, bool targetmd.getQualifier(), Item.ANY); } itemService.addMetadata(context, item, targetmd, null, value); + return true; } else { LOGGER.error("No valid metadatavalue to enhance specified"); } + return false; } public void setSourceEntityType(String sourceEntityType) { From 5ab97e40dd8367bdd2a0a48092696b4652ebb300 Mon Sep 17 00:00:00 2001 From: "Gantner, Florian Klaus" Date: Tue, 20 Feb 2024 15:48:52 +0100 Subject: [PATCH 16/19] implement deepMode in MapConverterValueEnhancer --- .../enhancer/impl/MapConverterValueEnhancer.java | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/dspace-api/src/main/java/org/dspace/content/enhancer/impl/MapConverterValueEnhancer.java b/dspace-api/src/main/java/org/dspace/content/enhancer/impl/MapConverterValueEnhancer.java index b27a5e772fe0..73c6f66a13ab 100644 --- a/dspace-api/src/main/java/org/dspace/content/enhancer/impl/MapConverterValueEnhancer.java +++ b/dspace-api/src/main/java/org/dspace/content/enhancer/impl/MapConverterValueEnhancer.java @@ -61,11 +61,11 @@ public boolean canEnhance(Context context, Item item) { } @Override - public void enhance(Context context, Item item) { + public boolean enhance(Context context, Item item, boolean deepMode) { try { if (StringUtils.isBlank(sourceItemMetadataField) || Objects.isNull(converter) || StringUtils.isBlank(targetItemMetadataField)) { - return; + return false; } String sourceval; String targetval; @@ -78,18 +78,28 @@ public void enhance(Context context, Item item) { // replace mdv if it's different removeTargetMetadata(context, item); addTargetMetadata(context, item, calculatedval); + return true; } else if (StringUtils.isBlank(targetval)) { // set new value addTargetMetadata(context, item, calculatedval); + return true; + } else if (StringUtils.isNotBlank(sourceval) && StringUtils.isNotBlank(targetval) + && sourceval.contentEquals(targetval) && deepMode) { + //When both values are equal and deepMode is active, recalculate the value + removeTargetMetadata(context, item); + addTargetMetadata(context, item, calculatedval); + return true; } } else if (StringUtils.isBlank(sourceval) && StringUtils.isNotBlank(targetval)) { // remove value removeTargetMetadata(context, item); + return true; } } catch (Exception e) { LOGGER.error("An error occurs enhancing item with id {}: {}", item.getID(), e.getMessage(), e); //throw new SQLRuntimeException(e); } + return false; } private void addTargetMetadata(Context context, Item item, String value) throws Exception { MetadataField targetmd = metadatafieldService.findByString(context, targetItemMetadataField, '.'); From 81aa4a49127a87a926a88664bae6fbc494ae96be Mon Sep 17 00:00:00 2001 From: "Gantner, Florian Klaus" Date: Thu, 22 Feb 2024 15:18:43 +0100 Subject: [PATCH 17/19] align enhancer integration tests --- .../script/ItemEnhancerByDateScriptIT.java | 20 +++++++++++------- .../ItemEnhancerEntityTypeScriptIT.java | 21 ++++++++++++------- 2 files changed, 26 insertions(+), 15 deletions(-) diff --git a/dspace-api/src/test/java/org/dspace/content/enhancer/script/ItemEnhancerByDateScriptIT.java b/dspace-api/src/test/java/org/dspace/content/enhancer/script/ItemEnhancerByDateScriptIT.java index 56f59115197d..7ec65d794508 100644 --- a/dspace-api/src/test/java/org/dspace/content/enhancer/script/ItemEnhancerByDateScriptIT.java +++ b/dspace-api/src/test/java/org/dspace/content/enhancer/script/ItemEnhancerByDateScriptIT.java @@ -9,6 +9,7 @@ import static org.dspace.app.matcher.MetadataValueMatcher.with; import static org.dspace.content.Item.ANY; +import static org.dspace.content.enhancer.consumer.ItemEnhancerConsumer.ITEMENHANCER_ENABLED; import static org.dspace.core.CrisConstants.PLACEHOLDER_PARENT_METADATA_VALUE; import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.Matchers.empty; @@ -47,6 +48,10 @@ public class ItemEnhancerByDateScriptIT extends AbstractIntegrationTestWithDatabase { + private static ConfigurationService configService = DSpaceServicesFactory.getInstance().getConfigurationService(); + + private static final EventService eventService = EventServiceFactory.getInstance().getEventService(); + private static boolean isEnabled; private static String[] consumers; private ItemService itemService; @@ -60,13 +65,13 @@ public class ItemEnhancerByDateScriptIT extends AbstractIntegrationTestWithDatab */ @BeforeClass public static void initConsumers() { - ConfigurationService configService = DSpaceServicesFactory.getInstance().getConfigurationService(); consumers = configService.getArrayProperty("event.dispatcher.default.consumers"); Set consumersSet = new HashSet(Arrays.asList(consumers)); - consumersSet.remove("itemenhancer"); - configService.setProperty("event.dispatcher.default.consumers", consumersSet.toArray()); - EventService eventService = EventServiceFactory.getInstance().getEventService(); - eventService.reloadConfiguration(); + if (!consumersSet.contains("itemenhancer")) { + consumersSet.add("itemenhancer"); + configService.setProperty("event.dispatcher.default.consumers", consumersSet.toArray()); + eventService.reloadConfiguration(); + } } /** @@ -74,18 +79,19 @@ public static void initConsumers() { */ @AfterClass public static void resetDefaultConsumers() { - ConfigurationService configService = DSpaceServicesFactory.getInstance().getConfigurationService(); configService.setProperty("event.dispatcher.default.consumers", consumers); - EventService eventService = EventServiceFactory.getInstance().getEventService(); eventService.reloadConfiguration(); } @Before public void setup() { + configService.setProperty(ITEMENHANCER_ENABLED, false); + itemService = ContentServiceFactory.getInstance().getItemService(); context.turnOffAuthorisationSystem(); + parentCommunity = CommunityBuilder.createCommunity(context) .withName("Parent Community") .build(); diff --git a/dspace-api/src/test/java/org/dspace/content/enhancer/script/ItemEnhancerEntityTypeScriptIT.java b/dspace-api/src/test/java/org/dspace/content/enhancer/script/ItemEnhancerEntityTypeScriptIT.java index b7659f8672fe..e45124137d88 100644 --- a/dspace-api/src/test/java/org/dspace/content/enhancer/script/ItemEnhancerEntityTypeScriptIT.java +++ b/dspace-api/src/test/java/org/dspace/content/enhancer/script/ItemEnhancerEntityTypeScriptIT.java @@ -9,6 +9,7 @@ import static org.dspace.app.matcher.MetadataValueMatcher.with; import static org.dspace.content.Item.ANY; +import static org.dspace.content.enhancer.consumer.ItemEnhancerConsumer.ITEMENHANCER_ENABLED; import static org.dspace.core.CrisConstants.PLACEHOLDER_PARENT_METADATA_VALUE; import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.Matchers.empty; @@ -47,8 +48,11 @@ public class ItemEnhancerEntityTypeScriptIT extends AbstractIntegrationTestWithDatabase { + private static final ConfigurationService configService = + DSpaceServicesFactory.getInstance().getConfigurationService(); + private static final EventService eventService = EventServiceFactory.getInstance().getEventService(); private static String[] consumers; - + private static boolean isEnabled; private ItemService itemService; private Collection collection; @@ -60,13 +64,13 @@ public class ItemEnhancerEntityTypeScriptIT extends AbstractIntegrationTestWithD */ @BeforeClass public static void initConsumers() { - ConfigurationService configService = DSpaceServicesFactory.getInstance().getConfigurationService(); consumers = configService.getArrayProperty("event.dispatcher.default.consumers"); Set consumersSet = new HashSet(Arrays.asList(consumers)); - consumersSet.remove("itemenhancer"); - configService.setProperty("event.dispatcher.default.consumers", consumersSet.toArray()); - EventService eventService = EventServiceFactory.getInstance().getEventService(); - eventService.reloadConfiguration(); + if (!consumersSet.contains("itemenhancer")) { + consumersSet.add("itemenhancer"); + configService.setProperty("event.dispatcher.default.consumers", consumersSet.toArray()); + eventService.reloadConfiguration(); + } } /** @@ -74,18 +78,19 @@ public static void initConsumers() { */ @AfterClass public static void resetDefaultConsumers() { - ConfigurationService configService = DSpaceServicesFactory.getInstance().getConfigurationService(); configService.setProperty("event.dispatcher.default.consumers", consumers); - EventService eventService = EventServiceFactory.getInstance().getEventService(); eventService.reloadConfiguration(); } @Before public void setup() { + configService.setProperty(ITEMENHANCER_ENABLED, false); + itemService = ContentServiceFactory.getInstance().getItemService(); context.turnOffAuthorisationSystem(); + parentCommunity = CommunityBuilder.createCommunity(context) .withName("Parent Community") .build(); From dfa2a9196cb6b1fef9d36c334e3010c2cd27eb69 Mon Sep 17 00:00:00 2001 From: "Gantner, Florian Klaus" Date: Fri, 23 Feb 2024 14:30:48 +0100 Subject: [PATCH 18/19] integration tests for enhancer test entitytype, collection, limit and max and solr query parameter on the enhancement scripts. lower progress handler infos to debug --- .../script/ItemEnhancerByDateScript.java | 67 ++-- .../script/ItemEnhancerEntityTypeScript.java | 153 ++++---- .../script/ItemEnhancerByDateScriptIT.java | 302 +++++++++++++++ .../ItemEnhancerEntityTypeScriptIT.java | 346 ++++++++++++++++++ 4 files changed, 783 insertions(+), 85 deletions(-) diff --git a/dspace-api/src/main/java/org/dspace/content/enhancer/script/ItemEnhancerByDateScript.java b/dspace-api/src/main/java/org/dspace/content/enhancer/script/ItemEnhancerByDateScript.java index 7a44a10a577f..258884a02e3d 100644 --- a/dspace-api/src/main/java/org/dspace/content/enhancer/script/ItemEnhancerByDateScript.java +++ b/dspace-api/src/main/java/org/dspace/content/enhancer/script/ItemEnhancerByDateScript.java @@ -86,6 +86,10 @@ public class ItemEnhancerByDateScript private int limit; + private int counter = 0; + + private int countertotal = 0; + private EntityTypeService entityTypeService; private static final Logger log = LoggerFactory.getLogger(ItemEnhancerByDateScript.class); @@ -99,36 +103,35 @@ public void setup() throws ParseException { itemEnhancerService = new DSpace().getSingletonService(ItemEnhancerService.class); this.solrSearchCore = DSpaceServicesFactory.getInstance().getServiceManager().getServicesByType(SolrSearchCore.class).get(0); - //this.solrSearchCore = new DSpace().getSingletonService(SolrSearchCore.class); this.force = commandLine.hasOption('f'); if (commandLine.hasOption('c')) { - this.collection = UUIDUtils.fromString(commandLine.getOptionValue('c')); + this.collection = UUIDUtils.fromString(commandLine.getOptionValue('c').trim()); } if (commandLine.hasOption('e')) { - this.entitytype = commandLine.getOptionValue('e'); + this.entitytype = commandLine.getOptionValue('e').trim(); } if (commandLine.hasOption('q')) { - this.query = commandLine.getOptionValue('q'); + this.query = commandLine.getOptionValue('q').trim(); } if (commandLine.hasOption('d')) { - this.dateupper = commandLine.getOptionValue('d'); + this.dateupper = commandLine.getOptionValue('d').trim(); } if (commandLine.hasOption('s')) { - this.datelower = commandLine.getOptionValue('s'); + this.datelower = commandLine.getOptionValue('s').trim(); } if (commandLine.hasOption('m')) { try { - this.max = Integer.parseInt(commandLine.getOptionValue('m')); + this.max = Integer.parseInt(commandLine.getOptionValue('m').trim()); } catch (Exception e) { - // + handler.logError(e.getMessage()); } } if (commandLine.hasOption('l')) { try { - this.limit = Integer.parseInt(commandLine.getOptionValue('l')); + this.limit = Integer.parseInt(commandLine.getOptionValue('l').trim()); } catch (Exception e) { - // + handler.logError(e.getMessage()); } } } @@ -165,7 +168,8 @@ public void internalRun() throws Exception { private void searchItems() { - + int maximum = 0; //maximum items to be processed + int total = 0; //results of search/query List items = new ArrayList<>(); try { SolrDocumentList results = searchItemsInSolr(this.query, this.dateupper, this.datelower); @@ -179,43 +183,59 @@ private void searchItems() { handler.logError(e.getMessage(), e); log.error(e.getMessage()); } - int total = items.size(); + total = items.size(); if (total == 0) { handler.logInfo("No results in solr-Query"); log.info("No results in solr-Query"); return; + } else if (this.max > 0) { + maximum = this.max; + if (this.max < items.size()) { + items = items.subList(0, (this.max - 1)); + total = this.max - 1; + } } + // split list and commit after limit entries if (this.limit > 0) { - // Split through every list - int counter = 0; + if (limit > total) { + limit = total; + } + // counting variables for pagination + int tempcounter = 0; int start = 0; int end = 0; - while (counter < total) { - start = counter; - end = counter + limit; - if (end > (total - 1)) { - end = total - 1; + while (tempcounter < total) { + start = tempcounter; + end = tempcounter + limit; + if (end > total) { + end = total; + limit = total - tempcounter; } try { this.itemService.findByIds(context, items.subList(start, end)).forEachRemaining(this::enhanceItem); + tempcounter += limit; context.commit(); - counter += limit; + handler.logInfo("enhanced " + tempcounter + " out of max " + maximum + " items"); + log.info("enhanced " + tempcounter + " out of max " + maximum + " items"); } catch (Exception e) { + tempcounter += limit; handler.logError(e.getMessage()); - counter += limit; + handler.logInfo("enhanced " + tempcounter + " out of max " + maximum + " items"); + log.info("enhanced " + tempcounter + " out of max " + maximum + " items"); } } } else { + // enhance all found items try { this.itemService.findByIds(context, items).forEachRemaining(this::enhanceItem); } catch (SQLException e) { handler.logError(e.getMessage()); } } - handler.logInfo("enhanced " + total + " items"); - log.info("enhanced " + total + " items"); + handler.logInfo("enhanced " + counter + " items"); + log.info("enhanced " + counter + " items"); } private SolrDocumentList searchItemsInSolr(String query, String datequeryupper, String datequerylower) @@ -253,6 +273,7 @@ private SolrDocumentList searchItemsInSolr(String query, String datequeryupper, } private void enhanceItem(Item item) { + counter++; itemEnhancerService.enhance(context, item, force); uncacheItem(item); } diff --git a/dspace-api/src/main/java/org/dspace/content/enhancer/script/ItemEnhancerEntityTypeScript.java b/dspace-api/src/main/java/org/dspace/content/enhancer/script/ItemEnhancerEntityTypeScript.java index 1a021fe3c154..7add8b2541d6 100644 --- a/dspace-api/src/main/java/org/dspace/content/enhancer/script/ItemEnhancerEntityTypeScript.java +++ b/dspace-api/src/main/java/org/dspace/content/enhancer/script/ItemEnhancerEntityTypeScript.java @@ -65,6 +65,8 @@ public class ItemEnhancerEntityTypeScript private int offset; + private int counter; + private EntityTypeService entityTypeService; private static final Logger log = LoggerFactory.getLogger(ItemEnhancerEntityTypeScript.class); @@ -79,30 +81,30 @@ public void setup() throws ParseException { this.force = commandLine.hasOption('f'); if (commandLine.hasOption('c')) { - this.collection = UUIDUtils.fromString(commandLine.getOptionValue('c')); + this.collection = UUIDUtils.fromString(commandLine.getOptionValue('c').trim()); } if (commandLine.hasOption('e')) { - this.entitytype = commandLine.getOptionValue('e'); + this.entitytype = commandLine.getOptionValue('e').trim(); } if (commandLine.hasOption('l')) { try { - this.limit = Integer.parseInt(commandLine.getOptionValue('l')); + this.limit = Integer.parseInt(commandLine.getOptionValue('l').trim()); } catch (Exception e) { - // + handler.logError(e.getMessage()); } } if (commandLine.hasOption('m')) { try { - this.max = Integer.parseInt(commandLine.getOptionValue('m')); + this.max = Integer.parseInt(commandLine.getOptionValue('m').trim()); } catch (Exception e) { - // + handler.logError(e.getMessage()); } } if (commandLine.hasOption('o')) { try { - this.offset = Integer.parseInt(commandLine.getOptionValue('o')); + this.offset = Integer.parseInt(commandLine.getOptionValue('o').trim()); } catch (Exception e) { - // + handler.logError(e.getMessage()); } } } @@ -149,7 +151,7 @@ private void enhanceItems() throws SQLException { Collection coll = this.collectionService.find(context, collection); findItemsToEnhance(coll); } else if (Objects.nonNull(entitytype)) { - //for each colelction with entity type + //for each collection with entity type for (Collection coll : collectionService.findAll(context).stream() .filter(collection1 -> collection1.getEntityType().contentEquals(entitytype)).collect( Collectors.toList())) { @@ -160,9 +162,13 @@ private void enhanceItems() throws SQLException { } } + /** + * enhance the items in this collection with the given numeric restrictions + * @param coll + */ private void findItemsToEnhance(Collection coll) { int total = 0; - int counter = 0; + int maximal = 0; if (Objects.nonNull(coll)) { //Paginate through items in one (given) collection try { @@ -171,64 +177,72 @@ private void findItemsToEnhance(Collection coll) { handler.logError(e.getMessage()); return; } - if ( this.max > 0) { + if (this.max > 0) { total = this.max; + maximal = this.max; } if (this.offset > 0) { //offset is being added to counter and offset total += offset; - counter += offset; if (limit > 0) { - handler.logInfo("offset " + offset + " added. Range: [" + handler.logDebug("offset " + offset + " added. Range: [" + counter + " to " + total + "] in " + limit + " steps"); - log.info("offset " + offset + " added. Range: [" + log.debug("offset " + offset + " added. Range: [" + counter + " to " + total + "] in " + limit + " steps"); } else { - handler.logInfo("offset " + offset + " added. Range: [" + handler.logDebug("offset " + offset + " added. Range: [" + counter + " to " + total + "]"); - log.info("offset " + offset + " added. Range: [" + log.debug("offset " + offset + " added. Range: [" + counter + " to " + total + "]"); } } else { if (limit > 0) { - handler.logInfo("Range: [" + counter + " to " + handler.logDebug("Range: [" + counter + " to " + total + "] in " + limit + " steps"); - log.info("Range: [" + counter + " to " + total + "] in " + limit + " steps"); + log.debug("Range: [" + counter + " to " + total + "] in " + limit + " steps"); } else { - handler.logInfo("Range: [" + counter + " to " + handler.logDebug("Range: [" + counter + " to " + total + "]"); - log.info("Range: [" + counter + " to " + total + "]"); + log.debug("Range: [" + counter + " to " + total + "]"); } } - while (counter < total) { + int tempcounter = 0; + if (limit > total) { + limit = total; + } + while (tempcounter < total) { if (limit > 0) { try { - itemService.findAllByCollection(context, coll, limit, counter) + itemService.findAllByCollection(context, coll, limit, tempcounter) .forEachRemaining(this::enhanceItem); - counter += limit; context.commit(); - handler.logInfo("processed " + counter + " out of total " + total + " items"); - log.info("processed " + counter + " out of total " + total + " items"); + handler.logInfo("enhanced " + counter + " out of max " + maximal + " items"); + log.info("enhanced " + counter + " out of max " + maximal + " items"); + + tempcounter += limit; } catch (SQLException e) { handler.logError(e.getMessage()); - counter += limit; - handler.logInfo("processed " + counter + " out of total " + total + " items"); - log.info("processed " + counter + " out of total " + total + " items"); + handler.logError("enhanced " + counter + " out of max " + maximal + " items"); + log.error("enhanced " + counter + " out of max " + maximal + " items"); + + tempcounter += limit; } } else { try { // no limit, so process all items in one commit - itemService.findAllByCollection(context, coll, total, counter) + itemService.findAllByCollection(context, coll, total, 0) .forEachRemaining(this::enhanceItem); - counter += total; context.commit(); - handler.logInfo("processed " + counter + " out of total " + total + " items"); - log.info("processed " + counter + " out of total " + total + " items"); + handler.logInfo("enhanced " + counter + " out of max " + maximal + " items"); + log.info("enhanced " + counter + " out of max " + maximal + " items"); + + tempcounter += total; } catch (SQLException e) { handler.logError(e.getMessage()); - counter += limit; - handler.logInfo("processed " + counter + " out of total " + total + " items"); - log.info("processed " + counter + " out of total " + total + " items"); + handler.logError("enhanced " + counter + " out of max " + maximal + " items"); + log.error("enhanced " + counter + " out of max " + maximal + " items"); + + tempcounter += total; } } } @@ -240,78 +254,90 @@ private void findItemsToEnhance(Collection coll) { handler.logError(e.getMessage()); return; } + if (this.max > 0) { total = this.max; + maximal = this.max; } if (this.offset > 0) { //offset is being added to counter and offset total += offset; - counter += offset; if (limit > 0) { - handler.logInfo("offset " + offset + " added. Range: [" + handler.logDebug("offset " + offset + " added. Range: [" + counter + " to " + total + "] in " + limit + " steps"); - log.info("offset " + offset + " added. Range: [" + log.debug("offset " + offset + " added. Range: [" + counter + " to " + total + "] in " + limit + " steps"); } else { - handler.logInfo("offset " + offset + " added. Range: [" + handler.logDebug("offset " + offset + " added. Range: [" + counter + " to " + total + "]"); - log.info("offset " + offset + " added. Range: [" + log.debug("offset " + offset + " added. Range: [" + counter + " to " + total + "]"); } } else { if (limit > 0) { - handler.logInfo("Range: [" + counter + " to " + handler.logDebug("Range: [" + counter + " to " + total + "] in " + limit + " steps"); - log.info("Range: [" + counter + " to " + total + "] in " + limit + " steps"); + log.debug("Range: [" + counter + " to " + total + "] in " + limit + " steps"); } else { - handler.logInfo("Range: [" + counter + " to " + handler.logDebug("Range: [" + counter + " to " + total + "]"); - log.info("Range: [" + counter + " to " + total + "]"); + log.debug("Range: [" + counter + " to " + total + "]"); } } - while (counter < total) { + //Counting variables for pagination + int tempcounter = 0; + while (tempcounter < total) { if (limit > 0) { try { // Check for entity type in enhanceItem method if (Objects.nonNull(this.entitytype)) { - itemService.findAll(context, limit, counter).forEachRemaining(this::enhanceItemEntityCheck); + itemService.findAll(context, limit, tempcounter) + .forEachRemaining(this::enhanceItemEntityCheck); } else { - itemService.findAll(context, limit, counter).forEachRemaining(this::enhanceItem); + itemService.findAll(context, limit, tempcounter).forEachRemaining(this::enhanceItem); } - counter += limit; context.commit(); - handler.logInfo("processed " + counter + " out of total " + total + " items"); - log.info("processed " + counter + " out of total " + total + " items"); + handler.logInfo("enhanced " + counter + " out of max " + maximal + " items"); + log.info("enhanced " + counter + " out of max " + maximal + " items"); + + tempcounter += limit; } catch (SQLException e) { handler.logError(e.getMessage()); - counter += limit; - handler.logInfo("processed " + counter + " out of total " + total + " items"); - log.info("processed " + counter + " out of total " + total + " items"); + handler.logError("enhanced " + counter + " out of max " + maximal + " items"); + log.error("enhanced " + counter + " out of max " + maximal + " items"); + + tempcounter += limit; } } else { try { // Check for entity type in enhanceItem method if (Objects.nonNull(this.entitytype)) { - itemService.findAll(context, total, counter).forEachRemaining(this::enhanceItemEntityCheck); + itemService.findAll(context, total, 0).forEachRemaining(this::enhanceItemEntityCheck); } else { - itemService.findAll(context, total, counter).forEachRemaining(this::enhanceItem); + itemService.findAll(context, total, 0).forEachRemaining(this::enhanceItem); } - counter += total; context.commit(); - handler.logInfo("processed " + counter + " out of total " + total + " items"); - log.info("processed " + counter + " out of total " + total + " items"); + handler.logInfo("enhanced " + counter + " out of max " + maximal + " items"); + log.info("enhanced " + counter + " out of max " + maximal + " items"); + + tempcounter += total; } catch (SQLException e) { + counter++; handler.logError(e.getMessage()); - counter += total; - handler.logInfo("processed " + counter + " out of total " + total + " items"); - log.info("processed " + counter + " out of total " + total + " items"); + handler.logError("enhanced " + counter + " out of max " + maximal + " items"); + log.error("enhanced " + counter + " out of max " + maximal + " items"); + + tempcounter += total; } } } } + handler.logInfo("enhanced " + counter + " items"); + log.info("enhanced " + counter + " items"); } private void enhanceItem(Item item) { + counter++; itemEnhancerService.enhance(context, item, force); uncacheItem(item); } @@ -321,10 +347,13 @@ private void enhanceItem(Item item) { */ private void enhanceItemEntityCheck(Item item) { if (Objects.nonNull(entitytype)) { - if (!entitytype.contentEquals(itemService.getEntityType(item))) { + if (entitytype.contentEquals(itemService.getEntityType(item))) { + counter++; + itemEnhancerService.enhance(context, item, force); uncacheItem(item); } } else { + counter++; itemEnhancerService.enhance(context, item, force); uncacheItem(item); } diff --git a/dspace-api/src/test/java/org/dspace/content/enhancer/script/ItemEnhancerByDateScriptIT.java b/dspace-api/src/test/java/org/dspace/content/enhancer/script/ItemEnhancerByDateScriptIT.java index 7ec65d794508..f6bd1ceb3a65 100644 --- a/dspace-api/src/test/java/org/dspace/content/enhancer/script/ItemEnhancerByDateScriptIT.java +++ b/dspace-api/src/test/java/org/dspace/content/enhancer/script/ItemEnhancerByDateScriptIT.java @@ -12,16 +12,22 @@ import static org.dspace.content.enhancer.consumer.ItemEnhancerConsumer.ITEMENHANCER_ENABLED; import static org.dspace.core.CrisConstants.PLACEHOLDER_PARENT_METADATA_VALUE; import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.empty; +import static org.hamcrest.Matchers.equalToIgnoringCase; import static org.hamcrest.Matchers.hasItem; +import static org.hamcrest.Matchers.hasItems; import static org.hamcrest.Matchers.hasSize; +import static org.hamcrest.Matchers.notNullValue; import java.sql.SQLException; +import java.util.ArrayList; import java.util.Arrays; import java.util.HashSet; import java.util.List; import java.util.Set; +import org.apache.commons.lang.StringUtils; import org.dspace.AbstractIntegrationTestWithDatabase; import org.dspace.app.launcher.ScriptLauncher; import org.dspace.app.scripts.handler.impl.TestDSpaceRunnableHandler; @@ -461,6 +467,272 @@ public void testItemEnhancementWithoutAuthorities() throws Exception { } + @Test + public void testItemEnhancementEntityTypeInvalidCollectionUUID() throws Exception { + + context.turnOffAuthorisationSystem(); + + Item publication = ItemBuilder.createItem(context, collection) + .withTitle("Test publication 2 ") + .withEntityType("Publication") + .withAuthor("Jesse Pinkman") + .withAuthor("Jesse Smith") + .build(); + + context.commit(); + publication = reload(publication); + + TestDSpaceRunnableHandler runnableHandler = + runScript(false, publication.getID().toString(), null, "*", null, null); + + assertThat(runnableHandler.getException(), notNullValue()); + assertThat(runnableHandler.getException().getMessage(), + equalToIgnoringCase("specified Collection does not exist")); + } + + @Test + public void testItemEnhancementEntityTypeAbortWhenInvalidSolrQuery() throws Exception { + + context.turnOffAuthorisationSystem(); + context.commit(); + String query = "(test : info"; + + TestDSpaceRunnableHandler runnableHandler = + runScript(false, null, null, query, null, null); + + assertThat(runnableHandler.getErrorMessages(), hasItems()); + assertThat(runnableHandler.getErrorMessages(), hasItem(containsString("ParseException"))); + assertThat(runnableHandler.getErrorMessages(), + hasItem("An error occurs during enhancement. The process is aborted")); + } + + @Test + public void testItemEnhancementEntityTypeAbortWhenEmptySolrQuery() throws Exception { + + context.turnOffAuthorisationSystem(); + + ItemBuilder.createItem(context, collection) + .withTitle("Test publication 2 ") + .withEntityType("Publication") + .withAuthor("Jesse Pinkman") + .withAuthor("Jesse Smith") + .build(); + + context.commit(); + String query = "neverexistingdsolrindexfield:true"; + + TestDSpaceRunnableHandler runnableHandler = + runScript(false, null, null, query, null, null); + + assertThat(runnableHandler.getInfoMessages(), hasItems()); + assertThat(runnableHandler.getInfoMessages(), hasItem("No results in solr-Query")); + } + + @Test + public void testItemEnhancementEntityTypeInvalidEntityType() throws Exception { + + context.turnOffAuthorisationSystem(); + context.commit(); + + TestDSpaceRunnableHandler runnableHandler + = runScript(false, null, "ResearchData", "*", null, null); + + assertThat(runnableHandler.getException(), notNullValue()); + assertThat(runnableHandler.getException().getMessage(), equalToIgnoringCase("unknown EntityType ResearchData")); + } + + @Test + public void testItemEnhancementEntityTypeMaxMatches() throws Exception { + + context.turnOffAuthorisationSystem(); + + ItemBuilder.createItem(context, collection) + .withTitle("Test publication") + .withEntityType("Publication") + .withAuthor("Jesse Pinkman") + .withAuthor("Jesse Smith") + .build(); + + ItemBuilder.createItem(context, collection) + .withTitle("Test publication 2 ") + .withEntityType("Publication") + .withAuthor("Jesse Pinkman") + .withAuthor("Jesse Smith") + .build(); + + ItemBuilder.createItem(context, collection) + .withTitle("Test publication 3") + .withEntityType("Publication") + .withAuthor("Jesse Pinkman") + .withAuthor("Jesse Smith") + .build(); + + context.commit(); + String max = "2"; + + TestDSpaceRunnableHandler runnableHandler + = runScript(false, collection.getID().toString(), null, "*", null, max); + + assertThat(runnableHandler.getInfoMessages(), hasItems()); + assertThat(runnableHandler.getInfoMessages(), hasItem("enhanced " + max + " items")); + } + + @Test + public void testItemEnhancementEntityTypeLimitAndMaxMatches() throws Exception { + + context.turnOffAuthorisationSystem(); + + ItemBuilder.createItem(context, collection) + .withTitle("Test publication") + .withEntityType("Publication") + .withAuthor("Jesse Pinkman") + .withAuthor("Jesse Smith") + .build(); + + ItemBuilder.createItem(context, collection) + .withTitle("Test publication 2 ") + .withEntityType("Publication") + .withAuthor("Jesse Pinkman") + .withAuthor("Jesse Smith") + .build(); + + ItemBuilder.createItem(context, collection) + .withTitle("Test publication 3") + .withEntityType("Publication") + .withAuthor("Jesse Pinkman") + .withAuthor("Jesse Smith") + .build(); + + ItemBuilder.createItem(context, collection) + .withTitle("Test publication 4") + .withEntityType("Publication") + .withAuthor("Jesse Pinkman") + .withAuthor("Jesse Smith") + .build(); + + ItemBuilder.createItem(context, collection) + .withTitle("Test publication 5") + .withEntityType("Publication") + .withAuthor("Jesse Pinkman") + .withAuthor("Jesse Smith") + .build(); + + context.commit(); + + String limit = "2"; + String max = "3"; + + TestDSpaceRunnableHandler runnableHandler + = runScript(false, collection.getID().toString(), null, "*", limit, max); + + assertThat(runnableHandler.getInfoMessages(), hasItems()); + assertThat(runnableHandler.getInfoMessages(), hasItem("enhanced 2 out of max " + max + " items")); + assertThat(runnableHandler.getInfoMessages(), hasItem("enhanced 3 out of max " + max + " items")); + assertThat(runnableHandler.getInfoMessages(), hasItem("enhanced " + max + " items")); + } + + @Test + public void testItemEnhancementEntityTypeLimitAndMaxMatches2() throws Exception { + + context.turnOffAuthorisationSystem(); + + ItemBuilder.createItem(context, collection) + .withTitle("Test publication") + .withEntityType("Publication") + .withAuthor("Jesse Pinkman") + .withAuthor("Jesse Smith") + .build(); + + ItemBuilder.createItem(context, collection) + .withTitle("Test publication 2 ") + .withEntityType("Publication") + .withAuthor("Jesse Pinkman") + .withAuthor("Jesse Smith") + .build(); + + ItemBuilder.createItem(context, collection) + .withTitle("Test publication 3") + .withEntityType("Publication") + .withAuthor("Jesse Pinkman") + .withAuthor("Jesse Smith") + .build(); + + ItemBuilder.createItem(context, collection) + .withTitle("Test publication 4") + .withEntityType("Publication") + .withAuthor("Jesse Pinkman") + .withAuthor("Jesse Smith") + .build(); + + ItemBuilder.createItem(context, collection) + .withTitle("Test publication 5") + .withEntityType("Publication") + .withAuthor("Jesse Pinkman") + .withAuthor("Jesse Smith") + .build(); + + context.commit(); + + String limit = "2"; + String max = "3"; + + TestDSpaceRunnableHandler runnableHandler + = runScript(false, collection.getID().toString(), null, "*", limit, max); + + assertThat(runnableHandler.getInfoMessages(), hasItems()); + assertThat(runnableHandler.getInfoMessages(), hasItem("enhanced 2 out of max " + max + " items")); + assertThat(runnableHandler.getInfoMessages(), hasItem("enhanced 3 out of max " + max + " items")); + assertThat(runnableHandler.getInfoMessages(), hasItem("enhanced " + max + " items")); + } + + @Test + public void testItemEnhancementEntityTypeLimitAndMaxMatches3() throws Exception { + + context.turnOffAuthorisationSystem(); + + ItemBuilder.createItem(context, collection) + .withTitle("Test publication") + .withEntityType("Publication") + .withAuthor("Jesse Pinkman") + .withAuthor("Jesse Smith") + .build(); + + ItemBuilder.createItem(context, collection) + .withTitle("Test publication 2 ") + .withEntityType("Publication") + .withAuthor("Jesse Pinkman") + .withAuthor("Jesse Smith") + .build(); + + ItemBuilder.createItem(context, collection) + .withTitle("Test publication 3") + .withEntityType("Publication") + .withAuthor("Jesse Pinkman") + .withAuthor("Jesse Smith") + .build(); + + ItemBuilder.createItem(context, collection) + .withTitle("Test publication 4") + .withEntityType("Publication") + .withAuthor("Jesse Pinkman") + .withAuthor("Jesse Smith") + .build(); + + context.commit(); + + String limit = "2"; + String max = "5"; + + TestDSpaceRunnableHandler runnableHandler + = runScript(false, collection.getID().toString(), null, "*", limit, max); + + assertThat(runnableHandler.getInfoMessages(), hasItems()); + assertThat(runnableHandler.getInfoMessages(), hasItem("enhanced 2 out of max " + max + " items")); + assertThat(runnableHandler.getInfoMessages(), hasItem("enhanced 4 out of max " + max + " items")); + assertThat(runnableHandler.getInfoMessages(), hasItem("enhanced 4 items")); + } + + private TestDSpaceRunnableHandler runScript(boolean force) throws InstantiationException, IllegalAccessException { TestDSpaceRunnableHandler runnableHandler = new TestDSpaceRunnableHandler(); String[] args = force ? new String[] { "item-enhancer-date", "-f" } : new String[] { "item-enhancer-date" }; @@ -468,6 +740,36 @@ private TestDSpaceRunnableHandler runScript(boolean force) throws InstantiationE return runnableHandler; } + private TestDSpaceRunnableHandler runScript(boolean force, String collectionuuid, String entitytype, String query, + String limit, String max) + throws InstantiationException, IllegalAccessException { + TestDSpaceRunnableHandler runnableHandler = new TestDSpaceRunnableHandler(); + List argslist = new ArrayList<>(); + argslist.add("item-enhancer-date"); + if (force) { + argslist.add("-f"); + } + if (StringUtils.isNotBlank(collectionuuid)) { + argslist.add("-c " + collectionuuid); + } + if (StringUtils.isNotBlank(entitytype)) { + argslist.add("-e " + entitytype); + } + if (StringUtils.isNotBlank(query)) { + argslist.add("-q " + query); + } + if (StringUtils.isNotBlank(limit)) { + argslist.add("-l " + limit); + } + if (StringUtils.isNotBlank(max)) { + argslist.add("-m " + max); + } + String[] args = argslist.toArray(new String[0]); + + ScriptLauncher.handleScript(args, ScriptLauncher.getConfig(kernelImpl), runnableHandler, kernelImpl); + return runnableHandler; + } + @SuppressWarnings("rawtypes") private T reload(T entity) throws SQLException, AuthorizeException { return context.reloadEntity(entity); diff --git a/dspace-api/src/test/java/org/dspace/content/enhancer/script/ItemEnhancerEntityTypeScriptIT.java b/dspace-api/src/test/java/org/dspace/content/enhancer/script/ItemEnhancerEntityTypeScriptIT.java index e45124137d88..1475f97b2875 100644 --- a/dspace-api/src/test/java/org/dspace/content/enhancer/script/ItemEnhancerEntityTypeScriptIT.java +++ b/dspace-api/src/test/java/org/dspace/content/enhancer/script/ItemEnhancerEntityTypeScriptIT.java @@ -13,24 +13,31 @@ import static org.dspace.core.CrisConstants.PLACEHOLDER_PARENT_METADATA_VALUE; import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.Matchers.empty; +import static org.hamcrest.Matchers.equalToIgnoringCase; import static org.hamcrest.Matchers.hasItem; +import static org.hamcrest.Matchers.hasItems; import static org.hamcrest.Matchers.hasSize; +import static org.hamcrest.Matchers.notNullValue; import java.sql.SQLException; +import java.util.ArrayList; import java.util.Arrays; import java.util.HashSet; import java.util.List; import java.util.Set; +import org.apache.commons.lang.StringUtils; import org.dspace.AbstractIntegrationTestWithDatabase; import org.dspace.app.launcher.ScriptLauncher; import org.dspace.app.scripts.handler.impl.TestDSpaceRunnableHandler; import org.dspace.authorize.AuthorizeException; import org.dspace.builder.CollectionBuilder; import org.dspace.builder.CommunityBuilder; +import org.dspace.builder.EntityTypeBuilder; import org.dspace.builder.ItemBuilder; import org.dspace.builder.WorkspaceItemBuilder; import org.dspace.content.Collection; +import org.dspace.content.EntityType; import org.dspace.content.Item; import org.dspace.content.MetadataValue; import org.dspace.content.WorkspaceItem; @@ -57,6 +64,14 @@ public class ItemEnhancerEntityTypeScriptIT extends AbstractIntegrationTestWithD private Collection collection; + private Collection publications; + + private Collection publications2; + + private Collection publications3; + + private Collection persons; + /** * This method will be run before the first test as per @BeforeClass. It will * configure the event.dispatcher.default.consumers property to remove the @@ -98,6 +113,27 @@ public void setup() { collection = CollectionBuilder.createCollection(context, parentCommunity) .withName("Collection") .build(); + + publications = CollectionBuilder.createCollection(context, parentCommunity) + .withName("Collection Publication") + .withEntityType("Publication") + .build(); + + publications2 = CollectionBuilder.createCollection(context, parentCommunity) + .withName("Collection Publication2") + .withEntityType("Publication") + .build(); + + publications3 = CollectionBuilder.createCollection(context, parentCommunity) + .withName("Collection Publication3") + .withEntityType("Publication") + .build(); + + persons = CollectionBuilder.createCollection(context, parentCommunity) + .withName("Collection Person") + .withEntityType("Person") + .build(); + context.restoreAuthSystemState(); } @@ -460,6 +496,290 @@ public void testItemEnhancementWithoutAuthorities() throws Exception { } + @Test + public void testItemEnhancementEntityTypeInvalidCollectionUUID() throws Exception { + + context.turnOffAuthorisationSystem(); + + Item publication = ItemBuilder.createItem(context, collection) + .withTitle("Test publication 2 ") + .withEntityType("Publication") + .withAuthor("Jesse Pinkman") + .withAuthor("Jesse Smith") + .build(); + + context.commit(); + publication = reload(publication); + + TestDSpaceRunnableHandler runnableHandler = runScript(false, publication.getID().toString(), null, null, null); + + assertThat(runnableHandler.getException(), notNullValue()); + assertThat(runnableHandler.getException().getMessage(), + equalToIgnoringCase("specified Collection does not exist")); + } + + @Test + public void testItemEnhancementEntityTypeNoCollection() throws Exception { + context.turnOffAuthorisationSystem(); + EntityType projectType = EntityTypeBuilder.createEntityTypeBuilder(context, "Project").build(); + context.restoreAuthSystemState(); + + TestDSpaceRunnableHandler runnableHandler = runScript(false, null, projectType.getLabel(), null, null); + + assertThat(runnableHandler.getException(), notNullValue()); + assertThat(runnableHandler.getException().getMessage(), + equalToIgnoringCase("no Collections with EntityType " + projectType.getLabel())); + } + + @Test + public void testItemEnhancementEntityTypeInvalidForCollection() throws Exception { + context.turnOffAuthorisationSystem(); + EntityType personType = EntityTypeBuilder.createEntityTypeBuilder(context, "Person").build(); + context.commit(); + context.restoreAuthSystemState(); + + TestDSpaceRunnableHandler runnableHandler = runScript(false, publications.getID().toString(), + personType.getLabel(), null, null); + + assertThat(runnableHandler.getException(), notNullValue()); + assertThat(runnableHandler.getException().getMessage(), + equalToIgnoringCase("the specified Collection does not match with the specified EntityType")); + } + + @Test + public void testItemEnhancementEntityTypeInvalidEntityType() throws Exception { + + context.turnOffAuthorisationSystem(); + context.commit(); + + TestDSpaceRunnableHandler runnableHandler = runScript(false, null, "ResearchData", null, null); + + assertThat(runnableHandler.getException(), notNullValue()); + assertThat(runnableHandler.getException().getMessage(), equalToIgnoringCase("unknown EntityType ResearchData")); + } + + @Test + public void testItemEnhancementEntityTypeMaxMatches() throws Exception { + + context.turnOffAuthorisationSystem(); + + ItemBuilder.createItem(context, collection) + .withTitle("Test publication") + .withEntityType("Publication") + .withAuthor("Jesse Pinkman") + .withAuthor("Jesse Smith") + .build(); + + ItemBuilder.createItem(context, collection) + .withTitle("Test publication 2 ") + .withEntityType("Publication") + .withAuthor("Jesse Pinkman") + .withAuthor("Jesse Smith") + .build(); + + ItemBuilder.createItem(context, collection) + .withTitle("Test publication 3") + .withEntityType("Publication") + .withAuthor("Jesse Pinkman") + .withAuthor("Jesse Smith") + .build(); + + context.commit(); + String max = "2"; + + TestDSpaceRunnableHandler runnableHandler + = runScript(false, collection.getID().toString(), null, null, max); + + assertThat(runnableHandler.getInfoMessages(), hasItems()); + assertThat(runnableHandler.getInfoMessages(), hasItem("enhanced " + max + " items")); + assertThat(runnableHandler.getInfoMessages(), hasItem("Enhancement completed with success")); + } + + @Test + public void testItemEnhancementEntityTypeLimitAndMaxMatches() throws Exception { + + context.turnOffAuthorisationSystem(); + + ItemBuilder.createItem(context, collection) + .withTitle("Test publication") + .withEntityType("Publication") + .withAuthor("Jesse Pinkman") + .withAuthor("Jesse Smith") + .build(); + + ItemBuilder.createItem(context, collection) + .withTitle("Test publication 2 ") + .withEntityType("Publication") + .withAuthor("Jesse Pinkman") + .withAuthor("Jesse Smith") + .build(); + + ItemBuilder.createItem(context, collection) + .withTitle("Test publication 3") + .withEntityType("Publication") + .withAuthor("Jesse Pinkman") + .withAuthor("Jesse Smith") + .build(); + + ItemBuilder.createItem(context, collection) + .withTitle("Test publication 4") + .withEntityType("Publication") + .withAuthor("Jesse Pinkman") + .withAuthor("Jesse Smith") + .build(); + + ItemBuilder.createItem(context, collection) + .withTitle("Test publication 5") + .withEntityType("Publication") + .withAuthor("Jesse Pinkman") + .withAuthor("Jesse Smith") + .build(); + + context.commit(); + + String limit = "2"; + String max = "4"; //smaller than collection size + + TestDSpaceRunnableHandler runnableHandler + = runScript(false, collection.getID().toString(), null, limit, max); + + assertThat(runnableHandler.getInfoMessages(), hasItems()); + assertThat(runnableHandler.getInfoMessages(), hasItem("enhanced 2 out of max " + max + " items")); + assertThat(runnableHandler.getInfoMessages(), hasItem("enhanced 4 out of max " + max + " items")); + assertThat(runnableHandler.getInfoMessages(), hasItem("enhanced " + max + " items")); + assertThat(runnableHandler.getInfoMessages(), hasItem("Enhancement completed with success")); + } + + @Test + public void testItemEnhancementEntityTypeLimitAndMaxMatches2() throws Exception { + + context.turnOffAuthorisationSystem(); + + List list = new ArrayList<>(); + + list.add(ItemBuilder.createItem(context, collection) + .withTitle("Test publication") + .withEntityType("Publication") + .withAuthor("Jesse Pinkman") + .withAuthor("Jesse Smith") + .build()); + + list.add(ItemBuilder.createItem(context, collection) + .withTitle("Test publication 2 ") + .withEntityType("Publication") + .withAuthor("Jesse Pinkman") + .withAuthor("Jesse Smith") + .build()); + + list.add(ItemBuilder.createItem(context, collection) + .withTitle("Test publication 3") + .withEntityType("Publication") + .withAuthor("Jesse Pinkman") + .withAuthor("Jesse Smith") + .build()); + + list.add(ItemBuilder.createItem(context, collection) + .withTitle("Test publication 4") + .withEntityType("Publication") + .withAuthor("Jesse Pinkman") + .withAuthor("Jesse Smith") + .build()); + + list.add(ItemBuilder.createItem(context, collection) + .withTitle("Test publication 5") + .withEntityType("Publication") + .withAuthor("Jesse Pinkman") + .withAuthor("Jesse Smith") + .build()); + + context.commit(); + + String limit = "3"; + String max = "7"; + + TestDSpaceRunnableHandler runnableHandler + = runScript(false, collection.getID().toString(), null, limit, max); + + assertThat(runnableHandler.getInfoMessages(), hasItems()); + assertThat(runnableHandler.getInfoMessages(), hasItem("enhanced 3 out of max " + max + " items")); + assertThat(runnableHandler.getInfoMessages(), hasItem("enhanced 5 out of max " + max + " items")); + assertThat(runnableHandler.getInfoMessages(), hasItem("enhanced " + list.size() + " items")); + assertThat(runnableHandler.getInfoMessages(), hasItem("Enhancement completed with success")); + } + + @Test + public void testItemEnhancementEntityTypeMaxMatchesPerCollectionsAndEntityType() throws Exception { + + context.turnOffAuthorisationSystem(); + EntityType publicationType = EntityTypeBuilder.createEntityTypeBuilder(context, "Publication").build(); + + List list = new ArrayList<>(); + + list.add(ItemBuilder.createItem(context, publications) + .withTitle("Test publication") + .withEntityType(publicationType.getLabel()) + .withAuthor("Jesse Pinkman") + .withAuthor("Jesse Smith") + .build()); + + list.add(ItemBuilder.createItem(context, publications2) + .withTitle("Test publication 2 ") + .withEntityType(publicationType.getLabel()) + .withAuthor("Jesse Pinkman") + .withAuthor("Jesse Smith") + .build()); + + list.add(ItemBuilder.createItem(context, publications2) + .withTitle("Test publication 3") + .withEntityType(publicationType.getLabel()) + .withAuthor("Jesse Pinkman") + .withAuthor("Jesse Smith") + .build()); + + list.add(ItemBuilder.createItem(context, publications2) + .withTitle("Test publication 4 ") + .withEntityType(publicationType.getLabel()) + .withAuthor("Jesse Pinkman") + .withAuthor("Jesse Smith") + .build()); + + list.add(ItemBuilder.createItem(context, publications3) + .withTitle("Test publication 5") + .withEntityType(publicationType.getLabel()) + .withAuthor("Jesse Pinkman") + .withAuthor("Jesse Smith") + .build()); + + list.add(ItemBuilder.createItem(context, publications3) + .withTitle("Test publication 6") + .withEntityType(publicationType.getLabel()) + .withAuthor("Jesse Pinkman") + .withAuthor("Jesse Smith") + .build()); + + list.add(ItemBuilder.createItem(context, publications3) + .withTitle("Test publication 7") + .withEntityType(publicationType.getLabel()) + .withAuthor("Jesse Pinkman") + .withAuthor("Jesse Smith") + .build()); + + context.commit(); + + // Max 2 per collection + String max = "2"; + + TestDSpaceRunnableHandler runnableHandler + = runScript(false, null, "Publication", null, max); + + // every collection max two items + assertThat(runnableHandler.getInfoMessages(), hasItems()); + assertThat(runnableHandler.getInfoMessages(), hasItem("enhanced 1 out of max 2 items")); + assertThat(runnableHandler.getInfoMessages(), hasItem("enhanced 3 out of max 2 items")); + assertThat(runnableHandler.getInfoMessages(), hasItem("enhanced 5 items")); + assertThat(runnableHandler.getInfoMessages(), hasItem("Enhancement completed with success")); + } + private TestDSpaceRunnableHandler runScript(boolean force) throws InstantiationException, IllegalAccessException { TestDSpaceRunnableHandler runnableHandler = new TestDSpaceRunnableHandler(); String[] args = force ? new String[] { "item-enhancer-type", "-f" } : new String[] { "item-enhancer-type" }; @@ -467,6 +787,32 @@ private TestDSpaceRunnableHandler runScript(boolean force) throws InstantiationE return runnableHandler; } + private TestDSpaceRunnableHandler runScript(boolean force, String collectionuuid, String entitytype, String limit, + String max) throws InstantiationException, IllegalAccessException { + TestDSpaceRunnableHandler runnableHandler = new TestDSpaceRunnableHandler(); + List argslist = new ArrayList<>(); + argslist.add("item-enhancer-type"); + if (force) { + argslist.add("-f"); + } + if (StringUtils.isNotBlank(collectionuuid)) { + argslist.add("-c " + collectionuuid); + } + if (StringUtils.isNotBlank(entitytype)) { + argslist.add("-e " + entitytype); + } + if (StringUtils.isNotBlank(limit)) { + argslist.add("-l " + limit); + } + if (StringUtils.isNotBlank(max)) { + argslist.add("-m " + max); + } + String[] args = argslist.toArray(new String[0]); + + ScriptLauncher.handleScript(args, ScriptLauncher.getConfig(kernelImpl), runnableHandler, kernelImpl); + return runnableHandler; + } + @SuppressWarnings("rawtypes") private T reload(T entity) throws SQLException, AuthorizeException { return context.reloadEntity(entity); From 8600d1be41fe31f11b532436ef302e79ee2e277e Mon Sep 17 00:00:00 2001 From: "Gantner, Florian Klaus" Date: Wed, 28 Feb 2024 18:02:40 +0100 Subject: [PATCH 19/19] integration test for nameenhancer provide simple integration test for the nameenhancer using common cris metadatafields --- .../script/ItemEnhancerByDateScriptIT.java | 72 +++++++++++++++++++ .../ItemEnhancerEntityTypeScriptIT.java | 64 +++++++++++++++++ .../enhancer/script/ItemEnhancerScriptIT.java | 71 ++++++++++++++++++ .../config/spring/api/metadata-enhancers.xml | 5 +- 4 files changed, 210 insertions(+), 2 deletions(-) diff --git a/dspace-api/src/test/java/org/dspace/content/enhancer/script/ItemEnhancerByDateScriptIT.java b/dspace-api/src/test/java/org/dspace/content/enhancer/script/ItemEnhancerByDateScriptIT.java index f6bd1ceb3a65..4faead266941 100644 --- a/dspace-api/src/test/java/org/dspace/content/enhancer/script/ItemEnhancerByDateScriptIT.java +++ b/dspace-api/src/test/java/org/dspace/content/enhancer/script/ItemEnhancerByDateScriptIT.java @@ -64,6 +64,8 @@ public class ItemEnhancerByDateScriptIT extends AbstractIntegrationTestWithDatab private Collection collection; + private Collection persons; + /** * This method will be run before the first test as per @BeforeClass. It will * configure the event.dispatcher.default.consumers property to remove the @@ -105,6 +107,12 @@ public void setup() { collection = CollectionBuilder.createCollection(context, parentCommunity) .withName("Collection") .build(); + + persons = CollectionBuilder.createCollection(context, parentCommunity) + .withName("Collection") + .withEntityType("Person") + .build(); + context.restoreAuthSystemState(); } @@ -256,6 +264,70 @@ public void testItemEnhancementWithoutForce() throws Exception { } + @Test + public void testItemEnhancementNameWithoutForce() throws Exception { + + context.turnOffAuthorisationSystem(); + + Item firstPerson = ItemBuilder.createItem(context, persons) + .withMetadata("crisrp", "name", null, "Walter White") + .build(); + + Item secondPerson = ItemBuilder.createItem(context, persons) + .withMetadata("crisrp", "name", null, "Alois White") + .withMetadata("crisrp", "name", "translated", "Alois W. White") + .build(); + + Item thirdPerson = ItemBuilder.createItem(context, persons) + .withMetadata("crisrp", "name", "translated", "Walt Alternative") + .build(); + + context.commit(); + + + assertThat(getMetadataValues(firstPerson, "dc.title"), empty()); + assertThat(getMetadataValues(secondPerson, "dc.title"), empty()); + assertThat(getMetadataValues(thirdPerson, "dc.title"), empty()); + + TestDSpaceRunnableHandler runnableHandler = runScript(false); + + assertThat(runnableHandler.getErrorMessages(), empty()); + assertThat(runnableHandler.getInfoMessages(), hasItem("Enhancement completed with success")); + + firstPerson = reload(firstPerson); + secondPerson = reload(secondPerson); + thirdPerson = reload(thirdPerson); + + assertThat(getMetadataValues(firstPerson, "dc.title"), hasSize(1)); + assertThat(getMetadataValues(secondPerson, "dc.title"), hasSize(1)); + assertThat(getMetadataValues(thirdPerson, "dc.title"), hasSize(1)); + + assertThat(firstPerson.getMetadata(), hasItem(with("dc.title", "Walter White"))); + assertThat(secondPerson.getMetadata(), hasItem(with("dc.title", "Alois White"))); + assertThat(thirdPerson.getMetadata(), hasItem(with("dc.title", "Walt Alternative"))); + + context.turnOffAuthorisationSystem(); + + MetadataValue nameToRemove = getMetadataValues(secondPerson, "crisrp.name").get(0); + itemService.removeMetadataValues(context, secondPerson, List.of(nameToRemove)); + + replaceMetadata(thirdPerson, "crisrp", "name", "translated", "Walt D. Alternative"); + + context.restoreAuthSystemState(); + + runnableHandler = runScript(false); + assertThat(runnableHandler.getErrorMessages(), empty()); + assertThat(runnableHandler.getInfoMessages(), hasItem("Enhancement completed with success")); + + firstPerson = reload(firstPerson); + secondPerson = reload(secondPerson); + thirdPerson = reload(thirdPerson); + + assertThat(firstPerson.getMetadata(), hasItem(with("dc.title", "Walter White"))); + assertThat(secondPerson.getMetadata(), hasItem(with("dc.title", "Alois W. White"))); + assertThat(thirdPerson.getMetadata(), hasItem(with("dc.title", "Walt D. Alternative"))); + } + @Test public void testItemEnhancementWithForce() throws Exception { diff --git a/dspace-api/src/test/java/org/dspace/content/enhancer/script/ItemEnhancerEntityTypeScriptIT.java b/dspace-api/src/test/java/org/dspace/content/enhancer/script/ItemEnhancerEntityTypeScriptIT.java index 1475f97b2875..3a703198c6a9 100644 --- a/dspace-api/src/test/java/org/dspace/content/enhancer/script/ItemEnhancerEntityTypeScriptIT.java +++ b/dspace-api/src/test/java/org/dspace/content/enhancer/script/ItemEnhancerEntityTypeScriptIT.java @@ -285,6 +285,70 @@ public void testItemEnhancementWithoutForce() throws Exception { } + @Test + public void testItemEnhancementNameWithoutForce() throws Exception { + + context.turnOffAuthorisationSystem(); + + Item firstPerson = ItemBuilder.createItem(context, persons) + .withMetadata("crisrp", "name", null, "Walter White") + .build(); + + Item secondPerson = ItemBuilder.createItem(context, persons) + .withMetadata("crisrp", "name", null, "Alois White") + .withMetadata("crisrp", "name", "translated", "Alois W. White") + .build(); + + Item thirdPerson = ItemBuilder.createItem(context, persons) + .withMetadata("crisrp", "name", "translated", "Walt Alternative") + .build(); + + context.commit(); + + + assertThat(getMetadataValues(firstPerson, "dc.title"), empty()); + assertThat(getMetadataValues(secondPerson, "dc.title"), empty()); + assertThat(getMetadataValues(thirdPerson, "dc.title"), empty()); + + TestDSpaceRunnableHandler runnableHandler = runScript(false); + + assertThat(runnableHandler.getErrorMessages(), empty()); + assertThat(runnableHandler.getInfoMessages(), hasItem("Enhancement completed with success")); + + firstPerson = reload(firstPerson); + secondPerson = reload(secondPerson); + thirdPerson = reload(thirdPerson); + + assertThat(getMetadataValues(firstPerson, "dc.title"), hasSize(1)); + assertThat(getMetadataValues(secondPerson, "dc.title"), hasSize(1)); + assertThat(getMetadataValues(thirdPerson, "dc.title"), hasSize(1)); + + assertThat(firstPerson.getMetadata(), hasItem(with("dc.title", "Walter White"))); + assertThat(secondPerson.getMetadata(), hasItem(with("dc.title", "Alois White"))); + assertThat(thirdPerson.getMetadata(), hasItem(with("dc.title", "Walt Alternative"))); + + context.turnOffAuthorisationSystem(); + + MetadataValue nameToRemove = getMetadataValues(secondPerson, "crisrp.name").get(0); + itemService.removeMetadataValues(context, secondPerson, List.of(nameToRemove)); + + replaceMetadata(thirdPerson, "crisrp", "name", "translated", "Walt D. Alternative"); + + context.restoreAuthSystemState(); + + runnableHandler = runScript(false); + assertThat(runnableHandler.getErrorMessages(), empty()); + assertThat(runnableHandler.getInfoMessages(), hasItem("Enhancement completed with success")); + + firstPerson = reload(firstPerson); + secondPerson = reload(secondPerson); + thirdPerson = reload(thirdPerson); + + assertThat(firstPerson.getMetadata(), hasItem(with("dc.title", "Walter White"))); + assertThat(secondPerson.getMetadata(), hasItem(with("dc.title", "Alois W. White"))); + assertThat(thirdPerson.getMetadata(), hasItem(with("dc.title", "Walt D. Alternative"))); + } + @Test public void testItemEnhancementWithForce() throws Exception { diff --git a/dspace-api/src/test/java/org/dspace/content/enhancer/script/ItemEnhancerScriptIT.java b/dspace-api/src/test/java/org/dspace/content/enhancer/script/ItemEnhancerScriptIT.java index 33913368b0a1..b95242910f44 100644 --- a/dspace-api/src/test/java/org/dspace/content/enhancer/script/ItemEnhancerScriptIT.java +++ b/dspace-api/src/test/java/org/dspace/content/enhancer/script/ItemEnhancerScriptIT.java @@ -58,6 +58,7 @@ public class ItemEnhancerScriptIT extends AbstractIntegrationTestWithDatabase { private Collection collection; + private Collection persons; /** * This method will be run before the first test as per @BeforeClass. It will @@ -100,6 +101,12 @@ public void setup() { collection = CollectionBuilder.createCollection(context, parentCommunity) .withName("Collection") .build(); + + persons = CollectionBuilder.createCollection(context, parentCommunity) + .withName("Collection") + .withEntityType("Person") + .build(); + context.restoreAuthSystemState(); } @@ -321,6 +328,70 @@ public void testItemEnhancementWithForce() throws Exception { } + @Test + public void testItemEnhancementNameWithoutForce() throws Exception { + + context.turnOffAuthorisationSystem(); + + Item firstPerson = ItemBuilder.createItem(context, persons) + .withMetadata("crisrp", "name", null, "Walter White") + .build(); + + Item secondPerson = ItemBuilder.createItem(context, persons) + .withMetadata("crisrp", "name", null, "Alois White") + .withMetadata("crisrp", "name", "translated", "Alois W. White") + .build(); + + Item thirdPerson = ItemBuilder.createItem(context, persons) + .withMetadata("crisrp", "name", "translated", "Walt Alternative") + .build(); + + context.commit(); + + + assertThat(getMetadataValues(firstPerson, "dc.title"), empty()); + assertThat(getMetadataValues(secondPerson, "dc.title"), empty()); + assertThat(getMetadataValues(thirdPerson, "dc.title"), empty()); + + TestDSpaceRunnableHandler runnableHandler = runScript(false); + + assertThat(runnableHandler.getErrorMessages(), empty()); + assertThat(runnableHandler.getInfoMessages(), hasItem("Enhancement completed with success")); + + firstPerson = reload(firstPerson); + secondPerson = reload(secondPerson); + thirdPerson = reload(thirdPerson); + + assertThat(getMetadataValues(firstPerson, "dc.title"), hasSize(1)); + assertThat(getMetadataValues(secondPerson, "dc.title"), hasSize(1)); + assertThat(getMetadataValues(thirdPerson, "dc.title"), hasSize(1)); + + assertThat(firstPerson.getMetadata(), hasItem(with("dc.title", "Walter White"))); + assertThat(secondPerson.getMetadata(), hasItem(with("dc.title", "Alois White"))); + assertThat(thirdPerson.getMetadata(), hasItem(with("dc.title", "Walt Alternative"))); + + context.turnOffAuthorisationSystem(); + + MetadataValue nameToRemove = getMetadataValues(secondPerson, "crisrp.name").get(0); + itemService.removeMetadataValues(context, secondPerson, List.of(nameToRemove)); + + replaceMetadata(thirdPerson, "crisrp", "name", "translated", "Walt D. Alternative"); + + context.restoreAuthSystemState(); + + runnableHandler = runScript(false); + assertThat(runnableHandler.getErrorMessages(), empty()); + assertThat(runnableHandler.getInfoMessages(), hasItem("Enhancement completed with success")); + + firstPerson = reload(firstPerson); + secondPerson = reload(secondPerson); + thirdPerson = reload(thirdPerson); + + assertThat(firstPerson.getMetadata(), hasItem(with("dc.title", "Walter White"))); + assertThat(secondPerson.getMetadata(), hasItem(with("dc.title", "Alois W. White"))); + assertThat(thirdPerson.getMetadata(), hasItem(with("dc.title", "Walt D. Alternative"))); + } + @Test public void testItemEnhancementMetadataPositions() throws Exception { diff --git a/dspace/config/spring/api/metadata-enhancers.xml b/dspace/config/spring/api/metadata-enhancers.xml index 58ad367c5193..acc44f57acfd 100644 --- a/dspace/config/spring/api/metadata-enhancers.xml +++ b/dspace/config/spring/api/metadata-enhancers.xml @@ -64,8 +64,9 @@ - crisrp.preferredName - crisrp.fullName + crisrp.name + crisrp.name.translated + crisrp.name.variants