diff --git a/mailbox/plugin/deleted-messages-vault-cassandra/src/main/java/org/apache/james/vault/metadata/StorageInformationDAO.java b/mailbox/plugin/deleted-messages-vault-cassandra/src/main/java/org/apache/james/vault/metadata/StorageInformationDAO.java index 2998ac03339..c6a4ed01484 100644 --- a/mailbox/plugin/deleted-messages-vault-cassandra/src/main/java/org/apache/james/vault/metadata/StorageInformationDAO.java +++ b/mailbox/plugin/deleted-messages-vault-cassandra/src/main/java/org/apache/james/vault/metadata/StorageInformationDAO.java @@ -32,8 +32,8 @@ import jakarta.inject.Inject; import org.apache.james.backends.cassandra.utils.CassandraAsyncExecutor; -import org.apache.james.blob.api.BlobId; import org.apache.james.blob.api.BucketName; +import org.apache.james.blob.api.PlainBlobId; import org.apache.james.core.Username; import org.apache.james.mailbox.model.MessageId; @@ -47,15 +47,13 @@ public class StorageInformationDAO { private final PreparedStatement addStatement; private final PreparedStatement removeStatement; private final PreparedStatement readStatement; - private final BlobId.Factory blobIdFactory; @Inject - StorageInformationDAO(CqlSession session, BlobId.Factory blobIdFactory) { + StorageInformationDAO(CqlSession session) { this.cassandraAsyncExecutor = new CassandraAsyncExecutor(session); this.addStatement = prepareAdd(session); this.removeStatement = prepareRemove(session); this.readStatement = prepareRead(session); - this.blobIdFactory = blobIdFactory; } private PreparedStatement prepareRead(CqlSession session) { @@ -102,6 +100,6 @@ Mono retrieveStorageInformation(Username username, MessageId .setString(MESSAGE_ID, messageId.serialize())) .map(row -> StorageInformation.builder() .bucketName(BucketName.of(row.getString(BUCKET_NAME))) - .blobId(blobIdFactory.parse(row.getString(BLOB_ID)))); + .blobId(new PlainBlobId(row.getString(BLOB_ID)))); } } diff --git a/mailbox/plugin/deleted-messages-vault-cassandra/src/test/java/org/apache/james/vault/metadata/CassandraDeletedMessageMetadataVaultTest.java b/mailbox/plugin/deleted-messages-vault-cassandra/src/test/java/org/apache/james/vault/metadata/CassandraDeletedMessageMetadataVaultTest.java index 73cff9198f0..ad0ec8694fa 100644 --- a/mailbox/plugin/deleted-messages-vault-cassandra/src/test/java/org/apache/james/vault/metadata/CassandraDeletedMessageMetadataVaultTest.java +++ b/mailbox/plugin/deleted-messages-vault-cassandra/src/test/java/org/apache/james/vault/metadata/CassandraDeletedMessageMetadataVaultTest.java @@ -32,7 +32,6 @@ import org.apache.james.backends.cassandra.CassandraCluster; import org.apache.james.backends.cassandra.CassandraClusterExtension; -import org.apache.james.blob.api.PlainBlobId; import org.apache.james.mailbox.inmemory.InMemoryId; import org.apache.james.mailbox.inmemory.InMemoryMessageId; import org.apache.james.vault.dto.DeletedMessageWithStorageInformationConverter; @@ -56,12 +55,11 @@ public class CassandraDeletedMessageMetadataVaultTest implements DeletedMessageM @BeforeEach void setUp(CassandraCluster cassandra) { - PlainBlobId.Factory blobIdFactory = new PlainBlobId.Factory(); InMemoryMessageId.Factory messageIdFactory = new InMemoryMessageId.Factory(); - DeletedMessageWithStorageInformationConverter dtoConverter = new DeletedMessageWithStorageInformationConverter(blobIdFactory, messageIdFactory, new InMemoryId.Factory()); + DeletedMessageWithStorageInformationConverter dtoConverter = new DeletedMessageWithStorageInformationConverter(messageIdFactory, new InMemoryId.Factory()); metadataDAO = new MetadataDAO(cassandra.getConf(), messageIdFactory, new MetadataSerializer(dtoConverter)); - storageInformationDAO = new StorageInformationDAO(cassandra.getConf(), blobIdFactory); + storageInformationDAO = new StorageInformationDAO(cassandra.getConf()); userPerBucketDAO = new UserPerBucketDAO(cassandra.getConf()); testee = new CassandraDeletedMessageMetadataVault(metadataDAO, storageInformationDAO, userPerBucketDAO); diff --git a/mailbox/plugin/deleted-messages-vault-cassandra/src/test/java/org/apache/james/vault/metadata/MetadataDAOTest.java b/mailbox/plugin/deleted-messages-vault-cassandra/src/test/java/org/apache/james/vault/metadata/MetadataDAOTest.java index 35760cb772b..760ffe0c2c5 100644 --- a/mailbox/plugin/deleted-messages-vault-cassandra/src/test/java/org/apache/james/vault/metadata/MetadataDAOTest.java +++ b/mailbox/plugin/deleted-messages-vault-cassandra/src/test/java/org/apache/james/vault/metadata/MetadataDAOTest.java @@ -31,7 +31,6 @@ import org.apache.james.backends.cassandra.CassandraCluster; import org.apache.james.backends.cassandra.CassandraClusterExtension; -import org.apache.james.blob.api.PlainBlobId; import org.apache.james.mailbox.inmemory.InMemoryId; import org.apache.james.mailbox.inmemory.InMemoryMessageId; import org.apache.james.mailbox.model.MessageId; @@ -49,7 +48,8 @@ class MetadataDAOTest { @BeforeEach void setUp(CassandraCluster cassandra) { DeletedMessageWithStorageInformationConverter dtoConverter = new DeletedMessageWithStorageInformationConverter( - new PlainBlobId.Factory(), new InMemoryMessageId.Factory(), new InMemoryId.Factory()); + new InMemoryMessageId.Factory(), + new InMemoryId.Factory()); testee = new MetadataDAO(cassandra.getConf(), new InMemoryMessageId.Factory(), new MetadataSerializer(dtoConverter)); diff --git a/mailbox/plugin/deleted-messages-vault-cassandra/src/test/java/org/apache/james/vault/metadata/StorageInformationDAOTest.java b/mailbox/plugin/deleted-messages-vault-cassandra/src/test/java/org/apache/james/vault/metadata/StorageInformationDAOTest.java index a91384fc3bf..1ce67d95d42 100644 --- a/mailbox/plugin/deleted-messages-vault-cassandra/src/test/java/org/apache/james/vault/metadata/StorageInformationDAOTest.java +++ b/mailbox/plugin/deleted-messages-vault-cassandra/src/test/java/org/apache/james/vault/metadata/StorageInformationDAOTest.java @@ -39,7 +39,6 @@ class StorageInformationDAOTest { private static final BucketName BUCKET_NAME = BucketName.of("deletedMessages-2019-06-01"); private static final BucketName BUCKET_NAME_2 = BucketName.of("deletedMessages-2019-07-01"); - private static final PlainBlobId.Factory BLOB_ID_FACTORY = new PlainBlobId.Factory(); private static final Username OWNER = Username.of("owner"); private static final TestMessageId MESSAGE_ID = TestMessageId.of(36); private static final BlobId BLOB_ID = new PlainBlobId.Factory().parse("05dcb33b-8382-4744-923a-bc593ad84d23"); @@ -54,7 +53,7 @@ class StorageInformationDAOTest { @BeforeEach void setUp(CassandraCluster cassandra) { - testee = new StorageInformationDAO(cassandra.getConf(), BLOB_ID_FACTORY); + testee = new StorageInformationDAO(cassandra.getConf()); } @Test diff --git a/mailbox/plugin/deleted-messages-vault-postgres/src/main/java/org/apache/james/vault/metadata/PostgresDeletedMessageMetadataVault.java b/mailbox/plugin/deleted-messages-vault-postgres/src/main/java/org/apache/james/vault/metadata/PostgresDeletedMessageMetadataVault.java index 38a5bc9e6c2..db36fc25424 100644 --- a/mailbox/plugin/deleted-messages-vault-postgres/src/main/java/org/apache/james/vault/metadata/PostgresDeletedMessageMetadataVault.java +++ b/mailbox/plugin/deleted-messages-vault-postgres/src/main/java/org/apache/james/vault/metadata/PostgresDeletedMessageMetadataVault.java @@ -33,8 +33,8 @@ import jakarta.inject.Inject; import org.apache.james.backends.postgres.utils.PostgresExecutor; -import org.apache.james.blob.api.BlobId; import org.apache.james.blob.api.BucketName; +import org.apache.james.blob.api.PlainBlobId; import org.apache.james.core.Username; import org.apache.james.mailbox.model.MessageId; import org.jooq.Record; @@ -46,15 +46,12 @@ public class PostgresDeletedMessageMetadataVault implements DeletedMessageMetadataVault { private final PostgresExecutor postgresExecutor; private final MetadataSerializer metadataSerializer; - private final BlobId.Factory blobIdFactory; @Inject public PostgresDeletedMessageMetadataVault(PostgresExecutor postgresExecutor, - MetadataSerializer metadataSerializer, - BlobId.Factory blobIdFactory) { + MetadataSerializer metadataSerializer) { this.postgresExecutor = postgresExecutor; this.metadataSerializer = metadataSerializer; - this.blobIdFactory = blobIdFactory; } @Override @@ -93,7 +90,7 @@ public Publisher retrieveStorageInformation(Username usernam private Function toStorageInformation() { return record -> StorageInformation.builder() .bucketName(BucketName.of(record.get(BUCKET_NAME))) - .blobId(blobIdFactory.parse(record.get(BLOB_ID))); + .blobId(new PlainBlobId(record.get(BLOB_ID))); } @Override diff --git a/mailbox/plugin/deleted-messages-vault-postgres/src/test/java/org/apache/james/vault/metadata/PostgresDeletedMessageMetadataVaultTest.java b/mailbox/plugin/deleted-messages-vault-postgres/src/test/java/org/apache/james/vault/metadata/PostgresDeletedMessageMetadataVaultTest.java index 71e06863f8b..c27c386238d 100644 --- a/mailbox/plugin/deleted-messages-vault-postgres/src/test/java/org/apache/james/vault/metadata/PostgresDeletedMessageMetadataVaultTest.java +++ b/mailbox/plugin/deleted-messages-vault-postgres/src/test/java/org/apache/james/vault/metadata/PostgresDeletedMessageMetadataVaultTest.java @@ -21,8 +21,6 @@ import org.apache.james.backends.postgres.PostgresDataDefinition; import org.apache.james.backends.postgres.PostgresExtension; -import org.apache.james.blob.api.BlobId; -import org.apache.james.blob.api.PlainBlobId; import org.apache.james.mailbox.inmemory.InMemoryId; import org.apache.james.mailbox.inmemory.InMemoryMessageId; import org.apache.james.vault.dto.DeletedMessageWithStorageInformationConverter; @@ -35,13 +33,11 @@ class PostgresDeletedMessageMetadataVaultTest implements DeletedMessageMetadataV @Override public DeletedMessageMetadataVault metadataVault() { - BlobId.Factory blobIdFactory = new PlainBlobId.Factory(); InMemoryMessageId.Factory messageIdFactory = new InMemoryMessageId.Factory(); - DeletedMessageWithStorageInformationConverter dtoConverter = new DeletedMessageWithStorageInformationConverter(blobIdFactory, + DeletedMessageWithStorageInformationConverter dtoConverter = new DeletedMessageWithStorageInformationConverter( messageIdFactory, new InMemoryId.Factory()); return new PostgresDeletedMessageMetadataVault(postgresExtension.getDefaultPostgresExecutor(), - new MetadataSerializer(dtoConverter), - blobIdFactory); + new MetadataSerializer(dtoConverter)); } } diff --git a/mailbox/plugin/deleted-messages-vault/pom.xml b/mailbox/plugin/deleted-messages-vault/pom.xml index bda2754acfc..4d9f421c258 100644 --- a/mailbox/plugin/deleted-messages-vault/pom.xml +++ b/mailbox/plugin/deleted-messages-vault/pom.xml @@ -85,6 +85,10 @@ blob-memory test + + ${james.groupId} + blob-storage-strategy + ${james.groupId} @@ -96,6 +100,15 @@ ${james.groupId} james-server-core + + ${james.groupId} + james-server-data-api + + + ${james.groupId} + james-server-data-memory + test + ${james.groupId} james-server-task-json diff --git a/mailbox/plugin/deleted-messages-vault/src/main/java/org/apache/james/vault/VaultConfiguration.java b/mailbox/plugin/deleted-messages-vault/src/main/java/org/apache/james/vault/VaultConfiguration.java index c6e46fb0d82..8781128df63 100644 --- a/mailbox/plugin/deleted-messages-vault/src/main/java/org/apache/james/vault/VaultConfiguration.java +++ b/mailbox/plugin/deleted-messages-vault/src/main/java/org/apache/james/vault/VaultConfiguration.java @@ -31,10 +31,11 @@ import com.google.common.base.Preconditions; public class VaultConfiguration { + public static final String DEFAULT_SINGLE_BUCKET_NAME = "james-deleted-message-vault"; public static final VaultConfiguration DEFAULT = - new VaultConfiguration(false, ChronoUnit.YEARS.getDuration(), DefaultMailboxes.RESTORED_MESSAGES); + new VaultConfiguration(false, ChronoUnit.YEARS.getDuration(), DefaultMailboxes.RESTORED_MESSAGES, DEFAULT_SINGLE_BUCKET_NAME); public static final VaultConfiguration ENABLED_DEFAULT = - new VaultConfiguration(true, ChronoUnit.YEARS.getDuration(), DefaultMailboxes.RESTORED_MESSAGES); + new VaultConfiguration(true, ChronoUnit.YEARS.getDuration(), DefaultMailboxes.RESTORED_MESSAGES, DEFAULT_SINGLE_BUCKET_NAME); public static VaultConfiguration from(Configuration propertiesConfiguration) { Duration retentionPeriod = Optional.ofNullable(propertiesConfiguration.getString("retentionPeriod")) @@ -42,21 +43,26 @@ public static VaultConfiguration from(Configuration propertiesConfiguration) { .orElse(DEFAULT.getRetentionPeriod()); String restoreLocation = Optional.ofNullable(propertiesConfiguration.getString("restoreLocation")) .orElse(DEFAULT.getRestoreLocation()); + String singleBucketName = Optional.ofNullable(propertiesConfiguration.getString("singleBucketName")) + .orElse(DEFAULT.getSingleBucketName()); boolean enabled = propertiesConfiguration.getBoolean("enabled", false); - return new VaultConfiguration(enabled, retentionPeriod, restoreLocation); + return new VaultConfiguration(enabled, retentionPeriod, restoreLocation, singleBucketName); } private final boolean enabled; private final Duration retentionPeriod; private final String restoreLocation; + private final String singleBucketName; - VaultConfiguration(boolean enabled, Duration retentionPeriod, String restoreLocation) { + VaultConfiguration(boolean enabled, Duration retentionPeriod, String restoreLocation, String singleBucketName) { this.enabled = enabled; Preconditions.checkNotNull(retentionPeriod); Preconditions.checkNotNull(restoreLocation); + Preconditions.checkNotNull(singleBucketName); this.retentionPeriod = retentionPeriod; this.restoreLocation = restoreLocation; + this.singleBucketName = singleBucketName; } public boolean isEnabled() { @@ -71,6 +77,10 @@ public String getRestoreLocation() { return restoreLocation; } + public String getSingleBucketName() { + return singleBucketName; + } + @Override public final boolean equals(Object o) { if (o instanceof VaultConfiguration) { @@ -78,13 +88,14 @@ public final boolean equals(Object o) { return Objects.equals(this.retentionPeriod, that.retentionPeriod) && Objects.equals(this.restoreLocation, that.restoreLocation) - && Objects.equals(this.enabled, that.enabled); + && Objects.equals(this.enabled, that.enabled) + && Objects.equals(this.singleBucketName, that.singleBucketName); } return false; } @Override public final int hashCode() { - return Objects.hash(retentionPeriod, restoreLocation, enabled); + return Objects.hash(retentionPeriod, restoreLocation, enabled, singleBucketName); } } diff --git a/mailbox/plugin/deleted-messages-vault/src/main/java/org/apache/james/vault/blob/BlobIdTimeGenerator.java b/mailbox/plugin/deleted-messages-vault/src/main/java/org/apache/james/vault/blob/BlobIdTimeGenerator.java new file mode 100644 index 00000000000..5204621ed03 --- /dev/null +++ b/mailbox/plugin/deleted-messages-vault/src/main/java/org/apache/james/vault/blob/BlobIdTimeGenerator.java @@ -0,0 +1,42 @@ +/**************************************************************** + * Licensed to the Apache Software Foundation (ASF) under one * + * or more contributor license agreements. See the NOTICE file * + * distributed with this work for additional information * + * regarding copyright ownership. The ASF licenses this file * + * to you under the Apache License, Version 2.0 (the * + * "License"); you may not use this file except in compliance * + * with the License. You may obtain a copy of the License at * + * * + * http://www.apache.org/licenses/LICENSE-2.0 * + * * + * Unless required by applicable law or agreed to in writing, * + * software distributed under the License is distributed on an * + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY * + * KIND, either express or implied. See the License for the * + * specific language governing permissions and limitations * + * under the License. * + ****************************************************************/ + +package org.apache.james.vault.blob; + +import java.time.Clock; +import java.time.ZonedDateTime; +import java.util.UUID; + +import org.apache.commons.lang3.RandomStringUtils; +import org.apache.james.blob.api.BlobId; +import org.apache.james.blob.api.PlainBlobId; + +public class BlobIdTimeGenerator { + public static final String BLOB_ID_GENERATING_FORMAT = "%d/%02d/%s/%s/%s"; + + static BlobId currentBlobId(Clock clock) { + ZonedDateTime now = ZonedDateTime.now(clock); + int month = now.getMonthValue(); + int year = now.getYear(); + String randomizerA = RandomStringUtils.insecure().nextAlphanumeric(1); + String randomizerB = RandomStringUtils.insecure().nextAlphanumeric(1); + + return new PlainBlobId(String.format(BLOB_ID_GENERATING_FORMAT, year, month, randomizerA, randomizerB, UUID.randomUUID())); + } +} diff --git a/mailbox/plugin/deleted-messages-vault/src/main/java/org/apache/james/vault/blob/BlobStoreDeletedMessageVault.java b/mailbox/plugin/deleted-messages-vault/src/main/java/org/apache/james/vault/blob/BlobStoreDeletedMessageVault.java index b7556ca5d93..c3391c4e9a0 100644 --- a/mailbox/plugin/deleted-messages-vault/src/main/java/org/apache/james/vault/blob/BlobStoreDeletedMessageVault.java +++ b/mailbox/plugin/deleted-messages-vault/src/main/java/org/apache/james/vault/blob/BlobStoreDeletedMessageVault.java @@ -33,14 +33,18 @@ import org.apache.james.blob.api.BlobStoreDAO; import org.apache.james.blob.api.BucketName; import org.apache.james.blob.api.ObjectNotFoundException; +import org.apache.james.blob.api.PlainBlobId; import org.apache.james.core.Username; import org.apache.james.mailbox.model.MessageId; import org.apache.james.metrics.api.MetricFactory; +import org.apache.james.server.blob.deduplication.BlobStoreFactory; import org.apache.james.task.Task; +import org.apache.james.user.api.UsersRepository; import org.apache.james.vault.DeletedMessage; import org.apache.james.vault.DeletedMessageContentNotFoundException; import org.apache.james.vault.DeletedMessageVault; import org.apache.james.vault.VaultConfiguration; +import org.apache.james.vault.blob.BlobStoreVaultGarbageCollectionTask.BlobStoreVaultGarbageCollectionContext; import org.apache.james.vault.metadata.DeletedMessageMetadataVault; import org.apache.james.vault.metadata.DeletedMessageWithStorageInformation; import org.apache.james.vault.metadata.StorageInformation; @@ -54,6 +58,7 @@ import reactor.core.publisher.Flux; import reactor.core.publisher.Mono; +import reactor.util.function.Tuples; public class BlobStoreDeletedMessageVault implements DeletedMessageVault { private static final Logger LOGGER = LoggerFactory.getLogger(BlobStoreDeletedMessageVault.class); @@ -72,28 +77,55 @@ public class BlobStoreDeletedMessageVault implements DeletedMessageVault { private final BucketNameGenerator nameGenerator; private final Clock clock; private final VaultConfiguration vaultConfiguration; + private final UsersRepository usersRepository; private final BlobStoreVaultGarbageCollectionTask.Factory taskFactory; @Inject public BlobStoreDeletedMessageVault(MetricFactory metricFactory, DeletedMessageMetadataVault messageMetadataVault, - BlobStore blobStore, BlobStoreDAO blobStoreDAO, BucketNameGenerator nameGenerator, - Clock clock, - VaultConfiguration vaultConfiguration) { + BlobStoreDAO blobStoreDAO, BucketNameGenerator nameGenerator, + Clock clock, VaultConfiguration vaultConfiguration, UsersRepository usersRepository) { this.metricFactory = metricFactory; this.messageMetadataVault = messageMetadataVault; - this.blobStore = blobStore; + this.blobStore = BlobStoreFactory.builder() + .blobStoreDAO(blobStoreDAO) + .blobIdFactory(new PlainBlobId.Factory()) + .defaultBucketName() + .passthrough(); this.blobStoreDAO = blobStoreDAO; this.nameGenerator = nameGenerator; this.clock = clock; this.vaultConfiguration = vaultConfiguration; + this.usersRepository = usersRepository; this.taskFactory = new BlobStoreVaultGarbageCollectionTask.Factory(this); } + @Deprecated + @VisibleForTesting + public Publisher appendV1(DeletedMessage deletedMessage, InputStream mimeMessage) { + Preconditions.checkNotNull(deletedMessage); + Preconditions.checkNotNull(mimeMessage); + BucketName bucketName = nameGenerator.currentBucket(); + + return metricFactory.decoratePublisherWithTimerMetric( + APPEND_METRIC_NAME, + appendMessageV1(deletedMessage, mimeMessage, bucketName)); + } + + private Mono appendMessageV1(DeletedMessage deletedMessage, InputStream mimeMessage, BucketName bucketName) { + return Mono.from(blobStore.save(bucketName, mimeMessage, LOW_COST)) + .map(blobId -> StorageInformation.builder() + .bucketName(bucketName) + .blobId(blobId)) + .map(storageInformation -> new DeletedMessageWithStorageInformation(deletedMessage, storageInformation)) + .flatMap(message -> Mono.from(messageMetadataVault.store(message))) + .then(); + } + @Override public Publisher append(DeletedMessage deletedMessage, InputStream mimeMessage) { Preconditions.checkNotNull(deletedMessage); Preconditions.checkNotNull(mimeMessage); - BucketName bucketName = nameGenerator.currentBucket(); + BucketName bucketName = BucketName.of(vaultConfiguration.getSingleBucketName()); return metricFactory.decoratePublisherWithTimerMetric( APPEND_METRIC_NAME, @@ -101,7 +133,7 @@ public Publisher append(DeletedMessage deletedMessage, InputStream mimeMes } private Mono appendMessage(DeletedMessage deletedMessage, InputStream mimeMessage, BucketName bucketName) { - return Mono.from(blobStore.save(bucketName, mimeMessage, LOW_COST)) + return Mono.from(blobStore.save(bucketName, mimeMessage, withTimePrefixBlobId(), LOW_COST)) .map(blobId -> StorageInformation.builder() .bucketName(bucketName) .blobId(blobId)) @@ -110,6 +142,12 @@ private Mono appendMessage(DeletedMessage deletedMessage, InputStream mime .then(); } + private BlobStore.BlobIdProvider withTimePrefixBlobId() { + return data -> Mono.just(Tuples.of( + BlobIdTimeGenerator.currentBlobId(clock), + data)); + } + @Override public Publisher loadMimeMessage(Username username, MessageId messageId) { Preconditions.checkNotNull(username); @@ -160,9 +198,9 @@ public Publisher delete(Username username, MessageId messageId) { private Mono deleteMessage(Username username, MessageId messageId) { return Mono.from(messageMetadataVault.retrieveStorageInformation(username, messageId)) - .flatMap(storageInformation -> Mono.from(messageMetadataVault.remove(storageInformation.getBucketName(), username, messageId)) - .thenReturn(storageInformation)) - .flatMap(storageInformation -> Mono.from(blobStoreDAO.delete(storageInformation.getBucketName(), storageInformation.getBlobId()))); + .flatMap(storageInformation -> Mono.from(blobStoreDAO.delete(storageInformation.getBucketName(), storageInformation.getBlobId())) + .onErrorResume(ObjectNotFoundException.class, e -> Mono.empty()) + .then(Mono.from(messageMetadataVault.remove(storageInformation.getBucketName(), username, messageId)))); } @Override @@ -170,13 +208,20 @@ public Task deleteExpiredMessagesTask() { return taskFactory.create(); } - - Flux deleteExpiredMessages(ZonedDateTime beginningOfRetentionPeriod) { + Mono deleteExpiredMessages(ZonedDateTime beginningOfRetentionPeriod, BlobStoreVaultGarbageCollectionContext context) { return Flux.from( metricFactory.decoratePublisherWithTimerMetric( DELETE_EXPIRED_MESSAGES_METRIC_NAME, - retentionQualifiedBuckets(beginningOfRetentionPeriod) - .flatMap(bucketName -> deleteBucketData(bucketName).then(Mono.just(bucketName)), DEFAULT_CONCURRENCY))); + deleteUserExpiredMessages(beginningOfRetentionPeriod, context) + .then(deletedExpiredMessagesFromOldBuckets(beginningOfRetentionPeriod, context).then()))) + .then(); + } + + @Deprecated + private Flux deletedExpiredMessagesFromOldBuckets(ZonedDateTime beginningOfRetentionPeriod, BlobStoreVaultGarbageCollectionContext context) { + return retentionQualifiedBuckets(beginningOfRetentionPeriod) + .flatMap(bucketName -> deleteBucketData(bucketName).then(Mono.just(bucketName)), DEFAULT_CONCURRENCY) + .doOnNext(context::recordDeletedBucketSuccess); } ZonedDateTime getBeginningOfRetentionPeriod() { @@ -185,6 +230,7 @@ ZonedDateTime getBeginningOfRetentionPeriod() { } @VisibleForTesting + @Deprecated Flux retentionQualifiedBuckets(ZonedDateTime beginningOfRetentionPeriod) { return Flux.from(messageMetadataVault.listRelatedBuckets()) .filter(bucketName -> isFullyExpired(beginningOfRetentionPeriod, bucketName)); @@ -204,4 +250,23 @@ private Mono deleteBucketData(BucketName bucketName) { return Mono.from(blobStore.deleteBucket(bucketName)) .then(Mono.from(messageMetadataVault.removeMetadataRelatedToBucket(bucketName))); } + + private Mono deleteUserExpiredMessages(ZonedDateTime beginningOfRetentionPeriod, BlobStoreVaultGarbageCollectionContext context) { + BucketName bucketName = BucketName.of(vaultConfiguration.getSingleBucketName()); + + return Flux.from(usersRepository.listReactive()) + .flatMap(username -> Flux.from(messageMetadataVault.listMessages(bucketName, username)) + .filter(deletedMessage -> isMessageFullyExpired(beginningOfRetentionPeriod, deletedMessage)) + .flatMap(deletedMessage -> Mono.from(blobStoreDAO.delete(bucketName, deletedMessage.getStorageInformation().getBlobId())) + .onErrorResume(ObjectNotFoundException.class, e -> Mono.empty()) + .then(Mono.from(messageMetadataVault.remove(bucketName, username, deletedMessage.getDeletedMessage().getMessageId()))) + .doOnSuccess(any -> context.recordDeletedBlobSuccess()))) + .then(); + } + + private boolean isMessageFullyExpired(ZonedDateTime beginningOfRetentionPeriod, DeletedMessageWithStorageInformation deletedMessage) { + ZonedDateTime deletionDate = deletedMessage.getDeletedMessage().getDeletionDate(); + + return deletionDate.isBefore(beginningOfRetentionPeriod); + } } diff --git a/mailbox/plugin/deleted-messages-vault/src/main/java/org/apache/james/vault/blob/BlobStoreVaultGarbageCollectionTask.java b/mailbox/plugin/deleted-messages-vault/src/main/java/org/apache/james/vault/blob/BlobStoreVaultGarbageCollectionTask.java index a455cc88fff..6b03f93c9cd 100644 --- a/mailbox/plugin/deleted-messages-vault/src/main/java/org/apache/james/vault/blob/BlobStoreVaultGarbageCollectionTask.java +++ b/mailbox/plugin/deleted-messages-vault/src/main/java/org/apache/james/vault/blob/BlobStoreVaultGarbageCollectionTask.java @@ -26,6 +26,7 @@ import java.util.List; import java.util.Optional; import java.util.concurrent.ConcurrentLinkedQueue; +import java.util.concurrent.atomic.AtomicInteger; import jakarta.inject.Inject; @@ -39,14 +40,42 @@ public class BlobStoreVaultGarbageCollectionTask implements Task { + static class BlobStoreVaultGarbageCollectionContext { + private final Collection deletedBuckets; + private final AtomicInteger deletedBlobs; + + BlobStoreVaultGarbageCollectionContext() { + this.deletedBuckets = new ConcurrentLinkedQueue<>(); + this.deletedBlobs = new AtomicInteger(0); + } + + void recordDeletedBlobSuccess() { + deletedBlobs.incrementAndGet(); + } + + int deletedBlobsCount() { + return deletedBlobs.get(); + } + + void recordDeletedBucketSuccess(BucketName bucketName) { + deletedBuckets.add(bucketName); + } + + ImmutableSet getDeletedBuckets() { + return ImmutableSet.copyOf(deletedBuckets); + } + } + public static class AdditionalInformation implements TaskExecutionDetails.AdditionalInformation { private final ZonedDateTime beginningOfRetentionPeriod; private final ImmutableSet deletedBuckets; + private final int deletedBlobs; private final Instant timestamp; - AdditionalInformation(ZonedDateTime beginningOfRetentionPeriod, ImmutableSet deletedBuckets, Instant timestamp) { + AdditionalInformation(ZonedDateTime beginningOfRetentionPeriod, ImmutableSet deletedBuckets, int deletedBlobs, Instant timestamp) { this.beginningOfRetentionPeriod = beginningOfRetentionPeriod; this.deletedBuckets = deletedBuckets; + this.deletedBlobs = deletedBlobs; this.timestamp = timestamp; } @@ -60,6 +89,10 @@ public List getDeletedBuckets() { .collect(ImmutableList.toImmutableList()); } + public int getDeletedBlobs() { + return deletedBlobs; + } + @Override public Instant timestamp() { return timestamp; @@ -67,7 +100,7 @@ public Instant timestamp() { } static final TaskType TYPE = TaskType.of("deleted-messages-blob-store-based-garbage-collection"); - private final Collection deletedBuckets; + private final BlobStoreVaultGarbageCollectionContext context; private final BlobStoreDeletedMessageVault deletedMessageVault; private final ZonedDateTime beginningOfRetentionPeriod; @@ -87,14 +120,12 @@ public BlobStoreVaultGarbageCollectionTask create() { private BlobStoreVaultGarbageCollectionTask(BlobStoreDeletedMessageVault deletedMessageVault) { this.beginningOfRetentionPeriod = deletedMessageVault.getBeginningOfRetentionPeriod(); this.deletedMessageVault = deletedMessageVault; - this.deletedBuckets = new ConcurrentLinkedQueue<>(); + this.context = new BlobStoreVaultGarbageCollectionContext(); } @Override public Result run() { - deletedMessageVault.deleteExpiredMessages(beginningOfRetentionPeriod) - .doOnNext(deletedBuckets::add) - .then() + deletedMessageVault.deleteExpiredMessages(beginningOfRetentionPeriod, context) .block(); return Result.COMPLETED; @@ -107,6 +138,10 @@ public TaskType type() { @Override public Optional details() { - return Optional.of(new AdditionalInformation(beginningOfRetentionPeriod, ImmutableSet.copyOf(deletedBuckets), Clock.systemUTC().instant())); + return Optional.of(new AdditionalInformation( + beginningOfRetentionPeriod, + context.getDeletedBuckets(), + context.deletedBlobsCount(), + Clock.systemUTC().instant())); } } diff --git a/mailbox/plugin/deleted-messages-vault/src/main/java/org/apache/james/vault/blob/BlobStoreVaultGarbageCollectionTaskAdditionalInformationDTO.java b/mailbox/plugin/deleted-messages-vault/src/main/java/org/apache/james/vault/blob/BlobStoreVaultGarbageCollectionTaskAdditionalInformationDTO.java index 169f56b17b9..e75e5dd82a9 100644 --- a/mailbox/plugin/deleted-messages-vault/src/main/java/org/apache/james/vault/blob/BlobStoreVaultGarbageCollectionTaskAdditionalInformationDTO.java +++ b/mailbox/plugin/deleted-messages-vault/src/main/java/org/apache/james/vault/blob/BlobStoreVaultGarbageCollectionTaskAdditionalInformationDTO.java @@ -37,6 +37,7 @@ static BlobStoreVaultGarbageCollectionTaskAdditionalInformationDTO fromDomainObj type, additionalInformation.getBeginningOfRetentionPeriod().toString(), additionalInformation.getDeletedBuckets(), + additionalInformation.getDeletedBlobs(), additionalInformation.timestamp() ); } @@ -61,6 +62,7 @@ public static final AdditionalInformationDTOModule deletedBuckets; + private final int deletedBlobs; private final String type; private final Instant timestamp; @@ -68,10 +70,12 @@ public static final AdditionalInformationDTOModule deletedBuckets, + @JsonProperty("deletedBlobs") int deletedBlobs, @JsonProperty("timestamp") Instant timestamp) { this.type = type; this.beginningOfRetentionPeriod = beginningOfRetentionPeriod; this.deletedBuckets = deletedBuckets; + this.deletedBlobs = deletedBlobs; this.timestamp = timestamp; } @@ -82,6 +86,7 @@ BlobStoreVaultGarbageCollectionTask.AdditionalInformation toDomainObject() { .stream() .map(BucketName::of) .collect(ImmutableSet.toImmutableSet()), + deletedBlobs, timestamp); } @@ -93,6 +98,10 @@ public Collection getDeletedBuckets() { return deletedBuckets; } + public int getDeletedBlobs() { + return deletedBlobs; + } + @Override public String getType() { return type; diff --git a/mailbox/plugin/deleted-messages-vault/src/main/java/org/apache/james/vault/dto/DeletedMessageWithStorageInformationConverter.java b/mailbox/plugin/deleted-messages-vault/src/main/java/org/apache/james/vault/dto/DeletedMessageWithStorageInformationConverter.java index 2e74c58dec1..2ce1146d01b 100644 --- a/mailbox/plugin/deleted-messages-vault/src/main/java/org/apache/james/vault/dto/DeletedMessageWithStorageInformationConverter.java +++ b/mailbox/plugin/deleted-messages-vault/src/main/java/org/apache/james/vault/dto/DeletedMessageWithStorageInformationConverter.java @@ -25,8 +25,8 @@ import jakarta.inject.Inject; import jakarta.mail.internet.AddressException; -import org.apache.james.blob.api.BlobId; import org.apache.james.blob.api.BucketName; +import org.apache.james.blob.api.PlainBlobId; import org.apache.james.core.MailAddress; import org.apache.james.core.MaybeSender; import org.apache.james.core.Username; @@ -40,15 +40,12 @@ import com.google.common.collect.ImmutableList; public class DeletedMessageWithStorageInformationConverter { - private final BlobId.Factory blobFactory; private final MessageId.Factory messageIdFactory; private final MailboxId.Factory mailboxIdFactory; @Inject - public DeletedMessageWithStorageInformationConverter(BlobId.Factory blobFactory, - MessageId.Factory messageIdFactory, + public DeletedMessageWithStorageInformationConverter(MessageId.Factory messageIdFactory, MailboxId.Factory mailboxIdFactory) { - this.blobFactory = blobFactory; this.messageIdFactory = messageIdFactory; this.mailboxIdFactory = mailboxIdFactory; } @@ -56,7 +53,7 @@ public DeletedMessageWithStorageInformationConverter(BlobId.Factory blobFactory, public StorageInformation toDomainObject(DeletedMessageWithStorageInformationDTO.StorageInformationDTO storageInformationDTO) { return StorageInformation.builder() .bucketName(BucketName.of(storageInformationDTO.getBucketName())) - .blobId(blobFactory.parse(storageInformationDTO.getBlobId())); + .blobId(new PlainBlobId(storageInformationDTO.getBlobId())); } public DeletedMessage toDomainObject(DeletedMessageWithStorageInformationDTO.DeletedMessageDTO deletedMessageDTO) throws AddressException { @@ -82,7 +79,7 @@ public DeletedMessageWithStorageInformation toDomainObject(DeletedMessageWithSto private ImmutableList deserializeOriginMailboxes(List originMailboxes) { return originMailboxes.stream() - .map(mailboxId -> mailboxIdFactory.fromString(mailboxId)) + .map(mailboxIdFactory::fromString) .collect(ImmutableList.toImmutableList()); } diff --git a/mailbox/plugin/deleted-messages-vault/src/test/java/org/apache/james/vault/DeletedMessageFixture.java b/mailbox/plugin/deleted-messages-vault/src/test/java/org/apache/james/vault/DeletedMessageFixture.java index a0a7f9c3976..99440a84000 100644 --- a/mailbox/plugin/deleted-messages-vault/src/test/java/org/apache/james/vault/DeletedMessageFixture.java +++ b/mailbox/plugin/deleted-messages-vault/src/test/java/org/apache/james/vault/DeletedMessageFixture.java @@ -43,6 +43,7 @@ public interface DeletedMessageFixture { InMemoryId MAILBOX_ID_3 = InMemoryId.of(45); Username USERNAME = Username.of("bob@apache.org"); Username USERNAME_2 = Username.of("dimitri@apache.org"); + String PASSWORD = "123456"; ZonedDateTime DELIVERY_DATE = ZonedDateTime.parse("2014-10-30T14:12:00Z"); ZonedDateTime DELETION_DATE = ZonedDateTime.parse("2015-10-30T14:12:00Z"); ZonedDateTime NOW = ZonedDateTime.parse("2015-10-30T16:12:00Z"); diff --git a/mailbox/plugin/deleted-messages-vault/src/test/java/org/apache/james/vault/DeletedMessageVaultContract.java b/mailbox/plugin/deleted-messages-vault/src/test/java/org/apache/james/vault/DeletedMessageVaultContract.java index cf82d47a4a3..368d9e7b9b6 100644 --- a/mailbox/plugin/deleted-messages-vault/src/test/java/org/apache/james/vault/DeletedMessageVaultContract.java +++ b/mailbox/plugin/deleted-messages-vault/src/test/java/org/apache/james/vault/DeletedMessageVaultContract.java @@ -35,7 +35,6 @@ import static org.assertj.core.api.Assertions.assertThatThrownBy; import java.io.ByteArrayInputStream; -import java.time.Clock; import java.time.Duration; import java.util.List; @@ -43,6 +42,7 @@ import org.apache.james.task.Task; import org.apache.james.util.concurrency.ConcurrentTestRunner; import org.apache.james.utils.UpdatableTickingClock; +import org.apache.james.vault.blob.BlobStoreDeletedMessageVault; import org.apache.james.vault.search.CriterionFactory; import org.apache.james.vault.search.Query; import org.junit.jupiter.api.Test; @@ -51,9 +51,7 @@ import reactor.core.publisher.Mono; public interface DeletedMessageVaultContract { - Clock CLOCK = Clock.fixed(NOW.toInstant(), NOW.getZone()); - - DeletedMessageVault getVault(); + BlobStoreDeletedMessageVault getVault(); UpdatableTickingClock getClock(); @@ -243,6 +241,16 @@ default void deleteExpiredMessagesTaskShouldCompleteWhenNoMail() throws Interrup @Test default void deleteExpiredMessagesTaskShouldCompleteWhenAllMailsDeleted() throws InterruptedException { + Mono.from(getVault().appendV1(DELETED_MESSAGE, new ByteArrayInputStream(CONTENT))).block(); + Mono.from(getVault().delete(USERNAME, DELETED_MESSAGE.getMessageId())).block(); + + Task.Result result = getVault().deleteExpiredMessagesTask().run(); + + assertThat(result).isEqualTo(Task.Result.COMPLETED); + } + + @Test + default void deleteExpiredMessagesSingleBucketTaskShouldCompleteWhenAllMailsDeleted() throws InterruptedException { Mono.from(getVault().append(DELETED_MESSAGE, new ByteArrayInputStream(CONTENT))).block(); Mono.from(getVault().delete(USERNAME, DELETED_MESSAGE.getMessageId())).block(); @@ -253,6 +261,15 @@ default void deleteExpiredMessagesTaskShouldCompleteWhenAllMailsDeleted() throws @Test default void deleteExpiredMessagesTaskShouldCompleteWhenOnlyRecentMails() throws InterruptedException { + Mono.from(getVault().appendV1(DELETED_MESSAGE, new ByteArrayInputStream(CONTENT))).block(); + + Task.Result result = getVault().deleteExpiredMessagesTask().run(); + + assertThat(result).isEqualTo(Task.Result.COMPLETED); + } + + @Test + default void deleteExpiredMessagesSingleBucketTaskShouldCompleteWhenOnlyRecentMails() throws InterruptedException { Mono.from(getVault().append(DELETED_MESSAGE, new ByteArrayInputStream(CONTENT))).block(); Task.Result result = getVault().deleteExpiredMessagesTask().run(); @@ -262,6 +279,15 @@ default void deleteExpiredMessagesTaskShouldCompleteWhenOnlyRecentMails() throws @Test default void deleteExpiredMessagesTaskShouldCompleteWhenOnlyOldMails() throws InterruptedException { + Mono.from(getVault().appendV1(OLD_DELETED_MESSAGE, new ByteArrayInputStream(CONTENT))).block(); + + Task.Result result = getVault().deleteExpiredMessagesTask().run(); + + assertThat(result).isEqualTo(Task.Result.COMPLETED); + } + + @Test + default void deleteExpiredMessagesSingleBucketTaskShouldCompleteWhenOnlyOldMails() throws InterruptedException { Mono.from(getVault().append(OLD_DELETED_MESSAGE, new ByteArrayInputStream(CONTENT))).block(); Task.Result result = getVault().deleteExpiredMessagesTask().run(); @@ -279,6 +305,16 @@ default void deleteExpiredMessagesTaskShouldDoNothingWhenEmpty() throws Interrup @Test default void deleteExpiredMessagesTaskShouldNotDeleteRecentMails() throws InterruptedException { + Mono.from(getVault().appendV1(DELETED_MESSAGE, new ByteArrayInputStream(CONTENT))).block(); + + getVault().deleteExpiredMessagesTask().run(); + + assertThat(Flux.from(getVault().search(USERNAME, ALL)).collectList().block()) + .containsOnly(DELETED_MESSAGE); + } + + @Test + default void deleteExpiredMessagesSingleBucketTaskShouldNotDeleteRecentMails() throws InterruptedException { Mono.from(getVault().append(DELETED_MESSAGE, new ByteArrayInputStream(CONTENT))).block(); getVault().deleteExpiredMessagesTask().run(); @@ -289,6 +325,17 @@ default void deleteExpiredMessagesTaskShouldNotDeleteRecentMails() throws Interr @Test default void deleteExpiredMessagesTaskShouldDeleteOldMails() throws InterruptedException { + Mono.from(getVault().appendV1(OLD_DELETED_MESSAGE, new ByteArrayInputStream(CONTENT))).block(); + + getClock().setInstant(NOW.plusYears(2).toInstant()); + getVault().deleteExpiredMessagesTask().run(); + + assertThat(Flux.from(getVault().search(USERNAME, ALL)).collectList().block()) + .isEmpty(); + } + + @Test + default void deleteExpiredMessagesSingleBucketTaskShouldDeleteOldMails() throws InterruptedException { Mono.from(getVault().append(OLD_DELETED_MESSAGE, new ByteArrayInputStream(CONTENT))).block(); getClock().setInstant(NOW.plusYears(2).toInstant()); @@ -300,6 +347,22 @@ default void deleteExpiredMessagesTaskShouldDeleteOldMails() throws InterruptedE @Test default void deleteExpiredMessagesTaskShouldDeleteOldMailsWhenRunSeveralTime() throws InterruptedException { + Mono.from(getVault().appendV1(OLD_DELETED_MESSAGE, new ByteArrayInputStream(CONTENT))).block(); + getClock().setInstant(NOW.plusYears(2).toInstant()); + getVault().deleteExpiredMessagesTask().run(); + + Mono.from(getVault().appendV1(OLD_DELETED_MESSAGE, new ByteArrayInputStream(CONTENT))).block(); + getClock().setInstant(NOW.plusYears(4).toInstant()); + getVault().deleteExpiredMessagesTask().run(); + + assertThat(Flux.from(getVault().search(USERNAME, ALL)).collectList().block()) + .isEmpty(); + assertThat(Flux.from(getVault().search(USERNAME_2, ALL)).collectList().block()) + .isEmpty(); + } + + @Test + default void deleteExpiredMessagesSingleBucketTaskShouldDeleteOldMailsWhenRunSeveralTime() throws InterruptedException { Mono.from(getVault().append(OLD_DELETED_MESSAGE, new ByteArrayInputStream(CONTENT))).block(); getClock().setInstant(NOW.plusYears(2).toInstant()); getVault().deleteExpiredMessagesTask().run(); @@ -313,4 +376,19 @@ default void deleteExpiredMessagesTaskShouldDeleteOldMailsWhenRunSeveralTime() t assertThat(Flux.from(getVault().search(USERNAME_2, ALL)).collectList().block()) .isEmpty(); } + + @Test + default void deleteExpiredMessagesTaskShouldDeleteOldMailsInBothCases() throws InterruptedException { + Mono.from(getVault().appendV1(OLD_DELETED_MESSAGE, new ByteArrayInputStream(CONTENT))).block(); + + getClock().setInstant(NOW.plusYears(2).toInstant()); + + Mono.from(getVault().append(OLD_DELETED_MESSAGE, new ByteArrayInputStream(CONTENT))).block(); + + getClock().setInstant(NOW.plusYears(4).toInstant()); + getVault().deleteExpiredMessagesTask().run(); + + assertThat(Flux.from(getVault().search(USERNAME, ALL)).collectList().block()) + .isEmpty(); + } } diff --git a/mailbox/plugin/deleted-messages-vault/src/test/java/org/apache/james/vault/DeletedMessageVaultHookTest.java b/mailbox/plugin/deleted-messages-vault/src/test/java/org/apache/james/vault/DeletedMessageVaultHookTest.java index 9149599fbaf..7f628d58114 100644 --- a/mailbox/plugin/deleted-messages-vault/src/test/java/org/apache/james/vault/DeletedMessageVaultHookTest.java +++ b/mailbox/plugin/deleted-messages-vault/src/test/java/org/apache/james/vault/DeletedMessageVaultHookTest.java @@ -24,6 +24,7 @@ import static org.apache.james.vault.DeletedMessageFixture.INTERNAL_DATE; import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatCode; +import static org.mockito.Mockito.mock; import java.nio.charset.StandardCharsets; import java.time.Clock; @@ -31,6 +32,7 @@ import java.util.List; import java.util.stream.IntStream; +import org.apache.james.blob.api.BlobId; import org.apache.james.blob.api.PlainBlobId; import org.apache.james.blob.memory.MemoryBlobStoreDAO; import org.apache.james.core.MailAddress; @@ -52,7 +54,7 @@ import org.apache.james.mailbox.model.SearchQuery; import org.apache.james.metrics.tests.RecordingMetricFactory; import org.apache.james.mime4j.dom.Message; -import org.apache.james.server.blob.deduplication.BlobStoreFactory; +import org.apache.james.user.api.UsersRepository; import org.apache.james.vault.blob.BlobStoreDeletedMessageVault; import org.apache.james.vault.blob.BucketNameGenerator; import org.apache.james.vault.memory.metadata.MemoryDeletedMessageMetadataVault; @@ -115,13 +117,13 @@ private ComposedMessageId appendMessage(MessageManager messageManager) throws Ex void setUp() throws Exception { clock = Clock.fixed(DELETION_DATE.toInstant(), ZoneOffset.UTC); MemoryBlobStoreDAO blobStoreDAO = new MemoryBlobStoreDAO(); + BlobId.Factory blobIdFactory = new PlainBlobId.Factory(); + + UsersRepository usersRepository = mock(UsersRepository.class); + messageVault = new BlobStoreDeletedMessageVault(new RecordingMetricFactory(), new MemoryDeletedMessageMetadataVault(), - BlobStoreFactory.builder() - .blobStoreDAO(blobStoreDAO) - .blobIdFactory(new PlainBlobId.Factory()) - .defaultBucketName() - .passthrough(), blobStoreDAO, new BucketNameGenerator(clock), clock, - VaultConfiguration.ENABLED_DEFAULT); + blobStoreDAO, new BucketNameGenerator(clock), clock, + VaultConfiguration.ENABLED_DEFAULT, usersRepository); DeletedMessageConverter deletedMessageConverter = new DeletedMessageConverter(); diff --git a/mailbox/plugin/deleted-messages-vault/src/test/java/org/apache/james/vault/VaultConfigurationTest.java b/mailbox/plugin/deleted-messages-vault/src/test/java/org/apache/james/vault/VaultConfigurationTest.java index 656af119d6a..25f13ddc51e 100644 --- a/mailbox/plugin/deleted-messages-vault/src/test/java/org/apache/james/vault/VaultConfigurationTest.java +++ b/mailbox/plugin/deleted-messages-vault/src/test/java/org/apache/james/vault/VaultConfigurationTest.java @@ -19,6 +19,7 @@ package org.apache.james.vault; +import static org.apache.james.vault.VaultConfiguration.DEFAULT_SINGLE_BUCKET_NAME; import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatThrownBy; @@ -40,13 +41,19 @@ void shouldMatchBeanContract() { @Test void constructorShouldThrowWhenRetentionPeriodIsNull() { - assertThatThrownBy(() -> new VaultConfiguration(true, null, DefaultMailboxes.RESTORED_MESSAGES)) + assertThatThrownBy(() -> new VaultConfiguration(true, null, DefaultMailboxes.RESTORED_MESSAGES, DEFAULT_SINGLE_BUCKET_NAME)) .isInstanceOf(NullPointerException.class); } @Test void constructorShouldThrowWhenRestoreLocationIsNull() { - assertThatThrownBy(() -> new VaultConfiguration(true, ChronoUnit.YEARS.getDuration(), null)) + assertThatThrownBy(() -> new VaultConfiguration(true, ChronoUnit.YEARS.getDuration(), null, DEFAULT_SINGLE_BUCKET_NAME)) + .isInstanceOf(NullPointerException.class); + } + + @Test + void constructorShouldThrowWhenSingleBucketNameIsNull() { + assertThatThrownBy(() -> new VaultConfiguration(true, ChronoUnit.YEARS.getDuration(), DefaultMailboxes.RESTORED_MESSAGES, null)) .isInstanceOf(NullPointerException.class); } @@ -62,7 +69,7 @@ void fromShouldReturnConfiguredRestoreLocation() { configuration.addProperty("restoreLocation", "INBOX"); assertThat(VaultConfiguration.from(configuration)).isEqualTo( - new VaultConfiguration(false, ChronoUnit.YEARS.getDuration(), DefaultMailboxes.INBOX)); + new VaultConfiguration(false, ChronoUnit.YEARS.getDuration(), DefaultMailboxes.INBOX, DEFAULT_SINGLE_BUCKET_NAME)); } @Test @@ -71,7 +78,7 @@ void fromShouldReturnConfiguredRetentionTime() { configuration.addProperty("retentionPeriod", "15d"); assertThat(VaultConfiguration.from(configuration)).isEqualTo( - new VaultConfiguration(false, Duration.ofDays(15), DefaultMailboxes.RESTORED_MESSAGES)); + new VaultConfiguration(false, Duration.ofDays(15), DefaultMailboxes.RESTORED_MESSAGES, DEFAULT_SINGLE_BUCKET_NAME)); } @Test @@ -80,7 +87,7 @@ void fromShouldHandleHours() { configuration.addProperty("retentionPeriod", "15h"); assertThat(VaultConfiguration.from(configuration)).isEqualTo( - new VaultConfiguration(false, Duration.ofHours(15), DefaultMailboxes.RESTORED_MESSAGES)); + new VaultConfiguration(false, Duration.ofHours(15), DefaultMailboxes.RESTORED_MESSAGES, DEFAULT_SINGLE_BUCKET_NAME)); } @Test @@ -89,7 +96,16 @@ void fromShouldUseDaysAsADefaultUnit() { configuration.addProperty("retentionPeriod", "15"); assertThat(VaultConfiguration.from(configuration)).isEqualTo( - new VaultConfiguration(false, Duration.ofDays(15), DefaultMailboxes.RESTORED_MESSAGES)); + new VaultConfiguration(false, Duration.ofDays(15), DefaultMailboxes.RESTORED_MESSAGES, DEFAULT_SINGLE_BUCKET_NAME)); + } + + @Test + void fromShouldReturnConfiguredSingleBucketName() { + PropertiesConfiguration configuration = new PropertiesConfiguration(); + configuration.addProperty("singleBucketName", "bucketBlabla"); + + assertThat(VaultConfiguration.from(configuration)).isEqualTo( + new VaultConfiguration(false, ChronoUnit.YEARS.getDuration(), DefaultMailboxes.RESTORED_MESSAGES, "bucketBlabla")); } @Test diff --git a/mailbox/plugin/deleted-messages-vault/src/test/java/org/apache/james/vault/blob/BlobIdTimeGeneratorTest.java b/mailbox/plugin/deleted-messages-vault/src/test/java/org/apache/james/vault/blob/BlobIdTimeGeneratorTest.java new file mode 100644 index 00000000000..98e66150bc9 --- /dev/null +++ b/mailbox/plugin/deleted-messages-vault/src/test/java/org/apache/james/vault/blob/BlobIdTimeGeneratorTest.java @@ -0,0 +1,40 @@ +/**************************************************************** + * Licensed to the Apache Software Foundation (ASF) under one * + * or more contributor license agreements. See the NOTICE file * + * distributed with this work for additional information * + * regarding copyright ownership. The ASF licenses this file * + * to you under the Apache License, Version 2.0 (the * + * "License"); you may not use this file except in compliance * + * with the License. You may obtain a copy of the License at * + * * + * http://www.apache.org/licenses/LICENSE-2.0 * + * * + * Unless required by applicable law or agreed to in writing, * + * software distributed under the License is distributed on an * + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY * + * KIND, either express or implied. See the License for the * + * specific language governing permissions and limitations * + * under the License. * + ****************************************************************/ + +package org.apache.james.vault.blob; + +import static org.assertj.core.api.Assertions.assertThat; + +import java.time.Clock; +import java.time.Instant; + +import org.apache.james.utils.UpdatableTickingClock; +import org.junit.jupiter.api.Test; + +class BlobIdTimeGeneratorTest { + private static final Instant NOW = Instant.parse("2007-07-03T10:15:30.00Z"); + private static final Clock CLOCK = new UpdatableTickingClock(NOW); + + @Test + void currentBlobIdShouldReturnBlobIdFormattedWithYearAndMonthPrefix() { + String currentBlobId = BlobIdTimeGenerator.currentBlobId(CLOCK).asString(); + + assertThat(currentBlobId).matches("2007/07/././.*"); + } +} diff --git a/mailbox/plugin/deleted-messages-vault/src/test/java/org/apache/james/vault/blob/BlobStoreDeletedMessageVaultTest.java b/mailbox/plugin/deleted-messages-vault/src/test/java/org/apache/james/vault/blob/BlobStoreDeletedMessageVaultTest.java index 328a5206607..3508d07b032 100644 --- a/mailbox/plugin/deleted-messages-vault/src/test/java/org/apache/james/vault/blob/BlobStoreDeletedMessageVaultTest.java +++ b/mailbox/plugin/deleted-messages-vault/src/test/java/org/apache/james/vault/blob/BlobStoreDeletedMessageVaultTest.java @@ -22,10 +22,15 @@ import static org.apache.james.vault.DeletedMessageFixture.CONTENT; import static org.apache.james.vault.DeletedMessageFixture.DELETED_MESSAGE; import static org.apache.james.vault.DeletedMessageFixture.DELETED_MESSAGE_2; +import static org.apache.james.vault.DeletedMessageFixture.DELETED_MESSAGE_GENERATOR; +import static org.apache.james.vault.DeletedMessageFixture.DELETED_MESSAGE_WITH_SUBJECT; import static org.apache.james.vault.DeletedMessageFixture.MESSAGE_ID; import static org.apache.james.vault.DeletedMessageFixture.NOW; import static org.apache.james.vault.DeletedMessageFixture.OLD_DELETED_MESSAGE; +import static org.apache.james.vault.DeletedMessageFixture.PASSWORD; +import static org.apache.james.vault.DeletedMessageFixture.SUBJECT; import static org.apache.james.vault.DeletedMessageFixture.USERNAME; +import static org.apache.james.vault.DeletedMessageFixture.USERNAME_2; import static org.apache.james.vault.blob.BlobStoreDeletedMessageVault.APPEND_METRIC_NAME; import static org.apache.james.vault.blob.BlobStoreDeletedMessageVault.DELETE_EXPIRED_MESSAGES_METRIC_NAME; import static org.apache.james.vault.blob.BlobStoreDeletedMessageVault.DELETE_METRIC_NAME; @@ -33,49 +38,59 @@ import static org.apache.james.vault.blob.BlobStoreDeletedMessageVault.SEARCH_METRIC_NAME; import static org.apache.james.vault.search.Query.ALL; import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.mock; import java.io.ByteArrayInputStream; import java.time.Instant; import java.time.ZonedDateTime; +import java.util.List; import org.apache.james.blob.api.BucketName; -import org.apache.james.blob.api.PlainBlobId; import org.apache.james.blob.memory.MemoryBlobStoreDAO; +import org.apache.james.domainlist.api.DomainList; +import org.apache.james.mailbox.inmemory.InMemoryMessageId; import org.apache.james.metrics.tests.RecordingMetricFactory; -import org.apache.james.server.blob.deduplication.BlobStoreFactory; +import org.apache.james.user.memory.MemoryUsersRepository; import org.apache.james.utils.UpdatableTickingClock; -import org.apache.james.vault.DeletedMessageVault; +import org.apache.james.vault.DeletedMessage; import org.apache.james.vault.DeletedMessageVaultContract; import org.apache.james.vault.DeletedMessageVaultSearchContract; import org.apache.james.vault.VaultConfiguration; import org.apache.james.vault.memory.metadata.MemoryDeletedMessageMetadataVault; +import org.apache.james.vault.search.CriterionFactory; +import org.apache.james.vault.search.Query; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; +import org.mockito.Mockito; +import reactor.core.publisher.Flux; import reactor.core.publisher.Mono; - class BlobStoreDeletedMessageVaultTest implements DeletedMessageVaultContract, DeletedMessageVaultSearchContract.AllContracts { private BlobStoreDeletedMessageVault messageVault; private UpdatableTickingClock clock; private RecordingMetricFactory metricFactory; @BeforeEach - void setUp() { + void setUp() throws Exception { clock = new UpdatableTickingClock(NOW.toInstant()); metricFactory = new RecordingMetricFactory(); MemoryBlobStoreDAO blobStoreDAO = new MemoryBlobStoreDAO(); + + DomainList domainList = mock(DomainList.class); + Mockito.when(domainList.containsDomain(any())).thenReturn(true); + MemoryUsersRepository usersRepository = MemoryUsersRepository.withVirtualHosting(domainList); + usersRepository.addUser(USERNAME, PASSWORD); + usersRepository.addUser(USERNAME_2, PASSWORD); + messageVault = new BlobStoreDeletedMessageVault(metricFactory, new MemoryDeletedMessageMetadataVault(), - BlobStoreFactory.builder() - .blobStoreDAO(blobStoreDAO) - .blobIdFactory(new PlainBlobId.Factory()) - .defaultBucketName() - .passthrough(), - blobStoreDAO, new BucketNameGenerator(clock), clock, VaultConfiguration.ENABLED_DEFAULT); + blobStoreDAO, new BucketNameGenerator(clock), clock, + VaultConfiguration.ENABLED_DEFAULT, usersRepository); } @Override - public DeletedMessageVault getVault() { + public BlobStoreDeletedMessageVault getVault() { return messageVault; } @@ -87,9 +102,9 @@ public UpdatableTickingClock getClock() { @Test void retentionQualifiedBucketsShouldReturnOnlyBucketsFullyBeforeBeginningOfRetentionPeriod() { clock.setInstant(Instant.parse("2007-12-03T10:15:30.00Z")); - Mono.from(getVault().append(OLD_DELETED_MESSAGE, new ByteArrayInputStream(CONTENT))).block(); + Mono.from(getVault().appendV1(OLD_DELETED_MESSAGE, new ByteArrayInputStream(CONTENT))).block(); clock.setInstant(Instant.parse("2008-01-03T10:15:30.00Z")); - Mono.from(getVault().append(DELETED_MESSAGE_2, new ByteArrayInputStream(CONTENT))).block(); + Mono.from(getVault().appendV1(DELETED_MESSAGE_2, new ByteArrayInputStream(CONTENT))).block(); ZonedDateTime beginningOfRetention = ZonedDateTime.parse("2008-01-30T10:15:30.00Z"); assertThat(messageVault.retentionQualifiedBuckets(beginningOfRetention).toStream()) @@ -99,9 +114,9 @@ void retentionQualifiedBucketsShouldReturnOnlyBucketsFullyBeforeBeginningOfReten @Test void retentionQualifiedBucketsShouldReturnAllWhenAllBucketMonthAreBeforeBeginningOfRetention() { clock.setInstant(Instant.parse("2007-12-03T10:15:30.00Z")); - Mono.from(getVault().append(OLD_DELETED_MESSAGE, new ByteArrayInputStream(CONTENT))).block(); + Mono.from(getVault().appendV1(OLD_DELETED_MESSAGE, new ByteArrayInputStream(CONTENT))).block(); clock.setInstant(Instant.parse("2008-01-30T10:15:30.00Z")); - Mono.from(getVault().append(DELETED_MESSAGE_2, new ByteArrayInputStream(CONTENT))).block(); + Mono.from(getVault().appendV1(DELETED_MESSAGE_2, new ByteArrayInputStream(CONTENT))).block(); assertThat(messageVault.retentionQualifiedBuckets(ZonedDateTime.parse("2008-02-01T10:15:30.00Z")).toStream()) .containsOnly( @@ -163,4 +178,67 @@ void deleteExpiredMessagesTaskShouldPublishRetentionTimerMetrics() throws Except assertThat(metricFactory.executionTimesFor(DELETE_EXPIRED_MESSAGES_METRIC_NAME)) .hasSize(1); } + + @Test + public void loadMimeMessageShouldReturnOldMessage() { + Mono.from(getVault().appendV1(DELETED_MESSAGE, new ByteArrayInputStream(CONTENT))).block(); + + assertThat(Mono.from(getVault().loadMimeMessage(USERNAME, MESSAGE_ID)).blockOptional()) + .isNotEmpty() + .satisfies(maybeContent -> assertThat(maybeContent.get()).hasSameContentAs(new ByteArrayInputStream(CONTENT))); + } + + @Test + public void loadMimeMessageShouldReturnEmptyWhenOldMessageDeleted() { + Mono.from(getVault().appendV1(DELETED_MESSAGE, new ByteArrayInputStream(CONTENT))).block(); + + Mono.from(getVault().delete(USERNAME, MESSAGE_ID)).block(); + + assertThat(Mono.from(getVault().loadMimeMessage(USERNAME, MESSAGE_ID)).blockOptional()) + .isEmpty(); + } + + @Test + public void searchAllShouldReturnOldMessage() { + Mono.from(getVault().appendV1(DELETED_MESSAGE, new ByteArrayInputStream(CONTENT))).block(); + + assertThat(Flux.from(getVault().search(USERNAME, ALL)).collectList().block()) + .containsOnly(DELETED_MESSAGE); + } + + @Test + public void searchAllShouldReturnOldAndNewMessages() { + Mono.from(getVault().appendV1(DELETED_MESSAGE, new ByteArrayInputStream(CONTENT))).block(); + Mono.from(getVault().append(DELETED_MESSAGE_2, new ByteArrayInputStream(CONTENT))).block(); + + assertThat(Flux.from(getVault().search(USERNAME, ALL)).collectList().block()) + .containsOnly(DELETED_MESSAGE, DELETED_MESSAGE_2); + } + + @Test + public void searchAllShouldSupportLimitQueryWithOldAndNewMessages() { + Mono.from(getVault().appendV1(DELETED_MESSAGE, new ByteArrayInputStream(CONTENT))).block(); + Mono.from(getVault().appendV1(DELETED_MESSAGE_2, new ByteArrayInputStream(CONTENT))).block(); + DeletedMessage deletedMessage3 = DELETED_MESSAGE_GENERATOR.apply(InMemoryMessageId.of(33).getRawId()); + Mono.from(getVault().append(deletedMessage3, new ByteArrayInputStream(CONTENT))).block(); + + assertThat(Flux.from(getVault().search(USERNAME, Query.of(1, List.of()))).collectList().block()) + .hasSize(1); + assertThat(Flux.from(getVault().search(USERNAME, Query.of(3, List.of()))).collectList().block()) + .containsExactlyInAnyOrder(DELETED_MESSAGE, DELETED_MESSAGE_2, deletedMessage3); + assertThat(Flux.from(getVault().search(USERNAME, Query.of(4, List.of()))).collectList().block()) + .containsExactlyInAnyOrder(DELETED_MESSAGE, DELETED_MESSAGE_2, deletedMessage3); + } + + @Test + public void searchShouldReturnMatchingOldMessages() { + Mono.from(getVault().appendV1(DELETED_MESSAGE_2, new ByteArrayInputStream(CONTENT))).block(); + Mono.from(getVault().appendV1(DELETED_MESSAGE_WITH_SUBJECT, new ByteArrayInputStream(CONTENT))).block(); + + assertThat( + Flux.from(getVault().search(USERNAME, + Query.of(CriterionFactory.subject().containsIgnoreCase(SUBJECT)))) + .collectList().block()) + .containsOnly(DELETED_MESSAGE_WITH_SUBJECT); + } } \ No newline at end of file diff --git a/mailbox/plugin/deleted-messages-vault/src/test/java/org/apache/james/vault/blob/BlobStoreVaultGarbageCollectionTaskSerializationTest.java b/mailbox/plugin/deleted-messages-vault/src/test/java/org/apache/james/vault/blob/BlobStoreVaultGarbageCollectionTaskSerializationTest.java index 26c8cbeee57..162eb9fa84e 100644 --- a/mailbox/plugin/deleted-messages-vault/src/test/java/org/apache/james/vault/blob/BlobStoreVaultGarbageCollectionTaskSerializationTest.java +++ b/mailbox/plugin/deleted-messages-vault/src/test/java/org/apache/james/vault/blob/BlobStoreVaultGarbageCollectionTaskSerializationTest.java @@ -21,7 +21,6 @@ import static net.javacrumbs.jsonunit.assertj.JsonAssertions.assertThatJson; import static org.assertj.core.api.Assertions.assertThat; -import static org.mockito.ArgumentMatchers.any; import java.io.IOException; import java.time.Instant; @@ -38,8 +37,6 @@ import com.fasterxml.jackson.core.JsonProcessingException; import com.google.common.collect.ImmutableSet; -import reactor.core.publisher.Flux; - class BlobStoreVaultGarbageCollectionTaskSerializationTest { private static final BlobStoreDeletedMessageVault DELETED_MESSAGE_VAULT = Mockito.mock(BlobStoreDeletedMessageVault.class); private static final BlobStoreVaultGarbageCollectionTask.Factory TASK_FACTORY = new BlobStoreVaultGarbageCollectionTask.Factory(DELETED_MESSAGE_VAULT); @@ -47,13 +44,13 @@ class BlobStoreVaultGarbageCollectionTaskSerializationTest { private static final JsonTaskSerializer TASK_SERIALIZER = JsonTaskSerializer.of(BlobStoreVaultGarbageCollectionTaskDTO.module(TASK_FACTORY)); private static final ZonedDateTime BEGINNING_OF_RETENTION_PERIOD = ZonedDateTime.parse("2019-09-03T15:26:13.356+02:00[Europe/Paris]"); private static final ImmutableSet BUCKET_IDS = ImmutableSet.of(BucketName.of("1"), BucketName.of("2"), BucketName.of("3")); - private static final Flux RETENTION_OPERATION = Flux.fromIterable(BUCKET_IDS); + private static final int DELETED_BLOBS = 5; private static final Instant TIMESTAMP = Instant.parse("2018-11-13T12:00:55Z"); - private static final BlobStoreVaultGarbageCollectionTask.AdditionalInformation DETAILS = new BlobStoreVaultGarbageCollectionTask.AdditionalInformation(BEGINNING_OF_RETENTION_PERIOD, BUCKET_IDS, TIMESTAMP); + private static final BlobStoreVaultGarbageCollectionTask.AdditionalInformation DETAILS = new BlobStoreVaultGarbageCollectionTask.AdditionalInformation(BEGINNING_OF_RETENTION_PERIOD, BUCKET_IDS, DELETED_BLOBS, TIMESTAMP); private static final BlobStoreVaultGarbageCollectionTask TASK = TASK_FACTORY.create(); private static final String SERIALIZED_TASK = "{\"type\":\"deleted-messages-blob-store-based-garbage-collection\"}"; - private static final String SERIALIZED_ADDITIONAL_INFORMATION_TASK = "{\"type\":\"deleted-messages-blob-store-based-garbage-collection\", \"beginningOfRetentionPeriod\":\"2019-09-03T15:26:13.356+02:00[Europe/Paris]\",\"deletedBuckets\":[\"1\", \"2\", \"3\"], \"timestamp\": \"2018-11-13T12:00:55Z\"}"; + private static final String SERIALIZED_ADDITIONAL_INFORMATION_TASK = "{\"type\":\"deleted-messages-blob-store-based-garbage-collection\", \"beginningOfRetentionPeriod\":\"2019-09-03T15:26:13.356+02:00[Europe/Paris]\",\"deletedBuckets\":[\"1\", \"2\", \"3\"], \"deletedBlobs\":5, \"timestamp\": \"2018-11-13T12:00:55Z\"}"; private static final JsonTaskAdditionalInformationSerializer JSON_TASK_ADDITIONAL_INFORMATION_SERIALIZER = JsonTaskAdditionalInformationSerializer.of(BlobStoreVaultGarbageCollectionTaskAdditionalInformationDTO.module()); @@ -61,8 +58,6 @@ class BlobStoreVaultGarbageCollectionTaskSerializationTest { static void setUp() { Mockito.when(DELETED_MESSAGE_VAULT.getBeginningOfRetentionPeriod()) .thenReturn(BEGINNING_OF_RETENTION_PERIOD); - Mockito.when(DELETED_MESSAGE_VAULT.deleteExpiredMessages(any())) - .thenReturn(RETENTION_OPERATION); } @Test diff --git a/mailbox/plugin/deleted-messages-vault/src/test/java/org/apache/james/vault/dto/DeletedMessageWithStorageInformationDTOTest.java b/mailbox/plugin/deleted-messages-vault/src/test/java/org/apache/james/vault/dto/DeletedMessageWithStorageInformationDTOTest.java index a05ef7885ec..a9099a7e70b 100644 --- a/mailbox/plugin/deleted-messages-vault/src/test/java/org/apache/james/vault/dto/DeletedMessageWithStorageInformationDTOTest.java +++ b/mailbox/plugin/deleted-messages-vault/src/test/java/org/apache/james/vault/dto/DeletedMessageWithStorageInformationDTOTest.java @@ -31,7 +31,6 @@ import jakarta.mail.internet.AddressException; -import org.apache.james.blob.api.PlainBlobId; import org.apache.james.mailbox.inmemory.InMemoryId; import org.apache.james.mailbox.inmemory.InMemoryMessageId; import org.apache.james.vault.metadata.DeletedMessageWithStorageInformation; @@ -73,7 +72,6 @@ void setup() { .setSerializationInclusion(JsonInclude.Include.NON_ABSENT); this.converter = new DeletedMessageWithStorageInformationConverter( - new PlainBlobId.Factory(), new InMemoryMessageId.Factory(), new InMemoryId.Factory()); } diff --git a/server/protocols/webadmin-integration-test/webadmin-integration-test-common/src/main/java/org/apache/james/webadmin/integration/vault/DeletedMessageVaultIntegrationTest.java b/server/protocols/webadmin-integration-test/webadmin-integration-test-common/src/main/java/org/apache/james/webadmin/integration/vault/DeletedMessageVaultIntegrationTest.java index abd5315a5d3..ff2a392bcb9 100644 --- a/server/protocols/webadmin-integration-test/webadmin-integration-test-common/src/main/java/org/apache/james/webadmin/integration/vault/DeletedMessageVaultIntegrationTest.java +++ b/server/protocols/webadmin-integration-test/webadmin-integration-test-common/src/main/java/org/apache/james/webadmin/integration/vault/DeletedMessageVaultIntegrationTest.java @@ -76,6 +76,7 @@ import org.hamcrest.Matchers; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtensionContext; @@ -770,6 +771,7 @@ void vaultDeleteShouldNotAppendMessageToTheUserMailbox() { .hasSize(0); } + @Disabled("JAMES-4156") @Test void vaultDeleteShouldDeleteAllMessagesHavingSameBlobContent() throws Exception { bartSendMessageToHomerAndJack(); diff --git a/server/protocols/webadmin/webadmin-mailbox-deleted-message-vault/src/test/java/org/apache/james/webadmin/vault/routes/DeletedMessagesVaultRoutesTest.java b/server/protocols/webadmin/webadmin-mailbox-deleted-message-vault/src/test/java/org/apache/james/webadmin/vault/routes/DeletedMessagesVaultRoutesTest.java index 33b75394494..0ce1bfc8fe4 100644 --- a/server/protocols/webadmin/webadmin-mailbox-deleted-message-vault/src/test/java/org/apache/james/webadmin/vault/routes/DeletedMessagesVaultRoutesTest.java +++ b/server/protocols/webadmin/webadmin-mailbox-deleted-message-vault/src/test/java/org/apache/james/webadmin/vault/routes/DeletedMessagesVaultRoutesTest.java @@ -110,7 +110,6 @@ import org.apache.james.user.memory.MemoryUsersRepository; import org.apache.james.utils.UpdatableTickingClock; import org.apache.james.vault.DeletedMessage; -import org.apache.james.vault.DeletedMessageVault; import org.apache.james.vault.DeletedMessageZipper; import org.apache.james.vault.VaultConfiguration; import org.apache.james.vault.blob.BlobStoreDeletedMessageVault; @@ -140,6 +139,8 @@ class DeletedMessagesVaultRoutesTest { + private MemoryBlobStoreDAO blobStoreDAO; + private static class NoopBlobExporting implements BlobExportMechanism { private Optional exportedBlobId = Optional.empty(); @@ -157,7 +158,7 @@ public Optional getExportedBlobId() { } } - private static final ZonedDateTime NOW = ZonedDateTime.parse("2015-10-30T16:12:00Z"); + private static final ZonedDateTime NOW = ZonedDateTime.parse("2016-10-30T16:12:00Z"); private static final ZonedDateTime OLD_DELETION_DATE = ZonedDateTime.parse("2010-10-30T15:12:00Z"); private static final String MATCH_ALL_QUERY = "{" + "\"combinator\": \"and\"," + @@ -169,7 +170,7 @@ public Optional getExportedBlobId() { private static final String BOB_DELETE_PATH = BOB_PATH + SEPARATOR + DELETED_MESSAGE_PARAM_PATH; private WebAdminServer webAdminServer; - private DeletedMessageVault vault; + private BlobStoreDeletedMessageVault vault; private InMemoryMailboxManager mailboxManager; private MemoryTaskManager taskManager; private NoopBlobExporting blobExporting; @@ -183,16 +184,17 @@ public Optional getExportedBlobId() { @BeforeEach void beforeEach() throws Exception { blobIdFactory = new PlainBlobId.Factory(); - MemoryBlobStoreDAO blobStoreDAO = new MemoryBlobStoreDAO(); + this.blobStoreDAO = spy(new MemoryBlobStoreDAO()); blobStore = spy(BlobStoreFactory.builder() .blobStoreDAO(blobStoreDAO) .blobIdFactory(blobIdFactory) .defaultBucketName() .passthrough()); clock = new UpdatableTickingClock(OLD_DELETION_DATE.toInstant()); + usersRepository = createUsersRepository(); vault = spy(new BlobStoreDeletedMessageVault(new RecordingMetricFactory(), new MemoryDeletedMessageMetadataVault(), - blobStore, blobStoreDAO, new BucketNameGenerator(clock), clock, - VaultConfiguration.ENABLED_DEFAULT)); + blobStoreDAO, new BucketNameGenerator(clock), clock, + VaultConfiguration.ENABLED_DEFAULT, usersRepository)); InMemoryIntegrationResources inMemoryResource = InMemoryIntegrationResources.defaultResources(); mailboxManager = spy(inMemoryResource.getMailboxManager()); @@ -204,7 +206,6 @@ blobStore, blobStoreDAO, new BucketNameGenerator(clock), clock, zipper = new DeletedMessageZipper(); exportService = new ExportService(blobExporting, blobStore, zipper, vault); QueryTranslator queryTranslator = new QueryTranslator(new InMemoryId.Factory()); - usersRepository = createUsersRepository(); MessageId.Factory messageIdFactory = new InMemoryMessageId.Factory(); webAdminServer = WebAdminUtils.createWebAdminServer( new TasksRoutes(taskManager, jsonTransformer, @@ -2014,9 +2015,9 @@ void purgeShouldReturnATaskCreated() { } @Test - void purgeShouldProduceASuccessfulTaskWithAdditionalInformation() { - Mono.from(vault.append(DELETED_MESSAGE, new ByteArrayInputStream(CONTENT))).block(); - Mono.from(vault.append(DELETED_MESSAGE_2, new ByteArrayInputStream(CONTENT))).block(); + void oldPurgeShouldProduceASuccessfulTaskWithAdditionalInformation() { + Mono.from(vault.appendV1(DELETED_MESSAGE, new ByteArrayInputStream(CONTENT))).block(); + Mono.from(vault.appendV1(DELETED_MESSAGE_2, new ByteArrayInputStream(CONTENT))).block(); clock.setInstant(NOW.toInstant()); @@ -2037,13 +2038,19 @@ void purgeShouldProduceASuccessfulTaskWithAdditionalInformation() { .body("type", is("deleted-messages-blob-store-based-garbage-collection")) .body("additionalInformation.beginningOfRetentionPeriod", is(notNullValue())) .body("additionalInformation.deletedBuckets", contains("deleted-messages-2010-10-01")) + .body("additionalInformation.deletedBlobs", is(0)) .body("startedDate", is(notNullValue())) .body("submitDate", is(notNullValue())) .body("completedDate", is(notNullValue())); } @Test - void purgeShouldNotDeleteNotExpiredMessagesInTheVault() { + void oldPurgeShouldNotDeleteNotExpiredMessagesInTheVault() { + + Mono.from(vault.appendV1(DELETED_MESSAGE, new ByteArrayInputStream(CONTENT))).block(); + Mono.from(vault.appendV1(DELETED_MESSAGE_2, new ByteArrayInputStream(CONTENT))).block(); + + clock.setInstant(NOW.toInstant()); DeletedMessage notExpiredMessage = DeletedMessage.builder() .messageId(InMemoryMessageId.of(46)) .originMailboxes(MAILBOX_ID_1, MAILBOX_ID_2) @@ -2056,11 +2063,92 @@ void purgeShouldNotDeleteNotExpiredMessagesInTheVault() { .size(CONTENT.length) .build(); + Mono.from(vault.appendV1(notExpiredMessage, new ByteArrayInputStream(CONTENT))).block(); + + String taskId = + with() + .queryParam("scope", "expired") + .delete() + .jsonPath() + .get("taskId"); + + with() + .basePath(TasksRoutes.BASE) + .get(taskId + "/await"); + + assertThat(Flux.from(vault.search(USERNAME, Query.ALL)).toStream()) + .containsOnly(notExpiredMessage); + } + + @Test + void oldPurgeShouldNotAppendMessagesToUserMailbox() throws Exception { + Mono.from(vault.appendV1(DELETED_MESSAGE, new ByteArrayInputStream(CONTENT))).block(); + Mono.from(vault.appendV1(DELETED_MESSAGE_2, new ByteArrayInputStream(CONTENT))).block(); + + String taskId = + with() + .queryParam("scope", "expired") + .delete() + .jsonPath() + .get("taskId"); + + with() + .basePath(TasksRoutes.BASE) + .get(taskId + "/await"); + + assertThat(hasAnyMail(USERNAME)) + .isFalse(); + } + + @Test + void purgeShouldProduceASuccessfulTaskWithAdditionalInformation() { Mono.from(vault.append(DELETED_MESSAGE, new ByteArrayInputStream(CONTENT))).block(); Mono.from(vault.append(DELETED_MESSAGE_2, new ByteArrayInputStream(CONTENT))).block(); clock.setInstant(NOW.toInstant()); + String taskId = + with() + .queryParam("scope", "expired") + .delete() + .jsonPath() + .get("taskId"); + + given() + .basePath(TasksRoutes.BASE) + .when() + .get(taskId + "/await") + .then() + .body("status", is("completed")) + .body("taskId", is(taskId)) + .body("type", is("deleted-messages-blob-store-based-garbage-collection")) + .body("additionalInformation.beginningOfRetentionPeriod", is(notNullValue())) + .body("additionalInformation.deletedBuckets", hasSize(0)) + .body("additionalInformation.deletedBlobs", is(2)) + .body("startedDate", is(notNullValue())) + .body("submitDate", is(notNullValue())) + .body("completedDate", is(notNullValue())); + } + + @Test + void purgeShouldNotDeleteNotExpiredMessagesInTheVault() { + Mono.from(vault.append(DELETED_MESSAGE, new ByteArrayInputStream(CONTENT))).block(); + Mono.from(vault.append(DELETED_MESSAGE_2, new ByteArrayInputStream(CONTENT))).block(); + + clock.setInstant(NOW.toInstant()); + + DeletedMessage notExpiredMessage = DeletedMessage.builder() + .messageId(InMemoryMessageId.of(46)) + .originMailboxes(MAILBOX_ID_1, MAILBOX_ID_2) + .user(USERNAME) + .deliveryDate(DELIVERY_DATE) + .deletionDate(ZonedDateTime.ofInstant(clock.instant(), ZoneOffset.UTC)) + .sender(MaybeSender.of(SENDER)) + .recipients(RECIPIENT1, RECIPIENT3) + .hasAttachment(false) + .size(CONTENT.length) + .build(); + Mono.from(vault.append(notExpiredMessage, new ByteArrayInputStream(CONTENT))).block(); String taskId = @@ -2101,12 +2189,12 @@ void purgeShouldNotAppendMessagesToUserMailbox() throws Exception { @Nested class FailingPurgeTest { @Test - void purgeShouldProduceAFailedTaskWhenFailingDeletingBucket() { - Mono.from(vault.append(DELETED_MESSAGE, new ByteArrayInputStream(CONTENT))).block(); - Mono.from(vault.append(DELETED_MESSAGE_2, new ByteArrayInputStream(CONTENT))).block(); + void oldPurgeShouldProduceAFailedTaskWhenFailingDeletingBucket() { + Mono.from(vault.appendV1(DELETED_MESSAGE, new ByteArrayInputStream(CONTENT))).block(); + Mono.from(vault.appendV1(DELETED_MESSAGE_2, new ByteArrayInputStream(CONTENT))).block(); doReturn(Mono.error(new RuntimeException("mock exception"))) - .when(blobStore) + .when(blobStoreDAO) .deleteBucket(BucketName.of("deleted-messages-2010-10-01")); clock.setInstant(NOW.toInstant()); @@ -2128,6 +2216,7 @@ void purgeShouldProduceAFailedTaskWhenFailingDeletingBucket() { .body("type", is("deleted-messages-blob-store-based-garbage-collection")) .body("additionalInformation.beginningOfRetentionPeriod", is(notNullValue())) .body("additionalInformation.deletedBuckets", hasSize(0)) + .body("additionalInformation.deletedBlobs", is(0)) .body("startedDate", is(notNullValue())) .body("submitDate", is(notNullValue())) .body("completedDate", is(nullValue()));