From 6f18cc9ad384b758ac035ccaabb5e2b4218e39fc Mon Sep 17 00:00:00 2001 From: Rene Cordier Date: Mon, 5 Jan 2026 14:40:44 +0700 Subject: [PATCH 01/14] JAMES-4156 add a property for single bucket name in vault configuration --- .../james/vault/VaultConfiguration.java | 23 +++++++++++---- .../james/vault/VaultConfigurationTest.java | 28 +++++++++++++++---- 2 files changed, 39 insertions(+), 12 deletions(-) diff --git a/mailbox/plugin/deleted-messages-vault/src/main/java/org/apache/james/vault/VaultConfiguration.java b/mailbox/plugin/deleted-messages-vault/src/main/java/org/apache/james/vault/VaultConfiguration.java index c6e46fb0d82..8781128df63 100644 --- a/mailbox/plugin/deleted-messages-vault/src/main/java/org/apache/james/vault/VaultConfiguration.java +++ b/mailbox/plugin/deleted-messages-vault/src/main/java/org/apache/james/vault/VaultConfiguration.java @@ -31,10 +31,11 @@ import com.google.common.base.Preconditions; public class VaultConfiguration { + public static final String DEFAULT_SINGLE_BUCKET_NAME = "james-deleted-message-vault"; public static final VaultConfiguration DEFAULT = - new VaultConfiguration(false, ChronoUnit.YEARS.getDuration(), DefaultMailboxes.RESTORED_MESSAGES); + new VaultConfiguration(false, ChronoUnit.YEARS.getDuration(), DefaultMailboxes.RESTORED_MESSAGES, DEFAULT_SINGLE_BUCKET_NAME); public static final VaultConfiguration ENABLED_DEFAULT = - new VaultConfiguration(true, ChronoUnit.YEARS.getDuration(), DefaultMailboxes.RESTORED_MESSAGES); + new VaultConfiguration(true, ChronoUnit.YEARS.getDuration(), DefaultMailboxes.RESTORED_MESSAGES, DEFAULT_SINGLE_BUCKET_NAME); public static VaultConfiguration from(Configuration propertiesConfiguration) { Duration retentionPeriod = Optional.ofNullable(propertiesConfiguration.getString("retentionPeriod")) @@ -42,21 +43,26 @@ public static VaultConfiguration from(Configuration propertiesConfiguration) { .orElse(DEFAULT.getRetentionPeriod()); String restoreLocation = Optional.ofNullable(propertiesConfiguration.getString("restoreLocation")) .orElse(DEFAULT.getRestoreLocation()); + String singleBucketName = Optional.ofNullable(propertiesConfiguration.getString("singleBucketName")) + .orElse(DEFAULT.getSingleBucketName()); boolean enabled = propertiesConfiguration.getBoolean("enabled", false); - return new VaultConfiguration(enabled, retentionPeriod, restoreLocation); + return new VaultConfiguration(enabled, retentionPeriod, restoreLocation, singleBucketName); } private final boolean enabled; private final Duration retentionPeriod; private final String restoreLocation; + private final String singleBucketName; - VaultConfiguration(boolean enabled, Duration retentionPeriod, String restoreLocation) { + VaultConfiguration(boolean enabled, Duration retentionPeriod, String restoreLocation, String singleBucketName) { this.enabled = enabled; Preconditions.checkNotNull(retentionPeriod); Preconditions.checkNotNull(restoreLocation); + Preconditions.checkNotNull(singleBucketName); this.retentionPeriod = retentionPeriod; this.restoreLocation = restoreLocation; + this.singleBucketName = singleBucketName; } public boolean isEnabled() { @@ -71,6 +77,10 @@ public String getRestoreLocation() { return restoreLocation; } + public String getSingleBucketName() { + return singleBucketName; + } + @Override public final boolean equals(Object o) { if (o instanceof VaultConfiguration) { @@ -78,13 +88,14 @@ public final boolean equals(Object o) { return Objects.equals(this.retentionPeriod, that.retentionPeriod) && Objects.equals(this.restoreLocation, that.restoreLocation) - && Objects.equals(this.enabled, that.enabled); + && Objects.equals(this.enabled, that.enabled) + && Objects.equals(this.singleBucketName, that.singleBucketName); } return false; } @Override public final int hashCode() { - return Objects.hash(retentionPeriod, restoreLocation, enabled); + return Objects.hash(retentionPeriod, restoreLocation, enabled, singleBucketName); } } diff --git a/mailbox/plugin/deleted-messages-vault/src/test/java/org/apache/james/vault/VaultConfigurationTest.java b/mailbox/plugin/deleted-messages-vault/src/test/java/org/apache/james/vault/VaultConfigurationTest.java index 656af119d6a..25f13ddc51e 100644 --- a/mailbox/plugin/deleted-messages-vault/src/test/java/org/apache/james/vault/VaultConfigurationTest.java +++ b/mailbox/plugin/deleted-messages-vault/src/test/java/org/apache/james/vault/VaultConfigurationTest.java @@ -19,6 +19,7 @@ package org.apache.james.vault; +import static org.apache.james.vault.VaultConfiguration.DEFAULT_SINGLE_BUCKET_NAME; import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatThrownBy; @@ -40,13 +41,19 @@ void shouldMatchBeanContract() { @Test void constructorShouldThrowWhenRetentionPeriodIsNull() { - assertThatThrownBy(() -> new VaultConfiguration(true, null, DefaultMailboxes.RESTORED_MESSAGES)) + assertThatThrownBy(() -> new VaultConfiguration(true, null, DefaultMailboxes.RESTORED_MESSAGES, DEFAULT_SINGLE_BUCKET_NAME)) .isInstanceOf(NullPointerException.class); } @Test void constructorShouldThrowWhenRestoreLocationIsNull() { - assertThatThrownBy(() -> new VaultConfiguration(true, ChronoUnit.YEARS.getDuration(), null)) + assertThatThrownBy(() -> new VaultConfiguration(true, ChronoUnit.YEARS.getDuration(), null, DEFAULT_SINGLE_BUCKET_NAME)) + .isInstanceOf(NullPointerException.class); + } + + @Test + void constructorShouldThrowWhenSingleBucketNameIsNull() { + assertThatThrownBy(() -> new VaultConfiguration(true, ChronoUnit.YEARS.getDuration(), DefaultMailboxes.RESTORED_MESSAGES, null)) .isInstanceOf(NullPointerException.class); } @@ -62,7 +69,7 @@ void fromShouldReturnConfiguredRestoreLocation() { configuration.addProperty("restoreLocation", "INBOX"); assertThat(VaultConfiguration.from(configuration)).isEqualTo( - new VaultConfiguration(false, ChronoUnit.YEARS.getDuration(), DefaultMailboxes.INBOX)); + new VaultConfiguration(false, ChronoUnit.YEARS.getDuration(), DefaultMailboxes.INBOX, DEFAULT_SINGLE_BUCKET_NAME)); } @Test @@ -71,7 +78,7 @@ void fromShouldReturnConfiguredRetentionTime() { configuration.addProperty("retentionPeriod", "15d"); assertThat(VaultConfiguration.from(configuration)).isEqualTo( - new VaultConfiguration(false, Duration.ofDays(15), DefaultMailboxes.RESTORED_MESSAGES)); + new VaultConfiguration(false, Duration.ofDays(15), DefaultMailboxes.RESTORED_MESSAGES, DEFAULT_SINGLE_BUCKET_NAME)); } @Test @@ -80,7 +87,7 @@ void fromShouldHandleHours() { configuration.addProperty("retentionPeriod", "15h"); assertThat(VaultConfiguration.from(configuration)).isEqualTo( - new VaultConfiguration(false, Duration.ofHours(15), DefaultMailboxes.RESTORED_MESSAGES)); + new VaultConfiguration(false, Duration.ofHours(15), DefaultMailboxes.RESTORED_MESSAGES, DEFAULT_SINGLE_BUCKET_NAME)); } @Test @@ -89,7 +96,16 @@ void fromShouldUseDaysAsADefaultUnit() { configuration.addProperty("retentionPeriod", "15"); assertThat(VaultConfiguration.from(configuration)).isEqualTo( - new VaultConfiguration(false, Duration.ofDays(15), DefaultMailboxes.RESTORED_MESSAGES)); + new VaultConfiguration(false, Duration.ofDays(15), DefaultMailboxes.RESTORED_MESSAGES, DEFAULT_SINGLE_BUCKET_NAME)); + } + + @Test + void fromShouldReturnConfiguredSingleBucketName() { + PropertiesConfiguration configuration = new PropertiesConfiguration(); + configuration.addProperty("singleBucketName", "bucketBlabla"); + + assertThat(VaultConfiguration.from(configuration)).isEqualTo( + new VaultConfiguration(false, ChronoUnit.YEARS.getDuration(), DefaultMailboxes.RESTORED_MESSAGES, "bucketBlabla")); } @Test From 4591193352c02b88250ee9779aa80ef53d65eef2 Mon Sep 17 00:00:00 2001 From: Rene Cordier Date: Tue, 6 Jan 2026 14:36:48 +0700 Subject: [PATCH 02/14] JAMES-4156 add a BlobIdTimeGenerator --- .../james/vault/blob/BlobIdTimeGenerator.java | 50 ++++++++++++ .../vault/blob/BlobIdTimeGeneratorTest.java | 78 +++++++++++++++++++ 2 files changed, 128 insertions(+) create mode 100644 mailbox/plugin/deleted-messages-vault/src/main/java/org/apache/james/vault/blob/BlobIdTimeGenerator.java create mode 100644 mailbox/plugin/deleted-messages-vault/src/test/java/org/apache/james/vault/blob/BlobIdTimeGeneratorTest.java diff --git a/mailbox/plugin/deleted-messages-vault/src/main/java/org/apache/james/vault/blob/BlobIdTimeGenerator.java b/mailbox/plugin/deleted-messages-vault/src/main/java/org/apache/james/vault/blob/BlobIdTimeGenerator.java new file mode 100644 index 00000000000..c84c18c8cf8 --- /dev/null +++ b/mailbox/plugin/deleted-messages-vault/src/main/java/org/apache/james/vault/blob/BlobIdTimeGenerator.java @@ -0,0 +1,50 @@ +/**************************************************************** + * Licensed to the Apache Software Foundation (ASF) under one * + * or more contributor license agreements. See the NOTICE file * + * distributed with this work for additional information * + * regarding copyright ownership. The ASF licenses this file * + * to you under the Apache License, Version 2.0 (the * + * "License"); you may not use this file except in compliance * + * with the License. You may obtain a copy of the License at * + * * + * http://www.apache.org/licenses/LICENSE-2.0 * + * * + * Unless required by applicable law or agreed to in writing, * + * software distributed under the License is distributed on an * + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY * + * KIND, either express or implied. See the License for the * + * specific language governing permissions and limitations * + * under the License. * + ****************************************************************/ + +package org.apache.james.vault.blob; + +import java.time.Clock; +import java.time.ZonedDateTime; +import java.util.UUID; + +import jakarta.inject.Inject; + +import org.apache.james.blob.api.BlobId; +import org.apache.james.blob.api.PlainBlobId; + +public class BlobIdTimeGenerator { + private static final String BLOB_ID_GENERATING_FORMAT = "%d/%02d/%s"; + + private final BlobId.Factory blobIdFactory; + private final Clock clock; + + @Inject + public BlobIdTimeGenerator(BlobId.Factory blobIdFactory, Clock clock) { + this.blobIdFactory = blobIdFactory; + this.clock = clock; + } + + BlobId currentBlobId() { + ZonedDateTime now = ZonedDateTime.now(clock); + int month = now.getMonthValue(); + int year = now.getYear(); + + return new PlainBlobId(String.format(BLOB_ID_GENERATING_FORMAT, year, month, blobIdFactory.of(UUID.randomUUID().toString()).asString())); + } +} diff --git a/mailbox/plugin/deleted-messages-vault/src/test/java/org/apache/james/vault/blob/BlobIdTimeGeneratorTest.java b/mailbox/plugin/deleted-messages-vault/src/test/java/org/apache/james/vault/blob/BlobIdTimeGeneratorTest.java new file mode 100644 index 00000000000..9391da5718b --- /dev/null +++ b/mailbox/plugin/deleted-messages-vault/src/test/java/org/apache/james/vault/blob/BlobIdTimeGeneratorTest.java @@ -0,0 +1,78 @@ +/**************************************************************** + * Licensed to the Apache Software Foundation (ASF) under one * + * or more contributor license agreements. See the NOTICE file * + * distributed with this work for additional information * + * regarding copyright ownership. The ASF licenses this file * + * to you under the Apache License, Version 2.0 (the * + * "License"); you may not use this file except in compliance * + * with the License. You may obtain a copy of the License at * + * * + * http://www.apache.org/licenses/LICENSE-2.0 * + * * + * Unless required by applicable law or agreed to in writing, * + * software distributed under the License is distributed on an * + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY * + * KIND, either express or implied. See the License for the * + * specific language governing permissions and limitations * + * under the License. * + ****************************************************************/ + +package org.apache.james.vault.blob; + +import static org.assertj.core.api.Assertions.assertThat; + +import java.time.Clock; +import java.time.Instant; + +import org.apache.james.blob.api.BlobId; +import org.apache.james.blob.api.PlainBlobId; +import org.apache.james.server.blob.deduplication.GenerationAwareBlobId; +import org.apache.james.server.blob.deduplication.MinIOGenerationAwareBlobId; +import org.apache.james.utils.UpdatableTickingClock; +import org.junit.jupiter.api.Nested; +import org.junit.jupiter.api.Test; + +public class BlobIdTimeGeneratorTest { + private static final Instant NOW = Instant.parse("2007-07-03T10:15:30.00Z"); + private static final Clock CLOCK = new UpdatableTickingClock(NOW); + + interface BlobIdTimeGeneratorContract { + BlobId.Factory blobIdFactory(); + + @Test + default void currentBlobIdShouldReturnBlobIdFormattedWithYearAndMonthPrefix() { + BlobIdTimeGenerator blobIdTimeGenerator = new BlobIdTimeGenerator(blobIdFactory(), CLOCK); + String currentBlobId = blobIdTimeGenerator.currentBlobId().asString(); + + int firstSlash = currentBlobId.indexOf('/'); + int secondSlash = currentBlobId.indexOf('/', firstSlash + 1); + String prefix = currentBlobId.substring(0, secondSlash); + + assertThat(prefix).isEqualTo("2007/07"); + } + } + + @Nested + class PlainBlobIdTimeGeneratorTest implements BlobIdTimeGeneratorContract { + @Override + public BlobId.Factory blobIdFactory() { + return new PlainBlobId.Factory(); + } + } + + @Nested + class GenerationAwareBlobIdTimeGeneratorTest implements BlobIdTimeGeneratorContract { + @Override + public BlobId.Factory blobIdFactory() { + return new GenerationAwareBlobId.Factory(CLOCK, new PlainBlobId.Factory(), GenerationAwareBlobId.Configuration.DEFAULT); + } + } + + @Nested + class MinIOGenerationAwareBlobIdTimeGeneratorTest implements BlobIdTimeGeneratorContract { + @Override + public BlobId.Factory blobIdFactory() { + return new MinIOGenerationAwareBlobId.Factory(CLOCK, GenerationAwareBlobId.Configuration.DEFAULT, new PlainBlobId.Factory()); + } + } +} From 51f4491470713e470c1ba7d1ff8a03bb2cca1d17 Mon Sep 17 00:00:00 2001 From: Rene Cordier Date: Wed, 7 Jan 2026 15:58:44 +0700 Subject: [PATCH 03/14] JAMES-4156 append deleted messages to the new deleted message vault single bucket Keep old code as appendV1 for testing retro-compatibility --- .../blob/BlobStoreDeletedMessageVault.java | 37 +++++++- .../vault/DeletedMessageVaultContract.java | 17 ++-- .../vault/DeletedMessageVaultHookTest.java | 6 +- .../BlobStoreDeletedMessageVaultTest.java | 91 +++++++++++++++++-- .../DeletedMessagesVaultRoutesTest.java | 23 ++--- 5 files changed, 142 insertions(+), 32 deletions(-) diff --git a/mailbox/plugin/deleted-messages-vault/src/main/java/org/apache/james/vault/blob/BlobStoreDeletedMessageVault.java b/mailbox/plugin/deleted-messages-vault/src/main/java/org/apache/james/vault/blob/BlobStoreDeletedMessageVault.java index b7556ca5d93..f3da89571b5 100644 --- a/mailbox/plugin/deleted-messages-vault/src/main/java/org/apache/james/vault/blob/BlobStoreDeletedMessageVault.java +++ b/mailbox/plugin/deleted-messages-vault/src/main/java/org/apache/james/vault/blob/BlobStoreDeletedMessageVault.java @@ -54,6 +54,7 @@ import reactor.core.publisher.Flux; import reactor.core.publisher.Mono; +import reactor.util.function.Tuples; public class BlobStoreDeletedMessageVault implements DeletedMessageVault { private static final Logger LOGGER = LoggerFactory.getLogger(BlobStoreDeletedMessageVault.class); @@ -72,12 +73,13 @@ public class BlobStoreDeletedMessageVault implements DeletedMessageVault { private final BucketNameGenerator nameGenerator; private final Clock clock; private final VaultConfiguration vaultConfiguration; + private final BlobIdTimeGenerator blobIdTimeGenerator; private final BlobStoreVaultGarbageCollectionTask.Factory taskFactory; @Inject public BlobStoreDeletedMessageVault(MetricFactory metricFactory, DeletedMessageMetadataVault messageMetadataVault, BlobStore blobStore, BlobStoreDAO blobStoreDAO, BucketNameGenerator nameGenerator, - Clock clock, + Clock clock, BlobIdTimeGenerator blobIdTimeGenerator, VaultConfiguration vaultConfiguration) { this.metricFactory = metricFactory; this.messageMetadataVault = messageMetadataVault; @@ -86,14 +88,37 @@ public BlobStoreDeletedMessageVault(MetricFactory metricFactory, DeletedMessageM this.nameGenerator = nameGenerator; this.clock = clock; this.vaultConfiguration = vaultConfiguration; + this.blobIdTimeGenerator = blobIdTimeGenerator; this.taskFactory = new BlobStoreVaultGarbageCollectionTask.Factory(this); } + @Deprecated + @VisibleForTesting + public Publisher appendV1(DeletedMessage deletedMessage, InputStream mimeMessage) { + Preconditions.checkNotNull(deletedMessage); + Preconditions.checkNotNull(mimeMessage); + BucketName bucketName = nameGenerator.currentBucket(); + + return metricFactory.decoratePublisherWithTimerMetric( + APPEND_METRIC_NAME, + appendMessageV1(deletedMessage, mimeMessage, bucketName)); + } + + private Mono appendMessageV1(DeletedMessage deletedMessage, InputStream mimeMessage, BucketName bucketName) { + return Mono.from(blobStore.save(bucketName, mimeMessage, LOW_COST)) + .map(blobId -> StorageInformation.builder() + .bucketName(bucketName) + .blobId(blobId)) + .map(storageInformation -> new DeletedMessageWithStorageInformation(deletedMessage, storageInformation)) + .flatMap(message -> Mono.from(messageMetadataVault.store(message))) + .then(); + } + @Override public Publisher append(DeletedMessage deletedMessage, InputStream mimeMessage) { Preconditions.checkNotNull(deletedMessage); Preconditions.checkNotNull(mimeMessage); - BucketName bucketName = nameGenerator.currentBucket(); + BucketName bucketName = BucketName.of(vaultConfiguration.getSingleBucketName()); return metricFactory.decoratePublisherWithTimerMetric( APPEND_METRIC_NAME, @@ -101,7 +126,7 @@ public Publisher append(DeletedMessage deletedMessage, InputStream mimeMes } private Mono appendMessage(DeletedMessage deletedMessage, InputStream mimeMessage, BucketName bucketName) { - return Mono.from(blobStore.save(bucketName, mimeMessage, LOW_COST)) + return Mono.from(blobStore.save(bucketName, mimeMessage, withTimePrefixBlobId(), LOW_COST)) .map(blobId -> StorageInformation.builder() .bucketName(bucketName) .blobId(blobId)) @@ -110,6 +135,12 @@ private Mono appendMessage(DeletedMessage deletedMessage, InputStream mime .then(); } + private BlobStore.BlobIdProvider withTimePrefixBlobId() { + return data -> Mono.just(Tuples.of( + blobIdTimeGenerator.currentBlobId(), + data)); + } + @Override public Publisher loadMimeMessage(Username username, MessageId messageId) { Preconditions.checkNotNull(username); diff --git a/mailbox/plugin/deleted-messages-vault/src/test/java/org/apache/james/vault/DeletedMessageVaultContract.java b/mailbox/plugin/deleted-messages-vault/src/test/java/org/apache/james/vault/DeletedMessageVaultContract.java index cf82d47a4a3..3524715a5af 100644 --- a/mailbox/plugin/deleted-messages-vault/src/test/java/org/apache/james/vault/DeletedMessageVaultContract.java +++ b/mailbox/plugin/deleted-messages-vault/src/test/java/org/apache/james/vault/DeletedMessageVaultContract.java @@ -43,6 +43,7 @@ import org.apache.james.task.Task; import org.apache.james.util.concurrency.ConcurrentTestRunner; import org.apache.james.utils.UpdatableTickingClock; +import org.apache.james.vault.blob.BlobStoreDeletedMessageVault; import org.apache.james.vault.search.CriterionFactory; import org.apache.james.vault.search.Query; import org.junit.jupiter.api.Test; @@ -53,7 +54,7 @@ public interface DeletedMessageVaultContract { Clock CLOCK = Clock.fixed(NOW.toInstant(), NOW.getZone()); - DeletedMessageVault getVault(); + BlobStoreDeletedMessageVault getVault(); UpdatableTickingClock getClock(); @@ -243,7 +244,7 @@ default void deleteExpiredMessagesTaskShouldCompleteWhenNoMail() throws Interrup @Test default void deleteExpiredMessagesTaskShouldCompleteWhenAllMailsDeleted() throws InterruptedException { - Mono.from(getVault().append(DELETED_MESSAGE, new ByteArrayInputStream(CONTENT))).block(); + Mono.from(getVault().appendV1(DELETED_MESSAGE, new ByteArrayInputStream(CONTENT))).block(); Mono.from(getVault().delete(USERNAME, DELETED_MESSAGE.getMessageId())).block(); Task.Result result = getVault().deleteExpiredMessagesTask().run(); @@ -253,7 +254,7 @@ default void deleteExpiredMessagesTaskShouldCompleteWhenAllMailsDeleted() throws @Test default void deleteExpiredMessagesTaskShouldCompleteWhenOnlyRecentMails() throws InterruptedException { - Mono.from(getVault().append(DELETED_MESSAGE, new ByteArrayInputStream(CONTENT))).block(); + Mono.from(getVault().appendV1(DELETED_MESSAGE, new ByteArrayInputStream(CONTENT))).block(); Task.Result result = getVault().deleteExpiredMessagesTask().run(); @@ -262,7 +263,7 @@ default void deleteExpiredMessagesTaskShouldCompleteWhenOnlyRecentMails() throws @Test default void deleteExpiredMessagesTaskShouldCompleteWhenOnlyOldMails() throws InterruptedException { - Mono.from(getVault().append(OLD_DELETED_MESSAGE, new ByteArrayInputStream(CONTENT))).block(); + Mono.from(getVault().appendV1(OLD_DELETED_MESSAGE, new ByteArrayInputStream(CONTENT))).block(); Task.Result result = getVault().deleteExpiredMessagesTask().run(); @@ -279,7 +280,7 @@ default void deleteExpiredMessagesTaskShouldDoNothingWhenEmpty() throws Interrup @Test default void deleteExpiredMessagesTaskShouldNotDeleteRecentMails() throws InterruptedException { - Mono.from(getVault().append(DELETED_MESSAGE, new ByteArrayInputStream(CONTENT))).block(); + Mono.from(getVault().appendV1(DELETED_MESSAGE, new ByteArrayInputStream(CONTENT))).block(); getVault().deleteExpiredMessagesTask().run(); @@ -289,7 +290,7 @@ default void deleteExpiredMessagesTaskShouldNotDeleteRecentMails() throws Interr @Test default void deleteExpiredMessagesTaskShouldDeleteOldMails() throws InterruptedException { - Mono.from(getVault().append(OLD_DELETED_MESSAGE, new ByteArrayInputStream(CONTENT))).block(); + Mono.from(getVault().appendV1(OLD_DELETED_MESSAGE, new ByteArrayInputStream(CONTENT))).block(); getClock().setInstant(NOW.plusYears(2).toInstant()); getVault().deleteExpiredMessagesTask().run(); @@ -300,11 +301,11 @@ default void deleteExpiredMessagesTaskShouldDeleteOldMails() throws InterruptedE @Test default void deleteExpiredMessagesTaskShouldDeleteOldMailsWhenRunSeveralTime() throws InterruptedException { - Mono.from(getVault().append(OLD_DELETED_MESSAGE, new ByteArrayInputStream(CONTENT))).block(); + Mono.from(getVault().appendV1(OLD_DELETED_MESSAGE, new ByteArrayInputStream(CONTENT))).block(); getClock().setInstant(NOW.plusYears(2).toInstant()); getVault().deleteExpiredMessagesTask().run(); - Mono.from(getVault().append(OLD_DELETED_MESSAGE, new ByteArrayInputStream(CONTENT))).block(); + Mono.from(getVault().appendV1(OLD_DELETED_MESSAGE, new ByteArrayInputStream(CONTENT))).block(); getClock().setInstant(NOW.plusYears(4).toInstant()); getVault().deleteExpiredMessagesTask().run(); diff --git a/mailbox/plugin/deleted-messages-vault/src/test/java/org/apache/james/vault/DeletedMessageVaultHookTest.java b/mailbox/plugin/deleted-messages-vault/src/test/java/org/apache/james/vault/DeletedMessageVaultHookTest.java index 9149599fbaf..320dcbd2a23 100644 --- a/mailbox/plugin/deleted-messages-vault/src/test/java/org/apache/james/vault/DeletedMessageVaultHookTest.java +++ b/mailbox/plugin/deleted-messages-vault/src/test/java/org/apache/james/vault/DeletedMessageVaultHookTest.java @@ -31,6 +31,7 @@ import java.util.List; import java.util.stream.IntStream; +import org.apache.james.blob.api.BlobId; import org.apache.james.blob.api.PlainBlobId; import org.apache.james.blob.memory.MemoryBlobStoreDAO; import org.apache.james.core.MailAddress; @@ -53,6 +54,7 @@ import org.apache.james.metrics.tests.RecordingMetricFactory; import org.apache.james.mime4j.dom.Message; import org.apache.james.server.blob.deduplication.BlobStoreFactory; +import org.apache.james.vault.blob.BlobIdTimeGenerator; import org.apache.james.vault.blob.BlobStoreDeletedMessageVault; import org.apache.james.vault.blob.BucketNameGenerator; import org.apache.james.vault.memory.metadata.MemoryDeletedMessageMetadataVault; @@ -115,12 +117,14 @@ private ComposedMessageId appendMessage(MessageManager messageManager) throws Ex void setUp() throws Exception { clock = Clock.fixed(DELETION_DATE.toInstant(), ZoneOffset.UTC); MemoryBlobStoreDAO blobStoreDAO = new MemoryBlobStoreDAO(); + BlobId.Factory blobIdFactory = new PlainBlobId.Factory(); messageVault = new BlobStoreDeletedMessageVault(new RecordingMetricFactory(), new MemoryDeletedMessageMetadataVault(), BlobStoreFactory.builder() .blobStoreDAO(blobStoreDAO) - .blobIdFactory(new PlainBlobId.Factory()) + .blobIdFactory(blobIdFactory) .defaultBucketName() .passthrough(), blobStoreDAO, new BucketNameGenerator(clock), clock, + new BlobIdTimeGenerator(blobIdFactory, clock), VaultConfiguration.ENABLED_DEFAULT); DeletedMessageConverter deletedMessageConverter = new DeletedMessageConverter(); diff --git a/mailbox/plugin/deleted-messages-vault/src/test/java/org/apache/james/vault/blob/BlobStoreDeletedMessageVaultTest.java b/mailbox/plugin/deleted-messages-vault/src/test/java/org/apache/james/vault/blob/BlobStoreDeletedMessageVaultTest.java index 328a5206607..873124e0b2b 100644 --- a/mailbox/plugin/deleted-messages-vault/src/test/java/org/apache/james/vault/blob/BlobStoreDeletedMessageVaultTest.java +++ b/mailbox/plugin/deleted-messages-vault/src/test/java/org/apache/james/vault/blob/BlobStoreDeletedMessageVaultTest.java @@ -22,9 +22,12 @@ import static org.apache.james.vault.DeletedMessageFixture.CONTENT; import static org.apache.james.vault.DeletedMessageFixture.DELETED_MESSAGE; import static org.apache.james.vault.DeletedMessageFixture.DELETED_MESSAGE_2; +import static org.apache.james.vault.DeletedMessageFixture.DELETED_MESSAGE_GENERATOR; +import static org.apache.james.vault.DeletedMessageFixture.DELETED_MESSAGE_WITH_SUBJECT; import static org.apache.james.vault.DeletedMessageFixture.MESSAGE_ID; import static org.apache.james.vault.DeletedMessageFixture.NOW; import static org.apache.james.vault.DeletedMessageFixture.OLD_DELETED_MESSAGE; +import static org.apache.james.vault.DeletedMessageFixture.SUBJECT; import static org.apache.james.vault.DeletedMessageFixture.USERNAME; import static org.apache.james.vault.blob.BlobStoreDeletedMessageVault.APPEND_METRIC_NAME; import static org.apache.james.vault.blob.BlobStoreDeletedMessageVault.DELETE_EXPIRED_MESSAGES_METRIC_NAME; @@ -37,24 +40,29 @@ import java.io.ByteArrayInputStream; import java.time.Instant; import java.time.ZonedDateTime; +import java.util.List; +import org.apache.james.blob.api.BlobId; import org.apache.james.blob.api.BucketName; import org.apache.james.blob.api.PlainBlobId; import org.apache.james.blob.memory.MemoryBlobStoreDAO; +import org.apache.james.mailbox.inmemory.InMemoryMessageId; import org.apache.james.metrics.tests.RecordingMetricFactory; import org.apache.james.server.blob.deduplication.BlobStoreFactory; import org.apache.james.utils.UpdatableTickingClock; -import org.apache.james.vault.DeletedMessageVault; +import org.apache.james.vault.DeletedMessage; import org.apache.james.vault.DeletedMessageVaultContract; import org.apache.james.vault.DeletedMessageVaultSearchContract; import org.apache.james.vault.VaultConfiguration; import org.apache.james.vault.memory.metadata.MemoryDeletedMessageMetadataVault; +import org.apache.james.vault.search.CriterionFactory; +import org.apache.james.vault.search.Query; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; +import reactor.core.publisher.Flux; import reactor.core.publisher.Mono; - class BlobStoreDeletedMessageVaultTest implements DeletedMessageVaultContract, DeletedMessageVaultSearchContract.AllContracts { private BlobStoreDeletedMessageVault messageVault; private UpdatableTickingClock clock; @@ -65,17 +73,19 @@ void setUp() { clock = new UpdatableTickingClock(NOW.toInstant()); metricFactory = new RecordingMetricFactory(); MemoryBlobStoreDAO blobStoreDAO = new MemoryBlobStoreDAO(); + BlobId.Factory blobIdFactory = new PlainBlobId.Factory(); messageVault = new BlobStoreDeletedMessageVault(metricFactory, new MemoryDeletedMessageMetadataVault(), BlobStoreFactory.builder() .blobStoreDAO(blobStoreDAO) - .blobIdFactory(new PlainBlobId.Factory()) + .blobIdFactory(blobIdFactory) .defaultBucketName() .passthrough(), - blobStoreDAO, new BucketNameGenerator(clock), clock, VaultConfiguration.ENABLED_DEFAULT); + blobStoreDAO, new BucketNameGenerator(clock), clock, new BlobIdTimeGenerator(blobIdFactory, clock), + VaultConfiguration.ENABLED_DEFAULT); } @Override - public DeletedMessageVault getVault() { + public BlobStoreDeletedMessageVault getVault() { return messageVault; } @@ -87,9 +97,9 @@ public UpdatableTickingClock getClock() { @Test void retentionQualifiedBucketsShouldReturnOnlyBucketsFullyBeforeBeginningOfRetentionPeriod() { clock.setInstant(Instant.parse("2007-12-03T10:15:30.00Z")); - Mono.from(getVault().append(OLD_DELETED_MESSAGE, new ByteArrayInputStream(CONTENT))).block(); + Mono.from(getVault().appendV1(OLD_DELETED_MESSAGE, new ByteArrayInputStream(CONTENT))).block(); clock.setInstant(Instant.parse("2008-01-03T10:15:30.00Z")); - Mono.from(getVault().append(DELETED_MESSAGE_2, new ByteArrayInputStream(CONTENT))).block(); + Mono.from(getVault().appendV1(DELETED_MESSAGE_2, new ByteArrayInputStream(CONTENT))).block(); ZonedDateTime beginningOfRetention = ZonedDateTime.parse("2008-01-30T10:15:30.00Z"); assertThat(messageVault.retentionQualifiedBuckets(beginningOfRetention).toStream()) @@ -99,9 +109,9 @@ void retentionQualifiedBucketsShouldReturnOnlyBucketsFullyBeforeBeginningOfReten @Test void retentionQualifiedBucketsShouldReturnAllWhenAllBucketMonthAreBeforeBeginningOfRetention() { clock.setInstant(Instant.parse("2007-12-03T10:15:30.00Z")); - Mono.from(getVault().append(OLD_DELETED_MESSAGE, new ByteArrayInputStream(CONTENT))).block(); + Mono.from(getVault().appendV1(OLD_DELETED_MESSAGE, new ByteArrayInputStream(CONTENT))).block(); clock.setInstant(Instant.parse("2008-01-30T10:15:30.00Z")); - Mono.from(getVault().append(DELETED_MESSAGE_2, new ByteArrayInputStream(CONTENT))).block(); + Mono.from(getVault().appendV1(DELETED_MESSAGE_2, new ByteArrayInputStream(CONTENT))).block(); assertThat(messageVault.retentionQualifiedBuckets(ZonedDateTime.parse("2008-02-01T10:15:30.00Z")).toStream()) .containsOnly( @@ -163,4 +173,67 @@ void deleteExpiredMessagesTaskShouldPublishRetentionTimerMetrics() throws Except assertThat(metricFactory.executionTimesFor(DELETE_EXPIRED_MESSAGES_METRIC_NAME)) .hasSize(1); } + + @Test + public void loadMimeMessageShouldReturnOldMessage() { + Mono.from(getVault().appendV1(DELETED_MESSAGE, new ByteArrayInputStream(CONTENT))).block(); + + assertThat(Mono.from(getVault().loadMimeMessage(USERNAME, MESSAGE_ID)).blockOptional()) + .isNotEmpty() + .satisfies(maybeContent -> assertThat(maybeContent.get()).hasSameContentAs(new ByteArrayInputStream(CONTENT))); + } + + @Test + public void loadMimeMessageShouldReturnEmptyWhenOldMessageDeleted() { + Mono.from(getVault().appendV1(DELETED_MESSAGE, new ByteArrayInputStream(CONTENT))).block(); + + Mono.from(getVault().delete(USERNAME, MESSAGE_ID)).block(); + + assertThat(Mono.from(getVault().loadMimeMessage(USERNAME, MESSAGE_ID)).blockOptional()) + .isEmpty(); + } + + @Test + public void searchAllShouldReturnOldMessage() { + Mono.from(getVault().appendV1(DELETED_MESSAGE, new ByteArrayInputStream(CONTENT))).block(); + + assertThat(Flux.from(getVault().search(USERNAME, ALL)).collectList().block()) + .containsOnly(DELETED_MESSAGE); + } + + @Test + public void searchAllShouldReturnOldAndNewMessages() { + Mono.from(getVault().appendV1(DELETED_MESSAGE, new ByteArrayInputStream(CONTENT))).block(); + Mono.from(getVault().appendV1(DELETED_MESSAGE_2, new ByteArrayInputStream(CONTENT))).block(); + + assertThat(Flux.from(getVault().search(USERNAME, ALL)).collectList().block()) + .containsOnly(DELETED_MESSAGE, DELETED_MESSAGE_2); + } + + @Test + public void searchAllShouldSupportLimitQueryWithOldAndNewMessages() { + Mono.from(getVault().appendV1(DELETED_MESSAGE, new ByteArrayInputStream(CONTENT))).block(); + Mono.from(getVault().appendV1(DELETED_MESSAGE_2, new ByteArrayInputStream(CONTENT))).block(); + DeletedMessage deletedMessage3 = DELETED_MESSAGE_GENERATOR.apply(InMemoryMessageId.of(33).getRawId()); + Mono.from(getVault().appendV1(deletedMessage3, new ByteArrayInputStream(CONTENT))).block(); + + assertThat(Flux.from(getVault().search(USERNAME, Query.of(1, List.of()))).collectList().block()) + .hasSize(1); + assertThat(Flux.from(getVault().search(USERNAME, Query.of(3, List.of()))).collectList().block()) + .containsExactlyInAnyOrder(DELETED_MESSAGE, DELETED_MESSAGE_2, deletedMessage3); + assertThat(Flux.from(getVault().search(USERNAME, Query.of(4, List.of()))).collectList().block()) + .containsExactlyInAnyOrder(DELETED_MESSAGE, DELETED_MESSAGE_2, deletedMessage3); + } + + @Test + public void searchShouldReturnMatchingOldMessages() { + Mono.from(getVault().appendV1(DELETED_MESSAGE_2, new ByteArrayInputStream(CONTENT))).block(); + Mono.from(getVault().appendV1(DELETED_MESSAGE_WITH_SUBJECT, new ByteArrayInputStream(CONTENT))).block(); + + assertThat( + Flux.from(getVault().search(USERNAME, + Query.of(CriterionFactory.subject().containsIgnoreCase(SUBJECT)))) + .collectList().block()) + .containsOnly(DELETED_MESSAGE_WITH_SUBJECT); + } } \ No newline at end of file diff --git a/server/protocols/webadmin/webadmin-mailbox-deleted-message-vault/src/test/java/org/apache/james/webadmin/vault/routes/DeletedMessagesVaultRoutesTest.java b/server/protocols/webadmin/webadmin-mailbox-deleted-message-vault/src/test/java/org/apache/james/webadmin/vault/routes/DeletedMessagesVaultRoutesTest.java index 33b75394494..c795ae763e5 100644 --- a/server/protocols/webadmin/webadmin-mailbox-deleted-message-vault/src/test/java/org/apache/james/webadmin/vault/routes/DeletedMessagesVaultRoutesTest.java +++ b/server/protocols/webadmin/webadmin-mailbox-deleted-message-vault/src/test/java/org/apache/james/webadmin/vault/routes/DeletedMessagesVaultRoutesTest.java @@ -110,9 +110,9 @@ import org.apache.james.user.memory.MemoryUsersRepository; import org.apache.james.utils.UpdatableTickingClock; import org.apache.james.vault.DeletedMessage; -import org.apache.james.vault.DeletedMessageVault; import org.apache.james.vault.DeletedMessageZipper; import org.apache.james.vault.VaultConfiguration; +import org.apache.james.vault.blob.BlobIdTimeGenerator; import org.apache.james.vault.blob.BlobStoreDeletedMessageVault; import org.apache.james.vault.blob.BlobStoreVaultGarbageCollectionTaskAdditionalInformationDTO; import org.apache.james.vault.blob.BucketNameGenerator; @@ -169,7 +169,7 @@ public Optional getExportedBlobId() { private static final String BOB_DELETE_PATH = BOB_PATH + SEPARATOR + DELETED_MESSAGE_PARAM_PATH; private WebAdminServer webAdminServer; - private DeletedMessageVault vault; + private BlobStoreDeletedMessageVault vault; private InMemoryMailboxManager mailboxManager; private MemoryTaskManager taskManager; private NoopBlobExporting blobExporting; @@ -192,6 +192,7 @@ void beforeEach() throws Exception { clock = new UpdatableTickingClock(OLD_DELETION_DATE.toInstant()); vault = spy(new BlobStoreDeletedMessageVault(new RecordingMetricFactory(), new MemoryDeletedMessageMetadataVault(), blobStore, blobStoreDAO, new BucketNameGenerator(clock), clock, + new BlobIdTimeGenerator(blobIdFactory, clock), VaultConfiguration.ENABLED_DEFAULT)); InMemoryIntegrationResources inMemoryResource = InMemoryIntegrationResources.defaultResources(); mailboxManager = spy(inMemoryResource.getMailboxManager()); @@ -2015,8 +2016,8 @@ void purgeShouldReturnATaskCreated() { @Test void purgeShouldProduceASuccessfulTaskWithAdditionalInformation() { - Mono.from(vault.append(DELETED_MESSAGE, new ByteArrayInputStream(CONTENT))).block(); - Mono.from(vault.append(DELETED_MESSAGE_2, new ByteArrayInputStream(CONTENT))).block(); + Mono.from(vault.appendV1(DELETED_MESSAGE, new ByteArrayInputStream(CONTENT))).block(); + Mono.from(vault.appendV1(DELETED_MESSAGE_2, new ByteArrayInputStream(CONTENT))).block(); clock.setInstant(NOW.toInstant()); @@ -2056,12 +2057,12 @@ void purgeShouldNotDeleteNotExpiredMessagesInTheVault() { .size(CONTENT.length) .build(); - Mono.from(vault.append(DELETED_MESSAGE, new ByteArrayInputStream(CONTENT))).block(); - Mono.from(vault.append(DELETED_MESSAGE_2, new ByteArrayInputStream(CONTENT))).block(); + Mono.from(vault.appendV1(DELETED_MESSAGE, new ByteArrayInputStream(CONTENT))).block(); + Mono.from(vault.appendV1(DELETED_MESSAGE_2, new ByteArrayInputStream(CONTENT))).block(); clock.setInstant(NOW.toInstant()); - Mono.from(vault.append(notExpiredMessage, new ByteArrayInputStream(CONTENT))).block(); + Mono.from(vault.appendV1(notExpiredMessage, new ByteArrayInputStream(CONTENT))).block(); String taskId = with() @@ -2080,8 +2081,8 @@ void purgeShouldNotDeleteNotExpiredMessagesInTheVault() { @Test void purgeShouldNotAppendMessagesToUserMailbox() throws Exception { - Mono.from(vault.append(DELETED_MESSAGE, new ByteArrayInputStream(CONTENT))).block(); - Mono.from(vault.append(DELETED_MESSAGE_2, new ByteArrayInputStream(CONTENT))).block(); + Mono.from(vault.appendV1(DELETED_MESSAGE, new ByteArrayInputStream(CONTENT))).block(); + Mono.from(vault.appendV1(DELETED_MESSAGE_2, new ByteArrayInputStream(CONTENT))).block(); String taskId = with() @@ -2102,8 +2103,8 @@ void purgeShouldNotAppendMessagesToUserMailbox() throws Exception { class FailingPurgeTest { @Test void purgeShouldProduceAFailedTaskWhenFailingDeletingBucket() { - Mono.from(vault.append(DELETED_MESSAGE, new ByteArrayInputStream(CONTENT))).block(); - Mono.from(vault.append(DELETED_MESSAGE_2, new ByteArrayInputStream(CONTENT))).block(); + Mono.from(vault.appendV1(DELETED_MESSAGE, new ByteArrayInputStream(CONTENT))).block(); + Mono.from(vault.appendV1(DELETED_MESSAGE_2, new ByteArrayInputStream(CONTENT))).block(); doReturn(Mono.error(new RuntimeException("mock exception"))) .when(blobStore) From f53bf5d1fc0abe1524aaf41dfa24158065903274 Mon Sep 17 00:00:00 2001 From: Rene Cordier Date: Thu, 8 Jan 2026 09:06:10 +0700 Subject: [PATCH 04/14] fixup! JAMES-4156 append deleted messages to the new deleted message vault single bucket --- .../james/vault/blob/BlobStoreDeletedMessageVaultTest.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mailbox/plugin/deleted-messages-vault/src/test/java/org/apache/james/vault/blob/BlobStoreDeletedMessageVaultTest.java b/mailbox/plugin/deleted-messages-vault/src/test/java/org/apache/james/vault/blob/BlobStoreDeletedMessageVaultTest.java index 873124e0b2b..e372dcc4bc9 100644 --- a/mailbox/plugin/deleted-messages-vault/src/test/java/org/apache/james/vault/blob/BlobStoreDeletedMessageVaultTest.java +++ b/mailbox/plugin/deleted-messages-vault/src/test/java/org/apache/james/vault/blob/BlobStoreDeletedMessageVaultTest.java @@ -204,7 +204,7 @@ public void searchAllShouldReturnOldMessage() { @Test public void searchAllShouldReturnOldAndNewMessages() { Mono.from(getVault().appendV1(DELETED_MESSAGE, new ByteArrayInputStream(CONTENT))).block(); - Mono.from(getVault().appendV1(DELETED_MESSAGE_2, new ByteArrayInputStream(CONTENT))).block(); + Mono.from(getVault().append(DELETED_MESSAGE_2, new ByteArrayInputStream(CONTENT))).block(); assertThat(Flux.from(getVault().search(USERNAME, ALL)).collectList().block()) .containsOnly(DELETED_MESSAGE, DELETED_MESSAGE_2); @@ -215,7 +215,7 @@ public void searchAllShouldSupportLimitQueryWithOldAndNewMessages() { Mono.from(getVault().appendV1(DELETED_MESSAGE, new ByteArrayInputStream(CONTENT))).block(); Mono.from(getVault().appendV1(DELETED_MESSAGE_2, new ByteArrayInputStream(CONTENT))).block(); DeletedMessage deletedMessage3 = DELETED_MESSAGE_GENERATOR.apply(InMemoryMessageId.of(33).getRawId()); - Mono.from(getVault().appendV1(deletedMessage3, new ByteArrayInputStream(CONTENT))).block(); + Mono.from(getVault().append(deletedMessage3, new ByteArrayInputStream(CONTENT))).block(); assertThat(Flux.from(getVault().search(USERNAME, Query.of(1, List.of()))).collectList().block()) .hasSize(1); From 4f8649ff1d0a6120bd5c7949729a132aeaeb04b3 Mon Sep 17 00:00:00 2001 From: Rene Cordier Date: Thu, 8 Jan 2026 16:44:51 +0700 Subject: [PATCH 05/14] JAMES-4156 add methods to calculate expirate date of blob and rebuild deleted blob id from string --- .../james/vault/blob/BlobIdTimeGenerator.java | 32 ++++++++- .../vault/blob/BlobIdTimeGeneratorTest.java | 65 ++++++++++++++++++- 2 files changed, 94 insertions(+), 3 deletions(-) diff --git a/mailbox/plugin/deleted-messages-vault/src/main/java/org/apache/james/vault/blob/BlobIdTimeGenerator.java b/mailbox/plugin/deleted-messages-vault/src/main/java/org/apache/james/vault/blob/BlobIdTimeGenerator.java index c84c18c8cf8..f8019b114d7 100644 --- a/mailbox/plugin/deleted-messages-vault/src/main/java/org/apache/james/vault/blob/BlobIdTimeGenerator.java +++ b/mailbox/plugin/deleted-messages-vault/src/main/java/org/apache/james/vault/blob/BlobIdTimeGenerator.java @@ -20,8 +20,12 @@ package org.apache.james.vault.blob; import java.time.Clock; +import java.time.LocalDate; import java.time.ZonedDateTime; +import java.util.Optional; import java.util.UUID; +import java.util.regex.Matcher; +import java.util.regex.Pattern; import jakarta.inject.Inject; @@ -29,7 +33,8 @@ import org.apache.james.blob.api.PlainBlobId; public class BlobIdTimeGenerator { - private static final String BLOB_ID_GENERATING_FORMAT = "%d/%02d/%s"; + public static final String BLOB_ID_GENERATING_FORMAT = "%d/%02d/%s"; + public static final Pattern BLOB_ID_TIME_PATTERN = Pattern.compile("^(\\d{4})/(\\d{2})/(.*)$"); private final BlobId.Factory blobIdFactory; private final Clock clock; @@ -47,4 +52,29 @@ BlobId currentBlobId() { return new PlainBlobId(String.format(BLOB_ID_GENERATING_FORMAT, year, month, blobIdFactory.of(UUID.randomUUID().toString()).asString())); } + + Optional blobIdEndTime(BlobId blobId) { + return Optional.of(BLOB_ID_TIME_PATTERN.matcher(blobId.asString())) + .filter(Matcher::matches) + .map(matcher -> { + int year = Integer.parseInt(matcher.group(1)); + int month = Integer.parseInt(matcher.group(2)); + return firstDayOfNextMonth(year, month); + }); + } + + private ZonedDateTime firstDayOfNextMonth(int year, int month) { + return LocalDate.of(year, month, 1).plusMonths(1).atStartOfDay(clock.getZone()); + } + + public BlobId toDeletedMessageBlobId(String blobId) { + return Optional.of(BLOB_ID_TIME_PATTERN.matcher(blobId)) + .filter(Matcher::matches) + .map(matcher -> { + int year = Integer.parseInt(matcher.group(1)); + int month = Integer.parseInt(matcher.group(2)); + String subBlobId = matcher.group(3); + return (BlobId) new PlainBlobId(String.format(BLOB_ID_GENERATING_FORMAT, year, month, blobIdFactory.parse(subBlobId).asString())); + }).orElseGet(() -> blobIdFactory.parse(blobId)); + } } diff --git a/mailbox/plugin/deleted-messages-vault/src/test/java/org/apache/james/vault/blob/BlobIdTimeGeneratorTest.java b/mailbox/plugin/deleted-messages-vault/src/test/java/org/apache/james/vault/blob/BlobIdTimeGeneratorTest.java index 9391da5718b..5e558b412c0 100644 --- a/mailbox/plugin/deleted-messages-vault/src/test/java/org/apache/james/vault/blob/BlobIdTimeGeneratorTest.java +++ b/mailbox/plugin/deleted-messages-vault/src/test/java/org/apache/james/vault/blob/BlobIdTimeGeneratorTest.java @@ -19,10 +19,13 @@ package org.apache.james.vault.blob; +import static org.apache.james.vault.blob.BlobIdTimeGenerator.BLOB_ID_GENERATING_FORMAT; import static org.assertj.core.api.Assertions.assertThat; import java.time.Clock; import java.time.Instant; +import java.time.ZonedDateTime; +import java.util.UUID; import org.apache.james.blob.api.BlobId; import org.apache.james.blob.api.PlainBlobId; @@ -31,6 +34,8 @@ import org.apache.james.utils.UpdatableTickingClock; import org.junit.jupiter.api.Nested; import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; public class BlobIdTimeGeneratorTest { private static final Instant NOW = Instant.parse("2007-07-03T10:15:30.00Z"); @@ -39,10 +44,11 @@ public class BlobIdTimeGeneratorTest { interface BlobIdTimeGeneratorContract { BlobId.Factory blobIdFactory(); + BlobIdTimeGenerator blobIdTimeGenerator(); + @Test default void currentBlobIdShouldReturnBlobIdFormattedWithYearAndMonthPrefix() { - BlobIdTimeGenerator blobIdTimeGenerator = new BlobIdTimeGenerator(blobIdFactory(), CLOCK); - String currentBlobId = blobIdTimeGenerator.currentBlobId().asString(); + String currentBlobId = blobIdTimeGenerator().currentBlobId().asString(); int firstSlash = currentBlobId.indexOf('/'); int secondSlash = currentBlobId.indexOf('/', firstSlash + 1); @@ -50,6 +56,30 @@ default void currentBlobIdShouldReturnBlobIdFormattedWithYearAndMonthPrefix() { assertThat(prefix).isEqualTo("2007/07"); } + + @Test + default void shouldReturnNextMonthAsEndTime() { + BlobId blobId = new PlainBlobId(String.format(BLOB_ID_GENERATING_FORMAT, 2018, 7, blobIdFactory().of(UUID.randomUUID().toString()).asString())); + + assertThat(blobIdTimeGenerator().blobIdEndTime(blobId)) + .contains(ZonedDateTime.parse("2018-08-01T00:00:00.000000000Z[UTC]")); + } + + @Test + default void shouldReturnProperDeletedMessageBlobIdFromString() { + BlobId currentBlobId = blobIdTimeGenerator().currentBlobId(); + BlobId deletedMessageBlobId = blobIdTimeGenerator().toDeletedMessageBlobId(currentBlobId.asString()); + + assertThat(deletedMessageBlobId).isEqualTo(currentBlobId); + } + + @Test + default void shouldFallbackToOldDeletedBlobIdFromString() { + BlobId currentBlobId = blobIdFactory().of(UUID.randomUUID().toString()); + BlobId deletedMessageBlobId = blobIdTimeGenerator().toDeletedMessageBlobId(currentBlobId.asString()); + + assertThat(deletedMessageBlobId).isEqualTo(currentBlobId); + } } @Nested @@ -58,6 +88,27 @@ class PlainBlobIdTimeGeneratorTest implements BlobIdTimeGeneratorContract { public BlobId.Factory blobIdFactory() { return new PlainBlobId.Factory(); } + + @Override + public BlobIdTimeGenerator blobIdTimeGenerator() { + return new BlobIdTimeGenerator(blobIdFactory(), CLOCK); + } + + @ParameterizedTest + @ValueSource(strings = { + "2018-07-cddf9c7c-12f1-4ce7-9993-8606b9fb8816", + "2018-07/cddf9c7c-12f1-4ce7-9993-8606b9fb8816", + "2018/07-cddf9c7c-12f1-4ce7-9993-8606b9fb8816", + "cddf9c7c-12f1-4ce7-9993-8606b9fb8816", + "18/07/cddf9c7c-12f1-4ce7-9993-8606b9fb8816", + "2018/7/cddf9c7c-12f1-4ce7-9993-8606b9fb8816", + "07/2018/cddf9c7c-12f1-4ce7-9993-8606b9fb8816", + }) + public void shouldBeEmptyWhenPassingNonWellFormattedBlobId(String blobIdAsString) { + BlobId blobId = blobIdFactory().of(blobIdAsString); + + assertThat(blobIdTimeGenerator().blobIdEndTime(blobId)).isEmpty(); + } } @Nested @@ -66,6 +117,11 @@ class GenerationAwareBlobIdTimeGeneratorTest implements BlobIdTimeGeneratorContr public BlobId.Factory blobIdFactory() { return new GenerationAwareBlobId.Factory(CLOCK, new PlainBlobId.Factory(), GenerationAwareBlobId.Configuration.DEFAULT); } + + @Override + public BlobIdTimeGenerator blobIdTimeGenerator() { + return new BlobIdTimeGenerator(blobIdFactory(), CLOCK); + } } @Nested @@ -74,5 +130,10 @@ class MinIOGenerationAwareBlobIdTimeGeneratorTest implements BlobIdTimeGenerator public BlobId.Factory blobIdFactory() { return new MinIOGenerationAwareBlobId.Factory(CLOCK, GenerationAwareBlobId.Configuration.DEFAULT, new PlainBlobId.Factory()); } + + @Override + public BlobIdTimeGenerator blobIdTimeGenerator() { + return new BlobIdTimeGenerator(blobIdFactory(), CLOCK); + } } } From 1533006ebff00846eb7c2318ec577bcfe715dd2f Mon Sep 17 00:00:00 2001 From: Rene Cordier Date: Fri, 9 Jan 2026 16:38:42 +0700 Subject: [PATCH 06/14] JAMES-4156 Refactoring the vault purge to take the single vault bucket design into account --- .../vault/metadata/StorageInformationDAO.java | 10 +-- ...sandraDeletedMessageMetadataVaultTest.java | 8 +- .../james/vault/metadata/MetadataDAOTest.java | 7 +- .../metadata/StorageInformationDAOTest.java | 7 +- .../PostgresDeletedMessageMetadataVault.java | 10 +-- ...stgresDeletedMessageMetadataVaultTest.java | 9 +- mailbox/plugin/deleted-messages-vault/pom.xml | 9 ++ .../blob/BlobStoreDeletedMessageVault.java | 35 +++++++- .../BlobStoreVaultGarbageCollectionTask.java | 2 +- ...essageWithStorageInformationConverter.java | 10 +-- .../james/vault/DeletedMessageFixture.java | 1 + .../vault/DeletedMessageVaultContract.java | 83 ++++++++++++++++++- .../vault/DeletedMessageVaultHookTest.java | 7 +- .../BlobStoreDeletedMessageVaultTest.java | 18 +++- ...dMessageWithStorageInformationDTOTest.java | 5 +- .../DeletedMessagesVaultRoutesTest.java | 4 +- 16 files changed, 191 insertions(+), 34 deletions(-) diff --git a/mailbox/plugin/deleted-messages-vault-cassandra/src/main/java/org/apache/james/vault/metadata/StorageInformationDAO.java b/mailbox/plugin/deleted-messages-vault-cassandra/src/main/java/org/apache/james/vault/metadata/StorageInformationDAO.java index 2998ac03339..902bd7fc422 100644 --- a/mailbox/plugin/deleted-messages-vault-cassandra/src/main/java/org/apache/james/vault/metadata/StorageInformationDAO.java +++ b/mailbox/plugin/deleted-messages-vault-cassandra/src/main/java/org/apache/james/vault/metadata/StorageInformationDAO.java @@ -32,10 +32,10 @@ import jakarta.inject.Inject; import org.apache.james.backends.cassandra.utils.CassandraAsyncExecutor; -import org.apache.james.blob.api.BlobId; import org.apache.james.blob.api.BucketName; import org.apache.james.core.Username; import org.apache.james.mailbox.model.MessageId; +import org.apache.james.vault.blob.BlobIdTimeGenerator; import com.datastax.oss.driver.api.core.CqlSession; import com.datastax.oss.driver.api.core.cql.PreparedStatement; @@ -47,15 +47,15 @@ public class StorageInformationDAO { private final PreparedStatement addStatement; private final PreparedStatement removeStatement; private final PreparedStatement readStatement; - private final BlobId.Factory blobIdFactory; + private final BlobIdTimeGenerator blobIdTimeGenerator; @Inject - StorageInformationDAO(CqlSession session, BlobId.Factory blobIdFactory) { + StorageInformationDAO(CqlSession session, BlobIdTimeGenerator blobIdTimeGenerator) { this.cassandraAsyncExecutor = new CassandraAsyncExecutor(session); this.addStatement = prepareAdd(session); this.removeStatement = prepareRemove(session); this.readStatement = prepareRead(session); - this.blobIdFactory = blobIdFactory; + this.blobIdTimeGenerator = blobIdTimeGenerator; } private PreparedStatement prepareRead(CqlSession session) { @@ -102,6 +102,6 @@ Mono retrieveStorageInformation(Username username, MessageId .setString(MESSAGE_ID, messageId.serialize())) .map(row -> StorageInformation.builder() .bucketName(BucketName.of(row.getString(BUCKET_NAME))) - .blobId(blobIdFactory.parse(row.getString(BLOB_ID)))); + .blobId(blobIdTimeGenerator.toDeletedMessageBlobId(row.getString(BLOB_ID)))); } } diff --git a/mailbox/plugin/deleted-messages-vault-cassandra/src/test/java/org/apache/james/vault/metadata/CassandraDeletedMessageMetadataVaultTest.java b/mailbox/plugin/deleted-messages-vault-cassandra/src/test/java/org/apache/james/vault/metadata/CassandraDeletedMessageMetadataVaultTest.java index 73cff9198f0..7f5640a3942 100644 --- a/mailbox/plugin/deleted-messages-vault-cassandra/src/test/java/org/apache/james/vault/metadata/CassandraDeletedMessageMetadataVaultTest.java +++ b/mailbox/plugin/deleted-messages-vault-cassandra/src/test/java/org/apache/james/vault/metadata/CassandraDeletedMessageMetadataVaultTest.java @@ -20,6 +20,7 @@ package org.apache.james.vault.metadata; import static org.apache.james.backends.cassandra.Scenario.Builder.fail; +import static org.apache.james.vault.DeletedMessageFixture.NOW; import static org.apache.james.vault.DeletedMessageFixture.USERNAME; import static org.apache.james.vault.metadata.DeletedMessageMetadataDataDefinition.MODULE; import static org.apache.james.vault.metadata.DeletedMessageVaultMetadataFixture.BUCKET_NAME; @@ -35,6 +36,8 @@ import org.apache.james.blob.api.PlainBlobId; import org.apache.james.mailbox.inmemory.InMemoryId; import org.apache.james.mailbox.inmemory.InMemoryMessageId; +import org.apache.james.utils.UpdatableTickingClock; +import org.apache.james.vault.blob.BlobIdTimeGenerator; import org.apache.james.vault.dto.DeletedMessageWithStorageInformationConverter; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Disabled; @@ -57,11 +60,12 @@ public class CassandraDeletedMessageMetadataVaultTest implements DeletedMessageM @BeforeEach void setUp(CassandraCluster cassandra) { PlainBlobId.Factory blobIdFactory = new PlainBlobId.Factory(); + BlobIdTimeGenerator blobIdTimeGenerator = new BlobIdTimeGenerator(blobIdFactory, new UpdatableTickingClock(NOW.toInstant())); InMemoryMessageId.Factory messageIdFactory = new InMemoryMessageId.Factory(); - DeletedMessageWithStorageInformationConverter dtoConverter = new DeletedMessageWithStorageInformationConverter(blobIdFactory, messageIdFactory, new InMemoryId.Factory()); + DeletedMessageWithStorageInformationConverter dtoConverter = new DeletedMessageWithStorageInformationConverter(blobIdTimeGenerator, messageIdFactory, new InMemoryId.Factory()); metadataDAO = new MetadataDAO(cassandra.getConf(), messageIdFactory, new MetadataSerializer(dtoConverter)); - storageInformationDAO = new StorageInformationDAO(cassandra.getConf(), blobIdFactory); + storageInformationDAO = new StorageInformationDAO(cassandra.getConf(), blobIdTimeGenerator); userPerBucketDAO = new UserPerBucketDAO(cassandra.getConf()); testee = new CassandraDeletedMessageMetadataVault(metadataDAO, storageInformationDAO, userPerBucketDAO); diff --git a/mailbox/plugin/deleted-messages-vault-cassandra/src/test/java/org/apache/james/vault/metadata/MetadataDAOTest.java b/mailbox/plugin/deleted-messages-vault-cassandra/src/test/java/org/apache/james/vault/metadata/MetadataDAOTest.java index 35760cb772b..2342db89a2a 100644 --- a/mailbox/plugin/deleted-messages-vault-cassandra/src/test/java/org/apache/james/vault/metadata/MetadataDAOTest.java +++ b/mailbox/plugin/deleted-messages-vault-cassandra/src/test/java/org/apache/james/vault/metadata/MetadataDAOTest.java @@ -20,6 +20,7 @@ package org.apache.james.vault.metadata; import static org.apache.james.vault.DeletedMessageFixture.MESSAGE_ID; +import static org.apache.james.vault.DeletedMessageFixture.NOW; import static org.apache.james.vault.DeletedMessageFixture.USERNAME; import static org.apache.james.vault.metadata.DeletedMessageMetadataDataDefinition.MODULE; import static org.apache.james.vault.metadata.DeletedMessageVaultMetadataFixture.BUCKET_NAME; @@ -35,6 +36,8 @@ import org.apache.james.mailbox.inmemory.InMemoryId; import org.apache.james.mailbox.inmemory.InMemoryMessageId; import org.apache.james.mailbox.model.MessageId; +import org.apache.james.utils.UpdatableTickingClock; +import org.apache.james.vault.blob.BlobIdTimeGenerator; import org.apache.james.vault.dto.DeletedMessageWithStorageInformationConverter; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @@ -49,7 +52,9 @@ class MetadataDAOTest { @BeforeEach void setUp(CassandraCluster cassandra) { DeletedMessageWithStorageInformationConverter dtoConverter = new DeletedMessageWithStorageInformationConverter( - new PlainBlobId.Factory(), new InMemoryMessageId.Factory(), new InMemoryId.Factory()); + new BlobIdTimeGenerator(new PlainBlobId.Factory(), new UpdatableTickingClock(NOW.toInstant())), + new InMemoryMessageId.Factory(), + new InMemoryId.Factory()); testee = new MetadataDAO(cassandra.getConf(), new InMemoryMessageId.Factory(), new MetadataSerializer(dtoConverter)); diff --git a/mailbox/plugin/deleted-messages-vault-cassandra/src/test/java/org/apache/james/vault/metadata/StorageInformationDAOTest.java b/mailbox/plugin/deleted-messages-vault-cassandra/src/test/java/org/apache/james/vault/metadata/StorageInformationDAOTest.java index a91384fc3bf..0cc66a6db25 100644 --- a/mailbox/plugin/deleted-messages-vault-cassandra/src/test/java/org/apache/james/vault/metadata/StorageInformationDAOTest.java +++ b/mailbox/plugin/deleted-messages-vault-cassandra/src/test/java/org/apache/james/vault/metadata/StorageInformationDAOTest.java @@ -19,6 +19,7 @@ package org.apache.james.vault.metadata; +import static org.apache.james.vault.DeletedMessageFixture.NOW; import static org.apache.james.vault.metadata.DeletedMessageMetadataDataDefinition.MODULE; import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatCode; @@ -32,6 +33,8 @@ import org.apache.james.blob.api.PlainBlobId; import org.apache.james.core.Username; import org.apache.james.mailbox.model.TestMessageId; +import org.apache.james.utils.UpdatableTickingClock; +import org.apache.james.vault.blob.BlobIdTimeGenerator; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.RegisterExtension; @@ -39,7 +42,7 @@ class StorageInformationDAOTest { private static final BucketName BUCKET_NAME = BucketName.of("deletedMessages-2019-06-01"); private static final BucketName BUCKET_NAME_2 = BucketName.of("deletedMessages-2019-07-01"); - private static final PlainBlobId.Factory BLOB_ID_FACTORY = new PlainBlobId.Factory(); + private static final BlobIdTimeGenerator BLOB_ID_TIME_GENERATOR = new BlobIdTimeGenerator(new PlainBlobId.Factory(), new UpdatableTickingClock(NOW.toInstant())); private static final Username OWNER = Username.of("owner"); private static final TestMessageId MESSAGE_ID = TestMessageId.of(36); private static final BlobId BLOB_ID = new PlainBlobId.Factory().parse("05dcb33b-8382-4744-923a-bc593ad84d23"); @@ -54,7 +57,7 @@ class StorageInformationDAOTest { @BeforeEach void setUp(CassandraCluster cassandra) { - testee = new StorageInformationDAO(cassandra.getConf(), BLOB_ID_FACTORY); + testee = new StorageInformationDAO(cassandra.getConf(), BLOB_ID_TIME_GENERATOR); } @Test diff --git a/mailbox/plugin/deleted-messages-vault-postgres/src/main/java/org/apache/james/vault/metadata/PostgresDeletedMessageMetadataVault.java b/mailbox/plugin/deleted-messages-vault-postgres/src/main/java/org/apache/james/vault/metadata/PostgresDeletedMessageMetadataVault.java index 38a5bc9e6c2..2f5c4c0b165 100644 --- a/mailbox/plugin/deleted-messages-vault-postgres/src/main/java/org/apache/james/vault/metadata/PostgresDeletedMessageMetadataVault.java +++ b/mailbox/plugin/deleted-messages-vault-postgres/src/main/java/org/apache/james/vault/metadata/PostgresDeletedMessageMetadataVault.java @@ -33,10 +33,10 @@ import jakarta.inject.Inject; import org.apache.james.backends.postgres.utils.PostgresExecutor; -import org.apache.james.blob.api.BlobId; import org.apache.james.blob.api.BucketName; import org.apache.james.core.Username; import org.apache.james.mailbox.model.MessageId; +import org.apache.james.vault.blob.BlobIdTimeGenerator; import org.jooq.Record; import org.reactivestreams.Publisher; @@ -46,15 +46,15 @@ public class PostgresDeletedMessageMetadataVault implements DeletedMessageMetadataVault { private final PostgresExecutor postgresExecutor; private final MetadataSerializer metadataSerializer; - private final BlobId.Factory blobIdFactory; + private final BlobIdTimeGenerator blobIdTimeGenerator; @Inject public PostgresDeletedMessageMetadataVault(PostgresExecutor postgresExecutor, MetadataSerializer metadataSerializer, - BlobId.Factory blobIdFactory) { + BlobIdTimeGenerator blobIdTimeGenerator) { this.postgresExecutor = postgresExecutor; this.metadataSerializer = metadataSerializer; - this.blobIdFactory = blobIdFactory; + this.blobIdTimeGenerator = blobIdTimeGenerator; } @Override @@ -93,7 +93,7 @@ public Publisher retrieveStorageInformation(Username usernam private Function toStorageInformation() { return record -> StorageInformation.builder() .bucketName(BucketName.of(record.get(BUCKET_NAME))) - .blobId(blobIdFactory.parse(record.get(BLOB_ID))); + .blobId(blobIdTimeGenerator.toDeletedMessageBlobId(record.get(BLOB_ID))); } @Override diff --git a/mailbox/plugin/deleted-messages-vault-postgres/src/test/java/org/apache/james/vault/metadata/PostgresDeletedMessageMetadataVaultTest.java b/mailbox/plugin/deleted-messages-vault-postgres/src/test/java/org/apache/james/vault/metadata/PostgresDeletedMessageMetadataVaultTest.java index 71e06863f8b..66fb7737730 100644 --- a/mailbox/plugin/deleted-messages-vault-postgres/src/test/java/org/apache/james/vault/metadata/PostgresDeletedMessageMetadataVaultTest.java +++ b/mailbox/plugin/deleted-messages-vault-postgres/src/test/java/org/apache/james/vault/metadata/PostgresDeletedMessageMetadataVaultTest.java @@ -19,12 +19,16 @@ package org.apache.james.vault.metadata; +import static org.apache.james.vault.DeletedMessageFixture.NOW; + import org.apache.james.backends.postgres.PostgresDataDefinition; import org.apache.james.backends.postgres.PostgresExtension; import org.apache.james.blob.api.BlobId; import org.apache.james.blob.api.PlainBlobId; import org.apache.james.mailbox.inmemory.InMemoryId; import org.apache.james.mailbox.inmemory.InMemoryMessageId; +import org.apache.james.utils.UpdatableTickingClock; +import org.apache.james.vault.blob.BlobIdTimeGenerator; import org.apache.james.vault.dto.DeletedMessageWithStorageInformationConverter; import org.junit.jupiter.api.extension.RegisterExtension; @@ -36,12 +40,13 @@ class PostgresDeletedMessageMetadataVaultTest implements DeletedMessageMetadataV @Override public DeletedMessageMetadataVault metadataVault() { BlobId.Factory blobIdFactory = new PlainBlobId.Factory(); + BlobIdTimeGenerator blobIdTimeGenerator = new BlobIdTimeGenerator(blobIdFactory, new UpdatableTickingClock(NOW.toInstant())); InMemoryMessageId.Factory messageIdFactory = new InMemoryMessageId.Factory(); - DeletedMessageWithStorageInformationConverter dtoConverter = new DeletedMessageWithStorageInformationConverter(blobIdFactory, + DeletedMessageWithStorageInformationConverter dtoConverter = new DeletedMessageWithStorageInformationConverter(blobIdTimeGenerator, messageIdFactory, new InMemoryId.Factory()); return new PostgresDeletedMessageMetadataVault(postgresExtension.getDefaultPostgresExecutor(), new MetadataSerializer(dtoConverter), - blobIdFactory); + blobIdTimeGenerator); } } diff --git a/mailbox/plugin/deleted-messages-vault/pom.xml b/mailbox/plugin/deleted-messages-vault/pom.xml index bda2754acfc..f2473bc773b 100644 --- a/mailbox/plugin/deleted-messages-vault/pom.xml +++ b/mailbox/plugin/deleted-messages-vault/pom.xml @@ -96,6 +96,15 @@ ${james.groupId} james-server-core + + ${james.groupId} + james-server-data-api + + + ${james.groupId} + james-server-data-memory + test + ${james.groupId} james-server-task-json diff --git a/mailbox/plugin/deleted-messages-vault/src/main/java/org/apache/james/vault/blob/BlobStoreDeletedMessageVault.java b/mailbox/plugin/deleted-messages-vault/src/main/java/org/apache/james/vault/blob/BlobStoreDeletedMessageVault.java index f3da89571b5..18b80c3dc0d 100644 --- a/mailbox/plugin/deleted-messages-vault/src/main/java/org/apache/james/vault/blob/BlobStoreDeletedMessageVault.java +++ b/mailbox/plugin/deleted-messages-vault/src/main/java/org/apache/james/vault/blob/BlobStoreDeletedMessageVault.java @@ -29,6 +29,7 @@ import jakarta.inject.Inject; +import org.apache.james.blob.api.BlobId; import org.apache.james.blob.api.BlobStore; import org.apache.james.blob.api.BlobStoreDAO; import org.apache.james.blob.api.BucketName; @@ -37,6 +38,7 @@ import org.apache.james.mailbox.model.MessageId; import org.apache.james.metrics.api.MetricFactory; import org.apache.james.task.Task; +import org.apache.james.user.api.UsersRepository; import org.apache.james.vault.DeletedMessage; import org.apache.james.vault.DeletedMessageContentNotFoundException; import org.apache.james.vault.DeletedMessageVault; @@ -74,13 +76,14 @@ public class BlobStoreDeletedMessageVault implements DeletedMessageVault { private final Clock clock; private final VaultConfiguration vaultConfiguration; private final BlobIdTimeGenerator blobIdTimeGenerator; + private final UsersRepository usersRepository; private final BlobStoreVaultGarbageCollectionTask.Factory taskFactory; @Inject public BlobStoreDeletedMessageVault(MetricFactory metricFactory, DeletedMessageMetadataVault messageMetadataVault, BlobStore blobStore, BlobStoreDAO blobStoreDAO, BucketNameGenerator nameGenerator, Clock clock, BlobIdTimeGenerator blobIdTimeGenerator, - VaultConfiguration vaultConfiguration) { + VaultConfiguration vaultConfiguration, UsersRepository usersRepository) { this.metricFactory = metricFactory; this.messageMetadataVault = messageMetadataVault; this.blobStore = blobStore; @@ -89,6 +92,7 @@ public BlobStoreDeletedMessageVault(MetricFactory metricFactory, DeletedMessageM this.clock = clock; this.vaultConfiguration = vaultConfiguration; this.blobIdTimeGenerator = blobIdTimeGenerator; + this.usersRepository = usersRepository; this.taskFactory = new BlobStoreVaultGarbageCollectionTask.Factory(this); } @@ -201,7 +205,7 @@ public Task deleteExpiredMessagesTask() { return taskFactory.create(); } - + @Deprecated Flux deleteExpiredMessages(ZonedDateTime beginningOfRetentionPeriod) { return Flux.from( metricFactory.decoratePublisherWithTimerMetric( @@ -216,6 +220,7 @@ ZonedDateTime getBeginningOfRetentionPeriod() { } @VisibleForTesting + @Deprecated Flux retentionQualifiedBuckets(ZonedDateTime beginningOfRetentionPeriod) { return Flux.from(messageMetadataVault.listRelatedBuckets()) .filter(bucketName -> isFullyExpired(beginningOfRetentionPeriod, bucketName)); @@ -235,4 +240,30 @@ private Mono deleteBucketData(BucketName bucketName) { return Mono.from(blobStore.deleteBucket(bucketName)) .then(Mono.from(messageMetadataVault.removeMetadataRelatedToBucket(bucketName))); } + + Mono deleteUserExpiredMessages(ZonedDateTime beginningOfRetentionPeriod) { + BucketName bucketName = BucketName.of(vaultConfiguration.getSingleBucketName()); + + return Flux.from(metricFactory.decoratePublisherWithTimerMetric( + DELETE_EXPIRED_MESSAGES_METRIC_NAME, + Flux.from(usersRepository.listReactive()) + .flatMap(username -> Flux.from(messageMetadataVault.listMessages(bucketName, username)) + .filter(deletedMessage -> isMessageFullyExpired(beginningOfRetentionPeriod, deletedMessage)) + .flatMap(deletedMessage -> messageMetadataVault.remove(bucketName, username, deletedMessage.getDeletedMessage().getMessageId()))))) + .then(); + } + + + + private boolean isMessageFullyExpired(ZonedDateTime beginningOfRetentionPeriod, DeletedMessageWithStorageInformation deletedMessage) { + BlobId blobId = deletedMessage.getStorageInformation().getBlobId(); + Optional maybeEndDate = blobIdTimeGenerator.blobIdEndTime(blobId); + + if (maybeEndDate.isEmpty()) { + LOGGER.error("Pattern used for blobId used in deletedMessageVault is invalid and end date cannot be parsed {}", blobId); + } + + return maybeEndDate.map(endDate -> endDate.isBefore(beginningOfRetentionPeriod)) + .orElse(false); + } } diff --git a/mailbox/plugin/deleted-messages-vault/src/main/java/org/apache/james/vault/blob/BlobStoreVaultGarbageCollectionTask.java b/mailbox/plugin/deleted-messages-vault/src/main/java/org/apache/james/vault/blob/BlobStoreVaultGarbageCollectionTask.java index a455cc88fff..75f8e7b3c99 100644 --- a/mailbox/plugin/deleted-messages-vault/src/main/java/org/apache/james/vault/blob/BlobStoreVaultGarbageCollectionTask.java +++ b/mailbox/plugin/deleted-messages-vault/src/main/java/org/apache/james/vault/blob/BlobStoreVaultGarbageCollectionTask.java @@ -94,7 +94,7 @@ private BlobStoreVaultGarbageCollectionTask(BlobStoreDeletedMessageVault deleted public Result run() { deletedMessageVault.deleteExpiredMessages(beginningOfRetentionPeriod) .doOnNext(deletedBuckets::add) - .then() + .then(deletedMessageVault.deleteUserExpiredMessages(beginningOfRetentionPeriod)) .block(); return Result.COMPLETED; diff --git a/mailbox/plugin/deleted-messages-vault/src/main/java/org/apache/james/vault/dto/DeletedMessageWithStorageInformationConverter.java b/mailbox/plugin/deleted-messages-vault/src/main/java/org/apache/james/vault/dto/DeletedMessageWithStorageInformationConverter.java index 2e74c58dec1..fe1492cd1d3 100644 --- a/mailbox/plugin/deleted-messages-vault/src/main/java/org/apache/james/vault/dto/DeletedMessageWithStorageInformationConverter.java +++ b/mailbox/plugin/deleted-messages-vault/src/main/java/org/apache/james/vault/dto/DeletedMessageWithStorageInformationConverter.java @@ -25,7 +25,6 @@ import jakarta.inject.Inject; import jakarta.mail.internet.AddressException; -import org.apache.james.blob.api.BlobId; import org.apache.james.blob.api.BucketName; import org.apache.james.core.MailAddress; import org.apache.james.core.MaybeSender; @@ -33,6 +32,7 @@ import org.apache.james.mailbox.model.MailboxId; import org.apache.james.mailbox.model.MessageId; import org.apache.james.vault.DeletedMessage; +import org.apache.james.vault.blob.BlobIdTimeGenerator; import org.apache.james.vault.metadata.DeletedMessageWithStorageInformation; import org.apache.james.vault.metadata.StorageInformation; @@ -40,15 +40,15 @@ import com.google.common.collect.ImmutableList; public class DeletedMessageWithStorageInformationConverter { - private final BlobId.Factory blobFactory; + private final BlobIdTimeGenerator blobIdTimeGenerator; private final MessageId.Factory messageIdFactory; private final MailboxId.Factory mailboxIdFactory; @Inject - public DeletedMessageWithStorageInformationConverter(BlobId.Factory blobFactory, + public DeletedMessageWithStorageInformationConverter(BlobIdTimeGenerator blobIdTimeGenerator, MessageId.Factory messageIdFactory, MailboxId.Factory mailboxIdFactory) { - this.blobFactory = blobFactory; + this.blobIdTimeGenerator = blobIdTimeGenerator; this.messageIdFactory = messageIdFactory; this.mailboxIdFactory = mailboxIdFactory; } @@ -56,7 +56,7 @@ public DeletedMessageWithStorageInformationConverter(BlobId.Factory blobFactory, public StorageInformation toDomainObject(DeletedMessageWithStorageInformationDTO.StorageInformationDTO storageInformationDTO) { return StorageInformation.builder() .bucketName(BucketName.of(storageInformationDTO.getBucketName())) - .blobId(blobFactory.parse(storageInformationDTO.getBlobId())); + .blobId(blobIdTimeGenerator.toDeletedMessageBlobId(storageInformationDTO.getBlobId())); } public DeletedMessage toDomainObject(DeletedMessageWithStorageInformationDTO.DeletedMessageDTO deletedMessageDTO) throws AddressException { diff --git a/mailbox/plugin/deleted-messages-vault/src/test/java/org/apache/james/vault/DeletedMessageFixture.java b/mailbox/plugin/deleted-messages-vault/src/test/java/org/apache/james/vault/DeletedMessageFixture.java index a0a7f9c3976..99440a84000 100644 --- a/mailbox/plugin/deleted-messages-vault/src/test/java/org/apache/james/vault/DeletedMessageFixture.java +++ b/mailbox/plugin/deleted-messages-vault/src/test/java/org/apache/james/vault/DeletedMessageFixture.java @@ -43,6 +43,7 @@ public interface DeletedMessageFixture { InMemoryId MAILBOX_ID_3 = InMemoryId.of(45); Username USERNAME = Username.of("bob@apache.org"); Username USERNAME_2 = Username.of("dimitri@apache.org"); + String PASSWORD = "123456"; ZonedDateTime DELIVERY_DATE = ZonedDateTime.parse("2014-10-30T14:12:00Z"); ZonedDateTime DELETION_DATE = ZonedDateTime.parse("2015-10-30T14:12:00Z"); ZonedDateTime NOW = ZonedDateTime.parse("2015-10-30T16:12:00Z"); diff --git a/mailbox/plugin/deleted-messages-vault/src/test/java/org/apache/james/vault/DeletedMessageVaultContract.java b/mailbox/plugin/deleted-messages-vault/src/test/java/org/apache/james/vault/DeletedMessageVaultContract.java index 3524715a5af..368d9e7b9b6 100644 --- a/mailbox/plugin/deleted-messages-vault/src/test/java/org/apache/james/vault/DeletedMessageVaultContract.java +++ b/mailbox/plugin/deleted-messages-vault/src/test/java/org/apache/james/vault/DeletedMessageVaultContract.java @@ -35,7 +35,6 @@ import static org.assertj.core.api.Assertions.assertThatThrownBy; import java.io.ByteArrayInputStream; -import java.time.Clock; import java.time.Duration; import java.util.List; @@ -52,8 +51,6 @@ import reactor.core.publisher.Mono; public interface DeletedMessageVaultContract { - Clock CLOCK = Clock.fixed(NOW.toInstant(), NOW.getZone()); - BlobStoreDeletedMessageVault getVault(); UpdatableTickingClock getClock(); @@ -252,6 +249,16 @@ default void deleteExpiredMessagesTaskShouldCompleteWhenAllMailsDeleted() throws assertThat(result).isEqualTo(Task.Result.COMPLETED); } + @Test + default void deleteExpiredMessagesSingleBucketTaskShouldCompleteWhenAllMailsDeleted() throws InterruptedException { + Mono.from(getVault().append(DELETED_MESSAGE, new ByteArrayInputStream(CONTENT))).block(); + Mono.from(getVault().delete(USERNAME, DELETED_MESSAGE.getMessageId())).block(); + + Task.Result result = getVault().deleteExpiredMessagesTask().run(); + + assertThat(result).isEqualTo(Task.Result.COMPLETED); + } + @Test default void deleteExpiredMessagesTaskShouldCompleteWhenOnlyRecentMails() throws InterruptedException { Mono.from(getVault().appendV1(DELETED_MESSAGE, new ByteArrayInputStream(CONTENT))).block(); @@ -261,6 +268,15 @@ default void deleteExpiredMessagesTaskShouldCompleteWhenOnlyRecentMails() throws assertThat(result).isEqualTo(Task.Result.COMPLETED); } + @Test + default void deleteExpiredMessagesSingleBucketTaskShouldCompleteWhenOnlyRecentMails() throws InterruptedException { + Mono.from(getVault().append(DELETED_MESSAGE, new ByteArrayInputStream(CONTENT))).block(); + + Task.Result result = getVault().deleteExpiredMessagesTask().run(); + + assertThat(result).isEqualTo(Task.Result.COMPLETED); + } + @Test default void deleteExpiredMessagesTaskShouldCompleteWhenOnlyOldMails() throws InterruptedException { Mono.from(getVault().appendV1(OLD_DELETED_MESSAGE, new ByteArrayInputStream(CONTENT))).block(); @@ -270,6 +286,15 @@ default void deleteExpiredMessagesTaskShouldCompleteWhenOnlyOldMails() throws In assertThat(result).isEqualTo(Task.Result.COMPLETED); } + @Test + default void deleteExpiredMessagesSingleBucketTaskShouldCompleteWhenOnlyOldMails() throws InterruptedException { + Mono.from(getVault().append(OLD_DELETED_MESSAGE, new ByteArrayInputStream(CONTENT))).block(); + + Task.Result result = getVault().deleteExpiredMessagesTask().run(); + + assertThat(result).isEqualTo(Task.Result.COMPLETED); + } + @Test default void deleteExpiredMessagesTaskShouldDoNothingWhenEmpty() throws InterruptedException { getVault().deleteExpiredMessagesTask().run(); @@ -288,6 +313,16 @@ default void deleteExpiredMessagesTaskShouldNotDeleteRecentMails() throws Interr .containsOnly(DELETED_MESSAGE); } + @Test + default void deleteExpiredMessagesSingleBucketTaskShouldNotDeleteRecentMails() throws InterruptedException { + Mono.from(getVault().append(DELETED_MESSAGE, new ByteArrayInputStream(CONTENT))).block(); + + getVault().deleteExpiredMessagesTask().run(); + + assertThat(Flux.from(getVault().search(USERNAME, ALL)).collectList().block()) + .containsOnly(DELETED_MESSAGE); + } + @Test default void deleteExpiredMessagesTaskShouldDeleteOldMails() throws InterruptedException { Mono.from(getVault().appendV1(OLD_DELETED_MESSAGE, new ByteArrayInputStream(CONTENT))).block(); @@ -299,6 +334,17 @@ default void deleteExpiredMessagesTaskShouldDeleteOldMails() throws InterruptedE .isEmpty(); } + @Test + default void deleteExpiredMessagesSingleBucketTaskShouldDeleteOldMails() throws InterruptedException { + Mono.from(getVault().append(OLD_DELETED_MESSAGE, new ByteArrayInputStream(CONTENT))).block(); + + getClock().setInstant(NOW.plusYears(2).toInstant()); + getVault().deleteExpiredMessagesTask().run(); + + assertThat(Flux.from(getVault().search(USERNAME, ALL)).collectList().block()) + .isEmpty(); + } + @Test default void deleteExpiredMessagesTaskShouldDeleteOldMailsWhenRunSeveralTime() throws InterruptedException { Mono.from(getVault().appendV1(OLD_DELETED_MESSAGE, new ByteArrayInputStream(CONTENT))).block(); @@ -314,4 +360,35 @@ default void deleteExpiredMessagesTaskShouldDeleteOldMailsWhenRunSeveralTime() t assertThat(Flux.from(getVault().search(USERNAME_2, ALL)).collectList().block()) .isEmpty(); } + + @Test + default void deleteExpiredMessagesSingleBucketTaskShouldDeleteOldMailsWhenRunSeveralTime() throws InterruptedException { + Mono.from(getVault().append(OLD_DELETED_MESSAGE, new ByteArrayInputStream(CONTENT))).block(); + getClock().setInstant(NOW.plusYears(2).toInstant()); + getVault().deleteExpiredMessagesTask().run(); + + Mono.from(getVault().append(OLD_DELETED_MESSAGE, new ByteArrayInputStream(CONTENT))).block(); + getClock().setInstant(NOW.plusYears(4).toInstant()); + getVault().deleteExpiredMessagesTask().run(); + + assertThat(Flux.from(getVault().search(USERNAME, ALL)).collectList().block()) + .isEmpty(); + assertThat(Flux.from(getVault().search(USERNAME_2, ALL)).collectList().block()) + .isEmpty(); + } + + @Test + default void deleteExpiredMessagesTaskShouldDeleteOldMailsInBothCases() throws InterruptedException { + Mono.from(getVault().appendV1(OLD_DELETED_MESSAGE, new ByteArrayInputStream(CONTENT))).block(); + + getClock().setInstant(NOW.plusYears(2).toInstant()); + + Mono.from(getVault().append(OLD_DELETED_MESSAGE, new ByteArrayInputStream(CONTENT))).block(); + + getClock().setInstant(NOW.plusYears(4).toInstant()); + getVault().deleteExpiredMessagesTask().run(); + + assertThat(Flux.from(getVault().search(USERNAME, ALL)).collectList().block()) + .isEmpty(); + } } diff --git a/mailbox/plugin/deleted-messages-vault/src/test/java/org/apache/james/vault/DeletedMessageVaultHookTest.java b/mailbox/plugin/deleted-messages-vault/src/test/java/org/apache/james/vault/DeletedMessageVaultHookTest.java index 320dcbd2a23..e80675b8b2f 100644 --- a/mailbox/plugin/deleted-messages-vault/src/test/java/org/apache/james/vault/DeletedMessageVaultHookTest.java +++ b/mailbox/plugin/deleted-messages-vault/src/test/java/org/apache/james/vault/DeletedMessageVaultHookTest.java @@ -24,6 +24,7 @@ import static org.apache.james.vault.DeletedMessageFixture.INTERNAL_DATE; import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatCode; +import static org.mockito.Mockito.mock; import java.nio.charset.StandardCharsets; import java.time.Clock; @@ -54,6 +55,7 @@ import org.apache.james.metrics.tests.RecordingMetricFactory; import org.apache.james.mime4j.dom.Message; import org.apache.james.server.blob.deduplication.BlobStoreFactory; +import org.apache.james.user.api.UsersRepository; import org.apache.james.vault.blob.BlobIdTimeGenerator; import org.apache.james.vault.blob.BlobStoreDeletedMessageVault; import org.apache.james.vault.blob.BucketNameGenerator; @@ -118,6 +120,9 @@ void setUp() throws Exception { clock = Clock.fixed(DELETION_DATE.toInstant(), ZoneOffset.UTC); MemoryBlobStoreDAO blobStoreDAO = new MemoryBlobStoreDAO(); BlobId.Factory blobIdFactory = new PlainBlobId.Factory(); + + UsersRepository usersRepository = mock(UsersRepository.class); + messageVault = new BlobStoreDeletedMessageVault(new RecordingMetricFactory(), new MemoryDeletedMessageMetadataVault(), BlobStoreFactory.builder() .blobStoreDAO(blobStoreDAO) @@ -125,7 +130,7 @@ void setUp() throws Exception { .defaultBucketName() .passthrough(), blobStoreDAO, new BucketNameGenerator(clock), clock, new BlobIdTimeGenerator(blobIdFactory, clock), - VaultConfiguration.ENABLED_DEFAULT); + VaultConfiguration.ENABLED_DEFAULT, usersRepository); DeletedMessageConverter deletedMessageConverter = new DeletedMessageConverter(); diff --git a/mailbox/plugin/deleted-messages-vault/src/test/java/org/apache/james/vault/blob/BlobStoreDeletedMessageVaultTest.java b/mailbox/plugin/deleted-messages-vault/src/test/java/org/apache/james/vault/blob/BlobStoreDeletedMessageVaultTest.java index e372dcc4bc9..e7bcedb9779 100644 --- a/mailbox/plugin/deleted-messages-vault/src/test/java/org/apache/james/vault/blob/BlobStoreDeletedMessageVaultTest.java +++ b/mailbox/plugin/deleted-messages-vault/src/test/java/org/apache/james/vault/blob/BlobStoreDeletedMessageVaultTest.java @@ -27,8 +27,10 @@ import static org.apache.james.vault.DeletedMessageFixture.MESSAGE_ID; import static org.apache.james.vault.DeletedMessageFixture.NOW; import static org.apache.james.vault.DeletedMessageFixture.OLD_DELETED_MESSAGE; +import static org.apache.james.vault.DeletedMessageFixture.PASSWORD; import static org.apache.james.vault.DeletedMessageFixture.SUBJECT; import static org.apache.james.vault.DeletedMessageFixture.USERNAME; +import static org.apache.james.vault.DeletedMessageFixture.USERNAME_2; import static org.apache.james.vault.blob.BlobStoreDeletedMessageVault.APPEND_METRIC_NAME; import static org.apache.james.vault.blob.BlobStoreDeletedMessageVault.DELETE_EXPIRED_MESSAGES_METRIC_NAME; import static org.apache.james.vault.blob.BlobStoreDeletedMessageVault.DELETE_METRIC_NAME; @@ -36,6 +38,8 @@ import static org.apache.james.vault.blob.BlobStoreDeletedMessageVault.SEARCH_METRIC_NAME; import static org.apache.james.vault.search.Query.ALL; import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.mock; import java.io.ByteArrayInputStream; import java.time.Instant; @@ -46,9 +50,11 @@ import org.apache.james.blob.api.BucketName; import org.apache.james.blob.api.PlainBlobId; import org.apache.james.blob.memory.MemoryBlobStoreDAO; +import org.apache.james.domainlist.api.DomainList; import org.apache.james.mailbox.inmemory.InMemoryMessageId; import org.apache.james.metrics.tests.RecordingMetricFactory; import org.apache.james.server.blob.deduplication.BlobStoreFactory; +import org.apache.james.user.memory.MemoryUsersRepository; import org.apache.james.utils.UpdatableTickingClock; import org.apache.james.vault.DeletedMessage; import org.apache.james.vault.DeletedMessageVaultContract; @@ -59,6 +65,7 @@ import org.apache.james.vault.search.Query; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; +import org.mockito.Mockito; import reactor.core.publisher.Flux; import reactor.core.publisher.Mono; @@ -69,11 +76,18 @@ class BlobStoreDeletedMessageVaultTest implements DeletedMessageVaultContract, D private RecordingMetricFactory metricFactory; @BeforeEach - void setUp() { + void setUp() throws Exception { clock = new UpdatableTickingClock(NOW.toInstant()); metricFactory = new RecordingMetricFactory(); MemoryBlobStoreDAO blobStoreDAO = new MemoryBlobStoreDAO(); BlobId.Factory blobIdFactory = new PlainBlobId.Factory(); + + DomainList domainList = mock(DomainList.class); + Mockito.when(domainList.containsDomain(any())).thenReturn(true); + MemoryUsersRepository usersRepository = MemoryUsersRepository.withVirtualHosting(domainList); + usersRepository.addUser(USERNAME, PASSWORD); + usersRepository.addUser(USERNAME_2, PASSWORD); + messageVault = new BlobStoreDeletedMessageVault(metricFactory, new MemoryDeletedMessageMetadataVault(), BlobStoreFactory.builder() .blobStoreDAO(blobStoreDAO) @@ -81,7 +95,7 @@ void setUp() { .defaultBucketName() .passthrough(), blobStoreDAO, new BucketNameGenerator(clock), clock, new BlobIdTimeGenerator(blobIdFactory, clock), - VaultConfiguration.ENABLED_DEFAULT); + VaultConfiguration.ENABLED_DEFAULT, usersRepository); } @Override diff --git a/mailbox/plugin/deleted-messages-vault/src/test/java/org/apache/james/vault/dto/DeletedMessageWithStorageInformationDTOTest.java b/mailbox/plugin/deleted-messages-vault/src/test/java/org/apache/james/vault/dto/DeletedMessageWithStorageInformationDTOTest.java index a05ef7885ec..9e0abcc9ab2 100644 --- a/mailbox/plugin/deleted-messages-vault/src/test/java/org/apache/james/vault/dto/DeletedMessageWithStorageInformationDTOTest.java +++ b/mailbox/plugin/deleted-messages-vault/src/test/java/org/apache/james/vault/dto/DeletedMessageWithStorageInformationDTOTest.java @@ -23,6 +23,7 @@ import static org.apache.james.util.ClassLoaderUtils.getSystemResourceAsString; import static org.apache.james.vault.DeletedMessageFixture.DELETED_MESSAGE; import static org.apache.james.vault.DeletedMessageFixture.DELETED_MESSAGE_WITH_SUBJECT; +import static org.apache.james.vault.DeletedMessageFixture.NOW; import static org.apache.james.vault.dto.DeletedMessageWithStorageInformationDTO.DeletedMessageDTO; import static org.apache.james.vault.dto.DeletedMessageWithStorageInformationDTO.StorageInformationDTO; import static org.apache.james.vault.metadata.DeletedMessageVaultMetadataFixture.STORAGE_INFORMATION; @@ -34,6 +35,8 @@ import org.apache.james.blob.api.PlainBlobId; import org.apache.james.mailbox.inmemory.InMemoryId; import org.apache.james.mailbox.inmemory.InMemoryMessageId; +import org.apache.james.utils.UpdatableTickingClock; +import org.apache.james.vault.blob.BlobIdTimeGenerator; import org.apache.james.vault.metadata.DeletedMessageWithStorageInformation; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @@ -73,7 +76,7 @@ void setup() { .setSerializationInclusion(JsonInclude.Include.NON_ABSENT); this.converter = new DeletedMessageWithStorageInformationConverter( - new PlainBlobId.Factory(), + new BlobIdTimeGenerator(new PlainBlobId.Factory(), new UpdatableTickingClock(NOW.toInstant())), new InMemoryMessageId.Factory(), new InMemoryId.Factory()); } diff --git a/server/protocols/webadmin/webadmin-mailbox-deleted-message-vault/src/test/java/org/apache/james/webadmin/vault/routes/DeletedMessagesVaultRoutesTest.java b/server/protocols/webadmin/webadmin-mailbox-deleted-message-vault/src/test/java/org/apache/james/webadmin/vault/routes/DeletedMessagesVaultRoutesTest.java index c795ae763e5..0fa41057b28 100644 --- a/server/protocols/webadmin/webadmin-mailbox-deleted-message-vault/src/test/java/org/apache/james/webadmin/vault/routes/DeletedMessagesVaultRoutesTest.java +++ b/server/protocols/webadmin/webadmin-mailbox-deleted-message-vault/src/test/java/org/apache/james/webadmin/vault/routes/DeletedMessagesVaultRoutesTest.java @@ -190,10 +190,11 @@ void beforeEach() throws Exception { .defaultBucketName() .passthrough()); clock = new UpdatableTickingClock(OLD_DELETION_DATE.toInstant()); + usersRepository = createUsersRepository(); vault = spy(new BlobStoreDeletedMessageVault(new RecordingMetricFactory(), new MemoryDeletedMessageMetadataVault(), blobStore, blobStoreDAO, new BucketNameGenerator(clock), clock, new BlobIdTimeGenerator(blobIdFactory, clock), - VaultConfiguration.ENABLED_DEFAULT)); + VaultConfiguration.ENABLED_DEFAULT, usersRepository)); InMemoryIntegrationResources inMemoryResource = InMemoryIntegrationResources.defaultResources(); mailboxManager = spy(inMemoryResource.getMailboxManager()); @@ -205,7 +206,6 @@ blobStore, blobStoreDAO, new BucketNameGenerator(clock), clock, zipper = new DeletedMessageZipper(); exportService = new ExportService(blobExporting, blobStore, zipper, vault); QueryTranslator queryTranslator = new QueryTranslator(new InMemoryId.Factory()); - usersRepository = createUsersRepository(); MessageId.Factory messageIdFactory = new InMemoryMessageId.Factory(); webAdminServer = WebAdminUtils.createWebAdminServer( new TasksRoutes(taskManager, jsonTransformer, From eba0223541c03108f42b4fe3c7001b959c3ac482 Mon Sep 17 00:00:00 2001 From: Rene Cordier Date: Mon, 12 Jan 2026 09:57:52 +0700 Subject: [PATCH 07/14] fixup! JAMES-4156 Refactoring the vault purge to take the single vault bucket design into account --- .../james/vault/blob/BlobStoreDeletedMessageVaultTest.java | 2 +- .../integration/vault/DeletedMessageVaultIntegrationTest.java | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/mailbox/plugin/deleted-messages-vault/src/test/java/org/apache/james/vault/blob/BlobStoreDeletedMessageVaultTest.java b/mailbox/plugin/deleted-messages-vault/src/test/java/org/apache/james/vault/blob/BlobStoreDeletedMessageVaultTest.java index e7bcedb9779..a521f9b80f3 100644 --- a/mailbox/plugin/deleted-messages-vault/src/test/java/org/apache/james/vault/blob/BlobStoreDeletedMessageVaultTest.java +++ b/mailbox/plugin/deleted-messages-vault/src/test/java/org/apache/james/vault/blob/BlobStoreDeletedMessageVaultTest.java @@ -185,7 +185,7 @@ void deleteExpiredMessagesTaskShouldPublishRetentionTimerMetrics() throws Except getVault().deleteExpiredMessagesTask().run(); assertThat(metricFactory.executionTimesFor(DELETE_EXPIRED_MESSAGES_METRIC_NAME)) - .hasSize(1); + .hasSize(2); } @Test diff --git a/server/protocols/webadmin-integration-test/webadmin-integration-test-common/src/main/java/org/apache/james/webadmin/integration/vault/DeletedMessageVaultIntegrationTest.java b/server/protocols/webadmin-integration-test/webadmin-integration-test-common/src/main/java/org/apache/james/webadmin/integration/vault/DeletedMessageVaultIntegrationTest.java index abd5315a5d3..ff2a392bcb9 100644 --- a/server/protocols/webadmin-integration-test/webadmin-integration-test-common/src/main/java/org/apache/james/webadmin/integration/vault/DeletedMessageVaultIntegrationTest.java +++ b/server/protocols/webadmin-integration-test/webadmin-integration-test-common/src/main/java/org/apache/james/webadmin/integration/vault/DeletedMessageVaultIntegrationTest.java @@ -76,6 +76,7 @@ import org.hamcrest.Matchers; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtensionContext; @@ -770,6 +771,7 @@ void vaultDeleteShouldNotAppendMessageToTheUserMailbox() { .hasSize(0); } + @Disabled("JAMES-4156") @Test void vaultDeleteShouldDeleteAllMessagesHavingSameBlobContent() throws Exception { bartSendMessageToHomerAndJack(); From 205cbc6a1fd54d1f8c863b8467310bdd25f49975 Mon Sep 17 00:00:00 2001 From: Rene Cordier Date: Mon, 12 Jan 2026 11:18:23 +0700 Subject: [PATCH 08/14] JAMES-4156 add a value to count deleted blobs during BlobStoreVaultGarbageCollectionTask execution --- .../blob/BlobStoreDeletedMessageVault.java | 6 +- .../BlobStoreVaultGarbageCollectionTask.java | 35 ++++++- ...ollectionTaskAdditionalInformationDTO.java | 9 ++ ...arbageCollectionTaskSerializationTest.java | 5 +- .../DeletedMessagesVaultRoutesTest.java | 96 ++++++++++++++++++- 5 files changed, 140 insertions(+), 11 deletions(-) diff --git a/mailbox/plugin/deleted-messages-vault/src/main/java/org/apache/james/vault/blob/BlobStoreDeletedMessageVault.java b/mailbox/plugin/deleted-messages-vault/src/main/java/org/apache/james/vault/blob/BlobStoreDeletedMessageVault.java index 18b80c3dc0d..963f704ce88 100644 --- a/mailbox/plugin/deleted-messages-vault/src/main/java/org/apache/james/vault/blob/BlobStoreDeletedMessageVault.java +++ b/mailbox/plugin/deleted-messages-vault/src/main/java/org/apache/james/vault/blob/BlobStoreDeletedMessageVault.java @@ -43,6 +43,7 @@ import org.apache.james.vault.DeletedMessageContentNotFoundException; import org.apache.james.vault.DeletedMessageVault; import org.apache.james.vault.VaultConfiguration; +import org.apache.james.vault.blob.BlobStoreVaultGarbageCollectionTask.BlobStoreVaultGarbageCollectionContext; import org.apache.james.vault.metadata.DeletedMessageMetadataVault; import org.apache.james.vault.metadata.DeletedMessageWithStorageInformation; import org.apache.james.vault.metadata.StorageInformation; @@ -241,7 +242,7 @@ private Mono deleteBucketData(BucketName bucketName) { .then(Mono.from(messageMetadataVault.removeMetadataRelatedToBucket(bucketName))); } - Mono deleteUserExpiredMessages(ZonedDateTime beginningOfRetentionPeriod) { + Mono deleteUserExpiredMessages(ZonedDateTime beginningOfRetentionPeriod, BlobStoreVaultGarbageCollectionContext context) { BucketName bucketName = BucketName.of(vaultConfiguration.getSingleBucketName()); return Flux.from(metricFactory.decoratePublisherWithTimerMetric( @@ -249,7 +250,8 @@ Mono deleteUserExpiredMessages(ZonedDateTime beginningOfRetentionPeriod) { Flux.from(usersRepository.listReactive()) .flatMap(username -> Flux.from(messageMetadataVault.listMessages(bucketName, username)) .filter(deletedMessage -> isMessageFullyExpired(beginningOfRetentionPeriod, deletedMessage)) - .flatMap(deletedMessage -> messageMetadataVault.remove(bucketName, username, deletedMessage.getDeletedMessage().getMessageId()))))) + .flatMap(deletedMessage -> Mono.from(messageMetadataVault.remove(bucketName, username, deletedMessage.getDeletedMessage().getMessageId())) + .doOnSuccess(any -> context.recordDeletedBlobSuccess()))))) .then(); } diff --git a/mailbox/plugin/deleted-messages-vault/src/main/java/org/apache/james/vault/blob/BlobStoreVaultGarbageCollectionTask.java b/mailbox/plugin/deleted-messages-vault/src/main/java/org/apache/james/vault/blob/BlobStoreVaultGarbageCollectionTask.java index 75f8e7b3c99..830b684610c 100644 --- a/mailbox/plugin/deleted-messages-vault/src/main/java/org/apache/james/vault/blob/BlobStoreVaultGarbageCollectionTask.java +++ b/mailbox/plugin/deleted-messages-vault/src/main/java/org/apache/james/vault/blob/BlobStoreVaultGarbageCollectionTask.java @@ -26,6 +26,7 @@ import java.util.List; import java.util.Optional; import java.util.concurrent.ConcurrentLinkedQueue; +import java.util.concurrent.atomic.AtomicInteger; import jakarta.inject.Inject; @@ -39,14 +40,32 @@ public class BlobStoreVaultGarbageCollectionTask implements Task { + static class BlobStoreVaultGarbageCollectionContext { + private final AtomicInteger deletedBlobs; + + BlobStoreVaultGarbageCollectionContext() { + this.deletedBlobs = new AtomicInteger(0); + } + + void recordDeletedBlobSuccess() { + deletedBlobs.incrementAndGet(); + } + + int deletedBlobsCount() { + return deletedBlobs.get(); + } + } + public static class AdditionalInformation implements TaskExecutionDetails.AdditionalInformation { private final ZonedDateTime beginningOfRetentionPeriod; private final ImmutableSet deletedBuckets; + private final int deletedBlobs; private final Instant timestamp; - AdditionalInformation(ZonedDateTime beginningOfRetentionPeriod, ImmutableSet deletedBuckets, Instant timestamp) { + AdditionalInformation(ZonedDateTime beginningOfRetentionPeriod, ImmutableSet deletedBuckets, int deletedBlobs, Instant timestamp) { this.beginningOfRetentionPeriod = beginningOfRetentionPeriod; this.deletedBuckets = deletedBuckets; + this.deletedBlobs = deletedBlobs; this.timestamp = timestamp; } @@ -60,6 +79,10 @@ public List getDeletedBuckets() { .collect(ImmutableList.toImmutableList()); } + public int getDeletedBlobs() { + return deletedBlobs; + } + @Override public Instant timestamp() { return timestamp; @@ -68,6 +91,7 @@ public Instant timestamp() { static final TaskType TYPE = TaskType.of("deleted-messages-blob-store-based-garbage-collection"); private final Collection deletedBuckets; + private final BlobStoreVaultGarbageCollectionContext context; private final BlobStoreDeletedMessageVault deletedMessageVault; private final ZonedDateTime beginningOfRetentionPeriod; @@ -88,13 +112,14 @@ private BlobStoreVaultGarbageCollectionTask(BlobStoreDeletedMessageVault deleted this.beginningOfRetentionPeriod = deletedMessageVault.getBeginningOfRetentionPeriod(); this.deletedMessageVault = deletedMessageVault; this.deletedBuckets = new ConcurrentLinkedQueue<>(); + this.context = new BlobStoreVaultGarbageCollectionContext(); } @Override public Result run() { deletedMessageVault.deleteExpiredMessages(beginningOfRetentionPeriod) .doOnNext(deletedBuckets::add) - .then(deletedMessageVault.deleteUserExpiredMessages(beginningOfRetentionPeriod)) + .then(deletedMessageVault.deleteUserExpiredMessages(beginningOfRetentionPeriod, context)) .block(); return Result.COMPLETED; @@ -107,6 +132,10 @@ public TaskType type() { @Override public Optional details() { - return Optional.of(new AdditionalInformation(beginningOfRetentionPeriod, ImmutableSet.copyOf(deletedBuckets), Clock.systemUTC().instant())); + return Optional.of(new AdditionalInformation( + beginningOfRetentionPeriod, + ImmutableSet.copyOf(deletedBuckets), + context.deletedBlobsCount(), + Clock.systemUTC().instant())); } } diff --git a/mailbox/plugin/deleted-messages-vault/src/main/java/org/apache/james/vault/blob/BlobStoreVaultGarbageCollectionTaskAdditionalInformationDTO.java b/mailbox/plugin/deleted-messages-vault/src/main/java/org/apache/james/vault/blob/BlobStoreVaultGarbageCollectionTaskAdditionalInformationDTO.java index 169f56b17b9..e75e5dd82a9 100644 --- a/mailbox/plugin/deleted-messages-vault/src/main/java/org/apache/james/vault/blob/BlobStoreVaultGarbageCollectionTaskAdditionalInformationDTO.java +++ b/mailbox/plugin/deleted-messages-vault/src/main/java/org/apache/james/vault/blob/BlobStoreVaultGarbageCollectionTaskAdditionalInformationDTO.java @@ -37,6 +37,7 @@ static BlobStoreVaultGarbageCollectionTaskAdditionalInformationDTO fromDomainObj type, additionalInformation.getBeginningOfRetentionPeriod().toString(), additionalInformation.getDeletedBuckets(), + additionalInformation.getDeletedBlobs(), additionalInformation.timestamp() ); } @@ -61,6 +62,7 @@ public static final AdditionalInformationDTOModule deletedBuckets; + private final int deletedBlobs; private final String type; private final Instant timestamp; @@ -68,10 +70,12 @@ public static final AdditionalInformationDTOModule deletedBuckets, + @JsonProperty("deletedBlobs") int deletedBlobs, @JsonProperty("timestamp") Instant timestamp) { this.type = type; this.beginningOfRetentionPeriod = beginningOfRetentionPeriod; this.deletedBuckets = deletedBuckets; + this.deletedBlobs = deletedBlobs; this.timestamp = timestamp; } @@ -82,6 +86,7 @@ BlobStoreVaultGarbageCollectionTask.AdditionalInformation toDomainObject() { .stream() .map(BucketName::of) .collect(ImmutableSet.toImmutableSet()), + deletedBlobs, timestamp); } @@ -93,6 +98,10 @@ public Collection getDeletedBuckets() { return deletedBuckets; } + public int getDeletedBlobs() { + return deletedBlobs; + } + @Override public String getType() { return type; diff --git a/mailbox/plugin/deleted-messages-vault/src/test/java/org/apache/james/vault/blob/BlobStoreVaultGarbageCollectionTaskSerializationTest.java b/mailbox/plugin/deleted-messages-vault/src/test/java/org/apache/james/vault/blob/BlobStoreVaultGarbageCollectionTaskSerializationTest.java index 26c8cbeee57..b1fbab1720c 100644 --- a/mailbox/plugin/deleted-messages-vault/src/test/java/org/apache/james/vault/blob/BlobStoreVaultGarbageCollectionTaskSerializationTest.java +++ b/mailbox/plugin/deleted-messages-vault/src/test/java/org/apache/james/vault/blob/BlobStoreVaultGarbageCollectionTaskSerializationTest.java @@ -47,13 +47,14 @@ class BlobStoreVaultGarbageCollectionTaskSerializationTest { private static final JsonTaskSerializer TASK_SERIALIZER = JsonTaskSerializer.of(BlobStoreVaultGarbageCollectionTaskDTO.module(TASK_FACTORY)); private static final ZonedDateTime BEGINNING_OF_RETENTION_PERIOD = ZonedDateTime.parse("2019-09-03T15:26:13.356+02:00[Europe/Paris]"); private static final ImmutableSet BUCKET_IDS = ImmutableSet.of(BucketName.of("1"), BucketName.of("2"), BucketName.of("3")); + private static final int DELETED_BLOBS = 5; private static final Flux RETENTION_OPERATION = Flux.fromIterable(BUCKET_IDS); private static final Instant TIMESTAMP = Instant.parse("2018-11-13T12:00:55Z"); - private static final BlobStoreVaultGarbageCollectionTask.AdditionalInformation DETAILS = new BlobStoreVaultGarbageCollectionTask.AdditionalInformation(BEGINNING_OF_RETENTION_PERIOD, BUCKET_IDS, TIMESTAMP); + private static final BlobStoreVaultGarbageCollectionTask.AdditionalInformation DETAILS = new BlobStoreVaultGarbageCollectionTask.AdditionalInformation(BEGINNING_OF_RETENTION_PERIOD, BUCKET_IDS, DELETED_BLOBS, TIMESTAMP); private static final BlobStoreVaultGarbageCollectionTask TASK = TASK_FACTORY.create(); private static final String SERIALIZED_TASK = "{\"type\":\"deleted-messages-blob-store-based-garbage-collection\"}"; - private static final String SERIALIZED_ADDITIONAL_INFORMATION_TASK = "{\"type\":\"deleted-messages-blob-store-based-garbage-collection\", \"beginningOfRetentionPeriod\":\"2019-09-03T15:26:13.356+02:00[Europe/Paris]\",\"deletedBuckets\":[\"1\", \"2\", \"3\"], \"timestamp\": \"2018-11-13T12:00:55Z\"}"; + private static final String SERIALIZED_ADDITIONAL_INFORMATION_TASK = "{\"type\":\"deleted-messages-blob-store-based-garbage-collection\", \"beginningOfRetentionPeriod\":\"2019-09-03T15:26:13.356+02:00[Europe/Paris]\",\"deletedBuckets\":[\"1\", \"2\", \"3\"], \"deletedBlobs\":5, \"timestamp\": \"2018-11-13T12:00:55Z\"}"; private static final JsonTaskAdditionalInformationSerializer JSON_TASK_ADDITIONAL_INFORMATION_SERIALIZER = JsonTaskAdditionalInformationSerializer.of(BlobStoreVaultGarbageCollectionTaskAdditionalInformationDTO.module()); diff --git a/server/protocols/webadmin/webadmin-mailbox-deleted-message-vault/src/test/java/org/apache/james/webadmin/vault/routes/DeletedMessagesVaultRoutesTest.java b/server/protocols/webadmin/webadmin-mailbox-deleted-message-vault/src/test/java/org/apache/james/webadmin/vault/routes/DeletedMessagesVaultRoutesTest.java index 0fa41057b28..65f7ae1a970 100644 --- a/server/protocols/webadmin/webadmin-mailbox-deleted-message-vault/src/test/java/org/apache/james/webadmin/vault/routes/DeletedMessagesVaultRoutesTest.java +++ b/server/protocols/webadmin/webadmin-mailbox-deleted-message-vault/src/test/java/org/apache/james/webadmin/vault/routes/DeletedMessagesVaultRoutesTest.java @@ -2015,7 +2015,7 @@ void purgeShouldReturnATaskCreated() { } @Test - void purgeShouldProduceASuccessfulTaskWithAdditionalInformation() { + void oldPurgeShouldProduceASuccessfulTaskWithAdditionalInformation() { Mono.from(vault.appendV1(DELETED_MESSAGE, new ByteArrayInputStream(CONTENT))).block(); Mono.from(vault.appendV1(DELETED_MESSAGE_2, new ByteArrayInputStream(CONTENT))).block(); @@ -2038,13 +2038,14 @@ void purgeShouldProduceASuccessfulTaskWithAdditionalInformation() { .body("type", is("deleted-messages-blob-store-based-garbage-collection")) .body("additionalInformation.beginningOfRetentionPeriod", is(notNullValue())) .body("additionalInformation.deletedBuckets", contains("deleted-messages-2010-10-01")) + .body("additionalInformation.deletedBlobs", is(0)) .body("startedDate", is(notNullValue())) .body("submitDate", is(notNullValue())) .body("completedDate", is(notNullValue())); } @Test - void purgeShouldNotDeleteNotExpiredMessagesInTheVault() { + void oldPurgeShouldNotDeleteNotExpiredMessagesInTheVault() { DeletedMessage notExpiredMessage = DeletedMessage.builder() .messageId(InMemoryMessageId.of(46)) .originMailboxes(MAILBOX_ID_1, MAILBOX_ID_2) @@ -2080,7 +2081,7 @@ void purgeShouldNotDeleteNotExpiredMessagesInTheVault() { } @Test - void purgeShouldNotAppendMessagesToUserMailbox() throws Exception { + void oldPurgeShouldNotAppendMessagesToUserMailbox() throws Exception { Mono.from(vault.appendV1(DELETED_MESSAGE, new ByteArrayInputStream(CONTENT))).block(); Mono.from(vault.appendV1(DELETED_MESSAGE_2, new ByteArrayInputStream(CONTENT))).block(); @@ -2099,10 +2100,96 @@ void purgeShouldNotAppendMessagesToUserMailbox() throws Exception { .isFalse(); } + @Test + void purgeShouldProduceASuccessfulTaskWithAdditionalInformation() { + Mono.from(vault.append(DELETED_MESSAGE, new ByteArrayInputStream(CONTENT))).block(); + Mono.from(vault.append(DELETED_MESSAGE_2, new ByteArrayInputStream(CONTENT))).block(); + + clock.setInstant(NOW.toInstant()); + + String taskId = + with() + .queryParam("scope", "expired") + .delete() + .jsonPath() + .get("taskId"); + + given() + .basePath(TasksRoutes.BASE) + .when() + .get(taskId + "/await") + .then() + .body("status", is("completed")) + .body("taskId", is(taskId)) + .body("type", is("deleted-messages-blob-store-based-garbage-collection")) + .body("additionalInformation.beginningOfRetentionPeriod", is(notNullValue())) + .body("additionalInformation.deletedBuckets", hasSize(0)) + .body("additionalInformation.deletedBlobs", is(2)) + .body("startedDate", is(notNullValue())) + .body("submitDate", is(notNullValue())) + .body("completedDate", is(notNullValue())); + } + + @Test + void purgeShouldNotDeleteNotExpiredMessagesInTheVault() { + DeletedMessage notExpiredMessage = DeletedMessage.builder() + .messageId(InMemoryMessageId.of(46)) + .originMailboxes(MAILBOX_ID_1, MAILBOX_ID_2) + .user(USERNAME) + .deliveryDate(DELIVERY_DATE) + .deletionDate(ZonedDateTime.ofInstant(clock.instant(), ZoneOffset.UTC)) + .sender(MaybeSender.of(SENDER)) + .recipients(RECIPIENT1, RECIPIENT3) + .hasAttachment(false) + .size(CONTENT.length) + .build(); + + Mono.from(vault.append(DELETED_MESSAGE, new ByteArrayInputStream(CONTENT))).block(); + Mono.from(vault.append(DELETED_MESSAGE_2, new ByteArrayInputStream(CONTENT))).block(); + + clock.setInstant(NOW.toInstant()); + + Mono.from(vault.append(notExpiredMessage, new ByteArrayInputStream(CONTENT))).block(); + + String taskId = + with() + .queryParam("scope", "expired") + .delete() + .jsonPath() + .get("taskId"); + + with() + .basePath(TasksRoutes.BASE) + .get(taskId + "/await"); + + assertThat(Flux.from(vault.search(USERNAME, Query.ALL)).toStream()) + .containsOnly(notExpiredMessage); + } + + @Test + void purgeShouldNotAppendMessagesToUserMailbox() throws Exception { + Mono.from(vault.append(DELETED_MESSAGE, new ByteArrayInputStream(CONTENT))).block(); + Mono.from(vault.append(DELETED_MESSAGE_2, new ByteArrayInputStream(CONTENT))).block(); + + String taskId = + with() + .queryParam("scope", "expired") + .delete() + .jsonPath() + .get("taskId"); + + with() + .basePath(TasksRoutes.BASE) + .get(taskId + "/await"); + + assertThat(hasAnyMail(USERNAME)) + .isFalse(); + } + @Nested class FailingPurgeTest { @Test - void purgeShouldProduceAFailedTaskWhenFailingDeletingBucket() { + void oldPurgeShouldProduceAFailedTaskWhenFailingDeletingBucket() { Mono.from(vault.appendV1(DELETED_MESSAGE, new ByteArrayInputStream(CONTENT))).block(); Mono.from(vault.appendV1(DELETED_MESSAGE_2, new ByteArrayInputStream(CONTENT))).block(); @@ -2129,6 +2216,7 @@ void purgeShouldProduceAFailedTaskWhenFailingDeletingBucket() { .body("type", is("deleted-messages-blob-store-based-garbage-collection")) .body("additionalInformation.beginningOfRetentionPeriod", is(notNullValue())) .body("additionalInformation.deletedBuckets", hasSize(0)) + .body("additionalInformation.deletedBlobs", is(0)) .body("startedDate", is(notNullValue())) .body("submitDate", is(notNullValue())) .body("completedDate", is(nullValue())); From d6195e986858c59120ee4744b41163db3b4deae2 Mon Sep 17 00:00:00 2001 From: Rene Cordier Date: Mon, 12 Jan 2026 14:48:37 +0700 Subject: [PATCH 09/14] fixup! JAMES-4156 add a value to count deleted blobs during BlobStoreVaultGarbageCollectionTask execution --- .../blob/BlobStoreDeletedMessageVault.java | 31 ++++++++++--------- .../BlobStoreVaultGarbageCollectionTask.java | 18 +++++++---- .../BlobStoreDeletedMessageVaultTest.java | 2 +- ...arbageCollectionTaskSerializationTest.java | 6 ---- 4 files changed, 30 insertions(+), 27 deletions(-) diff --git a/mailbox/plugin/deleted-messages-vault/src/main/java/org/apache/james/vault/blob/BlobStoreDeletedMessageVault.java b/mailbox/plugin/deleted-messages-vault/src/main/java/org/apache/james/vault/blob/BlobStoreDeletedMessageVault.java index 963f704ce88..391a6fbb632 100644 --- a/mailbox/plugin/deleted-messages-vault/src/main/java/org/apache/james/vault/blob/BlobStoreDeletedMessageVault.java +++ b/mailbox/plugin/deleted-messages-vault/src/main/java/org/apache/james/vault/blob/BlobStoreDeletedMessageVault.java @@ -206,13 +206,20 @@ public Task deleteExpiredMessagesTask() { return taskFactory.create(); } - @Deprecated - Flux deleteExpiredMessages(ZonedDateTime beginningOfRetentionPeriod) { + Mono deleteExpiredMessages(ZonedDateTime beginningOfRetentionPeriod, BlobStoreVaultGarbageCollectionContext context) { return Flux.from( metricFactory.decoratePublisherWithTimerMetric( DELETE_EXPIRED_MESSAGES_METRIC_NAME, - retentionQualifiedBuckets(beginningOfRetentionPeriod) - .flatMap(bucketName -> deleteBucketData(bucketName).then(Mono.just(bucketName)), DEFAULT_CONCURRENCY))); + deletedExpiredMessagesFromOldBuckets(beginningOfRetentionPeriod, context) + .then(deleteUserExpiredMessages(beginningOfRetentionPeriod, context)))) + .then(); + } + + @Deprecated + private Flux deletedExpiredMessagesFromOldBuckets(ZonedDateTime beginningOfRetentionPeriod, BlobStoreVaultGarbageCollectionContext context) { + return retentionQualifiedBuckets(beginningOfRetentionPeriod) + .flatMap(bucketName -> deleteBucketData(bucketName).then(Mono.just(bucketName)), DEFAULT_CONCURRENCY) + .doOnNext(context::recordDeletedBucketSuccess); } ZonedDateTime getBeginningOfRetentionPeriod() { @@ -242,21 +249,17 @@ private Mono deleteBucketData(BucketName bucketName) { .then(Mono.from(messageMetadataVault.removeMetadataRelatedToBucket(bucketName))); } - Mono deleteUserExpiredMessages(ZonedDateTime beginningOfRetentionPeriod, BlobStoreVaultGarbageCollectionContext context) { + private Mono deleteUserExpiredMessages(ZonedDateTime beginningOfRetentionPeriod, BlobStoreVaultGarbageCollectionContext context) { BucketName bucketName = BucketName.of(vaultConfiguration.getSingleBucketName()); - return Flux.from(metricFactory.decoratePublisherWithTimerMetric( - DELETE_EXPIRED_MESSAGES_METRIC_NAME, - Flux.from(usersRepository.listReactive()) - .flatMap(username -> Flux.from(messageMetadataVault.listMessages(bucketName, username)) - .filter(deletedMessage -> isMessageFullyExpired(beginningOfRetentionPeriod, deletedMessage)) - .flatMap(deletedMessage -> Mono.from(messageMetadataVault.remove(bucketName, username, deletedMessage.getDeletedMessage().getMessageId())) - .doOnSuccess(any -> context.recordDeletedBlobSuccess()))))) + return Flux.from(usersRepository.listReactive()) + .flatMap(username -> Flux.from(messageMetadataVault.listMessages(bucketName, username)) + .filter(deletedMessage -> isMessageFullyExpired(beginningOfRetentionPeriod, deletedMessage)) + .flatMap(deletedMessage -> Mono.from(messageMetadataVault.remove(bucketName, username, deletedMessage.getDeletedMessage().getMessageId())) + .doOnSuccess(any -> context.recordDeletedBlobSuccess()))) .then(); } - - private boolean isMessageFullyExpired(ZonedDateTime beginningOfRetentionPeriod, DeletedMessageWithStorageInformation deletedMessage) { BlobId blobId = deletedMessage.getStorageInformation().getBlobId(); Optional maybeEndDate = blobIdTimeGenerator.blobIdEndTime(blobId); diff --git a/mailbox/plugin/deleted-messages-vault/src/main/java/org/apache/james/vault/blob/BlobStoreVaultGarbageCollectionTask.java b/mailbox/plugin/deleted-messages-vault/src/main/java/org/apache/james/vault/blob/BlobStoreVaultGarbageCollectionTask.java index 830b684610c..6b03f93c9cd 100644 --- a/mailbox/plugin/deleted-messages-vault/src/main/java/org/apache/james/vault/blob/BlobStoreVaultGarbageCollectionTask.java +++ b/mailbox/plugin/deleted-messages-vault/src/main/java/org/apache/james/vault/blob/BlobStoreVaultGarbageCollectionTask.java @@ -41,9 +41,11 @@ public class BlobStoreVaultGarbageCollectionTask implements Task { static class BlobStoreVaultGarbageCollectionContext { + private final Collection deletedBuckets; private final AtomicInteger deletedBlobs; BlobStoreVaultGarbageCollectionContext() { + this.deletedBuckets = new ConcurrentLinkedQueue<>(); this.deletedBlobs = new AtomicInteger(0); } @@ -54,6 +56,14 @@ void recordDeletedBlobSuccess() { int deletedBlobsCount() { return deletedBlobs.get(); } + + void recordDeletedBucketSuccess(BucketName bucketName) { + deletedBuckets.add(bucketName); + } + + ImmutableSet getDeletedBuckets() { + return ImmutableSet.copyOf(deletedBuckets); + } } public static class AdditionalInformation implements TaskExecutionDetails.AdditionalInformation { @@ -90,7 +100,6 @@ public Instant timestamp() { } static final TaskType TYPE = TaskType.of("deleted-messages-blob-store-based-garbage-collection"); - private final Collection deletedBuckets; private final BlobStoreVaultGarbageCollectionContext context; private final BlobStoreDeletedMessageVault deletedMessageVault; private final ZonedDateTime beginningOfRetentionPeriod; @@ -111,15 +120,12 @@ public BlobStoreVaultGarbageCollectionTask create() { private BlobStoreVaultGarbageCollectionTask(BlobStoreDeletedMessageVault deletedMessageVault) { this.beginningOfRetentionPeriod = deletedMessageVault.getBeginningOfRetentionPeriod(); this.deletedMessageVault = deletedMessageVault; - this.deletedBuckets = new ConcurrentLinkedQueue<>(); this.context = new BlobStoreVaultGarbageCollectionContext(); } @Override public Result run() { - deletedMessageVault.deleteExpiredMessages(beginningOfRetentionPeriod) - .doOnNext(deletedBuckets::add) - .then(deletedMessageVault.deleteUserExpiredMessages(beginningOfRetentionPeriod, context)) + deletedMessageVault.deleteExpiredMessages(beginningOfRetentionPeriod, context) .block(); return Result.COMPLETED; @@ -134,7 +140,7 @@ public TaskType type() { public Optional details() { return Optional.of(new AdditionalInformation( beginningOfRetentionPeriod, - ImmutableSet.copyOf(deletedBuckets), + context.getDeletedBuckets(), context.deletedBlobsCount(), Clock.systemUTC().instant())); } diff --git a/mailbox/plugin/deleted-messages-vault/src/test/java/org/apache/james/vault/blob/BlobStoreDeletedMessageVaultTest.java b/mailbox/plugin/deleted-messages-vault/src/test/java/org/apache/james/vault/blob/BlobStoreDeletedMessageVaultTest.java index a521f9b80f3..e7bcedb9779 100644 --- a/mailbox/plugin/deleted-messages-vault/src/test/java/org/apache/james/vault/blob/BlobStoreDeletedMessageVaultTest.java +++ b/mailbox/plugin/deleted-messages-vault/src/test/java/org/apache/james/vault/blob/BlobStoreDeletedMessageVaultTest.java @@ -185,7 +185,7 @@ void deleteExpiredMessagesTaskShouldPublishRetentionTimerMetrics() throws Except getVault().deleteExpiredMessagesTask().run(); assertThat(metricFactory.executionTimesFor(DELETE_EXPIRED_MESSAGES_METRIC_NAME)) - .hasSize(2); + .hasSize(1); } @Test diff --git a/mailbox/plugin/deleted-messages-vault/src/test/java/org/apache/james/vault/blob/BlobStoreVaultGarbageCollectionTaskSerializationTest.java b/mailbox/plugin/deleted-messages-vault/src/test/java/org/apache/james/vault/blob/BlobStoreVaultGarbageCollectionTaskSerializationTest.java index b1fbab1720c..162eb9fa84e 100644 --- a/mailbox/plugin/deleted-messages-vault/src/test/java/org/apache/james/vault/blob/BlobStoreVaultGarbageCollectionTaskSerializationTest.java +++ b/mailbox/plugin/deleted-messages-vault/src/test/java/org/apache/james/vault/blob/BlobStoreVaultGarbageCollectionTaskSerializationTest.java @@ -21,7 +21,6 @@ import static net.javacrumbs.jsonunit.assertj.JsonAssertions.assertThatJson; import static org.assertj.core.api.Assertions.assertThat; -import static org.mockito.ArgumentMatchers.any; import java.io.IOException; import java.time.Instant; @@ -38,8 +37,6 @@ import com.fasterxml.jackson.core.JsonProcessingException; import com.google.common.collect.ImmutableSet; -import reactor.core.publisher.Flux; - class BlobStoreVaultGarbageCollectionTaskSerializationTest { private static final BlobStoreDeletedMessageVault DELETED_MESSAGE_VAULT = Mockito.mock(BlobStoreDeletedMessageVault.class); private static final BlobStoreVaultGarbageCollectionTask.Factory TASK_FACTORY = new BlobStoreVaultGarbageCollectionTask.Factory(DELETED_MESSAGE_VAULT); @@ -48,7 +45,6 @@ class BlobStoreVaultGarbageCollectionTaskSerializationTest { private static final ZonedDateTime BEGINNING_OF_RETENTION_PERIOD = ZonedDateTime.parse("2019-09-03T15:26:13.356+02:00[Europe/Paris]"); private static final ImmutableSet BUCKET_IDS = ImmutableSet.of(BucketName.of("1"), BucketName.of("2"), BucketName.of("3")); private static final int DELETED_BLOBS = 5; - private static final Flux RETENTION_OPERATION = Flux.fromIterable(BUCKET_IDS); private static final Instant TIMESTAMP = Instant.parse("2018-11-13T12:00:55Z"); private static final BlobStoreVaultGarbageCollectionTask.AdditionalInformation DETAILS = new BlobStoreVaultGarbageCollectionTask.AdditionalInformation(BEGINNING_OF_RETENTION_PERIOD, BUCKET_IDS, DELETED_BLOBS, TIMESTAMP); private static final BlobStoreVaultGarbageCollectionTask TASK = TASK_FACTORY.create(); @@ -62,8 +58,6 @@ class BlobStoreVaultGarbageCollectionTaskSerializationTest { static void setUp() { Mockito.when(DELETED_MESSAGE_VAULT.getBeginningOfRetentionPeriod()) .thenReturn(BEGINNING_OF_RETENTION_PERIOD); - Mockito.when(DELETED_MESSAGE_VAULT.deleteExpiredMessages(any())) - .thenReturn(RETENTION_OPERATION); } @Test From a8e419b43b5f81910f562f514d4d1fb36fca054a Mon Sep 17 00:00:00 2001 From: Benoit TELLIER Date: Thu, 15 Jan 2026 17:30:19 +0100 Subject: [PATCH 10/14] JAMES-4164 Remove unneeded BlobIdTimeGenerator::blobIdEndTime This information can be deduced without hacks from the storage information. --- .../james/vault/blob/BlobIdTimeGenerator.java | 15 ------- .../blob/BlobStoreDeletedMessageVault.java | 11 +---- .../vault/blob/BlobIdTimeGeneratorTest.java | 40 ------------------- 3 files changed, 2 insertions(+), 64 deletions(-) diff --git a/mailbox/plugin/deleted-messages-vault/src/main/java/org/apache/james/vault/blob/BlobIdTimeGenerator.java b/mailbox/plugin/deleted-messages-vault/src/main/java/org/apache/james/vault/blob/BlobIdTimeGenerator.java index f8019b114d7..d25de425ca5 100644 --- a/mailbox/plugin/deleted-messages-vault/src/main/java/org/apache/james/vault/blob/BlobIdTimeGenerator.java +++ b/mailbox/plugin/deleted-messages-vault/src/main/java/org/apache/james/vault/blob/BlobIdTimeGenerator.java @@ -20,7 +20,6 @@ package org.apache.james.vault.blob; import java.time.Clock; -import java.time.LocalDate; import java.time.ZonedDateTime; import java.util.Optional; import java.util.UUID; @@ -53,20 +52,6 @@ BlobId currentBlobId() { return new PlainBlobId(String.format(BLOB_ID_GENERATING_FORMAT, year, month, blobIdFactory.of(UUID.randomUUID().toString()).asString())); } - Optional blobIdEndTime(BlobId blobId) { - return Optional.of(BLOB_ID_TIME_PATTERN.matcher(blobId.asString())) - .filter(Matcher::matches) - .map(matcher -> { - int year = Integer.parseInt(matcher.group(1)); - int month = Integer.parseInt(matcher.group(2)); - return firstDayOfNextMonth(year, month); - }); - } - - private ZonedDateTime firstDayOfNextMonth(int year, int month) { - return LocalDate.of(year, month, 1).plusMonths(1).atStartOfDay(clock.getZone()); - } - public BlobId toDeletedMessageBlobId(String blobId) { return Optional.of(BLOB_ID_TIME_PATTERN.matcher(blobId)) .filter(Matcher::matches) diff --git a/mailbox/plugin/deleted-messages-vault/src/main/java/org/apache/james/vault/blob/BlobStoreDeletedMessageVault.java b/mailbox/plugin/deleted-messages-vault/src/main/java/org/apache/james/vault/blob/BlobStoreDeletedMessageVault.java index 391a6fbb632..67652f908e1 100644 --- a/mailbox/plugin/deleted-messages-vault/src/main/java/org/apache/james/vault/blob/BlobStoreDeletedMessageVault.java +++ b/mailbox/plugin/deleted-messages-vault/src/main/java/org/apache/james/vault/blob/BlobStoreDeletedMessageVault.java @@ -29,7 +29,6 @@ import jakarta.inject.Inject; -import org.apache.james.blob.api.BlobId; import org.apache.james.blob.api.BlobStore; import org.apache.james.blob.api.BlobStoreDAO; import org.apache.james.blob.api.BucketName; @@ -261,14 +260,8 @@ private Mono deleteUserExpiredMessages(ZonedDateTime beginningOfRetentionP } private boolean isMessageFullyExpired(ZonedDateTime beginningOfRetentionPeriod, DeletedMessageWithStorageInformation deletedMessage) { - BlobId blobId = deletedMessage.getStorageInformation().getBlobId(); - Optional maybeEndDate = blobIdTimeGenerator.blobIdEndTime(blobId); + ZonedDateTime deletionDate = deletedMessage.getDeletedMessage().getDeletionDate(); - if (maybeEndDate.isEmpty()) { - LOGGER.error("Pattern used for blobId used in deletedMessageVault is invalid and end date cannot be parsed {}", blobId); - } - - return maybeEndDate.map(endDate -> endDate.isBefore(beginningOfRetentionPeriod)) - .orElse(false); + return deletionDate.isBefore(beginningOfRetentionPeriod); } } diff --git a/mailbox/plugin/deleted-messages-vault/src/test/java/org/apache/james/vault/blob/BlobIdTimeGeneratorTest.java b/mailbox/plugin/deleted-messages-vault/src/test/java/org/apache/james/vault/blob/BlobIdTimeGeneratorTest.java index 5e558b412c0..bdd6082459c 100644 --- a/mailbox/plugin/deleted-messages-vault/src/test/java/org/apache/james/vault/blob/BlobIdTimeGeneratorTest.java +++ b/mailbox/plugin/deleted-messages-vault/src/test/java/org/apache/james/vault/blob/BlobIdTimeGeneratorTest.java @@ -19,12 +19,10 @@ package org.apache.james.vault.blob; -import static org.apache.james.vault.blob.BlobIdTimeGenerator.BLOB_ID_GENERATING_FORMAT; import static org.assertj.core.api.Assertions.assertThat; import java.time.Clock; import java.time.Instant; -import java.time.ZonedDateTime; import java.util.UUID; import org.apache.james.blob.api.BlobId; @@ -34,8 +32,6 @@ import org.apache.james.utils.UpdatableTickingClock; import org.junit.jupiter.api.Nested; import org.junit.jupiter.api.Test; -import org.junit.jupiter.params.ParameterizedTest; -import org.junit.jupiter.params.provider.ValueSource; public class BlobIdTimeGeneratorTest { private static final Instant NOW = Instant.parse("2007-07-03T10:15:30.00Z"); @@ -57,14 +53,6 @@ default void currentBlobIdShouldReturnBlobIdFormattedWithYearAndMonthPrefix() { assertThat(prefix).isEqualTo("2007/07"); } - @Test - default void shouldReturnNextMonthAsEndTime() { - BlobId blobId = new PlainBlobId(String.format(BLOB_ID_GENERATING_FORMAT, 2018, 7, blobIdFactory().of(UUID.randomUUID().toString()).asString())); - - assertThat(blobIdTimeGenerator().blobIdEndTime(blobId)) - .contains(ZonedDateTime.parse("2018-08-01T00:00:00.000000000Z[UTC]")); - } - @Test default void shouldReturnProperDeletedMessageBlobIdFromString() { BlobId currentBlobId = blobIdTimeGenerator().currentBlobId(); @@ -82,34 +70,6 @@ default void shouldFallbackToOldDeletedBlobIdFromString() { } } - @Nested - class PlainBlobIdTimeGeneratorTest implements BlobIdTimeGeneratorContract { - @Override - public BlobId.Factory blobIdFactory() { - return new PlainBlobId.Factory(); - } - - @Override - public BlobIdTimeGenerator blobIdTimeGenerator() { - return new BlobIdTimeGenerator(blobIdFactory(), CLOCK); - } - - @ParameterizedTest - @ValueSource(strings = { - "2018-07-cddf9c7c-12f1-4ce7-9993-8606b9fb8816", - "2018-07/cddf9c7c-12f1-4ce7-9993-8606b9fb8816", - "2018/07-cddf9c7c-12f1-4ce7-9993-8606b9fb8816", - "cddf9c7c-12f1-4ce7-9993-8606b9fb8816", - "18/07/cddf9c7c-12f1-4ce7-9993-8606b9fb8816", - "2018/7/cddf9c7c-12f1-4ce7-9993-8606b9fb8816", - "07/2018/cddf9c7c-12f1-4ce7-9993-8606b9fb8816", - }) - public void shouldBeEmptyWhenPassingNonWellFormattedBlobId(String blobIdAsString) { - BlobId blobId = blobIdFactory().of(blobIdAsString); - - assertThat(blobIdTimeGenerator().blobIdEndTime(blobId)).isEmpty(); - } - } @Nested class GenerationAwareBlobIdTimeGeneratorTest implements BlobIdTimeGeneratorContract { From febcdf04a0e3b61b36bddb8dd16382b9dd10e2d5 Mon Sep 17 00:00:00 2001 From: Benoit TELLIER Date: Thu, 15 Jan 2026 17:54:19 +0100 Subject: [PATCH 11/14] JAMES-4164 Simplification for BlobIdTimeGenerator by using underlying PlainBlobId Care is taken not to concentrate all blobs in a single folder --- .../vault/metadata/StorageInformationDAO.java | 8 +-- ...sandraDeletedMessageMetadataVaultTest.java | 10 +-- .../james/vault/metadata/MetadataDAOTest.java | 5 -- .../metadata/StorageInformationDAOTest.java | 6 +- .../PostgresDeletedMessageMetadataVault.java | 9 +-- ...stgresDeletedMessageMetadataVaultTest.java | 13 +--- .../james/vault/blob/BlobIdTimeGenerator.java | 26 ++----- ...essageWithStorageInformationConverter.java | 11 ++- .../vault/DeletedMessageVaultHookTest.java | 2 +- .../vault/blob/BlobIdTimeGeneratorTest.java | 69 ++----------------- .../BlobStoreDeletedMessageVaultTest.java | 2 +- ...dMessageWithStorageInformationDTOTest.java | 5 -- .../DeletedMessagesVaultRoutesTest.java | 2 +- 13 files changed, 29 insertions(+), 139 deletions(-) diff --git a/mailbox/plugin/deleted-messages-vault-cassandra/src/main/java/org/apache/james/vault/metadata/StorageInformationDAO.java b/mailbox/plugin/deleted-messages-vault-cassandra/src/main/java/org/apache/james/vault/metadata/StorageInformationDAO.java index 902bd7fc422..c6a4ed01484 100644 --- a/mailbox/plugin/deleted-messages-vault-cassandra/src/main/java/org/apache/james/vault/metadata/StorageInformationDAO.java +++ b/mailbox/plugin/deleted-messages-vault-cassandra/src/main/java/org/apache/james/vault/metadata/StorageInformationDAO.java @@ -33,9 +33,9 @@ import org.apache.james.backends.cassandra.utils.CassandraAsyncExecutor; import org.apache.james.blob.api.BucketName; +import org.apache.james.blob.api.PlainBlobId; import org.apache.james.core.Username; import org.apache.james.mailbox.model.MessageId; -import org.apache.james.vault.blob.BlobIdTimeGenerator; import com.datastax.oss.driver.api.core.CqlSession; import com.datastax.oss.driver.api.core.cql.PreparedStatement; @@ -47,15 +47,13 @@ public class StorageInformationDAO { private final PreparedStatement addStatement; private final PreparedStatement removeStatement; private final PreparedStatement readStatement; - private final BlobIdTimeGenerator blobIdTimeGenerator; @Inject - StorageInformationDAO(CqlSession session, BlobIdTimeGenerator blobIdTimeGenerator) { + StorageInformationDAO(CqlSession session) { this.cassandraAsyncExecutor = new CassandraAsyncExecutor(session); this.addStatement = prepareAdd(session); this.removeStatement = prepareRemove(session); this.readStatement = prepareRead(session); - this.blobIdTimeGenerator = blobIdTimeGenerator; } private PreparedStatement prepareRead(CqlSession session) { @@ -102,6 +100,6 @@ Mono retrieveStorageInformation(Username username, MessageId .setString(MESSAGE_ID, messageId.serialize())) .map(row -> StorageInformation.builder() .bucketName(BucketName.of(row.getString(BUCKET_NAME))) - .blobId(blobIdTimeGenerator.toDeletedMessageBlobId(row.getString(BLOB_ID)))); + .blobId(new PlainBlobId(row.getString(BLOB_ID)))); } } diff --git a/mailbox/plugin/deleted-messages-vault-cassandra/src/test/java/org/apache/james/vault/metadata/CassandraDeletedMessageMetadataVaultTest.java b/mailbox/plugin/deleted-messages-vault-cassandra/src/test/java/org/apache/james/vault/metadata/CassandraDeletedMessageMetadataVaultTest.java index 7f5640a3942..ad0ec8694fa 100644 --- a/mailbox/plugin/deleted-messages-vault-cassandra/src/test/java/org/apache/james/vault/metadata/CassandraDeletedMessageMetadataVaultTest.java +++ b/mailbox/plugin/deleted-messages-vault-cassandra/src/test/java/org/apache/james/vault/metadata/CassandraDeletedMessageMetadataVaultTest.java @@ -20,7 +20,6 @@ package org.apache.james.vault.metadata; import static org.apache.james.backends.cassandra.Scenario.Builder.fail; -import static org.apache.james.vault.DeletedMessageFixture.NOW; import static org.apache.james.vault.DeletedMessageFixture.USERNAME; import static org.apache.james.vault.metadata.DeletedMessageMetadataDataDefinition.MODULE; import static org.apache.james.vault.metadata.DeletedMessageVaultMetadataFixture.BUCKET_NAME; @@ -33,11 +32,8 @@ import org.apache.james.backends.cassandra.CassandraCluster; import org.apache.james.backends.cassandra.CassandraClusterExtension; -import org.apache.james.blob.api.PlainBlobId; import org.apache.james.mailbox.inmemory.InMemoryId; import org.apache.james.mailbox.inmemory.InMemoryMessageId; -import org.apache.james.utils.UpdatableTickingClock; -import org.apache.james.vault.blob.BlobIdTimeGenerator; import org.apache.james.vault.dto.DeletedMessageWithStorageInformationConverter; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Disabled; @@ -59,13 +55,11 @@ public class CassandraDeletedMessageMetadataVaultTest implements DeletedMessageM @BeforeEach void setUp(CassandraCluster cassandra) { - PlainBlobId.Factory blobIdFactory = new PlainBlobId.Factory(); - BlobIdTimeGenerator blobIdTimeGenerator = new BlobIdTimeGenerator(blobIdFactory, new UpdatableTickingClock(NOW.toInstant())); InMemoryMessageId.Factory messageIdFactory = new InMemoryMessageId.Factory(); - DeletedMessageWithStorageInformationConverter dtoConverter = new DeletedMessageWithStorageInformationConverter(blobIdTimeGenerator, messageIdFactory, new InMemoryId.Factory()); + DeletedMessageWithStorageInformationConverter dtoConverter = new DeletedMessageWithStorageInformationConverter(messageIdFactory, new InMemoryId.Factory()); metadataDAO = new MetadataDAO(cassandra.getConf(), messageIdFactory, new MetadataSerializer(dtoConverter)); - storageInformationDAO = new StorageInformationDAO(cassandra.getConf(), blobIdTimeGenerator); + storageInformationDAO = new StorageInformationDAO(cassandra.getConf()); userPerBucketDAO = new UserPerBucketDAO(cassandra.getConf()); testee = new CassandraDeletedMessageMetadataVault(metadataDAO, storageInformationDAO, userPerBucketDAO); diff --git a/mailbox/plugin/deleted-messages-vault-cassandra/src/test/java/org/apache/james/vault/metadata/MetadataDAOTest.java b/mailbox/plugin/deleted-messages-vault-cassandra/src/test/java/org/apache/james/vault/metadata/MetadataDAOTest.java index 2342db89a2a..760ffe0c2c5 100644 --- a/mailbox/plugin/deleted-messages-vault-cassandra/src/test/java/org/apache/james/vault/metadata/MetadataDAOTest.java +++ b/mailbox/plugin/deleted-messages-vault-cassandra/src/test/java/org/apache/james/vault/metadata/MetadataDAOTest.java @@ -20,7 +20,6 @@ package org.apache.james.vault.metadata; import static org.apache.james.vault.DeletedMessageFixture.MESSAGE_ID; -import static org.apache.james.vault.DeletedMessageFixture.NOW; import static org.apache.james.vault.DeletedMessageFixture.USERNAME; import static org.apache.james.vault.metadata.DeletedMessageMetadataDataDefinition.MODULE; import static org.apache.james.vault.metadata.DeletedMessageVaultMetadataFixture.BUCKET_NAME; @@ -32,12 +31,9 @@ import org.apache.james.backends.cassandra.CassandraCluster; import org.apache.james.backends.cassandra.CassandraClusterExtension; -import org.apache.james.blob.api.PlainBlobId; import org.apache.james.mailbox.inmemory.InMemoryId; import org.apache.james.mailbox.inmemory.InMemoryMessageId; import org.apache.james.mailbox.model.MessageId; -import org.apache.james.utils.UpdatableTickingClock; -import org.apache.james.vault.blob.BlobIdTimeGenerator; import org.apache.james.vault.dto.DeletedMessageWithStorageInformationConverter; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @@ -52,7 +48,6 @@ class MetadataDAOTest { @BeforeEach void setUp(CassandraCluster cassandra) { DeletedMessageWithStorageInformationConverter dtoConverter = new DeletedMessageWithStorageInformationConverter( - new BlobIdTimeGenerator(new PlainBlobId.Factory(), new UpdatableTickingClock(NOW.toInstant())), new InMemoryMessageId.Factory(), new InMemoryId.Factory()); diff --git a/mailbox/plugin/deleted-messages-vault-cassandra/src/test/java/org/apache/james/vault/metadata/StorageInformationDAOTest.java b/mailbox/plugin/deleted-messages-vault-cassandra/src/test/java/org/apache/james/vault/metadata/StorageInformationDAOTest.java index 0cc66a6db25..1ce67d95d42 100644 --- a/mailbox/plugin/deleted-messages-vault-cassandra/src/test/java/org/apache/james/vault/metadata/StorageInformationDAOTest.java +++ b/mailbox/plugin/deleted-messages-vault-cassandra/src/test/java/org/apache/james/vault/metadata/StorageInformationDAOTest.java @@ -19,7 +19,6 @@ package org.apache.james.vault.metadata; -import static org.apache.james.vault.DeletedMessageFixture.NOW; import static org.apache.james.vault.metadata.DeletedMessageMetadataDataDefinition.MODULE; import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatCode; @@ -33,8 +32,6 @@ import org.apache.james.blob.api.PlainBlobId; import org.apache.james.core.Username; import org.apache.james.mailbox.model.TestMessageId; -import org.apache.james.utils.UpdatableTickingClock; -import org.apache.james.vault.blob.BlobIdTimeGenerator; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.RegisterExtension; @@ -42,7 +39,6 @@ class StorageInformationDAOTest { private static final BucketName BUCKET_NAME = BucketName.of("deletedMessages-2019-06-01"); private static final BucketName BUCKET_NAME_2 = BucketName.of("deletedMessages-2019-07-01"); - private static final BlobIdTimeGenerator BLOB_ID_TIME_GENERATOR = new BlobIdTimeGenerator(new PlainBlobId.Factory(), new UpdatableTickingClock(NOW.toInstant())); private static final Username OWNER = Username.of("owner"); private static final TestMessageId MESSAGE_ID = TestMessageId.of(36); private static final BlobId BLOB_ID = new PlainBlobId.Factory().parse("05dcb33b-8382-4744-923a-bc593ad84d23"); @@ -57,7 +53,7 @@ class StorageInformationDAOTest { @BeforeEach void setUp(CassandraCluster cassandra) { - testee = new StorageInformationDAO(cassandra.getConf(), BLOB_ID_TIME_GENERATOR); + testee = new StorageInformationDAO(cassandra.getConf()); } @Test diff --git a/mailbox/plugin/deleted-messages-vault-postgres/src/main/java/org/apache/james/vault/metadata/PostgresDeletedMessageMetadataVault.java b/mailbox/plugin/deleted-messages-vault-postgres/src/main/java/org/apache/james/vault/metadata/PostgresDeletedMessageMetadataVault.java index 2f5c4c0b165..db36fc25424 100644 --- a/mailbox/plugin/deleted-messages-vault-postgres/src/main/java/org/apache/james/vault/metadata/PostgresDeletedMessageMetadataVault.java +++ b/mailbox/plugin/deleted-messages-vault-postgres/src/main/java/org/apache/james/vault/metadata/PostgresDeletedMessageMetadataVault.java @@ -34,9 +34,9 @@ import org.apache.james.backends.postgres.utils.PostgresExecutor; import org.apache.james.blob.api.BucketName; +import org.apache.james.blob.api.PlainBlobId; import org.apache.james.core.Username; import org.apache.james.mailbox.model.MessageId; -import org.apache.james.vault.blob.BlobIdTimeGenerator; import org.jooq.Record; import org.reactivestreams.Publisher; @@ -46,15 +46,12 @@ public class PostgresDeletedMessageMetadataVault implements DeletedMessageMetadataVault { private final PostgresExecutor postgresExecutor; private final MetadataSerializer metadataSerializer; - private final BlobIdTimeGenerator blobIdTimeGenerator; @Inject public PostgresDeletedMessageMetadataVault(PostgresExecutor postgresExecutor, - MetadataSerializer metadataSerializer, - BlobIdTimeGenerator blobIdTimeGenerator) { + MetadataSerializer metadataSerializer) { this.postgresExecutor = postgresExecutor; this.metadataSerializer = metadataSerializer; - this.blobIdTimeGenerator = blobIdTimeGenerator; } @Override @@ -93,7 +90,7 @@ public Publisher retrieveStorageInformation(Username usernam private Function toStorageInformation() { return record -> StorageInformation.builder() .bucketName(BucketName.of(record.get(BUCKET_NAME))) - .blobId(blobIdTimeGenerator.toDeletedMessageBlobId(record.get(BLOB_ID))); + .blobId(new PlainBlobId(record.get(BLOB_ID))); } @Override diff --git a/mailbox/plugin/deleted-messages-vault-postgres/src/test/java/org/apache/james/vault/metadata/PostgresDeletedMessageMetadataVaultTest.java b/mailbox/plugin/deleted-messages-vault-postgres/src/test/java/org/apache/james/vault/metadata/PostgresDeletedMessageMetadataVaultTest.java index 66fb7737730..c27c386238d 100644 --- a/mailbox/plugin/deleted-messages-vault-postgres/src/test/java/org/apache/james/vault/metadata/PostgresDeletedMessageMetadataVaultTest.java +++ b/mailbox/plugin/deleted-messages-vault-postgres/src/test/java/org/apache/james/vault/metadata/PostgresDeletedMessageMetadataVaultTest.java @@ -19,16 +19,10 @@ package org.apache.james.vault.metadata; -import static org.apache.james.vault.DeletedMessageFixture.NOW; - import org.apache.james.backends.postgres.PostgresDataDefinition; import org.apache.james.backends.postgres.PostgresExtension; -import org.apache.james.blob.api.BlobId; -import org.apache.james.blob.api.PlainBlobId; import org.apache.james.mailbox.inmemory.InMemoryId; import org.apache.james.mailbox.inmemory.InMemoryMessageId; -import org.apache.james.utils.UpdatableTickingClock; -import org.apache.james.vault.blob.BlobIdTimeGenerator; import org.apache.james.vault.dto.DeletedMessageWithStorageInformationConverter; import org.junit.jupiter.api.extension.RegisterExtension; @@ -39,14 +33,11 @@ class PostgresDeletedMessageMetadataVaultTest implements DeletedMessageMetadataV @Override public DeletedMessageMetadataVault metadataVault() { - BlobId.Factory blobIdFactory = new PlainBlobId.Factory(); - BlobIdTimeGenerator blobIdTimeGenerator = new BlobIdTimeGenerator(blobIdFactory, new UpdatableTickingClock(NOW.toInstant())); InMemoryMessageId.Factory messageIdFactory = new InMemoryMessageId.Factory(); - DeletedMessageWithStorageInformationConverter dtoConverter = new DeletedMessageWithStorageInformationConverter(blobIdTimeGenerator, + DeletedMessageWithStorageInformationConverter dtoConverter = new DeletedMessageWithStorageInformationConverter( messageIdFactory, new InMemoryId.Factory()); return new PostgresDeletedMessageMetadataVault(postgresExtension.getDefaultPostgresExecutor(), - new MetadataSerializer(dtoConverter), - blobIdTimeGenerator); + new MetadataSerializer(dtoConverter)); } } diff --git a/mailbox/plugin/deleted-messages-vault/src/main/java/org/apache/james/vault/blob/BlobIdTimeGenerator.java b/mailbox/plugin/deleted-messages-vault/src/main/java/org/apache/james/vault/blob/BlobIdTimeGenerator.java index d25de425ca5..bad695e31dd 100644 --- a/mailbox/plugin/deleted-messages-vault/src/main/java/org/apache/james/vault/blob/BlobIdTimeGenerator.java +++ b/mailbox/plugin/deleted-messages-vault/src/main/java/org/apache/james/vault/blob/BlobIdTimeGenerator.java @@ -21,26 +21,21 @@ import java.time.Clock; import java.time.ZonedDateTime; -import java.util.Optional; import java.util.UUID; -import java.util.regex.Matcher; -import java.util.regex.Pattern; import jakarta.inject.Inject; +import org.apache.commons.lang3.RandomStringUtils; import org.apache.james.blob.api.BlobId; import org.apache.james.blob.api.PlainBlobId; public class BlobIdTimeGenerator { - public static final String BLOB_ID_GENERATING_FORMAT = "%d/%02d/%s"; - public static final Pattern BLOB_ID_TIME_PATTERN = Pattern.compile("^(\\d{4})/(\\d{2})/(.*)$"); + public static final String BLOB_ID_GENERATING_FORMAT = "%d/%02d/%s/%s/%s"; - private final BlobId.Factory blobIdFactory; private final Clock clock; @Inject - public BlobIdTimeGenerator(BlobId.Factory blobIdFactory, Clock clock) { - this.blobIdFactory = blobIdFactory; + public BlobIdTimeGenerator(Clock clock) { this.clock = clock; } @@ -48,18 +43,9 @@ BlobId currentBlobId() { ZonedDateTime now = ZonedDateTime.now(clock); int month = now.getMonthValue(); int year = now.getYear(); + String randomizerA = RandomStringUtils.insecure().nextAlphanumeric(1); + String randomizerB = RandomStringUtils.insecure().nextAlphanumeric(1); - return new PlainBlobId(String.format(BLOB_ID_GENERATING_FORMAT, year, month, blobIdFactory.of(UUID.randomUUID().toString()).asString())); - } - - public BlobId toDeletedMessageBlobId(String blobId) { - return Optional.of(BLOB_ID_TIME_PATTERN.matcher(blobId)) - .filter(Matcher::matches) - .map(matcher -> { - int year = Integer.parseInt(matcher.group(1)); - int month = Integer.parseInt(matcher.group(2)); - String subBlobId = matcher.group(3); - return (BlobId) new PlainBlobId(String.format(BLOB_ID_GENERATING_FORMAT, year, month, blobIdFactory.parse(subBlobId).asString())); - }).orElseGet(() -> blobIdFactory.parse(blobId)); + return new PlainBlobId(String.format(BLOB_ID_GENERATING_FORMAT, year, month, randomizerA, randomizerB, UUID.randomUUID())); } } diff --git a/mailbox/plugin/deleted-messages-vault/src/main/java/org/apache/james/vault/dto/DeletedMessageWithStorageInformationConverter.java b/mailbox/plugin/deleted-messages-vault/src/main/java/org/apache/james/vault/dto/DeletedMessageWithStorageInformationConverter.java index fe1492cd1d3..2ce1146d01b 100644 --- a/mailbox/plugin/deleted-messages-vault/src/main/java/org/apache/james/vault/dto/DeletedMessageWithStorageInformationConverter.java +++ b/mailbox/plugin/deleted-messages-vault/src/main/java/org/apache/james/vault/dto/DeletedMessageWithStorageInformationConverter.java @@ -26,13 +26,13 @@ import jakarta.mail.internet.AddressException; import org.apache.james.blob.api.BucketName; +import org.apache.james.blob.api.PlainBlobId; import org.apache.james.core.MailAddress; import org.apache.james.core.MaybeSender; import org.apache.james.core.Username; import org.apache.james.mailbox.model.MailboxId; import org.apache.james.mailbox.model.MessageId; import org.apache.james.vault.DeletedMessage; -import org.apache.james.vault.blob.BlobIdTimeGenerator; import org.apache.james.vault.metadata.DeletedMessageWithStorageInformation; import org.apache.james.vault.metadata.StorageInformation; @@ -40,15 +40,12 @@ import com.google.common.collect.ImmutableList; public class DeletedMessageWithStorageInformationConverter { - private final BlobIdTimeGenerator blobIdTimeGenerator; private final MessageId.Factory messageIdFactory; private final MailboxId.Factory mailboxIdFactory; @Inject - public DeletedMessageWithStorageInformationConverter(BlobIdTimeGenerator blobIdTimeGenerator, - MessageId.Factory messageIdFactory, + public DeletedMessageWithStorageInformationConverter(MessageId.Factory messageIdFactory, MailboxId.Factory mailboxIdFactory) { - this.blobIdTimeGenerator = blobIdTimeGenerator; this.messageIdFactory = messageIdFactory; this.mailboxIdFactory = mailboxIdFactory; } @@ -56,7 +53,7 @@ public DeletedMessageWithStorageInformationConverter(BlobIdTimeGenerator blobIdT public StorageInformation toDomainObject(DeletedMessageWithStorageInformationDTO.StorageInformationDTO storageInformationDTO) { return StorageInformation.builder() .bucketName(BucketName.of(storageInformationDTO.getBucketName())) - .blobId(blobIdTimeGenerator.toDeletedMessageBlobId(storageInformationDTO.getBlobId())); + .blobId(new PlainBlobId(storageInformationDTO.getBlobId())); } public DeletedMessage toDomainObject(DeletedMessageWithStorageInformationDTO.DeletedMessageDTO deletedMessageDTO) throws AddressException { @@ -82,7 +79,7 @@ public DeletedMessageWithStorageInformation toDomainObject(DeletedMessageWithSto private ImmutableList deserializeOriginMailboxes(List originMailboxes) { return originMailboxes.stream() - .map(mailboxId -> mailboxIdFactory.fromString(mailboxId)) + .map(mailboxIdFactory::fromString) .collect(ImmutableList.toImmutableList()); } diff --git a/mailbox/plugin/deleted-messages-vault/src/test/java/org/apache/james/vault/DeletedMessageVaultHookTest.java b/mailbox/plugin/deleted-messages-vault/src/test/java/org/apache/james/vault/DeletedMessageVaultHookTest.java index e80675b8b2f..edfbeeab7c4 100644 --- a/mailbox/plugin/deleted-messages-vault/src/test/java/org/apache/james/vault/DeletedMessageVaultHookTest.java +++ b/mailbox/plugin/deleted-messages-vault/src/test/java/org/apache/james/vault/DeletedMessageVaultHookTest.java @@ -129,7 +129,7 @@ void setUp() throws Exception { .blobIdFactory(blobIdFactory) .defaultBucketName() .passthrough(), blobStoreDAO, new BucketNameGenerator(clock), clock, - new BlobIdTimeGenerator(blobIdFactory, clock), + new BlobIdTimeGenerator(clock), VaultConfiguration.ENABLED_DEFAULT, usersRepository); DeletedMessageConverter deletedMessageConverter = new DeletedMessageConverter(); diff --git a/mailbox/plugin/deleted-messages-vault/src/test/java/org/apache/james/vault/blob/BlobIdTimeGeneratorTest.java b/mailbox/plugin/deleted-messages-vault/src/test/java/org/apache/james/vault/blob/BlobIdTimeGeneratorTest.java index bdd6082459c..fc50ac8a50b 100644 --- a/mailbox/plugin/deleted-messages-vault/src/test/java/org/apache/james/vault/blob/BlobIdTimeGeneratorTest.java +++ b/mailbox/plugin/deleted-messages-vault/src/test/java/org/apache/james/vault/blob/BlobIdTimeGeneratorTest.java @@ -23,77 +23,18 @@ import java.time.Clock; import java.time.Instant; -import java.util.UUID; -import org.apache.james.blob.api.BlobId; -import org.apache.james.blob.api.PlainBlobId; -import org.apache.james.server.blob.deduplication.GenerationAwareBlobId; -import org.apache.james.server.blob.deduplication.MinIOGenerationAwareBlobId; import org.apache.james.utils.UpdatableTickingClock; -import org.junit.jupiter.api.Nested; import org.junit.jupiter.api.Test; -public class BlobIdTimeGeneratorTest { +class BlobIdTimeGeneratorTest { private static final Instant NOW = Instant.parse("2007-07-03T10:15:30.00Z"); private static final Clock CLOCK = new UpdatableTickingClock(NOW); - interface BlobIdTimeGeneratorContract { - BlobId.Factory blobIdFactory(); + @Test + void currentBlobIdShouldReturnBlobIdFormattedWithYearAndMonthPrefix() { + String currentBlobId = new BlobIdTimeGenerator(CLOCK).currentBlobId().asString(); - BlobIdTimeGenerator blobIdTimeGenerator(); - - @Test - default void currentBlobIdShouldReturnBlobIdFormattedWithYearAndMonthPrefix() { - String currentBlobId = blobIdTimeGenerator().currentBlobId().asString(); - - int firstSlash = currentBlobId.indexOf('/'); - int secondSlash = currentBlobId.indexOf('/', firstSlash + 1); - String prefix = currentBlobId.substring(0, secondSlash); - - assertThat(prefix).isEqualTo("2007/07"); - } - - @Test - default void shouldReturnProperDeletedMessageBlobIdFromString() { - BlobId currentBlobId = blobIdTimeGenerator().currentBlobId(); - BlobId deletedMessageBlobId = blobIdTimeGenerator().toDeletedMessageBlobId(currentBlobId.asString()); - - assertThat(deletedMessageBlobId).isEqualTo(currentBlobId); - } - - @Test - default void shouldFallbackToOldDeletedBlobIdFromString() { - BlobId currentBlobId = blobIdFactory().of(UUID.randomUUID().toString()); - BlobId deletedMessageBlobId = blobIdTimeGenerator().toDeletedMessageBlobId(currentBlobId.asString()); - - assertThat(deletedMessageBlobId).isEqualTo(currentBlobId); - } - } - - - @Nested - class GenerationAwareBlobIdTimeGeneratorTest implements BlobIdTimeGeneratorContract { - @Override - public BlobId.Factory blobIdFactory() { - return new GenerationAwareBlobId.Factory(CLOCK, new PlainBlobId.Factory(), GenerationAwareBlobId.Configuration.DEFAULT); - } - - @Override - public BlobIdTimeGenerator blobIdTimeGenerator() { - return new BlobIdTimeGenerator(blobIdFactory(), CLOCK); - } - } - - @Nested - class MinIOGenerationAwareBlobIdTimeGeneratorTest implements BlobIdTimeGeneratorContract { - @Override - public BlobId.Factory blobIdFactory() { - return new MinIOGenerationAwareBlobId.Factory(CLOCK, GenerationAwareBlobId.Configuration.DEFAULT, new PlainBlobId.Factory()); - } - - @Override - public BlobIdTimeGenerator blobIdTimeGenerator() { - return new BlobIdTimeGenerator(blobIdFactory(), CLOCK); - } + assertThat(currentBlobId).matches("2007/07/././.*"); } } diff --git a/mailbox/plugin/deleted-messages-vault/src/test/java/org/apache/james/vault/blob/BlobStoreDeletedMessageVaultTest.java b/mailbox/plugin/deleted-messages-vault/src/test/java/org/apache/james/vault/blob/BlobStoreDeletedMessageVaultTest.java index e7bcedb9779..bfa430740a6 100644 --- a/mailbox/plugin/deleted-messages-vault/src/test/java/org/apache/james/vault/blob/BlobStoreDeletedMessageVaultTest.java +++ b/mailbox/plugin/deleted-messages-vault/src/test/java/org/apache/james/vault/blob/BlobStoreDeletedMessageVaultTest.java @@ -94,7 +94,7 @@ void setUp() throws Exception { .blobIdFactory(blobIdFactory) .defaultBucketName() .passthrough(), - blobStoreDAO, new BucketNameGenerator(clock), clock, new BlobIdTimeGenerator(blobIdFactory, clock), + blobStoreDAO, new BucketNameGenerator(clock), clock, new BlobIdTimeGenerator(clock), VaultConfiguration.ENABLED_DEFAULT, usersRepository); } diff --git a/mailbox/plugin/deleted-messages-vault/src/test/java/org/apache/james/vault/dto/DeletedMessageWithStorageInformationDTOTest.java b/mailbox/plugin/deleted-messages-vault/src/test/java/org/apache/james/vault/dto/DeletedMessageWithStorageInformationDTOTest.java index 9e0abcc9ab2..a9099a7e70b 100644 --- a/mailbox/plugin/deleted-messages-vault/src/test/java/org/apache/james/vault/dto/DeletedMessageWithStorageInformationDTOTest.java +++ b/mailbox/plugin/deleted-messages-vault/src/test/java/org/apache/james/vault/dto/DeletedMessageWithStorageInformationDTOTest.java @@ -23,7 +23,6 @@ import static org.apache.james.util.ClassLoaderUtils.getSystemResourceAsString; import static org.apache.james.vault.DeletedMessageFixture.DELETED_MESSAGE; import static org.apache.james.vault.DeletedMessageFixture.DELETED_MESSAGE_WITH_SUBJECT; -import static org.apache.james.vault.DeletedMessageFixture.NOW; import static org.apache.james.vault.dto.DeletedMessageWithStorageInformationDTO.DeletedMessageDTO; import static org.apache.james.vault.dto.DeletedMessageWithStorageInformationDTO.StorageInformationDTO; import static org.apache.james.vault.metadata.DeletedMessageVaultMetadataFixture.STORAGE_INFORMATION; @@ -32,11 +31,8 @@ import jakarta.mail.internet.AddressException; -import org.apache.james.blob.api.PlainBlobId; import org.apache.james.mailbox.inmemory.InMemoryId; import org.apache.james.mailbox.inmemory.InMemoryMessageId; -import org.apache.james.utils.UpdatableTickingClock; -import org.apache.james.vault.blob.BlobIdTimeGenerator; import org.apache.james.vault.metadata.DeletedMessageWithStorageInformation; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @@ -76,7 +72,6 @@ void setup() { .setSerializationInclusion(JsonInclude.Include.NON_ABSENT); this.converter = new DeletedMessageWithStorageInformationConverter( - new BlobIdTimeGenerator(new PlainBlobId.Factory(), new UpdatableTickingClock(NOW.toInstant())), new InMemoryMessageId.Factory(), new InMemoryId.Factory()); } diff --git a/server/protocols/webadmin/webadmin-mailbox-deleted-message-vault/src/test/java/org/apache/james/webadmin/vault/routes/DeletedMessagesVaultRoutesTest.java b/server/protocols/webadmin/webadmin-mailbox-deleted-message-vault/src/test/java/org/apache/james/webadmin/vault/routes/DeletedMessagesVaultRoutesTest.java index 65f7ae1a970..f53851e5c3d 100644 --- a/server/protocols/webadmin/webadmin-mailbox-deleted-message-vault/src/test/java/org/apache/james/webadmin/vault/routes/DeletedMessagesVaultRoutesTest.java +++ b/server/protocols/webadmin/webadmin-mailbox-deleted-message-vault/src/test/java/org/apache/james/webadmin/vault/routes/DeletedMessagesVaultRoutesTest.java @@ -193,7 +193,7 @@ void beforeEach() throws Exception { usersRepository = createUsersRepository(); vault = spy(new BlobStoreDeletedMessageVault(new RecordingMetricFactory(), new MemoryDeletedMessageMetadataVault(), blobStore, blobStoreDAO, new BucketNameGenerator(clock), clock, - new BlobIdTimeGenerator(blobIdFactory, clock), + new BlobIdTimeGenerator(clock), VaultConfiguration.ENABLED_DEFAULT, usersRepository)); InMemoryIntegrationResources inMemoryResource = InMemoryIntegrationResources.defaultResources(); mailboxManager = spy(inMemoryResource.getMailboxManager()); From e21ce2b1a4c40f354552d7e8dbac6b7a2addcf0f Mon Sep 17 00:00:00 2001 From: Benoit TELLIER Date: Thu, 15 Jan 2026 18:08:59 +0100 Subject: [PATCH 12/14] JAMES-4164 Simplify BlobStoreDeletedMessageVault constructor - Build a tailor made blobStore from blobDAO - BlobIdTimeGenerator can be static --- mailbox/plugin/deleted-messages-vault/pom.xml | 4 ++++ .../james/vault/blob/BlobIdTimeGenerator.java | 11 +---------- .../blob/BlobStoreDeletedMessageVault.java | 17 ++++++++++------- .../vault/DeletedMessageVaultHookTest.java | 9 +-------- .../vault/blob/BlobIdTimeGeneratorTest.java | 2 +- .../blob/BlobStoreDeletedMessageVaultTest.java | 11 +---------- .../routes/DeletedMessagesVaultRoutesTest.java | 4 +--- 7 files changed, 19 insertions(+), 39 deletions(-) diff --git a/mailbox/plugin/deleted-messages-vault/pom.xml b/mailbox/plugin/deleted-messages-vault/pom.xml index f2473bc773b..4d9f421c258 100644 --- a/mailbox/plugin/deleted-messages-vault/pom.xml +++ b/mailbox/plugin/deleted-messages-vault/pom.xml @@ -85,6 +85,10 @@ blob-memory test + + ${james.groupId} + blob-storage-strategy + ${james.groupId} diff --git a/mailbox/plugin/deleted-messages-vault/src/main/java/org/apache/james/vault/blob/BlobIdTimeGenerator.java b/mailbox/plugin/deleted-messages-vault/src/main/java/org/apache/james/vault/blob/BlobIdTimeGenerator.java index bad695e31dd..5204621ed03 100644 --- a/mailbox/plugin/deleted-messages-vault/src/main/java/org/apache/james/vault/blob/BlobIdTimeGenerator.java +++ b/mailbox/plugin/deleted-messages-vault/src/main/java/org/apache/james/vault/blob/BlobIdTimeGenerator.java @@ -23,8 +23,6 @@ import java.time.ZonedDateTime; import java.util.UUID; -import jakarta.inject.Inject; - import org.apache.commons.lang3.RandomStringUtils; import org.apache.james.blob.api.BlobId; import org.apache.james.blob.api.PlainBlobId; @@ -32,14 +30,7 @@ public class BlobIdTimeGenerator { public static final String BLOB_ID_GENERATING_FORMAT = "%d/%02d/%s/%s/%s"; - private final Clock clock; - - @Inject - public BlobIdTimeGenerator(Clock clock) { - this.clock = clock; - } - - BlobId currentBlobId() { + static BlobId currentBlobId(Clock clock) { ZonedDateTime now = ZonedDateTime.now(clock); int month = now.getMonthValue(); int year = now.getYear(); diff --git a/mailbox/plugin/deleted-messages-vault/src/main/java/org/apache/james/vault/blob/BlobStoreDeletedMessageVault.java b/mailbox/plugin/deleted-messages-vault/src/main/java/org/apache/james/vault/blob/BlobStoreDeletedMessageVault.java index 67652f908e1..83c5a0e8c8f 100644 --- a/mailbox/plugin/deleted-messages-vault/src/main/java/org/apache/james/vault/blob/BlobStoreDeletedMessageVault.java +++ b/mailbox/plugin/deleted-messages-vault/src/main/java/org/apache/james/vault/blob/BlobStoreDeletedMessageVault.java @@ -33,9 +33,11 @@ import org.apache.james.blob.api.BlobStoreDAO; import org.apache.james.blob.api.BucketName; import org.apache.james.blob.api.ObjectNotFoundException; +import org.apache.james.blob.api.PlainBlobId; import org.apache.james.core.Username; import org.apache.james.mailbox.model.MessageId; import org.apache.james.metrics.api.MetricFactory; +import org.apache.james.server.blob.deduplication.BlobStoreFactory; import org.apache.james.task.Task; import org.apache.james.user.api.UsersRepository; import org.apache.james.vault.DeletedMessage; @@ -75,23 +77,24 @@ public class BlobStoreDeletedMessageVault implements DeletedMessageVault { private final BucketNameGenerator nameGenerator; private final Clock clock; private final VaultConfiguration vaultConfiguration; - private final BlobIdTimeGenerator blobIdTimeGenerator; private final UsersRepository usersRepository; private final BlobStoreVaultGarbageCollectionTask.Factory taskFactory; @Inject public BlobStoreDeletedMessageVault(MetricFactory metricFactory, DeletedMessageMetadataVault messageMetadataVault, - BlobStore blobStore, BlobStoreDAO blobStoreDAO, BucketNameGenerator nameGenerator, - Clock clock, BlobIdTimeGenerator blobIdTimeGenerator, - VaultConfiguration vaultConfiguration, UsersRepository usersRepository) { + BlobStoreDAO blobStoreDAO, BucketNameGenerator nameGenerator, + Clock clock, VaultConfiguration vaultConfiguration, UsersRepository usersRepository) { this.metricFactory = metricFactory; this.messageMetadataVault = messageMetadataVault; - this.blobStore = blobStore; + this.blobStore = BlobStoreFactory.builder() + .blobStoreDAO(blobStoreDAO) + .blobIdFactory(new PlainBlobId.Factory()) + .defaultBucketName() + .passthrough(); this.blobStoreDAO = blobStoreDAO; this.nameGenerator = nameGenerator; this.clock = clock; this.vaultConfiguration = vaultConfiguration; - this.blobIdTimeGenerator = blobIdTimeGenerator; this.usersRepository = usersRepository; this.taskFactory = new BlobStoreVaultGarbageCollectionTask.Factory(this); } @@ -141,7 +144,7 @@ private Mono appendMessage(DeletedMessage deletedMessage, InputStream mime private BlobStore.BlobIdProvider withTimePrefixBlobId() { return data -> Mono.just(Tuples.of( - blobIdTimeGenerator.currentBlobId(), + BlobIdTimeGenerator.currentBlobId(clock), data)); } diff --git a/mailbox/plugin/deleted-messages-vault/src/test/java/org/apache/james/vault/DeletedMessageVaultHookTest.java b/mailbox/plugin/deleted-messages-vault/src/test/java/org/apache/james/vault/DeletedMessageVaultHookTest.java index edfbeeab7c4..7f628d58114 100644 --- a/mailbox/plugin/deleted-messages-vault/src/test/java/org/apache/james/vault/DeletedMessageVaultHookTest.java +++ b/mailbox/plugin/deleted-messages-vault/src/test/java/org/apache/james/vault/DeletedMessageVaultHookTest.java @@ -54,9 +54,7 @@ import org.apache.james.mailbox.model.SearchQuery; import org.apache.james.metrics.tests.RecordingMetricFactory; import org.apache.james.mime4j.dom.Message; -import org.apache.james.server.blob.deduplication.BlobStoreFactory; import org.apache.james.user.api.UsersRepository; -import org.apache.james.vault.blob.BlobIdTimeGenerator; import org.apache.james.vault.blob.BlobStoreDeletedMessageVault; import org.apache.james.vault.blob.BucketNameGenerator; import org.apache.james.vault.memory.metadata.MemoryDeletedMessageMetadataVault; @@ -124,12 +122,7 @@ void setUp() throws Exception { UsersRepository usersRepository = mock(UsersRepository.class); messageVault = new BlobStoreDeletedMessageVault(new RecordingMetricFactory(), new MemoryDeletedMessageMetadataVault(), - BlobStoreFactory.builder() - .blobStoreDAO(blobStoreDAO) - .blobIdFactory(blobIdFactory) - .defaultBucketName() - .passthrough(), blobStoreDAO, new BucketNameGenerator(clock), clock, - new BlobIdTimeGenerator(clock), + blobStoreDAO, new BucketNameGenerator(clock), clock, VaultConfiguration.ENABLED_DEFAULT, usersRepository); DeletedMessageConverter deletedMessageConverter = new DeletedMessageConverter(); diff --git a/mailbox/plugin/deleted-messages-vault/src/test/java/org/apache/james/vault/blob/BlobIdTimeGeneratorTest.java b/mailbox/plugin/deleted-messages-vault/src/test/java/org/apache/james/vault/blob/BlobIdTimeGeneratorTest.java index fc50ac8a50b..98e66150bc9 100644 --- a/mailbox/plugin/deleted-messages-vault/src/test/java/org/apache/james/vault/blob/BlobIdTimeGeneratorTest.java +++ b/mailbox/plugin/deleted-messages-vault/src/test/java/org/apache/james/vault/blob/BlobIdTimeGeneratorTest.java @@ -33,7 +33,7 @@ class BlobIdTimeGeneratorTest { @Test void currentBlobIdShouldReturnBlobIdFormattedWithYearAndMonthPrefix() { - String currentBlobId = new BlobIdTimeGenerator(CLOCK).currentBlobId().asString(); + String currentBlobId = BlobIdTimeGenerator.currentBlobId(CLOCK).asString(); assertThat(currentBlobId).matches("2007/07/././.*"); } diff --git a/mailbox/plugin/deleted-messages-vault/src/test/java/org/apache/james/vault/blob/BlobStoreDeletedMessageVaultTest.java b/mailbox/plugin/deleted-messages-vault/src/test/java/org/apache/james/vault/blob/BlobStoreDeletedMessageVaultTest.java index bfa430740a6..3508d07b032 100644 --- a/mailbox/plugin/deleted-messages-vault/src/test/java/org/apache/james/vault/blob/BlobStoreDeletedMessageVaultTest.java +++ b/mailbox/plugin/deleted-messages-vault/src/test/java/org/apache/james/vault/blob/BlobStoreDeletedMessageVaultTest.java @@ -46,14 +46,11 @@ import java.time.ZonedDateTime; import java.util.List; -import org.apache.james.blob.api.BlobId; import org.apache.james.blob.api.BucketName; -import org.apache.james.blob.api.PlainBlobId; import org.apache.james.blob.memory.MemoryBlobStoreDAO; import org.apache.james.domainlist.api.DomainList; import org.apache.james.mailbox.inmemory.InMemoryMessageId; import org.apache.james.metrics.tests.RecordingMetricFactory; -import org.apache.james.server.blob.deduplication.BlobStoreFactory; import org.apache.james.user.memory.MemoryUsersRepository; import org.apache.james.utils.UpdatableTickingClock; import org.apache.james.vault.DeletedMessage; @@ -80,7 +77,6 @@ void setUp() throws Exception { clock = new UpdatableTickingClock(NOW.toInstant()); metricFactory = new RecordingMetricFactory(); MemoryBlobStoreDAO blobStoreDAO = new MemoryBlobStoreDAO(); - BlobId.Factory blobIdFactory = new PlainBlobId.Factory(); DomainList domainList = mock(DomainList.class); Mockito.when(domainList.containsDomain(any())).thenReturn(true); @@ -89,12 +85,7 @@ void setUp() throws Exception { usersRepository.addUser(USERNAME_2, PASSWORD); messageVault = new BlobStoreDeletedMessageVault(metricFactory, new MemoryDeletedMessageMetadataVault(), - BlobStoreFactory.builder() - .blobStoreDAO(blobStoreDAO) - .blobIdFactory(blobIdFactory) - .defaultBucketName() - .passthrough(), - blobStoreDAO, new BucketNameGenerator(clock), clock, new BlobIdTimeGenerator(clock), + blobStoreDAO, new BucketNameGenerator(clock), clock, VaultConfiguration.ENABLED_DEFAULT, usersRepository); } diff --git a/server/protocols/webadmin/webadmin-mailbox-deleted-message-vault/src/test/java/org/apache/james/webadmin/vault/routes/DeletedMessagesVaultRoutesTest.java b/server/protocols/webadmin/webadmin-mailbox-deleted-message-vault/src/test/java/org/apache/james/webadmin/vault/routes/DeletedMessagesVaultRoutesTest.java index f53851e5c3d..907e282eb81 100644 --- a/server/protocols/webadmin/webadmin-mailbox-deleted-message-vault/src/test/java/org/apache/james/webadmin/vault/routes/DeletedMessagesVaultRoutesTest.java +++ b/server/protocols/webadmin/webadmin-mailbox-deleted-message-vault/src/test/java/org/apache/james/webadmin/vault/routes/DeletedMessagesVaultRoutesTest.java @@ -112,7 +112,6 @@ import org.apache.james.vault.DeletedMessage; import org.apache.james.vault.DeletedMessageZipper; import org.apache.james.vault.VaultConfiguration; -import org.apache.james.vault.blob.BlobIdTimeGenerator; import org.apache.james.vault.blob.BlobStoreDeletedMessageVault; import org.apache.james.vault.blob.BlobStoreVaultGarbageCollectionTaskAdditionalInformationDTO; import org.apache.james.vault.blob.BucketNameGenerator; @@ -192,8 +191,7 @@ void beforeEach() throws Exception { clock = new UpdatableTickingClock(OLD_DELETION_DATE.toInstant()); usersRepository = createUsersRepository(); vault = spy(new BlobStoreDeletedMessageVault(new RecordingMetricFactory(), new MemoryDeletedMessageMetadataVault(), - blobStore, blobStoreDAO, new BucketNameGenerator(clock), clock, - new BlobIdTimeGenerator(clock), + blobStoreDAO, new BucketNameGenerator(clock), clock, VaultConfiguration.ENABLED_DEFAULT, usersRepository)); InMemoryIntegrationResources inMemoryResource = InMemoryIntegrationResources.defaultResources(); mailboxManager = spy(inMemoryResource.getMailboxManager()); From 07768b04bd1210c12e3b89fc0aa4faef3ce695a3 Mon Sep 17 00:00:00 2001 From: Benoit TELLIER Date: Thu, 15 Jan 2026 21:53:12 +0100 Subject: [PATCH 13/14] fixup! JAMES-4164 Remove unneeded BlobIdTimeGenerator::blobIdEndTime --- .../DeletedMessagesVaultRoutesTest.java | 28 ++++++++++--------- 1 file changed, 15 insertions(+), 13 deletions(-) diff --git a/server/protocols/webadmin/webadmin-mailbox-deleted-message-vault/src/test/java/org/apache/james/webadmin/vault/routes/DeletedMessagesVaultRoutesTest.java b/server/protocols/webadmin/webadmin-mailbox-deleted-message-vault/src/test/java/org/apache/james/webadmin/vault/routes/DeletedMessagesVaultRoutesTest.java index 907e282eb81..0ce1bfc8fe4 100644 --- a/server/protocols/webadmin/webadmin-mailbox-deleted-message-vault/src/test/java/org/apache/james/webadmin/vault/routes/DeletedMessagesVaultRoutesTest.java +++ b/server/protocols/webadmin/webadmin-mailbox-deleted-message-vault/src/test/java/org/apache/james/webadmin/vault/routes/DeletedMessagesVaultRoutesTest.java @@ -139,6 +139,8 @@ class DeletedMessagesVaultRoutesTest { + private MemoryBlobStoreDAO blobStoreDAO; + private static class NoopBlobExporting implements BlobExportMechanism { private Optional exportedBlobId = Optional.empty(); @@ -156,7 +158,7 @@ public Optional getExportedBlobId() { } } - private static final ZonedDateTime NOW = ZonedDateTime.parse("2015-10-30T16:12:00Z"); + private static final ZonedDateTime NOW = ZonedDateTime.parse("2016-10-30T16:12:00Z"); private static final ZonedDateTime OLD_DELETION_DATE = ZonedDateTime.parse("2010-10-30T15:12:00Z"); private static final String MATCH_ALL_QUERY = "{" + "\"combinator\": \"and\"," + @@ -182,7 +184,7 @@ public Optional getExportedBlobId() { @BeforeEach void beforeEach() throws Exception { blobIdFactory = new PlainBlobId.Factory(); - MemoryBlobStoreDAO blobStoreDAO = new MemoryBlobStoreDAO(); + this.blobStoreDAO = spy(new MemoryBlobStoreDAO()); blobStore = spy(BlobStoreFactory.builder() .blobStoreDAO(blobStoreDAO) .blobIdFactory(blobIdFactory) @@ -2044,6 +2046,11 @@ void oldPurgeShouldProduceASuccessfulTaskWithAdditionalInformation() { @Test void oldPurgeShouldNotDeleteNotExpiredMessagesInTheVault() { + + Mono.from(vault.appendV1(DELETED_MESSAGE, new ByteArrayInputStream(CONTENT))).block(); + Mono.from(vault.appendV1(DELETED_MESSAGE_2, new ByteArrayInputStream(CONTENT))).block(); + + clock.setInstant(NOW.toInstant()); DeletedMessage notExpiredMessage = DeletedMessage.builder() .messageId(InMemoryMessageId.of(46)) .originMailboxes(MAILBOX_ID_1, MAILBOX_ID_2) @@ -2056,11 +2063,6 @@ void oldPurgeShouldNotDeleteNotExpiredMessagesInTheVault() { .size(CONTENT.length) .build(); - Mono.from(vault.appendV1(DELETED_MESSAGE, new ByteArrayInputStream(CONTENT))).block(); - Mono.from(vault.appendV1(DELETED_MESSAGE_2, new ByteArrayInputStream(CONTENT))).block(); - - clock.setInstant(NOW.toInstant()); - Mono.from(vault.appendV1(notExpiredMessage, new ByteArrayInputStream(CONTENT))).block(); String taskId = @@ -2130,6 +2132,11 @@ void purgeShouldProduceASuccessfulTaskWithAdditionalInformation() { @Test void purgeShouldNotDeleteNotExpiredMessagesInTheVault() { + Mono.from(vault.append(DELETED_MESSAGE, new ByteArrayInputStream(CONTENT))).block(); + Mono.from(vault.append(DELETED_MESSAGE_2, new ByteArrayInputStream(CONTENT))).block(); + + clock.setInstant(NOW.toInstant()); + DeletedMessage notExpiredMessage = DeletedMessage.builder() .messageId(InMemoryMessageId.of(46)) .originMailboxes(MAILBOX_ID_1, MAILBOX_ID_2) @@ -2142,11 +2149,6 @@ void purgeShouldNotDeleteNotExpiredMessagesInTheVault() { .size(CONTENT.length) .build(); - Mono.from(vault.append(DELETED_MESSAGE, new ByteArrayInputStream(CONTENT))).block(); - Mono.from(vault.append(DELETED_MESSAGE_2, new ByteArrayInputStream(CONTENT))).block(); - - clock.setInstant(NOW.toInstant()); - Mono.from(vault.append(notExpiredMessage, new ByteArrayInputStream(CONTENT))).block(); String taskId = @@ -2192,7 +2194,7 @@ void oldPurgeShouldProduceAFailedTaskWhenFailingDeletingBucket() { Mono.from(vault.appendV1(DELETED_MESSAGE_2, new ByteArrayInputStream(CONTENT))).block(); doReturn(Mono.error(new RuntimeException("mock exception"))) - .when(blobStore) + .when(blobStoreDAO) .deleteBucket(BucketName.of("deleted-messages-2010-10-01")); clock.setInstant(NOW.toInstant()); From 1cc7a87fe2c461993cc44466ca4c2b7b96b18eef Mon Sep 17 00:00:00 2001 From: Benoit TELLIER Date: Fri, 16 Jan 2026 10:56:30 +0100 Subject: [PATCH 14/14] Improve deletion logic --- .../vault/blob/BlobStoreDeletedMessageVault.java | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/mailbox/plugin/deleted-messages-vault/src/main/java/org/apache/james/vault/blob/BlobStoreDeletedMessageVault.java b/mailbox/plugin/deleted-messages-vault/src/main/java/org/apache/james/vault/blob/BlobStoreDeletedMessageVault.java index 83c5a0e8c8f..c3391c4e9a0 100644 --- a/mailbox/plugin/deleted-messages-vault/src/main/java/org/apache/james/vault/blob/BlobStoreDeletedMessageVault.java +++ b/mailbox/plugin/deleted-messages-vault/src/main/java/org/apache/james/vault/blob/BlobStoreDeletedMessageVault.java @@ -198,9 +198,9 @@ public Publisher delete(Username username, MessageId messageId) { private Mono deleteMessage(Username username, MessageId messageId) { return Mono.from(messageMetadataVault.retrieveStorageInformation(username, messageId)) - .flatMap(storageInformation -> Mono.from(messageMetadataVault.remove(storageInformation.getBucketName(), username, messageId)) - .thenReturn(storageInformation)) - .flatMap(storageInformation -> Mono.from(blobStoreDAO.delete(storageInformation.getBucketName(), storageInformation.getBlobId()))); + .flatMap(storageInformation -> Mono.from(blobStoreDAO.delete(storageInformation.getBucketName(), storageInformation.getBlobId())) + .onErrorResume(ObjectNotFoundException.class, e -> Mono.empty()) + .then(Mono.from(messageMetadataVault.remove(storageInformation.getBucketName(), username, messageId)))); } @Override @@ -212,8 +212,8 @@ Mono deleteExpiredMessages(ZonedDateTime beginningOfRetentionPeriod, BlobS return Flux.from( metricFactory.decoratePublisherWithTimerMetric( DELETE_EXPIRED_MESSAGES_METRIC_NAME, - deletedExpiredMessagesFromOldBuckets(beginningOfRetentionPeriod, context) - .then(deleteUserExpiredMessages(beginningOfRetentionPeriod, context)))) + deleteUserExpiredMessages(beginningOfRetentionPeriod, context) + .then(deletedExpiredMessagesFromOldBuckets(beginningOfRetentionPeriod, context).then()))) .then(); } @@ -257,7 +257,9 @@ private Mono deleteUserExpiredMessages(ZonedDateTime beginningOfRetentionP return Flux.from(usersRepository.listReactive()) .flatMap(username -> Flux.from(messageMetadataVault.listMessages(bucketName, username)) .filter(deletedMessage -> isMessageFullyExpired(beginningOfRetentionPeriod, deletedMessage)) - .flatMap(deletedMessage -> Mono.from(messageMetadataVault.remove(bucketName, username, deletedMessage.getDeletedMessage().getMessageId())) + .flatMap(deletedMessage -> Mono.from(blobStoreDAO.delete(bucketName, deletedMessage.getStorageInformation().getBlobId())) + .onErrorResume(ObjectNotFoundException.class, e -> Mono.empty()) + .then(Mono.from(messageMetadataVault.remove(bucketName, username, deletedMessage.getDeletedMessage().getMessageId()))) .doOnSuccess(any -> context.recordDeletedBlobSuccess()))) .then(); }