From d9d75c915701ea0b3736b56498c1051dcc1ad94e Mon Sep 17 00:00:00 2001 From: Mark Payne Date: Mon, 18 May 2026 16:43:58 -0400 Subject: [PATCH] NIFI-16046: Initial implementation for Processor / Connector Backlog --- .../org/apache/nifi/util/FormatUtils.java | 75 +++ .../org/apache/nifi/util/TestFormatUtils.java | 55 +++ .../nifi/connectors/kafkas3/KafkaToS3.java | 23 +- nifi-docs/src/main/asciidoc/user-guide.adoc | 1 + .../aws/kinesis/ConsumeKinesis.java | 196 ++++++-- .../aws/kinesis/EnhancedFanOutClient.java | 17 +- .../aws/kinesis/KinesisConsumerClient.java | 21 + .../aws/kinesis/PollingKinesisClient.java | 8 +- .../aws/kinesis/ConsumeKinesisIT.java | 164 +++++++ .../aws/kinesis/ConsumeKinesisTest.java | 114 ++++- .../kinesis/KinesisConsumerClientTest.java | 4 +- .../aws/kinesis/PollingKinesisClientTest.java | 32 ++ .../apache/nifi/processors/aws/s3/ListS3.java | 225 ++++++++- .../nifi/processors/aws/s3/ITListS3.java | 320 +++++++++++++ .../nifi/processors/aws/s3/TestListS3.java | 207 +++++++++ .../processors/ConsumeKafkaBacklogIT.java | 275 +++++++++++ .../nifi/kafka/processors/ConsumeKafka.java | 71 ++- .../kafka/processors/ConsumeKafkaTest.java | 90 ++++ .../service/api/KafkaConnectionService.java | 23 + .../service/Kafka3ConnectionService.java | 133 ++++++ .../nifi/controller/queue/FlowFileQueue.java | 14 + .../queue/FlowFileQueueSnapshot.java | 54 +++ .../apache/nifi/web/api/dto/BacklogDTO.java | 123 +++++ .../nifi/web/api/dto/BacklogRequestDTO.java | 55 +++ .../web/api/dto/BacklogUpdateStepDTO.java | 23 + .../apache/nifi/web/api/dto/ProcessorDTO.java | 13 + .../nifi/web/api/entity/BacklogEntity.java | 54 +++ .../web/api/entity/BacklogRequestEntity.java | 36 ++ .../http/StandardHttpResponseMapper.java | 2 + .../BacklogRequestEndpointMerger.java | 308 +++++++++++++ .../BacklogRequestEndpointMergerTest.java | 434 ++++++++++++++++++ .../controller/StandardProcessorNode.java | 43 ++ .../apache/nifi/controller/ProcessorNode.java | 40 ++ .../StandaloneConnectionFacade.java | 24 +- .../standalone/StandaloneProcessorFacade.java | 20 +- .../queue/StandardFlowFileQueue.java | 5 + .../queue/SwappablePriorityQueue.java | 33 ++ .../SocketLoadBalancedFlowFileQueue.java | 44 ++ .../partition/LocalQueuePartition.java | 11 + .../clustered/partition/QueuePartition.java | 12 + .../partition/RemoteQueuePartition.java | 10 + .../StandardRebalancingPartition.java | 10 + .../SwappablePriorityQueueLocalPartition.java | 16 + .../StandaloneConnectionFacadeTest.java | 91 ++++ .../StandaloneProcessorFacadeTest.java | 96 ++++ .../TestSocketLoadBalancedFlowFileQueue.java | 118 +++++ .../clustered/TestSwappablePriorityQueue.java | 59 +++ .../TestWriteAheadFlowFileRepository.java | 6 + .../apache/nifi/web/NiFiServiceFacade.java | 35 ++ .../nifi/web/StandardNiFiServiceFacade.java | 75 +++ .../nifi/web/api/ConnectorResource.java | 254 ++++++++++ .../nifi/web/api/ProcessorResource.java | 247 ++++++++++ .../StandardAsynchronousWebRequest.java | 29 +- .../apache/nifi/web/api/dto/DtoFactory.java | 67 +++ .../AuthorizingProcessorFacade.java | 2 +- .../org/apache/nifi/web/dao/ProcessorDAO.java | 28 ++ .../web/dao/impl/StandardProcessorDAO.java | 23 + .../web/StandardNiFiServiceFacadeTest.java | 140 ++++++ .../nifi/web/api/ProcessorResourceTest.java | 249 ++++++++++ .../nifi/web/api/TestConnectorResource.java | 144 ++++++ .../StandardAsynchronousWebRequestTest.java | 73 +++ .../dao/impl/StandardProcessorDAOTest.java | 40 ++ .../service/canvas-context-menu.service.ts | 22 + .../flow-designer/service/flow.service.ts | 17 + .../flow-designer/state/flow/flow.actions.ts | 5 + .../state/flow/flow.effects.spec.ts | 55 ++- .../flow-designer/state/flow/flow.effects.ts | 32 ++ .../pages/flow-designer/state/flow/index.ts | 7 + .../backlog-dialog.component.html | 62 +++ .../backlog-dialog.component.scss | 33 ++ .../backlog-dialog.component.spec.ts | 385 ++++++++++++++++ .../backlog-dialog.component.ts | 256 +++++++++++ .../apps/nifi/src/app/state/shared/index.ts | 29 ++ .../queue/StatelessFlowFileQueue.java | 12 + .../system/BacklogReportingTestConnector.java | 343 ++++++++++++++ .../system/BacklogReportingTestProcessor.java | 140 ++++++ ...apache.nifi.components.connector.Connector | 1 + .../org.apache.nifi.processor.Processor | 1 + .../nifi/tests/system/NiFiClientUtil.java | 56 +++ .../ClusteredConnectorBacklogIT.java | 111 +++++ .../system/connectors/ConnectorBacklogIT.java | 190 ++++++++ .../nifi/toolkit/client/ConnectorClient.java | 33 ++ .../nifi/toolkit/client/ProcessorClient.java | 7 + .../client/impl/JerseyConnectorClient.java | 54 +++ .../client/impl/JerseyProcessorClient.java | 54 +++ 85 files changed, 7046 insertions(+), 73 deletions(-) create mode 100644 nifi-extension-bundles/nifi-kafka-bundle/nifi-kafka-3-integration/src/test/java/org/apache/nifi/kafka/processors/ConsumeKafkaBacklogIT.java create mode 100644 nifi-framework-api/src/main/java/org/apache/nifi/controller/queue/FlowFileQueueSnapshot.java create mode 100644 nifi-framework-bundle/nifi-framework/nifi-client-dto/src/main/java/org/apache/nifi/web/api/dto/BacklogDTO.java create mode 100644 nifi-framework-bundle/nifi-framework/nifi-client-dto/src/main/java/org/apache/nifi/web/api/dto/BacklogRequestDTO.java create mode 100644 nifi-framework-bundle/nifi-framework/nifi-client-dto/src/main/java/org/apache/nifi/web/api/dto/BacklogUpdateStepDTO.java create mode 100644 nifi-framework-bundle/nifi-framework/nifi-client-dto/src/main/java/org/apache/nifi/web/api/entity/BacklogEntity.java create mode 100644 nifi-framework-bundle/nifi-framework/nifi-client-dto/src/main/java/org/apache/nifi/web/api/entity/BacklogRequestEntity.java create mode 100644 nifi-framework-bundle/nifi-framework/nifi-framework-cluster/src/main/java/org/apache/nifi/cluster/coordination/http/endpoints/BacklogRequestEndpointMerger.java create mode 100644 nifi-framework-bundle/nifi-framework/nifi-framework-cluster/src/test/java/org/apache/nifi/cluster/coordination/http/endpoints/BacklogRequestEndpointMergerTest.java create mode 100644 nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/components/connector/facades/standalone/StandaloneConnectionFacadeTest.java create mode 100644 nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/components/connector/facades/standalone/StandaloneProcessorFacadeTest.java create mode 100644 nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/test/java/org/apache/nifi/web/api/ProcessorResourceTest.java create mode 100644 nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/test/java/org/apache/nifi/web/api/concurrent/StandardAsynchronousWebRequestTest.java create mode 100644 nifi-frontend/src/main/frontend/apps/nifi/src/app/pages/flow-designer/ui/canvas/items/processor/backlog-dialog/backlog-dialog.component.html create mode 100644 nifi-frontend/src/main/frontend/apps/nifi/src/app/pages/flow-designer/ui/canvas/items/processor/backlog-dialog/backlog-dialog.component.scss create mode 100644 nifi-frontend/src/main/frontend/apps/nifi/src/app/pages/flow-designer/ui/canvas/items/processor/backlog-dialog/backlog-dialog.component.spec.ts create mode 100644 nifi-frontend/src/main/frontend/apps/nifi/src/app/pages/flow-designer/ui/canvas/items/processor/backlog-dialog/backlog-dialog.component.ts create mode 100644 nifi-system-tests/nifi-system-test-extensions-bundle/nifi-system-test-extensions/src/main/java/org/apache/nifi/connectors/tests/system/BacklogReportingTestConnector.java create mode 100644 nifi-system-tests/nifi-system-test-extensions-bundle/nifi-system-test-extensions/src/main/java/org/apache/nifi/processors/tests/system/BacklogReportingTestProcessor.java create mode 100644 nifi-system-tests/nifi-system-test-suite/src/test/java/org/apache/nifi/tests/system/connectors/ClusteredConnectorBacklogIT.java create mode 100644 nifi-system-tests/nifi-system-test-suite/src/test/java/org/apache/nifi/tests/system/connectors/ConnectorBacklogIT.java diff --git a/nifi-commons/nifi-utils/src/main/java/org/apache/nifi/util/FormatUtils.java b/nifi-commons/nifi-utils/src/main/java/org/apache/nifi/util/FormatUtils.java index 5fda79cdc3f4..8520c330d9a4 100644 --- a/nifi-commons/nifi-utils/src/main/java/org/apache/nifi/util/FormatUtils.java +++ b/nifi-commons/nifi-utils/src/main/java/org/apache/nifi/util/FormatUtils.java @@ -134,6 +134,81 @@ public static String formatDataSize(final double dataSize) { return format.format(dataSize) + " bytes"; } + /** + * Formats the gap between two instants as a coarse-grained, human-readable relative-time + * string, for example {@code "5 secs ago"}, {@code "in 2 mins"}, {@code "yesterday"}, + * {@code "tomorrow"}, {@code "3 days ago"}, {@code "2 weeks ago"}, {@code "4 months ago"}, + * or {@code "1 year ago"}. When {@code from} is before {@code to}, the result reads as + * "X ago"; when {@code from} is after {@code to}, the result reads as "in X". The unit + * granularity is chosen so that the displayed number is small but informative — sub-minute + * gaps surface as seconds, sub-hour gaps as minutes, and so on up through years. + * + *

This method does not return a special {@code "now"} string when the gap is small; + * callers that want to collapse a small window around {@code to} (for example, "(now)" when + * a backlog is current within a few seconds) should test for that condition themselves + * before invoking this method.

+ * + * @param from the instant to describe relative to {@code to}, never {@code null} + * @param to the reference instant (usually "now"), never {@code null} + * @return a relative-time string describing the gap from {@code from} to {@code to} + */ + public static String formatRelativeTime(final Instant from, final Instant to) { + final long differenceMillis = to.toEpochMilli() - from.toEpochMilli(); + final boolean past = differenceMillis >= 0; + final long absoluteDifferenceMillis = Math.abs(differenceMillis); + + final long secondMillis = 1000L; + final long minuteMillis = 60L * secondMillis; + final long hourMillis = 60L * minuteMillis; + final long dayMillis = 24L * hourMillis; + final long weekMillis = 7L * dayMillis; + final long monthMillis = 30L * dayMillis; + final long yearMillis = 365L * dayMillis; + + // Round the difference into each candidate unit before deciding which bucket it belongs in. + // Rounding first keeps the chosen bucket and the displayed number consistent, so a value near a + // boundary (for example 59.5 seconds) is promoted to the next unit ("1 min ago") instead of + // producing a rounded number that overflows its own bucket ("60 secs ago"). + final long seconds = Math.round((double) absoluteDifferenceMillis / secondMillis); + if (seconds < 60L) { + return formatRelativeUnit(Math.max(1L, seconds), "sec", past); + } + + final long minutes = Math.round((double) absoluteDifferenceMillis / minuteMillis); + if (minutes < 60L) { + return formatRelativeUnit(minutes, "min", past); + } + + final long hours = Math.round((double) absoluteDifferenceMillis / hourMillis); + if (hours < 24L) { + return formatRelativeUnit(hours, "hour", past); + } + + final long days = Math.round((double) absoluteDifferenceMillis / dayMillis); + if (days == 1L) { + return past ? "yesterday" : "tomorrow"; + } + if (days < 7L) { + return formatRelativeUnit(days, "day", past); + } + if (absoluteDifferenceMillis < monthMillis) { + final long weeks = Math.round((double) absoluteDifferenceMillis / weekMillis); + return formatRelativeUnit(weeks, "week", past); + } + if (absoluteDifferenceMillis < yearMillis) { + final long months = Math.round((double) absoluteDifferenceMillis / monthMillis); + return formatRelativeUnit(months, "month", past); + } + + final long years = Math.round((double) absoluteDifferenceMillis / yearMillis); + return formatRelativeUnit(years, "year", past); + } + + private static String formatRelativeUnit(final long value, final String unit, final boolean past) { + final String suffix = (value == 1L) ? unit : unit + "s"; + return past ? value + " " + suffix + " ago" : "in " + value + " " + suffix; + } + /** * Returns a time duration in the requested {@link TimeUnit} after parsing the {@code String} * input. If the resulting value is a decimal (i.e. diff --git a/nifi-commons/nifi-utils/src/test/java/org/apache/nifi/util/TestFormatUtils.java b/nifi-commons/nifi-utils/src/test/java/org/apache/nifi/util/TestFormatUtils.java index b2c1dba37d2c..2f3b7763dd8f 100644 --- a/nifi-commons/nifi-utils/src/test/java/org/apache/nifi/util/TestFormatUtils.java +++ b/nifi-commons/nifi-utils/src/test/java/org/apache/nifi/util/TestFormatUtils.java @@ -177,6 +177,61 @@ public void testFormatTime(long sourceDuration, TimeUnit sourceUnit, String expe assertEquals(expected, FormatUtils.formatHoursMinutesSeconds(sourceDuration, sourceUnit)); } + @ParameterizedTest + @MethodSource("getRelativeTimeArguments") + public void testFormatRelativeTime(final long differenceMillis, final String expected) { + final Instant reference = Instant.parse("2026-05-15T10:00:00Z"); + final Instant from = reference.minusMillis(differenceMillis); + assertEquals(expected, FormatUtils.formatRelativeTime(from, reference)); + } + + private static Stream getRelativeTimeArguments() { + final long second = 1_000L; + final long minute = 60L * second; + final long hour = 60L * minute; + final long day = 24L * hour; + final long week = 7L * day; + final long month = 30L * day; + final long year = 365L * day; + return Stream.of( + Arguments.of(0L, "1 sec ago"), + Arguments.of(500L, "1 sec ago"), + Arguments.of(second, "1 sec ago"), + Arguments.of(45L * second, "45 secs ago"), + Arguments.of(minute, "1 min ago"), + Arguments.of(2L * minute, "2 mins ago"), + Arguments.of(59L * minute, "59 mins ago"), + Arguments.of(hour, "1 hour ago"), + Arguments.of(5L * hour, "5 hours ago"), + Arguments.of(day, "yesterday"), + Arguments.of(2L * day, "2 days ago"), + Arguments.of(6L * day, "6 days ago"), + Arguments.of(week, "1 week ago"), + Arguments.of(3L * week, "3 weeks ago"), + Arguments.of(month, "1 month ago"), + Arguments.of(6L * month, "6 months ago"), + Arguments.of(year, "1 year ago"), + Arguments.of(3L * year, "3 years ago"), + // Half-unit boundary values: a difference that rounds up to a full next unit must be + // promoted to that unit rather than rendering an overflowed count in the smaller unit. + Arguments.of(minute - second / 2 - 1L, "59 secs ago"), + Arguments.of(minute - second / 2, "1 min ago"), + Arguments.of(hour - minute / 2 - second, "59 mins ago"), + Arguments.of(hour - minute / 2, "1 hour ago"), + Arguments.of(day - hour / 2 - minute, "23 hours ago"), + Arguments.of(day - hour / 2, "yesterday"), + Arguments.of(week - day / 2 - hour, "6 days ago"), + Arguments.of(week - day / 2, "1 week ago"), + Arguments.of(month - week / 2, "4 weeks ago"), + // Negative offset (from is after to): future-tense rendering + Arguments.of(-1L * second, "in 1 sec"), + Arguments.of(-2L * minute, "in 2 mins"), + Arguments.of(-1L * day, "tomorrow"), + Arguments.of(-3L * day, "in 3 days"), + Arguments.of(-1L * year, "in 1 year") + ); + } + private static Stream getFormatTime() { return Stream.of(Arguments.of(0L, TimeUnit.DAYS, "00:00:00.000"), Arguments.of(1L, TimeUnit.HOURS, "01:00:00.000"), diff --git a/nifi-connectors/nifi-kafka-to-s3-bundle/nifi-kafka-to-s3-connector/src/main/java/org/apache/nifi/connectors/kafkas3/KafkaToS3.java b/nifi-connectors/nifi-kafka-to-s3-bundle/nifi-kafka-to-s3-connector/src/main/java/org/apache/nifi/connectors/kafkas3/KafkaToS3.java index f3871fdea1fa..84961a9a1cd2 100644 --- a/nifi-connectors/nifi-kafka-to-s3-bundle/nifi-kafka-to-s3-connector/src/main/java/org/apache/nifi/connectors/kafkas3/KafkaToS3.java +++ b/nifi-connectors/nifi-kafka-to-s3-bundle/nifi-kafka-to-s3-connector/src/main/java/org/apache/nifi/connectors/kafkas3/KafkaToS3.java @@ -20,10 +20,13 @@ import org.apache.nifi.annotation.documentation.CapabilityDescription; import org.apache.nifi.annotation.documentation.Tags; import org.apache.nifi.components.AllowableValue; +import org.apache.nifi.components.Backlog; +import org.apache.nifi.components.BacklogReportingException; import org.apache.nifi.components.ConfigVerificationResult; import org.apache.nifi.components.ConfigVerificationResult.Outcome; import org.apache.nifi.components.DescribedValue; import org.apache.nifi.components.connector.AbstractConnector; +import org.apache.nifi.components.connector.BacklogReportingConnector; import org.apache.nifi.components.connector.ConfigurationStep; import org.apache.nifi.components.connector.ConnectorConfigurationContext; import org.apache.nifi.components.connector.FlowUpdateException; @@ -41,6 +44,7 @@ import java.util.HashSet; import java.util.List; import java.util.Map; +import java.util.Optional; import java.util.Set; import java.util.concurrent.CompletableFuture; import java.util.concurrent.ExecutionException; @@ -51,7 +55,7 @@ @CapabilityDescription("Provides the ability to ingest data from Apache Kafka topics, merge it together into an object of reasonable " + "size, and write that data to Amazon S3.") @Tags({"kafka", "s3"}) -public class KafkaToS3 extends AbstractConnector { +public class KafkaToS3 extends AbstractConnector implements BacklogReportingConnector { private static final List configurationSteps = List.of( KafkaConnectionStep.KAFKA_CONNECTION_STEP, @@ -146,6 +150,20 @@ public List verifyConfigurationStep(final String stepN return Collections.emptyList(); } + @Override + public Optional getBacklog(final FlowContext activeFlowContext) throws BacklogReportingException { + final Optional consumeKafkaProcessor = findConsumeKafkaProcessor(activeFlowContext); + if (consumeKafkaProcessor.isEmpty()) { + return Optional.empty(); + } + + return consumeKafkaProcessor.get().getBacklog(); + } + + private Optional findConsumeKafkaProcessor(final FlowContext flowContext) { + return findProcessors(flowContext.getRootGroup(), processor -> processor.getDefinition().getType().endsWith("ConsumeKafka")).stream().findFirst(); + } + private List verifyKafkaParsability(final FlowContext workingFlowContext, final VersionedExternalFlow flow) { // Enable Controller Services necessary for parsing records. // We determine which Controller Services are referenced by the flow and enable them, but we do not use @@ -170,8 +188,7 @@ private List verifyKafkaParsability(final FlowContext } try { - final ProcessorFacade consumeKafkaFacade = findProcessors(rootGroup, - processor -> processor.getDefinition().getType().endsWith("ConsumeKafka")).getFirst(); + final ProcessorFacade consumeKafkaFacade = findConsumeKafkaProcessor(workingFlowContext).orElseThrow(); final List configVerificationResults = consumeKafkaFacade.verify(flow, Map.of()); for (final ConfigVerificationResult result : configVerificationResults) { diff --git a/nifi-docs/src/main/asciidoc/user-guide.adoc b/nifi-docs/src/main/asciidoc/user-guide.adoc index c95492882825..b5c0eee2c966 100644 --- a/nifi-docs/src/main/asciidoc/user-guide.adoc +++ b/nifi-docs/src/main/asciidoc/user-guide.adoc @@ -306,6 +306,7 @@ NOTE: For Processors, Ports, Remote Process Groups, Connections and Labels, it i - *View data provenance*: This option displays the NiFi Data Provenance table, with information about data provenance events for the FlowFiles routed through that Processor (see <>). - *Replay last event*: This option will replay the last Provenance event, effectively requeuing the last FlowFile that was processed by the Processor (see <>). - *View status history*: This option opens a graphical representation of the Processor's statistical information over time. +- *View Backlog*: For Processors that support reporting backlog against an external system (such as a queue or topic), this option displays the current backlog information. Because computing backlog may require querying the external system, this action requires *modify the component* access, not merely *view the component* access. - *View usage*: This option takes the user to the Processor's usage documentation. - *View connections->Upstream*: This option allows the user to see and "jump to" upstream connections that are coming into the Processor. This is particularly useful when processors connect into and out of other Process Groups. - *View connections->Downstream*: This option allows the user to see and "jump to" downstream connections that are going out of the Processor. This is particularly useful when processors connect into and out of other Process Groups. diff --git a/nifi-extension-bundles/nifi-aws-bundle/nifi-aws-kinesis/src/main/java/org/apache/nifi/processors/aws/kinesis/ConsumeKinesis.java b/nifi-extension-bundles/nifi-aws-bundle/nifi-aws-kinesis/src/main/java/org/apache/nifi/processors/aws/kinesis/ConsumeKinesis.java index 313802def6b3..ed51da3a802c 100644 --- a/nifi-extension-bundles/nifi-aws-bundle/nifi-aws-kinesis/src/main/java/org/apache/nifi/processors/aws/kinesis/ConsumeKinesis.java +++ b/nifi-extension-bundles/nifi-aws-bundle/nifi-aws-kinesis/src/main/java/org/apache/nifi/processors/aws/kinesis/ConsumeKinesis.java @@ -27,6 +27,8 @@ import org.apache.nifi.annotation.lifecycle.OnRemoved; import org.apache.nifi.annotation.lifecycle.OnScheduled; import org.apache.nifi.annotation.lifecycle.OnStopped; +import org.apache.nifi.components.Backlog; +import org.apache.nifi.components.BacklogReportingException; import org.apache.nifi.components.DescribedValue; import org.apache.nifi.components.PropertyDescriptor; import org.apache.nifi.components.Validator; @@ -37,6 +39,7 @@ import org.apache.nifi.migration.PropertyConfiguration; import org.apache.nifi.migration.ProxyServiceMigration; import org.apache.nifi.processor.AbstractProcessor; +import org.apache.nifi.processor.BacklogReportingProcessor; import org.apache.nifi.processor.DataUnit; import org.apache.nifi.processor.ProcessContext; import org.apache.nifi.processor.ProcessSession; @@ -93,7 +96,10 @@ import java.util.LinkedHashMap; import java.util.List; import java.util.Map; +import java.util.Optional; import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicLong; @@ -152,7 +158,7 @@ buffer is typically much smaller because fetch threads block when the queue is \ full and most responses are well below the maximum size. """) -public class ConsumeKinesis extends AbstractProcessor { +public class ConsumeKinesis extends AbstractProcessor implements BacklogReportingProcessor { static final String ATTR_STREAM_NAME = "aws.kinesis.stream.name"; static final String ATTR_SHARD_ID = "aws.kinesis.shard.id"; @@ -352,6 +358,9 @@ Specifies the string (interpreted as UTF-8) used to separate multiple Kinesis me private volatile String efoConsumerArn; private final AtomicLong shardRoundRobinCounter = new AtomicLong(); + private final ConcurrentMap shardMillisBehindLatest = new ConcurrentHashMap<>(); + private volatile Instant lastCaughtUpTimestamp; + @Override protected List getSupportedPropertyDescriptors() { return PROPERTY_DESCRIPTORS; @@ -382,41 +391,11 @@ public void migrateProperties(final PropertyConfiguration config) { @OnScheduled public void onScheduled(final ProcessContext context) { - final Region region = RegionUtil.getRegion(context); - final AwsCredentialsProvider credentialsProvider = context.getProperty(AWS_CREDENTIALS_PROVIDER_SERVICE) - .asControllerService(AwsCredentialsProviderService.class).getAwsCredentialsProvider(); - final String endpointOverride = context.getProperty(ENDPOINT_OVERRIDE).getValue(); - - final ClientOverrideConfiguration clientConfig = ClientOverrideConfiguration.builder() - .apiCallTimeout(API_CALL_TIMEOUT) - .apiCallAttemptTimeout(API_CALL_ATTEMPT_TIMEOUT) - .build(); - - final KinesisClientBuilder kinesisBuilder = KinesisClient.builder() - .region(region) - .credentialsProvider(credentialsProvider) - .overrideConfiguration(clientConfig); - - final DynamoDbClientBuilder dynamoBuilder = DynamoDbClient.builder() - .region(region) - .credentialsProvider(credentialsProvider) - .overrideConfiguration(clientConfig); - - if (endpointOverride != null && !endpointOverride.isEmpty()) { - final URI endpointUri = URI.create(endpointOverride); - kinesisBuilder.endpointOverride(endpointUri); - dynamoBuilder.endpointOverride(endpointUri); - } - - final ProxyConfiguration proxyConfig = ProxyConfiguration.getConfiguration(context); - - kinesisHttpClient = buildApacheHttpClient(proxyConfig, PollingKinesisClient.MAX_CONCURRENT_FETCHES + 10); - dynamoHttpClient = buildApacheHttpClient(proxyConfig, 50); - kinesisBuilder.httpClient(kinesisHttpClient); - dynamoBuilder.httpClient(dynamoHttpClient); - - kinesisClient = kinesisBuilder.build(); - dynamoDbClient = dynamoBuilder.build(); + final AwsClientBundle clientBundle = buildAwsClientBundle(context, PollingKinesisClient.MAX_CONCURRENT_FETCHES + 10, 50); + kinesisHttpClient = clientBundle.kinesisHttpClient(); + dynamoHttpClient = clientBundle.dynamoHttpClient(); + kinesisClient = clientBundle.kinesisClient(); + dynamoDbClient = clientBundle.dynamoDbClient(); final String checkpointTableName = context.getProperty(APPLICATION_NAME).getValue(); streamName = context.getProperty(STREAM_NAME).getValue(); @@ -430,6 +409,7 @@ public void onScheduled(final ProcessContext context) { shardManager = createShardManager(kinesisClient, dynamoDbClient, getLogger(), checkpointTableName, streamName); shardManager.ensureCheckpointTableExists(); consumerClient = createConsumerClient(kinesisClient, getLogger(), efoMode); + consumerClient.setShardLagObserver(shardMillisBehindLatest::put); final Instant timestampForPosition = resolveTimestampPosition(context); if (timestampForPosition != null) { @@ -437,6 +417,12 @@ public void onScheduled(final ProcessContext context) { } if (efoMode) { + final Region region = RegionUtil.getRegion(context); + final AwsCredentialsProvider credentialsProvider = context.getProperty(AWS_CREDENTIALS_PROVIDER_SERVICE) + .asControllerService(AwsCredentialsProviderService.class).getAwsCredentialsProvider(); + final String endpointOverride = context.getProperty(ENDPOINT_OVERRIDE).getValue(); + final ProxyConfiguration proxyConfig = ProxyConfiguration.getConfiguration(context); + final NettyNioAsyncHttpClient.Builder nettyBuilder = NettyNioAsyncHttpClient.builder() .protocol(Protocol.HTTP2) .maxConcurrency(500) @@ -539,6 +525,9 @@ public void onStopped() { kinesisHttpClient = null; closeQuietly(dynamoHttpClient); dynamoHttpClient = null; + + // Per-shard lag is tied to the now-released owned-shard set; lastCaughtUpTimestamp is preserved deliberately. + shardMillisBehindLatest.clear(); } @OnRemoved @@ -662,6 +651,141 @@ public void onTrigger(final ProcessContext context, final ProcessSession session } } + /** + * Returns this node's view of how much data remains on the source Kinesis stream for the shards + * it currently owns. Kinesis does not expose a per-shard records-behind or bytes-behind count; + * the only available lag signal is {@link ShardFetchResult#millisBehindLatest()}. Backlog is + * therefore reported as the number of owned shards whose most recently observed + * {@code millisBehindLatest} is greater than zero. Because that count is a lower bound on the + * work remaining (each behind shard holds at least one un-fetched record and yields at least one + * FlowFile), both the {@code flowFiles} and {@code records} dimensions use + * {@link Backlog.Precision#AT_LEAST}, and every owned shard is presumed behind until a fetch + * result has been observed. When this node owns no shards (for example while stopped, or when + * another cluster node holds the shards), only a remembered last-caught-up timestamp is surfaced, + * or {@link Optional#empty()} when none has been observed, to avoid double-counting the stream + * backlog across shard-less nodes. + */ + @Override + public Optional getBacklog(final ProcessContext context) throws BacklogReportingException { + final KinesisShardManager currentShardManager = shardManager; + if (currentShardManager == null) { + final Instant remembered = lastCaughtUpTimestamp; + if (remembered == null) { + return Optional.empty(); + } + return Optional.of(Backlog.lastCaughtUp(remembered)); + } + + final List ownedShards = currentShardManager.getOwnedShards(); + if (ownedShards.isEmpty()) { + // The Processor is running on this node but owns no shards, which only happens in a + // clustered deployment where shard ownership is held by a different node. Reporting the + // full-stream backlog here would be double-counted at the cluster merge layer (every + // shard-less node would report the same number). Surface only the remembered + // last-caught-up timestamp, or Optional.empty() when no caught-up moment has been seen. + final Instant remembered = lastCaughtUpTimestamp; + if (remembered == null) { + return Optional.empty(); + } + return Optional.of(Backlog.lastCaughtUp(remembered)); + } + + final Set ownedShardIds = new HashSet<>(); + for (final Shard shard : ownedShards) { + ownedShardIds.add(shard.shardId()); + } + final int ownedShardCount = ownedShardIds.size(); + + if (shardMillisBehindLatest.isEmpty()) { + getLogger().warn("Reporting lower-bound backlog estimate for stream [{}] because no shard fetch results have been observed yet: ownedShards={}, precision=AT_LEAST", + streamName, ownedShardCount); + } + + // A shard with no reported lag value is presumed behind, the same as a shard whose most recently + // observed millisBehindLatest is greater than zero. This ensures a shard that has not yet reported + // (for example, immediately after startup or a shard reassignment) cannot cause the stream to be + // reported as caught up before its true state is known. + int shardsBehindCount = 0; + for (final String shardId : ownedShardIds) { + final Long millisBehind = shardMillisBehindLatest.get(shardId); + if (millisBehind == null || millisBehind > 0L) { + shardsBehindCount++; + } + } + + if (shardsBehindCount == 0) { + final Backlog caughtUp = Backlog.caughtUp(); + lastCaughtUpTimestamp = caughtUp.getLastCaughtUp().orElseThrow(); + getLogger().debug("Reporting backlog for stream [{}] as caught up across {} owned shard(s)", streamName, ownedShardCount); + return Optional.of(caughtUp); + } + + final Backlog backlog = Backlog.builder() + .flowFiles(shardsBehindCount) + .records(shardsBehindCount) + .precision(Backlog.Precision.AT_LEAST) + .lastCaughtUp(lastCaughtUpTimestamp) + .build(); + getLogger().debug("Reporting backlog for stream [{}]: shardsBehind={}, ownedShards={}, lastCaughtUp={}, precision=AT_LEAST", + streamName, shardsBehindCount, ownedShardCount, lastCaughtUpTimestamp); + return Optional.of(backlog); + } + + /** + * Builds a matched pair of {@link KinesisClient}/{@link DynamoDbClient} from the supplied + * {@link ProcessContext}, along with the underlying {@link SdkHttpClient}s that back them. Used by + * {@link #onScheduled(ProcessContext)}, which retains the returned clients for the duration of the + * scheduled run. + */ + private AwsClientBundle buildAwsClientBundle(final ProcessContext context, final int kinesisConnectionPool, final int dynamoConnectionPool) { + final Region region = RegionUtil.getRegion(context); + final AwsCredentialsProvider credentialsProvider = context.getProperty(AWS_CREDENTIALS_PROVIDER_SERVICE) + .asControllerService(AwsCredentialsProviderService.class).getAwsCredentialsProvider(); + final String endpointOverride = context.getProperty(ENDPOINT_OVERRIDE).getValue(); + final ProxyConfiguration proxyConfig = ProxyConfiguration.getConfiguration(context); + + final SdkHttpClient kinesisHttp = buildApacheHttpClient(proxyConfig, kinesisConnectionPool); + final SdkHttpClient dynamoHttp = buildApacheHttpClient(proxyConfig, dynamoConnectionPool); + + final ClientOverrideConfiguration clientConfig = ClientOverrideConfiguration.builder() + .apiCallTimeout(API_CALL_TIMEOUT) + .apiCallAttemptTimeout(API_CALL_ATTEMPT_TIMEOUT) + .build(); + + final KinesisClientBuilder kinesisBuilder = KinesisClient.builder() + .region(region) + .credentialsProvider(credentialsProvider) + .overrideConfiguration(clientConfig) + .httpClient(kinesisHttp); + final DynamoDbClientBuilder dynamoBuilder = DynamoDbClient.builder() + .region(region) + .credentialsProvider(credentialsProvider) + .overrideConfiguration(clientConfig) + .httpClient(dynamoHttp); + + if (endpointOverride != null && !endpointOverride.isEmpty()) { + final URI endpointUri = URI.create(endpointOverride); + kinesisBuilder.endpointOverride(endpointUri); + dynamoBuilder.endpointOverride(endpointUri); + } + + return new AwsClientBundle(kinesisBuilder.build(), dynamoBuilder.build(), kinesisHttp, dynamoHttp); + } + + /** + * Matched set of AWS SDK v2 clients built together so that they share configuration (region, + * credentials, endpoint override, proxy, override configuration). Both {@link KinesisClient} and + * {@link DynamoDbClient} are produced; the underlying {@link SdkHttpClient}s are returned so the caller + * can close them when it closes the SDK clients themselves. + */ + private record AwsClientBundle( + KinesisClient kinesisClient, + DynamoDbClient dynamoDbClient, + SdkHttpClient kinesisHttpClient, + SdkHttpClient dynamoHttpClient + ) { + } + private List discardRelinquishedResults(final List consumedResults, final Set claimedShards) { final List accepted = new ArrayList<>(); final List discarded = new ArrayList<>(); diff --git a/nifi-extension-bundles/nifi-aws-bundle/nifi-aws-kinesis/src/main/java/org/apache/nifi/processors/aws/kinesis/EnhancedFanOutClient.java b/nifi-extension-bundles/nifi-aws-bundle/nifi-aws-kinesis/src/main/java/org/apache/nifi/processors/aws/kinesis/EnhancedFanOutClient.java index dbb340907d9a..e2bef7294c58 100644 --- a/nifi-extension-bundles/nifi-aws-bundle/nifi-aws-kinesis/src/main/java/org/apache/nifi/processors/aws/kinesis/EnhancedFanOutClient.java +++ b/nifi-extension-bundles/nifi-aws-bundle/nifi-aws-kinesis/src/main/java/org/apache/nifi/processors/aws/kinesis/EnhancedFanOutClient.java @@ -57,6 +57,7 @@ import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicLong; import java.util.function.Consumer; +import java.util.function.LongConsumer; /** * Enhanced Fan-Out Kinesis consumer that uses SubscribeToShard with dedicated throughput @@ -100,7 +101,8 @@ void startFetches(final List shards, final String streamName, final int b final BigInteger lastSeq = checkpoint == null ? null : new BigInteger(checkpoint); final StartingPosition startingPosition = buildStartingPosition(lastSeq, initialStreamPosition); logger.info("Creating Enhanced Fan-Out subscription for stream [{}] shard [{}] type [{}] seq [{}]", streamName, shardId, startingPosition.type(), lastSeq); - final ShardConsumer shardConsumer = new ShardConsumer(shardId, result -> enqueueIfActiveConsumer(shardId, result), pausedConsumers, logger); + final ShardConsumer shardConsumer = new ShardConsumer(shardId, result -> enqueueIfActiveConsumer(shardId, result), + millisBehindLatest -> recordShardLag(shardId, millisBehindLatest), pausedConsumers, logger); final ShardConsumer prior = shardConsumers.putIfAbsent(shardId, shardConsumer); if (prior == null) { try { @@ -349,6 +351,7 @@ private StartingPosition buildStartingPosition(final BigInteger sequenceNumber, static final class ShardConsumer { private final String shardId; private final Consumer resultSink; + private final LongConsumer lagObserver; private final Queue pausedConsumers; private final ComponentLog consumerLogger; private final AtomicBoolean subscribing = new AtomicBoolean(false); @@ -361,10 +364,11 @@ static final class ShardConsumer { private volatile BigInteger lastQueuedSequenceNumber; private volatile boolean shardNotFound; - ShardConsumer(final String shardId, final Consumer resultSink, + ShardConsumer(final String shardId, final Consumer resultSink, final LongConsumer lagObserver, final Queue pausedConsumers, final ComponentLog consumerLogger) { this.shardId = shardId; this.resultSink = resultSink; + this.lagObserver = lagObserver; this.pausedConsumers = pausedConsumers; this.consumerLogger = consumerLogger; } @@ -599,18 +603,25 @@ public void onNext(final SubscribeToShardEventStream eventStream) { return; } + final Long millisBehindLatest = event.millisBehindLatest(); + if (millisBehindLatest != null) { + // Notify the backlog tracker on every event, including events that carry no records. + // Empty events are the only way a caught-up shard transitions from "behind" back to 0. + lagObserver.accept(millisBehindLatest); + } + if (event.records().isEmpty()) { requestNext(); return; } - final long millisBehind = event.millisBehindLatest() != null ? event.millisBehindLatest() : -1; final List records = deduplicateRecords(event.records()); if (records.isEmpty()) { requestNext(); return; } + final long millisBehind = millisBehindLatest != null ? millisBehindLatest : -1; final ShardFetchResult result = createFetchResult(shardId, records, millisBehind); lastQueuedSequenceNumber = result.lastSequenceNumber(); resultSink.accept(result); diff --git a/nifi-extension-bundles/nifi-aws-bundle/nifi-aws-kinesis/src/main/java/org/apache/nifi/processors/aws/kinesis/KinesisConsumerClient.java b/nifi-extension-bundles/nifi-aws-bundle/nifi-aws-kinesis/src/main/java/org/apache/nifi/processors/aws/kinesis/KinesisConsumerClient.java index ff1e8a0e4fa6..3671e65d24d2 100644 --- a/nifi-extension-bundles/nifi-aws-bundle/nifi-aws-kinesis/src/main/java/org/apache/nifi/processors/aws/kinesis/KinesisConsumerClient.java +++ b/nifi-extension-bundles/nifi-aws-bundle/nifi-aws-kinesis/src/main/java/org/apache/nifi/processors/aws/kinesis/KinesisConsumerClient.java @@ -56,6 +56,7 @@ abstract class KinesisConsumerClient { private volatile long lastDiagnosticLogNanos; private volatile Instant timestampForInitialPosition; + private volatile ShardLagObserver shardLagObserver = (shardId, millisBehindLatest) -> { }; KinesisConsumerClient(final KinesisClient kinesisClient, final ComponentLog logger) { this.kinesisClient = kinesisClient; @@ -70,6 +71,26 @@ Instant getTimestampForInitialPosition() { return timestampForInitialPosition; } + /** + * Registers a callback that is invoked from the fetch threads (polling or EFO) with the + * {@code millisBehindLatest} value reported by Kinesis on every successful poll, including + * polls that returned no records. This is the only signal by which a shard transitions back + * to "caught up" from the backlog tracker's point of view; without it, a previously-observed + * lag would persist in the tracker until the next poll that happened to also return records. + */ + void setShardLagObserver(final ShardLagObserver observer) { + this.shardLagObserver = observer == null ? (shardId, millisBehindLatest) -> { } : observer; + } + + protected void recordShardLag(final String shardId, final long millisBehindLatest) { + shardLagObserver.observe(shardId, millisBehindLatest); + } + + @FunctionalInterface + interface ShardLagObserver { + void observe(String shardId, long millisBehindLatest); + } + void initialize(final KinesisAsyncClient asyncClient, final String streamName, final String consumerName) { } diff --git a/nifi-extension-bundles/nifi-aws-bundle/nifi-aws-kinesis/src/main/java/org/apache/nifi/processors/aws/kinesis/PollingKinesisClient.java b/nifi-extension-bundles/nifi-aws-bundle/nifi-aws-kinesis/src/main/java/org/apache/nifi/processors/aws/kinesis/PollingKinesisClient.java index 8cb21e46b956..8a1da8b8c195 100644 --- a/nifi-extension-bundles/nifi-aws-bundle/nifi-aws-kinesis/src/main/java/org/apache/nifi/processors/aws/kinesis/PollingKinesisClient.java +++ b/nifi-extension-bundles/nifi-aws-bundle/nifi-aws-kinesis/src/main/java/org/apache/nifi/processors/aws/kinesis/PollingKinesisClient.java @@ -263,8 +263,14 @@ private void runFetchLoop(final PollingShardState state, final String shardId, } final List records = response.records(); + final Long millisBehindLatest = response.millisBehindLatest(); + if (millisBehindLatest != null) { + // Notify the backlog tracker on every successful poll, including empty polls. Empty + // polls are the only way a caught-up shard transitions from "behind" back to 0. + recordShardLag(shardId, millisBehindLatest); + } if (!records.isEmpty()) { - final long millisBehind = response.millisBehindLatest() != null ? response.millisBehindLatest() : -1; + final long millisBehind = millisBehindLatest != null ? millisBehindLatest : -1; queuePermitConsumed = enqueueIfActive(shardId, state, createFetchResult(shardId, records, millisBehind)); } diff --git a/nifi-extension-bundles/nifi-aws-bundle/nifi-aws-kinesis/src/test/java/org/apache/nifi/processors/aws/kinesis/ConsumeKinesisIT.java b/nifi-extension-bundles/nifi-aws-bundle/nifi-aws-kinesis/src/test/java/org/apache/nifi/processors/aws/kinesis/ConsumeKinesisIT.java index 28e833222083..89237627bf23 100644 --- a/nifi-extension-bundles/nifi-aws-bundle/nifi-aws-kinesis/src/test/java/org/apache/nifi/processors/aws/kinesis/ConsumeKinesisIT.java +++ b/nifi-extension-bundles/nifi-aws-bundle/nifi-aws-kinesis/src/test/java/org/apache/nifi/processors/aws/kinesis/ConsumeKinesisIT.java @@ -24,6 +24,7 @@ import org.apache.avro.generic.GenericRecord; import org.apache.nifi.avro.AvroReader; import org.apache.nifi.avro.AvroRecordSetWriter; +import org.apache.nifi.components.Backlog; import org.apache.nifi.controller.AbstractControllerService; import org.apache.nifi.json.JsonRecordSetWriter; import org.apache.nifi.json.JsonTreeReader; @@ -64,11 +65,15 @@ import java.io.IOException; import java.io.OutputStream; import java.security.MessageDigest; +import java.time.Instant; import java.util.ArrayList; import java.util.HashSet; import java.util.List; import java.util.Map; +import java.util.Optional; +import java.util.OptionalLong; import java.util.Set; +import java.util.function.Predicate; import java.util.stream.Collectors; import static org.junit.jupiter.api.Assertions.assertEquals; @@ -810,6 +815,165 @@ void testAllCorruptRecordsRouteToParseFailure() throws Exception { assertEquals(3, failureFlowFiles.size(), "All corrupt records should route to parse failure"); } + @Test + void testBacklogReporting() throws Exception { + final String streamName = "backlog-test"; + createStream(streamName); + + runner.setProperty(ConsumeKinesis.STREAM_NAME, streamName); + runner.setProperty(ConsumeKinesis.PROCESSING_STRATEGY, "FLOW_FILE"); + runner.setProperty(ConsumeKinesis.CONSUMER_TYPE, "SHARED_THROUGHPUT"); + runner.setProperty(ConsumeKinesis.MAX_RECORDS_PER_REQUEST, "1"); + runner.setProperty(ConsumeKinesis.MAX_BATCH_DURATION, "100 ms"); + + final ConsumeKinesis processor = (ConsumeKinesis) runner.getProcessor(); + + // Schedule the processor and wait until the polling threads have observed at least one + // empty GetRecords response per shard. Each empty response carries millisBehindLatest=0, + // which the polling client forwards to the backlog tracker so the shard transitions to + // "caught up" without any records ever having been fetched. + runner.run(1, false, true); + + final Backlog emptyStreamCaughtUp = pollForBacklog(processor, runner, candidate -> + candidate.getPrecision() == Backlog.Precision.EXACT + && candidate.getLastCaughtUp().isPresent() + && candidate.getFlowFileCount().equals(OptionalLong.of(0L)) + && candidate.getRecordCount().equals(OptionalLong.of(0L)) + && candidate.getByteCount().equals(OptionalLong.of(0L)), + () -> runner.run(1, false, false)); + assertTrue(emptyStreamCaughtUp.getLastCaughtUp().isPresent()); + + publishRecords(streamName, 3); + final Backlog caughtUpBacklog = drainUntilCaughtUp(processor, runner); + assertEquals(Backlog.Precision.EXACT, caughtUpBacklog.getPrecision()); + assertEquals(OptionalLong.of(0L), caughtUpBacklog.getFlowFileCount()); + assertEquals(OptionalLong.of(0L), caughtUpBacklog.getRecordCount()); + assertEquals(OptionalLong.of(0L), caughtUpBacklog.getByteCount()); + assertTrue(caughtUpBacklog.getLastCaughtUp().isPresent()); + assertFalse(caughtUpBacklog.getLastCaughtUp().get().isBefore(emptyStreamCaughtUp.getLastCaughtUp().get())); + + runner.run(1, true, false); + } + + @Test + void testBacklogReportingWithEnhancedFanOut() throws Exception { + final String streamName = "backlog-efo-test"; + createStream(streamName); + + runner.setProperty(ConsumeKinesis.STREAM_NAME, streamName); + runner.setProperty(ConsumeKinesis.PROCESSING_STRATEGY, "FLOW_FILE"); + runner.setProperty(ConsumeKinesis.CONSUMER_TYPE, "ENHANCED_FAN_OUT"); + runner.setProperty(ConsumeKinesis.MAX_BATCH_DURATION, "100 ms"); + + final ConsumeKinesis processor = (ConsumeKinesis) runner.getProcessor(); + + // Schedule the processor and wait until the enhanced fan-out subscription's lag observer has + // reported a caught-up (millisBehindLatest=0) signal for the owned shard. This exercises the + // EnhancedFanOutClient lag-observer wiring that the shared-throughput backlog tests do not. + runner.run(1, false, true); + + final Backlog emptyStreamCaughtUp = pollForBacklog(processor, runner, candidate -> + candidate.getPrecision() == Backlog.Precision.EXACT + && candidate.getLastCaughtUp().isPresent() + && candidate.getFlowFileCount().equals(OptionalLong.of(0L)) + && candidate.getRecordCount().equals(OptionalLong.of(0L)) + && candidate.getByteCount().equals(OptionalLong.of(0L)), + () -> runner.run(1, false, false)); + assertTrue(emptyStreamCaughtUp.getLastCaughtUp().isPresent()); + + publishRecords(streamName, 3); + final Backlog caughtUpBacklog = drainUntilCaughtUp(processor, runner); + assertEquals(Backlog.Precision.EXACT, caughtUpBacklog.getPrecision()); + assertEquals(OptionalLong.of(0L), caughtUpBacklog.getFlowFileCount()); + assertEquals(OptionalLong.of(0L), caughtUpBacklog.getRecordCount()); + assertEquals(OptionalLong.of(0L), caughtUpBacklog.getByteCount()); + assertTrue(caughtUpBacklog.getLastCaughtUp().isPresent()); + assertFalse(caughtUpBacklog.getLastCaughtUp().get().isBefore(emptyStreamCaughtUp.getLastCaughtUp().get())); + + runner.run(1, true, false); + } + + /** + * Validates the stopped-state contract for a processor that has never been scheduled and has therefore + * never observed a caught-up moment. The Kinesis service does not expose enough information from + * outside the running consumer to determine a reliable backlog quickly, so {@link Optional#empty()} + * is the documented response. The framework surfaces this to the UI as "no data available". + */ + @Test + void testBacklogWhileStoppedWithoutPriorRunReturnsEmpty() throws Exception { + final String streamName = "stopped-never-run-test"; + createStream(streamName); + + runner.setProperty(ConsumeKinesis.STREAM_NAME, streamName); + runner.setProperty(ConsumeKinesis.PROCESSING_STRATEGY, "FLOW_FILE"); + runner.setProperty(ConsumeKinesis.CONSUMER_TYPE, "SHARED_THROUGHPUT"); + + final ConsumeKinesis processor = (ConsumeKinesis) runner.getProcessor(); + final Optional reported = processor.getBacklog(runner.getProcessContext()); + assertTrue(reported.isEmpty()); + } + + /** + * Validates that once the running processor has observed a caught-up moment, stopping the processor + * yields a {@link Backlog} that carries only the remembered {@code lastCaughtUp} timestamp — no + * numeric dimensions. The UI still shows "last caught up at X" even though the stopped processor + * cannot derive a fresh count from Kinesis. + */ + @Test + void testBacklogWhileStoppedAfterCaughtUpReturnsLastCaughtUpOnly() throws Exception { + final String streamName = "stopped-caught-up-test"; + createStream(streamName); + + runner.setProperty(ConsumeKinesis.STREAM_NAME, streamName); + runner.setProperty(ConsumeKinesis.PROCESSING_STRATEGY, "FLOW_FILE"); + runner.setProperty(ConsumeKinesis.CONSUMER_TYPE, "SHARED_THROUGHPUT"); + runner.setProperty(ConsumeKinesis.MAX_RECORDS_PER_REQUEST, "1"); + runner.setProperty(ConsumeKinesis.MAX_BATCH_DURATION, "100 ms"); + + publishRecords(streamName, 3); + + runner.run(1, false, true); + final Backlog caughtUpBacklog = drainUntilCaughtUp((ConsumeKinesis) runner.getProcessor(), runner); + assertTrue(caughtUpBacklog.getLastCaughtUp().isPresent()); + final Instant priorCaughtUp = caughtUpBacklog.getLastCaughtUp().get(); + + runner.run(1, true, false); + + final ConsumeKinesis processor = (ConsumeKinesis) runner.getProcessor(); + final Optional reported = processor.getBacklog(runner.getProcessContext()); + assertTrue(reported.isPresent()); + final Backlog backlog = reported.get(); + assertTrue(backlog.getLastCaughtUp().isPresent()); + assertFalse(backlog.getLastCaughtUp().get().isBefore(priorCaughtUp)); + assertEquals(OptionalLong.empty(), backlog.getFlowFileCount()); + assertEquals(OptionalLong.empty(), backlog.getByteCount()); + assertEquals(OptionalLong.empty(), backlog.getRecordCount()); + } + + private Backlog pollForBacklog(final ConsumeKinesis processor, final TestRunner testRunner, + final Predicate condition, final Runnable preProbe) throws Exception { + final long deadline = System.currentTimeMillis() + 60_000; + Optional last = Optional.empty(); + while (System.currentTimeMillis() < deadline) { + preProbe.run(); + Thread.sleep(100); + final Optional probe = processor.getBacklog(testRunner.getProcessContext()); + last = probe; + if (probe.isPresent() && condition.test(probe.get())) { + return probe.get(); + } + } + throw new AssertionError("Timed out waiting for backlog matching condition; last observed value: " + last); + } + + private Backlog drainUntilCaughtUp(final ConsumeKinesis processor, final TestRunner testRunner) throws Exception { + return pollForBacklog(processor, testRunner, candidate -> + candidate.getPrecision() == Backlog.Precision.EXACT + && candidate.getFlowFileCount().equals(OptionalLong.of(0L)) + && candidate.getLastCaughtUp().isPresent(), + () -> testRunner.run(1, false, false)); + } + private void addCredentialService(final TestRunner testRunner, final String serviceId) throws Exception { final AWSCredentialsProviderControllerService credsSvc = new AWSCredentialsProviderControllerService(); testRunner.addControllerService(serviceId, credsSvc); diff --git a/nifi-extension-bundles/nifi-aws-bundle/nifi-aws-kinesis/src/test/java/org/apache/nifi/processors/aws/kinesis/ConsumeKinesisTest.java b/nifi-extension-bundles/nifi-aws-bundle/nifi-aws-kinesis/src/test/java/org/apache/nifi/processors/aws/kinesis/ConsumeKinesisTest.java index 351f161ff6c0..ef538ed5fd1b 100644 --- a/nifi-extension-bundles/nifi-aws-bundle/nifi-aws-kinesis/src/test/java/org/apache/nifi/processors/aws/kinesis/ConsumeKinesisTest.java +++ b/nifi-extension-bundles/nifi-aws-bundle/nifi-aws-kinesis/src/test/java/org/apache/nifi/processors/aws/kinesis/ConsumeKinesisTest.java @@ -16,6 +16,7 @@ */ package org.apache.nifi.processors.aws.kinesis; +import org.apache.nifi.components.Backlog; import org.apache.nifi.json.JsonRecordSetWriter; import org.apache.nifi.json.JsonTreeReader; import org.apache.nifi.logging.ComponentLog; @@ -34,8 +35,12 @@ import java.nio.charset.StandardCharsets; import java.time.Instant; import java.util.ArrayList; +import java.util.LinkedHashMap; import java.util.LinkedHashSet; import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.OptionalLong; import java.util.Set; import static org.junit.jupiter.api.Assertions.assertEquals; @@ -63,15 +68,71 @@ void setUp() throws Exception { } private void setCommonProperties() throws Exception { + setCommonProperties(runner); + } + + private void setCommonProperties(final TestRunner targetRunner) throws Exception { final AWSCredentialsProviderControllerService credentialsService = new AWSCredentialsProviderControllerService(); - runner.addControllerService("creds", credentialsService); - runner.setProperty(credentialsService, AWSCredentialsProviderControllerService.ACCESS_KEY_ID, "AK_STUB"); - runner.setProperty(credentialsService, AWSCredentialsProviderControllerService.SECRET_KEY, "SK_STUB"); - runner.enableControllerService(credentialsService); + targetRunner.addControllerService("creds", credentialsService); + targetRunner.setProperty(credentialsService, AWSCredentialsProviderControllerService.ACCESS_KEY_ID, "AK_STUB"); + targetRunner.setProperty(credentialsService, AWSCredentialsProviderControllerService.SECRET_KEY, "SK_STUB"); + targetRunner.enableControllerService(credentialsService); + + targetRunner.setProperty(ConsumeKinesis.APPLICATION_NAME, "test-app"); + targetRunner.setProperty(ConsumeKinesis.STREAM_NAME, "test-stream"); + targetRunner.setProperty(ConsumeKinesis.AWS_CREDENTIALS_PROVIDER_SERVICE, "creds"); + } + + @Test + void testPartialShardLagReportingPresumesUnreportedShardsAreBehind() throws Exception { + // The node owns three shards. Lag is reported only for shard-A (behind) and shard-B (caught up); + // shard-C never reports a lag value. getBacklog must count both shard-A (reported behind) and + // shard-C (not yet reported, so presumed behind) and return a lower-bound (AT_LEAST) estimate. + final Map someBehind = new LinkedHashMap<>(); + someBehind.put("shard-A", 500L); + someBehind.put("shard-B", 0L); + final Backlog someBehindBacklog = reportBacklogWithShardLag(List.of("shard-A", "shard-B", "shard-C"), someBehind); + assertEquals(Backlog.Precision.AT_LEAST, someBehindBacklog.getPrecision()); + assertEquals(OptionalLong.of(2L), someBehindBacklog.getFlowFileCount()); + assertEquals(OptionalLong.of(2L), someBehindBacklog.getRecordCount()); + + // Every shard that has reported is caught up, but shard-C still has not reported. Because + // shard-C's true state is unknown, it is presumed behind, so getBacklog must not emit the exact + // caught-up Backlog, which would wrongly claim the whole stream is drained. It reports a + // conservative lower-bound of one (the presumed-behind shard-C) instead. + final Map allReportedCaughtUp = new LinkedHashMap<>(); + allReportedCaughtUp.put("shard-A", 0L); + allReportedCaughtUp.put("shard-B", 0L); + final Backlog reportedCaughtUpBacklog = reportBacklogWithShardLag(List.of("shard-A", "shard-B", "shard-C"), allReportedCaughtUp); + assertEquals(Backlog.Precision.AT_LEAST, reportedCaughtUpBacklog.getPrecision()); + assertEquals(OptionalLong.of(1L), reportedCaughtUpBacklog.getFlowFileCount()); + assertEquals(OptionalLong.of(1L), reportedCaughtUpBacklog.getRecordCount()); + } + + @Test + void testShardLagReportingClaimsCaughtUpOnlyWhenEveryOwnedShardHasReportedZeroLag() throws Exception { + final Map allCaughtUp = new LinkedHashMap<>(); + allCaughtUp.put("shard-A", 0L); + allCaughtUp.put("shard-B", 0L); + final Backlog backlog = reportBacklogWithShardLag(List.of("shard-A", "shard-B"), allCaughtUp); + assertEquals(Backlog.Precision.EXACT, backlog.getPrecision()); + assertEquals(OptionalLong.of(0L), backlog.getFlowFileCount()); + assertEquals(OptionalLong.of(0L), backlog.getRecordCount()); + } + + private Backlog reportBacklogWithShardLag(final List ownedShardIds, final Map shardLag) throws Exception { + final KinesisShardManager mockShardManager = buildShardManager(ownedShardIds.toArray(new String[0])); + final LagReportingConsumeKinesis processor = new LagReportingConsumeKinesis(mockShardManager, shardLag); + final TestRunner lagRunner = TestRunners.newTestRunner(processor); + setCommonProperties(lagRunner); + lagRunner.setProperty(ConsumeKinesis.PROCESSING_STRATEGY, "FLOW_FILE"); + lagRunner.setProperty(ConsumeKinesis.CONSUMER_TYPE, "SHARED_THROUGHPUT"); - runner.setProperty(ConsumeKinesis.APPLICATION_NAME, "test-app"); - runner.setProperty(ConsumeKinesis.STREAM_NAME, "test-stream"); - runner.setProperty(ConsumeKinesis.AWS_CREDENTIALS_PROVIDER_SERVICE, "creds"); + lagRunner.run(1, false, true); + + final Optional backlog = processor.getBacklog(lagRunner.getProcessContext()); + assertTrue(backlog.isPresent()); + return backlog.get(); } @Test @@ -515,6 +576,45 @@ protected KinesisConsumerClient createConsumerClient(final KinesisClient kinesis } } + static class LagReportingConsumeKinesis extends ConsumeKinesis { + private final KinesisShardManager mockShardManager; + private final Map shardLag; + + LagReportingConsumeKinesis(final KinesisShardManager mockShardManager, final Map shardLag) { + this.mockShardManager = mockShardManager; + this.shardLag = shardLag; + } + + @Override + protected KinesisShardManager createShardManager(final KinesisClient kinesisClient, final DynamoDbClient dynamoDbClient, + final ComponentLog logger, final String checkpointTableName, final String streamName) { + return mockShardManager; + } + + @Override + protected KinesisConsumerClient createConsumerClient(final KinesisClient kinesisClient, final ComponentLog logger, + final boolean efoMode) { + return new LagReportingConsumerClient(mock(KinesisClient.class), logger, shardLag); + } + } + + static class LagReportingConsumerClient extends StubConsumerClient { + private final Map shardLag; + + LagReportingConsumerClient(final KinesisClient kinesisClient, final ComponentLog logger, final Map shardLag) { + super(kinesisClient, logger); + this.shardLag = shardLag; + } + + @Override + void startFetches(final List shards, final String streamName, final int batchSize, + final String initialStreamPosition, final KinesisShardManager shardManager) { + for (final Map.Entry entry : shardLag.entrySet()) { + recordShardLag(entry.getKey(), entry.getValue()); + } + } + } + static class StubConsumerClient extends KinesisConsumerClient { StubConsumerClient(final KinesisClient kinesisClient, final ComponentLog logger) { super(kinesisClient, logger); diff --git a/nifi-extension-bundles/nifi-aws-bundle/nifi-aws-kinesis/src/test/java/org/apache/nifi/processors/aws/kinesis/KinesisConsumerClientTest.java b/nifi-extension-bundles/nifi-aws-bundle/nifi-aws-kinesis/src/test/java/org/apache/nifi/processors/aws/kinesis/KinesisConsumerClientTest.java index 66fbee0334f3..ea9e0c5d73e5 100644 --- a/nifi-extension-bundles/nifi-aws-bundle/nifi-aws-kinesis/src/test/java/org/apache/nifi/processors/aws/kinesis/KinesisConsumerClientTest.java +++ b/nifi-extension-bundles/nifi-aws-bundle/nifi-aws-kinesis/src/test/java/org/apache/nifi/processors/aws/kinesis/KinesisConsumerClientTest.java @@ -342,7 +342,7 @@ void testStaleErrorCallbackDoesNotCorruptNewSubscription() throws Exception { .thenReturn(CompletableFuture.completedFuture(null)); final EnhancedFanOutClient.ShardConsumer consumer = - new EnhancedFanOutClient.ShardConsumer("shardId-000000000001", result -> { }, new ConcurrentLinkedQueue<>(), mockLogger); + new EnhancedFanOutClient.ShardConsumer("shardId-000000000001", result -> { }, millisBehind -> { }, new ConcurrentLinkedQueue<>(), mockLogger); final StartingPosition pos = StartingPosition.builder() .type(ShardIteratorType.TRIM_HORIZON) @@ -404,7 +404,7 @@ void testResourceNotFoundExceptionEndsSubscriptionCleanly() { }); final EnhancedFanOutClient.ShardConsumer consumer = - new EnhancedFanOutClient.ShardConsumer("shardId-000000000001", result -> { }, new ConcurrentLinkedQueue<>(), mockLogger); + new EnhancedFanOutClient.ShardConsumer("shardId-000000000001", result -> { }, millisBehind -> { }, new ConcurrentLinkedQueue<>(), mockLogger); final StartingPosition pos = StartingPosition.builder() .type(ShardIteratorType.TRIM_HORIZON) diff --git a/nifi-extension-bundles/nifi-aws-bundle/nifi-aws-kinesis/src/test/java/org/apache/nifi/processors/aws/kinesis/PollingKinesisClientTest.java b/nifi-extension-bundles/nifi-aws-bundle/nifi-aws-kinesis/src/test/java/org/apache/nifi/processors/aws/kinesis/PollingKinesisClientTest.java index 0dba001e6c43..de1a3b7de139 100644 --- a/nifi-extension-bundles/nifi-aws-bundle/nifi-aws-kinesis/src/test/java/org/apache/nifi/processors/aws/kinesis/PollingKinesisClientTest.java +++ b/nifi-extension-bundles/nifi-aws-bundle/nifi-aws-kinesis/src/test/java/org/apache/nifi/processors/aws/kinesis/PollingKinesisClientTest.java @@ -39,6 +39,8 @@ import java.time.Instant; import java.util.Arrays; import java.util.List; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; @@ -79,6 +81,36 @@ void tearDown() { } } + /** + * Verifies the contract that the shard lag observer is notified on every successful + * {@code GetRecords} response — including responses that carry zero records — so that a shard which + * has caught up to the live tail can publish {@code millisBehindLatest = 0} and clear any prior + * lag observation held by the backlog tracker. + */ + @Test + void testEmptyPollNotifiesShardLagObserver() throws Exception { + when(mockShardManager.readCheckpoint(anyString())).thenReturn(null); + when(mockKinesisClient.getShardIterator(any(GetShardIteratorRequest.class))).thenReturn(ITERATOR_RESPONSE); + + final GetRecordsResponse firstResponse = GetRecordsResponse.builder() + .records(record("100", "data")).nextShardIterator("iter-next").millisBehindLatest(5000L).build(); + final GetRecordsResponse caughtUpResponse = GetRecordsResponse.builder() + .records(List.of()).nextShardIterator("iter-next").millisBehindLatest(0L).build(); + when(mockKinesisClient.getRecords(any(GetRecordsRequest.class))) + .thenReturn(firstResponse).thenReturn(caughtUpResponse); + + final ConcurrentMap observed = new ConcurrentHashMap<>(); + consumer.setShardLagObserver(observed::put); + + consumer.startFetches(shards("shard-1"), "test-stream", 1000, "TRIM_HORIZON", mockShardManager); + + final long deadline = System.currentTimeMillis() + 5_000L; + while (System.currentTimeMillis() < deadline && !Long.valueOf(0L).equals(observed.get("shard-1"))) { + Thread.sleep(20L); + } + assertEquals(0L, observed.get("shard-1")); + } + /** * When a shard reaches exhaustion (nextShardIterator is null), the records from the * final GetRecords response must be queued before the loop exits. A regression would diff --git a/nifi-extension-bundles/nifi-aws-bundle/nifi-aws-processors/src/main/java/org/apache/nifi/processors/aws/s3/ListS3.java b/nifi-extension-bundles/nifi-aws-bundle/nifi-aws-processors/src/main/java/org/apache/nifi/processors/aws/s3/ListS3.java index 113fe88fe16a..429e8cdc0911 100644 --- a/nifi-extension-bundles/nifi-aws-bundle/nifi-aws-processors/src/main/java/org/apache/nifi/processors/aws/s3/ListS3.java +++ b/nifi-extension-bundles/nifi-aws-bundle/nifi-aws-processors/src/main/java/org/apache/nifi/processors/aws/s3/ListS3.java @@ -32,6 +32,9 @@ import org.apache.nifi.annotation.notification.OnPrimaryNodeStateChange; import org.apache.nifi.annotation.notification.PrimaryNodeState; import org.apache.nifi.components.AllowableValue; +import org.apache.nifi.components.Backlog; +import org.apache.nifi.components.Backlog.Precision; +import org.apache.nifi.components.BacklogReportingException; import org.apache.nifi.components.ConfigVerificationResult; import org.apache.nifi.components.ConfigVerificationResult.Outcome; import org.apache.nifi.components.PropertyDescriptor; @@ -45,6 +48,7 @@ import org.apache.nifi.flowfile.attributes.CoreAttributes; import org.apache.nifi.logging.ComponentLog; import org.apache.nifi.migration.PropertyConfiguration; +import org.apache.nifi.processor.BacklogReportingProcessor; import org.apache.nifi.processor.ProcessContext; import org.apache.nifi.processor.ProcessSession; import org.apache.nifi.processor.Relationship; @@ -83,6 +87,7 @@ import java.io.IOException; import java.io.OutputStream; +import java.time.Instant; import java.util.ArrayList; import java.util.Collection; import java.util.Collections; @@ -90,6 +95,7 @@ import java.util.HashSet; import java.util.List; import java.util.Map; +import java.util.Optional; import java.util.Set; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicReference; @@ -131,7 +137,7 @@ @WritesAttribute(attribute = "s3.user.metadata.___", description = "If 'Write User Metadata' is set to 'True', the user defined metadata associated to the S3 object that is being listed " + "will be written as part of the FlowFile attributes")}) @DefaultSchedule(strategy = SchedulingStrategy.TIMER_DRIVEN, period = "1 min") -public class ListS3 extends AbstractS3Processor implements VerifiableProcessor { +public class ListS3 extends AbstractS3Processor implements VerifiableProcessor, BacklogReportingProcessor { public static final AllowableValue BY_TIMESTAMPS = new AllowableValue("timestamps", "Tracking Timestamps", "This strategy tracks the latest timestamp of listed entity to determine new/updated entities." + @@ -334,6 +340,15 @@ public class ListS3 extends AbstractS3Processor implements VerifiableProcessor { private volatile Long minObjectAgeMilliseconds; private volatile Long maxObjectAgeMilliseconds; + /** + * The most recent moment at which {@link #getBacklog(ProcessContext)} observed that the bucket was fully + * caught up. Held in memory for the JVM lifetime — preserved across stop/start cycles so that a stopped + * Processor still reports its last-known caught-up moment, and surfaced on every subsequent backlog + * response (including responses where the backlog has since grown back above zero). Reset only when the + * JVM restarts. + */ + private volatile Instant lastCaughtUpTimestamp; + @OnPrimaryNodeStateChange public void onPrimaryNodeChange(final PrimaryNodeState newState) { justElectedPrimaryNode = (newState == PrimaryNodeState.ELECTED_PRIMARY_NODE); @@ -374,7 +389,7 @@ public void initTrackingStrategy(ProcessContext context) throws IOException { } @OnScheduled - public void initObjectAgeThresholds(ProcessContext context) { + public void initObjectAgeThresholds(final ProcessContext context) { minObjectAgeMilliseconds = context.getProperty(MIN_AGE).asTimePeriod(TimeUnit.MILLISECONDS); maxObjectAgeMilliseconds = context.getProperty(MAX_AGE) != null ? context.getProperty(MAX_AGE).asTimePeriod(TimeUnit.MILLISECONDS) : null; } @@ -502,6 +517,77 @@ public void onTrigger(final ProcessContext context, final ProcessSession session } } + /** + *

+ * Reports backlog against the configured S3 bucket. The two tracking strategies + * ({@link #BY_TIMESTAMPS} and {@link #BY_ENTITIES}) compare the configured state of the + * processor to the listing of the bucket and report the unprocessed objects as remaining + * work. + *

+ * + *

+ * The {@link #NO_TRACKING} strategy is intentionally and unconditionally treated as + * {@link Backlog#caughtUp()}. In that mode the processor keeps no persistent state about + * which objects it has previously listed: every invocation re-lists the configured prefix + * from scratch and emits every matching object. There is therefore no notion of an + * "unprocessed set" that backlog could measure — the set is empty by definition, not by + * observation. Querying S3 to "verify" that fact would not produce different information, + * because any object the query found would, by definition, be emitted on the very next + * invocation and is not part of any backlog. + *

+ * + *

+ * This is intentionally different from {@link #BY_TIMESTAMPS} and {@link #BY_ENTITIES}, + * which do maintain "what has been listed" state in {@link Scope#CLUSTER cluster-scoped} + * state and where backlog has a meaningful definition (objects on the bucket whose + * identifiers/timestamps are not yet recorded in that state). Callers that want a Backlog + * derived from an actual bucket listing must configure one of those strategies. + *

+ */ + @Override + public Optional getBacklog(final ProcessContext context) throws BacklogReportingException { + final String listingStrategy = context.getProperty(LISTING_STRATEGY).getValue(); + if (NO_TRACKING.getValue().equals(listingStrategy)) { + // No Tracking has no "unprocessed" set by design; see the method javadoc for the rationale. + final Backlog backlog = Backlog.caughtUp(); + getLogger().debug("Computed S3 backlog using {} strategy: {}", NO_TRACKING.getDisplayName(), backlog); + return Optional.of(backlog); + } + + final S3Client client; + final String bucketName; + try { + client = getClient(context); + bucketName = context.getProperty(BUCKET_WITHOUT_DEFAULT_VALUE).evaluateAttributeExpressions().getValue(); + } catch (final RuntimeException e) { + throw new BacklogReportingException("Failed to create S3 client or resolve bucket name while reporting backlog", e); + } + + final long currentTime = System.currentTimeMillis(); + final List backlogCandidates; + try { + backlogCandidates = listBacklogCandidates(context, client, currentTime); + } catch (final RuntimeException e) { + throw new BacklogReportingException("Failed to list S3 bucket " + bucketName + " while reporting backlog", e); + } + + final Backlog backlog; + try { + if (BY_TIMESTAMPS.getValue().equals(listingStrategy)) { + backlog = getTimestampTrackingBacklog(context, backlogCandidates); + } else if (BY_ENTITIES.getValue().equals(listingStrategy)) { + backlog = getEntityTrackingBacklog(context, client, bucketName, currentTime, backlogCandidates); + } else { + throw new BacklogReportingException("Unknown listing strategy: " + listingStrategy); + } + } catch (final IOException | RuntimeException e) { + throw new BacklogReportingException("Failed to derive " + listingStrategy + " backlog for S3 bucket " + bucketName, e); + } + + getLogger().debug("Computed S3 backlog using {} strategy: {}", listingStrategy, backlog); + return Optional.of(backlog); + } + private void listNoTracking(ProcessContext context, ProcessSession session) { final S3Client client = getClient(context); @@ -715,11 +801,146 @@ && includeObjectInListing(objectVersion, currentTime)) justElectedPrimaryNode = false; } + private Backlog getTimestampTrackingBacklog(final ProcessContext context, final List backlogCandidates) throws IOException { + final StateMap stateMap = context.getStateManager().getState(Scope.CLUSTER); + if (stateMap.getStateVersion().isEmpty() || stateMap.get(CURRENT_TIMESTAMP) == null || stateMap.get(CURRENT_KEY_PREFIX + "0") == null) { + getLogger().debug("No timestamp tracking state recorded; every S3 object that satisfies the listing filters is unread, so the backlog equals the candidate count"); + return buildBacklog(backlogCandidates.size(), Precision.EXACT); + } + + final long currentTimestamp = Long.parseLong(stateMap.get(CURRENT_TIMESTAMP)); + final Set currentKeys = extractKeys(stateMap); + + long backlogObjectCount = 0L; + for (final ObjectVersion objectVersion : backlogCandidates) { + final long lastModified = objectVersion.lastModified().toEpochMilli(); + if (lastModified < currentTimestamp || (lastModified == currentTimestamp && currentKeys.contains(objectVersion.key()))) { + continue; + } + + backlogObjectCount++; + } + + return buildBacklog(backlogObjectCount, Precision.EXACT); + } + + private Backlog getEntityTrackingBacklog(final ProcessContext context, final S3Client client, final String bucketName, final long currentTime, final List backlogCandidates) { + final ListedS3VersionSummaryTracker s3Tracker = (listedEntityTracker instanceof final ListedS3VersionSummaryTracker tracker) ? tracker : null; + if (s3Tracker == null || !s3Tracker.hasTrackedEntities()) { + return buildBacklog(backlogCandidates.size(), Precision.EXACT); + } + + final long trackingWindowMilliseconds = context.getProperty(TRACKING_TIME_WINDOW).evaluateAttributeExpressions().asTimePeriod(TimeUnit.MILLISECONDS); + final long minimumTrackedTimestamp = currentTime - trackingWindowMilliseconds; + long backlogObjectCount = 0L; + for (final ObjectVersion objectVersion : backlogCandidates) { + final long lastModified = objectVersion.lastModified().toEpochMilli(); + if (lastModified < minimumTrackedTimestamp) { + continue; + } + + final String entityIdentifier = getBacklogEntityIdentifier(objectVersion); + if (s3Tracker.isBacklogEntity(entityIdentifier, lastModified, objectVersion.size())) { + backlogObjectCount++; + } + } + + return buildBacklog(backlogObjectCount, Precision.EXACT); + } + + /** + * Builds a {@link Backlog} from a freshly-computed object count. When the count is zero the + * remembered {@link #lastCaughtUpTimestamp} is refreshed to "now". The remembered timestamp — whether + * just-refreshed or carried over from an earlier caught-up moment — is always attached to the result so + * that a non-zero backlog still surfaces "the last time this Processor was caught up". + */ + private Backlog buildBacklog(final long backlogObjectCount, final Precision precision) { + if (backlogObjectCount == 0L) { + lastCaughtUpTimestamp = Instant.now(); + } + + final Backlog.Builder backlogBuilder = Backlog.builder() + .flowFiles(backlogObjectCount) + .precision(precision); + + final Instant rememberedCaughtUp = lastCaughtUpTimestamp; + if (rememberedCaughtUp != null) { + backlogBuilder.lastCaughtUp(rememberedCaughtUp); + } + + return backlogBuilder.build(); + } + + /** + * Identifies a backlog candidate by S3 key and version ID. When Use Versions is enabled, {@code + * objectVersion} comes from {@link S3VersionBucketLister}, which always populates a version ID: either + * a real version ID for a versioned bucket, or the literal string {@code "null"} that S3 assigns to + * objects in a bucket that has never had versioning enabled. When Use Versions is disabled, {@code + * objectVersion} is synthesized from a plain object listing that carries no version information, so + * the version ID is absent from the resulting identifier. + */ + private String getBacklogEntityIdentifier(final ObjectVersion objectVersion) { + return objectVersion.key() + "_" + objectVersion.versionId(); + } + + private List listBacklogCandidates(final ProcessContext context, final S3Client client, final long currentTime) { + // Use the same S3BucketLister abstraction the normal onTrigger path uses, so backlog + // reporting sees the same objects/versions the processor would actually emit. This honors + // both Use Versions (which switches to ListObjectVersions and therefore surfaces + // non-current versions) and List Type (1 vs 2, which selects ListObjects vs ListObjectsV2). + // Reporting backlog from a hard-coded ListObjectsV2 would otherwise under-count work in a + // versioned bucket because it would ignore non-current versions the processor can still emit. + final S3BucketLister bucketLister = getS3BucketLister(context, client); + final List matchingObjects = new ArrayList<>(); + do { + final List page = bucketLister.listVersions(); + for (final ObjectVersion objectVersion : page) { + if (includeObjectInListing(objectVersion, currentTime)) { + matchingObjects.add(objectVersion); + } + } + bucketLister.setNextMarker(); + } while (bucketLister.isTruncated()); + + return matchingObjects; + } + private class ListedS3VersionSummaryTracker extends ListedEntityTracker> { public ListedS3VersionSummaryTracker() { super(getIdentifier(), getLogger(), RecordObjectWriter.RECORD_SCHEMA); } + boolean hasTrackedEntities() { + return alreadyListedEntities != null && !alreadyListedEntities.isEmpty(); + } + + /** + * Determines whether an object should be counted as backlog for the Tracking Entities strategy, + * applying the same rule the real listing path uses in {@link ListedEntityTracker}'s + * {@code trackEntities}: an object is unprocessed work when it has never been tracked, when its + * last-modified timestamp is newer than the tracked timestamp, or when its size differs from the + * tracked size. This catches an object that was already tracked but has since been overwritten in + * place with a new timestamp or size, which the running processor would re-list and re-emit. The + * timestamp and size come from the object listing that already enumerated the candidate, so no + * additional S3 request is made. + */ + boolean isBacklogEntity(final String entityIdentifier, final long lastModifiedTimestamp, final long size) { + if (alreadyListedEntities == null) { + return true; + } + + final ListedEntity alreadyListedEntity = alreadyListedEntities.get(entityIdentifier); + if (alreadyListedEntity == null) { + return true; + } + + if (lastModifiedTimestamp > alreadyListedEntity.getTimestamp()) { + return true; + } + + return size != alreadyListedEntity.getSize(); + } + @Override protected void createRecordsForEntities( ProcessContext context, diff --git a/nifi-extension-bundles/nifi-aws-bundle/nifi-aws-processors/src/test/java/org/apache/nifi/processors/aws/s3/ITListS3.java b/nifi-extension-bundles/nifi-aws-bundle/nifi-aws-processors/src/test/java/org/apache/nifi/processors/aws/s3/ITListS3.java index f697cc775c06..1d3c39bb2659 100644 --- a/nifi-extension-bundles/nifi-aws-bundle/nifi-aws-processors/src/test/java/org/apache/nifi/processors/aws/s3/ITListS3.java +++ b/nifi-extension-bundles/nifi-aws-bundle/nifi-aws-processors/src/test/java/org/apache/nifi/processors/aws/s3/ITListS3.java @@ -16,18 +16,50 @@ */ package org.apache.nifi.processors.aws.s3; +import org.apache.nifi.components.Backlog; +import org.apache.nifi.components.BacklogReportingException; +import org.apache.nifi.controller.AbstractControllerService; +import org.apache.nifi.distributed.cache.client.Deserializer; +import org.apache.nifi.distributed.cache.client.DistributedMapCacheClient; +import org.apache.nifi.distributed.cache.client.Serializer; +import org.apache.nifi.processor.BacklogReportingProcessor; +import org.apache.nifi.processor.ProcessContext; +import org.apache.nifi.processors.aws.region.RegionUtil; import org.apache.nifi.reporting.InitializationException; import org.apache.nifi.util.MockFlowFile; import org.apache.nifi.util.TestRunner; +import org.apache.nifi.util.TestRunners; import org.junit.jupiter.api.Test; +import software.amazon.awssdk.services.s3.S3Client; +import software.amazon.awssdk.services.s3.model.HeadObjectRequest; +import software.amazon.awssdk.services.s3.model.ListObjectsV2Request; +import software.amazon.awssdk.services.s3.model.S3Exception; import software.amazon.awssdk.services.s3.model.Tag; +import java.io.ByteArrayOutputStream; import java.io.FileNotFoundException; import java.io.IOException; +import java.io.UncheckedIOException; +import java.nio.ByteBuffer; import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Optional; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertInstanceOf; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.Mockito.any; +import static org.mockito.Mockito.atLeastOnce; +import static org.mockito.Mockito.clearInvocations; +import static org.mockito.Mockito.doThrow; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.spy; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.verifyNoInteractions; /** * Provides integration level testing with actual AWS S3 resources for {@link ListS3} and requires additional configuration and resources to work. @@ -173,4 +205,292 @@ public void testUserMetadataWritten() throws FileNotFoundException, Initializati flowFiles.assertAttributeEquals("s3.user.metadata.dummy.metadata.2", "dummyvalue2"); } + @Test + public void testNoTrackingBacklogDoesNotListS3() throws BacklogReportingException, InitializationException { + final S3Client spyClient = spy(getClient()); + final TestRunner runner = initBacklogRunner(new TestableListS3(spyClient)); + runner.setProperty(ListS3.LISTING_STRATEGY, ListS3.NO_TRACKING.getValue()); + runner.run(0, false, true); + + clearInvocations(spyClient); + + final Backlog backlog = reportBacklog(runner).orElseThrow(); + assertTrue(backlog.getFlowFileCount().isPresent()); + assertEquals(0L, backlog.getFlowFileCount().getAsLong()); + assertTrue(backlog.getByteCount().isPresent()); + assertEquals(0L, backlog.getByteCount().getAsLong()); + assertTrue(backlog.getRecordCount().isPresent()); + assertEquals(0L, backlog.getRecordCount().getAsLong()); + assertEquals(Backlog.Precision.EXACT, backlog.getPrecision()); + assertTrue(backlog.getLastCaughtUp().isPresent()); + verifyNoInteractions(spyClient); + } + + @Test + public void testTimestampTrackingBacklogUsesPersistedWatermark() throws BacklogReportingException, InitializationException { + putTestFile("timestamps/first.txt", getFileFromResourceName(SAMPLE_FILE_RESOURCE_NAME)); + putTestFile("timestamps/second.txt", getFileFromResourceName(SAMPLE_FILE_RESOURCE_NAME)); + + final S3Client spyClient = spy(getClient()); + final TestRunner runner = initBacklogRunner(new TestableListS3(spyClient)); + runner.setProperty(ListS3.LISTING_STRATEGY, ListS3.BY_TIMESTAMPS.getValue()); + runner.setProperty(ListS3.PREFIX, "timestamps/"); + runner.run(0, false, true); + + final Backlog firstRunBacklog = reportBacklog(runner).orElseThrow(); + assertEquals(Backlog.Precision.EXACT, firstRunBacklog.getPrecision()); + assertTrue(firstRunBacklog.getFlowFileCount().isPresent()); + assertEquals(2L, firstRunBacklog.getFlowFileCount().getAsLong()); + assertFalse(firstRunBacklog.getLastCaughtUp().isPresent()); + + runner.run(1, false, false); + + putTestFile("timestamps/third.txt", getFileFromResourceName(SAMPLE_FILE_RESOURCE_NAME)); + + final Backlog backlog = reportBacklog(runner).orElseThrow(); + assertEquals(Backlog.Precision.EXACT, backlog.getPrecision()); + assertTrue(backlog.getFlowFileCount().isPresent()); + assertEquals(1L, backlog.getFlowFileCount().getAsLong()); + assertFalse(backlog.getLastCaughtUp().isPresent()); + verify(spyClient, atLeastOnce()).listObjectsV2(any(ListObjectsV2Request.class)); + + runner.run(1, false, false); + + final Backlog caughtUpBacklog = reportBacklog(runner).orElseThrow(); + assertEquals(Backlog.Precision.EXACT, caughtUpBacklog.getPrecision()); + assertTrue(caughtUpBacklog.getFlowFileCount().isPresent()); + assertEquals(0L, caughtUpBacklog.getFlowFileCount().getAsLong()); + assertTrue(caughtUpBacklog.getLastCaughtUp().isPresent()); + } + + @Test + public void testEntityTrackingBacklogUsesListedEntityTracker() throws BacklogReportingException, InitializationException { + putTestFile("entities/first.txt", getFileFromResourceName(SAMPLE_FILE_RESOURCE_NAME)); + putTestFile("entities/second.txt", getFileFromResourceName(SAMPLE_FILE_RESOURCE_NAME)); + + final S3Client spyClient = spy(getClient()); + final TestRunner runner = initEntityTrackingBacklogRunner(new TestableListS3(spyClient)); + runner.setProperty(ListS3.LISTING_STRATEGY, ListS3.BY_ENTITIES.getValue()); + runner.setProperty(ListS3.PREFIX, "entities/"); + runner.run(0, false, true); + + final Backlog firstRunBacklog = reportBacklog(runner).orElseThrow(); + assertEquals(Backlog.Precision.EXACT, firstRunBacklog.getPrecision()); + assertTrue(firstRunBacklog.getFlowFileCount().isPresent()); + assertEquals(2L, firstRunBacklog.getFlowFileCount().getAsLong()); + assertFalse(firstRunBacklog.getLastCaughtUp().isPresent()); + + runner.run(1, false, false); + + putTestFile("entities/third.txt", getFileFromResourceName(SAMPLE_FILE_RESOURCE_NAME)); + + final Backlog backlog = reportBacklog(runner).orElseThrow(); + assertEquals(Backlog.Precision.EXACT, backlog.getPrecision()); + assertTrue(backlog.getFlowFileCount().isPresent()); + assertEquals(1L, backlog.getFlowFileCount().getAsLong()); + assertFalse(backlog.getLastCaughtUp().isPresent()); + verify(spyClient, atLeastOnce()).listObjectsV2(any(ListObjectsV2Request.class)); + + runner.run(1, false, false); + + final Backlog caughtUpBacklog = reportBacklog(runner).orElseThrow(); + assertEquals(Backlog.Precision.EXACT, caughtUpBacklog.getPrecision()); + assertTrue(caughtUpBacklog.getFlowFileCount().isPresent()); + assertEquals(0L, caughtUpBacklog.getFlowFileCount().getAsLong()); + assertTrue(caughtUpBacklog.getLastCaughtUp().isPresent()); + } + + @Test + public void testBacklogWrapsListObjectsV2Errors() throws InitializationException { + final S3Client spyClient = spy(getClient()); + doThrow(S3Exception.builder().message("boom").build()).when(spyClient).listObjectsV2(any(ListObjectsV2Request.class)); + + final TestRunner runner = initBacklogRunner(new TestableListS3(spyClient)); + runner.setProperty(ListS3.LISTING_STRATEGY, ListS3.BY_TIMESTAMPS.getValue()); + runner.run(0, false, true); + + final BacklogReportingException exception = assertThrows(BacklogReportingException.class, () -> reportBacklog(runner)); + assertTrue(exception.getMessage().contains("Failed to list S3 bucket")); + assertTrue(exception.getMessage().contains("while reporting backlog")); + assertTrue(exception.getMessage().contains(BUCKET_NAME)); + assertInstanceOf(S3Exception.class, exception.getCause()); + } + + @Test + public void testBacklogWhileStoppedWithLagBeforeOnScheduled() throws BacklogReportingException, InitializationException { + putTestFile("stopped-lag/first.txt", getFileFromResourceName(SAMPLE_FILE_RESOURCE_NAME)); + putTestFile("stopped-lag/second.txt", getFileFromResourceName(SAMPLE_FILE_RESOURCE_NAME)); + + final S3Client spyClient = spy(getClient()); + final TestRunner runner = initBacklogRunner(new TestableListS3(spyClient)); + runner.setProperty(ListS3.LISTING_STRATEGY, ListS3.BY_TIMESTAMPS.getValue()); + runner.setProperty(ListS3.PREFIX, "stopped-lag/"); + + // Never call runner.run(), so @OnScheduled is never invoked. The fresh-query path must still answer. + // Because no listing state has been recorded yet, every object that satisfies the listing filters is + // unread, so the count is reported as EXACT. + final Backlog backlog = reportBacklog(runner).orElseThrow(); + assertEquals(Backlog.Precision.EXACT, backlog.getPrecision()); + assertTrue(backlog.getFlowFileCount().isPresent()); + assertEquals(2L, backlog.getFlowFileCount().getAsLong()); + assertFalse(backlog.getLastCaughtUp().isPresent()); + verify(spyClient, atLeastOnce()).listObjectsV2(any(ListObjectsV2Request.class)); + } + + @Test + public void testBacklogWhileStoppedAfterCaughtUp() throws BacklogReportingException, InitializationException { + putTestFile("stopped-caught-up/first.txt", getFileFromResourceName(SAMPLE_FILE_RESOURCE_NAME)); + putTestFile("stopped-caught-up/second.txt", getFileFromResourceName(SAMPLE_FILE_RESOURCE_NAME)); + + final S3Client spyClient = spy(getClient()); + final TestRunner runner = initBacklogRunner(new TestableListS3(spyClient)); + runner.setProperty(ListS3.LISTING_STRATEGY, ListS3.BY_TIMESTAMPS.getValue()); + runner.setProperty(ListS3.PREFIX, "stopped-caught-up/"); + + // Drain the listing while running, then stop the processor; the subsequent backlog call + // happens with no @OnScheduled context active and must still produce a meaningful answer. + runner.run(2, true, true); + + final Backlog backlog = reportBacklog(runner).orElseThrow(); + assertEquals(Backlog.Precision.EXACT, backlog.getPrecision()); + assertTrue(backlog.getFlowFileCount().isPresent()); + assertEquals(0L, backlog.getFlowFileCount().getAsLong()); + assertTrue(backlog.getLastCaughtUp().isPresent()); + } + + @Test + public void testEntityTrackingBacklogWithVersionsEnabledIdentifiesEntitiesByVersionId() throws BacklogReportingException, InitializationException { + putTestFile("versioned/first.txt", getFileFromResourceName(SAMPLE_FILE_RESOURCE_NAME)); + + final S3Client spyClient = spy(getClient()); + final TestRunner runner = initEntityTrackingBacklogRunner(new TestableListS3(spyClient)); + runner.setProperty(ListS3.LISTING_STRATEGY, ListS3.BY_ENTITIES.getValue()); + runner.setProperty(ListS3.PREFIX, "versioned/"); + runner.setProperty(ListS3.USE_VERSIONS, "true"); + runner.run(1, false, true); + + final Backlog firstRunBacklog = reportBacklog(runner).orElseThrow(); + assertEquals(Backlog.Precision.EXACT, firstRunBacklog.getPrecision()); + assertTrue(firstRunBacklog.getFlowFileCount().isPresent()); + assertEquals(0L, firstRunBacklog.getFlowFileCount().getAsLong()); + + putTestFile("versioned/second.txt", getFileFromResourceName(SAMPLE_FILE_RESOURCE_NAME)); + + final Backlog backlog = reportBacklog(runner).orElseThrow(); + assertEquals(Backlog.Precision.EXACT, backlog.getPrecision()); + assertTrue(backlog.getFlowFileCount().isPresent()); + assertEquals(1L, backlog.getFlowFileCount().getAsLong()); + verify(spyClient, never()).headObject(any(HeadObjectRequest.class)); + } + + private TestRunner initBacklogRunner(final ListS3 processor) throws InitializationException { + final TestRunner runner = TestRunners.newTestRunner(processor); + setSecureProperties(runner); + runner.setProperty(RegionUtil.REGION, getRegion()); + runner.setProperty(ListS3.ENDPOINT_OVERRIDE, getEndpointOverride()); + runner.setProperty(ListS3.BUCKET_WITHOUT_DEFAULT_VALUE, BUCKET_NAME); + // These backlog tests assert against the List Objects V2 endpoint, so List Type must be set + // explicitly since the Processor's default is List Objects V1. + runner.setProperty(ListS3.LIST_TYPE, "2"); + return runner; + } + + private TestRunner initEntityTrackingBacklogRunner(final ListS3 processor) throws InitializationException { + final TestRunner runner = initBacklogRunner(processor); + final EphemeralMapCacheClientService trackingStateCache = new EphemeralMapCacheClientService(); + runner.addControllerService("tracking-state-cache", trackingStateCache); + runner.enableControllerService(trackingStateCache); + runner.setProperty(ListS3.TRACKING_STATE_CACHE, "tracking-state-cache"); + return runner; + } + + private Optional reportBacklog(final TestRunner runner) throws BacklogReportingException { + final BacklogReportingProcessor backlogReportingProcessor = (BacklogReportingProcessor) runner.getProcessor(); + return backlogReportingProcessor.getBacklog(runner.getProcessContext()); + } + + private static final class TestableListS3 extends ListS3 { + private final S3Client client; + + private TestableListS3(final S3Client client) { + this.client = client; + } + + @Override + protected S3Client getClient(final ProcessContext context) { + return client; + } + + @Override + protected S3Client getClient(final ProcessContext context, final Map attributes) { + return client; + } + + @Override + protected S3Client createClient(final ProcessContext context, final Map attributes) { + return client; + } + } + + /** + * Minimal in-memory {@link DistributedMapCacheClient} backing the {@link ListS3} entity-tracker for the + * tests in this class. Only the cache methods actually exercised by + * {@link org.apache.nifi.processor.util.list.ListedEntityTracker} ({@code put}, {@code get}, and + * {@code remove}) are implemented; every other method on the interface throws to make accidental usage + * obvious during test development. + */ + private static final class EphemeralMapCacheClientService extends AbstractControllerService implements DistributedMapCacheClient { + private final Map stored = new HashMap<>(); + + @Override + public void put(final K key, final V value, final Serializer keySerializer, final Serializer valueSerializer) { + stored.put(serializeAsByteBuffer(key, keySerializer), serializeAsBytes(value, valueSerializer)); + } + + @Override + public V get(final K key, final Serializer keySerializer, final Deserializer valueDeserializer) throws IOException { + final byte[] storedValue = stored.get(serializeAsByteBuffer(key, keySerializer)); + if (storedValue == null) { + return null; + } + return valueDeserializer.deserialize(storedValue); + } + + @Override + public boolean remove(final K key, final Serializer serializer) { + return stored.remove(serializeAsByteBuffer(key, serializer)) != null; + } + + @Override + public void close() throws IOException { + } + + @Override + public boolean putIfAbsent(final K key, final V value, final Serializer keySerializer, final Serializer valueSerializer) { + throw new UnsupportedOperationException("putIfAbsent is not exercised by these tests"); + } + + @Override + public V getAndPutIfAbsent(final K key, final V value, final Serializer keySerializer, final Serializer valueSerializer, final Deserializer valueDeserializer) { + throw new UnsupportedOperationException("getAndPutIfAbsent is not exercised by these tests"); + } + + @Override + public boolean containsKey(final K key, final Serializer keySerializer) { + throw new UnsupportedOperationException("containsKey is not exercised by these tests"); + } + + private static ByteBuffer serializeAsByteBuffer(final T value, final Serializer serializer) { + return ByteBuffer.wrap(serializeAsBytes(value, serializer)); + } + + private static byte[] serializeAsBytes(final T value, final Serializer serializer) { + try (final ByteArrayOutputStream outputStream = new ByteArrayOutputStream()) { + serializer.serialize(value, outputStream); + return outputStream.toByteArray(); + } catch (final IOException exception) { + throw new UncheckedIOException("Failed to serialize cache value", exception); + } + } + } } diff --git a/nifi-extension-bundles/nifi-aws-bundle/nifi-aws-processors/src/test/java/org/apache/nifi/processors/aws/s3/TestListS3.java b/nifi-extension-bundles/nifi-aws-bundle/nifi-aws-processors/src/test/java/org/apache/nifi/processors/aws/s3/TestListS3.java index 6160c19db209..7e119193e702 100644 --- a/nifi-extension-bundles/nifi-aws-bundle/nifi-aws-processors/src/test/java/org/apache/nifi/processors/aws/s3/TestListS3.java +++ b/nifi-extension-bundles/nifi-aws-bundle/nifi-aws-processors/src/test/java/org/apache/nifi/processors/aws/s3/TestListS3.java @@ -17,6 +17,8 @@ package org.apache.nifi.processors.aws.s3; import org.apache.nifi.components.AllowableValue; +import org.apache.nifi.components.Backlog; +import org.apache.nifi.components.BacklogReportingException; import org.apache.nifi.components.ConfigVerificationResult; import org.apache.nifi.components.state.Scope; import org.apache.nifi.distributed.cache.client.DistributedMapCacheClient; @@ -61,6 +63,7 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Optional; import java.util.Set; import static org.apache.nifi.processors.aws.region.RegionUtil.REGION; @@ -68,6 +71,8 @@ import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertSame; +import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.eq; @@ -403,6 +408,208 @@ public void testListVersions() { ff1.assertAttributeEquals("s3.version", "2"); } + @Test + public void testBacklogWithUseVersionsEnumeratesAllObjectVersions() throws BacklogReportingException { + // When Use Versions=true, backlog reporting must follow the processor's configured listing + // mode and enumerate via ListObjectVersions so that non-current versions are included in + // the count. Going through ListObjectsV2 in this configuration would under-report the + // backlog because non-current versions, which the processor itself can still emit, would + // not be visible. + runner.setProperty(RegionUtil.REGION, "eu-west-1"); + runner.setProperty(ListS3.BUCKET_WITHOUT_DEFAULT_VALUE, "test-bucket"); + runner.setProperty(ListS3.LISTING_STRATEGY, ListS3.BY_TIMESTAMPS.getValue()); + runner.setProperty(ListS3.USE_VERSIONS, "true"); + + final Instant lastModified = Instant.now().minus(1, ChronoUnit.HOURS); + final ObjectVersion currentVersion = ObjectVersion.builder() + .key("versioned-key") + .versionId("v2") + .lastModified(lastModified) + .size(10L) + .isLatest(true) + .build(); + final ObjectVersion priorVersion = ObjectVersion.builder() + .key("versioned-key") + .versionId("v1") + .lastModified(lastModified.minus(1, ChronoUnit.HOURS)) + .size(20L) + .isLatest(false) + .build(); + final ListObjectVersionsResponse response = ListObjectVersionsResponse.builder() + .versions(currentVersion, priorVersion) + .isTruncated(false) + .build(); + when(mockS3Client.listObjectVersions(any(ListObjectVersionsRequest.class))).thenReturn(response); + + final Optional reported = listS3.getBacklog(runner.getProcessContext()); + + assertTrue(reported.isPresent()); + final Backlog backlog = reported.get(); + assertEquals(Backlog.Precision.EXACT, backlog.getPrecision()); + assertTrue(backlog.getFlowFileCount().isPresent()); + assertEquals(2L, backlog.getFlowFileCount().getAsLong()); + + verify(mockS3Client, Mockito.atLeastOnce()).listObjectVersions(any(ListObjectVersionsRequest.class)); + verify(mockS3Client, Mockito.never()).listObjectsV2(any(ListObjectsV2Request.class)); + verify(mockS3Client, Mockito.never()).listObjects(any(ListObjectsRequest.class)); + } + + @Test + public void testBacklogWithListType1EnumeratesWithListObjects() throws BacklogReportingException { + // List Type=1 selects the legacy ListObjects API. Backlog reporting must follow that mode + // rather than always going through ListObjectsV2, so that what is counted matches what the + // processor would actually emit. + runner.setProperty(RegionUtil.REGION, "eu-west-1"); + runner.setProperty(ListS3.BUCKET_WITHOUT_DEFAULT_VALUE, "test-bucket"); + runner.setProperty(ListS3.LISTING_STRATEGY, ListS3.BY_TIMESTAMPS.getValue()); + runner.setProperty(ListS3.LIST_TYPE, "1"); + + final Instant lastModified = Instant.now().minus(1, ChronoUnit.HOURS); + final S3Object first = S3Object.builder().key("a").lastModified(lastModified).size(5L).build(); + final S3Object second = S3Object.builder().key("b").lastModified(lastModified).size(7L).build(); + final ListObjectsResponse response = ListObjectsResponse.builder() + .contents(first, second) + .isTruncated(false) + .build(); + when(mockS3Client.listObjects(any(ListObjectsRequest.class))).thenReturn(response); + + final Optional reported = listS3.getBacklog(runner.getProcessContext()); + + assertTrue(reported.isPresent()); + final Backlog backlog = reported.get(); + assertEquals(Backlog.Precision.EXACT, backlog.getPrecision()); + assertTrue(backlog.getFlowFileCount().isPresent()); + assertEquals(2L, backlog.getFlowFileCount().getAsLong()); + + verify(mockS3Client, Mockito.atLeastOnce()).listObjects(any(ListObjectsRequest.class)); + verify(mockS3Client, Mockito.never()).listObjectsV2(any(ListObjectsV2Request.class)); + verify(mockS3Client, Mockito.never()).listObjectVersions(any(ListObjectVersionsRequest.class)); + } + + @Test + public void testEntityTrackingBacklogCountsOverwrittenObject() throws Exception { + final String serviceId = "DistributedMapCacheClient"; + when(mockCache.getIdentifier()).thenReturn(serviceId); + runner.addControllerService(serviceId, mockCache); + runner.enableControllerService(mockCache); + + runner.setProperty(RegionUtil.REGION, "eu-west-1"); + runner.setProperty(ListS3.BUCKET_WITHOUT_DEFAULT_VALUE, "test-bucket"); + runner.setProperty(ListS3.LISTING_STRATEGY, ListS3.BY_ENTITIES); + runner.setProperty(ListS3.LIST_TYPE, "1"); + runner.setProperty(ListS3.TRACKING_STATE_CACHE, serviceId); + + final Instant baseTime = Instant.now(); + final S3Object original = S3Object.builder() + .key("overwritten-key") + .lastModified(baseTime.minus(5, ChronoUnit.MINUTES)) + .size(10L) + .build(); + when(mockS3Client.listObjects(any(ListObjectsRequest.class))) + .thenReturn(ListObjectsResponse.builder().contents(original).isTruncated(false).build()); + + // Populate the entity tracker with the original object by exercising the normal listing path. + // stopOnFinish is false so the tracker's in-memory listed-entity state remains available to getBacklog. + runner.run(1, false, true); + runner.assertAllFlowFilesTransferred(ListS3.REL_SUCCESS, 1); + + // The tracked object has not changed, so it is not part of the backlog. + final Optional caughtUp = listS3.getBacklog(runner.getProcessContext()); + assertTrue(caughtUp.isPresent()); + assertTrue(caughtUp.get().getFlowFileCount().isPresent()); + assertEquals(0L, caughtUp.get().getFlowFileCount().getAsLong()); + + // The same key is overwritten in place with a newer timestamp and a different size. The running + // processor's ListedEntityTracker would re-list and re-emit it, so backlog must count it even + // though the identifier was already tracked. + final S3Object overwritten = S3Object.builder() + .key("overwritten-key") + .lastModified(baseTime.minus(1, ChronoUnit.MINUTES)) + .size(20L) + .build(); + when(mockS3Client.listObjects(any(ListObjectsRequest.class))) + .thenReturn(ListObjectsResponse.builder().contents(overwritten).isTruncated(false).build()); + + final Optional reported = listS3.getBacklog(runner.getProcessContext()); + assertTrue(reported.isPresent()); + assertTrue(reported.get().getFlowFileCount().isPresent()); + assertEquals(1L, reported.get().getFlowFileCount().getAsLong()); + } + + @Test + public void testGetBacklogWrapsClientCreationFailure() { + final RuntimeException clientFailure = new RuntimeException("credentials provider failed"); + final ListS3 failingListS3 = new ListS3() { + @Override + protected S3Client getClient(final ProcessContext context, final Map attributes) { + throw clientFailure; + } + + @Override + protected S3Client createClient(final ProcessContext context, final Map attributes) { + throw clientFailure; + } + }; + final TestRunner failingRunner = TestRunners.newTestRunner(failingListS3); + AuthUtils.enableAccessKey(failingRunner, "accessKeyId", "secretKey"); + failingRunner.setProperty(RegionUtil.REGION, "eu-west-1"); + failingRunner.setProperty(ListS3.BUCKET_WITHOUT_DEFAULT_VALUE, "test-bucket"); + failingRunner.setProperty(ListS3.LISTING_STRATEGY, ListS3.BY_TIMESTAMPS.getValue()); + + final BacklogReportingException thrown = assertThrows(BacklogReportingException.class, + () -> failingListS3.getBacklog(failingRunner.getProcessContext())); + assertSame(clientFailure, thrown.getCause()); + } + + @Test + public void testBacklogCountsAllPagesWhenListingPaginates() throws BacklogReportingException { + runner.setProperty(RegionUtil.REGION, "eu-west-1"); + runner.setProperty(ListS3.BUCKET_WITHOUT_DEFAULT_VALUE, "test-bucket"); + runner.setProperty(ListS3.LISTING_STRATEGY, ListS3.BY_TIMESTAMPS.getValue()); + runner.setProperty(ListS3.LIST_TYPE, "1"); + + final ListObjectsResponse firstPage = buildListObjectsResponse(List.of("key-0001", "key-0002", "key-0003"), true, null); + final ListObjectsResponse secondPage = buildListObjectsResponse(List.of("key-0004", "key-0005"), false, null); + when(mockS3Client.listObjects(any(ListObjectsRequest.class))) + .thenReturn(firstPage) + .thenReturn(secondPage); + + final Optional reported = listS3.getBacklog(runner.getProcessContext()); + + assertTrue(reported.isPresent()); + assertTrue(reported.get().getFlowFileCount().isPresent()); + assertEquals(5L, reported.get().getFlowFileCount().getAsLong()); + verify(mockS3Client, Mockito.times(2)).listObjects(any(ListObjectsRequest.class)); + } + + @Test + public void testBacklogSkipsAgeFilteringBeforeOnScheduled() throws BacklogReportingException { + // getBacklog can be invoked before the processor has ever been scheduled, for example from the + // UI while the processor is stopped. The minimum and maximum object-age thresholds are resolved + // in @OnScheduled, so before the processor is scheduled they are unset and age filtering does not + // run: every listed object is counted regardless of the configured age thresholds. + runner.setProperty(RegionUtil.REGION, "eu-west-1"); + runner.setProperty(ListS3.BUCKET_WITHOUT_DEFAULT_VALUE, "test-bucket"); + runner.setProperty(ListS3.LISTING_STRATEGY, ListS3.BY_TIMESTAMPS.getValue()); + runner.setProperty(ListS3.LIST_TYPE, "1"); + runner.setProperty(ListS3.MIN_AGE, "1 hour"); + runner.setProperty(ListS3.MAX_AGE, "2 hours"); + + // The "young" object is newer than the one-hour minimum age and the "old" object is older than + // the two-hour maximum age, so both would be excluded if the age filter were applied. + final Instant now = Instant.now(); + final S3Object young = S3Object.builder().key("young").lastModified(now).size(1L).build(); + final S3Object old = S3Object.builder().key("old").lastModified(now.minus(3, ChronoUnit.HOURS)).size(1L).build(); + when(mockS3Client.listObjects(any(ListObjectsRequest.class))) + .thenReturn(ListObjectsResponse.builder().contents(young, old).isTruncated(false).build()); + + final Optional reported = listS3.getBacklog(runner.getProcessContext()); + + assertTrue(reported.isPresent()); + assertTrue(reported.get().getFlowFileCount().isPresent()); + assertEquals(2L, reported.get().getFlowFileCount().getAsLong()); + } + @Test public void testListObjectsNothingNew() throws IOException { runner.setProperty(RegionUtil.REGION, "eu-west-1"); diff --git a/nifi-extension-bundles/nifi-kafka-bundle/nifi-kafka-3-integration/src/test/java/org/apache/nifi/kafka/processors/ConsumeKafkaBacklogIT.java b/nifi-extension-bundles/nifi-kafka-bundle/nifi-kafka-3-integration/src/test/java/org/apache/nifi/kafka/processors/ConsumeKafkaBacklogIT.java new file mode 100644 index 000000000000..62f719b3ff8f --- /dev/null +++ b/nifi-extension-bundles/nifi-kafka-bundle/nifi-kafka-3-integration/src/test/java/org/apache/nifi/kafka/processors/ConsumeKafkaBacklogIT.java @@ -0,0 +1,275 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.kafka.processors; + +import org.apache.kafka.clients.admin.Admin; +import org.apache.kafka.clients.admin.AdminClientConfig; +import org.apache.kafka.clients.admin.NewTopic; +import org.apache.nifi.components.Backlog; +import org.apache.nifi.components.BacklogReportingException; +import org.apache.nifi.controller.ControllerService; +import org.apache.nifi.kafka.processors.consumer.ProcessingStrategy; +import org.apache.nifi.kafka.service.Kafka3ConnectionService; +import org.apache.nifi.kafka.service.api.consumer.AutoOffsetReset; +import org.apache.nifi.reporting.InitializationException; +import org.apache.nifi.util.TestRunner; +import org.apache.nifi.util.TestRunners; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.Timeout; + +import java.time.Duration; +import java.time.Instant; +import java.util.Collections; +import java.util.Optional; +import java.util.Properties; +import java.util.UUID; +import java.util.concurrent.TimeUnit; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertSame; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * Integration tests for {@link ConsumeKafka#getBacklog(org.apache.nifi.processor.ProcessContext)} backed by a Testcontainers Kafka + * broker. Covers the {@link Backlog} payload reported for non-zero lag, the caught-up shape + * after a full drain, the persistence of {@code lastCaughtUp} across subsequent re-lagged + * reports, and pattern-based subscriptions. + */ +@Timeout(value = 90, unit = TimeUnit.SECONDS) +class ConsumeKafkaBacklogIT extends AbstractConsumeKafkaIT { + + private static final String MESSAGE_VALUE = "abcdefghij"; + + private TestRunner runner; + + @BeforeEach + void setRunner() throws InitializationException { + runner = TestRunners.newTestRunner(ConsumeKafka.class); + addKafkaConnectionService(runner); + runner.setProperty(ConsumeKafka.CONNECTION_SERVICE, CONNECTION_SERVICE_ID); + runner.setProperty(ConsumeKafka.AUTO_OFFSET_RESET, AutoOffsetReset.EARLIEST.getValue()); + runner.setProperty(ConsumeKafka.PROCESSING_STRATEGY, ProcessingStrategy.FLOW_FILE.getValue()); + } + + @Test + void testBacklogReportsLagBeforeDrain() throws Exception { + final String topic = newTopic(); + configureProcessor(topic); + configurePartialDrain(); + + final int seedCount = 5; + seedMessages(topic, seedCount); + + runner.run(1, false, true); + while (runner.getFlowFilesForRelationship(ConsumeKafka.SUCCESS).isEmpty()) { + runner.run(1, false, false); + } + + final int delivered = runner.getFlowFilesForRelationship(ConsumeKafka.SUCCESS).size(); + assertTrue(delivered > 0 && delivered < seedCount, "Partial drain expected; delivered=" + delivered); + + final Backlog backlog = requireBacklog(); + assertEquals(Backlog.Precision.EXACT, backlog.getPrecision()); + assertTrue(backlog.getRecordCount().isPresent()); + assertEquals((long) (seedCount - delivered), backlog.getRecordCount().getAsLong()); + assertFalse(backlog.getLastCaughtUp().isPresent()); + } + + @Test + void testBacklogReportsCaughtUpAfterDrain() throws Exception { + final String topic = newTopic(); + configureProcessor(topic); + + final int seedCount = 4; + seedMessages(topic, seedCount); + drainAll(seedCount); + + final Instant beforeBacklog = Instant.now(); + final Backlog backlog = requireBacklog(); + final Instant afterBacklog = Instant.now(); + + assertEquals(Backlog.Precision.EXACT, backlog.getPrecision()); + // A caught-up group has an exactly-known lag of zero. The Backlog populates the records + // dimension with that zero alongside lastCaughtUp so clients render "0 records" rather + // than treating the record count as unknown. flowFileCount and byteCount remain omitted + // because Kafka measures lag in records, not FlowFiles or bytes. + assertTrue(backlog.getRecordCount().isPresent()); + assertEquals(0L, backlog.getRecordCount().getAsLong()); + assertFalse(backlog.getFlowFileCount().isPresent()); + assertFalse(backlog.getByteCount().isPresent()); + assertTrue(backlog.getLastCaughtUp().isPresent()); + + final Instant lastCaughtUp = backlog.getLastCaughtUp().get(); + assertFalse(lastCaughtUp.isBefore(beforeBacklog), "lastCaughtUp=" + lastCaughtUp + " before window start=" + beforeBacklog); + assertFalse(lastCaughtUp.isAfter(afterBacklog), "lastCaughtUp=" + lastCaughtUp + " after window end=" + afterBacklog); + } + + @Test + void testBacklogPreservesLastCaughtUpAfterReSeed() throws Exception { + final String topic = newTopic(); + configureProcessor(topic); + configurePartialDrain(); + + final int initialSeed = 4; + seedMessages(topic, initialSeed); + drainAll(initialSeed); + + final Backlog drainedBacklog = requireBacklog(); + assertTrue(drainedBacklog.getLastCaughtUp().isPresent()); + final Instant priorCaughtUp = drainedBacklog.getLastCaughtUp().get(); + + final int reSeed = 3; + seedMessages(topic, reSeed); + + final int previouslyDelivered = runner.getFlowFilesForRelationship(ConsumeKafka.SUCCESS).size(); + while (runner.getFlowFilesForRelationship(ConsumeKafka.SUCCESS).size() == previouslyDelivered) { + runner.run(1, false, false); + } + + final int redelivered = runner.getFlowFilesForRelationship(ConsumeKafka.SUCCESS).size() - previouslyDelivered; + assertTrue(redelivered > 0 && redelivered < reSeed, "Partial re-drain expected; redelivered=" + redelivered); + + final Backlog reLagBacklog = requireBacklog(); + assertEquals(Backlog.Precision.EXACT, reLagBacklog.getPrecision()); + assertTrue(reLagBacklog.getRecordCount().isPresent()); + assertEquals((long) (reSeed - redelivered), reLagBacklog.getRecordCount().getAsLong()); + assertTrue(reLagBacklog.getLastCaughtUp().isPresent()); + assertSame(priorCaughtUp, reLagBacklog.getLastCaughtUp().get()); + } + + @Test + void testBacklogWhileStoppedReportsLag() throws Exception { + final String topic = newTopic(); + createTopic(topic, 1); + configureProcessor(topic); + + final int seedCount = 5; + seedMessages(topic, seedCount); + + // Never invoke @OnScheduled. getBacklog must build a fresh consumer service from the + // supplied ProcessContext and report the stream's lag. + final Backlog backlog = requireBacklog(); + assertEquals(Backlog.Precision.EXACT, backlog.getPrecision()); + assertTrue(backlog.getRecordCount().isPresent()); + assertEquals((long) seedCount, backlog.getRecordCount().getAsLong()); + assertFalse(backlog.getLastCaughtUp().isPresent()); + } + + @Test + void testBacklogWhileStoppedAfterCaughtUpPreservesLastCaughtUp() throws Exception { + final String topic = newTopic(); + configureProcessor(topic); + + final int seedCount = 3; + seedMessages(topic, seedCount); + drainAll(seedCount); + + // The drain call above triggered @OnScheduled and ran the processor; a caught-up moment has + // now been recorded. Stop the processor and ask for the backlog again — the fresh Admin-client + // query returns an exactly-known lag of zero with a lastCaughtUp timestamp populated. + runner.run(0, true, false); + + final ConsumeKafka consumeKafka = (ConsumeKafka) runner.getProcessor(); + final Optional reported = consumeKafka.getBacklog(runner.getProcessContext()); + assertTrue(reported.isPresent()); + final Backlog backlog = reported.get(); + assertTrue(backlog.getLastCaughtUp().isPresent()); + assertTrue(backlog.getRecordCount().isPresent()); + assertEquals(0L, backlog.getRecordCount().getAsLong()); + } + + @Test + void testBacklogFailsWellWithinDefaultClientTimeoutWhenBrokerUnreachable() { + final String topic = newTopic(); + configureProcessor(topic); + pointConnectionServiceAtUnreachableBroker(); + + final ConsumeKafka consumeKafka = (ConsumeKafka) runner.getProcessor(); + final Instant beforeBacklogAttempt = Instant.now(); + assertThrows(BacklogReportingException.class, () -> consumeKafka.getBacklog(runner.getProcessContext())); + final Duration elapsed = Duration.between(beforeBacklogAttempt, Instant.now()); + + // The Admin client used for backlog determination is bounded independently of the much longer + // default Client Timeout (60 seconds) configured for regular produce/consume operations, so an + // unreachable broker should not stall a backlog check for anywhere close to that long. + assertTrue(elapsed.toSeconds() < 30, "Expected backlog determination against an unreachable broker to fail in well under 30 seconds, but took " + elapsed); + } + + private void pointConnectionServiceAtUnreachableBroker() { + final ControllerService connectionService = runner.getControllerService(CONNECTION_SERVICE_ID); + runner.disableControllerService(connectionService); + runner.setProperty(connectionService, Kafka3ConnectionService.BOOTSTRAP_SERVERS, "localhost:1"); + runner.enableControllerService(connectionService); + } + + private void configureProcessor(final String topic) { + runner.setProperty(ConsumeKafka.TOPICS, topic); + runner.setProperty(ConsumeKafka.GROUP_ID, "ConsumeKafkaBacklogIT-" + UUID.randomUUID()); + } + + /** + * Constrains the inner poll loop to a single record per onTrigger so that seeded messages + * drain in small steps, leaving observable non-zero lag for assertions. + */ + private void configurePartialDrain() { + final ControllerService connectionService = runner.getControllerService(CONNECTION_SERVICE_ID); + runner.disableControllerService(connectionService); + runner.setProperty(connectionService, Kafka3ConnectionService.MAX_POLL_RECORDS, "1"); + runner.enableControllerService(connectionService); + + runner.setProperty(ConsumeKafka.MAX_UNCOMMITTED_SIZE, "1 B"); + runner.setProperty(ConsumeKafka.MAX_UNCOMMITTED_TIME, "1 s"); + } + + private void seedMessages(final String topic, final int count) throws Exception { + for (int index = 0; index < count; index++) { + produceOne(topic, 0, null, MESSAGE_VALUE, null); + } + } + + private void drainAll(final int expectedCount) { + runner.run(1, false, true); + while (runner.getFlowFilesForRelationship(ConsumeKafka.SUCCESS).size() < expectedCount) { + runner.run(1, false, false); + } + runner.assertTransferCount(ConsumeKafka.SUCCESS, expectedCount); + } + + private Backlog requireBacklog() throws Exception { + final ConsumeKafka consumeKafka = (ConsumeKafka) runner.getProcessor(); + final Optional reported = consumeKafka.getBacklog(runner.getProcessContext()); + assertTrue(reported.isPresent()); + return reported.get(); + } + + private static String newTopic() { + return "ConsumeKafkaBacklogIT-" + UUID.randomUUID(); + } + + private void createTopic(final String topic, final int numPartitions) throws Exception { + final Properties adminProps = new Properties(); + adminProps.put(AdminClientConfig.BOOTSTRAP_SERVERS_CONFIG, kafkaContainer.getBootstrapServers()); + + try (final Admin admin = Admin.create(adminProps)) { + final NewTopic topicCreationRequest = new NewTopic(topic, numPartitions, (short) 1); + admin.createTopics(Collections.singletonList(topicCreationRequest)).all().get(30, TimeUnit.SECONDS); + } + } +} diff --git a/nifi-extension-bundles/nifi-kafka-bundle/nifi-kafka-processors/src/main/java/org/apache/nifi/kafka/processors/ConsumeKafka.java b/nifi-extension-bundles/nifi-kafka-bundle/nifi-kafka-processors/src/main/java/org/apache/nifi/kafka/processors/ConsumeKafka.java index b489edbc6622..b853076af96e 100644 --- a/nifi-extension-bundles/nifi-kafka-bundle/nifi-kafka-processors/src/main/java/org/apache/nifi/kafka/processors/ConsumeKafka.java +++ b/nifi-extension-bundles/nifi-kafka-bundle/nifi-kafka-processors/src/main/java/org/apache/nifi/kafka/processors/ConsumeKafka.java @@ -25,6 +25,8 @@ import org.apache.nifi.annotation.lifecycle.OnScheduled; import org.apache.nifi.annotation.lifecycle.OnStopped; import org.apache.nifi.components.AllowableValue; +import org.apache.nifi.components.Backlog; +import org.apache.nifi.components.BacklogReportingException; import org.apache.nifi.components.ConfigVerificationResult; import org.apache.nifi.components.ConfigVerificationResult.Outcome; import org.apache.nifi.components.PropertyDescriptor; @@ -57,6 +59,7 @@ import org.apache.nifi.kafka.shared.property.OutputStrategy; import org.apache.nifi.logging.ComponentLog; import org.apache.nifi.processor.AbstractProcessor; +import org.apache.nifi.processor.BacklogReportingProcessor; import org.apache.nifi.processor.DataUnit; import org.apache.nifi.processor.ProcessContext; import org.apache.nifi.processor.ProcessSession; @@ -75,12 +78,14 @@ import java.nio.charset.Charset; import java.nio.charset.StandardCharsets; import java.time.Duration; +import java.time.Instant; import java.util.ArrayList; import java.util.Collection; import java.util.HexFormat; import java.util.Iterator; import java.util.List; import java.util.Map; +import java.util.Optional; import java.util.Queue; import java.util.Set; import java.util.concurrent.LinkedBlockingQueue; @@ -117,7 +122,7 @@ }) @InputRequirement(InputRequirement.Requirement.INPUT_FORBIDDEN) @SeeAlso({PublishKafka.class}) -public class ConsumeKafka extends AbstractProcessor implements VerifiableProcessor { +public class ConsumeKafka extends AbstractProcessor implements VerifiableProcessor, BacklogReportingProcessor { static final AllowableValue TOPIC_NAME = new AllowableValue("names", "names", "Topic is a full topic name or comma separated list of names"); static final AllowableValue TOPIC_PATTERN = new AllowableValue("pattern", "pattern", "Topic is a regular expression according to the Java Pattern syntax"); @@ -368,6 +373,14 @@ public class ConsumeKafka extends AbstractProcessor implements VerifiableProcess private volatile boolean maxUncommittedSizeConfigured; private volatile long maxUncommittedSize; + /** + * Most recent {@link Instant} at which {@link #getBacklog(ProcessContext)} observed the processor caught up. + * Stamped to {@code Instant.now()} when {@code getBacklog} computes a total lag of zero with + * {@link Backlog.Precision#EXACT} precision. Preserved across lifecycle stop so a stopped Processor still + * surfaces "Last caught up" in the UI. {@code null} when the Processor has never observed a caught up state. + */ + private volatile Instant lastCaughtUpTimestamp; + private final Queue consumerServices = new LinkedBlockingQueue<>(); private final AtomicInteger activeConsumerCount = new AtomicInteger(); @@ -541,6 +554,62 @@ public void onTrigger(final ProcessContext context, final ProcessSession session } } + /** + * Reports backlog by querying committed and end offsets for the Processor's consumer group via the + * connection service's Admin client. The Admin-client path is used regardless of whether any + * consumers are pooled in {@link #consumerServices} or checked out by a running {@code onTrigger} + * task: the broker is the source of truth for the consumer group's committed offsets across all + * assigned partitions, so the reported lag covers the entire group rather than only the partitions + * owned by whichever consumer instances happen to be idle in this Processor at this moment. This + * also avoids any per-partition undercount that would occur if some assigned partitions were owned + * by checked-out consumers and others by pooled consumers. The Admin client does not join the + * consumer group, so reporting backlog does not trigger a rebalance against running consumers. + */ + @Override + public Optional getBacklog(final ProcessContext context) throws BacklogReportingException { + final KafkaConnectionService connectionService = context.getProperty(CONNECTION_SERVICE).asControllerService(KafkaConnectionService.class); + if (connectionService == null) { + // Defensive: the Kafka Connection Service property is required, so the validator + // would normally have rejected the configuration before reaching this code path. + return Optional.empty(); + } + + final PollingContext backlogPollingContext = createPollingContext(context); + try { + final long totalLag = connectionService.getCommittedOffsetLag(backlogPollingContext); + return Optional.of(buildBacklog(totalLag)); + } catch (final InterruptedException e) { + Thread.currentThread().interrupt(); + throw new BacklogReportingException("Interrupted while determining Kafka backlog", e); + } catch (final Exception e) { + throw new BacklogReportingException("Failed to determine Kafka backlog", e); + } + } + + private Backlog buildBacklog(final long total) { + final Backlog.Builder builder = Backlog.builder().precision(Backlog.Precision.EXACT); + // Always populate the records dimension, including the known-zero case. A populated zero is + // a positive statement ("the lag is exactly 0 records") that clients can render as "0"; + // omitting the dimension would force clients to render the value as "unknown" alongside the + // lastCaughtUp timestamp, which is a misleading mixed signal for a caught-up consumer group. + builder.records(total); + if (total == 0L) { + final Instant caughtUpInstant = Instant.now(); + lastCaughtUpTimestamp = caughtUpInstant; + builder.lastCaughtUp(caughtUpInstant); + } else { + final Instant priorCaughtUp = lastCaughtUpTimestamp; + if (priorCaughtUp != null) { + builder.lastCaughtUp(priorCaughtUp); + } + } + + final Backlog backlog = builder.build(); + getLogger().debug("Computed Kafka backlog records [{}] precision [{}] lastCaughtUp [{}]", + backlog.getRecordCount(), backlog.getPrecision(), backlog.getLastCaughtUp()); + return backlog; + } + private void commitOffsets(final KafkaConsumerService consumerService, final OffsetTracker offsetTracker, final PollingContext pollingContext, final ProcessSession session) { try { if (commitOffsets) { diff --git a/nifi-extension-bundles/nifi-kafka-bundle/nifi-kafka-processors/src/test/java/org/apache/nifi/kafka/processors/ConsumeKafkaTest.java b/nifi-extension-bundles/nifi-kafka-bundle/nifi-kafka-processors/src/test/java/org/apache/nifi/kafka/processors/ConsumeKafkaTest.java index 14c50d45babe..92e825164154 100644 --- a/nifi-extension-bundles/nifi-kafka-bundle/nifi-kafka-processors/src/test/java/org/apache/nifi/kafka/processors/ConsumeKafkaTest.java +++ b/nifi-extension-bundles/nifi-kafka-bundle/nifi-kafka-processors/src/test/java/org/apache/nifi/kafka/processors/ConsumeKafkaTest.java @@ -16,6 +16,7 @@ */ package org.apache.nifi.kafka.processors; +import org.apache.nifi.components.Backlog; import org.apache.nifi.components.ConfigVerificationResult; import org.apache.nifi.components.ConfigVerificationResult.Outcome; import org.apache.nifi.kafka.service.api.KafkaConnectionService; @@ -30,8 +31,14 @@ import org.mockito.Mock; import org.mockito.junit.jupiter.MockitoExtension; +import java.lang.reflect.Field; +import java.time.Instant; import java.util.Collections; import java.util.List; +import java.util.Optional; +import java.util.OptionalLong; +import java.util.Queue; +import java.util.concurrent.atomic.AtomicInteger; import static org.apache.nifi.kafka.processors.ConsumeKafka.CONNECTION_SERVICE; import static org.apache.nifi.kafka.processors.ConsumeKafka.GROUP_ID; @@ -39,7 +46,10 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertTrue; import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; @ExtendWith(MockitoExtension.class) @@ -141,6 +151,86 @@ public void testDynamicProperties() throws InitializationException { runner.enableControllerService(kafkaConnectionService); } + @Test + public void testGetBacklogQueriesAdminClientRegardlessOfPoolState() throws Exception { + // getBacklog always queries committed/end offsets via the connection service's Admin + // client. The broker is the source of truth for the consumer group's committed offsets + // across every assigned partition, so the returned lag covers the entire group rather + // than only the partitions owned by whichever consumer instances happen to be pooled or + // checked out at this moment. Per-partition undercounts cannot occur because the consumer + // pool is not consulted on this path. + setConnectionService(); + runner.setProperty(TOPICS, TEST_TOPIC_NAME); + runner.setProperty(GROUP_ID, CONSUMER_GROUP_ID); + when(kafkaConnectionService.getCommittedOffsetLag(any())).thenReturn(42L); + + // Reflectively place one consumer in the pool and mark one additional consumer as checked + // out by a running onTrigger task. The Admin-client answer must be returned even when the + // pool contains a consumer that could be inspected directly. + final Field consumerServicesField = ConsumeKafka.class.getDeclaredField("consumerServices"); + consumerServicesField.setAccessible(true); + @SuppressWarnings("unchecked") + final Queue consumerServices = (Queue) consumerServicesField.get(processor); + consumerServices.offer(kafkaConsumerService); + + final Field activeConsumerCountField = ConsumeKafka.class.getDeclaredField("activeConsumerCount"); + activeConsumerCountField.setAccessible(true); + final AtomicInteger activeConsumerCount = (AtomicInteger) activeConsumerCountField.get(processor); + activeConsumerCount.set(1); + + final Optional backlog = processor.getBacklog(runner.getProcessContext()); + + assertTrue(backlog.isPresent()); + assertEquals(OptionalLong.of(42L), backlog.get().getRecordCount()); + assertEquals(Backlog.Precision.EXACT, backlog.get().getPrecision()); + assertTrue(backlog.get().getLastCaughtUp().isEmpty()); + // The pooled consumer must not be interrogated for partition state or lag; that path is + // bypassed entirely by the Admin-client design. + verify(kafkaConsumerService, never()).getPartitionStates(); + verify(kafkaConsumerService, never()).currentLag(any()); + } + + @Test + public void testGetBacklogPopulatesZeroRecordsOnCaughtUp() throws Exception { + // A caught-up consumer group has an exactly-known lag of zero. The returned Backlog must + // populate the records dimension with that zero alongside lastCaughtUp, so that clients + // can render "0 records" rather than treating the record count as unknown. + setConnectionService(); + runner.setProperty(TOPICS, TEST_TOPIC_NAME); + runner.setProperty(GROUP_ID, CONSUMER_GROUP_ID); + when(kafkaConnectionService.getCommittedOffsetLag(any())).thenReturn(0L); + + final Optional backlog = processor.getBacklog(runner.getProcessContext()); + + assertTrue(backlog.isPresent()); + assertEquals(OptionalLong.of(0L), backlog.get().getRecordCount()); + assertEquals(Backlog.Precision.EXACT, backlog.get().getPrecision()); + assertTrue(backlog.get().getLastCaughtUp().isPresent()); + } + + @Test + public void testOnStoppedPreservesLastCaughtUp() throws Exception { + setConnectionService(); + runner.setProperty(TOPICS, TEST_TOPIC_NAME); + runner.setProperty(GROUP_ID, CONSUMER_GROUP_ID); + when(kafkaConnectionService.getCommittedOffsetLag(any())).thenReturn(0L).thenReturn(5L); + + final Optional initialBacklog = processor.getBacklog(runner.getProcessContext()); + assertTrue(initialBacklog.isPresent()); + final Instant rememberedFirst = initialBacklog.get().getLastCaughtUp().orElse(null); + assertNotNull(rememberedFirst); + + processor.onStopped(); + + final Optional afterStop = processor.getBacklog(runner.getProcessContext()); + assertTrue(afterStop.isPresent()); + // After the processor is stopped while non-zero lag exists, the remembered caught-up + // timestamp from the prior caught-up observation must still be surfaced — onStopped() must + // not clear lastCaughtUpTimestamp. + assertEquals(Optional.of(rememberedFirst), afterStop.get().getLastCaughtUp()); + assertEquals(OptionalLong.of(5L), afterStop.get().getRecordCount()); + } + private void setConnectionService() throws InitializationException { when(kafkaConnectionService.getIdentifier()).thenReturn(SERVICE_ID); diff --git a/nifi-extension-bundles/nifi-kafka-bundle/nifi-kafka-service-api/src/main/java/org/apache/nifi/kafka/service/api/KafkaConnectionService.java b/nifi-extension-bundles/nifi-kafka-bundle/nifi-kafka-service-api/src/main/java/org/apache/nifi/kafka/service/api/KafkaConnectionService.java index e669c008117a..3da3307b4708 100644 --- a/nifi-extension-bundles/nifi-kafka-bundle/nifi-kafka-service-api/src/main/java/org/apache/nifi/kafka/service/api/KafkaConnectionService.java +++ b/nifi-extension-bundles/nifi-kafka-bundle/nifi-kafka-service-api/src/main/java/org/apache/nifi/kafka/service/api/KafkaConnectionService.java @@ -22,6 +22,8 @@ import org.apache.nifi.kafka.service.api.producer.KafkaProducerService; import org.apache.nifi.kafka.service.api.producer.ProducerConfiguration; +import java.util.concurrent.ExecutionException; + public interface KafkaConnectionService extends ControllerService { KafkaConsumerService getConsumerService(PollingContext pollingContext); @@ -29,4 +31,25 @@ public interface KafkaConnectionService extends ControllerService { KafkaProducerService getProducerService(ProducerConfiguration producerConfiguration); String getBrokerUri(); + + /** + * Compute the total committed-offset lag (records remaining) for the consumer group and topics described + * by the supplied {@link PollingContext}. Uses a Kafka Admin client internally and does NOT create a + * consumer or join the consumer group, so calling this method does not trigger a group rebalance and is + * safe to invoke while other live consumers are reading from the same group. + * + *

For every partition of every topic that matches the supplied context, the value contributed to the + * total is {@code endOffset - committedOffset}. When the consumer group has no committed offset for a + * particular partition, the partition contributes its full {@code endOffset - beginningOffset} (i.e. + * every record in the partition is treated as backlog for that group). Partitions are discovered via + * {@code describeTopics} when the polling context lists explicit topic names, or via {@code listTopics} + * filtered by the polling context's topic pattern when a pattern is configured.

+ * + * @param pollingContext describes the topics or topic pattern and the consumer group id to query. + * @return total records remaining for the configured consumer group across every partition of every + * matched topic. A return value of {@code 0} means the group is currently caught up. + * @throws ExecutionException if any Kafka Admin operation fails + * @throws InterruptedException if the calling thread is interrupted while waiting for an Admin response + */ + long getCommittedOffsetLag(PollingContext pollingContext) throws ExecutionException, InterruptedException; } diff --git a/nifi-extension-bundles/nifi-kafka-bundle/nifi-kafka-service-shared/src/main/java/org/apache/nifi/kafka/service/Kafka3ConnectionService.java b/nifi-extension-bundles/nifi-kafka-bundle/nifi-kafka-service-shared/src/main/java/org/apache/nifi/kafka/service/Kafka3ConnectionService.java index 225a21ca0682..ba08b5c69c6f 100644 --- a/nifi-extension-bundles/nifi-kafka-bundle/nifi-kafka-service-shared/src/main/java/org/apache/nifi/kafka/service/Kafka3ConnectionService.java +++ b/nifi-extension-bundles/nifi-kafka-bundle/nifi-kafka-service-shared/src/main/java/org/apache/nifi/kafka/service/Kafka3ConnectionService.java @@ -18,11 +18,17 @@ import org.apache.kafka.clients.admin.Admin; import org.apache.kafka.clients.admin.AdminClientConfig; +import org.apache.kafka.clients.admin.ListConsumerGroupOffsetsSpec; +import org.apache.kafka.clients.admin.ListOffsetsResult; import org.apache.kafka.clients.admin.ListTopicsResult; +import org.apache.kafka.clients.admin.OffsetSpec; +import org.apache.kafka.clients.admin.TopicDescription; import org.apache.kafka.clients.consumer.Consumer; import org.apache.kafka.clients.consumer.ConsumerConfig; import org.apache.kafka.clients.consumer.KafkaConsumer; +import org.apache.kafka.clients.consumer.OffsetAndMetadata; import org.apache.kafka.clients.producer.ProducerConfig; +import org.apache.kafka.common.TopicPartition; import org.apache.kafka.common.config.SslConfigs; import org.apache.kafka.common.security.auth.SecurityProtocol; import org.apache.kafka.common.serialization.ByteArrayDeserializer; @@ -67,6 +73,9 @@ import java.time.Duration; import java.util.ArrayList; +import java.util.Collection; +import java.util.HashMap; +import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Objects; @@ -174,6 +183,19 @@ public class Kafka3ConnectionService extends AbstractControllerService implement private static final KafkaConnectionVerifier kafkaConnectionVerifier = new KafkaConnectionVerifier(); + /** + * Bound applied to the Admin client used only for backlog determination, in place of the + * potentially much longer Client Timeout configured for regular produce/consume operations. + */ + private static final int BACKLOG_ADMIN_TIMEOUT_MS = 10_000; + + /** + * Delay between reconnect and retry attempts for the Admin client used only for backlog determination, + * so that a broker outage does not cause a burst of retries within the bounded BACKLOG_ADMIN_TIMEOUT_MS + * window. + */ + private static final int BACKLOG_ADMIN_BACKOFF_MS = 2_000; + private volatile ServiceConfiguration serviceConfiguration; private volatile Properties producerProperties; private volatile Properties consumerProperties; @@ -321,6 +343,117 @@ public List listTopicNames(final ConfigurationContext context) throws Ex } } + @Override + public long getCommittedOffsetLag(final PollingContext pollingContext) throws ExecutionException, InterruptedException { + Objects.requireNonNull(pollingContext, "Polling Context required"); + final String groupId = pollingContext.getGroupId(); + Objects.requireNonNull(groupId, "Group Id required to compute committed-offset lag"); + + try (final Admin admin = Admin.create(buildAdminProperties())) { + final Set topicNames = resolveTopicNames(admin, pollingContext); + if (topicNames.isEmpty()) { + return 0L; + } + + final Map topicDescriptions = admin.describeTopics(topicNames).allTopicNames().get(); + final Set partitions = new HashSet<>(); + for (final TopicDescription topicDescription : topicDescriptions.values()) { + topicDescription.partitions().forEach(partitionInfo -> + partitions.add(new TopicPartition(topicDescription.name(), partitionInfo.partition()))); + } + if (partitions.isEmpty()) { + return 0L; + } + + final Map endOffsetRequest = new HashMap<>(partitions.size()); + final Map beginningOffsetRequest = new HashMap<>(partitions.size()); + for (final TopicPartition partition : partitions) { + endOffsetRequest.put(partition, OffsetSpec.latest()); + beginningOffsetRequest.put(partition, OffsetSpec.earliest()); + } + final Map endOffsets = admin.listOffsets(endOffsetRequest).all().get(); + final Map beginningOffsets = admin.listOffsets(beginningOffsetRequest).all().get(); + + final Map groupRequest = + Map.of(groupId, new ListConsumerGroupOffsetsSpec().topicPartitions(partitions)); + final Map committedOffsets = + admin.listConsumerGroupOffsets(groupRequest).partitionsToOffsetAndMetadata(groupId).get(); + + long totalLag = 0L; + for (final TopicPartition partition : partitions) { + final ListOffsetsResult.ListOffsetsResultInfo endInfo = endOffsets.get(partition); + if (endInfo == null) { + continue; + } + final long endOffset = endInfo.offset(); + final OffsetAndMetadata committed = committedOffsets.get(partition); + final long lagForPartition; + if (committed == null) { + final ListOffsetsResult.ListOffsetsResultInfo beginningInfo = beginningOffsets.get(partition); + final long beginningOffset = beginningInfo == null ? 0L : beginningInfo.offset(); + lagForPartition = Math.max(0L, endOffset - beginningOffset); + } else { + lagForPartition = Math.max(0L, endOffset - committed.offset()); + } + totalLag = Math.addExact(totalLag, lagForPartition); + } + return totalLag; + } + } + + private Properties buildAdminProperties() { + final Properties adminProperties = new Properties(); + adminProperties.putAll(consumerProperties); + // Defensive/precautionary: the shared consumerProperties built by getConsumerProperties does not + // contain these consumer-only keys (getConsumerService applies group id, auto-offset-reset, and + // auto-commit to its own per-consumer Properties copy, not to this shared field), so in the current + // code path these removals are a no-op. They are kept so the Admin client can never inherit those + // consumer-only settings and log warnings about unknown configs should the shared properties ever change. + adminProperties.remove(ConsumerConfig.GROUP_ID_CONFIG); + adminProperties.remove(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG); + adminProperties.remove(ConsumerConfig.ENABLE_AUTO_COMMIT_CONFIG); + + // This Admin client is used exclusively to determine backlog, which is a best-effort, UI-facing + // check rather than part of the data plane. consumerProperties carries the Client Timeout + // configured for regular produce/consume operations, which may be tens of seconds or more. + // Overriding it here with a short, fixed bound keeps an unreachable or slow broker from stalling + // a backlog check for that entire duration, and limits how long the Admin client's background + // thread keeps retrying before this short-lived client is closed. + adminProperties.put(AdminClientConfig.DEFAULT_API_TIMEOUT_MS_CONFIG, BACKLOG_ADMIN_TIMEOUT_MS); + adminProperties.put(AdminClientConfig.REQUEST_TIMEOUT_MS_CONFIG, BACKLOG_ADMIN_TIMEOUT_MS / 2); + + // When no broker in the bootstrap list can be reached, the Admin client repeatedly rebootstraps + // and retries fetching metadata with minimal delay between attempts, which floods the log within + // the short window before BACKLOG_ADMIN_TIMEOUT_MS elapses. Widening the reconnect and retry + // backoff spaces out those attempts so a single failed backlog check produces a handful of log + // lines instead of a burst of them. + adminProperties.put(AdminClientConfig.RECONNECT_BACKOFF_MS_CONFIG, BACKLOG_ADMIN_BACKOFF_MS); + adminProperties.put(AdminClientConfig.RECONNECT_BACKOFF_MAX_MS_CONFIG, BACKLOG_ADMIN_BACKOFF_MS); + adminProperties.put(AdminClientConfig.RETRY_BACKOFF_MS_CONFIG, BACKLOG_ADMIN_BACKOFF_MS); + return adminProperties; + } + + private Set resolveTopicNames(final Admin admin, final PollingContext pollingContext) throws ExecutionException, InterruptedException { + final Optional topicPatternFound = pollingContext.getTopicPattern(); + if (topicPatternFound.isEmpty()) { + final Collection configuredTopics = pollingContext.getTopics(); + if (configuredTopics == null || configuredTopics.isEmpty()) { + return Set.of(); + } + return new HashSet<>(configuredTopics); + } + + final Pattern topicPattern = topicPatternFound.get(); + final Set allTopicNames = admin.listTopics().names().get(); + final Set matchingTopicNames = new HashSet<>(); + for (final String topicName : allTopicNames) { + if (topicPattern.matcher(topicName).matches()) { + matchingTopicNames.add(topicName); + } + } + return matchingTopicNames; + } + @Override public List verify(final ConfigurationContext configurationContext, final ComponentLog verificationLogger, final Map variables) { diff --git a/nifi-framework-api/src/main/java/org/apache/nifi/controller/queue/FlowFileQueue.java b/nifi-framework-api/src/main/java/org/apache/nifi/controller/queue/FlowFileQueue.java index 28b01de0cd20..d6cd4811539b 100644 --- a/nifi-framework-api/src/main/java/org/apache/nifi/controller/queue/FlowFileQueue.java +++ b/nifi-framework-api/src/main/java/org/apache/nifi/controller/queue/FlowFileQueue.java @@ -97,6 +97,20 @@ public interface FlowFileQueue { QueueSize size(); + /** + * Returns an atomic, point-in-time view of this queue's total {@link QueueSize} and the + * FlowFiles currently held in this node's active in-memory queue. Implementations freeze every + * partition that contributes to the {@link QueueSize} for the duration of the call, so the + * active list and the {@link QueueSize} are mutually consistent. When + * {@code snapshot.activeFlowFiles().size()} equals {@code snapshot.queueSize().getObjectCount()}, + * the active list is exhaustive. Any difference is accounted for by FlowFiles that are swapped to + * disk, assigned to remote partitions (destined for other cluster nodes), or in the rebalancing + * partition (being redistributed). + * + * @return a non-null snapshot of the queue's size and active in-memory FlowFiles + */ + FlowFileQueueSnapshot getQueueSnapshot(); + /** * @param fromTimestamp The timestamp in milliseconds from which to calculate durations. This will typically be the current timestamp. * @return the sum in milliseconds of how long all FlowFiles within this queue have currently been in this queue. diff --git a/nifi-framework-api/src/main/java/org/apache/nifi/controller/queue/FlowFileQueueSnapshot.java b/nifi-framework-api/src/main/java/org/apache/nifi/controller/queue/FlowFileQueueSnapshot.java new file mode 100644 index 000000000000..70b91d4df78d --- /dev/null +++ b/nifi-framework-api/src/main/java/org/apache/nifi/controller/queue/FlowFileQueueSnapshot.java @@ -0,0 +1,54 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.controller.queue; + +import org.apache.nifi.controller.repository.FlowFileRecord; + +import java.util.List; +import java.util.Objects; + +/** + * An atomic, point-in-time view of a {@link FlowFileQueue}'s state, returned by + * {@link FlowFileQueue#getQueueSnapshot()}. Bundles the queue's total {@link QueueSize} with the + * {@link FlowFileRecord}s held in this node's active in-memory queue at the moment the snapshot was + * taken. Both fields are captured while every partition that contributes to the {@link QueueSize} is + * locked, so callers can correlate them without race conditions: when {@code activeFlowFiles().size()} + * equals {@code queueSize().getObjectCount()}, the active list is exhaustive. + * + *

+ * The {@link #activeFlowFiles()} list is the in-memory active subset in poll order. It excludes + * FlowFiles that are swapped to disk or pending swap-out, held by the destination, or — for + * load-balanced connections — assigned to remote partitions (destined for other cluster nodes) or + * in the rebalancing partition (being redistributed). All of those FlowFiles are still counted in + * {@link #queueSize()}. The {@code activeFlowFiles} list is defensively copied into an immutable + * list, so callers may safely retain the snapshot beyond the original lock acquisition. + *

+ * + * @param queueSize the total {@link QueueSize} of the queue at the time the snapshot was taken, + * including any FlowFiles excluded from {@link #activeFlowFiles()} + * @param activeFlowFiles the active in-memory FlowFiles in poll order — the order in which the queue's + * configured {@link org.apache.nifi.flowfile.FlowFilePrioritizer}s would surface + * them to a consumer — never {@code null}; may be empty + */ +public record FlowFileQueueSnapshot(QueueSize queueSize, List activeFlowFiles) { + + public FlowFileQueueSnapshot { + Objects.requireNonNull(queueSize, "queueSize"); + Objects.requireNonNull(activeFlowFiles, "activeFlowFiles"); + activeFlowFiles = List.copyOf(activeFlowFiles); + } +} diff --git a/nifi-framework-bundle/nifi-framework/nifi-client-dto/src/main/java/org/apache/nifi/web/api/dto/BacklogDTO.java b/nifi-framework-bundle/nifi-framework/nifi-client-dto/src/main/java/org/apache/nifi/web/api/dto/BacklogDTO.java new file mode 100644 index 000000000000..54ef2c86663f --- /dev/null +++ b/nifi-framework-bundle/nifi-framework/nifi-client-dto/src/main/java/org/apache/nifi/web/api/dto/BacklogDTO.java @@ -0,0 +1,123 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.web.api.dto; + +import io.swagger.v3.oas.annotations.media.Schema; +import jakarta.xml.bind.annotation.XmlType; + +/** + * Snapshot of how much data remains on the source system for a Processor or Connector. Each numeric + * dimension is reported only when the component can determine it; null fields are omitted from the + * JSON representation rather than serialized as explicit nulls, in keeping with the framework-wide + * null-omission policy applied to REST entities. + */ +@XmlType(name = "backlog") +public class BacklogDTO { + + private Long flowFileCount; + private String formattedFlowFileCount; + private Long byteCount; + private String formattedByteCount; + private Long recordCount; + private String formattedRecordCount; + private String lastCaughtUp; + private String formattedLastCaughtUp; + private String precision; + + @Schema(description = "The number of FlowFiles remaining on the source, if the component can determine it.") + public Long getFlowFileCount() { + return flowFileCount; + } + + public void setFlowFileCount(final Long flowFileCount) { + this.flowFileCount = flowFileCount; + } + + @Schema(description = "The FlowFile count formatted for display with locale-appropriate grouping (for example, '1,025'). Populated whenever flowFileCount is populated.") + public String getFormattedFlowFileCount() { + return formattedFlowFileCount; + } + + public void setFormattedFlowFileCount(final String formattedFlowFileCount) { + this.formattedFlowFileCount = formattedFlowFileCount; + } + + @Schema(description = "The total number of bytes remaining on the source, if the component can determine it.") + public Long getByteCount() { + return byteCount; + } + + public void setByteCount(final Long byteCount) { + this.byteCount = byteCount; + } + + @Schema(description = "The byte count formatted for display in human-readable units (for example, '4.89 GB'). Populated whenever byteCount is populated.") + public String getFormattedByteCount() { + return formattedByteCount; + } + + public void setFormattedByteCount(final String formattedByteCount) { + this.formattedByteCount = formattedByteCount; + } + + @Schema(description = "The number of records remaining on the source, if the component can determine it.") + public Long getRecordCount() { + return recordCount; + } + + public void setRecordCount(final Long recordCount) { + this.recordCount = recordCount; + } + + @Schema(description = "The record count formatted for display with locale-appropriate grouping (for example, '1,025'). Populated whenever recordCount is populated.") + public String getFormattedRecordCount() { + return formattedRecordCount; + } + + public void setFormattedRecordCount(final String formattedRecordCount) { + this.formattedRecordCount = formattedRecordCount; + } + + @Schema(description = "The most recent moment the component observed itself as fully caught up with the source, formatted as an ISO-8601 UTC timestamp.") + public String getLastCaughtUp() { + return lastCaughtUp; + } + + public void setLastCaughtUp(final String lastCaughtUp) { + this.lastCaughtUp = lastCaughtUp; + } + + @Schema(description = "Human-readable description of how long ago the component was last caught up, " + + "for example '5 mins ago', 'in 2 hours', or 'now'. Populated whenever lastCaughtUp is populated.") + public String getFormattedLastCaughtUp() { + return formattedLastCaughtUp; + } + + public void setFormattedLastCaughtUp(final String formattedLastCaughtUp) { + this.formattedLastCaughtUp = formattedLastCaughtUp; + } + + @Schema(description = "The precision of the numeric dimensions of this backlog.", + allowableValues = {"EXACT", "AT_LEAST"}) + public String getPrecision() { + return precision; + } + + public void setPrecision(final String precision) { + this.precision = precision; + } +} diff --git a/nifi-framework-bundle/nifi-framework/nifi-client-dto/src/main/java/org/apache/nifi/web/api/dto/BacklogRequestDTO.java b/nifi-framework-bundle/nifi-framework/nifi-client-dto/src/main/java/org/apache/nifi/web/api/dto/BacklogRequestDTO.java new file mode 100644 index 000000000000..baa9f3e29370 --- /dev/null +++ b/nifi-framework-bundle/nifi-framework/nifi-client-dto/src/main/java/org/apache/nifi/web/api/dto/BacklogRequestDTO.java @@ -0,0 +1,55 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.web.api.dto; + +import io.swagger.v3.oas.annotations.media.Schema; +import jakarta.xml.bind.annotation.XmlType; + +/** + * A request to asynchronously determine the backlog of a Processor or Connector. Determining backlog typically + * requires issuing a request to an external source system, which may take an indeterminate amount of time or + * may never complete if the source system is unavailable. Rather than blocking the HTTP request thread for that + * duration, the framework creates a {@code BacklogRequestDTO}, performs the determination in the background, and + * expects the client to poll {@code GET} on the request's {@code uri} until {@code complete} is {@code true}, and + * then to {@code DELETE} the request. Deleting a request that has not yet completed cancels the background + * determination. + */ +@XmlType(name = "backlogRequest") +public class BacklogRequestDTO extends AsynchronousRequestDTO { + + private String componentId; + private BacklogDTO backlog; + + @Schema(description = "The ID of the Processor or Connector whose backlog is being determined") + public String getComponentId() { + return componentId; + } + + public void setComponentId(final String componentId) { + this.componentId = componentId; + } + + @Schema(description = "The backlog reported by the component, populated once the request has completed successfully.", + accessMode = Schema.AccessMode.READ_ONLY) + public BacklogDTO getBacklog() { + return backlog; + } + + public void setBacklog(final BacklogDTO backlog) { + this.backlog = backlog; + } +} diff --git a/nifi-framework-bundle/nifi-framework/nifi-client-dto/src/main/java/org/apache/nifi/web/api/dto/BacklogUpdateStepDTO.java b/nifi-framework-bundle/nifi-framework/nifi-client-dto/src/main/java/org/apache/nifi/web/api/dto/BacklogUpdateStepDTO.java new file mode 100644 index 000000000000..e3ea69ef6600 --- /dev/null +++ b/nifi-framework-bundle/nifi-framework/nifi-client-dto/src/main/java/org/apache/nifi/web/api/dto/BacklogUpdateStepDTO.java @@ -0,0 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.web.api.dto; + +import jakarta.xml.bind.annotation.XmlType; + +@XmlType(name = "backlogUpdateStep") +public class BacklogUpdateStepDTO extends UpdateStepDTO { +} diff --git a/nifi-framework-bundle/nifi-framework/nifi-client-dto/src/main/java/org/apache/nifi/web/api/dto/ProcessorDTO.java b/nifi-framework-bundle/nifi-framework/nifi-client-dto/src/main/java/org/apache/nifi/web/api/dto/ProcessorDTO.java index 284a82489b88..475a7818720b 100644 --- a/nifi-framework-bundle/nifi-framework/nifi-client-dto/src/main/java/org/apache/nifi/web/api/dto/ProcessorDTO.java +++ b/nifi-framework-bundle/nifi-framework/nifi-client-dto/src/main/java/org/apache/nifi/web/api/dto/ProcessorDTO.java @@ -42,6 +42,7 @@ public class ProcessorDTO extends ComponentDTO { private Boolean supportsParallelProcessing; private Boolean supportsBatching; private Boolean supportsSensitiveDynamicProperties; + private Boolean supportsBacklogReporting; private Boolean persistsState; private Boolean restricted; private Boolean deprecated; @@ -154,6 +155,18 @@ public void setSupportsSensitiveDynamicProperties(final Boolean supportsSensitiv this.supportsSensitiveDynamicProperties = supportsSensitiveDynamicProperties; } + /** + * @return whether the processor implements {@code BacklogReportingProcessor} and can report a backlog + */ + @Schema(description = "Whether the processor implements BacklogReportingProcessor and can report a backlog.") + public Boolean getSupportsBacklogReporting() { + return supportsBacklogReporting; + } + + public void setSupportsBacklogReporting(final Boolean supportsBacklogReporting) { + this.supportsBacklogReporting = supportsBacklogReporting; + } + /** * @return whether this processor persists state */ diff --git a/nifi-framework-bundle/nifi-framework/nifi-client-dto/src/main/java/org/apache/nifi/web/api/entity/BacklogEntity.java b/nifi-framework-bundle/nifi-framework/nifi-client-dto/src/main/java/org/apache/nifi/web/api/entity/BacklogEntity.java new file mode 100644 index 000000000000..de383f5847d3 --- /dev/null +++ b/nifi-framework-bundle/nifi-framework/nifi-client-dto/src/main/java/org/apache/nifi/web/api/entity/BacklogEntity.java @@ -0,0 +1,54 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.web.api.entity; + +import io.swagger.v3.oas.annotations.media.Schema; +import jakarta.xml.bind.annotation.XmlRootElement; +import org.apache.nifi.web.api.dto.BacklogDTO; + +/** + *

+ * Response entity wrapping a single {@link BacklogDTO}. The {@code backlog} field is {@code null} + * only when the underlying component supports backlog reporting but currently has no value to + * report (for example, the component opted out of reporting at this moment, or its source has + * not yet been polled). + *

+ * + *

+ * A component that does not support backlog reporting at all is rejected before this entity is + * produced: the framework checks support up front — Processors must implement + * {@code BacklogReportingProcessor} and Connectors must implement + * {@code BacklogReportingConnector} — and returns HTTP {@code 409 Conflict} when the component + * does not implement the corresponding capability interface. Clients can therefore treat a + * 2xx response with {@code backlog == null} as "supported but no value right now" and a 409 + * as "this component does not support backlog reporting". + *

+ */ +@XmlRootElement(name = "backlogEntity") +public class BacklogEntity extends Entity { + + private BacklogDTO backlog; + + @Schema(description = "The backlog reported by the component, or null if the component has no value to report at this time.") + public BacklogDTO getBacklog() { + return backlog; + } + + public void setBacklog(final BacklogDTO backlog) { + this.backlog = backlog; + } +} diff --git a/nifi-framework-bundle/nifi-framework/nifi-client-dto/src/main/java/org/apache/nifi/web/api/entity/BacklogRequestEntity.java b/nifi-framework-bundle/nifi-framework/nifi-client-dto/src/main/java/org/apache/nifi/web/api/entity/BacklogRequestEntity.java new file mode 100644 index 000000000000..b2241df2b79c --- /dev/null +++ b/nifi-framework-bundle/nifi-framework/nifi-client-dto/src/main/java/org/apache/nifi/web/api/entity/BacklogRequestEntity.java @@ -0,0 +1,36 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.web.api.entity; + +import io.swagger.v3.oas.annotations.media.Schema; +import jakarta.xml.bind.annotation.XmlRootElement; +import org.apache.nifi.web.api.dto.BacklogRequestDTO; + +@XmlRootElement(name = "backlogRequestEntity") +public class BacklogRequestEntity extends Entity { + + private BacklogRequestDTO request; + + @Schema(description = "The request") + public BacklogRequestDTO getRequest() { + return request; + } + + public void setRequest(final BacklogRequestDTO request) { + this.request = request; + } +} diff --git a/nifi-framework-bundle/nifi-framework/nifi-framework-cluster/src/main/java/org/apache/nifi/cluster/coordination/http/StandardHttpResponseMapper.java b/nifi-framework-bundle/nifi-framework/nifi-framework-cluster/src/main/java/org/apache/nifi/cluster/coordination/http/StandardHttpResponseMapper.java index 6c2b43a3239b..3ab00338f7a8 100644 --- a/nifi-framework-bundle/nifi-framework/nifi-framework-cluster/src/main/java/org/apache/nifi/cluster/coordination/http/StandardHttpResponseMapper.java +++ b/nifi-framework-bundle/nifi-framework/nifi-framework-cluster/src/main/java/org/apache/nifi/cluster/coordination/http/StandardHttpResponseMapper.java @@ -19,6 +19,7 @@ import jakarta.ws.rs.core.StreamingOutput; import org.apache.nifi.cluster.coordination.http.endpoints.AccessPolicyEndpointMerger; import org.apache.nifi.cluster.coordination.http.endpoints.AssetsEndpointMerger; +import org.apache.nifi.cluster.coordination.http.endpoints.BacklogRequestEndpointMerger; import org.apache.nifi.cluster.coordination.http.endpoints.BulletinBoardEndpointMerger; import org.apache.nifi.cluster.coordination.http.endpoints.ClearBulletinsEndpointMerger; import org.apache.nifi.cluster.coordination.http.endpoints.ClearBulletinsForGroupEndpointMerger; @@ -206,6 +207,7 @@ public StandardHttpResponseMapper(final NiFiProperties nifiProperties) { endpointMergers.add(new AccessPolicyEndpointMerger()); endpointMergers.add(new SearchUsersEndpointMerger()); endpointMergers.add(new ProcessorDiagnosticsEndpointMerger(snapshotMillis)); + endpointMergers.add(new BacklogRequestEndpointMerger()); endpointMergers.add(new ParameterContextValidationMerger()); endpointMergers.add(new ParameterContextsEndpointMerger()); endpointMergers.add(new ParameterContextEndpointMerger()); diff --git a/nifi-framework-bundle/nifi-framework/nifi-framework-cluster/src/main/java/org/apache/nifi/cluster/coordination/http/endpoints/BacklogRequestEndpointMerger.java b/nifi-framework-bundle/nifi-framework/nifi-framework-cluster/src/main/java/org/apache/nifi/cluster/coordination/http/endpoints/BacklogRequestEndpointMerger.java new file mode 100644 index 000000000000..19d8631fe8a3 --- /dev/null +++ b/nifi-framework-bundle/nifi-framework/nifi-framework-cluster/src/main/java/org/apache/nifi/cluster/coordination/http/endpoints/BacklogRequestEndpointMerger.java @@ -0,0 +1,308 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.cluster.coordination.http.endpoints; + +import org.apache.nifi.cluster.manager.NodeResponse; +import org.apache.nifi.cluster.protocol.NodeIdentifier; +import org.apache.nifi.util.FormatUtils; +import org.apache.nifi.web.api.dto.BacklogDTO; +import org.apache.nifi.web.api.dto.BacklogRequestDTO; +import org.apache.nifi.web.api.entity.BacklogEntity; +import org.apache.nifi.web.api.entity.BacklogRequestEntity; + +import java.net.URI; +import java.time.Instant; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.function.Function; +import java.util.regex.Pattern; +import java.util.stream.Collectors; + +/** + * Aggregates {@link BacklogRequestEntity} responses across cluster nodes for the asynchronous backlog request + * endpoints — {@code POST}/{@code GET}/{@code DELETE} on {@code /nifi-api/processors/{uuid}/backlog-requests} and + * {@code /nifi-api/connectors/{uuid}/backlog-requests}, each with an optional trailing {@code /{requestId}} — + * each node determines its own portion of the backlog independently in the background, keyed by the same + * request identifier across the cluster. + * + *

Aggregation rules:

+ *
    + *
  • If any node's request failed, the merged request is marked failed and complete, with a failure reason + * that lists the distinct per-node failure reasons. The merged {@code backlog} is left {@code null} because + * the cluster cannot report a trustworthy value when any node was unable to determine its own.
    • + *
  • Otherwise, the merged request is complete only once every node's request is complete, and the merged + * percent complete is the minimum percent complete across nodes, since the request as a whole cannot be + * further along than its slowest node.
    • + *
  • Once every node is complete without failure, the merged {@code backlog} is computed by + * {@link #mergeEntities(BacklogEntity, List)} using the cluster-wide aggregation rules described there.
    • + *
+ */ +public class BacklogRequestEndpointMerger extends AbstractSingleEntityEndpoint { + + public static final Pattern PROCESSOR_BACKLOG_REQUEST_URI_PATTERN = + Pattern.compile("/nifi-api/processors/[a-f0-9\\-]{36}/backlog-requests(/[a-f0-9\\-]{36})?"); + public static final Pattern CONNECTOR_BACKLOG_REQUEST_URI_PATTERN = + Pattern.compile("/nifi-api/connectors/[a-f0-9\\-]{36}/backlog-requests(/[a-f0-9\\-]{36})?"); + + public static final String PRECISION_EXACT = "EXACT"; + public static final String PRECISION_AT_LEAST = "AT_LEAST"; + + private static final long NOW_WINDOW_MILLISECONDS = 3000L; + + @Override + protected Class getEntityClass() { + return BacklogRequestEntity.class; + } + + @Override + public boolean canHandle(final URI uri, final String method) { + final String path = uri.getPath(); + return PROCESSOR_BACKLOG_REQUEST_URI_PATTERN.matcher(path).matches() + || CONNECTOR_BACKLOG_REQUEST_URI_PATTERN.matcher(path).matches(); + } + + @Override + protected void mergeResponses(final BacklogRequestEntity clientEntity, final Map entityMap, + final Set successfulResponses, final Set problematicResponses) { + + final BacklogRequestDTO clientDto = clientEntity.getRequest(); + final List nodeDtos = entityMap.values().stream() + .map(BacklogRequestEntity::getRequest) + .collect(Collectors.toList()); + + final List failureReasons = nodeDtos.stream() + .map(BacklogRequestDTO::getFailureReason) + .filter(reason -> reason != null) + .distinct() + .collect(Collectors.toList()); + + if (!failureReasons.isEmpty()) { + clientDto.setComplete(true); + clientDto.setPercentCompleted(100); + clientDto.setFailureReason(String.join("; ", failureReasons)); + clientDto.setState("Failed: " + String.join("; ", failureReasons)); + clientDto.setBacklog(null); + return; + } + + final boolean allComplete = nodeDtos.stream().allMatch(BacklogRequestDTO::isComplete); + final int minPercentCompleted = nodeDtos.stream().mapToInt(BacklogRequestDTO::getPercentCompleted).min().orElse(0); + + clientDto.setComplete(allComplete); + clientDto.setPercentCompleted(allComplete ? 100 : minPercentCompleted); + + if (!allComplete) { + clientDto.setState("In Progress"); + clientDto.setBacklog(null); + return; + } + + clientDto.setState("Complete"); + + final List backlogEntities = new ArrayList<>(nodeDtos.size()); + for (final BacklogRequestDTO nodeDto : nodeDtos) { + final BacklogEntity backlogEntity = new BacklogEntity(); + backlogEntity.setBacklog(nodeDto.getBacklog()); + backlogEntities.add(backlogEntity); + } + + final BacklogEntity mergedBacklogEntity = new BacklogEntity(); + mergeEntities(mergedBacklogEntity, backlogEntities); + clientDto.setBacklog(mergedBacklogEntity.getBacklog()); + } + + /** + * Mutates {@code clientEntity} in-place so that its {@code backlog} field reflects the cluster-wide + * aggregation across the provided entities. Visible for testing. + * + *

Aggregation rules:

+ *
    + *
  • + * Numeric dimensions ({@code flowFileCount}, {@code byteCount}, {@code recordCount}): treat a + * {@code null} per-node value as zero for summation. Emit the sum unless every reporting node + * had {@code null} for that dimension, in which case emit {@code null} on the merged DTO so + * the JSON output continues to omit the dimension. + *
    • + *
  • + * {@code precision}: emit {@code EXACT} only if every reporting node reported {@code EXACT}, + * every reporting node reported the same set of populated numeric dimensions, and no + * node returned {@code backlog == null}. If any node returned {@code backlog == null} the + * cluster total is by definition a lower bound — the unknown node could be holding additional + * work — so emit {@code AT_LEAST}. + *
    • + *
  • + * {@code lastCaughtUp}: emit the minimum (earliest) ISO-8601 timestamp across reporting nodes. + * Emit {@code null} if any reporting node returned {@code null} for {@code lastCaughtUp}, + * or if any node returned {@code backlog == null}. The cluster cannot claim it is + * caught up if any node has not reported. + *
    • + *
  • + * A per-node response whose {@code backlog} property is {@code null} counts as "no value to + * merge" for that node. If every node returns {@code backlog == null}, the merged entity is + * also {@code backlog == null}. Otherwise the {@code null}-backlog nodes are excluded from + * summation, but their absence forces the merged precision to {@code AT_LEAST} and clears + * the merged {@code lastCaughtUp} as described above. + *
    • + *
+ * + * @param clientEntity the entity returned to the caller; its {@code backlog} is replaced + * @param entities every per-node entity contributing to the merge, including the client entity + */ + static void mergeEntities(final BacklogEntity clientEntity, final List entities) { + final List reportingNodeDtos = new ArrayList<>(entities.size()); + boolean anyNodeMissingBacklog = false; + for (final BacklogEntity entity : entities) { + if (entity == null) { + // A null entity is treated as a non-reporting node for the same reason a null backlog is: + // the cluster has no value from that node and therefore cannot claim completeness. + anyNodeMissingBacklog = true; + continue; + } + + final BacklogDTO dto = entity.getBacklog(); + if (dto == null) { + anyNodeMissingBacklog = true; + } else { + reportingNodeDtos.add(dto); + } + } + + if (reportingNodeDtos.isEmpty()) { + clientEntity.setBacklog(null); + return; + } + + final BacklogDTO merged = new BacklogDTO(); + final Long mergedFlowFileCount = sumDimension(reportingNodeDtos, BacklogDTO::getFlowFileCount); + final Long mergedByteCount = sumDimension(reportingNodeDtos, BacklogDTO::getByteCount); + final Long mergedRecordCount = sumDimension(reportingNodeDtos, BacklogDTO::getRecordCount); + + merged.setFlowFileCount(mergedFlowFileCount); + if (mergedFlowFileCount != null) { + merged.setFormattedFlowFileCount(FormatUtils.formatCount(mergedFlowFileCount)); + } + + merged.setByteCount(mergedByteCount); + if (mergedByteCount != null) { + merged.setFormattedByteCount(FormatUtils.formatDataSize(mergedByteCount)); + } + + merged.setRecordCount(mergedRecordCount); + if (mergedRecordCount != null) { + merged.setFormattedRecordCount(FormatUtils.formatCount(mergedRecordCount)); + } + + // If any node was unable to report, the cluster cannot claim "caught up" — that claim + // requires every node to have observed itself caught up. + final String mergedLastCaughtUp = anyNodeMissingBacklog ? null : minLastCaughtUp(reportingNodeDtos); + merged.setLastCaughtUp(mergedLastCaughtUp); + if (mergedLastCaughtUp != null) { + merged.setFormattedLastCaughtUp(computeFormattedLastCaughtUp(mergedLastCaughtUp, merged)); + } + + merged.setPrecision(mergePrecision(reportingNodeDtos, anyNodeMissingBacklog)); + + clientEntity.setBacklog(merged); + } + + private static String computeFormattedLastCaughtUp(final String mergedLastCaughtUp, final BacklogDTO merged) { + final Instant lastCaughtUp = Instant.parse(mergedLastCaughtUp); + final Instant now = Instant.now(); + if (isNumericallyCaughtUp(merged) + && Math.abs(now.toEpochMilli() - lastCaughtUp.toEpochMilli()) <= NOW_WINDOW_MILLISECONDS) { + return "now"; + } + + return FormatUtils.formatRelativeTime(lastCaughtUp, now); + } + + private static boolean isNumericallyCaughtUp(final BacklogDTO dto) { + return isZeroOrNull(dto.getFlowFileCount()) + && isZeroOrNull(dto.getByteCount()) + && isZeroOrNull(dto.getRecordCount()); + } + + private static boolean isZeroOrNull(final Long value) { + return value == null || value.longValue() == 0L; + } + + private static Long sumDimension(final List dtos, final Function accessor) { + long sum = 0L; + boolean anyPresent = false; + for (final BacklogDTO dto : dtos) { + final Long value = accessor.apply(dto); + if (value != null) { + sum += value; + anyPresent = true; + } + } + + return anyPresent ? sum : null; + } + + /** + * Returns the earliest {@code lastCaughtUp} timestamp across the supplied DTOs, formatted as + * ISO-8601. If any DTO returns {@code null}, returns {@code null} so the merged DTO continues + * to omit the field — the cluster cannot claim it is caught up if any node has not reported. + */ + private static String minLastCaughtUp(final List dtos) { + Instant earliest = null; + for (final BacklogDTO dto : dtos) { + final String candidate = dto.getLastCaughtUp(); + if (candidate == null) { + return null; + } + + final Instant candidateInstant = Instant.parse(candidate); + if (earliest == null || candidateInstant.isBefore(earliest)) { + earliest = candidateInstant; + } + } + + return earliest == null ? null : earliest.toString(); + } + + private static String mergePrecision(final List dtos, final boolean anyNodeMissingBacklog) { + if (anyNodeMissingBacklog) { + return PRECISION_AT_LEAST; + } + + boolean allExact = true; + boolean shapesAgree = true; + + final BacklogDTO firstDto = dtos.getFirst(); + final boolean firstFlow = firstDto.getFlowFileCount() != null; + final boolean firstBytes = firstDto.getByteCount() != null; + final boolean firstRecords = firstDto.getRecordCount() != null; + + for (final BacklogDTO dto : dtos) { + if (!PRECISION_EXACT.equals(dto.getPrecision())) { + allExact = false; + } + + if ((dto.getFlowFileCount() != null) != firstFlow + || (dto.getByteCount() != null) != firstBytes + || (dto.getRecordCount() != null) != firstRecords) { + shapesAgree = false; + } + } + + return (allExact && shapesAgree) ? PRECISION_EXACT : PRECISION_AT_LEAST; + } +} diff --git a/nifi-framework-bundle/nifi-framework/nifi-framework-cluster/src/test/java/org/apache/nifi/cluster/coordination/http/endpoints/BacklogRequestEndpointMergerTest.java b/nifi-framework-bundle/nifi-framework/nifi-framework-cluster/src/test/java/org/apache/nifi/cluster/coordination/http/endpoints/BacklogRequestEndpointMergerTest.java new file mode 100644 index 000000000000..d88b8e35a6b7 --- /dev/null +++ b/nifi-framework-bundle/nifi-framework/nifi-framework-cluster/src/test/java/org/apache/nifi/cluster/coordination/http/endpoints/BacklogRequestEndpointMergerTest.java @@ -0,0 +1,434 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.cluster.coordination.http.endpoints; + +import org.apache.nifi.cluster.manager.NodeResponse; +import org.apache.nifi.cluster.protocol.NodeIdentifier; +import org.apache.nifi.util.FormatUtils; +import org.apache.nifi.web.api.dto.BacklogDTO; +import org.apache.nifi.web.api.dto.BacklogRequestDTO; +import org.apache.nifi.web.api.entity.BacklogEntity; +import org.apache.nifi.web.api.entity.BacklogRequestEntity; +import org.junit.jupiter.api.Test; + +import java.net.URI; +import java.time.Instant; +import java.util.Arrays; +import java.util.Collections; +import java.util.LinkedHashMap; +import java.util.Map; +import java.util.Set; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertTrue; + +public class BacklogRequestEndpointMergerTest { + + private static final String PROCESSOR_UUID = "12345678-1234-1234-1234-123456789012"; + private static final String CONNECTOR_UUID = "abcdef00-abcd-abcd-abcd-abcdef000000"; + private static final String REQUEST_UUID = "00000000-0000-0000-0000-000000000001"; + + private final BacklogRequestEndpointMerger merger = new BacklogRequestEndpointMerger(); + + @Test + public void testCanHandleAcceptsProcessorBacklogRequestCreationUri() { + assertTrue(merger.canHandle(URI.create("http://localhost:8080/nifi-api/processors/" + PROCESSOR_UUID + "/backlog-requests"), "POST")); + } + + @Test + public void testCanHandleAcceptsProcessorBacklogRequestPollingUri() { + assertTrue(merger.canHandle( + URI.create("http://localhost:8080/nifi-api/processors/" + PROCESSOR_UUID + "/backlog-requests/" + REQUEST_UUID), "GET")); + } + + @Test + public void testCanHandleAcceptsConnectorBacklogRequestUris() { + assertTrue(merger.canHandle(URI.create("http://localhost:8080/nifi-api/connectors/" + CONNECTOR_UUID + "/backlog-requests"), "POST")); + assertTrue(merger.canHandle( + URI.create("http://localhost:8080/nifi-api/connectors/" + CONNECTOR_UUID + "/backlog-requests/" + REQUEST_UUID), "DELETE")); + } + + @Test + public void testCanHandleRejectsUnrelatedUri() { + assertFalse(merger.canHandle(URI.create("http://localhost:8080/nifi-api/processors/" + PROCESSOR_UUID + "/backlog"), "GET")); + } + + @Test + public void testMergeWaitsForSlowestNodeBeforeReportingComplete() { + final BacklogRequestEntity client = requestEntity(false, 50, null, null); + final Map entityMap = nodeMap( + client, + requestEntity(false, 20, null, null)); + + merger.mergeResponses(client, entityMap, null, null); + + assertFalse(client.getRequest().isComplete()); + assertEquals(20, client.getRequest().getPercentCompleted()); + assertNull(client.getRequest().getBacklog()); + } + + @Test + public void testMergeReportsCompleteAndAggregatesBacklogOnceEveryNodeIsComplete() { + final BacklogRequestEntity client = requestEntity(true, 100, backlog(10L), null); + final Map entityMap = nodeMap( + client, + requestEntity(true, 100, backlog(5L), null)); + + merger.mergeResponses(client, entityMap, null, null); + + assertTrue(client.getRequest().isComplete()); + assertEquals(100, client.getRequest().getPercentCompleted()); + assertEquals(15L, client.getRequest().getBacklog().getFlowFileCount()); + } + + @Test + public void testMergeMarksFailedWhenAnyNodeFailsEvenIfOthersAreStillInProgress() { + final BacklogRequestEntity client = requestEntity(false, 40, null, null); + final Map entityMap = nodeMap( + client, + requestEntity(true, 100, null, "Interrupted while determining backlog")); + + merger.mergeResponses(client, entityMap, null, null); + + assertTrue(client.getRequest().isComplete()); + assertEquals(100, client.getRequest().getPercentCompleted()); + assertEquals("Interrupted while determining backlog", client.getRequest().getFailureReason()); + assertNull(client.getRequest().getBacklog()); + } + + @Test + public void testMergeCombinesDistinctFailureReasonsFromMultipleNodes() { + final BacklogRequestEntity client = requestEntity(true, 100, null, "Source unreachable"); + final Map entityMap = nodeMap( + client, + requestEntity(true, 100, null, "Interrupted while determining backlog")); + + merger.mergeResponses(client, entityMap, null, null); + + final String failureReason = client.getRequest().getFailureReason(); + assertTrue(failureReason.contains("Source unreachable")); + assertTrue(failureReason.contains("Interrupted while determining backlog")); + } + + @Test + public void testMergeIgnoresProblematicNodesAndAggregatesOnlyRespondingNodes() { + // AbstractSingleEntityEndpoint places only nodes that returned a successful response into entityMap, so a + // node that failed to respond (a "problematic" node) is absent from entityMap entirely. This test documents + // the merger's current behavior: it derives the merged request solely from entityMap and never inspects the + // problematicResponses parameter, so a non-responding node is silently excluded. As a result the cluster is + // reported complete with an aggregated backlog even though one node never reported. + final BacklogRequestEntity client = requestEntity(true, 100, backlog(10L), null); + final Map entityMap = new LinkedHashMap<>(); + entityMap.put(nodeIdentifier(1), client); + + final NodeResponse problematicResponse = new NodeResponse(nodeIdentifier(2), "GET", + URI.create("http://localhost:8082/nifi-api/processors/" + PROCESSOR_UUID + "/backlog-requests/" + REQUEST_UUID), + new RuntimeException("Node was unreachable")); + final Set problematicResponses = Collections.singleton(problematicResponse); + + merger.mergeResponses(client, entityMap, Collections.emptySet(), problematicResponses); + + assertTrue(client.getRequest().isComplete()); + assertEquals(100, client.getRequest().getPercentCompleted()); + assertNull(client.getRequest().getFailureReason()); + assertEquals(10L, client.getRequest().getBacklog().getFlowFileCount()); + } + + @Test + public void testMergeEntitiesAllNullBacklogsProduceNullBacklog() { + final BacklogEntity client = entityWith(null); + BacklogRequestEndpointMerger.mergeEntities(client, Arrays.asList(entityWith(null), entityWith(null), entityWith(null))); + assertNull(client.getBacklog()); + } + + @Test + public void testMergeEntitiesSumsNumericDimensionsAcrossNodes() { + final BacklogEntity client = entityWith(backlog(10L, 100L, 1000L, "2026-05-15T10:00:00Z", "EXACT")); + final BacklogEntity nodeTwo = entityWith(backlog(5L, 50L, 500L, "2026-05-15T09:00:00Z", "EXACT")); + final BacklogEntity nodeThree = entityWith(backlog(2L, 20L, 200L, "2026-05-15T08:30:00Z", "EXACT")); + + BacklogRequestEndpointMerger.mergeEntities(client, Arrays.asList(client, nodeTwo, nodeThree)); + + final BacklogDTO merged = client.getBacklog(); + assertNotNull(merged); + assertEquals(17L, merged.getFlowFileCount()); + assertEquals(170L, merged.getByteCount()); + assertEquals(1700L, merged.getRecordCount()); + assertEquals("2026-05-15T08:30:00Z", merged.getLastCaughtUp()); + assertEquals("EXACT", merged.getPrecision()); + } + + @Test + public void testMergeEntitiesPopulatesFormattedStringsFromMergedTotals() { + final BacklogEntity client = entityWith(backlog(1024L, 5_368_709_120L, 2050L, "2026-05-15T10:00:00Z", "EXACT")); + final BacklogEntity nodeTwo = entityWith(backlog(1L, 1024L, 1L, "2026-05-15T09:00:00Z", "EXACT")); + + BacklogRequestEndpointMerger.mergeEntities(client, Arrays.asList(client, nodeTwo)); + + final BacklogDTO merged = client.getBacklog(); + assertEquals(1025L, merged.getFlowFileCount()); + // The formatted strings are produced by the same FormatUtils helpers the runtime uses, so compare against + // those helpers directly to keep the assertions independent of the test machine's locale. + assertEquals(FormatUtils.formatCount(1025L), merged.getFormattedFlowFileCount()); + assertEquals(5_368_710_144L, merged.getByteCount()); + assertEquals(FormatUtils.formatDataSize(5_368_710_144L), merged.getFormattedByteCount()); + assertEquals(2051L, merged.getRecordCount()); + assertEquals(FormatUtils.formatCount(2051L), merged.getFormattedRecordCount()); + } + + @Test + public void testMergeEntitiesOmitsFormattedStringWhenDimensionIsNull() { + final BacklogEntity client = entityWith(backlog(null, 100L, null, "2026-05-15T10:00:00Z", "EXACT")); + final BacklogEntity nodeTwo = entityWith(backlog(null, 50L, null, "2026-05-15T09:00:00Z", "EXACT")); + + BacklogRequestEndpointMerger.mergeEntities(client, Arrays.asList(client, nodeTwo)); + + final BacklogDTO merged = client.getBacklog(); + assertNull(merged.getFormattedFlowFileCount()); + assertEquals(FormatUtils.formatDataSize(150L), merged.getFormattedByteCount()); + assertNull(merged.getFormattedRecordCount()); + } + + @Test + public void testMergeEntitiesEmitsNullForDimensionWhenEveryNodeIsNull() { + final BacklogEntity client = entityWith(backlog(null, 100L, null, "2026-05-15T10:00:00Z", "EXACT")); + final BacklogEntity nodeTwo = entityWith(backlog(null, 50L, null, "2026-05-15T09:00:00Z", "EXACT")); + + BacklogRequestEndpointMerger.mergeEntities(client, Arrays.asList(client, nodeTwo)); + + final BacklogDTO merged = client.getBacklog(); + assertNull(merged.getFlowFileCount()); + assertEquals(150L, merged.getByteCount()); + assertNull(merged.getRecordCount()); + } + + @Test + public void testMergeEntitiesTreatsNullDimensionAsZeroWhenAnyNodePopulatesIt() { + final BacklogEntity client = entityWith(backlog(10L, null, null, "2026-05-15T10:00:00Z", "EXACT")); + final BacklogEntity nodeTwo = entityWith(backlog(null, null, null, "2026-05-15T09:00:00Z", "EXACT")); + + BacklogRequestEndpointMerger.mergeEntities(client, Arrays.asList(client, nodeTwo)); + + assertEquals(10L, client.getBacklog().getFlowFileCount()); + } + + @Test + public void testMergeEntitiesDowngradesPrecisionWhenShapeDiffers() { + final BacklogEntity client = entityWith(backlog(10L, 100L, null, "2026-05-15T10:00:00Z", "EXACT")); + final BacklogEntity nodeTwo = entityWith(backlog(null, 50L, null, "2026-05-15T09:00:00Z", "EXACT")); + + BacklogRequestEndpointMerger.mergeEntities(client, Arrays.asList(client, nodeTwo)); + + assertEquals("AT_LEAST", client.getBacklog().getPrecision()); + } + + @Test + public void testMergeEntitiesDowngradesPrecisionWhenAnyNodeReportsAtLeast() { + final BacklogEntity client = entityWith(backlog(10L, 100L, 1000L, "2026-05-15T10:00:00Z", "EXACT")); + final BacklogEntity nodeTwo = entityWith(backlog(5L, 50L, 500L, "2026-05-15T09:00:00Z", "AT_LEAST")); + + BacklogRequestEndpointMerger.mergeEntities(client, Arrays.asList(client, nodeTwo)); + + assertEquals("AT_LEAST", client.getBacklog().getPrecision()); + } + + @Test + public void testMergeEntitiesNullLastCaughtUpFromAnyNodeProducesNullMergedLastCaughtUp() { + final BacklogEntity client = entityWith(backlog(10L, 100L, 1000L, "2026-05-15T10:00:00Z", "EXACT")); + final BacklogEntity nodeTwo = entityWith(backlog(5L, 50L, 500L, null, "EXACT")); + + BacklogRequestEndpointMerger.mergeEntities(client, Arrays.asList(client, nodeTwo)); + + assertNull(client.getBacklog().getLastCaughtUp()); + assertNull(client.getBacklog().getFormattedLastCaughtUp()); + } + + @Test + public void testMergeEntitiesFormattedLastCaughtUpComputedFromMergedTimestamp() { + // The earliest reported lastCaughtUp wins after merging; that wall-clock value is what + // formattedLastCaughtUp must describe relative to "now". Stamp the timestamp well into the + // past so the time-bucket assertion is stable regardless of when the test runs. + final String earliest = Instant.now().minusSeconds(120L).toString(); + final String later = Instant.now().minusSeconds(30L).toString(); + + final BacklogEntity client = entityWith(backlog(10L, 100L, 1000L, later, "EXACT")); + final BacklogEntity nodeTwo = entityWith(backlog(2L, 20L, 200L, earliest, "EXACT")); + + BacklogRequestEndpointMerger.mergeEntities(client, Arrays.asList(client, nodeTwo)); + + assertEquals(earliest, client.getBacklog().getLastCaughtUp()); + // The merged backlog is not numerically caught up (FlowFiles/bytes/records > 0), so the + // "now" collapse must not fire and the formatted value must be a relative-time string. + final String formatted = client.getBacklog().getFormattedLastCaughtUp(); + assertNotNull(formatted); + assertTrue(formatted.contains("min") || formatted.contains("hour"), + "Expected merged formattedLastCaughtUp to describe a multi-minute gap, got: " + formatted); + } + + @Test + public void testMergeEntitiesFormattedLastCaughtUpEmitsNowWhenAllNodesCaughtUpWithinWindow() { + final String nowIso = Instant.now().toString(); + final BacklogEntity client = entityWith(backlog(0L, 0L, 0L, nowIso, "EXACT")); + final BacklogEntity nodeTwo = entityWith(backlog(0L, 0L, 0L, nowIso, "EXACT")); + + BacklogRequestEndpointMerger.mergeEntities(client, Arrays.asList(client, nodeTwo)); + + assertEquals("now", client.getBacklog().getFormattedLastCaughtUp()); + } + + @Test + public void testMergeEntitiesFormattedLastCaughtUpUsesRelativeTimeWhenAnyNumericDimensionIsNonZero() { + final String nowIso = Instant.now().toString(); + final BacklogEntity client = entityWith(backlog(0L, 0L, 1L, nowIso, "EXACT")); + final BacklogEntity nodeTwo = entityWith(backlog(0L, 0L, 0L, nowIso, "EXACT")); + + BacklogRequestEndpointMerger.mergeEntities(client, Arrays.asList(client, nodeTwo)); + + // recordCount sums to 1, so the merged backlog is not caught up; even though the timestamp + // is within the "now" window, the formatted value must use the relative-time string. + final String formatted = client.getBacklog().getFormattedLastCaughtUp(); + assertNotNull(formatted); + assertTrue(formatted.endsWith("ago") || formatted.startsWith("in "), "Expected relative-time output, got: " + formatted); + } + + @Test + public void testMergeEntitiesExcludesNullBacklogEntitiesFromSummationButDowngradesPrecision() { + // A node that returned backlog == null is excluded from numeric summation, but its absence + // means the cluster total is by definition a lower bound — the missing node could be holding + // additional work. The merged precision must therefore be AT_LEAST and the merged + // lastCaughtUp must be cleared, because the cluster cannot claim it is caught up while any + // node has failed to report. + final BacklogEntity client = entityWith(backlog(10L, 100L, 1000L, "2026-05-15T10:00:00Z", "EXACT")); + final BacklogEntity nullEntity = entityWith(null); + final BacklogEntity nodeThree = entityWith(backlog(2L, 20L, 200L, "2026-05-15T08:30:00Z", "EXACT")); + + BacklogRequestEndpointMerger.mergeEntities(client, Arrays.asList(client, nullEntity, nodeThree)); + + final BacklogDTO merged = client.getBacklog(); + assertEquals(12L, merged.getFlowFileCount()); + assertEquals(120L, merged.getByteCount()); + assertEquals(1200L, merged.getRecordCount()); + assertEquals("AT_LEAST", merged.getPrecision()); + assertNull(merged.getLastCaughtUp()); + assertNull(merged.getFormattedLastCaughtUp()); + } + + @Test + public void testMergeEntitiesNullEntityElementForcesAtLeastEvenWhenOtherNodesAreExact() { + // A null BacklogEntity in the entity list (for example because that node's response failed to + // deserialize) is treated the same way as backlog == null: the merger has no value from that + // node, so it cannot claim cluster-wide completeness. + final BacklogEntity client = entityWith(backlog(7L, 70L, 700L, "2026-05-15T10:00:00Z", "EXACT")); + + BacklogRequestEndpointMerger.mergeEntities(client, Arrays.asList(client, null)); + + final BacklogDTO merged = client.getBacklog(); + assertEquals(7L, merged.getFlowFileCount()); + assertEquals("AT_LEAST", merged.getPrecision()); + assertNull(merged.getLastCaughtUp()); + } + + @Test + public void testMergeEntitiesNullBacklogFromAnyNodeForbidsZeroExactCaughtUpClaim() { + // Even if every reporting node says "zero, EXACT, caught up right now", a single + // non-reporting node must prevent the cluster from claiming caught up. + final String nowIso = Instant.now().toString(); + final BacklogEntity client = entityWith(backlog(0L, 0L, 0L, nowIso, "EXACT")); + final BacklogEntity nodeTwo = entityWith(backlog(0L, 0L, 0L, nowIso, "EXACT")); + final BacklogEntity nodeThree = entityWith(null); + + BacklogRequestEndpointMerger.mergeEntities(client, Arrays.asList(client, nodeTwo, nodeThree)); + + final BacklogDTO merged = client.getBacklog(); + assertNotNull(merged); + assertEquals(0L, merged.getFlowFileCount()); + assertEquals("AT_LEAST", merged.getPrecision()); + assertNull(merged.getLastCaughtUp()); + assertNull(merged.getFormattedLastCaughtUp()); + } + + @Test + public void testMergeEntitiesSingleReportingNodeProducesItsBacklog() { + // A single-node cluster (or single entity in the entities list) reports exactly what the + // node reported. There are no non-reporting nodes to taint the precision. + final BacklogEntity client = entityWith(backlog(3L, 30L, 300L, "2026-05-15T11:00:00Z", "EXACT")); + + BacklogRequestEndpointMerger.mergeEntities(client, Collections.singletonList(client)); + + final BacklogDTO merged = client.getBacklog(); + assertNotNull(merged); + assertEquals(3L, merged.getFlowFileCount()); + assertEquals("EXACT", merged.getPrecision()); + } + + @Test + public void testMergeEntitiesEmptyEntityListProducesNullBacklog() { + final BacklogEntity client = entityWith(backlog(99L, 999L, 9999L, Instant.now().toString(), "EXACT")); + BacklogRequestEndpointMerger.mergeEntities(client, Collections.emptyList()); + assertNull(client.getBacklog()); + } + + private static BacklogEntity entityWith(final BacklogDTO dto) { + final BacklogEntity entity = new BacklogEntity(); + entity.setBacklog(dto); + return entity; + } + + private static BacklogDTO backlog(final Long flowFiles, final Long bytes, final Long records, final String lastCaughtUp, final String precision) { + final BacklogDTO dto = new BacklogDTO(); + dto.setFlowFileCount(flowFiles); + dto.setByteCount(bytes); + dto.setRecordCount(records); + dto.setLastCaughtUp(lastCaughtUp); + dto.setPrecision(precision); + return dto; + } + + private static Map nodeMap(final BacklogRequestEntity clientEntity, final BacklogRequestEntity otherNodeEntity) { + final Map entityMap = new LinkedHashMap<>(); + entityMap.put(nodeIdentifier(1), clientEntity); + entityMap.put(nodeIdentifier(2), otherNodeEntity); + return entityMap; + } + + private static NodeIdentifier nodeIdentifier(final int index) { + return new NodeIdentifier("node-" + index, "localhost", 8000 + index, "localhost", 8100 + index, + "localhost", 8200 + index, 8300 + index, false); + } + + private static BacklogRequestEntity requestEntity(final boolean complete, final int percentCompleted, final BacklogDTO backlog, final String failureReason) { + final BacklogRequestDTO dto = new BacklogRequestDTO(); + dto.setComplete(complete); + dto.setPercentCompleted(percentCompleted); + dto.setBacklog(backlog); + dto.setFailureReason(failureReason); + + final BacklogRequestEntity entity = new BacklogRequestEntity(); + entity.setRequest(dto); + return entity; + } + + private static BacklogDTO backlog(final Long flowFileCount) { + final BacklogDTO dto = new BacklogDTO(); + dto.setFlowFileCount(flowFileCount); + dto.setPrecision("EXACT"); + return dto; + } +} diff --git a/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/java/org/apache/nifi/controller/StandardProcessorNode.java b/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/java/org/apache/nifi/controller/StandardProcessorNode.java index f8b0fde8afc8..7236343d91e0 100644 --- a/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/java/org/apache/nifi/controller/StandardProcessorNode.java +++ b/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/java/org/apache/nifi/controller/StandardProcessorNode.java @@ -40,6 +40,8 @@ import org.apache.nifi.authorization.resource.ResourceType; import org.apache.nifi.bundle.Bundle; import org.apache.nifi.bundle.BundleCoordinate; +import org.apache.nifi.components.Backlog; +import org.apache.nifi.components.BacklogReportingException; import org.apache.nifi.components.ConfigVerificationResult; import org.apache.nifi.components.ConfigVerificationResult.Outcome; import org.apache.nifi.components.ConfigurableComponent; @@ -87,6 +89,7 @@ import org.apache.nifi.parameter.ParameterParser; import org.apache.nifi.parameter.ParameterReference; import org.apache.nifi.parameter.ParameterTokenList; +import org.apache.nifi.processor.BacklogReportingProcessor; import org.apache.nifi.processor.ProcessContext; import org.apache.nifi.processor.ProcessSessionFactory; import org.apache.nifi.processor.Processor; @@ -1634,6 +1637,46 @@ public void verifyCanTerminate() { } } + @Override + public boolean supportsBacklogReporting() { + return getProcessor() instanceof BacklogReportingProcessor; + } + + @Override + public Optional getReportedBacklog(final ProcessContext context) throws BacklogReportingException { + final Processor processor = getProcessor(); + if (!(processor instanceof BacklogReportingProcessor)) { + return Optional.empty(); + } + + try (final NarCloseable ignored = NarCloseable.withComponentNarLoader(getExtensionManager(), processor.getClass(), processor.getIdentifier())) { + return ((BacklogReportingProcessor) processor).getBacklog(context); + } catch (final BacklogReportingException e) { + LOG.error("Failed to determine backlog for {}", this, e); + throw e; + } catch (final RuntimeException e) { + LOG.error("Failed to determine backlog for {}", this, e); + throw new BacklogReportingException("Failed to determine backlog for " + this, e); + } + } + + @Override + public void verifyCanReportBacklog() { + if (!supportsBacklogReporting()) { + throw new IllegalStateException("Cannot report backlog for " + this + " because the Processor does not implement BacklogReportingProcessor"); + } + + final ScheduledState state = getScheduledState(); + if (state == ScheduledState.DISABLED) { + throw new IllegalStateException("Cannot report backlog for " + this + " because the Processor is disabled"); + } + + final ValidationStatus validationStatus = getValidationStatus(); + if (validationStatus != ValidationStatus.VALID) { + throw new IllegalStateException("Cannot report backlog for " + this + " because the Processor is not valid (Validation State is " + validationStatus + ")"); + } + } + private void initiateStart(final ScheduledExecutorService taskScheduler, final long administrativeYieldMillis, final long timeoutMillis, final AtomicLong startupAttemptCount, final Supplier processContextFactory, final SchedulingAgentCallback schedulingAgentCallback, final boolean triggerLifecycleMethods) { diff --git a/nifi-framework-bundle/nifi-framework/nifi-framework-core-api/src/main/java/org/apache/nifi/controller/ProcessorNode.java b/nifi-framework-bundle/nifi-framework/nifi-framework-core-api/src/main/java/org/apache/nifi/controller/ProcessorNode.java index 25ce747b7365..e3db0b42de5b 100644 --- a/nifi-framework-bundle/nifi-framework/nifi-framework-core-api/src/main/java/org/apache/nifi/controller/ProcessorNode.java +++ b/nifi-framework-bundle/nifi-framework/nifi-framework-core-api/src/main/java/org/apache/nifi/controller/ProcessorNode.java @@ -19,6 +19,8 @@ import org.apache.nifi.annotation.behavior.InputRequirement.Requirement; import org.apache.nifi.annotation.lifecycle.OnUnscheduled; import org.apache.nifi.annotation.notification.PrimaryNodeState; +import org.apache.nifi.components.Backlog; +import org.apache.nifi.components.BacklogReportingException; import org.apache.nifi.components.ConfigVerificationResult; import org.apache.nifi.components.connector.InvocationFailedException; import org.apache.nifi.components.connector.components.ConnectorMethod; @@ -42,6 +44,7 @@ import java.util.List; import java.util.Map; +import java.util.Optional; import java.util.Set; import java.util.concurrent.CompletableFuture; import java.util.concurrent.ScheduledExecutorService; @@ -139,6 +142,43 @@ public void verifyCanPerformVerification() { } } + /** + * Indicates whether the wrapped Processor implements {@code BacklogReportingProcessor}. This is a + * cheap capability check that does not call into the extension and never throws. + * + * @return {@code true} if the wrapped Processor implements {@code BacklogReportingProcessor}; {@code false} otherwise + */ + public abstract boolean supportsBacklogReporting(); + + /** + * Invokes the wrapped Processor's backlog reporting method when it implements {@code BacklogReportingProcessor}. + * The method is invoked with the Processor's NAR ClassLoader active. Callers must invoke + * {@link #verifyCanReportBacklog()} (or an equivalent precondition check) before calling this method; + * the contract here is "report the backlog if you can". + * + *

+ * The supplied {@link ProcessContext} must be non-null and must be built fresh by the caller for + * each invocation. The framework intentionally does not reuse the live scheduled context here so + * that the Processor's backlog query is isolated from its running state and can be answered even + * when the Processor is stopped. + *

+ * + * @param context a non-null {@link ProcessContext} built fresh by the caller for this invocation + * @return the Processor's reported backlog, or {@link Optional#empty()} if the Processor does not implement + * {@code BacklogReportingProcessor} or currently has nothing to report + * @throws BacklogReportingException if the Processor attempted to determine its backlog and failed + */ + public abstract Optional getReportedBacklog(ProcessContext context) throws BacklogReportingException; + + /** + * Verifies that the Processor is in an appropriate state to be asked for a backlog. The Processor must + * be valid, must not be disabled, and must implement {@code BacklogReportingProcessor}. Failures throw + * {@link IllegalStateException} with a message describing the failed precondition + * + * @throws IllegalStateException if the Processor cannot be asked for a backlog + */ + public abstract void verifyCanReportBacklog(); + public abstract List verifyConfiguration(ProcessContext processContext, ComponentLog logger, Map attributes, ExtensionManager extensionManager, ParameterLookup parameterLookup); diff --git a/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/components/connector/facades/standalone/StandaloneConnectionFacade.java b/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/components/connector/facades/standalone/StandaloneConnectionFacade.java index a5d32a6c1bd6..a93cf86daeb9 100644 --- a/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/components/connector/facades/standalone/StandaloneConnectionFacade.java +++ b/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/components/connector/facades/standalone/StandaloneConnectionFacade.java @@ -21,11 +21,13 @@ import org.apache.nifi.components.connector.components.ConnectionFacade; import org.apache.nifi.components.connector.components.QueueSnapshot; import org.apache.nifi.connectable.Connection; +import org.apache.nifi.controller.queue.FlowFileQueueSnapshot; import org.apache.nifi.controller.queue.QueueSize; import org.apache.nifi.flow.VersionedConnection; import org.apache.nifi.flowfile.FlowFile; import java.io.IOException; +import java.util.List; import java.util.Objects; import java.util.function.Predicate; @@ -61,7 +63,27 @@ public DropFlowFileSummary dropFlowFiles(final Predicate predicate) th @Override public QueueSnapshot getQueueSnapshot() { - throw new UnsupportedOperationException("Queue snapshots not yet implemented"); + final FlowFileQueueSnapshot queueSnapshot = connection.getFlowFileQueue().getQueueSnapshot(); + return new StandaloneQueueSnapshot(queueSnapshot.queueSize(), List.copyOf(queueSnapshot.activeFlowFiles())); + } + + /** Adapts an atomically-captured {@link FlowFileQueueSnapshot} to the connector-facing {@link QueueSnapshot} contract. */ + private record StandaloneQueueSnapshot(QueueSize queueSize, List activeFlowFiles) implements QueueSnapshot { + + @Override + public QueueSize getQueueSize() { + return queueSize; + } + + @Override + public List getActiveFlowFiles() { + return activeFlowFiles; + } + + @Override + public boolean isActiveListExhaustive() { + return activeFlowFiles.size() == queueSize.getObjectCount(); + } } @Override diff --git a/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/components/connector/facades/standalone/StandaloneProcessorFacade.java b/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/components/connector/facades/standalone/StandaloneProcessorFacade.java index 9c33370cafa6..2fbf8525427a 100644 --- a/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/components/connector/facades/standalone/StandaloneProcessorFacade.java +++ b/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/components/connector/facades/standalone/StandaloneProcessorFacade.java @@ -179,6 +179,16 @@ private VersionedProcessor findVersionedProcessor(final VersionedProcessGroup gr return null; } + @Override + public boolean reportsBacklog() { + return processorNode.supportsBacklogReporting(); + } + + @Override + public Optional getBacklog() throws BacklogReportingException { + final ProcessContext processContext = componentContextProvider.createProcessContext(processorNode, parameterContext); + return processorNode.getReportedBacklog(processContext); + } @Override public Object invokeConnectorMethod(final String methodName, final Map arguments) throws InvocationFailedException { @@ -212,16 +222,6 @@ public T invokeConnectorMethod(final String methodName, final Map getBacklog() throws BacklogReportingException { - return Optional.empty(); - } - @Override public String toString() { final String type = versionedProcessor.getType(); diff --git a/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/queue/StandardFlowFileQueue.java b/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/queue/StandardFlowFileQueue.java index 8a2a9895d4b6..cbfe04cab2dd 100644 --- a/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/queue/StandardFlowFileQueue.java +++ b/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/queue/StandardFlowFileQueue.java @@ -156,6 +156,11 @@ public QueueSize size() { return queue.size(); } + @Override + public FlowFileQueueSnapshot getQueueSnapshot() { + return queue.getQueueSnapshot(); + } + @Override public long getTotalQueuedDuration(long fromTimestamp) { return queue.getTotalQueuedDuration(fromTimestamp); diff --git a/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/queue/SwappablePriorityQueue.java b/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/queue/SwappablePriorityQueue.java index 6e162d5aa260..d2f13071c792 100644 --- a/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/queue/SwappablePriorityQueue.java +++ b/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/queue/SwappablePriorityQueue.java @@ -178,6 +178,39 @@ public List getActiveFlowFiles() { } } + /** + * Acquires the queue's write lock so the caller can freeze all mutating operations + * (put/poll/swap/drop) on this queue until {@link #unlockForSnapshot()} is called. Used by + * load-balanced queues to capture a consistent snapshot across all of their partitions. + */ + public void lockForSnapshot() { + writeLock.lock(); + } + + public void unlockForSnapshot() { + writeLock.unlock("snapshot"); + } + + public FlowFileQueueSnapshot getQueueSnapshot() { + readLock.lock(); + try { + final QueueSize snapshotSize = getFlowFileQueueSize().toQueueSize(); + // java.util.PriorityQueue iterates in heap order, not priority/poll order. Re-heap into a fresh + // PriorityQueue using the same prioritizer and drain it to capture the FlowFiles in the order + // a caller would receive them via poll() — i.e. true queue order. + final Queue ordered = new PriorityQueue<>(Math.max(1, activeQueue.size()), new QueuePrioritizer(getPriorities())); + ordered.addAll(activeQueue); + final List snapshotActiveFlowFiles = new ArrayList<>(ordered.size()); + FlowFileRecord next; + while ((next = ordered.poll()) != null) { + snapshotActiveFlowFiles.add(next); + } + return new FlowFileQueueSnapshot(snapshotSize, snapshotActiveFlowFiles); + } finally { + readLock.unlock("getQueueSnapshot"); + } + } + public boolean isUnacknowledgedFlowFile() { return getFlowFileQueueSize().getUnacknowledgedCount() > 0; } diff --git a/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/queue/clustered/SocketLoadBalancedFlowFileQueue.java b/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/queue/clustered/SocketLoadBalancedFlowFileQueue.java index de1a8f7cfe58..18f403aaf296 100644 --- a/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/queue/clustered/SocketLoadBalancedFlowFileQueue.java +++ b/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/queue/clustered/SocketLoadBalancedFlowFileQueue.java @@ -28,6 +28,7 @@ import org.apache.nifi.controller.queue.DropFlowFileRequest; import org.apache.nifi.controller.queue.DropFlowFileState; import org.apache.nifi.controller.queue.FlowFileQueueContents; +import org.apache.nifi.controller.queue.FlowFileQueueSnapshot; import org.apache.nifi.controller.queue.IllegalClusterStateException; import org.apache.nifi.controller.queue.LoadBalanceStrategy; import org.apache.nifi.controller.queue.LoadBalancedFlowFileQueue; @@ -547,6 +548,49 @@ public QueueSize size() { return totalSize.get(); } + /** + * Returns an atomic, point-in-time snapshot of this queue by freezing every partition on this node for the + * duration of the call. The {@code partitionReadLock} prevents cluster-topology changes from replacing the + * partition array, and each partition's {@link QueuePartition#lockForSnapshot()} blocks that partition's puts, + * polls, swaps, and drops. Because the local partition, every remote partition, and the rebalancing partition + * are all frozen, the summed {@link QueueSize} and the captured local active list are consistent with each other. + */ + @Override + public FlowFileQueueSnapshot getQueueSnapshot() { + partitionReadLock.lock(); + try { + final List partitionsToSnapshot = new ArrayList<>(queuePartitions.length + 1); + Collections.addAll(partitionsToSnapshot, queuePartitions); + partitionsToSnapshot.add(rebalancingPartition); + + final List lockedPartitions = new ArrayList<>(partitionsToSnapshot.size()); + try { + for (final QueuePartition partition : partitionsToSnapshot) { + partition.lockForSnapshot(); + lockedPartitions.add(partition); + } + + long objectCount = 0L; + long byteCount = 0L; + for (final QueuePartition partition : partitionsToSnapshot) { + final QueueSize partitionSize = partition.size(); + objectCount += partitionSize.getObjectCount(); + byteCount += partitionSize.getByteCount(); + } + + final FlowFileQueueSnapshot localSnapshot = localPartition.getQueueSnapshot(); + final QueueSize totalQueueSize = new QueueSize(Math.toIntExact(objectCount), byteCount); + return new FlowFileQueueSnapshot(totalQueueSize, localSnapshot.activeFlowFiles()); + } finally { + for (int i = lockedPartitions.size() - 1; i >= 0; i--) { + lockedPartitions.get(i).unlockForSnapshot(); + } + } + } finally { + partitionReadLock.unlock(); + } + } + @Override public long getTotalQueuedDuration(long fromTimestamp) { long sum = 0L; diff --git a/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/queue/clustered/partition/LocalQueuePartition.java b/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/queue/clustered/partition/LocalQueuePartition.java index e15e62bc74ef..bd7a7ecab48a 100644 --- a/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/queue/clustered/partition/LocalQueuePartition.java +++ b/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/queue/clustered/partition/LocalQueuePartition.java @@ -18,6 +18,7 @@ package org.apache.nifi.controller.queue.clustered.partition; import org.apache.nifi.controller.queue.FlowFileQueueContents; +import org.apache.nifi.controller.queue.FlowFileQueueSnapshot; import org.apache.nifi.controller.queue.LocalQueuePartitionDiagnostics; import org.apache.nifi.controller.queue.PollStrategy; import org.apache.nifi.controller.repository.FlowFileRecord; @@ -111,6 +112,16 @@ public interface LocalQueuePartition extends QueuePartition { */ List getListableFlowFiles(); + /** + * Returns the partition's local queue size and its active in-memory FlowFiles atomically. + * Implementations must obtain both values under the same lock so callers see a coherent + * point-in-time view; see {@link org.apache.nifi.controller.queue.FlowFileQueue#getQueueSnapshot()} + * for the broader contract. + * + * @return a non-null snapshot of the partition's local queue size and active FlowFiles + */ + FlowFileQueueSnapshot getQueueSnapshot(); + /** * Inherits the contents of another queue/partition * @param queueContents the contents to inherit diff --git a/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/queue/clustered/partition/QueuePartition.java b/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/queue/clustered/partition/QueuePartition.java index 5b0e0d1889f0..e60401c89d49 100644 --- a/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/queue/clustered/partition/QueuePartition.java +++ b/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/queue/clustered/partition/QueuePartition.java @@ -123,4 +123,16 @@ public interface QueuePartition { * @return The minimum lastQueueDate in milliseconds of all FlowFiles currently enqueued. If no FlowFile is enqueued, this returns 0. */ long getMinLastQueueDate(); + + /** + * Acquires this partition's internal write-equivalent lock so that callers can freeze the + * partition's size and contents across multiple reads (for example while assembling a + * cluster-wide queue snapshot). Must be paired with {@link #unlockForSnapshot()} in a try/finally. + */ + void lockForSnapshot(); + + /** + * Releases the lock previously acquired by {@link #lockForSnapshot()}. + */ + void unlockForSnapshot(); } diff --git a/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/queue/clustered/partition/RemoteQueuePartition.java b/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/queue/clustered/partition/RemoteQueuePartition.java index 7b9bdce1d675..c0829a5ab830 100644 --- a/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/queue/clustered/partition/RemoteQueuePartition.java +++ b/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/queue/clustered/partition/RemoteQueuePartition.java @@ -104,6 +104,16 @@ public QueueSize size() { return priorityQueue.size(); } + @Override + public void lockForSnapshot() { + priorityQueue.lockForSnapshot(); + } + + @Override + public void unlockForSnapshot() { + priorityQueue.unlockForSnapshot(); + } + @Override public long getTotalActiveQueuedDuration(long fromTimestamp) { return priorityQueue.getTotalQueuedDuration(fromTimestamp); diff --git a/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/queue/clustered/partition/StandardRebalancingPartition.java b/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/queue/clustered/partition/StandardRebalancingPartition.java index 9fa3dced3eb5..f30157343e5f 100644 --- a/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/queue/clustered/partition/StandardRebalancingPartition.java +++ b/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/queue/clustered/partition/StandardRebalancingPartition.java @@ -75,6 +75,16 @@ public QueueSize size() { return queue.size(); } + @Override + public void lockForSnapshot() { + queue.lockForSnapshot(); + } + + @Override + public void unlockForSnapshot() { + queue.unlockForSnapshot(); + } + @Override public long getTotalActiveQueuedDuration(long fromTimestamp) { return queue.getTotalQueuedDuration(fromTimestamp); diff --git a/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/queue/clustered/partition/SwappablePriorityQueueLocalPartition.java b/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/queue/clustered/partition/SwappablePriorityQueueLocalPartition.java index 5d84760b4a51..e6a57bca32f1 100644 --- a/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/queue/clustered/partition/SwappablePriorityQueueLocalPartition.java +++ b/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/queue/clustered/partition/SwappablePriorityQueueLocalPartition.java @@ -22,6 +22,7 @@ import org.apache.nifi.controller.queue.DropFlowFileRequest; import org.apache.nifi.controller.queue.FlowFileQueue; import org.apache.nifi.controller.queue.FlowFileQueueContents; +import org.apache.nifi.controller.queue.FlowFileQueueSnapshot; import org.apache.nifi.controller.queue.LocalQueuePartitionDiagnostics; import org.apache.nifi.controller.queue.PollStrategy; import org.apache.nifi.controller.queue.QueueSize; @@ -155,6 +156,21 @@ public List getListableFlowFiles() { return priorityQueue.getActiveFlowFiles(); } + @Override + public FlowFileQueueSnapshot getQueueSnapshot() { + return priorityQueue.getQueueSnapshot(); + } + + @Override + public void lockForSnapshot() { + priorityQueue.lockForSnapshot(); + } + + @Override + public void unlockForSnapshot() { + priorityQueue.unlockForSnapshot(); + } + @Override public void dropFlowFiles(final DropFlowFileRequest dropRequest, final String requestor) { priorityQueue.dropFlowFiles(dropRequest, requestor); diff --git a/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/components/connector/facades/standalone/StandaloneConnectionFacadeTest.java b/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/components/connector/facades/standalone/StandaloneConnectionFacadeTest.java new file mode 100644 index 000000000000..a7f1f67418c4 --- /dev/null +++ b/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/components/connector/facades/standalone/StandaloneConnectionFacadeTest.java @@ -0,0 +1,91 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.components.connector.facades.standalone; + +import org.apache.nifi.components.connector.components.QueueSnapshot; +import org.apache.nifi.connectable.Connection; +import org.apache.nifi.controller.MockFlowFileRecord; +import org.apache.nifi.controller.queue.FlowFileQueue; +import org.apache.nifi.controller.queue.FlowFileQueueSnapshot; +import org.apache.nifi.controller.queue.QueueSize; +import org.apache.nifi.controller.repository.FlowFileRecord; +import org.apache.nifi.flow.VersionedConnection; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +import java.util.Collections; +import java.util.List; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +public class StandaloneConnectionFacadeTest { + + private FlowFileQueue flowFileQueue; + private StandaloneConnectionFacade facade; + + @BeforeEach + public void setUp() { + final Connection connection = mock(Connection.class); + flowFileQueue = mock(FlowFileQueue.class); + when(connection.getFlowFileQueue()).thenReturn(flowFileQueue); + + final VersionedConnection versionedConnection = mock(VersionedConnection.class); + facade = new StandaloneConnectionFacade(connection, versionedConnection); + } + + @Test + public void testGetQueueSnapshotWithActiveFlowFilesIsExhaustive() { + final List activeFlowFiles = List.of(new MockFlowFileRecord(10L), new MockFlowFileRecord(20L), new MockFlowFileRecord(30L)); + when(flowFileQueue.getQueueSnapshot()).thenReturn(new FlowFileQueueSnapshot(new QueueSize(3, 60L), activeFlowFiles)); + + final QueueSnapshot snapshot = facade.getQueueSnapshot(); + + assertEquals(new QueueSize(3, 60L), snapshot.getQueueSize()); + assertEquals(activeFlowFiles, snapshot.getActiveFlowFiles()); + assertTrue(snapshot.isActiveListExhaustive()); + } + + @Test + public void testGetQueueSnapshotEmptyQueueIsExhaustive() { + when(flowFileQueue.getQueueSnapshot()).thenReturn(new FlowFileQueueSnapshot(new QueueSize(0, 0L), Collections.emptyList())); + + final QueueSnapshot snapshot = facade.getQueueSnapshot(); + + assertEquals(new QueueSize(0, 0L), snapshot.getQueueSize()); + assertTrue(snapshot.getActiveFlowFiles().isEmpty()); + assertTrue(snapshot.isActiveListExhaustive()); + } + + @Test + public void testIsActiveListNotExhaustiveWhenQueueSizeExceedsActiveList() { + // The total queue size counts FlowFiles that are swapped to disk or held in remote/rebalancing partitions, + // none of which appear in the active in-memory list. When the total exceeds the active list, the active list + // is not exhaustive. + final List activeFlowFiles = List.of(new MockFlowFileRecord(10L), new MockFlowFileRecord(20L)); + when(flowFileQueue.getQueueSnapshot()).thenReturn(new FlowFileQueueSnapshot(new QueueSize(5, 150L), activeFlowFiles)); + + final QueueSnapshot snapshot = facade.getQueueSnapshot(); + + assertEquals(5, snapshot.getQueueSize().getObjectCount()); + assertEquals(2, snapshot.getActiveFlowFiles().size()); + assertFalse(snapshot.isActiveListExhaustive()); + } +} diff --git a/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/components/connector/facades/standalone/StandaloneProcessorFacadeTest.java b/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/components/connector/facades/standalone/StandaloneProcessorFacadeTest.java new file mode 100644 index 000000000000..a568d6607503 --- /dev/null +++ b/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/components/connector/facades/standalone/StandaloneProcessorFacadeTest.java @@ -0,0 +1,96 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.components.connector.facades.standalone; + +import org.apache.nifi.asset.AssetManager; +import org.apache.nifi.components.Backlog; +import org.apache.nifi.components.BacklogReportingException; +import org.apache.nifi.controller.ProcessScheduler; +import org.apache.nifi.controller.ProcessorNode; +import org.apache.nifi.flow.VersionedProcessor; +import org.apache.nifi.logging.ComponentLog; +import org.apache.nifi.nar.ExtensionManager; +import org.apache.nifi.parameter.ParameterContext; +import org.apache.nifi.processor.ProcessContext; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +import java.util.Optional; + +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertSame; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +public class StandaloneProcessorFacadeTest { + + private ProcessorNode processorNode; + private ComponentContextProvider componentContextProvider; + private ParameterContext parameterContext; + private ProcessContext processContext; + private StandaloneProcessorFacade facade; + + @BeforeEach + public void setUp() { + processorNode = mock(ProcessorNode.class); + final VersionedProcessor versionedProcessor = mock(VersionedProcessor.class); + final ProcessScheduler scheduler = mock(ProcessScheduler.class); + parameterContext = mock(ParameterContext.class); + componentContextProvider = mock(ComponentContextProvider.class); + final ComponentLog connectorLogger = mock(ComponentLog.class); + final ExtensionManager extensionManager = mock(ExtensionManager.class); + final AssetManager assetManager = mock(AssetManager.class); + + processContext = mock(ProcessContext.class); + when(componentContextProvider.createProcessContext(processorNode, parameterContext)).thenReturn(processContext); + + facade = new StandaloneProcessorFacade(processorNode, versionedProcessor, scheduler, parameterContext, + componentContextProvider, connectorLogger, extensionManager, assetManager); + } + + @Test + public void testReportsBacklogDelegatesToProcessorNode() { + when(processorNode.supportsBacklogReporting()).thenReturn(true); + assertTrue(facade.reportsBacklog()); + + when(processorNode.supportsBacklogReporting()).thenReturn(false); + assertFalse(facade.reportsBacklog()); + } + + @Test + public void testGetBacklogDelegatesToProcessorNode() throws BacklogReportingException { + final Optional reported = Optional.of(Backlog.builder().records(11L).precision(Backlog.Precision.EXACT).build()); + when(processorNode.getReportedBacklog(any(ProcessContext.class))).thenReturn(reported); + + assertSame(reported, facade.getBacklog()); + verify(processorNode).getReportedBacklog(processContext); + verify(componentContextProvider).createProcessContext(processorNode, parameterContext); + } + + @Test + public void testGetBacklogPropagatesBacklogReportingException() throws BacklogReportingException { + final BacklogReportingException failure = new BacklogReportingException("Source unreachable"); + when(processorNode.getReportedBacklog(any(ProcessContext.class))).thenThrow(failure); + + final BacklogReportingException thrown = assertThrows(BacklogReportingException.class, facade::getBacklog); + assertSame(failure, thrown); + } +} diff --git a/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/controller/queue/clustered/TestSocketLoadBalancedFlowFileQueue.java b/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/controller/queue/clustered/TestSocketLoadBalancedFlowFileQueue.java index f40c3f0827a2..72cbdd2009cc 100644 --- a/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/controller/queue/clustered/TestSocketLoadBalancedFlowFileQueue.java +++ b/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/controller/queue/clustered/TestSocketLoadBalancedFlowFileQueue.java @@ -26,6 +26,7 @@ import org.apache.nifi.controller.MockFlowFileRecord; import org.apache.nifi.controller.MockSwapManager; import org.apache.nifi.controller.ProcessScheduler; +import org.apache.nifi.controller.queue.FlowFileQueueSnapshot; import org.apache.nifi.controller.queue.QueueSize; import org.apache.nifi.controller.queue.clustered.client.async.AsyncLoadBalanceClientRegistry; import org.apache.nifi.controller.queue.clustered.partition.FlowFilePartitioner; @@ -61,6 +62,9 @@ import java.util.Map; import java.util.Set; import java.util.UUID; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; import static org.junit.jupiter.api.Assertions.assertEquals; @@ -677,6 +681,120 @@ public void testDataInRemotePartitionForLocalIdIsMovedToLocalPartition() { assertEquals(3, queue.getPartitionCount()); } + @Test + public void testGetQueueSnapshotReportsClusterWideTotalAcrossPartitions() { + // Push two FlowFiles into the local partition and one into a remote partition. The snapshot's + // QueueSize must sum every partition on this node, so all three FlowFiles must be counted; the + // active list must contain only the local partition's contents (the two locally-routed FlowFiles). + // This proves that the snapshot reflects every partition, not just the local one, and that the + // active list and the total are mutually consistent. + final int localPartitionIndex = determineLocalPartitionIndex(); + final int remotePartitionIndex = determineRemotePartitionIndex(); + final int[] sequence = new int[] {localPartitionIndex, localPartitionIndex, remotePartitionIndex}; + queue.setFlowFilePartitioner(new StaticSequencePartitioner(sequence, false)); + + final MockFlowFileRecord firstLocal = new MockFlowFileRecord(10L); + final MockFlowFileRecord secondLocal = new MockFlowFileRecord(20L); + final MockFlowFileRecord remote = new MockFlowFileRecord(40L); + queue.put(firstLocal); + queue.put(secondLocal); + queue.put(remote); + + final org.apache.nifi.controller.queue.FlowFileQueueSnapshot snapshot = queue.getQueueSnapshot(); + + assertEquals(3, snapshot.queueSize().getObjectCount()); + assertEquals(70L, snapshot.queueSize().getByteCount()); + assertEquals(2, snapshot.activeFlowFiles().size()); + assertTrue(snapshot.activeFlowFiles().contains(firstLocal)); + assertTrue(snapshot.activeFlowFiles().contains(secondLocal)); + assertFalse(snapshot.activeFlowFiles().contains(remote)); + } + + @Test + @Timeout(10) + public void testGetQueueSnapshotBlocksConcurrentPutsAndReturnsConsistentView() throws InterruptedException { + // Send every FlowFile to the local partition so the snapshot's active list and total + // {@link QueueSize} must match exactly: anything counted in the total must also be in the + // active list. While the snapshot is computing, manually hold the local partition's snapshot + // lock to simulate the lock window inside getQueueSnapshot(); concurrent puts must block + // until the lock is released. After the lock is released and the puts drain, the next + // snapshot must see the updated counts and the active list must still match the total. + final int localPartitionIndex = determineLocalPartitionIndex(); + queue.setFlowFilePartitioner(new StaticFlowFilePartitioner(localPartitionIndex)); + + for (int i = 0; i < 5; i++) { + queue.put(new MockFlowFileRecord(1L)); + } + + final org.apache.nifi.controller.queue.FlowFileQueueSnapshot initialSnapshot = queue.getQueueSnapshot(); + assertEquals(5, initialSnapshot.queueSize().getObjectCount()); + assertEquals(5, initialSnapshot.activeFlowFiles().size()); + assertEquals(initialSnapshot.queueSize().getObjectCount(), initialSnapshot.activeFlowFiles().size(), + "active list size must equal queueSize().getObjectCount() when every FlowFile is in the local partition"); + + // Hold the local partition's snapshot lock and confirm that a concurrent put cannot proceed. + // This is what the production getQueueSnapshot() relies on to keep the snapshot atomic. + final QueuePartition localPartition = queue.getLocalPartition(); + localPartition.lockForSnapshot(); + final CountDownLatch putThreadStarted = new CountDownLatch(1); + final AtomicBoolean putReturned = new AtomicBoolean(false); + final Thread putThread = new Thread(() -> { + putThreadStarted.countDown(); + queue.put(new MockFlowFileRecord(1L)); + putReturned.set(true); + }, "TestQueueSnapshotConcurrentPut"); + putThread.setDaemon(true); + putThread.start(); + try { + assertTrue(putThreadStarted.await(5, TimeUnit.SECONDS), "Put thread did not start"); + // Give the put thread plenty of time to attempt the put. If the lock is not honored, the + // put will complete and putReturned will flip to true before we release the lock. + Thread.sleep(200L); + assertFalse(putReturned.get(), "Put completed while local partition snapshot lock was held"); + } finally { + localPartition.unlockForSnapshot(); + } + + putThread.join(5_000L); + assertTrue(putReturned.get(), "Put did not complete after the snapshot lock was released"); + + final org.apache.nifi.controller.queue.FlowFileQueueSnapshot afterPutSnapshot = queue.getQueueSnapshot(); + assertEquals(6, afterPutSnapshot.queueSize().getObjectCount()); + assertEquals(6, afterPutSnapshot.activeFlowFiles().size()); + } + + @Test + public void testGetQueueSnapshotIncludesRebalancingPartition() { + // Route FlowFiles to the local partition, then move every partition's contents into the rebalancing + // partition and prevent it from redistributing them. FlowFiles sitting in the rebalancing partition + // (being redistributed across the cluster) must still be counted in the snapshot's total QueueSize. + final int localPartitionIndex = determineLocalPartitionIndex(); + queue.setFlowFilePartitioner(new StaticFlowFilePartitioner(localPartitionIndex)); + + // Toggle load balancing so the rebalancing partition is stopped and will not drain FlowFiles moved into it. + queue.startLoadBalancing(); + queue.stopLoadBalancing(); + + final long bytesPerFlowFile = 5L; + final int flowFileCount = 4; + for (int i = 0; i < flowFileCount; i++) { + queue.put(new MockFlowFileRecord(bytesPerFlowFile)); + } + + // Changing the partitioner packages every partition's contents and hands them to the rebalancing partition. + queue.setFlowFilePartitioner(new StaticFlowFilePartitioner(determineRemotePartitionIndex())); + + for (int i = 0; i < queue.getPartitionCount(); i++) { + assertEquals(0, queue.getPartition(i).size().getObjectCount()); + } + + final FlowFileQueueSnapshot snapshot = queue.getQueueSnapshot(); + assertEquals(flowFileCount, snapshot.queueSize().getObjectCount()); + assertEquals(flowFileCount * bytesPerFlowFile, snapshot.queueSize().getByteCount()); + assertEquals(queue.size(), snapshot.queueSize()); + assertTrue(snapshot.activeFlowFiles().isEmpty()); + } + private void assertPartitionSizes(final int[] expectedSizes) { final int[] partitionSizes = new int[queue.getPartitionCount()]; while (!Arrays.equals(expectedSizes, partitionSizes)) { diff --git a/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/controller/queue/clustered/TestSwappablePriorityQueue.java b/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/controller/queue/clustered/TestSwappablePriorityQueue.java index 05550aaa3b8c..46460d10c878 100644 --- a/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/controller/queue/clustered/TestSwappablePriorityQueue.java +++ b/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/controller/queue/clustered/TestSwappablePriorityQueue.java @@ -22,6 +22,7 @@ import org.apache.nifi.controller.queue.DropFlowFileAction; import org.apache.nifi.controller.queue.DropFlowFileRequest; import org.apache.nifi.controller.queue.FlowFileQueue; +import org.apache.nifi.controller.queue.FlowFileQueueSnapshot; import org.apache.nifi.controller.queue.PollStrategy; import org.apache.nifi.controller.queue.QueueSize; import org.apache.nifi.controller.queue.SelectiveDropResult; @@ -782,6 +783,64 @@ public void testGetActiveFlowFilesReturnsAllActiveFlowFiles() throws Interrupted assertEquals(9999, active.size()); } + @Test + public void testGetQueueSnapshotReturnsFlowFilesInPollOrder() { + final FlowFilePrioritizer iAttributePrioritizer = (o1, o2) -> { + final int i1 = Integer.parseInt(o1.getAttribute("i")); + final int i2 = Integer.parseInt(o2.getAttribute("i")); + return Integer.compare(i1, i2); + }; + queue.setPriorities(Collections.singletonList(iAttributePrioritizer)); + + for (final int i : new int[] {5, 1, 4, 2, 3}) { + queue.put(new MockFlowFileRecord(Map.of("i", String.valueOf(i)), i)); + } + + final FlowFileQueueSnapshot snapshot = queue.getQueueSnapshot(); + assertEquals(5, snapshot.queueSize().getObjectCount()); + assertEquals(5, snapshot.activeFlowFiles().size()); + + final List snapshotOrder = new ArrayList<>(); + for (final FlowFileRecord record : snapshot.activeFlowFiles()) { + snapshotOrder.add(record.getAttribute("i")); + } + assertEquals(List.of("1", "2", "3", "4", "5"), snapshotOrder); + + final List pollOrder = new ArrayList<>(); + FlowFileRecord polled; + while ((polled = queue.poll(Collections.emptySet(), 0L)) != null) { + pollOrder.add(polled.getAttribute("i")); + } + assertEquals(snapshotOrder, pollOrder); + } + + @Test + public void testGetQueueSnapshotExcludesSwappedFlowFiles() { + // Fill the queue past its swap threshold so the first 10,000 FlowFiles remain in the active in-memory queue + // and the remaining 10,000 are swapped out. The snapshot's active list must contain only the in-memory + // FlowFiles, while its total QueueSize still counts every FlowFile, including those swapped to disk. + final int swapThreshold = 10_000; + final int totalFlowFiles = 20_000; + for (int i = 0; i < totalFlowFiles; i++) { + queue.put(new MockFlowFileRecord(i)); + } + + assertEquals(1, swapManager.swappedOut.size()); + + final FlowFileQueueSnapshot snapshot = queue.getQueueSnapshot(); + assertEquals(totalFlowFiles, snapshot.queueSize().getObjectCount()); + assertEquals(swapThreshold, snapshot.activeFlowFiles().size()); + + final int swappedCount = snapshot.queueSize().getObjectCount() - snapshot.activeFlowFiles().size(); + assertEquals(totalFlowFiles - swapThreshold, swappedCount); + + // The active in-memory FlowFiles are the first 10,000 added (sizes 0 - 9,999); the swapped FlowFiles + // (sizes 10,000 - 19,999) must not appear in the active list. + for (final FlowFileRecord activeFlowFile : snapshot.activeFlowFiles()) { + assertTrue(activeFlowFile.getSize() < swapThreshold); + } + } + @Test @Timeout(5) public void testListFlowFilesResultsLimited() throws InterruptedException { diff --git a/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/controller/repository/TestWriteAheadFlowFileRepository.java b/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/controller/repository/TestWriteAheadFlowFileRepository.java index af5d92b3956d..fea7b00170bb 100644 --- a/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/controller/repository/TestWriteAheadFlowFileRepository.java +++ b/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/controller/repository/TestWriteAheadFlowFileRepository.java @@ -23,6 +23,7 @@ import org.apache.nifi.controller.queue.DropFlowFileStatus; import org.apache.nifi.controller.queue.FlowFileQueue; import org.apache.nifi.controller.queue.FlowFileQueueSize; +import org.apache.nifi.controller.queue.FlowFileQueueSnapshot; import org.apache.nifi.controller.queue.ListFlowFileStatus; import org.apache.nifi.controller.queue.LoadBalanceCompression; import org.apache.nifi.controller.queue.LoadBalanceStrategy; @@ -183,6 +184,11 @@ public QueueSize size() { return null; } + @Override + public FlowFileQueueSnapshot getQueueSnapshot() { + return new FlowFileQueueSnapshot(new QueueSize(0, 0L), List.of()); + } + @Override public long getTotalQueuedDuration(long fromTimestamp) { return 0; diff --git a/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/NiFiServiceFacade.java b/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/NiFiServiceFacade.java index 14b2e28ad957..da95b30496ac 100644 --- a/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/NiFiServiceFacade.java +++ b/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/NiFiServiceFacade.java @@ -30,6 +30,7 @@ import org.apache.nifi.c2.protocol.component.api.ProcessorDefinition; import org.apache.nifi.c2.protocol.component.api.ReportingTaskDefinition; import org.apache.nifi.c2.protocol.component.api.RuntimeManifest; +import org.apache.nifi.components.BacklogReportingException; import org.apache.nifi.components.ConfigurableComponent; import org.apache.nifi.controller.ScheduledState; import org.apache.nifi.controller.repository.claim.ContentDirection; @@ -105,6 +106,7 @@ import org.apache.nifi.web.api.entity.ActivateControllerServicesEntity; import org.apache.nifi.web.api.entity.AffectedComponentEntity; import org.apache.nifi.web.api.entity.AssetEntity; +import org.apache.nifi.web.api.entity.BacklogEntity; import org.apache.nifi.web.api.entity.BulletinEntity; import org.apache.nifi.web.api.entity.ClearBulletinsForGroupResultsEntity; import org.apache.nifi.web.api.entity.ClearBulletinsResultEntity; @@ -851,6 +853,39 @@ Set getControllerServiceTypes(final String serviceType, final */ ProcessorDiagnosticsEntity getProcessorDiagnostics(String id); + /** + * Verifies that the Processor with the given id is in an appropriate state to be asked for a backlog report. + * + * @param processorId the id of the Processor + */ + void verifyCanReportProcessorBacklog(String processorId); + + /** + * Returns a backlog snapshot for the Processor with the given id. Callers must invoke + * {@link #verifyCanReportProcessorBacklog(String)} before invoking this method. + * + * @param processorId the id of the Processor + * @return the backlog entity; the wrapped DTO is null when the Processor returns {@link Optional#empty()} + * @throws BacklogReportingException if the Processor attempts to determine its backlog and fails + */ + BacklogEntity getProcessorBacklog(String processorId) throws BacklogReportingException; + + /** + * Verifies that the Connector with the given id can be asked for a connector-scope backlog report. + * + * @param connectorId the id of the Connector + */ + void verifyCanReportConnectorBacklog(String connectorId); + + /** + * Returns a backlog snapshot for the Connector with the given id, computed against the Connector's active flow. + * + * @param connectorId the id of the Connector + * @return the backlog entity; the wrapped DTO is null when the Connector returns {@link Optional#empty()} + * @throws BacklogReportingException if the Connector attempts to determine its backlog and fails + */ + BacklogEntity getConnectorBacklog(String connectorId) throws BacklogReportingException; + /** * Gets the processor status. * diff --git a/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/StandardNiFiServiceFacade.java b/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/StandardNiFiServiceFacade.java index be5f17459e77..ea5e94d41efd 100644 --- a/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/StandardNiFiServiceFacade.java +++ b/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/StandardNiFiServiceFacade.java @@ -72,12 +72,15 @@ import org.apache.nifi.cluster.manager.exception.IllegalNodeDeletionException; import org.apache.nifi.cluster.manager.exception.UnknownNodeException; import org.apache.nifi.cluster.protocol.NodeIdentifier; +import org.apache.nifi.components.Backlog; +import org.apache.nifi.components.BacklogReportingException; import org.apache.nifi.components.ConfigVerificationResult; import org.apache.nifi.components.ConfigurableComponent; import org.apache.nifi.components.DescribedValue; import org.apache.nifi.components.PropertyDescriptor; import org.apache.nifi.components.ValidationResult; import org.apache.nifi.components.Validator; +import org.apache.nifi.components.connector.BacklogReportingConnector; import org.apache.nifi.components.connector.Connector; import org.apache.nifi.components.connector.ConnectorNode; import org.apache.nifi.components.connector.ConnectorState; @@ -89,6 +92,7 @@ import org.apache.nifi.components.state.StateManagerProvider; import org.apache.nifi.components.state.StateMap; import org.apache.nifi.components.validation.ValidationState; +import org.apache.nifi.components.validation.ValidationStatus; import org.apache.nifi.connectable.Connectable; import org.apache.nifi.connectable.Connection; import org.apache.nifi.connectable.Funnel; @@ -161,6 +165,7 @@ import org.apache.nifi.metrics.jvm.JmxJvmMetrics; import org.apache.nifi.nar.ExtensionDefinition; import org.apache.nifi.nar.ExtensionManager; +import org.apache.nifi.nar.NarCloseable; import org.apache.nifi.nar.NarInstallRequest; import org.apache.nifi.nar.NarManager; import org.apache.nifi.nar.NarNode; @@ -344,6 +349,7 @@ import org.apache.nifi.web.api.entity.AffectedComponentEntity; import org.apache.nifi.web.api.entity.AllowableValueEntity; import org.apache.nifi.web.api.entity.AssetEntity; +import org.apache.nifi.web.api.entity.BacklogEntity; import org.apache.nifi.web.api.entity.BulletinEntity; import org.apache.nifi.web.api.entity.ClearBulletinsForGroupResultsEntity; import org.apache.nifi.web.api.entity.ClearBulletinsResultEntity; @@ -7674,6 +7680,75 @@ public ProcessorDiagnosticsEntity getProcessorDiagnostics(final String id) { return entityFactory.createProcessorDiagnosticsEntity(dto, revisionDto, permissionsDto, processorStatusDto, bulletins); } + @Override + public void verifyCanReportProcessorBacklog(final String processorId) { + processorDAO.verifyReportBacklog(processorId); + } + + @Override + public BacklogEntity getProcessorBacklog(final String processorId) throws BacklogReportingException { + logger.debug("Fetching backlog for Processor {}", processorId); + return toBacklogEntity(processorDAO.getBacklog(processorId)); + } + + @Override + public void verifyCanReportConnectorBacklog(final String connectorId) { + final ConnectorNode connectorNode = connectorDAO.getConnector(connectorId, ConnectorSyncMode.LOCAL_ONLY); + if (!(connectorNode.getConnector() instanceof BacklogReportingConnector)) { + throw new IllegalStateException("Cannot report backlog for Connector " + connectorId + + " because the Connector implementation does not implement BacklogReportingConnector"); + } + + final ValidationStatus validationStatus = connectorNode.getValidationStatus(); + if (validationStatus != ValidationStatus.VALID) { + throw new IllegalStateException("Cannot report backlog for Connector " + connectorId + + " because the Connector is not valid (Validation State is " + validationStatus + ")"); + } + + final ConnectorState currentState = connectorNode.getCurrentState(); + if (currentState == ConnectorState.PREPARING_FOR_UPDATE || currentState == ConnectorState.UPDATING + || currentState == ConnectorState.UPDATE_FAILED) { + throw new IllegalStateException("Cannot report backlog for Connector " + connectorId + + " because the Connector is in state " + currentState); + } + } + + @Override + public BacklogEntity getConnectorBacklog(final String connectorId) throws BacklogReportingException { + logger.debug("Fetching backlog for Connector {}", connectorId); + final ConnectorNode connectorNode = connectorDAO.getConnector(connectorId, ConnectorSyncMode.LOCAL_ONLY); + final Connector connector = connectorNode.getConnector(); + if (!(connector instanceof final BacklogReportingConnector backlogReportingConnector)) { + // verifyCanReportConnectorBacklog is the documented precondition for this method and the + // REST layer always calls it first. This defensive check guards against callers that + // bypass the verify step and keeps the failure mode aligned with the Processor path. + throw new IllegalStateException("Cannot report backlog for Connector " + connectorId + + " because the Connector implementation does not implement BacklogReportingConnector"); + } + + final ExtensionManager extensionManager = controllerFacade.getExtensionManager(); + final Optional backlog; + try (final NarCloseable ignored = NarCloseable.withComponentNarLoader(extensionManager, connector.getClass(), connectorNode.getIdentifier())) { + backlog = backlogReportingConnector.getBacklog(connectorNode.getActiveFlowContext()); + } catch (final BacklogReportingException e) { + logger.error("Failed to fetch backlog for Connector {}", connectorId, e); + throw e; + } catch (final RuntimeException e) { + logger.error("Failed to fetch backlog for Connector {}", connectorId, e); + throw new BacklogReportingException("Failed to fetch backlog for Connector " + connectorId, e); + } + + return toBacklogEntity(backlog); + } + + private BacklogEntity toBacklogEntity(final Optional backlog) { + final BacklogEntity entity = new BacklogEntity(); + if (backlog.isPresent()) { + entity.setBacklog(dtoFactory.createBacklogDto(backlog.get())); + } + return entity; + } + protected Collection populateFlowMetrics(FlowMetricsReportingStrategy flowMetricsStrategy) { // Include registries which are fully refreshed upon each invocation NiFiMetricsRegistry nifiMetricsRegistry = new NiFiMetricsRegistry(); diff --git a/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/api/ConnectorResource.java b/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/api/ConnectorResource.java index dd8cd557597d..59de34b8b7f8 100644 --- a/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/api/ConnectorResource.java +++ b/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/api/ConnectorResource.java @@ -42,6 +42,7 @@ import jakarta.ws.rs.core.Response; import jakarta.ws.rs.core.StreamingOutput; import org.apache.commons.lang3.StringUtils; +import org.apache.commons.lang3.exception.ExceptionUtils; import org.apache.nifi.asset.Asset; import org.apache.nifi.authorization.Authorizer; import org.apache.nifi.authorization.RequestAction; @@ -74,6 +75,8 @@ import org.apache.nifi.web.api.concurrent.StandardUpdateStep; import org.apache.nifi.web.api.concurrent.UpdateStep; import org.apache.nifi.web.api.dto.AssetDTO; +import org.apache.nifi.web.api.dto.BacklogDTO; +import org.apache.nifi.web.api.dto.BacklogRequestDTO; import org.apache.nifi.web.api.dto.BundleDTO; import org.apache.nifi.web.api.dto.ComponentStateDTO; import org.apache.nifi.web.api.dto.ConfigVerificationResultDTO; @@ -84,6 +87,8 @@ import org.apache.nifi.web.api.dto.search.SearchResultsDTO; import org.apache.nifi.web.api.entity.AssetEntity; import org.apache.nifi.web.api.entity.AssetsEntity; +import org.apache.nifi.web.api.entity.BacklogEntity; +import org.apache.nifi.web.api.entity.BacklogRequestEntity; import org.apache.nifi.web.api.entity.ComponentStateEntity; import org.apache.nifi.web.api.entity.ConfigurationStepEntity; import org.apache.nifi.web.api.entity.ConfigurationStepNamesEntity; @@ -118,6 +123,7 @@ import java.util.Map; import java.util.Set; import java.util.UUID; +import java.util.concurrent.TimeUnit; import java.util.function.Consumer; /** @@ -131,6 +137,7 @@ public class ConnectorResource extends ApplicationResource { private static final Logger logger = LoggerFactory.getLogger(ConnectorResource.class); private static final String VERIFICATION_REQUEST_TYPE = "verification-request"; private static final String PURGE_REQUEST_TYPE = "purge-request"; + private static final String BACKLOG_REQUEST_TYPE = "backlog-request"; private static final String FILENAME_HEADER = "Filename"; private static final String CONTENT_TYPE_HEADER = "Content-Type"; private static final String UPLOAD_CONTENT_TYPE = "application/octet-stream"; @@ -147,6 +154,8 @@ public class ConnectorResource extends ApplicationResource { private final RequestManager purgeRequestManager = new AsyncRequestManager<>(100, 1L, "Connector FlowFile Purge"); + private final RequestManager backlogRequestManager = new AsyncRequestManager<>(100, TimeUnit.MINUTES.toMillis(1L), "Connector Backlog Thread"); + @Context private ServletContext servletContext; @@ -2806,6 +2815,251 @@ public Response clearConnectorControllerServiceState( ); } + // ----------------- + // Backlog + // ----------------- + + /** + * Initiates an asynchronous request to determine the current backlog for the specified Connector. + * + *

Determining backlog typically requires the Connector to issue a request to an external source system, + * which may take an indeterminate amount of time or may never complete if the source system is unavailable. + * Rather than blocking the HTTP request thread, this endpoint immediately returns a {@link BacklogRequestEntity} + * and performs the determination in the background. The client should poll + * {@code GET /connectors/{id}/backlog-requests/{requestId}} until the request completes, and then issue a + * {@code DELETE} to the same URI. Deleting the request before it completes cancels the background + * determination.

+ * + * @param connectorId the connector id + * @return a BacklogRequestEntity describing the newly created request + */ + @POST + @Consumes(MediaType.WILDCARD) + @Produces(MediaType.APPLICATION_JSON) + @Path("/{id}/backlog-requests") + @Operation( + summary = "Initiates a request to determine the current backlog for a Connector", + responses = { + @ApiResponse(responseCode = "200", content = @Content(schema = @Schema(implementation = BacklogRequestEntity.class))), + @ApiResponse(responseCode = "400", description = "NiFi was unable to complete the request because it was invalid. The request should not be retried without modification."), + @ApiResponse(responseCode = "401", description = "Client could not be authenticated."), + @ApiResponse(responseCode = "403", description = "Client is not authorized to make this request."), + @ApiResponse(responseCode = "404", description = "The specified resource could not be found."), + @ApiResponse(responseCode = "409", description = "The request was valid but NiFi was not in the appropriate state to process it.") + }, + description = "This will initiate the process of determining the backlog for a Connector. This may be a long-running task. As a result, this endpoint will immediately return a " + + "BacklogRequestEntity, and the process of determining the backlog will occur asynchronously in the background. The client may then periodically poll the status of the " + + "request by issuing a GET request to /connectors/{id}/backlog-requests/{requestId}. Once the request is completed, the client is expected to issue a DELETE " + + "request to /connectors/{id}/backlog-requests/{requestId}.", + security = { + @SecurityRequirement(name = "Write - /connectors/{uuid}") + } + ) + public Response submitConnectorBacklogRequest( + @Parameter(description = "The connector id.", required = true) + @PathParam("id") final String connectorId) { + + if (isReplicateRequest()) { + return replicate(HttpMethod.POST); + } + + final ConnectorEntity requestConnectorEntity = new ConnectorEntity(); + requestConnectorEntity.setId(connectorId); + + // A replicated write request is a two-phase commit: the framework replicates the same POST to every + // node once for validation and again for execution. withWriteLock() only invokes the action below + // during the execution phase (or immediately, when not part of a two-phase commit), so the backlog + // request is submitted exactly once per node instead of once during validation and once during + // execution, which would otherwise create two requests and cause the second submission to conflict + // with the first. + return withWriteLock( + serviceFacade, + requestConnectorEntity, + lookup -> { + final Authorizable connector = lookup.getConnector(connectorId); + connector.authorize(authorizer, RequestAction.WRITE, NiFiUserUtils.getNiFiUser()); + }, + () -> serviceFacade.verifyCanReportConnectorBacklog(connectorId), + (connectorEntity) -> performAsyncBacklog(connectorEntity.getId(), NiFiUserUtils.getNiFiUser()) + ); + } + + /** + * Returns the current status of the specified backlog request. + * + * @param connectorId the connector id + * @param requestId the backlog request id + * @return a BacklogRequestEntity describing the current status of the request + */ + @GET + @Consumes(MediaType.WILDCARD) + @Produces(MediaType.APPLICATION_JSON) + @Path("/{id}/backlog-requests/{requestId}") + @Operation( + summary = "Returns the Backlog Request with the given ID", + responses = { + @ApiResponse(responseCode = "200", content = @Content(schema = @Schema(implementation = BacklogRequestEntity.class))), + @ApiResponse(responseCode = "400", description = "NiFi was unable to complete the request because it was invalid. The request should not be retried without modification."), + @ApiResponse(responseCode = "401", description = "Client could not be authenticated."), + @ApiResponse(responseCode = "403", description = "Client is not authorized to make this request."), + @ApiResponse(responseCode = "404", description = "The specified resource could not be found."), + @ApiResponse(responseCode = "409", description = "The request was valid but NiFi was not in the appropriate state to process it.") + }, + description = "Returns the Backlog Request with the given ID. Once a Backlog Request has been created by performing a POST to /connectors/{id}/backlog-requests, " + + "that request can subsequently be retrieved via this endpoint, and the request that is fetched will contain the updated state, such as percent complete, the " + + "current state of the request, and the backlog once the request is complete.", + security = { + @SecurityRequirement(name = "Only the user that submitted the request can get it") + } + ) + public Response getConnectorBacklogRequest( + @Parameter(description = "The connector id.", required = true) + @PathParam("id") final String connectorId, + @Parameter(description = "The ID of the Backlog Request") @PathParam("requestId") final String requestId) { + + if (isReplicateRequest()) { + return replicate(HttpMethod.GET); + } + + final NiFiUser user = NiFiUserUtils.getNiFiUser(); + + // request manager will ensure that the current user is the user that submitted this request + final AsynchronousWebRequest asyncRequest = backlogRequestManager.getRequest(BACKLOG_REQUEST_TYPE, requestId, user); + final BacklogRequestEntity entity = createBacklogRequestEntity(asyncRequest, requestId); + return generateOkResponse(entity).build(); + } + + /** + * Cancels and/or removes the specified backlog request. + * + * @param connectorId the connector id + * @param requestId the backlog request id + * @return a BacklogRequestEntity describing the final status of the request + */ + @DELETE + @Consumes(MediaType.WILDCARD) + @Produces(MediaType.APPLICATION_JSON) + @Path("/{id}/backlog-requests/{requestId}") + @Operation( + summary = "Deletes the Backlog Request with the given ID", + responses = { + @ApiResponse(responseCode = "200", content = @Content(schema = @Schema(implementation = BacklogRequestEntity.class))), + @ApiResponse(responseCode = "400", description = "NiFi was unable to complete the request because it was invalid. The request should not be retried without modification."), + @ApiResponse(responseCode = "401", description = "Client could not be authenticated."), + @ApiResponse(responseCode = "403", description = "Client is not authorized to make this request."), + @ApiResponse(responseCode = "404", description = "The specified resource could not be found."), + @ApiResponse(responseCode = "409", description = "The request was valid but NiFi was not in the appropriate state to process it.") + }, + description = "Deletes the Backlog Request with the given ID. After a request is created, it is expected that the client will properly clean up the request by DELETE'ing " + + "it once the backlog determination has completed. If the request is deleted before it completes, the background determination is cancelled.", + security = { + @SecurityRequirement(name = "Only the user that submitted the request can remove it") + } + ) + public Response deleteConnectorBacklogRequest( + @Parameter(description = "The connector id.", required = true) + @PathParam("id") final String connectorId, + @Parameter(description = "The ID of the Backlog Request") @PathParam("requestId") final String requestId) { + + if (isReplicateRequest()) { + return replicate(HttpMethod.DELETE); + } + + final NiFiUser user = NiFiUserUtils.getNiFiUser(); + final boolean twoPhaseRequest = isTwoPhaseRequest(httpServletRequest); + final boolean executionPhase = isExecutionPhase(httpServletRequest); + + if (!twoPhaseRequest || executionPhase) { + // request manager will ensure that the current user is the user that submitted this request + final AsynchronousWebRequest asyncRequest = backlogRequestManager.removeRequest(BACKLOG_REQUEST_TYPE, requestId, user); + + if (!asyncRequest.isComplete()) { + asyncRequest.cancel(); + } + + final BacklogRequestEntity entity = createBacklogRequestEntity(asyncRequest, requestId); + return generateOkResponse(entity).build(); + } + + if (isValidationPhase(httpServletRequest)) { + backlogRequestManager.getRequest(BACKLOG_REQUEST_TYPE, requestId, user); + return generateContinueResponse().build(); + } else if (isCancellationPhase(httpServletRequest)) { + return generateOkResponse().build(); + } else { + throw new IllegalStateException("This request does not appear to be part of the two phase commit."); + } + } + + private Response performAsyncBacklog(final String connectorId, final NiFiUser user) { + final String requestId = generateUuid(); + logger.debug("Generated Backlog Request with ID {} for Connector {}", requestId, connectorId); + + final List updateSteps = Collections.singletonList(new StandardUpdateStep("Determining Backlog")); + final AsynchronousWebRequest request = new StandardAsynchronousWebRequest<>(requestId, connectorId, connectorId, user, updateSteps); + + final Consumer> backlogTask = asyncRequest -> { + final Thread currentThread = Thread.currentThread(); + asyncRequest.setCancelCallback(currentThread::interrupt); + + try { + final BacklogEntity backlogEntity = serviceFacade.getConnectorBacklog(connectorId); + asyncRequest.markStepComplete(backlogEntity.getBacklog()); + } catch (final Exception e) { + logger.error("Failed to determine backlog for Connector {}", connectorId, e); + asyncRequest.fail(describeBacklogFailure(e)); + } + }; + + backlogRequestManager.submitRequest(BACKLOG_REQUEST_TYPE, requestId, request, backlogTask); + + final BacklogRequestEntity resultsEntity = createBacklogRequestEntity(request, requestId); + return generateOkResponse(resultsEntity).build(); + } + + /** + * Builds a failure explanation for a backlog determination that includes both the top-level exception + * message and, when it differs, the message of the deepest cause in the exception chain. The top-level + * message alone is often a generic wrapper while the root cause carries the actionable detail (for + * example a Kafka {@code TimeoutException} explaining which call timed out), so both are surfaced to + * the user. + * + * @param e the exception raised while determining backlog + * @return an explanation combining the top-level message with the root cause message, when they differ + */ + private static String describeBacklogFailure(final Exception e) { + final String topLevelMessage = e.getMessage() != null ? e.getMessage() : e.getClass().getSimpleName(); + if (e.getCause() == null) { + return topLevelMessage; + } + + final String rootCauseMessage = ExceptionUtils.getRootCauseMessage(e); + if (rootCauseMessage == null || rootCauseMessage.isBlank()) { + return topLevelMessage; + } + return topLevelMessage + " (" + rootCauseMessage + ")"; + } + + private BacklogRequestEntity createBacklogRequestEntity(final AsynchronousWebRequest asyncRequest, final String requestId) { + final String connectorId = asyncRequest.getRequest(); + + final BacklogRequestDTO dto = new BacklogRequestDTO(); + dto.setComponentId(connectorId); + dto.setRequestId(requestId); + dto.setUri(generateResourceUri("connectors", connectorId, "backlog-requests", requestId)); + dto.setSubmissionTime(asyncRequest.getLastUpdated()); + dto.setLastUpdated(asyncRequest.getLastUpdated()); + dto.setComplete(asyncRequest.isComplete()); + dto.setFailureReason(asyncRequest.getFailureReason()); + dto.setPercentCompleted(asyncRequest.getPercentComplete()); + dto.setState(asyncRequest.getState()); + dto.setBacklog(asyncRequest.getResults()); + + final BacklogRequestEntity entity = new BacklogRequestEntity(); + entity.setRequest(dto); + return entity; + } + // ----------------- // setters // ----------------- diff --git a/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/api/ProcessorResource.java b/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/api/ProcessorResource.java index fd0aa5eec4c5..18dd9ed394d3 100644 --- a/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/api/ProcessorResource.java +++ b/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/api/ProcessorResource.java @@ -39,6 +39,7 @@ import jakarta.ws.rs.core.MediaType; import jakarta.ws.rs.core.Response; import org.apache.commons.lang3.StringUtils; +import org.apache.commons.lang3.exception.ExceptionUtils; import org.apache.nifi.authorization.AuthorizeComponentAnalysis; import org.apache.nifi.authorization.AuthorizeComponentReference; import org.apache.nifi.authorization.AuthorizeConfigVerification; @@ -63,6 +64,8 @@ import org.apache.nifi.web.api.concurrent.StandardAsynchronousWebRequest; import org.apache.nifi.web.api.concurrent.StandardUpdateStep; import org.apache.nifi.web.api.concurrent.UpdateStep; +import org.apache.nifi.web.api.dto.BacklogDTO; +import org.apache.nifi.web.api.dto.BacklogRequestDTO; import org.apache.nifi.web.api.dto.BundleDTO; import org.apache.nifi.web.api.dto.ComponentStateDTO; import org.apache.nifi.web.api.dto.ConfigVerificationResultDTO; @@ -72,6 +75,8 @@ import org.apache.nifi.web.api.dto.ProcessorDTO; import org.apache.nifi.web.api.dto.PropertyDescriptorDTO; import org.apache.nifi.web.api.dto.VerifyConfigRequestDTO; +import org.apache.nifi.web.api.entity.BacklogEntity; +import org.apache.nifi.web.api.entity.BacklogRequestEntity; import org.apache.nifi.web.api.entity.ClearBulletinsRequestEntity; import org.apache.nifi.web.api.entity.ClearBulletinsResultEntity; import org.apache.nifi.web.api.entity.ComponentStateEntity; @@ -111,6 +116,10 @@ public class ProcessorResource extends ApplicationResource { private RequestManager> configVerificationRequestManager = new AsyncRequestManager<>(100, TimeUnit.MINUTES.toMillis(1L), "Verify Processor Config Thread"); + private static final String BACKLOG_REQUEST_TYPE = "backlog-request"; + private final RequestManager backlogRequestManager = + new AsyncRequestManager<>(100, TimeUnit.MINUTES.toMillis(1L), "Processor Backlog Thread"); + private NiFiServiceFacade serviceFacade; private Authorizer authorizer; @@ -360,6 +369,244 @@ public Response getProcessorDiagnostics( return generateOkResponse(entity).build(); } + /** + * Initiates an asynchronous request to determine the current backlog for the specified processor. + * + *

Determining backlog typically requires the Processor to issue a request to an external source system, + * which may take an indeterminate amount of time or may never complete if the source system is unavailable. + * Rather than blocking the HTTP request thread, this endpoint immediately returns a {@link BacklogRequestEntity} + * and performs the determination in the background. The client should poll + * {@code GET /processors/{id}/backlog-requests/{requestId}} until the request completes, and then issue a + * {@code DELETE} to the same URI. Deleting the request before it completes cancels the background + * determination.

+ * + * @param id the processor id + * @return a BacklogRequestEntity describing the newly created request + */ + @POST + @Consumes(MediaType.WILDCARD) + @Produces(MediaType.APPLICATION_JSON) + @Path("/{id}/backlog-requests") + @Operation( + summary = "Initiates a request to determine the current backlog for a processor that implements BacklogReportingProcessor", + responses = { + @ApiResponse(responseCode = "200", content = @Content(schema = @Schema(implementation = BacklogRequestEntity.class))), + @ApiResponse(responseCode = "400", description = "NiFi was unable to complete the request because it was invalid. The request should not be retried without modification."), + @ApiResponse(responseCode = "401", description = "Client could not be authenticated."), + @ApiResponse(responseCode = "403", description = "Client is not authorized to make this request."), + @ApiResponse(responseCode = "404", description = "The specified resource could not be found."), + @ApiResponse(responseCode = "409", description = "The request was valid but NiFi was not in the appropriate state to process it.") + }, + description = "This will initiate the process of determining the backlog for a Processor. This may be a long-running task. As a result, this endpoint will immediately return a " + + "BacklogRequestEntity, and the process of determining the backlog will occur asynchronously in the background. The client may then periodically poll the status of the " + + "request by issuing a GET request to /processors/{processorId}/backlog-requests/{requestId}. Once the request is completed, the client is expected to issue a DELETE " + + "request to /processors/{processorId}/backlog-requests/{requestId}.", + security = { + @SecurityRequirement(name = "Write - /processors/{uuid}") + } + ) + public Response submitProcessorBacklogRequest( + @Parameter(description = "The processor id.", required = true) @PathParam("id") final String id) { + + if (isReplicateRequest()) { + return replicate(HttpMethod.POST); + } + + final ProcessorEntity requestProcessorEntity = new ProcessorEntity(); + requestProcessorEntity.setId(id); + + // A replicated write request is a two-phase commit: the framework replicates the same POST to every + // node once for validation and again for execution. withWriteLock() only invokes the action below + // during the execution phase (or immediately, when not part of a two-phase commit), so the backlog + // request is submitted exactly once per node instead of once during validation and once during + // execution, which would otherwise create two requests and cause the second submission to conflict + // with the first. + return withWriteLock( + serviceFacade, + requestProcessorEntity, + lookup -> { + final Authorizable processor = lookup.getProcessor(id).getAuthorizable(); + processor.authorize(authorizer, RequestAction.WRITE, NiFiUserUtils.getNiFiUser()); + }, + () -> serviceFacade.verifyCanReportProcessorBacklog(id), + (processorEntity) -> performAsyncBacklog(processorEntity.getId(), NiFiUserUtils.getNiFiUser()) + ); + } + + /** + * Returns the current status of the specified backlog request. + * + * @param id the processor id + * @param requestId the backlog request id + * @return a BacklogRequestEntity describing the current status of the request + */ + @GET + @Consumes(MediaType.WILDCARD) + @Produces(MediaType.APPLICATION_JSON) + @Path("/{id}/backlog-requests/{requestId}") + @Operation( + summary = "Returns the Backlog Request with the given ID", + responses = { + @ApiResponse(responseCode = "200", content = @Content(schema = @Schema(implementation = BacklogRequestEntity.class))), + @ApiResponse(responseCode = "400", description = "NiFi was unable to complete the request because it was invalid. The request should not be retried without modification."), + @ApiResponse(responseCode = "401", description = "Client could not be authenticated."), + @ApiResponse(responseCode = "403", description = "Client is not authorized to make this request."), + @ApiResponse(responseCode = "404", description = "The specified resource could not be found."), + @ApiResponse(responseCode = "409", description = "The request was valid but NiFi was not in the appropriate state to process it.") + }, + description = "Returns the Backlog Request with the given ID. Once a Backlog Request has been created by performing a POST to /processors/{processorId}/backlog-requests, " + + "that request can subsequently be retrieved via this endpoint, and the request that is fetched will contain the updated state, such as percent complete, the " + + "current state of the request, and the backlog once the request is complete.", + security = { + @SecurityRequirement(name = "Only the user that submitted the request can get it") + } + ) + public Response getBacklogRequest( + @Parameter(description = "The processor id.", required = true) @PathParam("id") final String id, + @Parameter(description = "The ID of the Backlog Request") @PathParam("requestId") final String requestId) { + + if (isReplicateRequest()) { + return replicate(HttpMethod.GET); + } + + final NiFiUser user = NiFiUserUtils.getNiFiUser(); + + // request manager will ensure that the current user is the user that submitted this request + final AsynchronousWebRequest asyncRequest = backlogRequestManager.getRequest(BACKLOG_REQUEST_TYPE, requestId, user); + final BacklogRequestEntity entity = createBacklogRequestEntity(asyncRequest, requestId); + return generateOkResponse(entity).build(); + } + + /** + * Cancels and/or removes the specified backlog request. + * + * @param id the processor id + * @param requestId the backlog request id + * @return a BacklogRequestEntity describing the final status of the request + */ + @DELETE + @Consumes(MediaType.WILDCARD) + @Produces(MediaType.APPLICATION_JSON) + @Path("/{id}/backlog-requests/{requestId}") + @Operation( + summary = "Deletes the Backlog Request with the given ID", + responses = { + @ApiResponse(responseCode = "200", content = @Content(schema = @Schema(implementation = BacklogRequestEntity.class))), + @ApiResponse(responseCode = "400", description = "NiFi was unable to complete the request because it was invalid. The request should not be retried without modification."), + @ApiResponse(responseCode = "401", description = "Client could not be authenticated."), + @ApiResponse(responseCode = "403", description = "Client is not authorized to make this request."), + @ApiResponse(responseCode = "404", description = "The specified resource could not be found."), + @ApiResponse(responseCode = "409", description = "The request was valid but NiFi was not in the appropriate state to process it.") + }, + description = "Deletes the Backlog Request with the given ID. After a request is created, it is expected that the client will properly clean up the request by DELETE'ing " + + "it once the backlog determination has completed. If the request is deleted before it completes, the background determination is cancelled.", + security = { + @SecurityRequirement(name = "Only the user that submitted the request can remove it") + } + ) + public Response deleteBacklogRequest( + @Parameter(description = "The processor id.", required = true) @PathParam("id") final String id, + @Parameter(description = "The ID of the Backlog Request") @PathParam("requestId") final String requestId) { + + if (isReplicateRequest()) { + return replicate(HttpMethod.DELETE); + } + + final NiFiUser user = NiFiUserUtils.getNiFiUser(); + final boolean twoPhaseRequest = isTwoPhaseRequest(httpServletRequest); + final boolean executionPhase = isExecutionPhase(httpServletRequest); + + if (!twoPhaseRequest || executionPhase) { + // request manager will ensure that the current user is the user that submitted this request + final AsynchronousWebRequest asyncRequest = backlogRequestManager.removeRequest(BACKLOG_REQUEST_TYPE, requestId, user); + + if (!asyncRequest.isComplete()) { + asyncRequest.cancel(); + } + + final BacklogRequestEntity entity = createBacklogRequestEntity(asyncRequest, requestId); + return generateOkResponse(entity).build(); + } + + if (isValidationPhase(httpServletRequest)) { + backlogRequestManager.getRequest(BACKLOG_REQUEST_TYPE, requestId, user); + return generateContinueResponse().build(); + } else if (isCancellationPhase(httpServletRequest)) { + return generateOkResponse().build(); + } else { + throw new IllegalStateException("This request does not appear to be part of the two phase commit."); + } + } + + private Response performAsyncBacklog(final String processorId, final NiFiUser user) { + final String requestId = generateUuid(); + logger.debug("Generated Backlog Request with ID {} for Processor {}", requestId, processorId); + + final List updateSteps = Collections.singletonList(new StandardUpdateStep("Determining Backlog")); + final AsynchronousWebRequest request = new StandardAsynchronousWebRequest<>(requestId, processorId, processorId, user, updateSteps); + + final Consumer> backlogTask = asyncRequest -> { + final Thread currentThread = Thread.currentThread(); + asyncRequest.setCancelCallback(currentThread::interrupt); + + try { + final BacklogEntity backlogEntity = serviceFacade.getProcessorBacklog(processorId); + asyncRequest.markStepComplete(backlogEntity.getBacklog()); + } catch (final Exception e) { + logger.error("Failed to determine backlog for Processor {}", processorId, e); + asyncRequest.fail(describeBacklogFailure(e)); + } + }; + + backlogRequestManager.submitRequest(BACKLOG_REQUEST_TYPE, requestId, request, backlogTask); + + final BacklogRequestEntity resultsEntity = createBacklogRequestEntity(request, requestId); + return generateOkResponse(resultsEntity).build(); + } + + /** + * Builds a failure explanation for a backlog determination that includes both the top-level exception + * message and, when it differs, the message of the deepest cause in the exception chain. The top-level + * message alone is often a generic wrapper (for example {@code "Failed to determine Kafka backlog"}) + * while the root cause carries the actionable detail (for example a Kafka {@code TimeoutException} + * explaining which call timed out), so both are surfaced to the user. + * + * @param e the exception raised while determining backlog + * @return an explanation combining the top-level message with the root cause message, when they differ + */ + private static String describeBacklogFailure(final Exception e) { + final String topLevelMessage = e.getMessage() != null ? e.getMessage() : e.getClass().getSimpleName(); + if (e.getCause() == null) { + return topLevelMessage; + } + + final String rootCauseMessage = ExceptionUtils.getRootCauseMessage(e); + if (rootCauseMessage == null || rootCauseMessage.isBlank()) { + return topLevelMessage; + } + return topLevelMessage + " (" + rootCauseMessage + ")"; + } + + private BacklogRequestEntity createBacklogRequestEntity(final AsynchronousWebRequest asyncRequest, final String requestId) { + final String processorId = asyncRequest.getRequest(); + + final BacklogRequestDTO dto = new BacklogRequestDTO(); + dto.setComponentId(processorId); + dto.setRequestId(requestId); + dto.setUri(generateResourceUri("processors", processorId, "backlog-requests", requestId)); + dto.setSubmissionTime(asyncRequest.getLastUpdated()); + dto.setLastUpdated(asyncRequest.getLastUpdated()); + dto.setComplete(asyncRequest.isComplete()); + dto.setFailureReason(asyncRequest.getFailureReason()); + dto.setPercentCompleted(asyncRequest.getPercentComplete()); + dto.setState(asyncRequest.getState()); + dto.setBacklog(asyncRequest.getResults()); + + final BacklogRequestEntity entity = new BacklogRequestEntity(); + entity.setRequest(dto); + return entity; + } + /** * Returns the descriptor for the specified property. * diff --git a/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/api/concurrent/StandardAsynchronousWebRequest.java b/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/api/concurrent/StandardAsynchronousWebRequest.java index 341dc9a9b000..02a77552c5b3 100644 --- a/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/api/concurrent/StandardAsynchronousWebRequest.java +++ b/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/api/concurrent/StandardAsynchronousWebRequest.java @@ -145,6 +145,18 @@ public NiFiUser getUser() { @Override public synchronized void fail(final String explanation) { + // Once the request has been cancelled, its failure reason is fixed. Without this check, a background + // task that is in the middle of handling its own interruption when cancel() runs could call fail() + // with its own explanation immediately afterward, overwriting the "cancelled by user" explanation the + // caller of cancel() expects to see. + if (isCancelled()) { + return; + } + + applyFailure(explanation); + } + + private void applyFailure(final String explanation) { this.failureReason = Objects.requireNonNull(explanation); this.complete = true; this.results = null; @@ -168,12 +180,19 @@ public T getResults() { @Override public void cancel() { - this.cancelled = true; - percentComplete = 100; - fail("Request cancelled by user"); + // The cancel callback is invoked outside the synchronized block so that a callback which itself + // interacts with this request (for example, by checking isComplete() from another thread) cannot + // be blocked waiting on a lock held by this method. + final Runnable callback; + synchronized (this) { + this.cancelled = true; + percentComplete = 100; + applyFailure("Request cancelled by user"); + callback = cancelCallback; + } - if (cancelCallback != null) { - cancelCallback.run(); + if (callback != null) { + callback.run(); } } diff --git a/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/api/dto/DtoFactory.java b/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/api/dto/DtoFactory.java index 1df40ad87db0..ac1c8ade06a1 100644 --- a/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/api/dto/DtoFactory.java +++ b/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/api/dto/DtoFactory.java @@ -60,6 +60,7 @@ import org.apache.nifi.cluster.manager.StatusMerger; import org.apache.nifi.cluster.protocol.NodeIdentifier; import org.apache.nifi.components.AllowableValue; +import org.apache.nifi.components.Backlog; import org.apache.nifi.components.ConfigVerificationResult; import org.apache.nifi.components.DescribedValue; import org.apache.nifi.components.PropertyDependency; @@ -283,6 +284,7 @@ import java.text.Collator; import java.text.NumberFormat; import java.time.Duration; +import java.time.Instant; import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; @@ -301,6 +303,7 @@ import java.util.Map.Entry; import java.util.Objects; import java.util.Optional; +import java.util.OptionalLong; import java.util.Set; import java.util.TimeZone; import java.util.TreeMap; @@ -317,6 +320,8 @@ public final class DtoFactory { private static final Comparator CLASS_NAME_COMPARATOR = (class1, class2) -> Collator.getInstance(Locale.US).compare(class1.getSimpleName(), class2.getSimpleName()); public static final String SENSITIVE_VALUE_MASK = "********"; + private static final long BACKLOG_NOW_WINDOW_MILLISECONDS = 3000L; + private BulletinRepository bulletinRepository; private ControllerServiceProvider controllerServiceProvider; private EntityFactory entityFactory; @@ -3387,6 +3392,66 @@ public Set fromDocumentedTypes(final Set return documentedTypes; } + /** + * Maps a {@link Backlog} report from the nifi-api into a {@link BacklogDTO} suitable for REST + * clients. Each present numeric dimension is copied verbatim and accompanied by a formatted + * string (e.g., {@code "4.89 GB"}, {@code "1,025"}); absent dimensions map to {@code null} on + * both the raw and formatted fields, which the JSON serializer omits from the response. The + * {@code lastCaughtUp} instant is serialized as its {@link Instant#toString()} form (ISO-8601 + * UTC) and accompanied by a relative-time string (e.g., {@code "5 mins ago"}, or {@code "now"} + * when all numeric dimensions are zero/absent and the timestamp falls within + * {@link #BACKLOG_NOW_WINDOW_MILLISECONDS} of the current server clock). Precision is the + * {@link Backlog.Precision#name() Backlog.Precision name}. + * + * @param backlog the source backlog + * @return a BacklogDTO mirroring the dimensions reported by {@code backlog}, or {@code null} if {@code backlog} is null + */ + public BacklogDTO createBacklogDto(final Backlog backlog) { + if (backlog == null) { + return null; + } + + final BacklogDTO dto = new BacklogDTO(); + dto.setPrecision(backlog.getPrecision().name()); + + if (backlog.getFlowFileCount().isPresent()) { + final long flowFileCount = backlog.getFlowFileCount().getAsLong(); + dto.setFlowFileCount(flowFileCount); + dto.setFormattedFlowFileCount(FormatUtils.formatCount(flowFileCount)); + } + if (backlog.getByteCount().isPresent()) { + final long byteCount = backlog.getByteCount().getAsLong(); + dto.setByteCount(byteCount); + dto.setFormattedByteCount(FormatUtils.formatDataSize(byteCount)); + } + if (backlog.getRecordCount().isPresent()) { + final long recordCount = backlog.getRecordCount().getAsLong(); + dto.setRecordCount(recordCount); + dto.setFormattedRecordCount(FormatUtils.formatCount(recordCount)); + } + if (backlog.getLastCaughtUp().isPresent()) { + final Instant lastCaughtUp = backlog.getLastCaughtUp().get(); + dto.setLastCaughtUp(lastCaughtUp.toString()); + final boolean numericallyCaughtUp = isZeroOrAbsent(backlog.getFlowFileCount()) + && isZeroOrAbsent(backlog.getByteCount()) + && isZeroOrAbsent(backlog.getRecordCount()); + dto.setFormattedLastCaughtUp(computeFormattedLastCaughtUp(lastCaughtUp, numericallyCaughtUp)); + } + return dto; + } + + private static String computeFormattedLastCaughtUp(final Instant lastCaughtUp, final boolean numericallyCaughtUp) { + final Instant now = Instant.now(); + if (numericallyCaughtUp && Math.abs(now.toEpochMilli() - lastCaughtUp.toEpochMilli()) <= BACKLOG_NOW_WINDOW_MILLISECONDS) { + return "now"; + } + return FormatUtils.formatRelativeTime(lastCaughtUp, now); + } + + private static boolean isZeroOrAbsent(final OptionalLong value) { + return value.isEmpty() || value.getAsLong() == 0L; + } + public ProcessorDTO createProcessorDto(final ProcessorNode node) { return createProcessorDto(node, false); } @@ -3423,6 +3488,7 @@ private ProcessorDTO createProcessorDto(final ProcessorNode node, final boolean dto.setInputRequirement(node.getInputRequirement().name()); dto.setPersistsState(processorClass.isAnnotationPresent(Stateful.class)); dto.setSupportsSensitiveDynamicProperties(node.isSupportsSensitiveDynamicProperties()); + dto.setSupportsBacklogReporting(node.supportsBacklogReporting()); dto.setRestricted(false); dto.setDeprecated(node.isDeprecated()); dto.setExecutionNodeRestricted(node.isExecutionNodeRestricted()); @@ -4523,6 +4589,7 @@ public ProcessorDTO copy(final ProcessorDTO original) { copy.setSupportsParallelProcessing(original.getSupportsParallelProcessing()); copy.setSupportsBatching(original.getSupportsBatching()); copy.setSupportsSensitiveDynamicProperties(original.getSupportsSensitiveDynamicProperties()); + copy.setSupportsBacklogReporting(original.getSupportsBacklogReporting()); copy.setPersistsState(original.getPersistsState()); copy.setExecutionNodeRestricted(original.isExecutionNodeRestricted()); copy.setExtensionMissing(original.getExtensionMissing()); diff --git a/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/connector/authorization/AuthorizingProcessorFacade.java b/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/connector/authorization/AuthorizingProcessorFacade.java index 14c8c4c6fedb..22a0e7f1c346 100644 --- a/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/connector/authorization/AuthorizingProcessorFacade.java +++ b/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/connector/authorization/AuthorizingProcessorFacade.java @@ -108,7 +108,7 @@ public boolean reportsBacklog() { @Override public Optional getBacklog() throws BacklogReportingException { - authContext.authorizeRead(); + authContext.authorizeWrite(); return delegate.getBacklog(); } diff --git a/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/dao/ProcessorDAO.java b/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/dao/ProcessorDAO.java index a5bed8eb1269..b6a569368350 100644 --- a/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/dao/ProcessorDAO.java +++ b/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/dao/ProcessorDAO.java @@ -16,6 +16,8 @@ */ package org.apache.nifi.web.dao; +import org.apache.nifi.components.Backlog; +import org.apache.nifi.components.BacklogReportingException; import org.apache.nifi.components.state.Scope; import org.apache.nifi.components.state.StateMap; import org.apache.nifi.controller.ProcessorNode; @@ -25,6 +27,7 @@ import java.util.List; import java.util.Map; +import java.util.Optional; import java.util.Set; public interface ProcessorDAO { @@ -81,6 +84,31 @@ public interface ProcessorDAO { */ void verifyConfigVerification(String processorId); + /** + * Verifies that the specified processor is in an appropriate state to be asked for a backlog report. + * + * @param processorId the ID of the Processor + */ + void verifyReportBacklog(String processorId); + + /** + * Returns the backlog reported by the specified processor. + * + *

+ * Implementations are responsible for first verifying that the processor is in an appropriate state + * (per {@link #verifyReportBacklog(String)}), then building a fresh {@code ProcessContext} for the + * processor and threading that context through to {@link ProcessorNode#getReportedBacklog(org.apache.nifi.processor.ProcessContext)}. + * A fresh context per call is intentional so that the processor's backlog query is isolated from + * its running state and can be answered even when the processor is stopped. + *

+ * + * @param processorId the ID of the processor + * @return the processor's reported backlog, or {@link Optional#empty()} if the processor does not + * implement {@code BacklogReportingProcessor} or has nothing to report + * @throws BacklogReportingException if the processor attempted to determine its backlog and failed + */ + Optional getBacklog(String processorId) throws BacklogReportingException; + /** * Verifies that the specified processor can be terminated at this time * diff --git a/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/dao/impl/StandardProcessorDAO.java b/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/dao/impl/StandardProcessorDAO.java index 3b386f244630..fe714026ef0f 100644 --- a/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/dao/impl/StandardProcessorDAO.java +++ b/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/dao/impl/StandardProcessorDAO.java @@ -18,6 +18,8 @@ import org.apache.commons.lang3.StringUtils; import org.apache.nifi.bundle.BundleCoordinate; +import org.apache.nifi.components.Backlog; +import org.apache.nifi.components.BacklogReportingException; import org.apache.nifi.components.ConfigVerificationResult; import org.apache.nifi.components.ConfigurableComponent; import org.apache.nifi.components.connector.ConnectorState; @@ -41,6 +43,7 @@ import org.apache.nifi.logging.repository.NopLogRepository; import org.apache.nifi.nar.ExtensionManager; import org.apache.nifi.processor.ProcessContext; +import org.apache.nifi.processor.Processor; import org.apache.nifi.processor.Relationship; import org.apache.nifi.processor.SimpleProcessLogger; import org.apache.nifi.processor.StandardProcessContext; @@ -67,6 +70,7 @@ import java.util.HashSet; import java.util.List; import java.util.Map; +import java.util.Optional; import java.util.Set; import java.util.concurrent.RejectedExecutionException; import java.util.concurrent.TimeUnit; @@ -402,6 +406,25 @@ public void verifyConfigVerification(final String processorId) { processor.verifyCanPerformVerification(); } + @Override + public void verifyReportBacklog(final String processorId) { + locateProcessor(processorId).verifyCanReportBacklog(); + } + + @Override + public Optional getBacklog(final String processorId) throws BacklogReportingException { + // Callers are required to invoke verifyReportBacklog first (the REST layer does this via + // NiFiServiceFacade.verifyCanReportProcessorBacklog). Re-verifying here would duplicate the + // capability/state check that the verify step already performed. + final ProcessorNode processor = locateProcessor(processorId); + final Processor componentProcessor = processor.getProcessor(); + final Class componentClass = componentProcessor == null ? null : componentProcessor.getClass(); + final ProcessContext processContext = new StandardProcessContext(processor, flowController.getControllerServiceProvider(), + flowController.getStateManagerProvider().getStateManager(processor.getIdentifier(), componentClass), () -> false, flowController); + + return processor.getReportedBacklog(processContext); + } + @Override public void verifyUpdate(final ProcessorDTO processorDTO) { verifyUpdate(locateProcessor(processorDTO.getId()), processorDTO); diff --git a/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/test/java/org/apache/nifi/web/StandardNiFiServiceFacadeTest.java b/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/test/java/org/apache/nifi/web/StandardNiFiServiceFacadeTest.java index 4cfce89af711..6ca161c4dbe7 100644 --- a/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/test/java/org/apache/nifi/web/StandardNiFiServiceFacadeTest.java +++ b/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/test/java/org/apache/nifi/web/StandardNiFiServiceFacadeTest.java @@ -36,6 +36,8 @@ import org.apache.nifi.authorization.user.NiFiUserDetails; import org.apache.nifi.authorization.user.StandardNiFiUser; import org.apache.nifi.authorization.user.StandardNiFiUser.Builder; +import org.apache.nifi.components.Backlog; +import org.apache.nifi.components.BacklogReportingException; import org.apache.nifi.components.PropertyDescriptor; import org.apache.nifi.components.connector.ConnectorNode; import org.apache.nifi.components.connector.ConnectorSyncMode; @@ -106,6 +108,7 @@ import org.apache.nifi.util.NiFiProperties; import org.apache.nifi.validation.RuleViolation; import org.apache.nifi.validation.RuleViolationsManager; +import org.apache.nifi.web.api.dto.BacklogDTO; import org.apache.nifi.web.api.dto.BulletinBoardDTO; import org.apache.nifi.web.api.dto.BulletinQueryDTO; import org.apache.nifi.web.api.dto.ComponentStateDTO; @@ -126,6 +129,7 @@ import org.apache.nifi.web.api.dto.status.StatusHistoryDTO; import org.apache.nifi.web.api.entity.ActionEntity; import org.apache.nifi.web.api.entity.AffectedComponentEntity; +import org.apache.nifi.web.api.entity.BacklogEntity; import org.apache.nifi.web.api.entity.ClearBulletinsForGroupResultsEntity; import org.apache.nifi.web.api.entity.ClearBulletinsResultEntity; import org.apache.nifi.web.api.entity.ConnectorEntity; @@ -144,6 +148,7 @@ import org.apache.nifi.web.dao.ConnectorManagedComponentLookup; import org.apache.nifi.web.dao.FlowRegistryDAO; import org.apache.nifi.web.dao.ProcessGroupDAO; +import org.apache.nifi.web.dao.ProcessorDAO; import org.apache.nifi.web.dao.RemoteProcessGroupDAO; import org.apache.nifi.web.dao.UserDAO; import org.apache.nifi.web.dao.UserGroupDAO; @@ -178,6 +183,7 @@ import java.util.LinkedHashMap; import java.util.List; import java.util.Map; +import java.util.Optional; import java.util.Set; import java.util.UUID; import java.util.concurrent.TimeUnit; @@ -2577,4 +2583,138 @@ public void testCreateFlowBranchPropagatesRegistryErrors() throws IOException, F verify(processGroup, never()).synchronizeWithFlowRegistry(any(FlowManager.class)); } + + // ----------------- + // Backlog Tests + // ----------------- + + @Test + public void testGetProcessorBacklog() throws BacklogReportingException { + final String processorId = "processor-backlog"; + final ProcessorDAO processorDAO = mock(ProcessorDAO.class); + final DtoFactory dtoFactory = mock(DtoFactory.class); + serviceFacade.setProcessorDAO(processorDAO); + serviceFacade.setDtoFactory(dtoFactory); + + final Backlog reportedBacklog = Backlog.builder().records(42L).precision(Backlog.Precision.EXACT).build(); + when(processorDAO.getBacklog(processorId)).thenReturn(Optional.of(reportedBacklog)); + + final BacklogDTO mappedDto = new BacklogDTO(); + mappedDto.setRecordCount(42L); + mappedDto.setPrecision("EXACT"); + when(dtoFactory.createBacklogDto(reportedBacklog)).thenReturn(mappedDto); + + final BacklogEntity entity = serviceFacade.getProcessorBacklog(processorId); + + verify(processorDAO).getBacklog(processorId); + assertNotNull(entity); + assertNotNull(entity.getBacklog()); + assertEquals(42L, entity.getBacklog().getRecordCount()); + assertEquals("EXACT", entity.getBacklog().getPrecision()); + } + + @Test + public void testGetProcessorBacklogMapsCaughtUpDirectly() throws BacklogReportingException { + final String processorId = "processor-caught-up"; + final ProcessorDAO processorDAO = mock(ProcessorDAO.class); + serviceFacade.setProcessorDAO(processorDAO); + serviceFacade.setDtoFactory(new DtoFactory()); + + final Backlog caughtUpBacklog = Backlog.caughtUp(); + when(processorDAO.getBacklog(processorId)).thenReturn(Optional.of(caughtUpBacklog)); + + final BacklogEntity entity = serviceFacade.getProcessorBacklog(processorId); + + assertNotNull(entity); + final BacklogDTO dto = entity.getBacklog(); + assertNotNull(dto); + assertEquals(0L, dto.getFlowFileCount()); + assertEquals(0L, dto.getByteCount()); + assertEquals(0L, dto.getRecordCount()); + assertEquals("0", dto.getFormattedFlowFileCount()); + assertEquals("0 bytes", dto.getFormattedByteCount()); + assertEquals("0", dto.getFormattedRecordCount()); + assertEquals("EXACT", dto.getPrecision()); + assertNotNull(dto.getLastCaughtUp()); + assertNotNull(dto.getFormattedLastCaughtUp()); + } + + @Test + public void testGetProcessorBacklogMapsEmptyOptionalToNullDto() throws BacklogReportingException { + final String processorId = "processor-empty"; + final ProcessorDAO processorDAO = mock(ProcessorDAO.class); + final DtoFactory dtoFactory = mock(DtoFactory.class); + serviceFacade.setProcessorDAO(processorDAO); + serviceFacade.setDtoFactory(dtoFactory); + + when(processorDAO.getBacklog(processorId)).thenReturn(Optional.empty()); + + final BacklogEntity entity = serviceFacade.getProcessorBacklog(processorId); + + verify(processorDAO).getBacklog(processorId); + assertNotNull(entity); + assertNull(entity.getBacklog()); + verify(dtoFactory, times(0)).createBacklogDto(any(Backlog.class)); + } + + @Test + public void testVerifyCanReportProcessorBacklogPropagatesIllegalState() { + final String processorId = "processor-disabled"; + final ProcessorDAO processorDAO = mock(ProcessorDAO.class); + serviceFacade.setProcessorDAO(processorDAO); + + doThrow(new IllegalStateException("Processor is disabled")).when(processorDAO).verifyReportBacklog(processorId); + + assertThrows(IllegalStateException.class, () -> serviceFacade.verifyCanReportProcessorBacklog(processorId)); + } + + @Test + public void testGetProcessorBacklogPropagatesBacklogReportingException() throws BacklogReportingException { + final String processorId = "processor-failure"; + final ProcessorDAO processorDAO = mock(ProcessorDAO.class); + serviceFacade.setProcessorDAO(processorDAO); + + final BacklogReportingException failure = new BacklogReportingException("Source unreachable"); + when(processorDAO.getBacklog(processorId)).thenThrow(failure); + + final BacklogReportingException thrown = assertThrows(BacklogReportingException.class, + () -> serviceFacade.getProcessorBacklog(processorId)); + assertEquals("Source unreachable", thrown.getMessage()); + } + + @Test + public void testVerifyCanReportConnectorBacklogRejectsConnectorWithoutBacklogCapability() { + final String connectorId = "connector-no-backlog"; + + final ConnectorDAO connectorDAO = mock(ConnectorDAO.class); + serviceFacade.setConnectorDAO(connectorDAO); + + final ConnectorNode connectorNode = mock(ConnectorNode.class); + final org.apache.nifi.components.connector.Connector connector = mock(org.apache.nifi.components.connector.Connector.class); + when(connectorDAO.getConnector(connectorId, ConnectorSyncMode.LOCAL_ONLY)).thenReturn(connectorNode); + when(connectorNode.getConnector()).thenReturn(connector); + + assertThrows(IllegalStateException.class, () -> serviceFacade.verifyCanReportConnectorBacklog(connectorId)); + } + + @Test + public void testVerifyCanReportConnectorBacklogAcceptsBacklogReportingConnector() { + final String connectorId = "connector-with-backlog"; + + final ConnectorDAO connectorDAO = mock(ConnectorDAO.class); + serviceFacade.setConnectorDAO(connectorDAO); + + final ConnectorNode connectorNode = mock(ConnectorNode.class); + // The capability is declared by implementing BacklogReportingConnector, not by a flag, so + // the mock must satisfy both Connector and BacklogReportingConnector for the instanceof + // check to succeed. + final org.apache.nifi.components.connector.Connector connector = mock(org.apache.nifi.components.connector.Connector.class, + org.mockito.Mockito.withSettings().extraInterfaces(org.apache.nifi.components.connector.BacklogReportingConnector.class)); + when(connectorDAO.getConnector(connectorId, ConnectorSyncMode.LOCAL_ONLY)).thenReturn(connectorNode); + when(connectorNode.getConnector()).thenReturn(connector); + when(connectorNode.getValidationStatus()).thenReturn(org.apache.nifi.components.validation.ValidationStatus.VALID); + when(connectorNode.getCurrentState()).thenReturn(org.apache.nifi.components.connector.ConnectorState.STOPPED); + + serviceFacade.verifyCanReportConnectorBacklog(connectorId); + } } diff --git a/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/test/java/org/apache/nifi/web/api/ProcessorResourceTest.java b/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/test/java/org/apache/nifi/web/api/ProcessorResourceTest.java new file mode 100644 index 000000000000..1589dcc13107 --- /dev/null +++ b/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/test/java/org/apache/nifi/web/api/ProcessorResourceTest.java @@ -0,0 +1,249 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.web.api; + +import jakarta.servlet.ServletContext; +import jakarta.servlet.http.HttpServletRequest; +import jakarta.ws.rs.core.Response; +import jakarta.ws.rs.core.UriBuilder; +import jakarta.ws.rs.core.UriInfo; +import org.apache.nifi.authorization.AccessDeniedException; +import org.apache.nifi.authorization.AuthorizeAccess; +import org.apache.nifi.authorization.Authorizer; +import org.apache.nifi.authorization.user.NiFiUser; +import org.apache.nifi.authorization.user.NiFiUserDetails; +import org.apache.nifi.authorization.user.StandardNiFiUser; +import org.apache.nifi.components.BacklogReportingException; +import org.apache.nifi.util.NiFiProperties; +import org.apache.nifi.web.NiFiServiceFacade; +import org.apache.nifi.web.api.dto.BacklogDTO; +import org.apache.nifi.web.api.entity.BacklogEntity; +import org.apache.nifi.web.api.entity.BacklogRequestEntity; +import org.apache.nifi.web.security.token.NiFiAuthenticationToken; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.InjectMocks; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; +import org.springframework.security.core.Authentication; +import org.springframework.security.core.context.SecurityContextHolder; + +import java.net.URI; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.junit.jupiter.api.Assertions.fail; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.Mockito.doThrow; +import static org.mockito.Mockito.lenient; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +@ExtendWith(MockitoExtension.class) +public class ProcessorResourceTest { + + @InjectMocks + private ProcessorResource processorResource; + + @Mock + private NiFiServiceFacade serviceFacade; + + @Mock + private Authorizer authorizer; + + @Mock + private HttpServletRequest httpServletRequest; + + @Mock + private ServletContext servletContext; + + @Mock + private NiFiProperties properties; + + @Mock + private UriInfo uriInfo; + + @Mock + private UriBuilder uriBuilder; + + private static final String PROCESSOR_ID = "test-processor-id"; + + @BeforeEach + public void setUp() throws Exception { + lenient().when(httpServletRequest.getHeader(any())).thenReturn(null); + lenient().when(httpServletRequest.getServletContext()).thenReturn(servletContext); + lenient().when(httpServletRequest.getContextPath()).thenReturn("/nifi-api"); + lenient().when(httpServletRequest.getScheme()).thenReturn("http"); + lenient().when(httpServletRequest.getServerName()).thenReturn("localhost"); + lenient().when(httpServletRequest.getServerPort()).thenReturn(8080); + lenient().when(servletContext.getInitParameter(any())).thenReturn(null); + lenient().when(properties.isNode()).thenReturn(Boolean.FALSE); + + lenient().when(uriInfo.getBaseUriBuilder()).thenReturn(uriBuilder); + lenient().when(uriBuilder.segment(any(String[].class))).thenReturn(uriBuilder); + lenient().when(uriBuilder.build()).thenReturn(new URI("http://localhost:8080/nifi-api/processors/" + PROCESSOR_ID)); + + processorResource.setServiceFacade(serviceFacade); + processorResource.setAuthorizer(authorizer); + processorResource.httpServletRequest = httpServletRequest; + processorResource.properties = properties; + processorResource.uriInfo = uriInfo; + } + + @AfterEach + public void tearDown() { + SecurityContextHolder.clearContext(); + } + + private NiFiUser authenticate() { + final NiFiUser user = new StandardNiFiUser.Builder().identity("unit-test-user").build(); + final Authentication authentication = new NiFiAuthenticationToken(new NiFiUserDetails(user)); + SecurityContextHolder.getContext().setAuthentication(authentication); + return user; + } + + @Test + public void testSubmitProcessorBacklogRequestHappyPath() throws Exception { + authenticate(); + + final BacklogEntity entity = new BacklogEntity(); + final BacklogDTO dto = new BacklogDTO(); + dto.setRecordCount(42L); + dto.setPrecision("EXACT"); + entity.setBacklog(dto); + when(serviceFacade.getProcessorBacklog(PROCESSOR_ID)).thenReturn(entity); + + final String requestId; + try (Response response = processorResource.submitProcessorBacklogRequest(PROCESSOR_ID)) { + assertEquals(200, response.getStatus()); + final BacklogRequestEntity body = (BacklogRequestEntity) response.getEntity(); + assertEquals(PROCESSOR_ID, body.getRequest().getComponentId()); + assertNotNull(body.getRequest().getRequestId()); + requestId = body.getRequest().getRequestId(); + } + + verify(serviceFacade).authorizeAccess(any(AuthorizeAccess.class)); + verify(serviceFacade).verifyCanReportProcessorBacklog(PROCESSOR_ID); + + final BacklogRequestEntity completedEntity = awaitBacklogRequestCompletion(requestId); + assertEquals(42L, completedEntity.getRequest().getBacklog().getRecordCount()); + + try (Response deleteResponse = processorResource.deleteBacklogRequest(PROCESSOR_ID, requestId)) { + assertEquals(200, deleteResponse.getStatus()); + } + } + + @Test + public void testSubmitProcessorBacklogRequestSurfacesRootCauseOfFailure() throws Exception { + authenticate(); + + final RuntimeException rootCause = new RuntimeException("Timed out waiting for a node assignment"); + when(serviceFacade.getProcessorBacklog(PROCESSOR_ID)) + .thenThrow(new BacklogReportingException("Failed to determine Kafka backlog", rootCause)); + + final String requestId; + try (Response response = processorResource.submitProcessorBacklogRequest(PROCESSOR_ID)) { + requestId = ((BacklogRequestEntity) response.getEntity()).getRequest().getRequestId(); + } + + final BacklogRequestEntity completedEntity = awaitBacklogRequestCompletion(requestId); + final String failureReason = completedEntity.getRequest().getFailureReason(); + assertTrue(failureReason.contains("Failed to determine Kafka backlog")); + assertTrue(failureReason.contains("Timed out waiting for a node assignment")); + } + + @Test + public void testSubmitProcessorBacklogRequestVerificationFailsNeverCallsServiceFacade() throws Exception { + authenticate(); + doThrow(new IllegalStateException("Processor is disabled")).when(serviceFacade).verifyCanReportProcessorBacklog(PROCESSOR_ID); + + assertThrows(IllegalStateException.class, () -> processorResource.submitProcessorBacklogRequest(PROCESSOR_ID)); + + verify(serviceFacade).authorizeAccess(any(AuthorizeAccess.class)); + verify(serviceFacade).verifyCanReportProcessorBacklog(PROCESSOR_ID); + verify(serviceFacade, never()).getProcessorBacklog(anyString()); + } + + @Test + public void testSubmitProcessorBacklogRequestNotAuthorized() throws Exception { + authenticate(); + doThrow(AccessDeniedException.class).when(serviceFacade).authorizeAccess(any(AuthorizeAccess.class)); + + assertThrows(AccessDeniedException.class, () -> processorResource.submitProcessorBacklogRequest(PROCESSOR_ID)); + + verify(serviceFacade, never()).verifyCanReportProcessorBacklog(anyString()); + } + + @Test + public void testDeleteBacklogRequestCancelsInProgressRequestAndInterruptsBackgroundThread() throws Exception { + authenticate(); + + final CountDownLatch backlogDeterminationStarted = new CountDownLatch(1); + final CountDownLatch backlogDeterminationInterrupted = new CountDownLatch(1); + when(serviceFacade.getProcessorBacklog(PROCESSOR_ID)).thenAnswer(invocation -> { + backlogDeterminationStarted.countDown(); + try { + Thread.sleep(30_000L); + } catch (final InterruptedException e) { + backlogDeterminationInterrupted.countDown(); + throw new BacklogReportingException("Interrupted while determining backlog"); + } + return new BacklogEntity(); + }); + + final String requestId; + try (Response response = processorResource.submitProcessorBacklogRequest(PROCESSOR_ID)) { + requestId = ((BacklogRequestEntity) response.getEntity()).getRequest().getRequestId(); + } + + assertTrue(backlogDeterminationStarted.await(5, TimeUnit.SECONDS)); + + try (Response deleteResponse = processorResource.deleteBacklogRequest(PROCESSOR_ID, requestId)) { + assertEquals(200, deleteResponse.getStatus()); + final BacklogRequestEntity body = (BacklogRequestEntity) deleteResponse.getEntity(); + assertTrue(body.getRequest().isComplete()); + assertEquals("Request cancelled by user", body.getRequest().getFailureReason()); + } + + assertTrue(backlogDeterminationInterrupted.await(5, TimeUnit.SECONDS)); + } + + private BacklogRequestEntity awaitBacklogRequestCompletion(final String requestId) throws Exception { + final long deadline = System.currentTimeMillis() + TimeUnit.SECONDS.toMillis(5); + BacklogRequestEntity entity; + do { + try (Response response = processorResource.getBacklogRequest(PROCESSOR_ID, requestId)) { + entity = (BacklogRequestEntity) response.getEntity(); + } + if (entity.getRequest().isComplete()) { + return entity; + } + Thread.sleep(20L); + } while (System.currentTimeMillis() < deadline); + + fail("Backlog request did not complete within the expected time"); + return entity; + } +} diff --git a/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/test/java/org/apache/nifi/web/api/TestConnectorResource.java b/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/test/java/org/apache/nifi/web/api/TestConnectorResource.java index 2fcac1d5dcdc..bc71d67b2609 100644 --- a/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/test/java/org/apache/nifi/web/api/TestConnectorResource.java +++ b/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/test/java/org/apache/nifi/web/api/TestConnectorResource.java @@ -25,11 +25,16 @@ import org.apache.nifi.authorization.AccessDeniedException; import org.apache.nifi.authorization.AuthorizeAccess; import org.apache.nifi.authorization.Authorizer; +import org.apache.nifi.authorization.user.NiFiUser; +import org.apache.nifi.authorization.user.NiFiUserDetails; +import org.apache.nifi.authorization.user.StandardNiFiUser; +import org.apache.nifi.components.BacklogReportingException; import org.apache.nifi.controller.ScheduledState; import org.apache.nifi.util.NiFiProperties; import org.apache.nifi.web.NiFiServiceFacade; import org.apache.nifi.web.Revision; import org.apache.nifi.web.api.dto.AllowableValueDTO; +import org.apache.nifi.web.api.dto.BacklogDTO; import org.apache.nifi.web.api.dto.ComponentStateDTO; import org.apache.nifi.web.api.dto.ConnectorDTO; import org.apache.nifi.web.api.dto.ParameterContextDTO; @@ -37,6 +42,8 @@ import org.apache.nifi.web.api.dto.RevisionDTO; import org.apache.nifi.web.api.dto.flow.ProcessGroupFlowDTO; import org.apache.nifi.web.api.entity.AllowableValueEntity; +import org.apache.nifi.web.api.entity.BacklogEntity; +import org.apache.nifi.web.api.entity.BacklogRequestEntity; import org.apache.nifi.web.api.entity.ComponentStateEntity; import org.apache.nifi.web.api.entity.ConnectorEntity; import org.apache.nifi.web.api.entity.ConnectorPropertyAllowableValuesEntity; @@ -49,6 +56,7 @@ import org.apache.nifi.web.api.entity.SecretsEntity; import org.apache.nifi.web.api.request.ClientIdParameter; import org.apache.nifi.web.api.request.LongParameter; +import org.apache.nifi.web.security.token.NiFiAuthenticationToken; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @@ -56,14 +64,20 @@ import org.mockito.InjectMocks; import org.mockito.Mock; import org.mockito.junit.jupiter.MockitoExtension; +import org.springframework.security.core.Authentication; import org.springframework.security.core.context.SecurityContextHolder; import java.net.URI; import java.util.List; import java.util.Set; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.junit.jupiter.api.Assertions.fail; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.anyString; import static org.mockito.ArgumentMatchers.eq; @@ -878,4 +892,134 @@ public void testClearConnectorControllerServiceStateNotAuthorized() { verify(serviceFacade, never()).verifyCanClearConnectorControllerServiceState(anyString(), anyString()); verify(serviceFacade, never()).clearConnectorControllerServiceState(anyString(), anyString(), any()); } + + private NiFiUser authenticate() { + final NiFiUser user = new StandardNiFiUser.Builder().identity("unit-test-user").build(); + final Authentication authentication = new NiFiAuthenticationToken(new NiFiUserDetails(user)); + SecurityContextHolder.getContext().setAuthentication(authentication); + return user; + } + + @Test + public void testSubmitConnectorBacklogRequestHappyPath() throws Exception { + authenticate(); + + final BacklogEntity entity = new BacklogEntity(); + final BacklogDTO dto = new BacklogDTO(); + dto.setFlowFileCount(5L); + dto.setPrecision("EXACT"); + entity.setBacklog(dto); + when(serviceFacade.getConnectorBacklog(CONNECTOR_ID)).thenReturn(entity); + + final String requestId; + try (Response response = connectorResource.submitConnectorBacklogRequest(CONNECTOR_ID)) { + assertEquals(200, response.getStatus()); + final BacklogRequestEntity body = (BacklogRequestEntity) response.getEntity(); + assertEquals(CONNECTOR_ID, body.getRequest().getComponentId()); + assertNotNull(body.getRequest().getRequestId()); + requestId = body.getRequest().getRequestId(); + } + + verify(serviceFacade).authorizeAccess(any(AuthorizeAccess.class)); + verify(serviceFacade).verifyCanReportConnectorBacklog(CONNECTOR_ID); + + final BacklogRequestEntity completedEntity = awaitBacklogRequestCompletion(requestId); + assertEquals(5L, completedEntity.getRequest().getBacklog().getFlowFileCount()); + + try (Response deleteResponse = connectorResource.deleteConnectorBacklogRequest(CONNECTOR_ID, requestId)) { + assertEquals(200, deleteResponse.getStatus()); + } + } + + @Test + public void testSubmitConnectorBacklogRequestSurfacesRootCauseOfFailure() throws Exception { + authenticate(); + + final RuntimeException rootCause = new RuntimeException("Timed out waiting for a node assignment"); + when(serviceFacade.getConnectorBacklog(CONNECTOR_ID)) + .thenThrow(new BacklogReportingException("Failed to determine Kafka backlog", rootCause)); + + final String requestId; + try (Response response = connectorResource.submitConnectorBacklogRequest(CONNECTOR_ID)) { + requestId = ((BacklogRequestEntity) response.getEntity()).getRequest().getRequestId(); + } + + final BacklogRequestEntity completedEntity = awaitBacklogRequestCompletion(requestId); + final String failureReason = completedEntity.getRequest().getFailureReason(); + assertTrue(failureReason.contains("Failed to determine Kafka backlog")); + assertTrue(failureReason.contains("Timed out waiting for a node assignment")); + } + + @Test + public void testSubmitConnectorBacklogRequestVerificationFailsNeverCallsServiceFacade() throws Exception { + authenticate(); + doThrow(new IllegalStateException("Connector is disabled")).when(serviceFacade).verifyCanReportConnectorBacklog(CONNECTOR_ID); + + assertThrows(IllegalStateException.class, () -> connectorResource.submitConnectorBacklogRequest(CONNECTOR_ID)); + + verify(serviceFacade).authorizeAccess(any(AuthorizeAccess.class)); + verify(serviceFacade).verifyCanReportConnectorBacklog(CONNECTOR_ID); + verify(serviceFacade, never()).getConnectorBacklog(anyString()); + } + + @Test + public void testSubmitConnectorBacklogRequestNotAuthorized() throws Exception { + authenticate(); + doThrow(AccessDeniedException.class).when(serviceFacade).authorizeAccess(any(AuthorizeAccess.class)); + + assertThrows(AccessDeniedException.class, () -> connectorResource.submitConnectorBacklogRequest(CONNECTOR_ID)); + + verify(serviceFacade, never()).verifyCanReportConnectorBacklog(anyString()); + } + + @Test + public void testDeleteConnectorBacklogRequestCancelsInProgressRequestAndInterruptsBackgroundThread() throws Exception { + authenticate(); + + final CountDownLatch backlogDeterminationStarted = new CountDownLatch(1); + final CountDownLatch backlogDeterminationInterrupted = new CountDownLatch(1); + when(serviceFacade.getConnectorBacklog(CONNECTOR_ID)).thenAnswer(invocation -> { + backlogDeterminationStarted.countDown(); + try { + Thread.sleep(30_000L); + } catch (final InterruptedException e) { + backlogDeterminationInterrupted.countDown(); + throw new BacklogReportingException("Interrupted while determining backlog"); + } + return new BacklogEntity(); + }); + + final String requestId; + try (Response response = connectorResource.submitConnectorBacklogRequest(CONNECTOR_ID)) { + requestId = ((BacklogRequestEntity) response.getEntity()).getRequest().getRequestId(); + } + + assertTrue(backlogDeterminationStarted.await(5, TimeUnit.SECONDS)); + + try (Response deleteResponse = connectorResource.deleteConnectorBacklogRequest(CONNECTOR_ID, requestId)) { + assertEquals(200, deleteResponse.getStatus()); + final BacklogRequestEntity body = (BacklogRequestEntity) deleteResponse.getEntity(); + assertTrue(body.getRequest().isComplete()); + assertEquals("Request cancelled by user", body.getRequest().getFailureReason()); + } + + assertTrue(backlogDeterminationInterrupted.await(5, TimeUnit.SECONDS)); + } + + private BacklogRequestEntity awaitBacklogRequestCompletion(final String requestId) throws Exception { + final long deadline = System.currentTimeMillis() + TimeUnit.SECONDS.toMillis(5); + BacklogRequestEntity entity; + do { + try (Response response = connectorResource.getConnectorBacklogRequest(CONNECTOR_ID, requestId)) { + entity = (BacklogRequestEntity) response.getEntity(); + } + if (entity.getRequest().isComplete()) { + return entity; + } + Thread.sleep(20L); + } while (System.currentTimeMillis() < deadline); + + fail("Backlog request did not complete within the expected time"); + return entity; + } } diff --git a/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/test/java/org/apache/nifi/web/api/concurrent/StandardAsynchronousWebRequestTest.java b/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/test/java/org/apache/nifi/web/api/concurrent/StandardAsynchronousWebRequestTest.java new file mode 100644 index 000000000000..80157dbcd3c8 --- /dev/null +++ b/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/test/java/org/apache/nifi/web/api/concurrent/StandardAsynchronousWebRequestTest.java @@ -0,0 +1,73 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.web.api.concurrent; + +import org.junit.jupiter.api.Test; + +import java.util.Collections; +import java.util.List; +import java.util.concurrent.atomic.AtomicBoolean; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + +public class StandardAsynchronousWebRequestTest { + + private static final String REQUEST_ID = "request-id"; + private static final String COMPONENT_ID = "component-id"; + + @Test + public void testFailAfterCancelDoesNotOverwriteCancellationReason() { + final StandardAsynchronousWebRequest request = createRequest(); + + request.cancel(); + request.fail("Failed while handling interruption from cancellation"); + + assertEquals("Request cancelled by user", request.getFailureReason()); + assertTrue(request.isCancelled()); + assertTrue(request.isComplete()); + } + + @Test + public void testCancelInvokesCancelCallback() { + final StandardAsynchronousWebRequest request = createRequest(); + final AtomicBoolean callbackInvoked = new AtomicBoolean(false); + request.setCancelCallback(() -> callbackInvoked.set(true)); + + request.cancel(); + + assertTrue(callbackInvoked.get()); + assertEquals("Request cancelled by user", request.getFailureReason()); + } + + @Test + public void testFailBeforeCancelSetsFailureReason() { + final StandardAsynchronousWebRequest request = createRequest(); + + request.fail("Backlog determination failed"); + + assertEquals("Backlog determination failed", request.getFailureReason()); + assertTrue(request.isComplete()); + assertFalse(request.isCancelled()); + } + + private StandardAsynchronousWebRequest createRequest() { + final List updateSteps = Collections.singletonList(new StandardUpdateStep("Determining Backlog")); + return new StandardAsynchronousWebRequest<>(REQUEST_ID, COMPONENT_ID, COMPONENT_ID, null, updateSteps); + } +} diff --git a/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/test/java/org/apache/nifi/web/dao/impl/StandardProcessorDAOTest.java b/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/test/java/org/apache/nifi/web/dao/impl/StandardProcessorDAOTest.java index 1ae61d3dce6c..0ed1c1632dc1 100644 --- a/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/test/java/org/apache/nifi/web/dao/impl/StandardProcessorDAOTest.java +++ b/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/test/java/org/apache/nifi/web/dao/impl/StandardProcessorDAOTest.java @@ -19,14 +19,17 @@ import org.apache.nifi.bundle.Bundle; import org.apache.nifi.bundle.BundleCoordinate; import org.apache.nifi.bundle.BundleDetails; +import org.apache.nifi.components.Backlog; import org.apache.nifi.components.state.StateManager; import org.apache.nifi.components.state.StateManagerProvider; import org.apache.nifi.controller.FlowController; import org.apache.nifi.controller.ProcessorNode; import org.apache.nifi.controller.ScheduledState; import org.apache.nifi.controller.flow.FlowManager; +import org.apache.nifi.controller.service.ControllerServiceProvider; import org.apache.nifi.groups.ProcessGroup; import org.apache.nifi.nar.ExtensionManager; +import org.apache.nifi.processor.ProcessContext; import org.apache.nifi.processor.Processor; import org.apache.nifi.web.ResourceNotFoundException; import org.apache.nifi.web.api.dto.ProcessorDTO; @@ -35,15 +38,21 @@ import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; import org.junit.jupiter.api.io.TempDir; +import org.mockito.ArgumentCaptor; import org.mockito.Mock; import org.mockito.junit.jupiter.MockitoExtension; import java.io.File; import java.util.List; +import java.util.Map; +import java.util.Optional; import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertSame; import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.anyString; import static org.mockito.ArgumentMatchers.eq; @@ -87,6 +96,9 @@ class StandardProcessorDAOTest { @Mock private StateManager stateManager; + @Mock + private ControllerServiceProvider controllerServiceProvider; + private StandardProcessorDAO dao; @BeforeEach @@ -195,6 +207,34 @@ void testVerifyUpdate_ProcessorNotFound() { assertThrows(ResourceNotFoundException.class, () -> dao.verifyUpdate(processorDTO)); } + @Test + void testGetBacklogProvidesRealStateManagerToProcessor() throws Exception { + final String processorId = "test-processor-id"; + final Backlog expectedBacklog = Backlog.caughtUp(); + + when(processorNode.getProcessor()).thenReturn(processor); + when(processorNode.getIdentifier()).thenReturn(processorId); + when(processorNode.getEffectivePropertyValues()).thenReturn(Map.of()); + when(processorNode.getAnnotationData()).thenReturn(null); + when(flowController.getControllerServiceProvider()).thenReturn(controllerServiceProvider); + when(flowController.getStateManagerProvider()).thenReturn(stateManagerProvider); + when(stateManagerProvider.getStateManager(eq(processorId), any())).thenReturn(stateManager); + when(processorNode.getReportedBacklog(any(ProcessContext.class))).thenReturn(Optional.of(expectedBacklog)); + + final Optional backlog = dao.getBacklog(processorId); + + assertTrue(backlog.isPresent()); + assertEquals(expectedBacklog, backlog.get()); + + final ArgumentCaptor contextCaptor = ArgumentCaptor.forClass(ProcessContext.class); + verify(processorNode).getReportedBacklog(contextCaptor.capture()); + final ProcessContext capturedContext = contextCaptor.getValue(); + assertNotNull(capturedContext.getStateManager()); + // The constructed ProcessContext must expose the real per-component StateManager so backlog + // implementations that consult cluster state observe the same state the running Processor would. + assertSame(stateManager, capturedContext.getStateManager()); + } + @Test void testCreateProcessor(@TempDir final File tempDir) { final String id = ProcessorDTO.class.getSimpleName(); diff --git a/nifi-frontend/src/main/frontend/apps/nifi/src/app/pages/flow-designer/service/canvas-context-menu.service.ts b/nifi-frontend/src/main/frontend/apps/nifi/src/app/pages/flow-designer/service/canvas-context-menu.service.ts index b85d1c06eb85..5eb220748949 100644 --- a/nifi-frontend/src/main/frontend/apps/nifi/src/app/pages/flow-designer/service/canvas-context-menu.service.ts +++ b/nifi-frontend/src/main/frontend/apps/nifi/src/app/pages/flow-designer/service/canvas-context-menu.service.ts @@ -943,6 +943,28 @@ export class CanvasContextMenu implements ContextMenuDefinitionProvider { ); } }, + { + condition: (selection: any) => { + // View Backlog is authorized as a WRITE action on the Processor, so it requires + // modify access (not just read access) to avoid offering an action that would fail with a 403. + return ( + this.canvasUtils.isProcessor(selection) && + selection.datum().component.supportsBacklogReporting === true && + this.canvasUtils.canRead(selection) && + this.canvasUtils.canModify(selection) + ); + }, + clazz: 'fa fa-list', + text: 'View Backlog', + action: (selection: any) => { + const selectionData = selection.datum(); + this.store.dispatch( + FlowActions.openProcessorBacklogDialog({ + id: selectionData.id + }) + ); + } + }, { condition: (selection: any) => { return this.canvasUtils.isConnection(selection); diff --git a/nifi-frontend/src/main/frontend/apps/nifi/src/app/pages/flow-designer/service/flow.service.ts b/nifi-frontend/src/main/frontend/apps/nifi/src/app/pages/flow-designer/service/flow.service.ts index 48eb10db1f12..b605afeda3f2 100644 --- a/nifi-frontend/src/main/frontend/apps/nifi/src/app/pages/flow-designer/service/flow.service.ts +++ b/nifi-frontend/src/main/frontend/apps/nifi/src/app/pages/flow-designer/service/flow.service.ts @@ -52,6 +52,7 @@ import { Client } from '../../../service/client.service'; import { ComponentType, NiFiCommon } from '@nifi/shared'; import { ClusterConnectionService } from '../../../service/cluster-connection.service'; import { + BacklogRequestEntity, ClearBulletinsRequest, DisableComponentRequest, EnableComponentRequest, @@ -95,6 +96,22 @@ export class FlowService implements PropertyDescriptorRetriever { return this.httpClient.get(`${FlowService.API}/processors/${id}`); } + submitProcessorBacklogRequest(id: string): Observable { + return this.httpClient.post(`${FlowService.API}/processors/${id}/backlog-requests`, null); + } + + pollProcessorBacklogRequest(id: string, requestId: string): Observable { + return this.httpClient.get( + `${FlowService.API}/processors/${id}/backlog-requests/${requestId}` + ); + } + + deleteProcessorBacklogRequest(id: string, requestId: string): Observable { + return this.httpClient.delete( + `${FlowService.API}/processors/${id}/backlog-requests/${requestId}` + ); + } + getInputPort(id: string): Observable { return this.httpClient.get(`${FlowService.API}/input-ports/${id}`); } diff --git a/nifi-frontend/src/main/frontend/apps/nifi/src/app/pages/flow-designer/state/flow/flow.actions.ts b/nifi-frontend/src/main/frontend/apps/nifi/src/app/pages/flow-designer/state/flow/flow.actions.ts index 39cb505c0677..76587c6a0cc4 100644 --- a/nifi-frontend/src/main/frontend/apps/nifi/src/app/pages/flow-designer/state/flow/flow.actions.ts +++ b/nifi-frontend/src/main/frontend/apps/nifi/src/app/pages/flow-designer/state/flow/flow.actions.ts @@ -458,6 +458,11 @@ export const openEditLabelDialog = createAction( props<{ request: EditComponentDialogRequest }>() ); +export const openProcessorBacklogDialog = createAction( + `${CANVAS_PREFIX} Open Processor Backlog Dialog`, + props<{ id: string }>() +); + export const navigateToManageRemotePorts = createAction( `${CANVAS_PREFIX} Open Remote Process Group Manage Remote Ports`, props<{ request: RpgManageRemotePortsRequest }>() diff --git a/nifi-frontend/src/main/frontend/apps/nifi/src/app/pages/flow-designer/state/flow/flow.effects.spec.ts b/nifi-frontend/src/main/frontend/apps/nifi/src/app/pages/flow-designer/state/flow/flow.effects.spec.ts index c8eac1a81301..e2f21e170fef 100644 --- a/nifi-frontend/src/main/frontend/apps/nifi/src/app/pages/flow-designer/state/flow/flow.effects.spec.ts +++ b/nifi-frontend/src/main/frontend/apps/nifi/src/app/pages/flow-designer/state/flow/flow.effects.spec.ts @@ -21,6 +21,7 @@ import { of, ReplaySubject, take, throwError } from 'rxjs'; import { MatDialog, MatDialogRef } from '@angular/material/dialog'; import { ComponentHistoryEntity } from '../../../../state/shared'; import { EditProcessor } from '../../../../ui/common/component-dialogs/edit-processor/edit-processor.component'; +import { ProcessorBacklogDialog } from '../../ui/canvas/items/processor/backlog-dialog/backlog-dialog.component'; import { PropertyTableHelperService } from '../../../../service/property-table-helper.service'; import { FlowEffects } from './flow.effects'; import { provideMockActions } from '@ngrx/effects/testing'; @@ -37,6 +38,7 @@ import { MoveToFrontRequest } from './index'; import { + BacklogRequestEntity, DisableComponentRequest, EnableComponentRequest, StartComponentRequest, @@ -830,7 +832,8 @@ describe('FlowEffects', () => { updateComponent: vi.fn(), createConnection: vi.fn(), createLabel: vi.fn(), - clearBulletinsForProcessGroup: vi.fn() + clearBulletinsForProcessGroup: vi.fn(), + submitProcessorBacklogRequest: vi.fn() } }, { @@ -1493,4 +1496,54 @@ describe('FlowEffects', () => { }); }); }); + + describe('openProcessorBacklogDialog$', () => { + const PROCESSOR_ID = 'proc-1'; + + it('submits a backlog request and opens the dialog with the resulting request entity', () => { + const requestEntity: BacklogRequestEntity = { + request: { + requestId: 'req-1', + uri: `https://localhost:4200/nifi-api/processors/${PROCESSOR_ID}/backlog-requests/req-1`, + componentId: PROCESSOR_ID, + complete: false, + percentCompleted: 0 + } + }; + + vi.spyOn(flowService, 'submitProcessorBacklogRequest').mockReturnValue(of(requestEntity)); + + effects.openProcessorBacklogDialog$.subscribe(); + action$.next(FlowActions.openProcessorBacklogDialog({ id: PROCESSOR_ID })); + + expect(flowService.submitProcessorBacklogRequest).toHaveBeenCalledWith(PROCESSOR_ID); + expect(dialog.open).toHaveBeenCalledWith( + ProcessorBacklogDialog, + expect.objectContaining({ + minWidth: '36rem', + maxWidth: '36rem', + width: '36rem', + data: { processorId: PROCESSOR_ID, requestEntity } + }) + ); + }); + + it('opens the dialog with an error message when the submit request fails', () => { + const errorHelper = TestBed.inject(ErrorHelper); + vi.spyOn(errorHelper, 'getErrorString').mockReturnValue('submit failed'); + vi.spyOn(flowService, 'submitProcessorBacklogRequest').mockReturnValue( + throwError(() => new HttpErrorResponse({ status: 500 })) + ); + + effects.openProcessorBacklogDialog$.subscribe(); + action$.next(FlowActions.openProcessorBacklogDialog({ id: PROCESSOR_ID })); + + expect(dialog.open).toHaveBeenCalledWith( + ProcessorBacklogDialog, + expect.objectContaining({ + data: { processorId: PROCESSOR_ID, errorMessage: 'submit failed' } + }) + ); + }); + }); }); diff --git a/nifi-frontend/src/main/frontend/apps/nifi/src/app/pages/flow-designer/state/flow/flow.effects.ts b/nifi-frontend/src/main/frontend/apps/nifi/src/app/pages/flow-designer/state/flow/flow.effects.ts index f888de8579fe..a66564ab5bc6 100644 --- a/nifi-frontend/src/main/frontend/apps/nifi/src/app/pages/flow-designer/state/flow/flow.effects.ts +++ b/nifi-frontend/src/main/frontend/apps/nifi/src/app/pages/flow-designer/state/flow/flow.effects.ts @@ -36,6 +36,7 @@ import { asyncScheduler, catchError, combineLatest, + exhaustMap, filter, from, interval, @@ -63,6 +64,7 @@ import { PasteRequestContext, PasteRequestEntity, ProcessGroupFlowEntity, + ProcessorBacklogDialogRequest, SaveVersionDialogRequest, SaveVersionRequest, SelectedComponent, @@ -158,6 +160,7 @@ import { SaveVersionDialog } from '../../ui/canvas/items/flow/save-version-dialo import { ChangeVersionDialog } from '../../ui/canvas/items/flow/change-version-dialog/change-version-dialog'; import { ChangeVersionProgressDialog } from '../../ui/canvas/items/flow/change-version-progress-dialog/change-version-progress-dialog'; import { LocalChangesDialog } from '../../ui/canvas/items/flow/local-changes-dialog/local-changes-dialog'; +import { ProcessorBacklogDialog } from '../../ui/canvas/items/processor/backlog-dialog/backlog-dialog.component'; import { ClusterConnectionService } from '../../../../service/cluster-connection.service'; import { ExtensionTypesService } from '../../../../service/extension-types.service'; import { ChangeComponentVersionDialog } from '../../../../ui/common/change-component-version-dialog/change-component-version-dialog'; @@ -3173,6 +3176,35 @@ export class FlowEffects { { dispatch: false } ); + openProcessorBacklogDialog$ = createEffect( + () => + this.actions$.pipe( + ofType(FlowActions.openProcessorBacklogDialog), + map((action) => action.id), + exhaustMap((id) => + this.flowService.submitProcessorBacklogRequest(id).pipe( + map((requestEntity) => ({ processorId: id, requestEntity }) as ProcessorBacklogDialogRequest), + catchError((errorResponse: HttpErrorResponse) => + of({ + processorId: id, + errorMessage: this.errorHelper.getErrorString(errorResponse) + } as ProcessorBacklogDialogRequest) + ), + tap((request) => { + this.dialog.open(ProcessorBacklogDialog, { + ...MEDIUM_DIALOG, + minWidth: '36rem', + maxWidth: '36rem', + width: '36rem', + data: request + }); + }) + ) + ) + ), + { dispatch: false } + ); + enableCurrentProcessGroup$ = createEffect(() => this.actions$.pipe( ofType(FlowActions.enableCurrentProcessGroup), diff --git a/nifi-frontend/src/main/frontend/apps/nifi/src/app/pages/flow-designer/state/flow/index.ts b/nifi-frontend/src/main/frontend/apps/nifi/src/app/pages/flow-designer/state/flow/index.ts index 867fdb42e225..c4e821d01524 100644 --- a/nifi-frontend/src/main/frontend/apps/nifi/src/app/pages/flow-designer/state/flow/index.ts +++ b/nifi-frontend/src/main/frontend/apps/nifi/src/app/pages/flow-designer/state/flow/index.ts @@ -17,6 +17,7 @@ import { Position } from '../shared'; import { + BacklogRequestEntity, BreadcrumbEntity, DisableComponentRequest, EnableComponentRequest, @@ -729,6 +730,12 @@ export interface LocalChangesDialogRequest { mode: 'SHOW' | 'REVERT'; } +export interface ProcessorBacklogDialogRequest { + processorId: string; + requestEntity?: BacklogRequestEntity; + errorMessage?: string; +} + export interface DownloadFlowRequest { processGroupId: string; includeReferencedServices: boolean; diff --git a/nifi-frontend/src/main/frontend/apps/nifi/src/app/pages/flow-designer/ui/canvas/items/processor/backlog-dialog/backlog-dialog.component.html b/nifi-frontend/src/main/frontend/apps/nifi/src/app/pages/flow-designer/ui/canvas/items/processor/backlog-dialog/backlog-dialog.component.html new file mode 100644 index 000000000000..a3431897c0ab --- /dev/null +++ b/nifi-frontend/src/main/frontend/apps/nifi/src/app/pages/flow-designer/ui/canvas/items/processor/backlog-dialog/backlog-dialog.component.html @@ -0,0 +1,62 @@ + + +

Backlog

+ +
+ @if (!complete) { +
+
Determining backlog...
+ +
+ } @else if (errorMessage) { +
{{ errorMessage }}
+ } @else { +
+ + + + + + + + + + + + + +
Measurement{{ row.label }}Value + @if (row.value === null) { + No value set + } @else if (row.tooltip) { + {{ row.value }} + } @else { + {{ row.value }} + } +
+
+ } +
+ + + @if (!complete) { + + } @else { + + } + diff --git a/nifi-frontend/src/main/frontend/apps/nifi/src/app/pages/flow-designer/ui/canvas/items/processor/backlog-dialog/backlog-dialog.component.scss b/nifi-frontend/src/main/frontend/apps/nifi/src/app/pages/flow-designer/ui/canvas/items/processor/backlog-dialog/backlog-dialog.component.scss new file mode 100644 index 000000000000..3f54e7fb42ff --- /dev/null +++ b/nifi-frontend/src/main/frontend/apps/nifi/src/app/pages/flow-designer/ui/canvas/items/processor/backlog-dialog/backlog-dialog.component.scss @@ -0,0 +1,33 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +.backlog-table { + width: 100%; + + table { + width: 100%; + } +} + +.backlog-measurement-column { + width: 12rem; + font-weight: 500; +} + +.backlog-value-column { + white-space: nowrap; +} diff --git a/nifi-frontend/src/main/frontend/apps/nifi/src/app/pages/flow-designer/ui/canvas/items/processor/backlog-dialog/backlog-dialog.component.spec.ts b/nifi-frontend/src/main/frontend/apps/nifi/src/app/pages/flow-designer/ui/canvas/items/processor/backlog-dialog/backlog-dialog.component.spec.ts new file mode 100644 index 000000000000..991f9667f981 --- /dev/null +++ b/nifi-frontend/src/main/frontend/apps/nifi/src/app/pages/flow-designer/ui/canvas/items/processor/backlog-dialog/backlog-dialog.component.spec.ts @@ -0,0 +1,385 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { ComponentFixture, fakeAsync, TestBed, tick } from '@angular/core/testing'; +import { MAT_DIALOG_DATA, MatDialogRef } from '@angular/material/dialog'; +import { HttpErrorResponse } from '@angular/common/http'; +import { Observable, of, throwError } from 'rxjs'; + +import { BacklogRow, ProcessorBacklogDialog } from './backlog-dialog.component'; +import { Backlog, BacklogRequest, BacklogRequestEntity } from '../../../../../../../state/shared'; +import { ProcessorBacklogDialogRequest } from '../../../../../state/flow'; +import { FlowService } from '../../../../../service/flow.service'; +import { ErrorHelper } from '../../../../../../../service/error-helper.service'; + +const PROCESSOR_ID = 'a1b2c3d4-0000-0000-0000-000000000000'; +const REQUEST_ID = 'e5f6a7b8-0000-0000-0000-000000000000'; + +function completedRequest(backlog: Backlog | null, failureReason?: string): BacklogRequest { + return { + requestId: REQUEST_ID, + uri: `../nifi-api/processors/${PROCESSOR_ID}/backlog-requests/${REQUEST_ID}`, + componentId: PROCESSOR_ID, + complete: true, + percentCompleted: 100, + failureReason: failureReason ?? null, + backlog + }; +} + +function inProgressRequest(): BacklogRequest { + return { + requestId: REQUEST_ID, + uri: `../nifi-api/processors/${PROCESSOR_ID}/backlog-requests/${REQUEST_ID}`, + componentId: PROCESSOR_ID, + complete: false, + percentCompleted: 0 + }; +} + +function requestEntity(request: BacklogRequest): BacklogRequestEntity { + return { request }; +} + +interface FlowServiceStub { + pollProcessorBacklogRequest: ReturnType; + deleteProcessorBacklogRequest: ReturnType; +} + +interface CreatedDialog { + component: ProcessorBacklogDialog; + fixture: ComponentFixture; + flowService: FlowServiceStub; +} + +function createDialog( + dialogRequest: ProcessorBacklogDialogRequest, + pollResponses: Observable[] = [] +): CreatedDialog { + let pollCallCount = 0; + const flowService: FlowServiceStub = { + pollProcessorBacklogRequest: vi.fn(() => { + const response = pollResponses[Math.min(pollCallCount, pollResponses.length - 1)]; + pollCallCount++; + return response; + }), + deleteProcessorBacklogRequest: vi.fn(() => of({}) as Observable) + }; + + TestBed.resetTestingModule(); + TestBed.configureTestingModule({ + imports: [ProcessorBacklogDialog], + providers: [ + { provide: MAT_DIALOG_DATA, useValue: dialogRequest }, + { provide: MatDialogRef, useValue: { close: vi.fn(), keydownEvents: () => of() } }, + { provide: FlowService, useValue: flowService }, + { provide: ErrorHelper, useValue: new ErrorHelper() } + ] + }); + const fixture: ComponentFixture = TestBed.createComponent(ProcessorBacklogDialog); + fixture.detectChanges(); + return { component: fixture.componentInstance, fixture, flowService }; +} + +function rowByLabel(rows: BacklogRow[], label: string): BacklogRow { + const match = rows.find((row) => row.label === label); + if (match == null) { + throw new Error(`Expected a row labelled '${label}', found: ${rows.map((row) => row.label).join(', ')}`); + } + return match; +} + +describe('ProcessorBacklogDialog', () => { + it('should create', () => { + const { component } = createDialog({ + processorId: PROCESSOR_ID, + requestEntity: requestEntity( + completedRequest({ + flowFileCount: 1, + byteCount: 2, + recordCount: 3, + precision: 'EXACT', + lastCaughtUp: '2026-05-15T12:00:00Z', + formattedLastCaughtUp: '5 mins ago' + }) + ) + }); + expect(component).toBeTruthy(); + }); + + describe('row building', () => { + it('uses server-provided formatted values and exposes raw values as tooltips', () => { + const { component } = createDialog({ + processorId: PROCESSOR_ID, + requestEntity: requestEntity( + completedRequest({ + flowFileCount: 1025, + formattedFlowFileCount: '1,025', + byteCount: 5247340800, + formattedByteCount: '4.89 GB', + recordCount: 2050, + formattedRecordCount: '2,050', + precision: 'EXACT', + lastCaughtUp: '2026-05-15T12:00:00Z', + formattedLastCaughtUp: '5 mins ago' + }) + ) + }); + + const flowFiles: BacklogRow = rowByLabel(component.rows, 'FlowFiles'); + expect(flowFiles.value).toBe('1,025'); + expect(flowFiles.tooltip).toBe('1025'); + + const bytes: BacklogRow = rowByLabel(component.rows, 'Bytes'); + expect(bytes.value).toBe('4.89 GB'); + expect(bytes.tooltip).toBe('5247340800 bytes'); + + const records: BacklogRow = rowByLabel(component.rows, 'Records'); + expect(records.value).toBe('2,050'); + expect(records.tooltip).toBe('2050'); + + const precision: BacklogRow = rowByLabel(component.rows, 'Precision'); + expect(precision.value).toBe('EXACT'); + expect(precision.tooltip).toBeUndefined(); + + const lastCaughtUp: BacklogRow = rowByLabel(component.rows, 'Last caught up'); + // The absolute timestamp is rendered in the test runner's local timezone, which varies by + // machine, so assert the shape — "yyyy-MM-dd HH:mm:ss (5 mins ago)" — instead of an + // exact string. The relative-time portion must be the server-provided string verbatim, and + // the raw ISO timestamp must be preserved as the tooltip. + expect(lastCaughtUp.value).toMatch(/^\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2} \S+ \(5 mins ago\)$/); + expect(lastCaughtUp.tooltip).toBe('2026-05-15T12:00:00Z'); + }); + + it('falls back to the raw numeric value when no formatted value is provided', () => { + const { component } = createDialog({ + processorId: PROCESSOR_ID, + requestEntity: requestEntity( + completedRequest({ + flowFileCount: 7, + byteCount: 42, + precision: 'AT_LEAST' + }) + ) + }); + + expect(rowByLabel(component.rows, 'FlowFiles').value).toBe('7'); + expect(rowByLabel(component.rows, 'Bytes').value).toBe('42 B'); + }); + + it('marks dimensions the backlog did not report as unset so the template renders "No value set"', () => { + const { component } = createDialog({ + processorId: PROCESSOR_ID, + requestEntity: requestEntity(completedRequest({ precision: 'EXACT' })) + }); + + expect(rowByLabel(component.rows, 'FlowFiles').value).toBeNull(); + expect(rowByLabel(component.rows, 'FlowFiles').tooltip).toBeUndefined(); + expect(rowByLabel(component.rows, 'Bytes').value).toBeNull(); + expect(rowByLabel(component.rows, 'Records').value).toBeNull(); + expect(rowByLabel(component.rows, 'Last caught up').value).toBeNull(); + }); + + it('marks every measurement as unset when the backlog is null', () => { + const { component } = createDialog({ + processorId: PROCESSOR_ID, + requestEntity: requestEntity(completedRequest(null)) + }); + + for (const row of component.rows) { + expect(row.value).toBeNull(); + expect(row.tooltip).toBeUndefined(); + } + }); + }); + + describe('Last caught up row formatting', () => { + it('renders the absolute timestamp paired with the server-provided "(now)" string', () => { + const { component } = createDialog({ + processorId: PROCESSOR_ID, + requestEntity: requestEntity( + completedRequest({ + flowFileCount: 0, + byteCount: 0, + recordCount: 0, + precision: 'EXACT', + lastCaughtUp: '2026-05-15T12:00:00Z', + formattedLastCaughtUp: 'now' + }) + ) + }); + expect(rowByLabel(component.rows, 'Last caught up').value).toMatch( + /^\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2} \S+ \(now\)$/ + ); + }); + + it('renders any server-provided relative-time string verbatim inside parentheses', () => { + const { component } = createDialog({ + processorId: PROCESSOR_ID, + requestEntity: requestEntity( + completedRequest({ + flowFileCount: 1, + precision: 'AT_LEAST', + lastCaughtUp: '2026-05-14T12:00:00Z', + formattedLastCaughtUp: 'yesterday' + }) + ) + }); + expect(rowByLabel(component.rows, 'Last caught up').value).toMatch(/^.+ \(yesterday\)$/); + }); + + it('renders just the absolute timestamp when the server omits the relative-time string', () => { + const { component } = createDialog({ + processorId: PROCESSOR_ID, + requestEntity: requestEntity( + completedRequest({ + flowFileCount: 1, + precision: 'AT_LEAST', + lastCaughtUp: '2026-05-15T12:00:00Z' + }) + ) + }); + const value = rowByLabel(component.rows, 'Last caught up').value; + expect(value).not.toBeNull(); + expect(value).toMatch(/^\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2} \S+$/); + }); + + it('renders "No value set" when the timestamp is unparseable', () => { + const { component } = createDialog({ + processorId: PROCESSOR_ID, + requestEntity: requestEntity( + completedRequest({ + lastCaughtUp: 'not a real timestamp', + formattedLastCaughtUp: '5 mins ago' + }) + ) + }); + expect(rowByLabel(component.rows, 'Last caught up').value).toBeNull(); + }); + }); + + describe('asynchronous request lifecycle', () => { + it('shows the initial error message without polling when the request could not be submitted', () => { + const { component, flowService } = createDialog({ + processorId: PROCESSOR_ID, + errorMessage: 'Not authorized to view this backlog.' + }); + + expect(component.complete).toBe(true); + expect(component.errorMessage).toBe('Not authorized to view this backlog.'); + expect(flowService.pollProcessorBacklogRequest).not.toHaveBeenCalled(); + }); + + it('polls until the request completes, populates the rows, and deletes the request', fakeAsync(() => { + const { component, fixture, flowService } = createDialog( + { + processorId: PROCESSOR_ID, + requestEntity: requestEntity(inProgressRequest()) + }, + [of(requestEntity(inProgressRequest())), of(requestEntity(completedRequest({ flowFileCount: 5 })))] + ); + + expect(component.complete).toBe(false); + + tick(2000); + fixture.detectChanges(false); + expect(component.complete).toBe(false); + expect(flowService.deleteProcessorBacklogRequest).not.toHaveBeenCalled(); + + tick(2000); + fixture.detectChanges(false); + + expect(component.complete).toBe(true); + expect(rowByLabel(component.rows, 'FlowFiles').value).toBe('5'); + expect(flowService.deleteProcessorBacklogRequest).toHaveBeenCalledTimes(1); + expect(flowService.deleteProcessorBacklogRequest).toHaveBeenCalledWith(PROCESSOR_ID, REQUEST_ID); + + fixture.destroy(); + })); + + it('surfaces the failure reason once the request completes unsuccessfully', fakeAsync(() => { + const { component, fixture } = createDialog( + { + processorId: PROCESSOR_ID, + requestEntity: requestEntity(inProgressRequest()) + }, + [of(requestEntity(completedRequest(null, 'Interrupted while determining Kafka backlog')))] + ); + + tick(2000); + fixture.detectChanges(false); + + expect(component.complete).toBe(true); + expect(component.errorMessage).toBe('Interrupted while determining Kafka backlog'); + expect(component.rows).toEqual([]); + + fixture.destroy(); + })); + + it('stops polling and surfaces an error when a poll request itself fails', fakeAsync(() => { + const { component, fixture, flowService } = createDialog( + { + processorId: PROCESSOR_ID, + requestEntity: requestEntity(inProgressRequest()) + }, + [throwError(() => new HttpErrorResponse({ status: 500, error: 'Server error' }))] + ); + + tick(2000); + fixture.detectChanges(false); + + expect(component.complete).toBe(true); + expect(component.errorMessage).toContain('Server error'); + + const callsBeforeFurtherTicks = flowService.pollProcessorBacklogRequest.mock.calls.length; + tick(4000); + expect(flowService.pollProcessorBacklogRequest.mock.calls.length).toBe(callsBeforeFurtherTicks); + + fixture.destroy(); + })); + + it('deletes the in-progress request immediately when the dialog is cancelled', () => { + const { component, fixture, flowService } = createDialog({ + processorId: PROCESSOR_ID, + requestEntity: requestEntity(inProgressRequest()) + }); + + component.cancel(); + fixture.destroy(); + + expect(flowService.deleteProcessorBacklogRequest).toHaveBeenCalledTimes(1); + expect(flowService.deleteProcessorBacklogRequest).toHaveBeenCalledWith(PROCESSOR_ID, REQUEST_ID); + }); + + it('does not delete an already-deleted request a second time when the dialog is destroyed', fakeAsync(() => { + const { fixture, flowService } = createDialog( + { + processorId: PROCESSOR_ID, + requestEntity: requestEntity(inProgressRequest()) + }, + [of(requestEntity(completedRequest({ flowFileCount: 1 })))] + ); + + tick(2000); + fixture.detectChanges(false); + expect(flowService.deleteProcessorBacklogRequest).toHaveBeenCalledTimes(1); + + fixture.destroy(); + expect(flowService.deleteProcessorBacklogRequest).toHaveBeenCalledTimes(1); + })); + }); +}); diff --git a/nifi-frontend/src/main/frontend/apps/nifi/src/app/pages/flow-designer/ui/canvas/items/processor/backlog-dialog/backlog-dialog.component.ts b/nifi-frontend/src/main/frontend/apps/nifi/src/app/pages/flow-designer/ui/canvas/items/processor/backlog-dialog/backlog-dialog.component.ts new file mode 100644 index 000000000000..7156e2c3741f --- /dev/null +++ b/nifi-frontend/src/main/frontend/apps/nifi/src/app/pages/flow-designer/ui/canvas/items/processor/backlog-dialog/backlog-dialog.component.ts @@ -0,0 +1,256 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { Component, DestroyRef, inject, OnDestroy } from '@angular/core'; +import { MAT_DIALOG_DATA, MatDialogModule, MatDialogRef } from '@angular/material/dialog'; +import { MatButton } from '@angular/material/button'; +import { MatTableModule } from '@angular/material/table'; +import { MatTooltipModule } from '@angular/material/tooltip'; +import { MatProgressBarModule } from '@angular/material/progress-bar'; +import { Backlog, BacklogRequest } from '../../../../../../../state/shared'; +import { ProcessorBacklogDialogRequest } from '../../../../../state/flow'; +import { CloseOnEscapeDialog } from '@nifi/shared'; +import { FlowService } from '../../../../../service/flow.service'; +import { HttpErrorResponse } from '@angular/common/http'; +import { ErrorHelper } from '../../../../../../../service/error-helper.service'; +import { asyncScheduler, catchError, interval, of, Subscription, switchMap } from 'rxjs'; +import { takeUntilDestroyed } from '@angular/core/rxjs-interop'; + +/** + * How often to poll the backlog request while it remains incomplete. + */ +const POLL_INTERVAL_MILLIS = 2000; + +/** + * Row in the vertical two-column backlog table. + * - {@code label}: shown in the "Measurement" column. + * - {@code value}: formatted display string. {@code null} signals that the backlog response did not + * carry a value for this row and the cell should render the standard "No value set" placeholder + * used throughout the NiFi UI for unset property values. + * - {@code tooltip}: optional raw underlying value (for example, the raw byte count or the raw ISO + * timestamp) revealed when the user hovers over the value cell. Omitted when no underlying raw + * form exists separate from the displayed value, or when the value itself is unset. + */ +export interface BacklogRow { + label: string; + value: string | null; + tooltip?: string; +} + +/** + * Displays the outcome of an asynchronous backlog request for a single processor. Determining a + * backlog may require contacting an external system and can take a while to complete, so the + * request is created on the server and polled here until it either completes or fails. Closing + * the dialog by any means (the Cancel button, the OK button, the Escape key, or the backdrop) + * deletes the request on the server, which cancels the background determination if it has not + * yet completed. + */ +@Component({ + selector: 'processor-backlog-dialog', + imports: [MatButton, MatDialogModule, MatTableModule, MatTooltipModule, MatProgressBarModule], + templateUrl: './backlog-dialog.component.html', + styleUrls: ['./backlog-dialog.component.scss'] +}) +export class ProcessorBacklogDialog extends CloseOnEscapeDialog implements OnDestroy { + private dialogRequest = inject(MAT_DIALOG_DATA); + private backlogDialogRef = inject>(MatDialogRef); + private flowService = inject(FlowService); + private errorHelper = inject(ErrorHelper); + private destroyRef = inject(DestroyRef); + + private readonly processorId: string; + private requestId: string | null = null; + private requestDeleted = false; + private pollingSubscription: Subscription | null = null; + + readonly displayedColumns: string[] = ['measurement', 'value']; + rows: BacklogRow[] = []; + errorMessage?: string; + complete = false; + + constructor() { + super(); + this.processorId = this.dialogRequest.processorId; + + if (this.dialogRequest.errorMessage) { + this.errorMessage = this.dialogRequest.errorMessage; + this.complete = true; + } else if (this.dialogRequest.requestEntity) { + this.applyRequest(this.dialogRequest.requestEntity.request); + if (!this.complete) { + this.startPolling(); + } + } + } + + ngOnDestroy(): void { + this.deleteRequestIfNecessary(); + } + + cancel(): void { + this.backlogDialogRef.close(); + } + + private startPolling(): void { + this.pollingSubscription = interval(POLL_INTERVAL_MILLIS, asyncScheduler) + .pipe( + switchMap(() => + this.flowService.pollProcessorBacklogRequest(this.processorId, this.requestId as string) + ), + catchError((errorResponse: HttpErrorResponse) => { + this.stopPolling(); + this.errorMessage = this.errorHelper.getErrorString(errorResponse); + this.complete = true; + return of(null); + }), + takeUntilDestroyed(this.destroyRef) + ) + .subscribe((entity) => { + if (entity) { + this.applyRequest(entity.request); + if (this.complete) { + this.stopPolling(); + this.deleteRequestIfNecessary(); + } + } + }); + } + + private stopPolling(): void { + this.pollingSubscription?.unsubscribe(); + this.pollingSubscription = null; + } + + private applyRequest(request: BacklogRequest): void { + this.requestId = request.requestId; + this.complete = request.complete; + if (request.complete) { + if (request.failureReason) { + this.errorMessage = request.failureReason; + } else { + this.rows = this.buildRows(request.backlog); + } + } + } + + private deleteRequestIfNecessary(): void { + this.stopPolling(); + if (this.requestId && !this.requestDeleted) { + this.requestDeleted = true; + this.flowService.deleteProcessorBacklogRequest(this.processorId, this.requestId).subscribe(); + } + } + + private buildRows(backlog: Backlog | null | undefined): BacklogRow[] { + return [ + this.buildCountRow('FlowFiles', backlog?.flowFileCount, backlog?.formattedFlowFileCount), + this.buildDataSizeRow('Bytes', backlog?.byteCount, backlog?.formattedByteCount), + this.buildCountRow('Records', backlog?.recordCount, backlog?.formattedRecordCount), + this.buildPrecisionRow(backlog?.precision), + this.buildLastCaughtUpRow(backlog?.lastCaughtUp, backlog?.formattedLastCaughtUp) + ]; + } + + private buildCountRow(label: string, rawValue: number | undefined, formattedValue: string | undefined): BacklogRow { + if (rawValue == null) { + return { label, value: null }; + } + const displayed = formattedValue ?? String(rawValue); + return { label, value: displayed, tooltip: String(rawValue) }; + } + + private buildDataSizeRow( + label: string, + rawValue: number | undefined, + formattedValue: string | undefined + ): BacklogRow { + if (rawValue == null) { + return { label, value: null }; + } + const displayed = formattedValue ?? `${rawValue} B`; + return { label, value: displayed, tooltip: `${rawValue} bytes` }; + } + + private buildPrecisionRow(precision: string | undefined): BacklogRow { + if (precision == null) { + return { label: 'Precision', value: null }; + } + return { label: 'Precision', value: precision }; + } + + /** + * Render the "Last caught up" row as the absolute timestamp in the user's local timezone, paired + * with the server-computed relative-time string. The server emits {@code formattedLastCaughtUp} + * alongside the raw {@code lastCaughtUp} so that the relative phrase ("5 mins ago", "yesterday", + * "now", etc.) is consistent with the values shown for {@code formattedFlowFileCount}, + * {@code formattedByteCount}, and {@code formattedRecordCount} — all computed once on the server + * rather than recomputed on every UI render. The absolute portion stays on the client because + * formatting the timestamp in the user's local timezone requires the browser's locale. + */ + private buildLastCaughtUpRow( + lastCaughtUp: string | null | undefined, + formattedLastCaughtUp: string | undefined + ): BacklogRow { + if (lastCaughtUp == null) { + return { label: 'Last caught up', value: null }; + } + + const lastCaughtUpDate = new Date(lastCaughtUp); + if (Number.isNaN(lastCaughtUpDate.getTime())) { + return { label: 'Last caught up', value: null }; + } + + const formattedDate = formatLocalTimestampWithZone(lastCaughtUpDate); + if (formattedLastCaughtUp == null) { + return { + label: 'Last caught up', + value: formattedDate, + tooltip: lastCaughtUp + }; + } + return { + label: 'Last caught up', + value: `${formattedDate} (${formattedLastCaughtUp})`, + tooltip: lastCaughtUp + }; + } +} + +/** + * Format an absolute {@link Date} in the browser's local time using {@link Intl.DateTimeFormat} with + * {@code timeZoneName: 'short'}, producing strings such as {@code "2026-05-15 08:45:30 EDT"}. The + * Angular {@code DatePipe} 'z' symbol is unreliable here because its locale data typically falls back + * to a {@code "GMT-4"}-style offset when a specific timezone abbreviation is unavailable, whereas the + * ICU-backed {@link Intl} formatter consistently emits the browser's short timezone name. + */ +export function formatLocalTimestampWithZone(date: Date): string { + const formatter = new Intl.DateTimeFormat(undefined, { + year: 'numeric', + month: '2-digit', + day: '2-digit', + hour: '2-digit', + minute: '2-digit', + second: '2-digit', + hour12: false, + hourCycle: 'h23', + timeZoneName: 'short' + }); + const parts = formatter.formatToParts(date); + const partOfType = (type: Intl.DateTimeFormatPartTypes): string => + parts.find((part) => part.type === type)?.value ?? ''; + return `${partOfType('year')}-${partOfType('month')}-${partOfType('day')} ${partOfType('hour')}:${partOfType('minute')}:${partOfType('second')} ${partOfType('timeZoneName')}`; +} diff --git a/nifi-frontend/src/main/frontend/apps/nifi/src/app/state/shared/index.ts b/nifi-frontend/src/main/frontend/apps/nifi/src/app/state/shared/index.ts index dd0bf773a4dc..2472558dccb0 100644 --- a/nifi-frontend/src/main/frontend/apps/nifi/src/app/state/shared/index.ts +++ b/nifi-frontend/src/main/frontend/apps/nifi/src/app/state/shared/index.ts @@ -38,6 +38,35 @@ export interface OkDialogRequest { message: string; } +export interface Backlog { + flowFileCount?: number; + formattedFlowFileCount?: string; + byteCount?: number; + formattedByteCount?: string; + recordCount?: number; + formattedRecordCount?: string; + precision?: string; + lastCaughtUp?: string | null; + formattedLastCaughtUp?: string; +} + +export interface BacklogRequest { + requestId: string; + uri: string; + componentId: string; + submissionTime?: string; + lastUpdated?: string; + complete: boolean; + failureReason?: string | null; + percentCompleted: number; + state?: string; + backlog?: Backlog | null; +} + +export interface BacklogRequestEntity { + request: BacklogRequest; +} + export interface CancelDialogRequest { title: string; message: string; diff --git a/nifi-stateless/nifi-stateless-bundle/nifi-stateless-engine/src/main/java/org/apache/nifi/stateless/queue/StatelessFlowFileQueue.java b/nifi-stateless/nifi-stateless-bundle/nifi-stateless-engine/src/main/java/org/apache/nifi/stateless/queue/StatelessFlowFileQueue.java index 2c30d9168392..9d6a231363b6 100644 --- a/nifi-stateless/nifi-stateless-bundle/nifi-stateless-engine/src/main/java/org/apache/nifi/stateless/queue/StatelessFlowFileQueue.java +++ b/nifi-stateless/nifi-stateless-bundle/nifi-stateless-engine/src/main/java/org/apache/nifi/stateless/queue/StatelessFlowFileQueue.java @@ -19,6 +19,7 @@ import org.apache.nifi.components.connector.DropFlowFileSummary; import org.apache.nifi.controller.queue.DropFlowFileStatus; +import org.apache.nifi.controller.queue.FlowFileQueueSnapshot; import org.apache.nifi.controller.queue.ListFlowFileStatus; import org.apache.nifi.controller.queue.LoadBalanceCompression; import org.apache.nifi.controller.queue.LoadBalanceStrategy; @@ -106,6 +107,17 @@ public QueueSize size() { return new QueueSize(flowFiles.size() + unacknowledgedCount.get(), totalBytes.get()); } + @Override + public FlowFileQueueSnapshot getQueueSnapshot() { + // Stateless queues are accessed from a single flow execution thread, so a snapshot built from the + // thread-safe LinkedBlockingQueue plus the AtomicInteger/AtomicLong counters is consistent enough + // for backlog reporting. No explicit lock is needed because the stateless engine does not run + // concurrent enqueue/dequeue activity against this queue. + final List snapshotActive = new ArrayList<>(flowFiles); + final QueueSize snapshotSize = new QueueSize(snapshotActive.size() + unacknowledgedCount.get(), totalBytes.get()); + return new FlowFileQueueSnapshot(snapshotSize, snapshotActive); + } + @Override public long getTotalQueuedDuration(long fromTimestamp) { long sum = 0L; diff --git a/nifi-system-tests/nifi-system-test-extensions-bundle/nifi-system-test-extensions/src/main/java/org/apache/nifi/connectors/tests/system/BacklogReportingTestConnector.java b/nifi-system-tests/nifi-system-test-extensions-bundle/nifi-system-test-extensions/src/main/java/org/apache/nifi/connectors/tests/system/BacklogReportingTestConnector.java new file mode 100644 index 000000000000..5a698e7c37d3 --- /dev/null +++ b/nifi-system-tests/nifi-system-test-extensions-bundle/nifi-system-test-extensions/src/main/java/org/apache/nifi/connectors/tests/system/BacklogReportingTestConnector.java @@ -0,0 +1,343 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.connectors.tests.system; + +import org.apache.nifi.components.Backlog; +import org.apache.nifi.components.BacklogReportingException; +import org.apache.nifi.components.ConfigVerificationResult; +import org.apache.nifi.components.connector.AbstractConnector; +import org.apache.nifi.components.connector.BacklogReportingConnector; +import org.apache.nifi.components.connector.BundleCompatibility; +import org.apache.nifi.components.connector.ConfigurationStep; +import org.apache.nifi.components.connector.ConnectorPropertyDescriptor; +import org.apache.nifi.components.connector.ConnectorPropertyGroup; +import org.apache.nifi.components.connector.FlowUpdateException; +import org.apache.nifi.components.connector.PropertyType; +import org.apache.nifi.components.connector.StepConfigurationContext; +import org.apache.nifi.components.connector.components.ConnectionFacade; +import org.apache.nifi.components.connector.components.FlowContext; +import org.apache.nifi.components.connector.components.ProcessGroupFacade; +import org.apache.nifi.components.connector.components.ProcessorFacade; +import org.apache.nifi.components.connector.components.QueueSnapshot; +import org.apache.nifi.components.connector.util.VersionedFlowUtils; +import org.apache.nifi.flow.Bundle; +import org.apache.nifi.flow.Position; +import org.apache.nifi.flow.ScheduledState; +import org.apache.nifi.flow.VersionedConnection; +import org.apache.nifi.flow.VersionedExternalFlow; +import org.apache.nifi.flow.VersionedProcessGroup; +import org.apache.nifi.flow.VersionedProcessor; +import org.apache.nifi.flowfile.FlowFile; +import org.apache.nifi.processor.util.StandardValidators; + +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.Set; + +/** + * Test Connector that exercises Connector-level backlog reporting in three ways: + *
    + *
  • {@code DELEGATE_TO_PROCESSOR}: forwards {@code getBacklog} to the embedded + * {@code BacklogReportingTestProcessor} via {@link ProcessorFacade#getBacklog()}. + * Combined with the Processor's own {@code Backlog Mode} this also covers the + * {@link BacklogReportingException} and {@link RuntimeException} propagation paths, which the + * asynchronous backlog request flow surfaces as the request's failure reason.
    • + *
  • {@code READ_QUEUE_ATTRIBUTES}: reads the atomic queue snapshot from the + * queued connection in the managed flow, inspects FlowFile attributes, and computes + * a Backlog from what it observes. Used to verify that a Connector can access + * queued FlowFile attributes via {@link ConnectionFacade#getQueueSnapshot()}.
    • + *
  • {@code RETURN_EMPTY}: returns {@link Optional#empty()}, the default opt-out.
    • + *
+ * + *

The managed flow is {@code GenerateFlowFile -> BacklogReportingTestProcessor -> TerminateFlowFile}, + * with {@code TerminateFlowFile} {@link ScheduledState#DISABLED} so FlowFiles queue up on the + * downstream connection. The downstream connection's load-balance strategy is controlled by + * the {@code Load Balance Strategy} property so the clustered system tests can exercise both + * {@code StandardFlowFileQueue} and {@code SocketLoadBalancedFlowFileQueue}.

+ */ +public class BacklogReportingTestConnector extends AbstractConnector implements BacklogReportingConnector { + + static final String DELEGATE_TO_PROCESSOR = "DELEGATE_TO_PROCESSOR"; + static final String READ_QUEUE_ATTRIBUTES = "READ_QUEUE_ATTRIBUTES"; + static final String RETURN_EMPTY = "RETURN_EMPTY"; + + static final String PROCESSOR_BACKLOG_MODE_NORMAL = "NORMAL"; + static final String PROCESSOR_BACKLOG_MODE_THROW_BACKLOG_REPORTING_EXCEPTION = "THROW_BACKLOG_REPORTING_EXCEPTION"; + static final String PROCESSOR_BACKLOG_MODE_THROW_RUNTIME_EXCEPTION = "THROW_RUNTIME_EXCEPTION"; + + static final String LOAD_BALANCE_DO_NOT = "DO_NOT_LOAD_BALANCE"; + static final String LOAD_BALANCE_ROUND_ROBIN = "ROUND_ROBIN"; + + static final ConnectorPropertyDescriptor DELEGATION_MODE = new ConnectorPropertyDescriptor.Builder() + .name("Delegation Mode") + .description("How this Connector should compute its Backlog") + .type(PropertyType.STRING) + .required(true) + .defaultValue(DELEGATE_TO_PROCESSOR) + .allowableValues(DELEGATE_TO_PROCESSOR, READ_QUEUE_ATTRIBUTES, RETURN_EMPTY) + .build(); + + static final ConnectorPropertyDescriptor PROCESSOR_BACKLOG_MODE = new ConnectorPropertyDescriptor.Builder() + .name("Processor Backlog Mode") + .description("Forwarded to the embedded BacklogReportingTestProcessor's Backlog Mode property") + .type(PropertyType.STRING) + .required(true) + .defaultValue(PROCESSOR_BACKLOG_MODE_NORMAL) + .allowableValues(PROCESSOR_BACKLOG_MODE_NORMAL, PROCESSOR_BACKLOG_MODE_THROW_BACKLOG_REPORTING_EXCEPTION, PROCESSOR_BACKLOG_MODE_THROW_RUNTIME_EXCEPTION) + .build(); + + static final ConnectorPropertyDescriptor FLOWFILE_BACKLOG = new ConnectorPropertyDescriptor.Builder() + .name("FlowFile Backlog") + .description("Number of FlowFiles to report on the source") + .type(PropertyType.STRING) + .required(true) + .defaultValue("42") + .addValidator(StandardValidators.NON_NEGATIVE_INTEGER_VALIDATOR) + .build(); + + static final ConnectorPropertyDescriptor BYTE_BACKLOG = new ConnectorPropertyDescriptor.Builder() + .name("Byte Backlog") + .description("Number of bytes to report on the source") + .type(PropertyType.STRING) + .required(true) + .defaultValue("1024") + .addValidator(StandardValidators.NON_NEGATIVE_INTEGER_VALIDATOR) + .build(); + + static final ConnectorPropertyDescriptor RECORD_BACKLOG = new ConnectorPropertyDescriptor.Builder() + .name("Record Backlog") + .description("Number of records to report on the source") + .type(PropertyType.STRING) + .required(true) + .defaultValue("7") + .addValidator(StandardValidators.NON_NEGATIVE_INTEGER_VALIDATOR) + .build(); + + static final ConnectorPropertyDescriptor EXCEPTION_MESSAGE = new ConnectorPropertyDescriptor.Builder() + .name("Exception Message") + .description("Message included on the thrown exception when Processor Backlog Mode is one of the throw modes") + .type(PropertyType.STRING) + .required(true) + .defaultValue("Simulated backlog reporting failure") + .addValidator(StandardValidators.NON_EMPTY_VALIDATOR) + .build(); + + static final ConnectorPropertyDescriptor LOAD_BALANCE_STRATEGY = new ConnectorPropertyDescriptor.Builder() + .name("Load Balance Strategy") + .description("Load balance strategy to apply to the connection between the BacklogReportingTestProcessor and the terminal Processor") + .type(PropertyType.STRING) + .required(true) + .defaultValue(LOAD_BALANCE_DO_NOT) + .allowableValues(LOAD_BALANCE_DO_NOT, LOAD_BALANCE_ROUND_ROBIN) + .build(); + + private static final ConnectorPropertyGroup PROPERTY_GROUP = new ConnectorPropertyGroup.Builder() + .name("Backlog Configuration") + .description("Controls how this Connector and its embedded Processor report backlog") + .properties(List.of(DELEGATION_MODE, PROCESSOR_BACKLOG_MODE, FLOWFILE_BACKLOG, BYTE_BACKLOG, RECORD_BACKLOG, EXCEPTION_MESSAGE, LOAD_BALANCE_STRATEGY)) + .build(); + + private static final ConfigurationStep CONFIG_STEP = new ConfigurationStep.Builder() + .name("Backlog Configuration") + .description("Configure the embedded Processor's backlog behavior and the managed flow's load balance strategy") + .propertyGroups(List.of(PROPERTY_GROUP)) + .build(); + + @Override + protected void onStepConfigured(final String stepName, final FlowContext workingContext) { + } + + @Override + public List getConfigurationSteps() { + return List.of(CONFIG_STEP); + } + + @Override + public List verifyConfigurationStep(final String stepName, final Map propertyValueOverrides, final FlowContext flowContext) { + return List.of(); + } + + @Override + public VersionedExternalFlow getInitialFlow() { + return buildFlow( + PROCESSOR_BACKLOG_MODE_NORMAL, + "0", "0", "0", + "Simulated backlog reporting failure", + LOAD_BALANCE_DO_NOT); + } + + @Override + public VersionedExternalFlow getActiveFlow(final FlowContext activeFlowContext) { + // The authoritative Active flow is the flow produced by applying the current configuration. This + // mirrors applyUpdate so that exiting Troubleshooting restores a flow equivalent to re-applying + // the active configuration. + return buildFlow(activeFlowContext.getConfigurationContext().scopedToStep(CONFIG_STEP)); + } + + @Override + public void applyUpdate(final FlowContext workingFlowContext, final FlowContext activeFlowContext) throws FlowUpdateException { + final VersionedExternalFlow flow = buildFlow(workingFlowContext.getConfigurationContext().scopedToStep(CONFIG_STEP)); + getInitializationContext().updateFlow(activeFlowContext, flow, BundleCompatibility.RESOLVE_BUNDLE); + } + + private VersionedExternalFlow buildFlow(final StepConfigurationContext stepContext) { + final String processorMode = stepContext.getProperty(PROCESSOR_BACKLOG_MODE).getValue(); + final String flowFileBacklog = stepContext.getProperty(FLOWFILE_BACKLOG).getValue(); + final String byteBacklog = stepContext.getProperty(BYTE_BACKLOG).getValue(); + final String recordBacklog = stepContext.getProperty(RECORD_BACKLOG).getValue(); + final String exceptionMessage = stepContext.getProperty(EXCEPTION_MESSAGE).getValue(); + final String loadBalanceStrategy = stepContext.getProperty(LOAD_BALANCE_STRATEGY).getValue(); + return buildFlow(processorMode, flowFileBacklog, byteBacklog, recordBacklog, exceptionMessage, loadBalanceStrategy); + } + + private VersionedExternalFlow buildFlow(final String processorMode, final String flowFileBacklog, final String byteBacklog, + final String recordBacklog, final String exceptionMessage, final String loadBalanceStrategy) { + final Bundle bundle = new Bundle("org.apache.nifi", "nifi-system-test-extensions-nar", "2.8.0-SNAPSHOT"); + final VersionedProcessGroup rootGroup = VersionedFlowUtils.createProcessGroup("backlog-reporting-test-connector", "Backlog Reporting Test Connector"); + + final VersionedProcessor generate = VersionedFlowUtils.addProcessor(rootGroup, + "org.apache.nifi.processors.tests.system.GenerateFlowFile", bundle, "GenerateFlowFile", new Position(0, 0)); + generate.getProperties().putAll(Map.of( + "File Size", "1 B", + "Batch Size", "100", + "Max FlowFiles", "1000", + "flowFileIndex", "${nextInt()}" + )); + generate.setSchedulingPeriod("100 millis"); + + final VersionedProcessor backlogProcessor = VersionedFlowUtils.addProcessor(rootGroup, + "org.apache.nifi.processors.tests.system.BacklogReportingTestProcessor", bundle, "BacklogReportingTestProcessor", new Position(0, 100)); + backlogProcessor.getProperties().putAll(Map.of( + "Backlog Mode", processorMode, + "FlowFile Backlog", flowFileBacklog, + "Byte Backlog", byteBacklog, + "Record Backlog", recordBacklog, + "Exception Message", exceptionMessage + )); + + final VersionedProcessor terminate = VersionedFlowUtils.addProcessor(rootGroup, + "org.apache.nifi.processors.tests.system.TerminateFlowFile", bundle, "TerminateFlowFile", new Position(0, 200)); + terminate.setScheduledState(ScheduledState.DISABLED); + + final VersionedConnection upstream = VersionedFlowUtils.addConnection(rootGroup, + VersionedFlowUtils.createConnectableComponent(generate), + VersionedFlowUtils.createConnectableComponent(backlogProcessor), + Set.of("success")); + upstream.setBackPressureDataSizeThreshold("100 GB"); + upstream.setBackPressureObjectThreshold(100_000L); + + final VersionedConnection downstream = VersionedFlowUtils.addConnection(rootGroup, + VersionedFlowUtils.createConnectableComponent(backlogProcessor), + VersionedFlowUtils.createConnectableComponent(terminate), + Set.of("success")); + downstream.setBackPressureDataSizeThreshold("100 GB"); + downstream.setBackPressureObjectThreshold(100_000L); + downstream.setLoadBalanceStrategy(loadBalanceStrategy); + + final VersionedExternalFlow flow = new VersionedExternalFlow(); + flow.setFlowContents(rootGroup); + flow.setParameterContexts(Collections.emptyMap()); + return flow; + } + + @Override + public Optional getBacklog(final FlowContext activeFlowContext) throws BacklogReportingException { + final StepConfigurationContext stepContext = activeFlowContext.getConfigurationContext().scopedToStep(CONFIG_STEP); + final String delegationMode = stepContext.getProperty(DELEGATION_MODE).getValue(); + + if (RETURN_EMPTY.equals(delegationMode)) { + return Optional.empty(); + } + + if (DELEGATE_TO_PROCESSOR.equals(delegationMode)) { + final ProcessorFacade processor = findBacklogReportingTestProcessor(activeFlowContext) + .orElseThrow(() -> new BacklogReportingException("BacklogReportingTestProcessor not found in managed flow")); + return processor.getBacklog(); + } + + if (READ_QUEUE_ATTRIBUTES.equals(delegationMode)) { + return Optional.of(readBacklogFromQueue(activeFlowContext)); + } + + throw new BacklogReportingException("Unrecognized Delegation Mode: " + delegationMode); + } + + private Optional findBacklogReportingTestProcessor(final FlowContext flowContext) { + return findProcessors(flowContext.getRootGroup(), + processor -> processor.getDefinition().getType().endsWith("BacklogReportingTestProcessor")).stream().findFirst(); + } + + private Backlog readBacklogFromQueue(final FlowContext activeFlowContext) { + final ConnectionFacade downstreamConnection = findDownstreamConnection(activeFlowContext.getRootGroup()) + .orElseThrow(() -> new IllegalStateException("Could not find connection downstream of BacklogReportingTestProcessor")); + + final QueueSnapshot snapshot = downstreamConnection.getQueueSnapshot(); + long evenIndexCount = 0L; + long evenIndexBytes = 0L; + for (final FlowFile flowFile : snapshot.getActiveFlowFiles()) { + final String indexAttribute = flowFile.getAttribute("flowFileIndex"); + if (indexAttribute == null) { + continue; + } + final int index; + try { + index = Integer.parseInt(indexAttribute); + } catch (final NumberFormatException ignored) { + continue; + } + if (index % 2 == 0) { + evenIndexCount++; + evenIndexBytes += flowFile.getSize(); + } + } + + final Backlog.Precision precision = snapshot.isActiveListExhaustive() ? Backlog.Precision.EXACT : Backlog.Precision.AT_LEAST; + + return Backlog.builder() + .flowFiles(evenIndexCount) + .bytes(evenIndexBytes) + .records(evenIndexCount) + .precision(precision) + .build(); + } + + private Optional findDownstreamConnection(final ProcessGroupFacade group) { + for (final ConnectionFacade connection : group.getConnections()) { + final String sourceType = connection.getDefinition().getSource().getType().name(); + if (!"PROCESSOR".equals(sourceType)) { + continue; + } + + final String sourceId = connection.getDefinition().getSource().getId(); + for (final ProcessorFacade processor : group.getProcessors()) { + if (processor.getDefinition().getIdentifier().equals(sourceId) + && processor.getDefinition().getType().endsWith("BacklogReportingTestProcessor")) { + return Optional.of(connection); + } + } + } + for (final ProcessGroupFacade child : group.getProcessGroups()) { + final Optional found = findDownstreamConnection(child); + if (found.isPresent()) { + return found; + } + } + return Optional.empty(); + } +} diff --git a/nifi-system-tests/nifi-system-test-extensions-bundle/nifi-system-test-extensions/src/main/java/org/apache/nifi/processors/tests/system/BacklogReportingTestProcessor.java b/nifi-system-tests/nifi-system-test-extensions-bundle/nifi-system-test-extensions/src/main/java/org/apache/nifi/processors/tests/system/BacklogReportingTestProcessor.java new file mode 100644 index 000000000000..204b9e78d0b7 --- /dev/null +++ b/nifi-system-tests/nifi-system-test-extensions-bundle/nifi-system-test-extensions/src/main/java/org/apache/nifi/processors/tests/system/BacklogReportingTestProcessor.java @@ -0,0 +1,140 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.processors.tests.system; + +import org.apache.nifi.annotation.behavior.InputRequirement; +import org.apache.nifi.annotation.behavior.InputRequirement.Requirement; +import org.apache.nifi.components.AllowableValue; +import org.apache.nifi.components.Backlog; +import org.apache.nifi.components.BacklogReportingException; +import org.apache.nifi.components.PropertyDescriptor; +import org.apache.nifi.flowfile.FlowFile; +import org.apache.nifi.processor.AbstractProcessor; +import org.apache.nifi.processor.BacklogReportingProcessor; +import org.apache.nifi.processor.ProcessContext; +import org.apache.nifi.processor.ProcessSession; +import org.apache.nifi.processor.Relationship; +import org.apache.nifi.processor.exception.ProcessException; +import org.apache.nifi.processor.util.StandardValidators; + +import java.util.List; +import java.util.Optional; +import java.util.Set; + +/** + * Test Processor that implements {@link BacklogReportingProcessor} and exposes its behavior + * through property values. Used by backlog system tests to verify happy-path delegation as + * well as how the framework handles {@link BacklogReportingException} and {@link RuntimeException} + * thrown from {@code getBacklog}. + */ +@InputRequirement(Requirement.INPUT_ALLOWED) +public class BacklogReportingTestProcessor extends AbstractProcessor implements BacklogReportingProcessor { + + static final AllowableValue MODE_NORMAL = new AllowableValue("NORMAL", "Normal", + "Return a Backlog populated from the FlowFile Backlog, Byte Backlog and Record Backlog properties"); + static final AllowableValue MODE_THROW_BACKLOG_REPORTING_EXCEPTION = new AllowableValue("THROW_BACKLOG_REPORTING_EXCEPTION", + "Throw BacklogReportingException", "Throw a BacklogReportingException with the configured Exception Message"); + static final AllowableValue MODE_THROW_RUNTIME_EXCEPTION = new AllowableValue("THROW_RUNTIME_EXCEPTION", + "Throw RuntimeException", "Throw a RuntimeException with the configured Exception Message"); + + static final PropertyDescriptor BACKLOG_MODE = new PropertyDescriptor.Builder() + .name("Backlog Mode") + .description("Controls what getBacklog returns or throws") + .required(true) + .allowableValues(MODE_NORMAL, MODE_THROW_BACKLOG_REPORTING_EXCEPTION, MODE_THROW_RUNTIME_EXCEPTION) + .defaultValue(MODE_NORMAL.getValue()) + .build(); + + static final PropertyDescriptor FLOWFILE_BACKLOG = new PropertyDescriptor.Builder() + .name("FlowFile Backlog") + .description("Number of FlowFiles to report on the source") + .required(true) + .addValidator(StandardValidators.NON_NEGATIVE_INTEGER_VALIDATOR) + .defaultValue("0") + .build(); + + static final PropertyDescriptor BYTE_BACKLOG = new PropertyDescriptor.Builder() + .name("Byte Backlog") + .description("Number of bytes to report on the source") + .required(true) + .addValidator(StandardValidators.NON_NEGATIVE_INTEGER_VALIDATOR) + .defaultValue("0") + .build(); + + static final PropertyDescriptor RECORD_BACKLOG = new PropertyDescriptor.Builder() + .name("Record Backlog") + .description("Number of records to report on the source") + .required(true) + .addValidator(StandardValidators.NON_NEGATIVE_INTEGER_VALIDATOR) + .defaultValue("0") + .build(); + + static final PropertyDescriptor EXCEPTION_MESSAGE = new PropertyDescriptor.Builder() + .name("Exception Message") + .description("Message to include on the thrown exception when Backlog Mode is one of the throw modes") + .required(false) + .addValidator(StandardValidators.NON_EMPTY_VALIDATOR) + .defaultValue("Simulated backlog reporting failure") + .build(); + + static final Relationship REL_SUCCESS = new Relationship.Builder() + .name("success") + .description("All incoming FlowFiles are transferred here") + .build(); + + @Override + protected List getSupportedPropertyDescriptors() { + return List.of(BACKLOG_MODE, FLOWFILE_BACKLOG, BYTE_BACKLOG, RECORD_BACKLOG, EXCEPTION_MESSAGE); + } + + @Override + public Set getRelationships() { + return Set.of(REL_SUCCESS); + } + + @Override + public void onTrigger(final ProcessContext context, final ProcessSession session) throws ProcessException { + final FlowFile flowFile = session.get(); + if (flowFile == null) { + return; + } + + session.transfer(flowFile, REL_SUCCESS); + } + + @Override + public Optional getBacklog(final ProcessContext context) throws BacklogReportingException { + final String mode = context.getProperty(BACKLOG_MODE).getValue(); + + if (MODE_THROW_BACKLOG_REPORTING_EXCEPTION.getValue().equals(mode)) { + throw new BacklogReportingException(context.getProperty(EXCEPTION_MESSAGE).getValue()); + } + if (MODE_THROW_RUNTIME_EXCEPTION.getValue().equals(mode)) { + throw new RuntimeException(context.getProperty(EXCEPTION_MESSAGE).getValue()); + } + + final long flowFiles = context.getProperty(FLOWFILE_BACKLOG).asLong(); + final long bytes = context.getProperty(BYTE_BACKLOG).asLong(); + final long records = context.getProperty(RECORD_BACKLOG).asLong(); + + return Optional.of(Backlog.builder() + .flowFiles(flowFiles) + .bytes(bytes) + .records(records) + .build()); + } +} diff --git a/nifi-system-tests/nifi-system-test-extensions-bundle/nifi-system-test-extensions/src/main/resources/META-INF/services/org.apache.nifi.components.connector.Connector b/nifi-system-tests/nifi-system-test-extensions-bundle/nifi-system-test-extensions/src/main/resources/META-INF/services/org.apache.nifi.components.connector.Connector index a2d46427433a..9c5d82e86d3f 100644 --- a/nifi-system-tests/nifi-system-test-extensions-bundle/nifi-system-test-extensions/src/main/resources/META-INF/services/org.apache.nifi.components.connector.Connector +++ b/nifi-system-tests/nifi-system-test-extensions-bundle/nifi-system-test-extensions/src/main/resources/META-INF/services/org.apache.nifi.components.connector.Connector @@ -14,6 +14,7 @@ # limitations under the License. org.apache.nifi.connectors.tests.system.AssetConnector +org.apache.nifi.connectors.tests.system.BacklogReportingTestConnector org.apache.nifi.connectors.tests.system.BundleResolutionConnector org.apache.nifi.connectors.tests.system.CalculateConnector org.apache.nifi.connectors.tests.system.ComponentLifecycleConnector diff --git a/nifi-system-tests/nifi-system-test-extensions-bundle/nifi-system-test-extensions/src/main/resources/META-INF/services/org.apache.nifi.processor.Processor b/nifi-system-tests/nifi-system-test-extensions-bundle/nifi-system-test-extensions/src/main/resources/META-INF/services/org.apache.nifi.processor.Processor index 80655da6f369..a0e104fb589b 100644 --- a/nifi-system-tests/nifi-system-test-extensions-bundle/nifi-system-test-extensions/src/main/resources/META-INF/services/org.apache.nifi.processor.Processor +++ b/nifi-system-tests/nifi-system-test-extensions-bundle/nifi-system-test-extensions/src/main/resources/META-INF/services/org.apache.nifi.processor.Processor @@ -13,6 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. +org.apache.nifi.processors.tests.system.BacklogReportingTestProcessor org.apache.nifi.processors.tests.system.ClassloaderIsolationWithServiceProperty org.apache.nifi.processors.tests.system.CountEvents org.apache.nifi.processors.tests.system.CountFlowFiles diff --git a/nifi-system-tests/nifi-system-test-suite/src/test/java/org/apache/nifi/tests/system/NiFiClientUtil.java b/nifi-system-tests/nifi-system-test-suite/src/test/java/org/apache/nifi/tests/system/NiFiClientUtil.java index 3cb5d07dbdd2..e6ea06cacae5 100644 --- a/nifi-system-tests/nifi-system-test-suite/src/test/java/org/apache/nifi/tests/system/NiFiClientUtil.java +++ b/nifi-system-tests/nifi-system-test-suite/src/test/java/org/apache/nifi/tests/system/NiFiClientUtil.java @@ -34,6 +34,7 @@ import org.apache.nifi.toolkit.client.ProcessorClient; import org.apache.nifi.toolkit.client.VersionsClient; import org.apache.nifi.web.api.dto.AssetReferenceDTO; +import org.apache.nifi.web.api.dto.BacklogDTO; import org.apache.nifi.web.api.dto.BundleDTO; import org.apache.nifi.web.api.dto.ConfigVerificationResultDTO; import org.apache.nifi.web.api.dto.ConfigurationStepConfigurationDTO; @@ -76,6 +77,7 @@ import org.apache.nifi.web.api.dto.status.ProcessGroupStatusSnapshotDTO; import org.apache.nifi.web.api.dto.status.ProcessorStatusSnapshotDTO; import org.apache.nifi.web.api.entity.ActivateControllerServicesEntity; +import org.apache.nifi.web.api.entity.BacklogRequestEntity; import org.apache.nifi.web.api.entity.ConfigurationStepEntity; import org.apache.nifi.web.api.entity.ConnectionEntity; import org.apache.nifi.web.api.entity.ConnectionStatusEntity; @@ -2292,6 +2294,60 @@ public List verifyControllerServiceConfig(final Str return results.getRequest().getResults(); } + /** + * Determines the backlog for the given Processor using the asynchronous backlog request flow: it submits a + * request, polls until the request is complete, deletes the request, and then returns the reported backlog. If + * the backlog determination failed, the request's failure reason is raised as a {@link NiFiClientException} so + * that callers observe the same failure they previously observed from the synchronous endpoint. + * + * @param processorId the Processor id + * @return the reported {@link BacklogDTO}, which may be {@code null} if the Processor reports no backlog + * @throws NiFiClientException if the request fails or the backlog determination reports a failure reason + */ + public BacklogDTO getProcessorBacklog(final String processorId) throws NiFiClientException, IOException, InterruptedException { + BacklogRequestEntity results = nifiClient.getProcessorClient().submitProcessorBacklogRequest(processorId); + while (!results.getRequest().isComplete()) { + Thread.sleep(50L); + results = nifiClient.getProcessorClient().getProcessorBacklogRequest(processorId, results.getRequest().getRequestId()); + } + + nifiClient.getProcessorClient().deleteProcessorBacklogRequest(processorId, results.getRequest().getRequestId()); + + final String failureReason = results.getRequest().getFailureReason(); + if (failureReason != null) { + throw new NiFiClientException(failureReason); + } + + return results.getRequest().getBacklog(); + } + + /** + * Determines the backlog for the given Connector using the asynchronous backlog request flow: it submits a + * request, polls until the request is complete, deletes the request, and then returns the reported backlog. If + * the backlog determination failed, the request's failure reason is raised as a {@link NiFiClientException} so + * that callers observe the same failure they previously observed from the synchronous endpoint. + * + * @param connectorId the Connector id + * @return the reported {@link BacklogDTO}, which may be {@code null} if the Connector reports no backlog + * @throws NiFiClientException if the request fails or the backlog determination reports a failure reason + */ + public BacklogDTO getConnectorBacklog(final String connectorId) throws NiFiClientException, IOException, InterruptedException { + BacklogRequestEntity results = nifiClient.getConnectorClient().submitConnectorBacklogRequest(connectorId); + while (!results.getRequest().isComplete()) { + Thread.sleep(50L); + results = nifiClient.getConnectorClient().getConnectorBacklogRequest(connectorId, results.getRequest().getRequestId()); + } + + nifiClient.getConnectorClient().deleteConnectorBacklogRequest(connectorId, results.getRequest().getRequestId()); + + final String failureReason = results.getRequest().getFailureReason(); + if (failureReason != null) { + throw new NiFiClientException(failureReason); + } + + return results.getRequest().getBacklog(); + } + public List verifyReportingTaskConfig(final String taskId, final Map properties) throws InterruptedException, IOException, NiFiClientException { diff --git a/nifi-system-tests/nifi-system-test-suite/src/test/java/org/apache/nifi/tests/system/connectors/ClusteredConnectorBacklogIT.java b/nifi-system-tests/nifi-system-test-suite/src/test/java/org/apache/nifi/tests/system/connectors/ClusteredConnectorBacklogIT.java new file mode 100644 index 000000000000..13b28c267dab --- /dev/null +++ b/nifi-system-tests/nifi-system-test-suite/src/test/java/org/apache/nifi/tests/system/connectors/ClusteredConnectorBacklogIT.java @@ -0,0 +1,111 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.nifi.tests.system.connectors; + +import org.apache.nifi.tests.system.NiFiInstanceFactory; +import org.apache.nifi.tests.system.NiFiSystemIT; +import org.apache.nifi.toolkit.client.NiFiClientException; +import org.apache.nifi.web.api.dto.BacklogDTO; +import org.apache.nifi.web.api.entity.ConnectorEntity; +import org.junit.jupiter.api.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; +import java.util.Map; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * Cluster-specific verification that Connector backlog aggregates correctly across nodes and that + * the queue-snapshot path works against both {@code StandardFlowFileQueue} (normal connections) + * and {@code SocketLoadBalancedFlowFileQueue} (load-balanced connections). + */ +public class ClusteredConnectorBacklogIT extends NiFiSystemIT { + + private static final Logger logger = LoggerFactory.getLogger(ClusteredConnectorBacklogIT.class); + + private static final int CLUSTER_SIZE = 2; + + @Override + public NiFiInstanceFactory getInstanceFactory() { + return createTwoNodeInstanceFactory(); + } + + @Test + public void testClusteredDelegateBacklogAggregatesAcrossNodes() throws NiFiClientException, IOException, InterruptedException { + final long perNodeFlowFiles = 42L; + final long perNodeBytes = 1024L; + final long perNodeRecords = 7L; + + final ConnectorEntity connector = ConnectorBacklogIT.configureBacklogConnector(getClientUtil(), Map.of( + "Delegation Mode", "DELEGATE_TO_PROCESSOR", + "Processor Backlog Mode", "NORMAL", + "FlowFile Backlog", String.valueOf(perNodeFlowFiles), + "Byte Backlog", String.valueOf(perNodeBytes), + "Record Backlog", String.valueOf(perNodeRecords), + "Load Balance Strategy", "DO_NOT_LOAD_BALANCE")); + + final BacklogDTO backlog = getClientUtil().getConnectorBacklog(connector.getId()); + + assertNotNull(backlog); + assertEquals(Long.valueOf(perNodeFlowFiles * CLUSTER_SIZE), backlog.getFlowFileCount()); + assertEquals(Long.valueOf(perNodeBytes * CLUSTER_SIZE), backlog.getByteCount()); + assertEquals(Long.valueOf(perNodeRecords * CLUSTER_SIZE), backlog.getRecordCount()); + } + + @Test + public void testClusteredReadQueueAttributesNormalConnection() throws NiFiClientException, IOException, InterruptedException { + runQueueAttributeAggregationScenario("DO_NOT_LOAD_BALANCE"); + } + + @Test + public void testClusteredReadQueueAttributesLoadBalancedConnection() throws NiFiClientException, IOException, InterruptedException { + runQueueAttributeAggregationScenario("ROUND_ROBIN"); + } + + private void runQueueAttributeAggregationScenario(final String loadBalanceStrategy) throws NiFiClientException, IOException, InterruptedException { + final ConnectorEntity connector = ConnectorBacklogIT.configureBacklogConnector(getClientUtil(), Map.of( + "Delegation Mode", "READ_QUEUE_ATTRIBUTES", + "Processor Backlog Mode", "NORMAL", + "FlowFile Backlog", "0", + "Byte Backlog", "0", + "Record Backlog", "0", + "Load Balance Strategy", loadBalanceStrategy)); + + getClientUtil().startConnector(connector.getId()); + waitForConnectorMinQueueCount(connector.getId(), 40); + getClientUtil().stopConnector(connector.getId()); + + final int totalQueued = getConnectorQueuedFlowFileCount(connector.getId()); + logger.info("Load Balance Strategy {}: cluster has {} queued FlowFiles before reading backlog", loadBalanceStrategy, totalQueued); + assertTrue(totalQueued > 0); + + final BacklogDTO backlog = getClientUtil().getConnectorBacklog(connector.getId()); + assertNotNull(backlog); + + final long flowFiles = backlog.getFlowFileCount(); + assertTrue(flowFiles >= 0); + assertTrue(flowFiles <= totalQueued, + "Cluster aggregate FlowFile count from queue snapshot (" + flowFiles + ") must not exceed total queued count (" + totalQueued + ")"); + assertEquals(Long.valueOf(flowFiles), backlog.getRecordCount()); + assertTrue(backlog.getByteCount() >= 0L); + } +} diff --git a/nifi-system-tests/nifi-system-test-suite/src/test/java/org/apache/nifi/tests/system/connectors/ConnectorBacklogIT.java b/nifi-system-tests/nifi-system-test-suite/src/test/java/org/apache/nifi/tests/system/connectors/ConnectorBacklogIT.java new file mode 100644 index 000000000000..a622dbf432da --- /dev/null +++ b/nifi-system-tests/nifi-system-test-suite/src/test/java/org/apache/nifi/tests/system/connectors/ConnectorBacklogIT.java @@ -0,0 +1,190 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.nifi.tests.system.connectors; + +import org.apache.nifi.tests.system.NiFiClientUtil; +import org.apache.nifi.tests.system.NiFiSystemIT; +import org.apache.nifi.toolkit.client.NiFiClientException; +import org.apache.nifi.web.api.dto.BacklogDTO; +import org.apache.nifi.web.api.entity.ConnectorEntity; +import org.junit.jupiter.api.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; +import java.util.Map; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * Verifies Connector backlog reporting end to end in a single-node NiFi: + *
    + *
  • Connector delegates {@code getBacklog} to an embedded {@code BacklogReportingTestProcessor}.
    • + *
  • Connector reads its own queue snapshot, accesses FlowFile attributes, and reports a Backlog from them.
    • + *
  • Errors thrown by the embedded Processor's {@code getBacklog} are surfaced as the backlog request's + * failure reason, both for {@link org.apache.nifi.components.BacklogReportingException} and for unexpected + * {@link RuntimeException}s.
    • + *
  • A Connector that opts out of backlog reporting returns a {@code null} backlog.
    • + *
+ */ +public class ConnectorBacklogIT extends NiFiSystemIT { + + private static final Logger logger = LoggerFactory.getLogger(ConnectorBacklogIT.class); + + private static final String STEP_NAME = "Backlog Configuration"; + + @Test + public void testConnectorDelegatesBacklogToProcessor() throws NiFiClientException, IOException, InterruptedException { + final ConnectorEntity connector = configureBacklogConnector(Map.of( + "Delegation Mode", "DELEGATE_TO_PROCESSOR", + "Processor Backlog Mode", "NORMAL", + "FlowFile Backlog", "42", + "Byte Backlog", "1024", + "Record Backlog", "7", + "Load Balance Strategy", "DO_NOT_LOAD_BALANCE")); + + final BacklogDTO backlog = getClientUtil().getConnectorBacklog(connector.getId()); + + assertNotNull(backlog); + assertEquals(Long.valueOf(42L), backlog.getFlowFileCount()); + assertEquals(Long.valueOf(1024L), backlog.getByteCount()); + assertEquals(Long.valueOf(7L), backlog.getRecordCount()); + assertEquals("EXACT", backlog.getPrecision()); + } + + @Test + public void testConnectorReadsFlowFileAttributesFromQueueSnapshot() throws NiFiClientException, IOException, InterruptedException { + final ConnectorEntity connector = configureBacklogConnector(Map.of( + "Delegation Mode", "READ_QUEUE_ATTRIBUTES", + "Processor Backlog Mode", "NORMAL", + "FlowFile Backlog", "0", + "Byte Backlog", "0", + "Record Backlog", "0", + "Load Balance Strategy", "DO_NOT_LOAD_BALANCE")); + + getClientUtil().startConnector(connector.getId()); + waitForConnectorMinQueueCount(connector.getId(), 20); + getClientUtil().stopConnector(connector.getId()); + + final int totalQueued = getConnectorQueuedFlowFileCount(connector.getId()); + logger.info("Total queued FlowFiles before reading backlog: {}", totalQueued); + assertTrue(totalQueued > 0); + + final BacklogDTO backlog = getClientUtil().getConnectorBacklog(connector.getId()); + assertNotNull(backlog); + + final long flowFiles = backlog.getFlowFileCount(); + assertTrue(flowFiles >= 0); + assertTrue(flowFiles <= totalQueued, "FlowFile count read from queue (" + flowFiles + ") must not exceed total queued count (" + totalQueued + ")"); + assertEquals(Long.valueOf(flowFiles), backlog.getRecordCount()); + assertTrue(backlog.getByteCount() >= 0L); + } + + @Test + public void testProcessorBacklogReportingExceptionReportedAsFailure() throws NiFiClientException, IOException, InterruptedException { + final String message = "Simulated BacklogReportingException for ConnectorBacklogIT"; + final ConnectorEntity connector = configureBacklogConnector(Map.of( + "Delegation Mode", "DELEGATE_TO_PROCESSOR", + "Processor Backlog Mode", "THROW_BACKLOG_REPORTING_EXCEPTION", + "FlowFile Backlog", "0", + "Byte Backlog", "0", + "Record Backlog", "0", + "Exception Message", message, + "Load Balance Strategy", "DO_NOT_LOAD_BALANCE")); + + final NiFiClientException thrown = assertThrows(NiFiClientException.class, + () -> getClientUtil().getConnectorBacklog(connector.getId())); + assertTrue(thrown.getMessage().contains(message), + "Expected backlog request failure reason to contain configured exception message but was: " + thrown.getMessage()); + } + + @Test + public void testProcessorRuntimeExceptionReportedAsFailure() throws NiFiClientException, IOException, InterruptedException { + final String message = "Simulated RuntimeException for ConnectorBacklogIT"; + final ConnectorEntity connector = configureBacklogConnector(Map.of( + "Delegation Mode", "DELEGATE_TO_PROCESSOR", + "Processor Backlog Mode", "THROW_RUNTIME_EXCEPTION", + "FlowFile Backlog", "0", + "Byte Backlog", "0", + "Record Backlog", "0", + "Exception Message", message, + "Load Balance Strategy", "DO_NOT_LOAD_BALANCE")); + + final NiFiClientException thrown = assertThrows(NiFiClientException.class, + () -> getClientUtil().getConnectorBacklog(connector.getId())); + assertTrue(thrown.getMessage().contains(message), + "Expected backlog request failure reason to contain configured exception message but was: " + thrown.getMessage()); + } + + @Test + public void testConnectorReturnsEmptyBacklog() throws NiFiClientException, IOException, InterruptedException { + final ConnectorEntity connector = configureBacklogConnector(Map.of( + "Delegation Mode", "RETURN_EMPTY", + "Processor Backlog Mode", "NORMAL", + "FlowFile Backlog", "0", + "Byte Backlog", "0", + "Record Backlog", "0", + "Load Balance Strategy", "DO_NOT_LOAD_BALANCE")); + + final BacklogDTO backlog = getClientUtil().getConnectorBacklog(connector.getId()); + assertNull(backlog); + } + + /** + * A Connector that delegates to a Processor reporting a fully caught-up source returns a non-null + * {@link BacklogDTO} carrying explicit zero counts. This "empty backlog with zero counts" result is + * distinct from {@link #testConnectorReturnsEmptyBacklog()}, where a Connector that opts out of backlog + * reporting returns a null {@code backlog}. A caller must be able to tell "the source has exactly zero + * remaining work" (zero counts) apart from "no backlog information is available" (null backlog). + */ + @Test + public void testConnectorDelegatesZeroCountBacklog() throws NiFiClientException, IOException, InterruptedException { + final ConnectorEntity connector = configureBacklogConnector(Map.of( + "Delegation Mode", "DELEGATE_TO_PROCESSOR", + "Processor Backlog Mode", "NORMAL", + "FlowFile Backlog", "0", + "Byte Backlog", "0", + "Record Backlog", "0", + "Load Balance Strategy", "DO_NOT_LOAD_BALANCE")); + + final BacklogDTO backlog = getClientUtil().getConnectorBacklog(connector.getId()); + + assertNotNull(backlog); + assertEquals(Long.valueOf(0L), backlog.getFlowFileCount()); + assertEquals(Long.valueOf(0L), backlog.getByteCount()); + assertEquals(Long.valueOf(0L), backlog.getRecordCount()); + assertEquals("EXACT", backlog.getPrecision()); + } + + static ConnectorEntity configureBacklogConnector(final NiFiClientUtil clientUtil, final Map properties) throws NiFiClientException, IOException, InterruptedException { + final ConnectorEntity connector = clientUtil.createConnector("BacklogReportingTestConnector"); + assertNotNull(connector); + clientUtil.configureConnector(connector, STEP_NAME, properties); + clientUtil.applyConnectorUpdate(connector); + clientUtil.waitForValidConnector(connector.getId()); + return connector; + } + + private ConnectorEntity configureBacklogConnector(final Map properties) throws NiFiClientException, IOException, InterruptedException { + return configureBacklogConnector(getClientUtil(), properties); + } +} diff --git a/nifi-toolkit/nifi-toolkit-client/src/main/java/org/apache/nifi/toolkit/client/ConnectorClient.java b/nifi-toolkit/nifi-toolkit-client/src/main/java/org/apache/nifi/toolkit/client/ConnectorClient.java index 76f49df4b07c..b674ed70723e 100644 --- a/nifi-toolkit/nifi-toolkit-client/src/main/java/org/apache/nifi/toolkit/client/ConnectorClient.java +++ b/nifi-toolkit/nifi-toolkit-client/src/main/java/org/apache/nifi/toolkit/client/ConnectorClient.java @@ -18,6 +18,7 @@ import org.apache.nifi.web.api.entity.AssetEntity; import org.apache.nifi.web.api.entity.AssetsEntity; +import org.apache.nifi.web.api.entity.BacklogRequestEntity; import org.apache.nifi.web.api.entity.ComponentStateEntity; import org.apache.nifi.web.api.entity.ConfigurationStepEntity; import org.apache.nifi.web.api.entity.ConfigurationStepNamesEntity; @@ -478,6 +479,38 @@ VerifyConnectorConfigStepRequestEntity deleteConfigStepVerificationRequest(Strin */ ComponentStateEntity clearControllerServiceState(String connectorId, String controllerServiceId) throws NiFiClientException, IOException; + /** + * Initiates an asynchronous request to determine the current backlog for the Connector with the given id. + * + * @param connectorId the Connector id + * @return the created backlog request entity + * @throws NiFiClientException if an error occurs during the request + * @throws IOException if an I/O error occurs + */ + BacklogRequestEntity submitConnectorBacklogRequest(String connectorId) throws NiFiClientException, IOException; + + /** + * Retrieves the current status of a previously submitted Connector backlog request. + * + * @param connectorId the Connector id + * @param requestId the backlog request id + * @return the backlog request entity describing the current status of the request + * @throws NiFiClientException if an error occurs during the request + * @throws IOException if an I/O error occurs + */ + BacklogRequestEntity getConnectorBacklogRequest(String connectorId, String requestId) throws NiFiClientException, IOException; + + /** + * Deletes a previously submitted Connector backlog request, cancelling it if it has not yet completed. + * + * @param connectorId the Connector id + * @param requestId the backlog request id + * @return the backlog request entity describing the deleted request + * @throws NiFiClientException if an error occurs during the request + * @throws IOException if an I/O error occurs + */ + BacklogRequestEntity deleteConnectorBacklogRequest(String connectorId, String requestId) throws NiFiClientException, IOException; + /** * Indicates that mutable requests should indicate that the client has * acknowledged that the node is disconnected. diff --git a/nifi-toolkit/nifi-toolkit-client/src/main/java/org/apache/nifi/toolkit/client/ProcessorClient.java b/nifi-toolkit/nifi-toolkit-client/src/main/java/org/apache/nifi/toolkit/client/ProcessorClient.java index 247065b31391..da4d2a41c85d 100644 --- a/nifi-toolkit/nifi-toolkit-client/src/main/java/org/apache/nifi/toolkit/client/ProcessorClient.java +++ b/nifi-toolkit/nifi-toolkit-client/src/main/java/org/apache/nifi/toolkit/client/ProcessorClient.java @@ -16,6 +16,7 @@ */ package org.apache.nifi.toolkit.client; +import org.apache.nifi.web.api.entity.BacklogRequestEntity; import org.apache.nifi.web.api.entity.ComponentStateEntity; import org.apache.nifi.web.api.entity.ProcessorEntity; import org.apache.nifi.web.api.entity.PropertyDescriptorEntity; @@ -68,6 +69,12 @@ default ComponentStateEntity clearProcessorState(String processorId) throws NiFi ComponentStateEntity getProcessorState(String processorId) throws NiFiClientException, IOException; + BacklogRequestEntity submitProcessorBacklogRequest(String processorId) throws NiFiClientException, IOException; + + BacklogRequestEntity getProcessorBacklogRequest(String processorId, String requestId) throws NiFiClientException, IOException; + + BacklogRequestEntity deleteProcessorBacklogRequest(String processorId, String requestId) throws NiFiClientException, IOException; + /** * Indicates that mutable requests should indicate that the client has * acknowledged that the node is disconnected. diff --git a/nifi-toolkit/nifi-toolkit-client/src/main/java/org/apache/nifi/toolkit/client/impl/JerseyConnectorClient.java b/nifi-toolkit/nifi-toolkit-client/src/main/java/org/apache/nifi/toolkit/client/impl/JerseyConnectorClient.java index 915091d08e58..35db04f05307 100644 --- a/nifi-toolkit/nifi-toolkit-client/src/main/java/org/apache/nifi/toolkit/client/impl/JerseyConnectorClient.java +++ b/nifi-toolkit/nifi-toolkit-client/src/main/java/org/apache/nifi/toolkit/client/impl/JerseyConnectorClient.java @@ -27,6 +27,7 @@ import org.apache.nifi.web.api.dto.RevisionDTO; import org.apache.nifi.web.api.entity.AssetEntity; import org.apache.nifi.web.api.entity.AssetsEntity; +import org.apache.nifi.web.api.entity.BacklogRequestEntity; import org.apache.nifi.web.api.entity.ComponentStateEntity; import org.apache.nifi.web.api.entity.ConfigurationStepEntity; import org.apache.nifi.web.api.entity.ConfigurationStepNamesEntity; @@ -805,4 +806,57 @@ public ComponentStateEntity clearControllerServiceState(final String connectorId return getRequestBuilder(target).post(null, ComponentStateEntity.class); }); } + + @Override + public BacklogRequestEntity submitConnectorBacklogRequest(final String connectorId) throws NiFiClientException, IOException { + if (StringUtils.isBlank(connectorId)) { + throw new IllegalArgumentException("Connector id cannot be null or blank"); + } + + return executeAction("Error submitting Backlog Request for Connector " + connectorId, () -> { + final WebTarget target = connectorTarget + .path("/backlog-requests") + .resolveTemplate("id", connectorId); + + return getRequestBuilder(target).post(null, BacklogRequestEntity.class); + }); + } + + @Override + public BacklogRequestEntity getConnectorBacklogRequest(final String connectorId, final String requestId) throws NiFiClientException, IOException { + if (StringUtils.isBlank(connectorId)) { + throw new IllegalArgumentException("Connector id cannot be null or blank"); + } + if (StringUtils.isBlank(requestId)) { + throw new IllegalArgumentException("Backlog Request id cannot be null or blank"); + } + + return executeAction("Error retrieving Backlog Request for Connector " + connectorId, () -> { + final WebTarget target = connectorTarget + .path("/backlog-requests/{requestId}") + .resolveTemplate("id", connectorId) + .resolveTemplate("requestId", requestId); + + return getRequestBuilder(target).get(BacklogRequestEntity.class); + }); + } + + @Override + public BacklogRequestEntity deleteConnectorBacklogRequest(final String connectorId, final String requestId) throws NiFiClientException, IOException { + if (StringUtils.isBlank(connectorId)) { + throw new IllegalArgumentException("Connector id cannot be null or blank"); + } + if (StringUtils.isBlank(requestId)) { + throw new IllegalArgumentException("Backlog Request id cannot be null or blank"); + } + + return executeAction("Error deleting Backlog Request for Connector " + connectorId, () -> { + final WebTarget target = connectorTarget + .path("/backlog-requests/{requestId}") + .resolveTemplate("id", connectorId) + .resolveTemplate("requestId", requestId); + + return getRequestBuilder(target).delete(BacklogRequestEntity.class); + }); + } } diff --git a/nifi-toolkit/nifi-toolkit-client/src/main/java/org/apache/nifi/toolkit/client/impl/JerseyProcessorClient.java b/nifi-toolkit/nifi-toolkit-client/src/main/java/org/apache/nifi/toolkit/client/impl/JerseyProcessorClient.java index a194e3c6237a..8d2f16917dc5 100644 --- a/nifi-toolkit/nifi-toolkit-client/src/main/java/org/apache/nifi/toolkit/client/impl/JerseyProcessorClient.java +++ b/nifi-toolkit/nifi-toolkit-client/src/main/java/org/apache/nifi/toolkit/client/impl/JerseyProcessorClient.java @@ -24,6 +24,7 @@ import org.apache.nifi.toolkit.client.ProcessorClient; import org.apache.nifi.toolkit.client.RequestConfig; import org.apache.nifi.web.api.dto.RevisionDTO; +import org.apache.nifi.web.api.entity.BacklogRequestEntity; import org.apache.nifi.web.api.entity.ComponentStateEntity; import org.apache.nifi.web.api.entity.ProcessorEntity; import org.apache.nifi.web.api.entity.ProcessorRunStatusEntity; @@ -311,4 +312,57 @@ public ComponentStateEntity getProcessorState(String processorId) throws NiFiCli return getRequestBuilder(target).get(ComponentStateEntity.class); }); } + + @Override + public BacklogRequestEntity submitProcessorBacklogRequest(final String processorId) throws NiFiClientException, IOException { + if (StringUtils.isBlank(processorId)) { + throw new IllegalArgumentException("Processor ID cannot be null or blank"); + } + + return executeAction("Error submitting Backlog Request for Processor " + processorId, () -> { + final WebTarget target = processorTarget + .path("/backlog-requests") + .resolveTemplate("id", processorId); + + return getRequestBuilder(target).post(null, BacklogRequestEntity.class); + }); + } + + @Override + public BacklogRequestEntity getProcessorBacklogRequest(final String processorId, final String requestId) throws NiFiClientException, IOException { + if (StringUtils.isBlank(processorId)) { + throw new IllegalArgumentException("Processor ID cannot be null or blank"); + } + if (StringUtils.isBlank(requestId)) { + throw new IllegalArgumentException("Backlog Request ID cannot be null or blank"); + } + + return executeAction("Error retrieving Backlog Request for Processor " + processorId, () -> { + final WebTarget target = processorTarget + .path("/backlog-requests/{requestId}") + .resolveTemplate("id", processorId) + .resolveTemplate("requestId", requestId); + + return getRequestBuilder(target).get(BacklogRequestEntity.class); + }); + } + + @Override + public BacklogRequestEntity deleteProcessorBacklogRequest(final String processorId, final String requestId) throws NiFiClientException, IOException { + if (StringUtils.isBlank(processorId)) { + throw new IllegalArgumentException("Processor ID cannot be null or blank"); + } + if (StringUtils.isBlank(requestId)) { + throw new IllegalArgumentException("Backlog Request ID cannot be null or blank"); + } + + return executeAction("Error deleting Backlog Request for Processor " + processorId, () -> { + final WebTarget target = processorTarget + .path("/backlog-requests/{requestId}") + .resolveTemplate("id", processorId) + .resolveTemplate("requestId", requestId); + + return getRequestBuilder(target).delete(BacklogRequestEntity.class); + }); + } }