Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 4 additions & 4 deletions flume-bom/pom.xml
Original file line number Diff line number Diff line change
Expand Up @@ -38,11 +38,11 @@
<flume-jms.version>2.0.0-SNAPSHOT</flume-jms.version>
<flume-kafka.version>2.0.0-SNAPSHOT</flume-kafka.version>
<flume-scribe.version>2.0.0-SNAPSHOT</flume-scribe.version>
<flume-spring-boot.version>2.0.0-SNAPSHOT</flume-spring-boot.version>
<flume-spring-boot.version>2.1.0-SNAPSHOT</flume-spring-boot.version>
<flume-taildir.version>2.0.0-SNAPSHOT</flume-taildir.version>
<flume-tools.version>2.0.0-SNAPSHOT</flume-tools.version>
<flume-legacy.version>2.0.0-SNAPSHOT</flume-legacy.version>
<flume-morphline.version>2.0.0-SNAPSHOT</flume-morphline.version>
<flume-rpc.version>2.0.0-SNAPSHOT</flume-rpc.version>
</properties>

<dependencyManagement>
Expand Down Expand Up @@ -105,12 +105,12 @@
<dependency>
<groupId>org.apache.flume</groupId>
<artifactId>flume-rpc-avro</artifactId>
<version>${project.version}</version>
<version>${flume-rpc.version}</version>
</dependency>
<dependency>
<groupId>org.apache.flume</groupId>
<artifactId>flume-rpc-thrift</artifactId>
<version>${project.version}</version>
<version>${flume-rpc.version}</version>
</dependency>
<dependency>
<groupId>org.apache.flume</groupId>
Expand Down
1 change: 0 additions & 1 deletion flume-ng-channels/flume-file-channel/pom.xml
Original file line number Diff line number Diff line change
Expand Up @@ -121,7 +121,6 @@
<dependency>
<groupId>com.google.code.findbugs</groupId>
<artifactId>jsr305</artifactId>
<version>${jsr305.version}</version>
<scope>provided</scope>
</dependency>

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -61,6 +61,7 @@
import org.apache.flume.conf.channel.ChannelType;
import org.apache.flume.conf.configfilter.ConfigFilterConfiguration;
import org.apache.flume.conf.configfilter.ConfigFilterType;
import org.apache.flume.conf.internal.SuppressFBWarnings;
import org.apache.flume.conf.sink.SinkConfiguration;
import org.apache.flume.conf.sink.SinkGroupConfiguration;
import org.apache.flume.conf.sink.SinkType;
Expand All @@ -83,6 +84,7 @@
* @see org.apache.flume.node.ConfigurationProvider
*
*/
@SuppressFBWarnings(value = {"EI_EXPOSE_REP"})
public class FlumeConfiguration {

private static final Logger logger = LogManager.getLogger();
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -41,7 +41,11 @@ public enum ChannelSelectorType implements ComponentWithClassName {
/**
* Multiplexing channel selector.
*/
MULTIPLEXING("org.apache.flume.channel.MultiplexingChannelSelector");
MULTIPLEXING("org.apache.flume.channel.MultiplexingChannelSelector"),
/**
* Routable proxy channel selector.
*/
ROUTABLE_PROXY("org.apache.flume.channel.RoutableProxyChannelSelector");

private final String channelSelectorClassName;

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -46,6 +46,11 @@ public enum ChannelType implements ComponentWithClassName {
*/
JDBC("org.apache.flume.channel.jdbc.JdbcChannel"),

/**
* Kafka channel.
*/
KAFKA("org.apache.flume.channel.kafka.KafkaChannel"),

/**
* Spillable Memory channel
*
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,38 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to you under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.flume.conf.internal;

import java.lang.annotation.ElementType;
import java.lang.annotation.Retention;
import java.lang.annotation.RetentionPolicy;
import java.lang.annotation.Target;

/**
* Used to suppress SpotBugs warnings in Flume artifacts.
*
* <p>SpotBugs recognizes any annotation whose simple name is {@code SuppressFBWarnings}, so this
* type lets us drop the dependency on the LGPL-licensed {@code spotbugs-annotations} artifact.</p>
*
* <p>This type is not exported via JPMS. Do <strong>not</strong> use in third-party modules.</p>
*/
@Retention(RetentionPolicy.CLASS)
@Target({ElementType.TYPE, ElementType.FIELD, ElementType.METHOD, ElementType.CONSTRUCTOR, ElementType.PARAMETER})
public @interface SuppressFBWarnings {

/** The set of SpotBugs warnings to suppress. */
String[] value() default {};
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,22 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to you under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
/**
* Contains types used only by Flume modules.
*
* <p>These types are not exported via JPMS and are not available to third-party modules.</p>
*/
package org.apache.flume.conf.internal;
1 change: 0 additions & 1 deletion flume-ng-core/pom.xml
Original file line number Diff line number Diff line change
Expand Up @@ -89,7 +89,6 @@
<dependency>
<groupId>com.google.code.findbugs</groupId>
<artifactId>jsr305</artifactId>
<version>${jsr305.version}</version>
<scope>provided</scope>
</dependency>

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -55,8 +55,15 @@ public class ChannelProcessor implements Configurable {
private final InterceptorChain interceptorChain;

public ChannelProcessor(ChannelSelector selector) {
this(selector, null);
}

public ChannelProcessor(ChannelSelector selector, List<Interceptor> interceptors) {
this.selector = selector;
this.interceptorChain = new InterceptorChain();
if (interceptors != null) {
interceptorChain.setInterceptors(interceptors);
}
}

public void initialize() {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@
import org.apache.flume.Channel;
import org.apache.flume.Context;
import org.apache.flume.Event;
import org.apache.flume.conf.internal.SuppressFBWarnings;

/**
* Load balancing channel selector. This selector allows for load balancing
Expand All @@ -39,6 +40,7 @@
* defaults to <tt>ROUND_ROBIN</tt> type, but can be overridden via
* configuration.</p>
*/
@SuppressFBWarnings("UWF_FIELD_NOT_INITIALIZED_IN_CONSTRUCTOR")
public class LoadBalancingChannelSelector extends AbstractChannelSelector {
private final List<Channel> emptyList = Collections.emptyList();
private ChannelPicker picker;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,9 @@
import org.apache.flume.Context;
import org.apache.flume.Event;
import org.apache.flume.FlumeException;
import org.apache.flume.conf.internal.SuppressFBWarnings;

@SuppressFBWarnings("UWF_FIELD_NOT_INITIALIZED_IN_CONSTRUCTOR")
public class MultiplexingChannelSelector extends AbstractChannelSelector {

public static final String CONFIG_MULTIPLEX_HEADER_NAME = "header";
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,127 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to you under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.flume.channel;

import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import org.apache.commons.lang3.StringUtils;
import org.apache.flume.Channel;
import org.apache.flume.ChannelSelector;
import org.apache.flume.Context;
import org.apache.flume.Event;
import org.apache.flume.conf.Configurables;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;

public class RoutableProxyChannelSelector extends LoadBalancingChannelSelector {
private static final Logger LOGGER = LogManager.getLogger(RoutableProxyChannelSelector.class);
private static final String HEADER_NAME = "headerName";
private static final String SELECTOR = ".selector.";
private static final String CHANNELS = "channels";
private static final String DEFAULT = "default";
private static final String TYPE = "type";

private final Map<String, ChannelSelector> selectorMap = new HashMap<>();
private ChannelSelector defaultSelector;
private String headerName;

public void addSelector(String headerName, ChannelSelector selector) {
selectorMap.put(headerName, selector);
}

public void setDefaultSelector(ChannelSelector defaultSelector) {
this.defaultSelector = defaultSelector;
}

public ChannelSelector getDefaultSelector() {
return defaultSelector;
}

@Override
public void configure(Context context) {
Configurables.ensureRequiredNonNull(context, HEADER_NAME);
List<Channel> allChannels = getAllChannels();
for (Map.Entry<String, String> entry : context.getParameters().entrySet()) {
if (entry.getKey().equals(HEADER_NAME)) {
this.headerName = entry.getValue();
} else if (!entry.getKey().equals(TYPE)) {
String key = StringUtils.substringBefore(entry.getKey(), ".");
Map<String, String> map = context.getSubProperties(key + SELECTOR);
if (map != null) {
String channelNames = getRequiredNonNull(map, CHANNELS, key);
Set<String> channelSet = new HashSet<>(Arrays.asList(channelNames.split("\\s+")));
List<Channel> channels = new ArrayList<>(channelSet.size());
for (String channelName : channelSet) {
for (Channel channel : allChannels) {
if (channelName.equals(channel.getName())) {
channels.add(channel);
}
}
}
ChannelSelector selector = ChannelSelectorFactory.create(channels, map);
if (DEFAULT.equals(key)) {
defaultSelector = selector;
} else {
selectorMap.put(key, selector);
}
}
}
}
if (headerName == null) {
throw new IllegalArgumentException("No header name specified for RoutableProxy");
}
}

@Override
public List<Channel> getRequiredChannels(Event event) {
ChannelSelector selector = getSelector(event);
if (selector != null) {
return selector.getRequiredChannels(event);
}
return new ArrayList<>();
}

@Override
public List<Channel> getOptionalChannels(Event event) {
ChannelSelector selector = getSelector(event);
if (selector != null) {
return selector.getOptionalChannels(event);
}
return new ArrayList<>();
}

private ChannelSelector getSelector(Event event) {
ChannelSelector channelSelector = selectorMap.get(event.getHeaders().get(headerName));
if (channelSelector == null) {
channelSelector = defaultSelector;
}
return channelSelector;
}
Comment thread
rgoers marked this conversation as resolved.

private String getRequiredNonNull(Map<String, String> map, String keyName, String prefix) {
String value = map.get(keyName);
if (value == null) {
throw new IllegalArgumentException(String.format("Missing key %s in %s", keyName, prefix));
}
return value;
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -26,9 +26,11 @@
import java.util.concurrent.ScheduledThreadPoolExecutor;
import java.util.concurrent.TimeUnit;
import org.apache.flume.FlumeException;
import org.apache.flume.conf.internal.SuppressFBWarnings;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;

@SuppressFBWarnings("UWF_FIELD_NOT_INITIALIZED_IN_CONSTRUCTOR")
public class LifecycleSupervisor implements LifecycleAware {

private static final Logger logger = LogManager.getLogger();
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -20,11 +20,13 @@
import java.util.List;
import org.apache.flume.Sink;
import org.apache.flume.SinkProcessor;
import org.apache.flume.conf.internal.SuppressFBWarnings;
import org.apache.flume.lifecycle.LifecycleState;

/**
* A Sink Processor that only accesses a single Sink.
*/
@SuppressFBWarnings("UWF_FIELD_NOT_INITIALIZED_IN_CONSTRUCTOR")
public abstract class AbstractSingleSinkProcessor implements SinkProcessor {
protected Sink sink;
private LifecycleState lifecycleState;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -21,11 +21,13 @@
import java.util.List;
import org.apache.flume.Sink;
import org.apache.flume.SinkProcessor;
import org.apache.flume.conf.internal.SuppressFBWarnings;
import org.apache.flume.lifecycle.LifecycleState;

/**
* A convenience base class for sink processors.
*/
@SuppressFBWarnings("UWF_FIELD_NOT_INITIALIZED_IN_CONSTRUCTOR")
public abstract class AbstractSinkProcessor implements SinkProcessor {

private LifecycleState state;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,7 @@
import org.apache.flume.EventDeliveryException;
import org.apache.flume.Sink;
import org.apache.flume.Sink.Status;
import org.apache.flume.conf.internal.SuppressFBWarnings;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;

Expand Down Expand Up @@ -59,6 +60,7 @@
* host1.sinkgroups.group1.processor.maxpenalty = 10000
*
*/
@SuppressFBWarnings("UWF_FIELD_NOT_INITIALIZED_IN_CONSTRUCTOR")
public class FailoverSinkProcessor extends AbstractSinkProcessor {
private static final int FAILURE_PENALTY = 1000;
private static final int DEFAULT_MAX_PENALTY = 30000;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@
import org.apache.flume.Sink;
import org.apache.flume.Sink.Status;
import org.apache.flume.conf.Configurable;
import org.apache.flume.conf.internal.SuppressFBWarnings;
import org.apache.flume.lifecycle.LifecycleAware;
import org.apache.flume.util.OrderSelector;
import org.apache.flume.util.RandomOrderSelector;
Expand Down Expand Up @@ -75,6 +76,7 @@
* @see FailoverSinkProcessor
* @see LoadBalancingSinkProcessor.SinkSelector
*/
@SuppressFBWarnings("UWF_FIELD_NOT_INITIALIZED_IN_CONSTRUCTOR")
public class LoadBalancingSinkProcessor extends AbstractSinkProcessor {
public static final String CONFIG_SELECTOR = "selector";
public static final String CONFIG_SELECTOR_PREFIX = CONFIG_SELECTOR + ".";
Expand Down
Loading
Loading