Skip to content

Commit cc6342d

Browse files
ddanielrddanielr
authored
Merge pull request #84 from Kidcredible300/update-to-newest-accumulo
Accumulo Proxy build with Accumulo 2.1.3
2 parents 63c9d52 + 521df74 commit cc6342d

File tree

6 files changed

+123
-50
lines changed

6 files changed

+123
-50
lines changed

‎Dockerfile‎

Lines changed: 14 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -19,30 +19,30 @@ EXPOSE 42424
1919

2020
WORKDIR /opt/accumulo-proxy
2121

22-
ARG HADOOP_VERSION=3.3.4
23-
ARG ZOOKEEPER_VERSION=3.8.0
24-
ARG ACCUMULO_VERSION=2.1.1
22+
ARG HADOOP_VERSION=3.3.6
23+
ARG ZOOKEEPER_VERSION=3.9.2
24+
ARG ACCUMULO_VERSION=2.1.3
2525
ARG ACCUMULO_PROXY_VERSION=2.0.0-SNAPSHOT
2626

27-
ARG HADOOP_HASH=ca5e12625679ca95b8fd7bb7babc2a8dcb2605979b901df9ad137178718821097b67555115fafc6dbf6bb32b61864ccb6786dbc555e589694a22bf69147780b4
28-
ARG ZOOKEEPER_HASH=d66e3a40451f840406901b2cd940992b001f92049a372ae48d8b420891605871cd1ae5f6cceb3b10665491e7abef36a4078dace158bd1e0938fcd3567b5234ca
29-
ARG ACCUMULO_HASH=adb23e56362c2e3e813d07791389b8ca2d5976df8b00a29b607e6ae05ea465eff80ada6d1ec9a9c596df8b4066c51078cd5a4006dc78568ac38f638a1d3895be
27+
ARG HADOOP_HASH=de3eaca2e0517e4b569a88b63c89fae19cb8ac6c01ff990f1ff8f0cc0f3128c8e8a23db01577ca562a0e0bb1b4a3889f8c74384e609cd55e537aada3dcaa9f8a
28+
ARG ZOOKEEPER_HASH=2b5ae02d618a27ca8cd54924855d5344263b7d9dee760181f9d66bafa9230324d2ad31786895f0654c969dc38d4a3d0077f74cc376b58b5fa2fb94beb1ab445f
29+
ARG ACCUMULO_HASH=1a27a144dc31f55ccc8e081b6c1bc6cc0362a8391838c53c166cb45291ff8f35867fd8e4729aa7b2c540f8b721f8c6953281bf589fc7fe320e4dc4d20b87abc4
3030

3131
# Download from Apache mirrors instead of archive #9
32-
ENV APACHE_DIST_URLS \
32+
ENV APACHE_DIST_URLS="\
3333
https://www.apache.org/dyn/closer.cgi?action=download&filename= \
3434
# if the version is outdated (or we're grabbing the .asc file), we might have to pull from the dist/archive :/
3535
https://www-us.apache.org/dist/ \
3636
https://www.apache.org/dist/ \
37-
https://archive.apache.org/dist/
37+
https://archive.apache.org/dist/"
3838

3939
RUN set -eux; \
4040
download_bin() { \
4141
local f="$1"; shift; \
4242
local hash="$1"; shift; \
4343
local distFile="$1"; shift; \
4444
local success=; \
45-
local distUrl=; \
45+
local distUrl=; \
4646
for distUrl in ${APACHE_DIST_URLS}; do \
4747
if wget -nv -O "/tmp/${f}" "${distUrl}${distFile}"; then \
4848
success=1; \
@@ -63,16 +63,16 @@ RUN tar xzf /tmp/hadoop.tar.gz -C /opt/ && ln -s /opt/hadoop-${HADOOP_VERSION} /
6363
RUN tar xzf /tmp/apache-zookeeper.tar.gz -C /opt/ && ln -s /opt/apache-zookeeper-${ZOOKEEPER_VERSION}-bin /opt/apache-zookeeper
6464
RUN tar xzf /tmp/accumulo.tar.gz -C /opt/ && ln -s /opt/accumulo-${ACCUMULO_VERSION} /opt/accumulo && sed -i 's/\${ZOOKEEPER_HOME}\/\*/\${ZOOKEEPER_HOME}\/\*\:\${ZOOKEEPER_HOME}\/lib\/\*/g' /opt/accumulo/conf/accumulo-env.sh
6565

66-
ENV HADOOP_HOME /opt/hadoop
67-
ENV ZOOKEEPER_HOME /opt/apache-zookeeper
68-
ENV ACCUMULO_HOME /opt/accumulo
66+
ENV HADOOP_HOME=/opt/hadoop
67+
ENV ZOOKEEPER_HOME=/opt/apache-zookeeper
68+
ENV ACCUMULO_HOME=/opt/accumulo
6969

7070
# Add the proxy binary
7171
COPY target/accumulo-proxy-${ACCUMULO_PROXY_VERSION}-bin.tar.gz /tmp/
7272
RUN tar xzf /tmp/accumulo-proxy-${ACCUMULO_PROXY_VERSION}-bin.tar.gz -C /opt/accumulo-proxy --strip 1
73-
ENV ACCUMULO_PROXY_HOME /opt/accumulo-proxy
73+
ENV ACCUMULO_PROXY_HOME=/opt/accumulo-proxy
7474

7575
# Ensure Accumulo is on the path.
76-
ENV PATH "${PATH}:${ACCUMULO_HOME}/bin"
76+
ENV PATH="${PATH}:${ACCUMULO_HOME}/bin"
7777

7878
CMD ["/opt/accumulo-proxy/bin/accumulo-proxy", "-p", "/opt/accumulo-proxy/conf/proxy.properties"]

‎pom.xml‎

Lines changed: 29 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,7 @@
2020
<parent>
2121
<groupId>org.apache</groupId>
2222
<artifactId>apache</artifactId>
23-
<version>29</version>
23+
<version>31</version>
2424
</parent>
2525
<groupId>org.apache.accumulo</groupId>
2626
<artifactId>accumulo-proxy</artifactId>
@@ -85,27 +85,33 @@
8585
<url>https://github.com/apache/accumulo-proxy/actions</url>
8686
</ciManagement>
8787
<properties>
88-
<accumulo.version>2.1.1</accumulo.version>
88+
<accumulo.version>2.1.3</accumulo.version>
8989
<eclipseFormatterStyle>contrib/Eclipse-Accumulo-Codestyle.xml</eclipseFormatterStyle>
9090
<!-- extra release args for testing -->
9191
<extraReleaseArguments />
92-
<hadoop.version>3.3.4</hadoop.version>
92+
<hadoop.version>3.3.6</hadoop.version>
9393
<it.failIfNoSpecifiedTests>false</it.failIfNoSpecifiedTests>
9494
<maven.compiler.release>11</maven.compiler.release>
9595
<maven.compiler.source>11</maven.compiler.source>
9696
<maven.compiler.target>11</maven.compiler.target>
97-
<slf4j.version>2.0.6</slf4j.version>
97+
<slf4j.version>2.0.12</slf4j.version>
9898
<sourceReleaseAssemblyDescriptor>source-release-tar</sourceReleaseAssemblyDescriptor>
9999
<spotbugs.excludeFilterFile>src/main/spotbugs/exclude-filter.xml</spotbugs.excludeFilterFile>
100-
<surefire.version>3.0.0-M8</surefire.version>
101100
<thrift.version>0.17.0</thrift.version>
102101
</properties>
103102
<dependencyManagement>
104103
<dependencies>
104+
<dependency>
105+
<groupId>io.micrometer</groupId>
106+
<artifactId>micrometer-bom</artifactId>
107+
<version>1.12.2</version>
108+
<type>pom</type>
109+
<scope>import</scope>
110+
</dependency>
105111
<dependency>
106112
<groupId>org.apache.logging.log4j</groupId>
107113
<artifactId>log4j-bom</artifactId>
108-
<version>2.19.0</version>
114+
<version>2.23.1</version>
109115
<type>pom</type>
110116
<scope>import</scope>
111117
</dependency>
@@ -121,13 +127,13 @@
121127
<dependency>
122128
<groupId>com.github.spotbugs</groupId>
123129
<artifactId>spotbugs-annotations</artifactId>
124-
<version>4.7.3</version>
130+
<version>4.8.3</version>
125131
<optional>true</optional>
126132
</dependency>
127133
<dependency>
128134
<groupId>com.google.auto.service</groupId>
129135
<artifactId>auto-service</artifactId>
130-
<version>1.0.1</version>
136+
<version>1.1.1</version>
131137
<optional>true</optional>
132138
</dependency>
133139
<dependency>
@@ -140,6 +146,10 @@
140146
<artifactId>commons-lang</artifactId>
141147
<version>2.6</version>
142148
</dependency>
149+
<dependency>
150+
<groupId>io.micrometer</groupId>
151+
<artifactId>micrometer-core</artifactId>
152+
</dependency>
143153
<dependency>
144154
<groupId>org.apache.accumulo</groupId>
145155
<artifactId>accumulo-core</artifactId>
@@ -214,7 +224,7 @@
214224
<dependency>
215225
<groupId>org.junit.jupiter</groupId>
216226
<artifactId>junit-jupiter-api</artifactId>
217-
<version>5.9.2</version>
227+
<version>5.10.1</version>
218228
<scope>test</scope>
219229
</dependency>
220230
</dependencies>
@@ -225,12 +235,12 @@
225235
<plugin>
226236
<groupId>org.codehaus.mojo</groupId>
227237
<artifactId>versions-maven-plugin</artifactId>
228-
<version>2.14.2</version>
238+
<version>2.16.2</version>
229239
</plugin>
230240
<plugin>
231241
<groupId>com.mycila</groupId>
232242
<artifactId>license-maven-plugin</artifactId>
233-
<version>4.1</version>
243+
<version>4.3</version>
234244
<configuration>
235245
<header>${session.executionRootDirectory}/contrib/license-header.txt</header>
236246
<excludes combine.children="append">
@@ -255,15 +265,15 @@
255265
<plugin>
256266
<groupId>org.gaul</groupId>
257267
<artifactId>modernizer-maven-plugin</artifactId>
258-
<version>2.5.0</version>
268+
<version>2.7.0</version>
259269
<configuration>
260270
<javaVersion>${maven.compiler.target}</javaVersion>
261271
</configuration>
262272
</plugin>
263273
<plugin>
264274
<groupId>com.github.spotbugs</groupId>
265275
<artifactId>spotbugs-maven-plugin</artifactId>
266-
<version>4.7.3.0</version>
276+
<version>4.7.3.6</version>
267277
<configuration>
268278
<xmlOutput>true</xmlOutput>
269279
<effort>Max</effort>
@@ -313,7 +323,7 @@
313323
<plugin>
314324
<groupId>com.github.koraktor</groupId>
315325
<artifactId>mavanagaiata</artifactId>
316-
<version>1.0.0</version>
326+
<version>1.0.1</version>
317327
<configuration>
318328
<skipNoGit>true</skipNoGit>
319329
</configuration>
@@ -337,7 +347,6 @@
337347
<groupId>org.apache.maven.plugins</groupId>
338348
<artifactId>maven-compiler-plugin</artifactId>
339349
<configuration>
340-
<optimize>true</optimize>
341350
<showDeprecation>true</showDeprecation>
342351
<showWarnings>true</showWarnings>
343352
<compilerArgs>
@@ -410,22 +419,22 @@
410419
<plugin>
411420
<groupId>org.codehaus.mojo</groupId>
412421
<artifactId>build-helper-maven-plugin</artifactId>
413-
<version>3.3.0</version>
422+
<version>3.5.0</version>
414423
</plugin>
415424
<plugin>
416425
<groupId>org.codehaus.mojo</groupId>
417426
<artifactId>exec-maven-plugin</artifactId>
418-
<version>3.0.0</version>
427+
<version>3.1.1</version>
419428
</plugin>
420429
<plugin>
421430
<groupId>net.revelc.code</groupId>
422431
<artifactId>apilyzer-maven-plugin</artifactId>
423-
<version>1.2.0</version>
432+
<version>1.3.0</version>
424433
</plugin>
425434
<plugin>
426435
<groupId>net.revelc.code.formatter</groupId>
427436
<artifactId>formatter-maven-plugin</artifactId>
428-
<version>2.21.0</version>
437+
<version>2.23.0</version>
429438
<configuration>
430439
<configFile>${eclipseFormatterStyle}</configFile>
431440
<compilerCompliance>${maven.compiler.source}</compilerCompliance>
@@ -435,7 +444,6 @@
435444
<exclude>**/thrift-gen-java/**/*.java</exclude>
436445
</excludes>
437446
<lineEnding>LF</lineEnding>
438-
<overrideConfigCompilerVersion>true</overrideConfigCompilerVersion>
439447
<skipJsFormatting>true</skipJsFormatting>
440448
<skipHtmlFormatting>true</skipHtmlFormatting>
441449
<skipXmlFormatting>true</skipXmlFormatting>
@@ -645,7 +653,7 @@
645653
<dependency>
646654
<groupId>com.puppycrawl.tools</groupId>
647655
<artifactId>checkstyle</artifactId>
648-
<version>10.6.0</version>
656+
<version>10.12.6</version>
649657
</dependency>
650658
</dependencies>
651659
<executions>

‎src/main/java/org/apache/accumulo/proxy/Proxy.java‎

Lines changed: 58 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,8 @@
2222
import java.io.InputStream;
2323
import java.io.UncheckedIOException;
2424
import java.nio.file.Files;
25+
import java.util.Collection;
26+
import java.util.List;
2527
import java.util.Properties;
2628

2729
import org.apache.accumulo.core.cli.Help;
@@ -30,6 +32,8 @@
3032
import org.apache.accumulo.core.clientImpl.ClientConfConverter;
3133
import org.apache.accumulo.core.conf.ClientProperty;
3234
import org.apache.accumulo.core.conf.ConfigurationTypeHelper;
35+
import org.apache.accumulo.core.metrics.MetricsInfo;
36+
import org.apache.accumulo.core.metrics.MetricsProducer;
3337
import org.apache.accumulo.core.rpc.SslConnectionParams;
3438
import org.apache.accumulo.core.trace.TraceUtil;
3539
import org.apache.accumulo.core.util.HostAndPort;
@@ -55,6 +59,8 @@
5559
import com.google.auto.service.AutoService;
5660

5761
import edu.umd.cs.findbugs.annotations.SuppressFBWarnings;
62+
import io.micrometer.core.instrument.MeterRegistry;
63+
import io.micrometer.core.instrument.Tag;
5864

5965
@AutoService(KeywordExecutable.class)
6066
public class Proxy implements KeywordExecutable {
@@ -250,12 +256,63 @@ public static ServerAddress createProxyServer(HostAndPort address,
250256
break;
251257
}
252258

253-
TimedProcessor timedProcessor = new TimedProcessor(processor);
259+
TimedProcessor timedProcessor = new TimedProcessor(processor, new ProxyMetricsInfo());
254260

255261
// Create the thrift server with our processor and properties
256262
return TServerUtils.startTServer(serverType, timedProcessor, protocolFactory, serverName,
257263
threadName, numThreads, ThreadPools.DEFAULT_TIMEOUT_MILLISECS,
258264
ClientConfConverter.toAccumuloConf(props), 1000L, maxFrameSize, sslParams, saslParams,
259265
serverSocketTimeout, address);
260266
}
267+
268+
// TODO: This MetricsInfo is a stub Metrics Info to allow the timed processor to build. If Metrics
269+
// are wanted or needed in a later version of the proxy, this can be updated.
270+
static private class ProxyMetricsInfo implements MetricsInfo {
271+
272+
@Override
273+
public boolean isMetricsEnabled() {
274+
return false;
275+
}
276+
277+
@Override
278+
public void addServiceTags(String applicationName, HostAndPort hostAndPort) {
279+
throw new UnsupportedOperationException("Unimplemented method 'addServiceTags'");
280+
}
281+
282+
@Override
283+
public void addCommonTags(List<Tag> updates) {
284+
throw new UnsupportedOperationException("Unimplemented method 'addCommonTags'");
285+
}
286+
287+
@Override
288+
public Collection<Tag> getCommonTags() {
289+
throw new UnsupportedOperationException("Unimplemented method 'getCommonTags'");
290+
}
291+
292+
@Override
293+
public void addRegistry(MeterRegistry registry) {
294+
throw new UnsupportedOperationException("Unimplemented method 'addRegistry'");
295+
}
296+
297+
@Override
298+
public void addMetricsProducers(MetricsProducer... producer) {
299+
return;
300+
}
301+
302+
@Override
303+
public void init() {
304+
throw new UnsupportedOperationException("Unimplemented method 'init'");
305+
}
306+
307+
@Override
308+
public MeterRegistry getRegistry() {
309+
throw new UnsupportedOperationException("Unimplemented method 'getRegistry'");
310+
}
311+
312+
@Override
313+
public void close() {
314+
throw new UnsupportedOperationException("Unimplemented method 'close'");
315+
}
316+
317+
}
261318
}

‎src/main/java/org/apache/accumulo/proxy/ProxyServer.java‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -222,6 +222,7 @@ public ProxyServer(Properties props) {
222222
}
223223

224224
protected AccumuloClient getClient(String sharedSecret) throws Exception {
225+
225226
if (sharedSecret.equals(this.sharedSecret)) {
226227
return client;
227228
} else {
@@ -2118,7 +2119,9 @@ public String createConditionalWriter(String sharedSecret, String tableName,
21182119
try {
21192120
ConditionalWriterConfig cwc = new ConditionalWriterConfig();
21202121
if (options.getMaxMemory() != 0) {
2121-
// TODO
2122+
// TODO - This was left blank, I'm not sure why it's here. For now, it will log the max
2123+
// memory
2124+
logger.info("Max memory: " + options.getMaxMemory());
21222125
}
21232126
if (options.isSetThreads() && options.getThreads() != 0) {
21242127
cwc.setMaxWriteThreads(options.getThreads());

‎src/main/java/org/apache/accumulo/proxy/TServerUtils.java‎

Lines changed: 6 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,7 @@
3131
import java.util.Map;
3232
import java.util.Set;
3333
import java.util.concurrent.ExecutorService;
34+
import java.util.concurrent.LinkedBlockingQueue;
3435
import java.util.concurrent.ThreadPoolExecutor;
3536
import java.util.concurrent.TimeUnit;
3637
import java.util.stream.Collectors;
@@ -187,8 +188,10 @@ private static ServerAddress createNonBlockingServer(HostAndPort address, TProce
187188
private static ThreadPoolExecutor createSelfResizingThreadPool(final String serverName,
188189
final int executorThreads, long threadTimeOut, final AccumuloConfiguration conf,
189190
long timeBetweenThreadChecks) {
190-
final ThreadPoolExecutor pool = ThreadPools.getServerThreadPools().createFixedThreadPool(
191-
executorThreads, threadTimeOut, TimeUnit.MILLISECONDS, serverName + "-ClientPool", true);
191+
final ThreadPoolExecutor pool = ThreadPools.getServerThreadPools()
192+
.getPoolBuilder(serverName + "-ClientPool").numCoreThreads(executorThreads)
193+
.numMaxThreads(executorThreads).withTimeOut(threadTimeOut, TimeUnit.MILLISECONDS)
194+
.enableThreadPoolMetrics(true).withQueue(new LinkedBlockingQueue<>()).build();
192195
// periodically adjust the number of threads we need by checking how busy our threads are
193196
ThreadPools.watchCriticalFixedDelay(conf, timeBetweenThreadChecks, () -> {
194197
// there is a minor race condition between sampling the current state of the thread pool
@@ -276,7 +279,7 @@ private static TServerSocket getSslServerSocket(int port, int timeout, InetAddre
276279
TSSLTransportFactory.getServerSocket(port, timeout, params.isClientAuth(), address);
277280
} else {
278281
tServerSock = TSSLTransportFactory.getServerSocket(port, timeout, address,
279-
params.getTTransportParams());
282+
params.getTSSLTransportParameters());
280283
}
281284

282285
final ServerSocket serverSock = tServerSock.getServerSocket();

0 commit comments

Comments
 (0)