Skip to content

Commit

Permalink
Update zookeeper version to fix CVE-2023-44981
Browse files Browse the repository at this point in the history
  • Loading branch information
bibith4 committed Jan 31, 2025
1 parent ef03de0 commit d72ddd1
Show file tree
Hide file tree
Showing 6 changed files with 116 additions and 142 deletions.
6 changes: 5 additions & 1 deletion pom.xml
Original file line number Diff line number Diff line change
Expand Up @@ -1936,7 +1936,7 @@
<dependency>
<groupId>org.apache.zookeeper</groupId>
<artifactId>zookeeper</artifactId>
<version>3.4.14</version>
<version>3.9.3</version>
<exclusions>
<exclusion>
<artifactId>jline</artifactId>
Expand All @@ -1950,6 +1950,10 @@
<groupId>org.slf4j</groupId>
<artifactId>slf4j-log4j12</artifactId>
</exclusion>
<exclusion>
<groupId>commons-io</groupId>
<artifactId>commons-io</artifactId>
</exclusion>
</exclusions>
</dependency>

Expand Down
10 changes: 8 additions & 2 deletions presto-accumulo/pom.xml
Original file line number Diff line number Diff line change
Expand Up @@ -236,8 +236,8 @@
</dependency>

<dependency>
<groupId>commons-io</groupId>
<artifactId>commons-io</artifactId>
<groupId>com.github.docker-java</groupId>
<artifactId>docker-java-api</artifactId>
</dependency>

<dependency>
Expand Down Expand Up @@ -341,6 +341,12 @@
<artifactId>testng</artifactId>
<scope>test</scope>
</dependency>

<dependency>
<groupId>org.testcontainers</groupId>
<artifactId>testcontainers</artifactId>
<scope>test</scope>
</dependency>
</dependencies>

<profiles>
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -18,48 +18,29 @@
import com.facebook.presto.accumulo.conf.AccumuloConfig;
import com.facebook.presto.accumulo.serializers.LexicoderRowSerializer;
import com.facebook.presto.common.QualifiedObjectName;
import com.facebook.presto.spi.PrestoException;
import com.facebook.presto.testing.QueryRunner;
import com.facebook.presto.tests.DistributedQueryRunner;
import com.facebook.presto.tpch.TpchPlugin;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.ImmutableSortedSet;
import io.airlift.tpch.TpchTable;
import org.apache.accumulo.core.client.AccumuloException;
import org.apache.accumulo.core.client.AccumuloSecurityException;
import org.apache.accumulo.core.client.Connector;
import org.apache.accumulo.core.client.Instance;
import org.apache.accumulo.core.client.ZooKeeperInstance;
import org.apache.accumulo.core.client.security.tokens.PasswordToken;
import org.apache.accumulo.minicluster.MiniAccumuloCluster;
import org.apache.commons.io.FileUtils;
import org.apache.hadoop.io.Text;
import org.intellij.lang.annotations.Language;

import java.io.File;
import java.io.IOException;
import java.nio.file.Files;
import java.util.Map;

import static com.facebook.presto.accumulo.AccumuloErrorCode.MINI_ACCUMULO;
import static com.facebook.presto.accumulo.AccumuloErrorCode.UNEXPECTED_ACCUMULO_ERROR;
import static com.facebook.presto.accumulo.MiniAccumuloConfigUtil.setConfigClassPath;
import static com.facebook.presto.common.type.BigintType.BIGINT;
import static com.facebook.presto.testing.TestingSession.testSessionBuilder;
import static com.facebook.presto.tpch.TpchMetadata.TINY_SCHEMA_NAME;
import static io.airlift.units.Duration.nanosSince;
import static java.lang.String.format;
import static java.util.concurrent.TimeUnit.SECONDS;
import static org.apache.accumulo.minicluster.MemoryUnit.MEGABYTE;

public final class AccumuloQueryRunner
{
private static final Logger LOG = Logger.get(AccumuloQueryRunner.class);
private static final String MAC_PASSWORD = "secret";
private static final String MAC_USER = "root";

private static boolean tpchLoaded;
private static Connector connector = getAccumuloConnector();

private AccumuloQueryRunner() {}

Expand All @@ -72,21 +53,22 @@ public static synchronized DistributedQueryRunner createAccumuloQueryRunner(Map<
queryRunner.installPlugin(new TpchPlugin());
queryRunner.createCatalog("tpch", "tpch");

TestingAccumuloServer server = TestingAccumuloServer.getInstance();
queryRunner.installPlugin(new AccumuloPlugin());
Map<String, String> accumuloProperties =
ImmutableMap.<String, String>builder()
.put(AccumuloConfig.INSTANCE, connector.getInstance().getInstanceName())
.put(AccumuloConfig.ZOOKEEPERS, connector.getInstance().getZooKeepers())
.put(AccumuloConfig.USERNAME, MAC_USER)
.put(AccumuloConfig.PASSWORD, MAC_PASSWORD)
.put(AccumuloConfig.INSTANCE, server.getInstanceName())
.put(AccumuloConfig.ZOOKEEPERS, server.getZooKeepers())
.put(AccumuloConfig.USERNAME, server.getUser())
.put(AccumuloConfig.PASSWORD, server.getPassword())
.put(AccumuloConfig.ZOOKEEPER_METADATA_ROOT, "/presto-accumulo-test")
.build();

queryRunner.createCatalog("accumulo", "accumulo", accumuloProperties);

if (!tpchLoaded) {
copyTpchTables(queryRunner, "tpch", TINY_SCHEMA_NAME, createSession(), TpchTable.getTables());
connector.tableOperations().addSplits("tpch.orders", ImmutableSortedSet.of(new Text(new LexicoderRowSerializer().encode(BIGINT, 7500L))));
server.getConnector().tableOperations().addSplits("tpch.orders", ImmutableSortedSet.of(new Text(new LexicoderRowSerializer().encode(BIGINT, 7500L))));
tpchLoaded = true;
}

Expand Down Expand Up @@ -156,68 +138,13 @@ public static Session createSession()
return testSessionBuilder().setCatalog("accumulo").setSchema("tpch").build();
}

/**
* Gets the AccumuloConnector singleton, starting the MiniAccumuloCluster on initialization.
* This singleton instance is required so all test cases access the same MiniAccumuloCluster.
*
* @return Accumulo connector
*/
public static Connector getAccumuloConnector()
{
if (connector != null) {
return connector;
}

try {
MiniAccumuloCluster accumulo = createMiniAccumuloCluster();
Instance instance = new ZooKeeperInstance(accumulo.getInstanceName(), accumulo.getZooKeepers());
connector = instance.getConnector(MAC_USER, new PasswordToken(MAC_PASSWORD));
LOG.info("Connection to MAC instance %s at %s established, user %s password %s", accumulo.getInstanceName(), accumulo.getZooKeepers(), MAC_USER, MAC_PASSWORD);
return connector;
}
catch (AccumuloException | AccumuloSecurityException | InterruptedException | IOException e) {
throw new PrestoException(UNEXPECTED_ACCUMULO_ERROR, "Failed to get connector to Accumulo", e);
}
}

/**
* Creates and starts an instance of MiniAccumuloCluster, returning the new instance.
*
* @return New MiniAccumuloCluster
*/
private static MiniAccumuloCluster createMiniAccumuloCluster()
throws IOException, InterruptedException
public static void main(String[] args)
throws Exception
{
// Create MAC directory
File macDir = Files.createTempDirectory("mac-").toFile();
LOG.info("MAC is enabled, starting MiniAccumuloCluster at %s", macDir);

// Start MAC and connect to it
MiniAccumuloCluster accumulo = new MiniAccumuloCluster(macDir, MAC_PASSWORD);
accumulo.getConfig().setDefaultMemory(512, MEGABYTE);
setConfigClassPath(accumulo.getConfig());
accumulo.start();

// Add shutdown hook to stop MAC and cleanup temporary files
Runtime.getRuntime().addShutdownHook(new Thread(() -> {
try {
LOG.info("Shutting down MAC");
accumulo.stop();
}
catch (IOException | InterruptedException e) {
Thread.currentThread().interrupt();
throw new PrestoException(MINI_ACCUMULO, "Failed to shut down MAC instance", e);
}

try {
LOG.info("Cleaning up MAC directory");
FileUtils.forceDelete(macDir);
}
catch (IOException e) {
throw new PrestoException(MINI_ACCUMULO, "Failed to clean up MAC directory", e);
}
}));

return accumulo;
DistributedQueryRunner queryRunner = createAccumuloQueryRunner(ImmutableMap.of("http-server.http.port", "8080"));
Thread.sleep(10);
Logger log = Logger.get(AccumuloQueryRunner.class);
log.info("======== SERVER STARTED ========");
log.info("\n====\n%s\n====", queryRunner.getCoordinator().getBaseUrl());
}
}

This file was deleted.

Original file line number Diff line number Diff line change
Expand Up @@ -46,7 +46,7 @@ public TestAccumuloClient()
.setUsername("root")
.setPassword("secret");

Connector connector = AccumuloQueryRunner.getAccumuloConnector();
Connector connector = TestingAccumuloServer.getInstance().getConnector();
config.setZooKeepers(connector.getInstance().getZooKeepers());
zooKeeperMetadataManager = new ZooKeeperMetadataManager(config, createTestFunctionAndTypeManager());
client = new AccumuloClient(connector, config, zooKeeperMetadataManager, new AccumuloTableManager(connector), new IndexLookup(connector, new ColumnCardinalityCache(connector, config)));
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,89 @@
package com.facebook.presto.accumulo;

/*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import org.apache.accumulo.core.client.AccumuloException;
import org.apache.accumulo.core.client.AccumuloSecurityException;
import org.apache.accumulo.core.client.Connector;
import org.apache.accumulo.core.client.ZooKeeperInstance;
import org.apache.accumulo.core.client.security.tokens.PasswordToken;
import org.testcontainers.containers.FixedHostPortGenericContainer;
import org.testcontainers.containers.wait.strategy.Wait;

import java.io.IOException;
import java.time.Duration;

import static java.lang.String.format;
public class TestingAccumuloServer
{
private static final int ACCUMULO_MASTER_PORT = 9999;
private static final int ACCUMULO_TSERVER_PORT = 9997;
private static final int ZOOKEEPER_PORT = 2181;
private static final TestingAccumuloServer instance = new TestingAccumuloServer();
private final FixedHostPortGenericContainer<?> accumuloContainer;
public static TestingAccumuloServer getInstance()
{
return instance;
}
private TestingAccumuloServer()
{
accumuloContainer = new FixedHostPortGenericContainer<>("ghcr.io/trinodb/testing/accumulo:74");
accumuloContainer.withFixedExposedPort(ACCUMULO_MASTER_PORT, ACCUMULO_MASTER_PORT);
accumuloContainer.withFixedExposedPort(ACCUMULO_TSERVER_PORT, ACCUMULO_TSERVER_PORT);
accumuloContainer.withExposedPorts(ZOOKEEPER_PORT);
accumuloContainer.withCreateContainerCmdModifier(cmd -> cmd
.withHostName("localhost")
.withEnv("ADDRESS=0.0.0.0")
.withEntrypoint("supervisord", "-c", "/etc/supervisord.conf"));
accumuloContainer.waitingFor(Wait.forHealthcheck().withStartupTimeout(Duration.ofMinutes(10)));
// No need for an explicit stop since this server is a singleton
// and the container will be stopped by TestContainers on shutdown
// TODO Change this class to not be a singleton
// https://github.com/prestosql/presto/issues/5842
accumuloContainer.start();
try {
accumuloContainer.execInContainer("wget", "-P", "/usr/local/lib/accumulo/lib/ext/", "https://repo1.maven.org/maven2/com/facebook/presto/presto-accumulo/0.289/presto-accumulo-0.289.jar");
accumuloContainer.execInContainer("mv", "/usr/local/lib/accumulo/lib/ext/presto-accumulo-0.289.jar", "/usr/local/lib/accumulo/lib/ext/presto-accumulo-0.290-SNAPSHOT.jar");
}
catch (IOException | InterruptedException e) {
throw new RuntimeException(e);
}
}
public String getInstanceName()
{
return "default";
}
public String getZooKeepers()
{
return format("%s:%s", accumuloContainer.getHost(), accumuloContainer.getMappedPort(ZOOKEEPER_PORT));
}
public String getUser()
{
return "root";
}
public String getPassword()
{
return "secret";
}
public Connector getConnector()
{
try {
ZooKeeperInstance instance = new ZooKeeperInstance(getInstanceName(), getZooKeepers());
return instance.getConnector(getUser(), new PasswordToken(getPassword()));
}
catch (AccumuloException | AccumuloSecurityException e) {
throw new RuntimeException(e);
}
}
}

0 comments on commit d72ddd1

Please sign in to comment.