From 15fdc3327aad1d4e586962362a3a2b39bb57d606 Mon Sep 17 00:00:00 2001 From: Konjac Huang Date: Wed, 16 Oct 2024 03:26:23 -0700 Subject: [PATCH] Add session property for jdbc metadata, pool size, caching enabled --- .../presto/plugin/jdbc/JdbcMetadataCache.java | 16 +++-- .../plugin/jdbc/JdbcMetadataConfig.java | 27 +++++++++ ...JdbcMetadataSessionPropertiesProvider.java | 58 +++++++++++++++++++ .../presto/plugin/jdbc/TestJdbcMetadata.java | 3 +- .../plugin/jdbc/TestJdbcMetadataConfig.java | 10 +++- 5 files changed, 107 insertions(+), 7 deletions(-) create mode 100644 presto-base-jdbc/src/main/java/com/facebook/presto/plugin/jdbc/JdbcMetadataSessionPropertiesProvider.java diff --git a/presto-base-jdbc/src/main/java/com/facebook/presto/plugin/jdbc/JdbcMetadataCache.java b/presto-base-jdbc/src/main/java/com/facebook/presto/plugin/jdbc/JdbcMetadataCache.java index 8ef7d92a0a1e2..ee47abd9f1e7d 100644 --- a/presto-base-jdbc/src/main/java/com/facebook/presto/plugin/jdbc/JdbcMetadataCache.java +++ b/presto-base-jdbc/src/main/java/com/facebook/presto/plugin/jdbc/JdbcMetadataCache.java @@ -13,6 +13,7 @@ */ package com.facebook.presto.plugin.jdbc; +import com.facebook.airlift.concurrent.BoundedExecutor; import com.facebook.presto.spi.ConnectorSession; import com.facebook.presto.spi.PrestoException; import com.facebook.presto.spi.SchemaTableName; @@ -27,9 +28,9 @@ import java.util.Objects; import java.util.Optional; import java.util.OptionalLong; -import java.util.concurrent.ExecutorService; import static com.facebook.airlift.concurrent.Threads.daemonThreadsNamed; +import static com.facebook.presto.plugin.jdbc.JdbcMetadataSessionPropertiesProvider.isJdbcMetadataCacheEnabled; import static com.google.common.base.MoreObjects.toStringHelper; import static com.google.common.base.Throwables.throwIfInstanceOf; import static com.google.common.cache.CacheLoader.asyncReloading; @@ -48,7 +49,7 @@ public class JdbcMetadataCache public JdbcMetadataCache(JdbcClient jdbcClient, JdbcMetadataConfig config, JdbcMetadataCacheStats stats) { this( - newCachedThreadPool(daemonThreadsNamed("jdbc-metadata-cache" + "-%s")), + new BoundedExecutor(newCachedThreadPool(daemonThreadsNamed("jdbc-metadata-cache" + "-%s")), config.getMetadataCacheThreadPoolSize()), jdbcClient, stats, OptionalLong.of(config.getMetadataCacheTtl().toMillis()), @@ -57,7 +58,7 @@ public JdbcMetadataCache(JdbcClient jdbcClient, JdbcMetadataConfig config, JdbcM } public JdbcMetadataCache( - ExecutorService executor, + BoundedExecutor executor, JdbcClient jdbcClient, JdbcMetadataCacheStats stats, OptionalLong cacheTtl, @@ -65,7 +66,6 @@ public JdbcMetadataCache( long cacheMaximumSize) { this.jdbcClient = requireNonNull(jdbcClient, "jdbcClient is null"); - this.tableHandleCache = newCacheBuilder(cacheTtl, refreshInterval, cacheMaximumSize) .build(asyncReloading(CacheLoader.from(this::loadTableHandle), executor)); stats.setTableHandleCache(tableHandleCache); @@ -77,11 +77,19 @@ public JdbcMetadataCache( public JdbcTableHandle getTableHandle(ConnectorSession session, SchemaTableName tableName) { + if (!isJdbcMetadataCacheEnabled(session)) { + return jdbcClient.getTableHandle(session, JdbcIdentity.from(session), tableName); + } + return get(tableHandleCache, new KeyAndSession<>(session, tableName)).orElse(null); } public List getColumns(ConnectorSession session, JdbcTableHandle jdbcTableHandle) { + if (!isJdbcMetadataCacheEnabled(session)) { + return jdbcClient.getColumns(session, jdbcTableHandle); + } + return get(columnHandlesCache, new KeyAndSession<>(session, jdbcTableHandle)); } diff --git a/presto-base-jdbc/src/main/java/com/facebook/presto/plugin/jdbc/JdbcMetadataConfig.java b/presto-base-jdbc/src/main/java/com/facebook/presto/plugin/jdbc/JdbcMetadataConfig.java index d2bd555f2c53b..37d1f95955043 100644 --- a/presto-base-jdbc/src/main/java/com/facebook/presto/plugin/jdbc/JdbcMetadataConfig.java +++ b/presto-base-jdbc/src/main/java/com/facebook/presto/plugin/jdbc/JdbcMetadataConfig.java @@ -29,6 +29,8 @@ public class JdbcMetadataConfig private Duration metadataCacheTtl = new Duration(0, TimeUnit.SECONDS); private Duration metadataCacheRefreshInterval = new Duration(0, TimeUnit.SECONDS); private long metadataCacheMaximumSize = 10000; + private int metadataCacheThreadPoolSize = 10; + private boolean jdbcMetadataCacheEnabled; public boolean isAllowDropTable() { @@ -83,4 +85,29 @@ public JdbcMetadataConfig setMetadataCacheMaximumSize(long metadataCacheMaximumS this.metadataCacheMaximumSize = metadataCacheMaximumSize; return this; } + + public int getMetadataCacheThreadPoolSize() + { + return metadataCacheThreadPoolSize; + } + + @Min(1) + @Config("metadata-cache-thread-pool-size") + public JdbcMetadataConfig setMetadataCacheThreadPoolSize(int threadPoolSize) + { + this.metadataCacheThreadPoolSize = threadPoolSize; + return this; + } + + public boolean isJdbcMetadataCacheEnabled() + { + return jdbcMetadataCacheEnabled; + } + + @Config("metadata-jdbc-cache-enabled") + public JdbcMetadataConfig setJdbcMetadataCacheEnabled(boolean jdbcMetadataCacheEnabled) + { + this.jdbcMetadataCacheEnabled = jdbcMetadataCacheEnabled; + return this; + } } diff --git a/presto-base-jdbc/src/main/java/com/facebook/presto/plugin/jdbc/JdbcMetadataSessionPropertiesProvider.java b/presto-base-jdbc/src/main/java/com/facebook/presto/plugin/jdbc/JdbcMetadataSessionPropertiesProvider.java new file mode 100644 index 0000000000000..27077164ac69d --- /dev/null +++ b/presto-base-jdbc/src/main/java/com/facebook/presto/plugin/jdbc/JdbcMetadataSessionPropertiesProvider.java @@ -0,0 +1,58 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.plugin.jdbc; + +import com.facebook.presto.spi.ConnectorSession; +import com.facebook.presto.spi.session.PropertyMetadata; +import com.google.common.collect.ImmutableList; + +import javax.inject.Inject; + +import java.util.List; + +import static com.facebook.presto.spi.session.PropertyMetadata.booleanProperty; + +public class JdbcMetadataSessionPropertiesProvider + implements JdbcSessionPropertiesProvider +{ + private final List> sessionProperties; + private static final String USE_JDBC_METADATA_CACHE = "use_jdbc_metadata_cache"; + + @Inject + public JdbcMetadataSessionPropertiesProvider(JdbcMetadataConfig jdbcMetadataConfig) + { + this.sessionProperties = ImmutableList.of( + booleanProperty( + USE_JDBC_METADATA_CACHE, + "Whether Jdbc Metadata In Memory Cache is Enabled", + jdbcMetadataConfig.isJdbcMetadataCacheEnabled(), + false)); + } + + @Override + public List> getSessionProperties() + { + return sessionProperties; + } + + public static boolean isJdbcMetadataCacheEnabled(ConnectorSession session) + { + try { + return session.getProperty(USE_JDBC_METADATA_CACHE, Boolean.class); + } + catch (Exception e) { + return false; + } + } +} diff --git a/presto-base-jdbc/src/test/java/com/facebook/presto/plugin/jdbc/TestJdbcMetadata.java b/presto-base-jdbc/src/test/java/com/facebook/presto/plugin/jdbc/TestJdbcMetadata.java index c03c6d1eeaafc..00c1b1a7b4de0 100644 --- a/presto-base-jdbc/src/test/java/com/facebook/presto/plugin/jdbc/TestJdbcMetadata.java +++ b/presto-base-jdbc/src/test/java/com/facebook/presto/plugin/jdbc/TestJdbcMetadata.java @@ -13,6 +13,7 @@ */ package com.facebook.presto.plugin.jdbc; +import com.facebook.airlift.concurrent.BoundedExecutor; import com.facebook.presto.spi.ColumnMetadata; import com.facebook.presto.spi.ConnectorTableMetadata; import com.facebook.presto.spi.PrestoException; @@ -65,7 +66,7 @@ public void setUp() throws Exception { database = new TestingDatabase(); - ListeningExecutorService executor = listeningDecorator(newCachedThreadPool(daemonThreadsNamed("test-%s"))); + BoundedExecutor executor = new BoundedExecutor(newCachedThreadPool(daemonThreadsNamed("jdbc-metadata-cache" + "-%s")), 1); jdbcMetadataCache = new JdbcMetadataCache(executor, database.getJdbcClient(), new JdbcMetadataCacheStats(), OptionalLong.of(0), OptionalLong.of(0), 100); metadata = new JdbcMetadata(jdbcMetadataCache, database.getJdbcClient(), false); tableHandle = metadata.getTableHandle(SESSION, new SchemaTableName("example", "numbers")); diff --git a/presto-base-jdbc/src/test/java/com/facebook/presto/plugin/jdbc/TestJdbcMetadataConfig.java b/presto-base-jdbc/src/test/java/com/facebook/presto/plugin/jdbc/TestJdbcMetadataConfig.java index 6477239814c8a..73f49fe9179d7 100644 --- a/presto-base-jdbc/src/test/java/com/facebook/presto/plugin/jdbc/TestJdbcMetadataConfig.java +++ b/presto-base-jdbc/src/test/java/com/facebook/presto/plugin/jdbc/TestJdbcMetadataConfig.java @@ -34,7 +34,9 @@ public void testDefaults() .setAllowDropTable(false) .setMetadataCacheTtl(new Duration(0, SECONDS)) .setMetadataCacheRefreshInterval(new Duration(0, SECONDS)) - .setMetadataCacheMaximumSize(10000)); + .setMetadataCacheMaximumSize(10000) + .setMetadataCacheThreadPoolSize(10) + .setJdbcMetadataCacheEnabled(false)); } @Test @@ -45,13 +47,17 @@ public void testExplicitPropertyMappings() .put("metadata-cache-ttl", "1h") .put("metadata-cache-refresh-interval", "10s") .put("metadata-cache-maximum-size", "100") + .put("metadata-cache-thread-pool-size", "50") + .put("metadata-jdbc-cache-enabled", "true") .build(); JdbcMetadataConfig expected = new JdbcMetadataConfig() .setAllowDropTable(true) .setMetadataCacheTtl(new Duration(1, HOURS)) .setMetadataCacheRefreshInterval(new Duration(10, SECONDS)) - .setMetadataCacheMaximumSize(100); + .setMetadataCacheMaximumSize(100) + .setMetadataCacheThreadPoolSize(50) + .setJdbcMetadataCacheEnabled(true); assertFullMapping(properties, expected); }