Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Fix getting views for Hive metastore 2.3+ #24466

Open
wants to merge 1 commit into
base: master
Choose a base branch
from
+46 −5
Open

Fix getting views for Hive metastore 2.3+ #24466

Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
51 changes: 46 additions & 5 deletions ...etastore/src/main/java/com/facebook/presto/hive/metastore/thrift/ThriftHiveMetastore.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -183,6 +183,9 @@ public class ThriftHiveMetastore
private final boolean isMetastoreAuthenticationEnabled;
private final boolean deleteFilesOnTableDrop;

private volatile boolean metastoreKnownToSupportTableParamEqualsPredicate;
private volatile boolean metastoreKnownToSupportTableParamLikePredicate;

@Inject
public ThriftHiveMetastore(HiveCluster hiveCluster, MetastoreClientConfig config, HdfsEnvironment hdfsEnvironment)
{
Expand Down Expand Up @@ -896,11 +899,9 @@ public Optional<List<String>> getAllViews(MetastoreContext metastoreContext, Str
return retry()
.stopOn(UnknownDBException.class)
.stopOnIllegalExceptions()
.run("getAllViews", stats.getGetAllViews().wrap(() ->
getMetastoreClientThenCall(metastoreContext, client -> {
String filter = HIVE_FILTER_FIELD_PARAMS + PRESTO_VIEW_FLAG + " = \"true\"";
return Optional.of(client.getTableNamesByFilter(databaseName, filter));
})));
.run("getAllViews", stats.getGetAllViews().wrap(() -> {
return Optional.of(getPrestoViews(databaseName));
}));
}
catch (UnknownDBException e) {
return Optional.empty();
Expand Down Expand Up @@ -1209,6 +1210,46 @@ public MetastoreOperationResult addPartitions(
return EMPTY_RESULT;
}

private List<String> getPrestoViews(String databaseName)
throws TException
{
/*
* Thrift call `get_table_names_by_filter` may be translated by Metastore to a SQL query against Metastore database.
* Hive 2.3 on some databases uses CLOB for table parameter value column and some databases disallow `=` predicate over
* CLOB values. At the same time, they allow `LIKE` predicates over them.
*/
String filterWithEquals = HIVE_FILTER_FIELD_PARAMS + PRESTO_VIEW_FLAG + " = \"true\"";
String filterWithLike = HIVE_FILTER_FIELD_PARAMS + PRESTO_VIEW_FLAG + " LIKE \"true\"";
if (metastoreKnownToSupportTableParamEqualsPredicate) {
try (HiveMetastoreClient client = clientProvider.createMetastoreClient(Optional.empty())) {
return client.getTableNamesByFilter(databaseName, filterWithEquals);
}
}
if (metastoreKnownToSupportTableParamLikePredicate) {
try (HiveMetastoreClient client = clientProvider.createMetastoreClient(Optional.empty())) {
return client.getTableNamesByFilter(databaseName, filterWithLike);
}
}
try (HiveMetastoreClient client = clientProvider.createMetastoreClient(Optional.empty())) {
List<String> views = client.getTableNamesByFilter(databaseName, filterWithEquals);
metastoreKnownToSupportTableParamEqualsPredicate = true;
return views;
}
catch (TException | RuntimeException firstException) {
try (HiveMetastoreClient client = clientProvider.createMetastoreClient(Optional.empty())) {
List<String> views = client.getTableNamesByFilter(databaseName, filterWithLike);
metastoreKnownToSupportTableParamLikePredicate = true;
return views;
}
catch (TException | RuntimeException secondException) {
if (firstException != secondException) {
firstException.addSuppressed(secondException);
}
}
throw firstException;
}
}

private void addPartitionsWithoutStatistics(MetastoreContext metastoreContext, String databaseName, String tableName, List<Partition> partitions)
{
if (partitions.isEmpty()) {
Expand Down
Loading