-
Notifications
You must be signed in to change notification settings - Fork 5.4k
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Subclass
FileHiveMetastore for Iceberg connector
- Loading branch information
1 parent
d7c0930
commit 4a0bbc9
Showing
12 changed files
with
454 additions
and
280 deletions.
There are no files selected for viewing
484 changes: 220 additions & 264 deletions
484
...ve-metastore/src/main/java/com/facebook/presto/hive/metastore/file/FileHiveMetastore.java
Large diffs are not rendered by default.
Oops, something went wrong.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
114 changes: 114 additions & 0 deletions
114
presto-iceberg/src/main/java/com/facebook/presto/iceberg/hive/IcebergFileHiveMetastore.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,114 @@ | ||
| /* | ||
| * Licensed under the Apache License, Version 2.0 (the "License"); | ||
| * you may not use this file except in compliance with the License. | ||
| * You may obtain a copy of the License at | ||
| * | ||
| * http://www.apache.org/licenses/LICENSE-2.0 | ||
| * | ||
| * Unless required by applicable law or agreed to in writing, software | ||
| * distributed under the License is distributed on an "AS IS" BASIS, | ||
| * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
| * See the License for the specific language governing permissions and | ||
| * limitations under the License. | ||
| */ | ||
| package com.facebook.presto.iceberg.hive; | ||
|
|
||
| import com.facebook.airlift.log.Logger; | ||
| import com.facebook.presto.hive.HdfsEnvironment; | ||
| import com.facebook.presto.hive.metastore.Table; | ||
| import com.facebook.presto.hive.metastore.file.FileHiveMetastore; | ||
| import com.facebook.presto.hive.metastore.file.FileHiveMetastoreConfig; | ||
| import com.facebook.presto.spi.PrestoException; | ||
| import org.apache.hadoop.fs.FileSystem; | ||
| import org.apache.hadoop.fs.FileUtil; | ||
| import org.apache.hadoop.fs.Path; | ||
|
|
||
| import javax.annotation.concurrent.ThreadSafe; | ||
| import javax.inject.Inject; | ||
|
|
||
| import java.io.IOException; | ||
| import java.util.Optional; | ||
|
|
||
| import static com.facebook.presto.hive.HiveErrorCode.HIVE_METASTORE_ERROR; | ||
| import static java.lang.String.format; | ||
|
|
||
| @ThreadSafe | ||
| public class IcebergFileHiveMetastore | ||
| extends FileHiveMetastore | ||
| { | ||
| private static final Logger LOG = Logger.get(IcebergFileHiveMetastore.class); | ||
|
|
||
| @Inject | ||
| public IcebergFileHiveMetastore(HdfsEnvironment hdfsEnvironment, FileHiveMetastoreConfig config) | ||
| { | ||
| this(hdfsEnvironment, config.getCatalogDirectory(), config.getMetastoreUser()); | ||
| } | ||
|
|
||
| public IcebergFileHiveMetastore(HdfsEnvironment hdfsEnvironment, String catalogDirectory, String metastoreUser) | ||
| { | ||
| super(hdfsEnvironment, catalogDirectory, metastoreUser); | ||
| } | ||
|
|
||
| @Override | ||
| protected void validateExternalLocation(Path externalLocation, Path catalogDirectory) | ||
| throws IOException | ||
| { | ||
| FileSystem externalFileSystem = hdfsEnvironment.getFileSystem(hdfsContext, externalLocation); | ||
| if (!externalFileSystem.isDirectory(externalLocation)) { | ||
| throw new PrestoException(HIVE_METASTORE_ERROR, "External table location does not exist"); | ||
| } | ||
| } | ||
|
|
||
| @Override | ||
| protected void validateReplaceTableType(Table originTable, Table newTable) | ||
| {} | ||
|
|
||
| @Override | ||
| protected void renameTable(Path originalMetadataDirectory, Path newMetadataDirectory) | ||
| { | ||
| Optional<Runnable> rollbackAction = Optional.empty(); | ||
| try { | ||
| // If the directory `.prestoPermissions` exists, copy it to the new table metadata directory | ||
| Path originTablePermissionDir = new Path(originalMetadataDirectory, PRESTO_PERMISSIONS_DIRECTORY_NAME); | ||
| Path newTablePermissionDir = new Path(newMetadataDirectory, PRESTO_PERMISSIONS_DIRECTORY_NAME); | ||
| if (metadataFileSystem.exists(originTablePermissionDir)) { | ||
| if (!FileUtil.copy(metadataFileSystem, originTablePermissionDir, | ||
| metadataFileSystem, newTablePermissionDir, false, metadataFileSystem.getConf())) { | ||
| throw new IOException(format("Could not rename table. Failed to copy directory: %s to %s", originTablePermissionDir, newTablePermissionDir)); | ||
| } | ||
| else { | ||
| rollbackAction = Optional.of(() -> { | ||
| try { | ||
| metadataFileSystem.delete(newTablePermissionDir, true); | ||
| } | ||
| catch (IOException e) { | ||
| // Ignore the exception and print a warn level log | ||
| LOG.warn("Could not delete table permission directory: %s", newTablePermissionDir); | ||
| } | ||
| }); | ||
| } | ||
| } | ||
|
|
||
| // Rename file `.prestoSchema` to change it to the new metadata path | ||
| // This will atomically execute the table renaming behavior | ||
| Path originMetadataFile = new Path(originalMetadataDirectory, PRESTO_SCHEMA_FILE_NAME); | ||
| Path newMetadataFile = new Path(newMetadataDirectory, PRESTO_SCHEMA_FILE_NAME); | ||
| renamePath(originMetadataFile, newMetadataFile, | ||
| format("Could not rename table. Failed to rename file %s to %s", originMetadataFile, newMetadataFile)); | ||
|
|
||
| // Subsequent action, delete the redundant directory `.prestoPermissions` from the original table metadata path | ||
| try { | ||
| metadataFileSystem.delete(new Path(originalMetadataDirectory, PRESTO_PERMISSIONS_DIRECTORY_NAME), true); | ||
| } | ||
| catch (IOException e) { | ||
| // Ignore the exception and print a warn level log | ||
| LOG.warn("Could not delete table permission directory: %s", originalMetadataDirectory); | ||
| } | ||
| } | ||
| catch (IOException e) { | ||
| // If table renaming fails and rollback action has already been recorded, perform the rollback action to clean up junk files | ||
| rollbackAction.ifPresent(Runnable::run); | ||
| throw new PrestoException(HIVE_METASTORE_ERROR, e); | ||
| } | ||
| } | ||
| } |
48 changes: 48 additions & 0 deletions
48
...ceberg/src/main/java/com/facebook/presto/iceberg/hive/IcebergHiveFileMetastoreModule.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,48 @@ | ||
| /* | ||
| * Licensed under the Apache License, Version 2.0 (the "License"); | ||
| * you may not use this file except in compliance with the License. | ||
| * You may obtain a copy of the License at | ||
| * | ||
| * http://www.apache.org/licenses/LICENSE-2.0 | ||
| * | ||
| * Unless required by applicable law or agreed to in writing, software | ||
| * distributed under the License is distributed on an "AS IS" BASIS, | ||
| * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
| * See the License for the specific language governing permissions and | ||
| * limitations under the License. | ||
| */ | ||
| package com.facebook.presto.iceberg.hive; | ||
|
|
||
| import com.facebook.presto.hive.ForCachingHiveMetastore; | ||
| import com.facebook.presto.hive.metastore.ExtendedHiveMetastore; | ||
| import com.facebook.presto.hive.metastore.InMemoryCachingHiveMetastore; | ||
| import com.facebook.presto.hive.metastore.file.FileHiveMetastoreConfig; | ||
| import com.google.inject.Binder; | ||
| import com.google.inject.Module; | ||
| import com.google.inject.Scopes; | ||
|
|
||
| import static com.facebook.airlift.configuration.ConfigBinder.configBinder; | ||
| import static java.util.Objects.requireNonNull; | ||
| import static org.weakref.jmx.ObjectNames.generatedNameOf; | ||
| import static org.weakref.jmx.guice.ExportBinder.newExporter; | ||
|
|
||
| public class IcebergHiveFileMetastoreModule | ||
| implements Module | ||
| { | ||
| private final String connectorId; | ||
|
|
||
| public IcebergHiveFileMetastoreModule(String connectorId) | ||
| { | ||
| this.connectorId = requireNonNull(connectorId, "connectorId is null"); | ||
| } | ||
|
|
||
| @Override | ||
| public void configure(Binder binder) | ||
| { | ||
| configBinder(binder).bindConfig(FileHiveMetastoreConfig.class); | ||
| binder.bind(ExtendedHiveMetastore.class).annotatedWith(ForCachingHiveMetastore.class).to(IcebergFileHiveMetastore.class).in(Scopes.SINGLETON); | ||
| binder.bind(ExtendedHiveMetastore.class).to(InMemoryCachingHiveMetastore.class).in(Scopes.SINGLETON); | ||
| newExporter(binder).export(ExtendedHiveMetastore.class) | ||
| .as(generatedNameOf(InMemoryCachingHiveMetastore.class, connectorId)); | ||
| } | ||
| } |
60 changes: 60 additions & 0 deletions
60
...to-iceberg/src/main/java/com/facebook/presto/iceberg/hive/IcebergHiveMetastoreModule.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,60 @@ | ||
| /* | ||
| * Licensed under the Apache License, Version 2.0 (the "License"); | ||
| * you may not use this file except in compliance with the License. | ||
| * You may obtain a copy of the License at | ||
| * | ||
| * http://www.apache.org/licenses/LICENSE-2.0 | ||
| * | ||
| * Unless required by applicable law or agreed to in writing, software | ||
| * distributed under the License is distributed on an "AS IS" BASIS, | ||
| * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
| * See the License for the specific language governing permissions and | ||
| * limitations under the License. | ||
| */ | ||
| package com.facebook.presto.iceberg.hive; | ||
|
|
||
| import com.facebook.airlift.configuration.AbstractConfigurationAwareModule; | ||
| import com.facebook.presto.hive.metastore.ExtendedHiveMetastore; | ||
| import com.facebook.presto.hive.metastore.MetastoreConfig; | ||
| import com.facebook.presto.hive.metastore.glue.GlueMetastoreModule; | ||
| import com.facebook.presto.hive.metastore.thrift.ThriftMetastoreModule; | ||
| import com.google.inject.Binder; | ||
| import com.google.inject.Module; | ||
|
|
||
| import java.util.Optional; | ||
|
|
||
| import static com.facebook.airlift.configuration.ConditionalModule.installModuleIf; | ||
|
|
||
| public class IcebergHiveMetastoreModule | ||
| extends AbstractConfigurationAwareModule | ||
| { | ||
| private final String connectorId; | ||
| private final Optional<ExtendedHiveMetastore> metastore; | ||
|
|
||
| public IcebergHiveMetastoreModule(String connectorId, Optional<ExtendedHiveMetastore> metastore) | ||
| { | ||
| this.connectorId = connectorId; | ||
| this.metastore = metastore; | ||
| } | ||
|
|
||
| @Override | ||
| protected void setup(Binder binder) | ||
| { | ||
| if (metastore.isPresent()) { | ||
| binder.bind(ExtendedHiveMetastore.class).toInstance(metastore.get()); | ||
| } | ||
| else { | ||
| bindMetastoreModule("thrift", new ThriftMetastoreModule(connectorId)); | ||
| bindMetastoreModule("file", new IcebergHiveFileMetastoreModule(connectorId)); | ||
| bindMetastoreModule("glue", new GlueMetastoreModule(connectorId)); | ||
| } | ||
| } | ||
|
|
||
| private void bindMetastoreModule(String name, Module module) | ||
| { | ||
| install(installModuleIf( | ||
| MetastoreConfig.class, | ||
| metastore -> name.equalsIgnoreCase(metastore.getMetastoreType()), | ||
| module)); | ||
| } | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters