Java tutorial
/* * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package com.facebook.presto.hive.metastore.file; import com.facebook.presto.hive.HdfsEnvironment; import com.facebook.presto.hive.HdfsEnvironment.HdfsContext; import com.facebook.presto.hive.HiveType; import com.facebook.presto.hive.SchemaAlreadyExistsException; import com.facebook.presto.hive.TableAlreadyExistsException; import com.facebook.presto.hive.metastore.Column; import com.facebook.presto.hive.metastore.Database; import com.facebook.presto.hive.metastore.ExtendedHiveMetastore; import com.facebook.presto.hive.metastore.HiveColumnStatistics; import com.facebook.presto.hive.metastore.HivePrivilegeInfo; import com.facebook.presto.hive.metastore.Partition; import com.facebook.presto.hive.metastore.PrincipalPrivileges; import com.facebook.presto.hive.metastore.PrincipalType; import com.facebook.presto.hive.metastore.Table; import com.facebook.presto.spi.ColumnNotFoundException; import com.facebook.presto.spi.PrestoException; import com.facebook.presto.spi.SchemaNotFoundException; import com.facebook.presto.spi.SchemaTableName; import com.facebook.presto.spi.TableNotFoundException; import com.facebook.presto.spi.security.Identity; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; import com.google.common.io.ByteStreams; import io.airlift.json.JsonCodec; import org.apache.hadoop.fs.FSDataInputStream; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hive.metastore.TableType; import javax.annotation.concurrent.ThreadSafe; import javax.inject.Inject; import java.io.IOException; import java.io.OutputStream; import java.util.ArrayDeque; import java.util.ArrayList; import java.util.Collection; import java.util.EnumSet; import java.util.HashSet; import java.util.LinkedHashMap; import java.util.LinkedHashSet; import java.util.List; import java.util.Locale; import java.util.Map; import java.util.Map.Entry; import java.util.Optional; import java.util.Set; import java.util.function.Function; import static com.facebook.presto.hive.HiveErrorCode.HIVE_METASTORE_ERROR; import static com.facebook.presto.hive.HiveUtil.toPartitionValues; import static com.facebook.presto.hive.metastore.Database.DEFAULT_DATABASE_NAME; import static com.facebook.presto.hive.metastore.HivePrivilegeInfo.HivePrivilege.OWNERSHIP; import static com.facebook.presto.hive.metastore.MetastoreUtil.makePartName; import static com.facebook.presto.hive.metastore.MetastoreUtil.verifyCanDropColumn; import static com.facebook.presto.hive.metastore.PrincipalType.ROLE; import static com.facebook.presto.hive.metastore.PrincipalType.USER; import static com.facebook.presto.spi.StandardErrorCode.ALREADY_EXISTS; import static com.facebook.presto.spi.StandardErrorCode.NOT_SUPPORTED; import static com.google.common.base.Preconditions.checkArgument; import static java.util.Objects.requireNonNull; import static java.util.stream.Collectors.toList; import static org.apache.hadoop.hive.metastore.TableType.EXTERNAL_TABLE; import static org.apache.hadoop.hive.metastore.TableType.MANAGED_TABLE; import static org.apache.hadoop.hive.metastore.TableType.VIRTUAL_VIEW; @ThreadSafe public class FileHiveMetastore implements ExtendedHiveMetastore { private static final String PUBLIC_ROLE_NAME = "public"; private static final String PRESTO_SCHEMA_FILE_NAME = ".prestoSchema"; private static final String PRESTO_PERMISSIONS_DIRECTORY_NAME = ".prestoPermissions"; private final HdfsEnvironment hdfsEnvironment; private final Path catalogDirectory; private final HdfsContext hdfsContext; private final FileSystem metadataFileSystem; private final JsonCodec<DatabaseMetadata> databaseCodec = JsonCodec.jsonCodec(DatabaseMetadata.class); private final JsonCodec<TableMetadata> tableCodec = JsonCodec.jsonCodec(TableMetadata.class); private final JsonCodec<PartitionMetadata> partitionCodec = JsonCodec.jsonCodec(PartitionMetadata.class); private final JsonCodec<List<PermissionMetadata>> permissionsCodec = JsonCodec .listJsonCodec(PermissionMetadata.class); @Inject public FileHiveMetastore(HdfsEnvironment hdfsEnvironment, FileHiveMetastoreConfig config) { this(hdfsEnvironment, config.getCatalogDirectory(), config.getMetastoreUser()); } public FileHiveMetastore(HdfsEnvironment hdfsEnvironment, String catalogDirectory, String metastoreUser) { this.hdfsEnvironment = requireNonNull(hdfsEnvironment, "hdfsEnvironment is null"); this.catalogDirectory = new Path(requireNonNull(catalogDirectory, "baseDirectory is null")); this.hdfsContext = new HdfsContext(new Identity(metastoreUser, Optional.empty())); try { metadataFileSystem = hdfsEnvironment.getFileSystem(hdfsContext, this.catalogDirectory); } catch (IOException e) { throw new PrestoException(HIVE_METASTORE_ERROR, e); } } @Override public synchronized void createDatabase(Database database) { requireNonNull(database, "database is null"); if (database.getLocation().isPresent()) { throw new PrestoException(HIVE_METASTORE_ERROR, "Database can not be created with a location set"); } verifyDatabaseNotExists(database.getDatabaseName()); Path databaseMetadataDirectory = getDatabaseMetadataDirectory(database.getDatabaseName()); writeSchemaFile("database", databaseMetadataDirectory, databaseCodec, new DatabaseMetadata(database), false); } @Override public synchronized void dropDatabase(String databaseName) { requireNonNull(databaseName, "databaseName is null"); getRequiredDatabase(databaseName); if (!getAllTables(databaseName).orElse(ImmutableList.of()).isEmpty()) { throw new PrestoException(HIVE_METASTORE_ERROR, "Database " + databaseName + " is not empty"); } deleteMetadataDirectory(getDatabaseMetadataDirectory(databaseName)); } @Override public synchronized void renameDatabase(String databaseName, String newDatabaseName) { requireNonNull(databaseName, "databaseName is null"); requireNonNull(newDatabaseName, "newDatabaseName is null"); getRequiredDatabase(databaseName); verifyDatabaseNotExists(newDatabaseName); try { if (!metadataFileSystem.rename(getDatabaseMetadataDirectory(databaseName), getDatabaseMetadataDirectory(newDatabaseName))) { throw new PrestoException(HIVE_METASTORE_ERROR, "Could not rename database metadata directory"); } } catch (IOException e) { throw new PrestoException(HIVE_METASTORE_ERROR, e); } } @Override public synchronized Optional<Database> getDatabase(String databaseName) { requireNonNull(databaseName, "databaseName is null"); Path databaseMetadataDirectory = getDatabaseMetadataDirectory(databaseName); return readSchemaFile("database", databaseMetadataDirectory, databaseCodec) .map(databaseMetadata -> databaseMetadata.toDatabase(databaseName, databaseMetadataDirectory.toString())); } private Database getRequiredDatabase(String databaseName) { return getDatabase(databaseName).orElseThrow(() -> new SchemaNotFoundException(databaseName)); } private void verifyDatabaseNotExists(String databaseName) { if (getDatabase(databaseName).isPresent()) { throw new SchemaAlreadyExistsException(databaseName); } } @Override public synchronized List<String> getAllDatabases() { List<String> databases = getChildSchemaDirectories(catalogDirectory).stream().map(Path::getName) .collect(toList()); return ImmutableList.copyOf(databases); } @Override public synchronized void createTable(Table table, PrincipalPrivileges principalPrivileges) { verifyTableNotExists(table.getDatabaseName(), table.getTableName()); Path tableMetadataDirectory = getTableMetadataDirectory(table); // validate table location if (table.getTableType().equals(VIRTUAL_VIEW.name())) { checkArgument(table.getStorage().getLocation().isEmpty(), "Storage location for view must be empty"); } else if (table.getTableType().equals(MANAGED_TABLE.name())) { if (!tableMetadataDirectory.equals(new Path(table.getStorage().getLocation()))) { throw new PrestoException(HIVE_METASTORE_ERROR, "Table directory must be " + tableMetadataDirectory); } } else if (table.getTableType().equals(EXTERNAL_TABLE.name())) { try { Path externalLocation = new Path(table.getStorage().getLocation()); FileSystem externalFileSystem = hdfsEnvironment.getFileSystem(hdfsContext, externalLocation); if (!externalFileSystem.isDirectory(externalLocation)) { throw new PrestoException(HIVE_METASTORE_ERROR, "External table location does not exist"); } if (isChildDirectory(catalogDirectory, externalLocation)) { throw new PrestoException(HIVE_METASTORE_ERROR, "External table location can not be inside the system metadata directory"); } } catch (IOException e) { throw new PrestoException(HIVE_METASTORE_ERROR, "Could not validate external location", e); } } else { throw new PrestoException(NOT_SUPPORTED, "Table type not supported: " + table.getTableType()); } writeSchemaFile("table", tableMetadataDirectory, tableCodec, new TableMetadata(table), false); for (Entry<String, Collection<HivePrivilegeInfo>> entry : principalPrivileges.getUserPrivileges().asMap() .entrySet()) { setTablePrivileges(entry.getKey(), USER, table.getDatabaseName(), table.getTableName(), entry.getValue()); } for (Entry<String, Collection<HivePrivilegeInfo>> entry : principalPrivileges.getRolePrivileges().asMap() .entrySet()) { setTablePrivileges(entry.getKey(), ROLE, table.getDatabaseName(), table.getTableName(), entry.getValue()); } } @Override public synchronized Optional<Table> getTable(String databaseName, String tableName) { requireNonNull(databaseName, "databaseName is null"); requireNonNull(tableName, "tableName is null"); Path tableMetadataDirectory = getTableMetadataDirectory(databaseName, tableName); return readSchemaFile("table", tableMetadataDirectory, tableCodec).map( tableMetadata -> tableMetadata.toTable(databaseName, tableName, tableMetadataDirectory.toString())); } @Override public Optional<Map<String, HiveColumnStatistics>> getTableColumnStatistics(String databaseName, String tableName, Set<String> columnNames) { return Optional.of(ImmutableMap.of()); } @Override public Optional<Map<String, Map<String, HiveColumnStatistics>>> getPartitionColumnStatistics( String databaseName, String tableName, Set<String> partitionNames, Set<String> columnNames) { return Optional.of(ImmutableMap.of()); } private Table getRequiredTable(String databaseName, String tableName) { return getTable(databaseName, tableName) .orElseThrow(() -> new TableNotFoundException(new SchemaTableName(databaseName, tableName))); } private void verifyTableNotExists(String newDatabaseName, String newTableName) { if (getTable(newDatabaseName, newTableName).isPresent()) { throw new TableAlreadyExistsException(new SchemaTableName(newDatabaseName, newTableName)); } } @Override public synchronized Optional<List<String>> getAllTables(String databaseName) { requireNonNull(databaseName, "databaseName is null"); Optional<Database> database = getDatabase(databaseName); if (!database.isPresent()) { return Optional.empty(); } Path databaseMetadataDirectory = getDatabaseMetadataDirectory(databaseName); List<String> tables = getChildSchemaDirectories(databaseMetadataDirectory).stream().map(Path::getName) .collect(toList()); return Optional.of(ImmutableList.copyOf(tables)); } @Override public synchronized Optional<List<String>> getAllViews(String databaseName) { Optional<List<String>> tables = getAllTables(databaseName); if (!tables.isPresent()) { return Optional.empty(); } List<String> views = tables.get().stream().map(tableName -> getTable(databaseName, tableName)) .filter(Optional::isPresent).map(Optional::get) .filter(table -> table.getTableType().equals(VIRTUAL_VIEW.name())).map(Table::getTableName) .collect(toList()); return Optional.of(ImmutableList.copyOf(views)); } @Override public synchronized void dropTable(String databaseName, String tableName, boolean deleteData) { requireNonNull(databaseName, "databaseName is null"); requireNonNull(tableName, "tableName is null"); Table table = getRequiredTable(databaseName, tableName); Path tableMetadataDirectory = getTableMetadataDirectory(databaseName, tableName); // It is safe to delete the whole meta directory for external tables and views if (!table.getTableType().equals(MANAGED_TABLE.name()) || deleteData) { deleteMetadataDirectory(tableMetadataDirectory); } else { // in this case we only wan to delete the metadata of a managed table deleteSchemaFile("table", tableMetadataDirectory); deleteTablePrivileges(table); } } @Override public synchronized void replaceTable(String databaseName, String tableName, Table newTable, PrincipalPrivileges principalPrivileges) { Table table = getRequiredTable(databaseName, tableName); if (!table.getTableType().equals(VIRTUAL_VIEW.name()) || !newTable.getTableType().equals(VIRTUAL_VIEW.name())) { throw new PrestoException(HIVE_METASTORE_ERROR, "Only views can be updated with replaceTable"); } if (!table.getDatabaseName().equals(databaseName) || !table.getTableName().equals(tableName)) { throw new PrestoException(HIVE_METASTORE_ERROR, "Replacement table must have same name"); } Path tableMetadataDirectory = getTableMetadataDirectory(table); writeSchemaFile("table", tableMetadataDirectory, tableCodec, new TableMetadata(newTable), true); // replace existing permissions deleteTablePrivileges(table); for (Entry<String, Collection<HivePrivilegeInfo>> entry : principalPrivileges.getUserPrivileges().asMap() .entrySet()) { setTablePrivileges(entry.getKey(), USER, table.getDatabaseName(), table.getTableName(), entry.getValue()); } for (Entry<String, Collection<HivePrivilegeInfo>> entry : principalPrivileges.getRolePrivileges().asMap() .entrySet()) { setTablePrivileges(entry.getKey(), ROLE, table.getDatabaseName(), table.getTableName(), entry.getValue()); } } @Override public synchronized void renameTable(String databaseName, String tableName, String newDatabaseName, String newTableName) { requireNonNull(databaseName, "databaseName is null"); requireNonNull(tableName, "tableName is null"); requireNonNull(newDatabaseName, "newDatabaseName is null"); requireNonNull(newTableName, "newTableName is null"); getRequiredTable(databaseName, tableName); getRequiredDatabase(newDatabaseName); // verify new table does not exist verifyTableNotExists(newDatabaseName, newTableName); try { if (!metadataFileSystem.rename(getTableMetadataDirectory(databaseName, tableName), getTableMetadataDirectory(newDatabaseName, newTableName))) { throw new PrestoException(HIVE_METASTORE_ERROR, "Could not rename table directory"); } } catch (IOException e) { throw new PrestoException(HIVE_METASTORE_ERROR, e); } } @Override public synchronized void addColumn(String databaseName, String tableName, String columnName, HiveType columnType, String columnComment) { alterTable(databaseName, tableName, oldTable -> { if (oldTable.getColumn(columnName).isPresent()) { throw new PrestoException(ALREADY_EXISTS, "Column already exists: " + columnName); } return oldTable.withDataColumns(ImmutableList.<Column>builder().addAll(oldTable.getDataColumns()) .add(new Column(columnName, columnType, Optional.ofNullable(columnComment))).build()); }); } @Override public synchronized void renameColumn(String databaseName, String tableName, String oldColumnName, String newColumnName) { alterTable(databaseName, tableName, oldTable -> { if (oldTable.getColumn(newColumnName).isPresent()) { throw new PrestoException(ALREADY_EXISTS, "Column already exists: " + newColumnName); } if (!oldTable.getColumn(oldColumnName).isPresent()) { SchemaTableName name = new SchemaTableName(databaseName, tableName); throw new ColumnNotFoundException(name, oldColumnName); } for (Column column : oldTable.getPartitionColumns()) { if (column.getName().equals(oldColumnName)) { throw new PrestoException(NOT_SUPPORTED, "Renaming partition columns is not supported"); } } ImmutableList.Builder<Column> newDataColumns = ImmutableList.builder(); for (Column fieldSchema : oldTable.getDataColumns()) { if (fieldSchema.getName().equals(oldColumnName)) { newDataColumns.add(new Column(newColumnName, fieldSchema.getType(), fieldSchema.getComment())); } else { newDataColumns.add(fieldSchema); } } return oldTable.withDataColumns(newDataColumns.build()); }); } @Override public synchronized void dropColumn(String databaseName, String tableName, String columnName) { alterTable(databaseName, tableName, oldTable -> { verifyCanDropColumn(this, databaseName, tableName, columnName); if (!oldTable.getColumn(columnName).isPresent()) { SchemaTableName name = new SchemaTableName(databaseName, tableName); throw new ColumnNotFoundException(name, columnName); } ImmutableList.Builder<Column> newDataColumns = ImmutableList.builder(); for (Column fieldSchema : oldTable.getDataColumns()) { if (!fieldSchema.getName().equals(columnName)) { newDataColumns.add(fieldSchema); } } return oldTable.withDataColumns(newDataColumns.build()); }); } private void alterTable(String databaseName, String tableName, Function<TableMetadata, TableMetadata> alterFunction) { requireNonNull(databaseName, "databaseName is null"); requireNonNull(tableName, "tableName is null"); Path tableMetadataDirectory = getTableMetadataDirectory(databaseName, tableName); TableMetadata oldTableSchema = readSchemaFile("table", tableMetadataDirectory, tableCodec) .orElseThrow(() -> new TableNotFoundException(new SchemaTableName(databaseName, tableName))); TableMetadata newTableSchema = alterFunction.apply(oldTableSchema); if (oldTableSchema == newTableSchema) { return; } writeSchemaFile("table", tableMetadataDirectory, tableCodec, newTableSchema, true); } @Override public synchronized void addPartitions(String databaseName, String tableName, List<Partition> partitions) { requireNonNull(databaseName, "databaseName is null"); requireNonNull(tableName, "tableName is null"); requireNonNull(partitions, "partitions is null"); Table table = getRequiredTable(databaseName, tableName); TableType tableType = TableType.valueOf(table.getTableType()); checkArgument(EnumSet.of(MANAGED_TABLE, EXTERNAL_TABLE).contains(tableType), "Invalid table type: %s", tableType); try { Map<Path, byte[]> schemaFiles = new LinkedHashMap<>(); for (Partition partition : partitions) { verifiedPartition(table, partition); Path partitionMetadataDirectory = getPartitionMetadataDirectory(table, partition.getValues()); Path schemaPath = new Path(partitionMetadataDirectory, PRESTO_SCHEMA_FILE_NAME); if (metadataFileSystem.exists(schemaPath)) { throw new PrestoException(HIVE_METASTORE_ERROR, "Partition already exists"); } byte[] schemaJson = partitionCodec.toJsonBytes(new PartitionMetadata(table, partition)); schemaFiles.put(schemaPath, schemaJson); } Set<Path> createdFiles = new LinkedHashSet<>(); try { for (Entry<Path, byte[]> entry : schemaFiles.entrySet()) { try (OutputStream outputStream = metadataFileSystem.create(entry.getKey())) { createdFiles.add(entry.getKey()); outputStream.write(entry.getValue()); } catch (IOException e) { throw new PrestoException(HIVE_METASTORE_ERROR, "Could not write partition schema", e); } } } catch (Throwable e) { for (Path createdFile : createdFiles) { try { metadataFileSystem.delete(createdFile, false); } catch (IOException ignored) { } } throw e; } } catch (IOException e) { throw new PrestoException(HIVE_METASTORE_ERROR, e); } } private void verifiedPartition(Table table, Partition partition) { Path partitionMetadataDirectory = getPartitionMetadataDirectory(table, partition.getValues()); if (table.getTableType().equals(MANAGED_TABLE.name())) { if (!partitionMetadataDirectory.equals(new Path(partition.getStorage().getLocation()))) { throw new PrestoException(HIVE_METASTORE_ERROR, "Partition directory must be " + partitionMetadataDirectory); } } else if (table.getTableType().equals(EXTERNAL_TABLE.name())) { try { Path externalLocation = new Path(partition.getStorage().getLocation()); FileSystem externalFileSystem = hdfsEnvironment.getFileSystem(hdfsContext, externalLocation); if (!externalFileSystem.isDirectory(externalLocation)) { throw new PrestoException(HIVE_METASTORE_ERROR, "External partition location does not exist"); } if (isChildDirectory(catalogDirectory, externalLocation)) { throw new PrestoException(HIVE_METASTORE_ERROR, "External partition location can not be inside the system metadata directory"); } } catch (IOException e) { throw new PrestoException(HIVE_METASTORE_ERROR, "Could not validate external partition location", e); } } else { throw new PrestoException(NOT_SUPPORTED, "Partitions can not be added to " + table.getTableType()); } } @Override public synchronized void dropPartition(String databaseName, String tableName, List<String> partitionValues, boolean deleteData) { requireNonNull(databaseName, "databaseName is null"); requireNonNull(tableName, "tableName is null"); requireNonNull(partitionValues, "partitionValues is null"); Optional<Table> tableReference = getTable(databaseName, tableName); if (!tableReference.isPresent()) { return; } Table table = tableReference.get(); Path partitionMetadataDirectory = getPartitionMetadataDirectory(table, partitionValues); if (deleteData) { deleteMetadataDirectory(partitionMetadataDirectory); } else { deleteSchemaFile("partition", partitionMetadataDirectory); } } @Override public synchronized void alterPartition(String databaseName, String tableName, Partition partition) { Table table = getRequiredTable(databaseName, tableName); verifiedPartition(table, partition); Path partitionMetadataDirectory = getPartitionMetadataDirectory(table, partition.getValues()); writeSchemaFile("partition", partitionMetadataDirectory, partitionCodec, new PartitionMetadata(table, partition), true); } @Override public synchronized Optional<List<String>> getPartitionNames(String databaseName, String tableName) { requireNonNull(databaseName, "databaseName is null"); requireNonNull(tableName, "tableName is null"); Optional<Table> tableReference = getTable(databaseName, tableName); if (!tableReference.isPresent()) { return Optional.empty(); } Table table = tableReference.get(); Path tableMetadataDirectory = getTableMetadataDirectory(table); List<ArrayDeque<String>> partitions = listPartitions(tableMetadataDirectory, table.getPartitionColumns()); List<String> partitionNames = partitions.stream().map( partitionValues -> makePartName(table.getPartitionColumns(), ImmutableList.copyOf(partitionValues))) .collect(toList()); return Optional.of(ImmutableList.copyOf(partitionNames)); } private List<ArrayDeque<String>> listPartitions(Path director, List<Column> partitionColumns) { if (partitionColumns.isEmpty()) { return ImmutableList.of(); } try { String directoryPrefix = partitionColumns.get(0).getName() + '='; List<ArrayDeque<String>> partitionValues = new ArrayList<>(); for (FileStatus fileStatus : metadataFileSystem.listStatus(director)) { if (!fileStatus.isDirectory()) { continue; } if (!fileStatus.getPath().getName().startsWith(directoryPrefix)) { continue; } List<ArrayDeque<String>> childPartitionValues; if (partitionColumns.size() == 1) { childPartitionValues = ImmutableList.of(new ArrayDeque<>()); } else { childPartitionValues = listPartitions(fileStatus.getPath(), partitionColumns.subList(1, partitionColumns.size())); } String value = fileStatus.getPath().getName().substring(directoryPrefix.length()); for (ArrayDeque<String> childPartition : childPartitionValues) { childPartition.addFirst(value); partitionValues.add(childPartition); } } return partitionValues; } catch (IOException e) { throw new PrestoException(HIVE_METASTORE_ERROR, "Error listing partition directories", e); } } @Override public synchronized Optional<Partition> getPartition(String databaseName, String tableName, List<String> partitionValues) { requireNonNull(databaseName, "databaseName is null"); requireNonNull(tableName, "tableName is null"); requireNonNull(partitionValues, "partitionValues is null"); Optional<Table> tableReference = getTable(databaseName, tableName); if (!tableReference.isPresent()) { return Optional.empty(); } Table table = tableReference.get(); Path partitionDirectory = getPartitionMetadataDirectory(table, partitionValues); return readSchemaFile("partition", partitionDirectory, partitionCodec) .map(partitionMetadata -> partitionMetadata.toPartition(databaseName, tableName, partitionValues, partitionDirectory.toString())); } @Override public synchronized Optional<List<String>> getPartitionNamesByParts(String databaseName, String tableName, List<String> parts) { // todo this should be more efficient by selectively walking the directory tree return getPartitionNames(databaseName, tableName).map(partitionNames -> partitionNames.stream() .filter(partitionName -> partitionMatches(partitionName, parts)).collect(toList())); } private static boolean partitionMatches(String partitionName, List<String> parts) { List<String> values = toPartitionValues(partitionName); if (values.size() != parts.size()) { return false; } for (int i = 0; i < values.size(); i++) { String part = parts.get(i); if (!part.isEmpty() && !values.get(i).equals(part)) { return false; } } return true; } @Override public synchronized Map<String, Optional<Partition>> getPartitionsByNames(String databaseName, String tableName, List<String> partitionNames) { ImmutableMap.Builder<String, Optional<Partition>> builder = ImmutableMap.builder(); for (String partitionName : partitionNames) { List<String> partitionValues = toPartitionValues(partitionName); builder.put(partitionName, getPartition(databaseName, tableName, partitionValues)); } return builder.build(); } @Override public synchronized Set<String> getRoles(String user) { return ImmutableSet.<String>builder().add(PUBLIC_ROLE_NAME).add("admin") // todo there should be a way to manage the admins list .build(); } @Override public synchronized Set<HivePrivilegeInfo> getDatabasePrivileges(String user, String databaseName) { Set<HivePrivilegeInfo> privileges = new HashSet<>(); if (isDatabaseOwner(user, databaseName)) { privileges.add(new HivePrivilegeInfo(OWNERSHIP, true)); } return privileges; } @Override public synchronized Set<HivePrivilegeInfo> getTablePrivileges(String user, String databaseName, String tableName) { Table table = getRequiredTable(databaseName, tableName); Set<HivePrivilegeInfo> privileges = new HashSet<>(); if (user.equals(table.getOwner())) { privileges.add(new HivePrivilegeInfo(OWNERSHIP, true)); } Path permissionsDirectory = getPermissionsDirectory(table); privileges.addAll(getTablePrivileges(permissionsDirectory, user, USER)); for (String role : getRoles(user)) { privileges.addAll(getTablePrivileges(permissionsDirectory, role, ROLE)); } return privileges; } private synchronized Collection<HivePrivilegeInfo> getTablePrivileges(Path permissionsDirectory, String principalName, PrincipalType principalType) { Path permissionFilePath = getPermissionsPath(permissionsDirectory, principalName, principalType); return readFile("permissions", permissionFilePath, permissionsCodec).orElse(ImmutableList.of()).stream() .map(PermissionMetadata::toHivePrivilegeInfo).collect(toList()); } @Override public synchronized void grantTablePrivileges(String databaseName, String tableName, String grantee, Set<HivePrivilegeInfo> privileges) { setTablePrivileges(grantee, USER, databaseName, tableName, privileges); } @Override public synchronized void revokeTablePrivileges(String databaseName, String tableName, String grantee, Set<HivePrivilegeInfo> privileges) { Set<HivePrivilegeInfo> currentPrivileges = getTablePrivileges(grantee, databaseName, tableName); currentPrivileges.removeAll(privileges); setTablePrivileges(grantee, USER, databaseName, tableName, currentPrivileges); } private synchronized void setTablePrivileges(String principalName, PrincipalType principalType, String databaseName, String tableName, Collection<HivePrivilegeInfo> privileges) { requireNonNull(principalName, "principalName is null"); requireNonNull(principalType, "principalType is null"); requireNonNull(databaseName, "databaseName is null"); requireNonNull(tableName, "tableName is null"); requireNonNull(privileges, "privileges is null"); try { Table table = getRequiredTable(databaseName, tableName); Path permissionsDirectory = getPermissionsDirectory(table); metadataFileSystem.mkdirs(permissionsDirectory); if (!metadataFileSystem.isDirectory(permissionsDirectory)) { throw new PrestoException(HIVE_METASTORE_ERROR, "Could not create permissions directory"); } Path permissionFilePath = getPermissionsPath(permissionsDirectory, principalName, principalType); List<PermissionMetadata> permissions = privileges.stream().map(PermissionMetadata::new) .collect(toList()); writeFile("permissions", permissionFilePath, permissionsCodec, permissions, true); } catch (IOException e) { throw new PrestoException(HIVE_METASTORE_ERROR, e); } } private synchronized void deleteTablePrivileges(Table table) { try { Path permissionsDirectory = getPermissionsDirectory(table); metadataFileSystem.delete(permissionsDirectory, true); } catch (IOException e) { throw new PrestoException(HIVE_METASTORE_ERROR, "Could not delete table permissions", e); } } private boolean isDatabaseOwner(String user, String databaseName) { // all users are "owners" of the default database if (DEFAULT_DATABASE_NAME.equalsIgnoreCase(databaseName)) { return true; } Optional<Database> databaseMetadata = getDatabase(databaseName); if (!databaseMetadata.isPresent()) { return false; } Database database = databaseMetadata.get(); // a database can be owned by a user or role if (database.getOwnerType() == USER && user.equals(database.getOwnerName())) { return true; } if (database.getOwnerType() == ROLE && getRoles(user).contains(database.getOwnerName())) { return true; } return false; } private Path getDatabaseMetadataDirectory(String databaseName) { return new Path(catalogDirectory, databaseName); } private Path getTableMetadataDirectory(Table table) { return getTableMetadataDirectory(table.getDatabaseName(), table.getTableName()); } private Path getTableMetadataDirectory(String databaseName, String tableName) { return new Path(getDatabaseMetadataDirectory(databaseName), tableName); } private Path getPartitionMetadataDirectory(Table table, List<String> values) { String partitionName = makePartName(table.getPartitionColumns(), values); return getPartitionMetadataDirectory(table, partitionName); } private Path getPartitionMetadataDirectory(Table table, String partitionName) { Path tableMetadataDirectory = getTableMetadataDirectory(table); return new Path(tableMetadataDirectory, partitionName); } private Path getPermissionsDirectory(Table table) { return new Path(getTableMetadataDirectory(table), PRESTO_PERMISSIONS_DIRECTORY_NAME); } private static Path getPermissionsPath(Path permissionsDirectory, String principalName, PrincipalType principalType) { return new Path(permissionsDirectory, principalType.name().toLowerCase(Locale.US) + "_" + principalName); } private List<Path> getChildSchemaDirectories(Path metadataDirectory) { try { if (!metadataFileSystem.isDirectory(metadataDirectory)) { return ImmutableList.of(); } ImmutableList.Builder<Path> childSchemaDirectories = ImmutableList.builder(); for (FileStatus child : metadataFileSystem.listStatus(metadataDirectory)) { if (!child.isDirectory()) { continue; } Path childPath = child.getPath(); if (childPath.getName().startsWith(".")) { continue; } if (metadataFileSystem.isFile(new Path(childPath, PRESTO_SCHEMA_FILE_NAME))) { childSchemaDirectories.add(childPath); } } return childSchemaDirectories.build(); } catch (IOException e) { throw new PrestoException(HIVE_METASTORE_ERROR, e); } } private void deleteMetadataDirectory(Path metadataDirectory) { try { Path schemaPath = new Path(metadataDirectory, PRESTO_SCHEMA_FILE_NAME); if (!metadataFileSystem.isFile(schemaPath)) { // if there is no schema file, assume this is not a database, partition or table return; } if (!metadataFileSystem.delete(metadataDirectory, true)) { throw new PrestoException(HIVE_METASTORE_ERROR, "Could not delete metadata directory"); } } catch (IOException e) { throw new PrestoException(HIVE_METASTORE_ERROR, e); } } private <T> Optional<T> readSchemaFile(String type, Path metadataDirectory, JsonCodec<T> codec) { Path schemaPath = new Path(metadataDirectory, PRESTO_SCHEMA_FILE_NAME); return readFile(type + " schema", schemaPath, codec); } private <T> Optional<T> readFile(String type, Path path, JsonCodec<T> codec) { try { if (!metadataFileSystem.isFile(path)) { return Optional.empty(); } try (FSDataInputStream inputStream = metadataFileSystem.open(path)) { byte[] json = ByteStreams.toByteArray(inputStream); return Optional.of(codec.fromJson(json)); } } catch (Exception e) { throw new PrestoException(HIVE_METASTORE_ERROR, "Could not read " + type, e); } } private <T> void writeSchemaFile(String type, Path directory, JsonCodec<T> codec, T value, boolean overwrite) { Path schemaPath = new Path(directory, PRESTO_SCHEMA_FILE_NAME); writeFile(type + " schema", schemaPath, codec, value, overwrite); } private <T> void writeFile(String type, Path path, JsonCodec<T> codec, T value, boolean overwrite) { try { byte[] json = codec.toJsonBytes(value); if (!overwrite) { if (metadataFileSystem.exists(path)) { throw new PrestoException(HIVE_METASTORE_ERROR, type + " file already exists"); } } metadataFileSystem.mkdirs(path.getParent()); // todo implement safer overwrite code try (OutputStream outputStream = metadataFileSystem.create(path, overwrite)) { outputStream.write(json); } } catch (Exception e) { throw new PrestoException(HIVE_METASTORE_ERROR, "Could not write " + type, e); } } private void deleteSchemaFile(String type, Path metadataDirectory) { try { if (!metadataFileSystem.delete(new Path(metadataDirectory, PRESTO_SCHEMA_FILE_NAME), false)) { throw new PrestoException(HIVE_METASTORE_ERROR, "Could not delete " + type + " schema"); } } catch (IOException e) { throw new PrestoException(HIVE_METASTORE_ERROR, "Could not delete " + type + " schema", e); } } private static boolean isChildDirectory(Path parentDirectory, Path childDirectory) { if (parentDirectory.equals(childDirectory)) { return true; } if (childDirectory.isRoot()) { return false; } return isChildDirectory(parentDirectory, childDirectory.getParent()); } }