List of usage examples for org.apache.hadoop.fs Path equals
@Override
public boolean equals(Object o)
From source file:com.cloudera.sqoop.lib.LobRef.java
License:Apache License
/** * Get access to the LOB data itself.//from ww w .ja v a 2 s . c om * This method returns a lazy reader of the LOB data, accessing the * filesystem for external LOB storage as necessary. * @param conf the Configuration used to access the filesystem * @param basePath the base directory where the table records are * stored. * @return an object that lazily streams the record to the client. * @throws IOException if it could not read the LOB from external storage. */ public ACCESSORTYPE getDataStream(Configuration conf, Path basePath) throws IOException { if (isExternal()) { // Read from external storage. Path pathToRead = LobReaderCache.qualify(new Path(basePath, fileName), conf); LOG.debug("Retreving data stream from external path: " + pathToRead); if (lobReader != null) { // We already have a reader open to a LobFile. Is it the correct file? if (!pathToRead.equals(lobReader.getPath())) { // No. Close this.lobReader and get the correct one. LOG.debug("Releasing previous external reader for " + lobReader.getPath()); LobReaderCache.getCache().recycle(lobReader); lobReader = LobReaderCache.getCache().get(pathToRead, conf); } } else { lobReader = LobReaderCache.getCache().get(pathToRead, conf); } // We now have a LobFile.Reader associated with the correct file. Get to // the correct offset and return an InputStream/Reader to the user. if (lobReader.tell() != offset) { LOG.debug("Seeking to record start offset " + offset); lobReader.seek(offset); } if (!lobReader.next()) { throw new IOException("Could not locate record at " + pathToRead + ":" + offset); } return getExternalSource(lobReader); } else { // This data is already materialized in memory; wrap it and return. return getInternalSource(realData); } }
From source file:com.datatorrent.lib.io.fs.AbstractHdfsRollingFileOutputOperator.java
License:Open Source License
/** * This checks if the new path is not same as old path. If it is then throw exception *//*from w ww .ja va 2 s . c om*/ private void validateNextFilePath() { Path filepath = nextFilePath(); if (currentFilePath != null && filepath.equals(currentFilePath)) { throw new IllegalArgumentException( "Rolling files require %() placeholders for unique names: " + filepath); } currentFilePath = filepath; }
From source file:com.datatorrent.lib.io.HdfsOutputOperator.java
License:Open Source License
/** * * @param context//from w ww. ja v a 2 s.co m */ @Override public void setup(OperatorContext context) { this.contextId = context.getId(); try { Path filepath = subFilePath(this.fileIndex); fs = FileSystem.get(filepath.toUri(), new Configuration()); if (bytesPerFile > 0) { // ensure file path generates unique names Path p1 = subFilePath(1); Path p2 = subFilePath(2); if (p1.equals(p2)) { throw new IllegalArgumentException( "Rolling files require %() placeholders for unique names: " + filepath); } } openFile(filepath); } catch (IOException ex) { throw new RuntimeException(ex); } }
From source file:com.facebook.presto.hive.AbstractTestHiveClient.java
License:Apache License
/** * @return query id// w w w. ja v a 2s . co m */ private String insertData(SchemaTableName tableName, MaterializedResult data) throws Exception { Path writePath; Path targetPath; String queryId; try (Transaction transaction = newTransaction()) { ConnectorMetadata metadata = transaction.getMetadata(); ConnectorSession session = newSession(); ConnectorTableHandle tableHandle = getTableHandle(metadata, tableName); ConnectorInsertTableHandle insertTableHandle = metadata.beginInsert(session, tableHandle); queryId = session.getQueryId(); writePath = getStagingPathRoot(insertTableHandle); targetPath = getTargetPathRoot(insertTableHandle); ConnectorPageSink sink = pageSinkProvider.createPageSink(transaction.getTransactionHandle(), session, insertTableHandle); // write data sink.appendPage(data.toPage()); Collection<Slice> fragments = getFutureValue(sink.finish()); // commit the insert metadata.finishInsert(session, insertTableHandle, fragments); transaction.commit(); } // check that temporary files are removed if (!writePath.equals(targetPath)) { FileSystem fileSystem = hdfsEnvironment.getFileSystem("user", writePath); assertFalse(fileSystem.exists(writePath)); } return queryId; }
From source file:com.facebook.presto.hive.AbstractTestHiveClient.java
License:Apache License
private void doTestTransactionDeleteInsert(HiveStorageFormat storageFormat, SchemaTableName tableName, Domain domainToDrop, MaterializedResult insertData, MaterializedResult expectedData, TransactionDeleteInsertTestTag tag, boolean expectQuerySucceed, Optional<ConflictTrigger> conflictTrigger) throws Exception { Path writePath = null; Path targetPath = null;/*from ww w .j a v a 2 s .c o m*/ try (Transaction transaction = newTransaction()) { try { ConnectorMetadata metadata = transaction.getMetadata(); ConnectorTableHandle tableHandle = getTableHandle(metadata, tableName); ConnectorSession session; rollbackIfEquals(tag, ROLLBACK_RIGHT_AWAY); // Query 1: delete session = newSession(); HiveColumnHandle dsColumnHandle = (HiveColumnHandle) metadata.getColumnHandles(session, tableHandle) .get("pk2"); TupleDomain<ColumnHandle> tupleDomain = TupleDomain .withColumnDomains(ImmutableMap.of(dsColumnHandle, domainToDrop)); Constraint<ColumnHandle> constraint = new Constraint<>(tupleDomain, convertToPredicate(tupleDomain)); List<ConnectorTableLayoutResult> tableLayoutResults = metadata.getTableLayouts(session, tableHandle, constraint, Optional.empty()); ConnectorTableLayoutHandle tableLayoutHandle = Iterables.getOnlyElement(tableLayoutResults) .getTableLayout().getHandle(); metadata.metadataDelete(session, tableHandle, tableLayoutHandle); rollbackIfEquals(tag, ROLLBACK_AFTER_DELETE); // Query 2: insert session = newSession(); ConnectorInsertTableHandle insertTableHandle = metadata.beginInsert(session, tableHandle); rollbackIfEquals(tag, ROLLBACK_AFTER_BEGIN_INSERT); writePath = getStagingPathRoot(insertTableHandle); targetPath = getTargetPathRoot(insertTableHandle); ConnectorPageSink sink = pageSinkProvider.createPageSink(transaction.getTransactionHandle(), session, insertTableHandle); sink.appendPage(insertData.toPage()); rollbackIfEquals(tag, ROLLBACK_AFTER_APPEND_PAGE); Collection<Slice> fragments = getFutureValue(sink.finish()); rollbackIfEquals(tag, ROLLBACK_AFTER_SINK_FINISH); metadata.finishInsert(session, insertTableHandle, fragments); rollbackIfEquals(tag, ROLLBACK_AFTER_FINISH_INSERT); assertEquals(tag, COMMIT); if (conflictTrigger.isPresent()) { JsonCodec<PartitionUpdate> partitionUpdateCodec = JsonCodec.jsonCodec(PartitionUpdate.class); List<PartitionUpdate> partitionUpdates = fragments.stream().map(Slice::getBytes) .map(partitionUpdateCodec::fromJson).collect(toList()); conflictTrigger.get().triggerConflict(session, tableName, insertTableHandle, partitionUpdates); } transaction.commit(); if (conflictTrigger.isPresent()) { assertTrue(expectQuerySucceed); conflictTrigger.get().verifyAndCleanup(tableName); } } catch (TestingRollbackException e) { transaction.rollback(); } catch (PrestoException e) { assertFalse(expectQuerySucceed); if (conflictTrigger.isPresent()) { conflictTrigger.get().verifyAndCleanup(tableName); } } } // check that temporary files are removed if (writePath != null && !writePath.equals(targetPath)) { FileSystem fileSystem = hdfsEnvironment.getFileSystem("user", writePath); assertFalse(fileSystem.exists(writePath)); } try (Transaction transaction = newTransaction()) { // verify partitions List<String> partitionNames = transaction.getMetastore(tableName.getSchemaName()) .getPartitionNames(tableName.getSchemaName(), tableName.getTableName()).orElseThrow( () -> new PrestoException(HIVE_METASTORE_ERROR, "Partition metadata not available")); assertEqualsIgnoreOrder(partitionNames, expectedData.getMaterializedRows().stream() .map(row -> format("pk1=%s/pk2=%s", row.getField(1), row.getField(2))).distinct() .collect(toList())); // load the new table ConnectorSession session = newSession(); ConnectorMetadata metadata = transaction.getMetadata(); ConnectorTableHandle tableHandle = getTableHandle(metadata, tableName); List<ColumnHandle> columnHandles = filterNonHiddenColumnHandles( metadata.getColumnHandles(session, tableHandle).values()); // verify the data MaterializedResult result = readTable(transaction, tableHandle, columnHandles, session, TupleDomain.all(), OptionalInt.empty(), Optional.of(storageFormat)); assertEqualsIgnoreOrder(result.getMaterializedRows(), expectedData.getMaterializedRows()); } }
From source file:com.facebook.presto.hive.HiveMetadata.java
License:Apache License
@Override public HiveOutputTableHandle beginCreateTable(ConnectorSession session, ConnectorTableMetadata tableMetadata, Optional<ConnectorNewTableLayout> layout) { verifyJvmTimeZone();/* w ww . j a v a2s .c om*/ if (getExternalLocation(tableMetadata.getProperties()) != null) { throw new PrestoException(NOT_SUPPORTED, "External tables cannot be created using CREATE TABLE AS"); } HiveStorageFormat tableStorageFormat = getHiveStorageFormat(tableMetadata.getProperties()); List<String> partitionedBy = getPartitionedBy(tableMetadata.getProperties()); Optional<HiveBucketProperty> bucketProperty = getBucketProperty(tableMetadata.getProperties()); Map<String, String> additionalTableParameters = tableParameterCodec.encode(tableMetadata.getProperties()); // get the root directory for the database SchemaTableName schemaTableName = tableMetadata.getTable(); String schemaName = schemaTableName.getSchemaName(); String tableName = schemaTableName.getTableName(); List<HiveColumnHandle> columnHandles = getColumnHandles(connectorId, tableMetadata, ImmutableSet.copyOf(partitionedBy), typeTranslator); HiveStorageFormat partitionStorageFormat = respectTableFormat ? tableStorageFormat : defaultStorageFormat; // unpartitioned tables ignore the partition storage format HiveStorageFormat actualStorageFormat = partitionedBy.isEmpty() ? tableStorageFormat : partitionStorageFormat; actualStorageFormat.validateColumns(columnHandles); LocationHandle locationHandle = locationService.forNewTable(metastore, session.getUser(), session.getQueryId(), schemaName, tableName); HiveOutputTableHandle result = new HiveOutputTableHandle(connectorId, schemaName, tableName, columnHandles, session.getQueryId(), metastore.generatePageSinkMetadata(schemaTableName), locationHandle, tableStorageFormat, partitionStorageFormat, partitionedBy, bucketProperty, session.getUser(), additionalTableParameters); Path writePathRoot = locationService.writePathRoot(locationHandle).get(); Path targetPathRoot = locationService.targetPathRoot(locationHandle); WriteMode mode = writePathRoot.equals(targetPathRoot) ? DIRECT_TO_TARGET_NEW_DIRECTORY : STAGE_AND_MOVE_TO_TARGET_DIRECTORY; metastore.declareIntentionToWrite(session, mode, writePathRoot, result.getFilePrefix(), schemaTableName); return result; }
From source file:com.facebook.presto.hive.HiveWriterFactory.java
License:Apache License
public HiveWriter createWriter(Page partitionColumns, int position, OptionalInt bucketNumber) { if (bucketCount.isPresent()) { checkArgument(bucketNumber.isPresent(), "Bucket not provided for bucketed table"); checkArgument(bucketNumber.getAsInt() < bucketCount.getAsInt(), "Bucket number %s must be less than bucket count %s", bucketNumber, bucketCount); } else {/* ww w . j a v a2 s .co m*/ checkArgument(!bucketNumber.isPresent(), "Bucket number provided by for table that is not bucketed"); } String fileName; if (bucketNumber.isPresent()) { fileName = computeBucketedFileName(filePrefix, bucketNumber.getAsInt()); } else { fileName = filePrefix + "_" + randomUUID(); } List<String> partitionValues = toPartitionValues(partitionColumns, position); Optional<String> partitionName; if (!partitionColumnNames.isEmpty()) { partitionName = Optional.of(FileUtils.makePartName(partitionColumnNames, partitionValues)); } else { partitionName = Optional.empty(); } // attempt to get the existing partition (if this is an existing partitioned table) Optional<Partition> partition = Optional.empty(); if (!partitionValues.isEmpty() && table != null) { partition = pageSinkMetadataProvider.getPartition(partitionValues); } boolean isNew; Properties schema; Path target; Path write; StorageFormat outputStorageFormat; if (!partition.isPresent()) { if (table == null) { // Write to: a new partition in a new partitioned table, // or a new unpartitioned table. isNew = true; schema = new Properties(); schema.setProperty(META_TABLE_COLUMNS, dataColumns.stream().map(DataColumn::getName).collect(joining(","))); schema.setProperty(META_TABLE_COLUMN_TYPES, dataColumns.stream().map(DataColumn::getHiveType) .map(HiveType::getHiveTypeName).collect(joining(":"))); target = locationService.targetPath(locationHandle, partitionName); write = locationService.writePath(locationHandle, partitionName).get(); if (partitionName.isPresent() && !target.equals(write)) { // When target path is different from write path, // verify that the target directory for the partition does not already exist if (HiveWriteUtils.pathExists(session.getUser(), hdfsEnvironment, target)) { throw new PrestoException(HIVE_PATH_ALREADY_EXISTS, format( "Target directory for new partition '%s' of table '%s.%s' already exists: %s", partitionName, schemaName, tableName, target)); } } } else { // Write to: a new partition in an existing partitioned table, // or an existing unpartitioned table if (partitionName.isPresent()) { isNew = true; } else { if (bucketNumber.isPresent()) { throw new PrestoException(HIVE_PARTITION_READ_ONLY, "Can not insert into bucketed unpartitioned Hive table"); } if (immutablePartitions) { throw new PrestoException(HIVE_PARTITION_READ_ONLY, "Unpartitioned Hive tables are immutable"); } isNew = false; } schema = getHiveSchema(table); target = locationService.targetPath(locationHandle, partitionName); write = locationService.writePath(locationHandle, partitionName).orElse(target); } if (partitionName.isPresent()) { // Write to a new partition outputStorageFormat = fromHiveStorageFormat(partitionStorageFormat); } else { // Write to a new/existing unpartitioned table outputStorageFormat = fromHiveStorageFormat(tableStorageFormat); } } else { // Write to: an existing partition in an existing partitioned table, if (bucketNumber.isPresent()) { throw new PrestoException(HIVE_PARTITION_READ_ONLY, "Can not insert into existing partitions of bucketed Hive table"); } if (immutablePartitions) { throw new PrestoException(HIVE_PARTITION_READ_ONLY, "Hive partitions are immutable"); } isNew = false; // Check the column types in partition schema match the column types in table schema List<Column> tableColumns = table.getDataColumns(); List<Column> existingPartitionColumns = partition.get().getColumns(); for (int i = 0; i < min(existingPartitionColumns.size(), tableColumns.size()); i++) { HiveType tableType = tableColumns.get(i).getType(); HiveType partitionType = existingPartitionColumns.get(i).getType(); if (!tableType.equals(partitionType)) { throw new PrestoException(HIVE_PARTITION_SCHEMA_MISMATCH, format("" + "There is a mismatch between the table and partition schemas. " + "The column '%s' in table '%s' is declared as type '%s', " + "but partition '%s' declared column '%s' as type '%s'.", tableColumns.get(i).getName(), tableName, tableType, partitionName, existingPartitionColumns.get(i).getName(), partitionType)); } } // Append to an existing partition HiveWriteUtils.checkPartitionIsWritable(partitionName.get(), partition.get()); outputStorageFormat = partition.get().getStorage().getStorageFormat(); schema = getHiveSchema(partition.get(), table); target = locationService.targetPath(locationHandle, partition.get(), partitionName.get()); write = locationService.writePath(locationHandle, partitionName).orElse(target); } validateSchema(partitionName, schema); String fileNameWithExtension = fileName + getFileExtension(conf, outputStorageFormat); HiveRecordWriter hiveRecordWriter = new HiveRecordWriter(new Path(write, fileNameWithExtension), dataColumns.stream().map(DataColumn::getName).collect(toList()), outputStorageFormat, schema, typeManager, conf); return new HiveWriter(hiveRecordWriter, partitionName, isNew, fileNameWithExtension, write.toString(), target.toString()); }
From source file:com.facebook.presto.hive.metastore.file.FileHiveMetastore.java
License:Apache License
@Override public synchronized void createTable(Table table, PrincipalPrivileges principalPrivileges) { verifyTableNotExists(table.getDatabaseName(), table.getTableName()); Path tableMetadataDirectory = getTableMetadataDirectory(table); // validate table location if (table.getTableType().equals(VIRTUAL_VIEW.name())) { checkArgument(table.getStorage().getLocation().isEmpty(), "Storage location for view must be empty"); } else if (table.getTableType().equals(MANAGED_TABLE.name())) { if (!tableMetadataDirectory.equals(new Path(table.getStorage().getLocation()))) { throw new PrestoException(HIVE_METASTORE_ERROR, "Table directory must be " + tableMetadataDirectory); }//from ww w.j a va2s . c om } else if (table.getTableType().equals(EXTERNAL_TABLE.name())) { try { Path externalLocation = new Path(table.getStorage().getLocation()); FileSystem externalFileSystem = hdfsEnvironment.getFileSystem(hdfsContext, externalLocation); if (!externalFileSystem.isDirectory(externalLocation)) { throw new PrestoException(HIVE_METASTORE_ERROR, "External table location does not exist"); } if (isChildDirectory(catalogDirectory, externalLocation)) { throw new PrestoException(HIVE_METASTORE_ERROR, "External table location can not be inside the system metadata directory"); } } catch (IOException e) { throw new PrestoException(HIVE_METASTORE_ERROR, "Could not validate external location", e); } } else { throw new PrestoException(NOT_SUPPORTED, "Table type not supported: " + table.getTableType()); } writeSchemaFile("table", tableMetadataDirectory, tableCodec, new TableMetadata(table), false); for (Entry<String, Collection<HivePrivilegeInfo>> entry : principalPrivileges.getUserPrivileges().asMap() .entrySet()) { setTablePrivileges(entry.getKey(), USER, table.getDatabaseName(), table.getTableName(), entry.getValue()); } for (Entry<String, Collection<HivePrivilegeInfo>> entry : principalPrivileges.getRolePrivileges().asMap() .entrySet()) { setTablePrivileges(entry.getKey(), ROLE, table.getDatabaseName(), table.getTableName(), entry.getValue()); } }
From source file:com.facebook.presto.hive.metastore.file.FileHiveMetastore.java
License:Apache License
private void verifiedPartition(Table table, Partition partition) { Path partitionMetadataDirectory = getPartitionMetadataDirectory(table, partition.getValues()); if (table.getTableType().equals(MANAGED_TABLE.name())) { if (!partitionMetadataDirectory.equals(new Path(partition.getStorage().getLocation()))) { throw new PrestoException(HIVE_METASTORE_ERROR, "Partition directory must be " + partitionMetadataDirectory); }/* w ww .java2 s . c o m*/ } else if (table.getTableType().equals(EXTERNAL_TABLE.name())) { try { Path externalLocation = new Path(partition.getStorage().getLocation()); FileSystem externalFileSystem = hdfsEnvironment.getFileSystem(hdfsContext, externalLocation); if (!externalFileSystem.isDirectory(externalLocation)) { throw new PrestoException(HIVE_METASTORE_ERROR, "External partition location does not exist"); } if (isChildDirectory(catalogDirectory, externalLocation)) { throw new PrestoException(HIVE_METASTORE_ERROR, "External partition location can not be inside the system metadata directory"); } } catch (IOException e) { throw new PrestoException(HIVE_METASTORE_ERROR, "Could not validate external partition location", e); } } else { throw new PrestoException(NOT_SUPPORTED, "Partitions can not be added to " + table.getTableType()); } }
From source file:com.facebook.presto.hive.metastore.file.FileHiveMetastore.java
License:Apache License
private static boolean isChildDirectory(Path parentDirectory, Path childDirectory) { if (parentDirectory.equals(childDirectory)) { return true; }//from w ww .j a v a 2s.c o m if (childDirectory.isRoot()) { return false; } return isChildDirectory(parentDirectory, childDirectory.getParent()); }