Example usage for org.apache.hadoop.fs Path toString

Introduction

In this page you can find the example usage for org.apache.hadoop.fs Path toString.

Prototype

@Override
    public String toString() 

Source Link

Usage

From source file:com.facebook.presto.hive.parquet.TestMapredParquetOutputFormat.java

License:Apache License

@Override
public FileSinkOperator.RecordWriter getHiveRecordWriter(JobConf jobConf, Path finalOutPath,
        Class<? extends Writable> valueClass, boolean isCompressed, Properties tableProperties,
        Progressable progress) throws IOException {
    if (schema.isPresent()) {
        DataWritableWriteSupport.setSchema(schema.get(), jobConf);
        return getParquerRecordWriterWrapper(realOutputFormat, jobConf, finalOutPath.toString(), progress,
                tableProperties);//www  . j a  v  a  2  s .c o  m
    }
    return super.getHiveRecordWriter(jobConf, finalOutPath, valueClass, isCompressed, tableProperties,
            progress);
}

---

From source file:com.facebook.presto.hive.rcfile.RcFilePageSourceFactory.java

License:Apache License

@Override
public Optional<? extends ConnectorPageSource> createPageSource(Configuration configuration,
        ConnectorSession session, Path path, long start, long length, Properties schema,
        List<HiveColumnHandle> columns, TupleDomain<HiveColumnHandle> effectivePredicate,
        DateTimeZone hiveStorageTimeZone) {
    if (!isRcfileOptimizedReaderEnabled(session)) {
        return Optional.empty();
    }//from www  .ja  v a  2  s. c  o m

    RcFileEncoding rcFileEncoding;
    String deserializerClassName = getDeserializerClassName(schema);
    if (deserializerClassName.equals(LazyBinaryColumnarSerDe.class.getName())) {
        rcFileEncoding = new BinaryRcFileEncoding();
    } else if (deserializerClassName.equals(ColumnarSerDe.class.getName())) {
        rcFileEncoding = createTextVectorEncoding(schema, hiveStorageTimeZone);
    } else {
        return Optional.empty();
    }

    long size;
    FSDataInputStream inputStream;
    try {
        FileSystem fileSystem = hdfsEnvironment.getFileSystem(session.getUser(), path, configuration);
        size = fileSystem.getFileStatus(path).getLen();
        inputStream = fileSystem.open(path);
    } catch (Exception e) {
        throw Throwables.propagate(e);
    }

    try {
        ImmutableMap.Builder<Integer, Type> readColumns = ImmutableMap.builder();
        for (HiveColumnHandle column : columns) {
            readColumns.put(column.getHiveColumnIndex(), column.getHiveType().getType(typeManager));
        }

        RcFileReader rcFileReader = new RcFileReader(
                new HdfsRcFileDataSource(path.toString(), inputStream, size), rcFileEncoding,
                readColumns.build(),
                new AircompressorCodecFactory(new HadoopCodecFactory(configuration.getClassLoader())), start,
                length, new DataSize(1, Unit.MEGABYTE));

        return Optional.of(new RcFilePageSource(rcFileReader, columns, hiveStorageTimeZone, typeManager));
    } catch (Throwable e) {
        try {
            inputStream.close();
        } catch (IOException ignored) {
        }
        throw Throwables.propagate(e);
    }
}

---

From source file:com.facebook.presto.hive.RcFileFileWriterFactory.java

License:Apache License

@Override
public Optional<HiveFileWriter> createFileWriter(Path path, List<String> inputColumnNames,
        StorageFormat storageFormat, Properties schema, JobConf configuration, ConnectorSession session) {
    if (!HiveSessionProperties.isRcfileOptimizedWriterEnabled(session)) {
        return Optional.empty();
    }/*w w  w .  j a v a  2  s  .  c o  m*/

    if (!RCFileOutputFormat.class.getName().equals(storageFormat.getOutputFormat())) {
        return Optional.empty();
    }

    RcFileEncoding rcFileEncoding;
    if (LazyBinaryColumnarSerDe.class.getName().equals(storageFormat.getSerDe())) {
        rcFileEncoding = new BinaryRcFileEncoding();
    } else if (ColumnarSerDe.class.getName().equals(storageFormat.getSerDe())) {
        rcFileEncoding = createTextVectorEncoding(schema, hiveStorageTimeZone);
    } else {
        return Optional.empty();
    }

    Optional<String> codecName = Optional.ofNullable(configuration.get(FileOutputFormat.COMPRESS_CODEC));

    // existing tables and partitions may have columns in a different order than the writer is providing, so build
    // an index to rearrange columns in the proper order
    List<String> fileColumnNames = Splitter.on(',').trimResults().omitEmptyStrings()
            .splitToList(schema.getProperty(META_TABLE_COLUMNS, ""));
    List<Type> fileColumnTypes = toHiveTypes(schema.getProperty(META_TABLE_COLUMN_TYPES, "")).stream()
            .map(hiveType -> hiveType.getType(typeManager)).collect(toList());

    int[] fileInputColumnIndexes = fileColumnNames.stream().mapToInt(inputColumnNames::indexOf).toArray();

    try {
        FileSystem fileSystem = hdfsEnvironment.getFileSystem(session.getUser(), path, configuration);
        OutputStream outputStream = fileSystem.create(path);

        Optional<Supplier<RcFileDataSource>> validationInputFactory = Optional.empty();
        if (HiveSessionProperties.isRcfileOptimizedWriterValidate(session)) {
            validationInputFactory = Optional.of(() -> {
                try {
                    return new HdfsRcFileDataSource(path.toString(), fileSystem.open(path),
                            fileSystem.getFileStatus(path).getLen(), stats);
                } catch (IOException e) {
                    throw new PrestoException(HIVE_WRITE_VALIDATION_FAILED, e);
                }
            });
        }

        Callable<Void> rollbackAction = () -> {
            fileSystem.delete(path, false);
            return null;
        };

        return Optional.of(new RcFileFileWriter(outputStream, rollbackAction, rcFileEncoding, fileColumnTypes,
                codecName, fileInputColumnIndexes,
                ImmutableMap.<String, String>builder()
                        .put(HiveMetadata.PRESTO_VERSION_NAME, nodeVersion.toString())
                        .put(HiveMetadata.PRESTO_QUERY_ID_NAME, session.getQueryId()).build(),
                validationInputFactory));
    } catch (Exception e) {
        throw new PrestoException(HIVE_WRITER_OPEN_ERROR, "Error creating RCFile file", e);
    }
}

---

From source file:com.facebook.presto.hive.s3.TestPrestoS3FileSystem.java

License:Apache License

@Test
public void testCreateWithNonexistentStagingDirectory() throws Exception {
    java.nio.file.Path stagingParent = createTempDirectory("test");
    java.nio.file.Path staging = Paths.get(stagingParent.toString(), "staging");
    // stagingParent = /tmp/testXXX
    // staging = /tmp/testXXX/staging

    try (PrestoS3FileSystem fs = new PrestoS3FileSystem()) {
        MockAmazonS3 s3 = new MockAmazonS3();
        Configuration conf = new Configuration();
        conf.set(PrestoS3FileSystem.S3_STAGING_DIRECTORY, staging.toString());
        fs.initialize(new URI("s3n://test-bucket/"), conf);
        fs.setS3Client(s3);/*from  ww w . j  av a 2 s  .c o m*/
        FSDataOutputStream stream = fs.create(new Path("s3n://test-bucket/test"));
        stream.close();
        assertTrue(Files.exists(staging));
    } finally {
        deleteRecursively(stagingParent, ALLOW_INSECURE);
    }
}

---

From source file:com.facebook.presto.hive.s3.TestPrestoS3FileSystem.java

License:Apache License

@Test(expectedExceptions = IOException.class, expectedExceptionsMessageRegExp = "Configured staging path is not a directory: .*")
public void testCreateWithStagingDirectoryFile() throws Exception {
    java.nio.file.Path staging = createTempFile("staging", null);
    // staging = /tmp/stagingXXX.tmp

    try (PrestoS3FileSystem fs = new PrestoS3FileSystem()) {
        MockAmazonS3 s3 = new MockAmazonS3();
        Configuration conf = new Configuration();
        conf.set(PrestoS3FileSystem.S3_STAGING_DIRECTORY, staging.toString());
        fs.initialize(new URI("s3n://test-bucket/"), conf);
        fs.setS3Client(s3);/*from w w w  .  j a  va 2 s. co m*/
        fs.create(new Path("s3n://test-bucket/test"));
    } finally {
        Files.deleteIfExists(staging);
    }
}

---

From source file:com.facebook.presto.hive.s3.TestPrestoS3FileSystem.java

License:Apache License

@Test
public void testCreateWithStagingDirectorySymlink() throws Exception {
    java.nio.file.Path staging = createTempDirectory("staging");
    java.nio.file.Path link = Paths.get(staging + ".symlink");
    // staging = /tmp/stagingXXX
    // link = /tmp/stagingXXX.symlink -> /tmp/stagingXXX

    try {//from   www .ja  va  2 s  .  c  o m
        try {
            Files.createSymbolicLink(link, staging);
        } catch (UnsupportedOperationException e) {
            throw new SkipException("Filesystem does not support symlinks", e);
        }

        try (PrestoS3FileSystem fs = new PrestoS3FileSystem()) {
            MockAmazonS3 s3 = new MockAmazonS3();
            Configuration conf = new Configuration();
            conf.set(PrestoS3FileSystem.S3_STAGING_DIRECTORY, link.toString());
            fs.initialize(new URI("s3n://test-bucket/"), conf);
            fs.setS3Client(s3);
            FSDataOutputStream stream = fs.create(new Path("s3n://test-bucket/test"));
            stream.close();
            assertTrue(Files.exists(link));
        }
    } finally {
        deleteRecursively(link, ALLOW_INSECURE);
        deleteRecursively(staging, ALLOW_INSECURE);
    }
}

---

From source file:com.facebook.presto.hive.SortingFileWriter.java

License:Apache License

private void mergeFiles(Iterable<TempFile> files, Consumer<Page> consumer) {
    try (Closer closer = Closer.create()) {
        Collection<Iterator<Page>> iterators = new ArrayList<>();

        for (TempFile tempFile : files) {
            Path file = tempFile.getPath();
            OrcDataSource dataSource = new HdfsOrcDataSource(new OrcDataSourceId(file.toString()),
                    fileSystem.getFileStatus(file).getLen(), new DataSize(1, MEGABYTE),
                    new DataSize(8, MEGABYTE), new DataSize(8, MEGABYTE), false, fileSystem.open(file),
                    new FileFormatDataSourceStats());
            closer.register(dataSource);
            iterators.add(new TempFileReader(types, dataSource));
        }//from   ww w.  j  a v  a 2 s  . com

        new MergingPageIterator(iterators, types, sortFields, sortOrders).forEachRemaining(consumer);

        for (TempFile tempFile : files) {
            Path file = tempFile.getPath();
            fileSystem.delete(file, false);
            if (fileSystem.exists(file)) {
                throw new IOException("Failed to delete temporary file: " + file);
            }
        }
    } catch (IOException e) {
        throw new UncheckedIOException(e);
    }
}

---

From source file:com.facebook.presto.hive.TestPrestoS3FileSystem.java

License:Apache License

@Test
public void testCreateWithNonexistentStagingDirectory() throws Exception {
    java.nio.file.Path tmpdir = Paths.get(StandardSystemProperty.JAVA_IO_TMPDIR.value());
    java.nio.file.Path stagingParent = Files.createTempDirectory(tmpdir, "test");
    java.nio.file.Path staging = Paths.get(stagingParent.toString(), "staging");
    // stagingParent = /tmp/testXXX
    // staging = /tmp/testXXX/staging

    try (PrestoS3FileSystem fs = new PrestoS3FileSystem()) {
        MockAmazonS3 s3 = new MockAmazonS3();
        Configuration conf = new Configuration();
        conf.set(PrestoS3FileSystem.S3_STAGING_DIRECTORY, staging.toString());
        fs.initialize(new URI("s3n://test-bucket/"), conf);
        fs.setS3Client(s3);//from w ww  .  jav  a2  s .  c  o  m
        FSDataOutputStream stream = fs.create(new Path("s3n://test-bucket/test"));
        stream.close();
        assertTrue(Files.exists(staging));
    } finally {
        deleteRecursively(stagingParent.toFile());
    }
}

---

From source file:com.facebook.presto.hive.TestPrestoS3FileSystem.java

License:Apache License

@Test(expectedExceptions = IOException.class, expectedExceptionsMessageRegExp = "Configured staging path is not a directory: .*")
public void testCreateWithStagingDirectoryFile() throws Exception {
    java.nio.file.Path tmpdir = Paths.get(StandardSystemProperty.JAVA_IO_TMPDIR.value());
    java.nio.file.Path staging = Files.createTempFile(tmpdir, "staging", null);
    // staging = /tmp/stagingXXX.tmp

    try (PrestoS3FileSystem fs = new PrestoS3FileSystem()) {
        MockAmazonS3 s3 = new MockAmazonS3();
        Configuration conf = new Configuration();
        conf.set(PrestoS3FileSystem.S3_STAGING_DIRECTORY, staging.toString());
        fs.initialize(new URI("s3n://test-bucket/"), conf);
        fs.setS3Client(s3);/*from   w w  w. j ava2  s .  c  o  m*/
        fs.create(new Path("s3n://test-bucket/test"));
    } finally {
        Files.deleteIfExists(staging);
    }
}

---

From source file:com.facebook.presto.hive.TestPrestoS3FileSystem.java

License:Apache License

@Test
public void testCreateWithStagingDirectorySymlink() throws Exception {
    java.nio.file.Path tmpdir = Paths.get(StandardSystemProperty.JAVA_IO_TMPDIR.value());
    java.nio.file.Path staging = Files.createTempDirectory(tmpdir, "staging");
    java.nio.file.Path link = Paths.get(staging + ".symlink");
    // staging = /tmp/stagingXXX
    // link = /tmp/stagingXXX.symlink -> /tmp/stagingXXX

    try {//from  w w w.  java 2s  .c o  m
        try {
            Files.createSymbolicLink(link, staging);
        } catch (UnsupportedOperationException e) {
            throw new SkipException("Filesystem does not support symlinks", e);
        }

        try (PrestoS3FileSystem fs = new PrestoS3FileSystem()) {
            MockAmazonS3 s3 = new MockAmazonS3();
            Configuration conf = new Configuration();
            conf.set(PrestoS3FileSystem.S3_STAGING_DIRECTORY, link.toString());
            fs.initialize(new URI("s3n://test-bucket/"), conf);
            fs.setS3Client(s3);
            FSDataOutputStream stream = fs.create(new Path("s3n://test-bucket/test"));
            stream.close();
            assertTrue(Files.exists(link));
        }
    } finally {
        deleteRecursively(link.toFile());
        deleteRecursively(staging.toFile());
    }
}

---

• «
• 1
• 2
• 3
• 4
• 5
• 6
• 7
• 8
• 9
• 10
• 11
• 12
• 13
• 14
• 15
• 16
• 17
• 18
• 19
• 20
• 21
• 22
• 23
• 24
• 25
• 26
• 27
• 28
• 29
• 30
• 31
• 32
• 33
• 34
• 35
• 36
• 37
• 38
• 39
• 40
• 41
• 42
• 43
• 44
• 45
• 46
• 47
• 48
• 49
• 50
• »