Example usage for org.apache.hadoop.fs Path getFileSystem

Introduction

In this page you can find the example usage for org.apache.hadoop.fs Path getFileSystem.

Prototype

public FileSystem getFileSystem(Configuration conf) throws IOException 

Source Link

Document

Return the FileSystem that owns this Path.

Usage

From source file:com.conductor.hadoop.WritableValueInputFormat.java

License:Apache License

@VisibleForTesting
static <V extends Writable> void doSetupInput(final List<V> values, final Class<V> clazz,
        final int inputsPerSplit, final Job job, final Path inputPath, final Writer writer) throws IOException {
    job.getConfiguration().setClass(VALUE_TYPE_CONF, clazz, Writable.class);
    job.getConfiguration().setInt(INPUTS_PER_SPLIT_CONF, inputsPerSplit);
    job.getConfiguration().set(INPUT_FILE_LOCATION_CONF, inputPath.toString());

    // write each value to the sequence file
    int syncCounter = 0;
    for (final V input : values) {
        // each entry in the sequence file is a map input
        writer.append(NullWritable.get(), input);
        // syncing indicates an input split boundary
        if (++syncCounter % inputsPerSplit == 0) {
            writer.sync();//from  w w  w.  j a v a  2  s.c om
        }
    }
    // close the input file
    writer.hflush();
    writer.close();

    // delete file when JVM exits
    inputPath.getFileSystem(job.getConfiguration()).deleteOnExit(inputPath);
}

---

From source file:com.congiu.load.csv.FSUtils.java

License:Open Source License

public static boolean fileExists(Configuration conf, String file) throws IOException {
    Path p = new Path(file);
    FileSystem fs = p.getFileSystem(conf);
    return fs.exists(p);
}

---

From source file:com.conversantmedia.mapreduce.output.BloomFilterOutputFormat.java

License:Apache License

@Override
public RecordWriter<K, V> getRecordWriter(TaskAttemptContext job) throws IOException, InterruptedException {
    if (writer == null) {
        int size = getExpectedInsertions(job);
        checkState(size > 0, "Expected insertion insertionSize not set.");

        Configuration conf = job.getConfiguration();
        String extension = "";
        Path file = getDefaultWorkFile(job, extension);
        FileSystem fs = file.getFileSystem(conf);
        FSDataOutputStream fileOut = fs.create(file, false);

        writer = new BloomFilterRecordWriter<>(fileOut, size);
    }//from  w w  w .  ja  va2s  .c om
    return writer;
}

---

From source file:com.cotdp.hadoop.BrotliFileRecordReader.java

License:Apache License

/**
 * Initialize and open the ZIP file from the FileSystem
 *//* w  ww .jav a  2  s .c  o m*/
@Override
public void initialize(InputSplit inputSplit, TaskAttemptContext taskAttemptContext)
        throws IOException, InterruptedException {
    FileSplit split = (FileSplit) inputSplit;
    Configuration conf = taskAttemptContext.getConfiguration();
    Path path = split.getPath();
    FileSystem fs = path.getFileSystem(conf);

    // Set the file path as the key
    currentKey.set(path.getName());
    // Open the stream
    fsin = fs.open(path);

    String cmd = "/bin/cat";
    ProcessBuilder pb = new ProcessBuilder();
    pb.redirectOutput();
    pb.command(cmd);

    try {
        decompressor = pb.start();

    } catch (IOException e) {
        // TODO Auto-generated catch block
        e.printStackTrace();
    }
}

---

From source file:com.cotdp.hadoop.ZipFileRecordReader.java

License:Apache License

/**
 * Initialise and open the ZIP file from the FileSystem
 *///from w  w  w. j av  a  2  s  .  co m

public void initialize(InputSplit inputSplit, TaskAttemptContext taskAttemptContext)
        throws IOException, InterruptedException {
    FileSplit split = (FileSplit) inputSplit;
    Configuration conf = taskAttemptContext.getConfiguration();
    Path path = split.getPath();
    FileSystem fs = path.getFileSystem(conf);

    // Open the stream
    fsin = fs.open(path);
    zip = new ZipInputStream(fsin);
    try {
        entry = zip.getNextEntry();
    } catch (ZipException e) {
        if (ZipFileInputFormat.getLenient() == false)
            throw e;
    }
}

---

From source file:com.csiro.hadoop.UFORecord.java

@Override
public int run(String[] args) throws Exception {
    JobConf conf = new JobConf(getConf(), getClass());
    conf.setJobName("UFO count");

    String[] otherArgs = new GenericOptionsParser(conf, args).getRemainingArgs();
    if (otherArgs.length != 2) {
        System.err.println("Usage: avro UFO counter <in> <out>");
        System.exit(2);/* w ww .j a v  a 2  s  .  co  m*/

    }

    org.apache.hadoop.mapred.FileInputFormat.addInputPath(conf, new Path(otherArgs[0]));
    Path outputPath = new Path(otherArgs[1]);
    org.apache.hadoop.mapred.FileOutputFormat.setOutputPath(conf, outputPath);
    outputPath.getFileSystem(conf).delete(outputPath);
    Schema input_schema = Schema.parse(getClass().getResourceAsStream("ufo.avsc"));
    AvroJob.setInputSchema(conf, input_schema);
    AvroJob.setMapOutputSchema(conf,
            Pair.getPairSchema(Schema.create(Schema.Type.STRING), Schema.create(Schema.Type.LONG)));

    AvroJob.setOutputSchema(conf, OUTPUT_SCHEMA);
    AvroJob.setMapperClass(conf, AvroRecordMapper.class);
    AvroJob.setReducerClass(conf, AvroRecordReducer.class);
    conf.setInputFormat(AvroInputFormat.class);
    JobClient.runJob(conf);

    return 0;
}

---

From source file:com.dalabs.droop.util.password.CredentialProviderPasswordLoader.java

License:Apache License

/**
 * If credential provider is available (made available as part of 2.6.0 and
 * 3.0, then use the credential provider to get the password. Else throw an
 * exception saying this provider is not available.
 *//*from  w ww .ja v  a 2  s . c o m*/
@Override
public String loadPassword(String p, Configuration configuration) throws IOException {
    if (!CredentialProviderHelper.isProviderAvailable()) {
        throw new IOException("CredentialProvider facility not available " + "in the hadoop environment used");
    }
    LOG.debug("Fetching alias from the specified path: " + p);
    Path path = new Path(p);
    FileSystem fs = path.getFileSystem(configuration);

    // Not closing FileSystem object because of SQOOP-1226
    verifyPath(fs, path);
    String alias = new String(readBytes(fs, path));
    String pass = CredentialProviderHelper.resolveAlias(configuration, alias);
    return pass;
}

---

From source file:com.dalabs.droop.util.password.CryptoFileLoader.java

License:Apache License

@Override
public String loadPassword(String p, Configuration configuration) throws IOException {
    LOG.debug("Fetching password from specified path: " + p);
    Path path = new Path(p);
    FileSystem fs = path.getFileSystem(configuration);

    byte[] encrypted;
    try {//from w w w  . j a  va 2 s . c  o  m
        verifyPath(fs, path);
        encrypted = readBytes(fs, path);
    } finally {
        fs.close();
    }

    String passPhrase = configuration.get(PROPERTY_CRYPTO_PASSPHRASE);
    if (passPhrase == null) {
        throw new IOException("Passphrase is missing in property " + PROPERTY_CRYPTO_PASSPHRASE);
    }

    String alg = configuration.get(PROPERTY_CRYPTO_ALG, DEFAULT_ALG);
    String algOnly = alg.split("/")[0];
    String salt = configuration.get(PROPERTY_CRYPTO_SALT, DEFAULT_SALT);
    int iterations = configuration.getInt(PROPERTY_CRYPTO_ITERATIONS, DEFAULT_ITERATIONS);
    int keyLen = configuration.getInt(PROPERTY_CRYPTO_KEY_LEN, DEFAULT_KEY_LEN);

    SecretKeyFactory factory = null;
    try {
        factory = SecretKeyFactory.getInstance("PBKDF2WithHmacSHA1");
    } catch (NoSuchAlgorithmException e) {
        throw new IOException("Can't load SecretKeyFactory", e);
    }

    SecretKeySpec key = null;
    try {
        key = new SecretKeySpec(factory
                .generateSecret(new PBEKeySpec(passPhrase.toCharArray(), salt.getBytes(), iterations, keyLen))
                .getEncoded(), algOnly);
    } catch (Exception e) {
        throw new IOException("Can't generate secret key", e);
    }

    Cipher crypto = null;

    try {
        crypto = Cipher.getInstance(alg);
    } catch (Exception e) {
        throw new IOException("Can't initialize the decryptor", e);
    }

    byte[] decryptedBytes;

    try {
        crypto.init(Cipher.DECRYPT_MODE, key);
        decryptedBytes = crypto.doFinal(encrypted);
    } catch (Exception e) {
        throw new IOException("Can't decrypt the password", e);
    }

    return new String(decryptedBytes);
}

---

From source file:com.dalabs.droop.util.password.FilePasswordLoader.java

License:Apache License

@Override
public String loadPassword(String p, Configuration configuration) throws IOException {
    LOG.debug("Fetching password from specified path: " + p);
    Path path = new Path(p);
    FileSystem fs = path.getFileSystem(configuration);

    // Not closing FileSystem object because of SQOOP-1226
    verifyPath(fs, path);//  ww w .  j  a va 2  s  .  c om
    return new String(readBytes(fs, path));
}

---

From source file:com.datamoin.tajo.tpcds.TpcDSTestUtil.java

License:Apache License

public static void createTables(String database, TajoClient client) throws Exception {
    String dataDir = getDataDir();
    if (dataDir == null || dataDir.isEmpty()) {
        throw new IOException("No TPCDS_DATA_DIR property. Use -DTPCDS_DATA_DIR=<data dir>");
    }/*w  w  w . jav  a  2s  . co m*/

    if (dataDir.startsWith("hdfs://")) {
        Path path = new Path(dataDir);
        FileSystem fs = path.getFileSystem(new Configuration());
        for (String eachTable : tableNames) {
            Path tableDataDir = new Path(path, eachTable);
            if (!fs.exists(tableDataDir)) {
                throw new IOException(eachTable + " data dir [" + tableDataDir + "] not exists.");
            }
        }
    } else {
        File dataDirFile = new File(dataDir);
        if (!dataDirFile.exists()) {
            throw new IOException("TPCDS_DATA_DIR [" + dataDir + "] not exists.");
        }
        if (dataDirFile.isFile()) {
            throw new IOException("TPCDS_DATA_DIR [" + dataDir + "] is not a directory.");
        }

        for (String eachTable : tableNames) {
            File tableDataDir = new File(dataDirFile, eachTable);
            if (!tableDataDir.exists()) {
                throw new IOException(eachTable + " data dir [" + tableDataDir + "] not exists.");
            }
        }
    }

    KeyValueSet opt = new KeyValueSet();
    opt.set(StorageConstants.TEXT_DELIMITER, StorageConstants.DEFAULT_FIELD_DELIMITER);

    LOG.info("Create database: " + database);
    client.executeQuery("create database if not exists " + database);

    Path tpcdsResourceURL = new Path(ClassLoader.getSystemResource("tpcds").toString());

    Path ddlPath = new Path(tpcdsResourceURL, "ddl");
    FileSystem localFs = FileSystem.getLocal(new Configuration());

    FileStatus[] files = localFs.listStatus(ddlPath);

    String dataDirWithPrefix = dataDir;
    if (dataDir.indexOf("://") < 0) {
        dataDirWithPrefix = "file://" + dataDir;
    }

    for (FileStatus eachFile : files) {
        if (eachFile.isFile()) {
            String tableName = eachFile.getPath().getName().split("\\.")[0];
            String query = FileUtil.readTextFile(new File(eachFile.getPath().toUri()));
            query = query.replace("${DB}", database);
            query = query.replace("${DATA_LOCATION}", dataDirWithPrefix + "/" + tableName);

            LOG.info("Create table:" + tableName + "," + query);
            client.executeQuery(query);
        }
    }
}

---

• «
• 1
• 2
• 3
• 4
• 5
• 6
• 7
• 8
• 9
• 10
• 11
• 12
• 13
• 14
• 15
• 16
• 17
• 18
• 19
• 20
• 21
• 22
• 23
• 24
• 25
• 26
• 27
• 28
• 29
• 30
• 31
• 32
• 33
• 34
• 35
• 36
• 37
• 38
• 39
• 40
• 41
• 42
• 43
• 44
• 45
• 46
• 47
• 48
• 49
• 50
• »