Example usage for org.apache.hadoop.fs FileSystem exists

List of usage examples for org.apache.hadoop.fs FileSystem exists

  1. HOME
  2. Java
  3. org.apache.hadoop
  4. org.apache.hadoop.fs.*
  5. FileSystem
  6. exists

Introduction

In this page you can find the example usage for org.apache.hadoop.fs FileSystem exists.

Prototype

public boolean exists(Path f) throws IOException

Source Link

Document

Check if a path exists.

Usage

From source file:JaqlShell.java

License:Apache License

/**
 * @param dir//from   w  w  w .ja va  2  s .  co  m
 * @param numNodes
 * @param format
 * @throws Exception
 */
public void init(String dir, int numNodes) throws Exception {
    String vInfo = VersionInfo.getVersion();
    System.setProperty("test.build.data", dir);
    m_conf = new Configuration();

    // setup conf according to the Hadoop version
    if (vInfo.indexOf("0.20") < 0) {
        throw new Exception("Unsupported Hadoop version: " + vInfo);
    }

    // setup the mini dfs cluster
    m_fs = new MiniDFSCluster(m_conf, numNodes, true, (String[]) null);
    FileSystem filesystem = m_fs.getFileSystem();
    m_conf.set("fs.default.name", filesystem.getUri().toString());
    Path parentdir = filesystem.getHomeDirectory();
    filesystem.mkdirs(parentdir);
    //FSUtils.setVersion(filesystem, parentdir);

    // setup hbase cluster (only if OS is not windows)
    //    if(!System.getProperty("os.name").toLowerCase().contains("win")) {
    //      m_conf.set(HConstants.HBASE_DIR, parentdir.toString());      
    //      Path hdfsTestDir = filesystem.makeQualified(new Path(m_conf.get(HConstants.HBASE_DIR)));
    //
    //      // prime the hdfs for hbase information...
    //      HRegion root = HRegion.createHRegion(HRegionInfo.ROOT_REGIONINFO, hdfsTestDir, (HBaseConfiguration)m_conf);
    //      HRegion meta = HRegion.createHRegion(HRegionInfo.FIRST_META_REGIONINFO, hdfsTestDir, (HBaseConfiguration)m_conf);
    //      HRegion.addRegionToMETA(root, meta);
    //
    //      // ... and close the root and meta
    //      if (meta != null) {
    //        meta.close();
    //        meta.getLog().closeAndDelete();
    //      }
    //      if (root != null) {
    //        root.close();
    //        root.getLog().closeAndDelete();
    //      }
    //
    //      try
    //      {
    //        this.zooKeeperCluster = new MiniZooKeeperCluster();
    //        File testDir = new File(dir);
    //        int clientPort = this.zooKeeperCluster.startup(testDir);
    //        m_conf.set("hbase.zookeeper.property.clientPort", Integer.toString(clientPort));
    //      } catch(Exception e) {
    //        LOG.error("Unable to startup zookeeper");
    //        throw new IOException(e);
    //      }
    //      try {
    //        // start the mini cluster
    //        m_base = new MiniHBaseCluster((HBaseConfiguration)m_conf, numNodes);
    //      } catch(Exception e) {
    //        LOG.error("Unable to startup hbase");
    //        throw new IOException(e);
    //      }
    //      try {
    //        // opening the META table ensures that cluster is running
    //        new HTable((HBaseConfiguration)m_conf, HConstants.META_TABLE_NAME);        
    //
    //        //setupOverride(conf);
    //      }
    //      catch (Exception e)
    //      {
    //        LOG.warn("Could not verify that hbase is up", e);
    //      }
    //      setupOverride();
    //    }

    m_mr = startMRCluster(numNodes, m_fs.getFileSystem().getName(), m_conf);

    Configuration conf = new Configuration();
    FileSystem fs = FileSystem.get(conf);

    // make the home directory if it does not exist
    Path hd = fs.getWorkingDirectory();
    if (!fs.exists(hd))
        fs.mkdirs(hd);

    // make the $USER/_temporary directory if it does not exist
    Path tmpPath = new Path(hd, "_temporary");
    if (!fs.exists(tmpPath))
        fs.mkdirs(tmpPath);

    //    if (m_base != null)
    //    {
    //      try {
    //        m_admin = new HBaseAdmin((HBaseConfiguration) m_conf);
    //        HTableDescriptor[] tables = m_admin.listTables();
    //        if (tables != null)
    //        {
    //          for (int i = 0; i < tables.length; i++)
    //          {
    //            m_admin.enableTable(tables[i].getName());
    //          }
    //        }
    //      } catch(Exception e) {
    //        LOG.warn("failed to enable hbase tables");
    //      }
    //    }
}

From source file:JaqlShell.java

License:Apache License

/**
 * @throws Exception/*ww w.j  a  v  a 2  s.  c  o m*/
 */
public void init() throws Exception {
    // do nothing in the case of cluster
    //m_conf = new HBaseConfiguration();
    //m_admin = new HBaseAdmin(m_conf);
    Configuration conf = new Configuration();
    FileSystem fs = FileSystem.get(conf);

    // make the home directory if it does not exist
    Path hd = fs.getWorkingDirectory();
    if (!fs.exists(hd))
        fs.mkdirs(hd);
}

From source file:LobFileStressTest.java

License:Apache License

private void writeIntegerFile(boolean compress) throws Exception {
    boolean passed = false;
    try {/*from  ww w. j av a2s .com*/
        System.out.print("Writing integers file. compress=" + compress + ". ");
        Path p = getPath(compress);
        FileSystem fs = FileSystem.getLocal(conf);
        if (fs.exists(p)) {
            fs.delete(p, false);
        }
        String codecName = compress ? "deflate" : null;
        LobFile.Writer w = LobFile.create(p, conf, false, codecName);

        int numRecords = getNumRecords(compress);
        for (int i = 0; i < numRecords; i++) {
            setLastRecordPos(w.tell(), compress);
            OutputStream os = w.writeBlobRecord(0);
            DataOutputStream dos = new DataOutputStream(os);
            dos.writeInt(i);
            dos.close();
            os.close();
        }

        w.close();
        System.out.println("PASS");
        passed = true;
    } finally {
        if (!passed) {
            allPassed = false;
            System.out.println("FAIL");
        }
    }
}

From source file:LobFileStressTest.java

License:Apache License

private void testBigFile(boolean compress) throws Exception {
    // Write a file containing 5 GB records.

    final int NUM_RECORDS = 5;
    boolean passed = false;

    try {//from  w w  w  .  ja  v  a2  s.c  om
        System.out.print("Testing large file operations. compress=" + compress + ". ");

        Path p = getBigFilePath(compress);
        long[] startOffsets = new long[NUM_RECORDS];

        // Write the file. Five records, 5 GB a piece.
        System.out.print("Testing write. ");
        FileSystem fs = FileSystem.getLocal(conf);
        if (fs.exists(p)) {
            fs.delete(p, false);
        }
        String codecName = compress ? "deflate" : null;
        System.out.println("record size: " + LARGE_RECORD_LEN);
        LobFile.Writer w = LobFile.create(p, conf, false, codecName);
        for (int i = 0; i < NUM_RECORDS; i++) {
            startOffsets[i] = w.tell();
            System.out.println("Starting record " + i + " at " + startOffsets[i]);
            OutputStream os = w.writeBlobRecord(0);
            for (long v = 0; v < LARGE_RECORD_LEN; v++) {
                long byteVal = (((long) i) + v) & 0xFF;
                os.write((int) byteVal);
            }
            os.close();
        }
        w.close();
        System.out.println("PASS");

        // Iterate past three records, read the fourth.
        System.out.print("Testing iterated skipping. ");
        LobFile.Reader r = LobFile.open(p, conf);
        for (int i = 0; i < 4; i++) {
            r.next();
        }

        checkBigRecord(r, 3);
        System.out.println("PASS");

        // Seek directly to record 2, read it through.
        System.out.print("Testing large backward seek. ");
        r.seek(startOffsets[2]);
        r.next();
        checkBigRecord(r, 2);
        System.out.println("PASS");

        passed = true;
    } finally {
        if (!passed) {
            allPassed = false;
            System.out.println("FAIL");
        }
    }
}

From source file:HoodieClientExample.java

License:Apache License

public void run() throws Exception {

    SparkConf sparkConf = new SparkConf().setAppName("hoodie-client-example");
    sparkConf.setMaster("local[1]");
    sparkConf.set("spark.serializer", "org.apache.spark.serializer.KryoSerializer");
    sparkConf.set("spark.kryoserializer.buffer.max", "512m");
    JavaSparkContext jsc = new JavaSparkContext(sparkConf);

    // Generator of some records to be loaded in.
    HoodieTestDataGenerator dataGen = new HoodieTestDataGenerator();

    // initialize the table, if not done already
    Path path = new Path(tablePath);
    FileSystem fs = FSUtils.getFs(tablePath, jsc.hadoopConfiguration());
    if (!fs.exists(path)) {
        HoodieTableMetaClient.initTableType(jsc.hadoopConfiguration(), tablePath,
                HoodieTableType.valueOf(tableType), tableName, HoodieAvroPayload.class.getName());
    }/*w w w . java 2  s. c om*/

    // Create the write client to write some records in
    HoodieWriteConfig cfg = HoodieWriteConfig.newBuilder().withPath(tablePath)
            .withSchema(HoodieTestDataGenerator.TRIP_EXAMPLE_SCHEMA).withParallelism(2, 2).forTable(tableName)
            .withIndexConfig(HoodieIndexConfig.newBuilder().withIndexType(IndexType.BLOOM).build())
            .withCompactionConfig(HoodieCompactionConfig.newBuilder().archiveCommitsWith(2, 3).build()).build();
    HoodieWriteClient client = new HoodieWriteClient(jsc, cfg);

    /**
     * Write 1 (only inserts)
     */
    String newCommitTime = client.startCommit();
    logger.info("Starting commit " + newCommitTime);

    List<HoodieRecord> records = dataGen.generateInserts(newCommitTime, 100);
    JavaRDD<HoodieRecord> writeRecords = jsc.<HoodieRecord>parallelize(records, 1);
    client.upsert(writeRecords, newCommitTime);

    /**
     * Write 2 (updates)
     */
    newCommitTime = client.startCommit();
    logger.info("Starting commit " + newCommitTime);
    records.addAll(dataGen.generateUpdates(newCommitTime, 100));
    writeRecords = jsc.<HoodieRecord>parallelize(records, 1);
    client.upsert(writeRecords, newCommitTime);

    /**
     * Schedule a compaction and also perform compaction on a MOR dataset
     */
    if (HoodieTableType.valueOf(tableType) == HoodieTableType.MERGE_ON_READ) {
        Optional<String> instant = client.scheduleCompaction(Optional.empty());
        JavaRDD<WriteStatus> writeStatues = client.compact(instant.get());
        client.commitCompaction(instant.get(), writeStatues, Optional.empty());
    }
}

From source file:acromusashi.stream.bolt.hdfs.HdfsPreProcessor.java

License:Open Source License

/**
 * ????<br>/*from w w  w . j  a  v  a2 s  . c om*/
 * ???????????
 * 
 * @param hdfs 
 * @param targetTmpPath ??
 * @param tmpSuffix ??
 */
private static void renameTmpFile(FileSystem hdfs, String targetTmpPath, String tmpSuffix) {
    String basePath = extractBasePath(targetTmpPath, tmpSuffix);

    boolean isFileExists = true;

    try {
        isFileExists = hdfs.exists(new Path(basePath));
    } catch (IOException ioex) {
        String logFormat = "Failed to search target file exists. Skip file rename. : TargetUri={0}";
        String logMessage = MessageFormat.format(logFormat, basePath);
        logger.warn(logMessage, ioex);
        return;
    }

    if (isFileExists) {
        String logFormat = "File exists renamed target. Skip file rename. : BeforeUri={0} , AfterUri={1}";
        String logMessage = MessageFormat.format(logFormat, targetTmpPath, basePath);
        logger.warn(logMessage);
    } else {
        try {
            hdfs.rename(new Path(targetTmpPath), new Path(basePath));
        } catch (IOException ioex) {
            String logFormat = "Failed to HDFS file rename. Skip rename file and continue preprocess. : BeforeUri={0} , AfterUri={1}";
            String logMessage = MessageFormat.format(logFormat, targetTmpPath, basePath);
            logger.warn(logMessage, ioex);
        }
    }
}

From source file:acromusashi.stream.bolt.hdfs.HdfsStreamWriter.java

License:Open Source License

/**
 * ??HDFS??Open?// w ww. j  a  v  a  2  s. c o m
 * 
 * @param filePath HDFS
 * @param fs 
 * @param isFileSyncEachTime ?????????
 * @throws IOException Open
 */
public void open(String filePath, FileSystem fs, boolean isFileSyncEachTime) throws IOException {
    Path dstPath = new Path(filePath);

    if (fs.exists(dstPath) == true) {
        this.delegateStream = fs.append(dstPath);
    } else {
        this.delegateStream = fs.create(dstPath);
    }

    this.isFileSyncEachTime = isFileSyncEachTime;
}

From source file:adts.HbaseClient.java

License:Open Source License

public static void main(String[] args) throws IOException {
    String[] keys = new String[5];
    int keywords_counter = 0;
    Configuration conf = new Configuration();
    FileSystem fs = FileSystem.get(conf);
    Path inFile = new Path(args[0]);
    if (!fs.exists(inFile))
        System.out.println("Input file not found");
    if (!fs.isFile(inFile))
        System.out.println("Input should be a file");
    else {//  ww  w. ja  v a 2  s .c  o m
        FSDataInputStream fsDataInputStream = fs.open(inFile);
        BufferedReader bufferedReader = new BufferedReader(new InputStreamReader(fsDataInputStream));
        String line;
        while (((line = bufferedReader.readLine()) != null) && (keywords_counter < 5)) {
            String[] array = line.split("\t");
            String keyword = array[0];
            System.out.println("Record :   " + keyword);
            keys[keywords_counter] = keyword;
            keywords_counter++;
        }
        bufferedReader.close();
        fs.close();

        Configuration config = HBaseConfiguration.create();
        HTable table = new HTable(config, "index");

        Random randomGenerator = new Random();
        for (int i = 0; i < 10; i++) {
            int randomInt = randomGenerator.nextInt(5);
            System.out.println("Random chosen keyword : " + keys[randomInt]);

            FilterList list = new FilterList(FilterList.Operator.MUST_PASS_ALL);
            SingleColumnValueFilter filter_by_name = new SingleColumnValueFilter(Bytes.toBytes("keyword"),
                    Bytes.toBytes(""), CompareOp.EQUAL, Bytes.toBytes(keys[randomInt]));
            //filter_by_name.setFilterIfMissing(true);
            list.addFilter(filter_by_name);

            Scan scan = new Scan();
            scan.setFilter(list);
            //scan.addFamily(Bytes.toBytes("keyword"));
            ResultScanner scanner = table.getScanner(scan);
            try {

                for (Result rr = scanner.next(); rr != null; rr = scanner.next()) {
                    // print out the row we found and the columns we were looking for
                    byte[] cells = rr.getValue(Bytes.toBytes("article"), Bytes.toBytes(""));
                    System.out.println("Keyword " + keys[randomInt] + "belonging to article with md5 : "
                            + Bytes.toString(cells));
                }
            } catch (Exception e) {
                e.printStackTrace();
            } finally {
                scanner.close();
            }

        }
        table.close();

    }

}

From source file:alluxio.hadoop.mapreduce.KeyValueOutputCommitter.java

License:Apache License

/**
 * @param taskContext MapReduce task configuration
 * @return true if the task output directory exists, otherwise false
 * @throws IOException if fails to determine whether the output directory exists
 *//* w w  w .  j a va2  s  .com*/
@Override
public boolean needsTaskCommit(TaskAttemptContext taskContext) throws IOException {
    Path taskOutputPath = new Path(KeyValueOutputFormat.getTaskOutputURI(taskContext).toString());
    FileSystem fs = taskOutputPath.getFileSystem(taskContext.getConfiguration());
    return fs.exists(taskOutputPath);
}

From source file:arpserver.HadoopTool.java

@Override
public int run(String[] strings) throws Exception {
    Configuration conf = new Configuration();
    String in = strings[0];/*from   ww w.  ja v a2  s  .co m*/
    String out = strings[1];
    FileSystem fs = FileSystem.get(conf);
    if (fs.exists(new Path(out))) {
        fs.delete(new Path(out), true);
        fs.delete(new Path(out + "Src"), true);
        fs.delete(new Path(out + "Mitm"), true);
        fs.delete(new Path(out + "ArpScn"), true);
        fs.delete(new Path(out + "s"), true);
        fs.delete(new Path(out + "d"), true);
        fs.delete(new Path(out + "t"), true);
    }
    Job job = new Job();
    Job job2 = new Job();
    Job job3 = new Job();
    Job job4 = new Job();
    Job job5 = new Job();
    Job job6 = new Job();
    Job job7 = new Job();
    job.setJobName("Q");
    job2.setJobName("Src");
    job3.setJobName("Mitm");
    job4.setJobName("ArpScn");
    job5.setJobName("s");
    job6.setJobName("d");
    job7.setJobName("time");
    job.setJarByClass(QuickDetect.class);

    job.setMapperClass(Qmapper.class);
    job.setReducerClass(Qreducer.class);

    job2.setMapperClass(Srcmapper.class);
    job2.setReducerClass(Srcreducer.class);

    job3.setMapperClass(ArpScanmapper.class);
    job3.setReducerClass(ArpScanreducer.class);

    job4.setMapperClass(Mitmmapper.class);
    job4.setReducerClass(Mitmreducer.class);

    job5.setMapperClass(Smapper.class);
    job5.setReducerClass(Sreducer.class);

    job6.setMapperClass(Dmapper.class);
    job6.setReducerClass(Dreducer.class);

    job7.setMapperClass(timemapper.class);
    job7.setReducerClass(timereducer.class);
    //testFinal168.txt
    job.setOutputKeyClass(NullWritable.class);
    job.setOutputValueClass(Text.class);

    job2.setOutputKeyClass(NullWritable.class);
    job2.setOutputValueClass(Text.class);

    job3.setOutputKeyClass(NullWritable.class);
    job3.setOutputValueClass(IntWritable.class);

    job4.setOutputKeyClass(NullWritable.class);
    job4.setOutputValueClass(Text.class);

    job5.setOutputKeyClass(NullWritable.class);
    job5.setOutputValueClass(Text.class);

    job6.setOutputKeyClass(NullWritable.class);
    job6.setOutputValueClass(Text.class);

    job7.setOutputKeyClass(NullWritable.class);
    job7.setOutputValueClass(Text.class);

    job.setMapOutputKeyClass(QuickDetect.class);
    job.setMapOutputValueClass(IntWritable.class);
    //job.setOutputFormatClass(YearMultipleTextOutputFormat.class);
    job2.setMapOutputKeyClass(DetectSrc.class);
    job2.setMapOutputValueClass(IntWritable.class);

    job3.setMapOutputKeyClass(DetectArpScan.class);
    job3.setMapOutputValueClass(IntWritable.class);

    job4.setMapOutputKeyClass(DetectMitm.class);
    job4.setMapOutputValueClass(IntWritable.class);

    job5.setMapOutputKeyClass(SMac.class);
    job5.setMapOutputValueClass(IntWritable.class);

    job6.setMapOutputKeyClass(DMac.class);
    job6.setMapOutputValueClass(IntWritable.class);

    job7.setMapOutputKeyClass(timeMac.class);
    job7.setMapOutputValueClass(IntWritable.class);

    FileInputFormat.addInputPath(job, new Path(in));
    FileOutputFormat.setOutputPath(job, new Path(out));
    if (job.waitForCompletion(true)) {
        FileInputFormat.addInputPath(job2, new Path(in));
        FileOutputFormat.setOutputPath(job2, new Path(out + "Src"));
        if (job2.waitForCompletion(true)) {
            FileInputFormat.addInputPath(job3, new Path(in));
            FileOutputFormat.setOutputPath(job3, new Path(out + "ArpScn"));
            if (job3.waitForCompletion(true)) {
                FileInputFormat.addInputPath(job4, new Path(in));
                FileOutputFormat.setOutputPath(job4, new Path(out + "Mitm"));
                if (job4.waitForCompletion(true)) {
                    FileInputFormat.addInputPath(job5, new Path(in));
                    FileOutputFormat.setOutputPath(job5, new Path(out + "s"));
                    if (job5.waitForCompletion(true)) {
                        FileInputFormat.addInputPath(job6, new Path(in));
                        FileOutputFormat.setOutputPath(job6, new Path(out + "d"));
                        if (job6.waitForCompletion(true)) {
                            FileInputFormat.addInputPath(job7, new Path(in));
                            FileOutputFormat.setOutputPath(job7, new Path(out + "t"));
                            job7.waitForCompletion(true);
                        } else {
                            return 1;
                        }
                    } else {
                        return 1;
                    }
                } else {
                    return 1;
                }
            } else {
                return 1;
            }
        } else {
            return 1;
        }
    } else {
        return 1;
    }
    return 0;
}