Example usage for org.apache.hadoop.fs FileSystem makeQualified

Introduction

In this page you can find the example usage for org.apache.hadoop.fs FileSystem makeQualified.

Prototype

public Path makeQualified(Path path) 

Source Link

Document

Qualify a path to one which uses this FileSystem and, if relative, made absolute.

Usage

From source file:pl.edu.icm.coansys.heeut.TestHDFS.java

License:Apache License

@Test(timeout = 1800000)
public void testMkdir() throws Exception {

    FileSystem dfs = UTIL.getDFSCluster().getFileSystem();
    String tmpDirName = getCurrentDateAppended("tmp");

    Path qualifiedTmpDir = dfs.makeQualified(new Path(tmpDirName));
    Assert.assertFalse(dfs.exists(qualifiedTmpDir));

    dfs.create(qualifiedTmpDir);/*ww  w  . j av a  2s.  co m*/
    Assert.assertTrue(dfs.exists(qualifiedTmpDir));
}

---

From source file:pl.edu.icm.coansys.heeut.TestMapReduce.java

License:Apache License

@Test(timeout = 1800000)
public void testWordCount() throws Exception {

    String prefix = getCurrentDateAppended("wordcount");
    String inputDirName = prefix + "-input";
    String outputDirName = prefix + "-output";

    FileSystem dfs = UTIL.getDFSCluster().getFileSystem();
    Path inputDir = new Path(inputDirName);
    Path qualifiedInputDir = dfs.makeQualified(inputDir);

    dfs.copyFromLocalFile(new Path("src/test/resource/input/wordcount/apache_projects.dat"), qualifiedInputDir);
    ToolRunner.run(UTIL.getConfiguration(), new WordCount(), new String[] { inputDirName, outputDirName });

    InputStream contentStream = dfs.open(new Path(outputDirName + "/part-00000"));
    BufferedReader contentReader = new BufferedReader(new InputStreamReader(contentStream));
    Assert.assertEquals("Apache\t3", contentReader.readLine());
    Assert.assertEquals("HBase\t1", contentReader.readLine());
    Assert.assertEquals("Hadoop\t1", contentReader.readLine());
    Assert.assertEquals("Pig\t1", contentReader.readLine());

    Assert.assertNull(contentReader.readLine());
    contentReader.close();/*from   w  ww.j av  a2  s.  c  om*/
}

---

From source file:pl.edu.icm.coansys.heeut.TestMapReduce.java

License:Apache License

@Test(timeout = 1800000)
public void testWordCountDiff() throws Exception {

    String prefix = getCurrentDateAppended("wordcount");
    String inputDirName = prefix + "-input";
    String outputDirName = prefix + "-output";

    FileSystem dfs = UTIL.getDFSCluster().getFileSystem();
    Path inputDir = new Path(inputDirName);
    Path qualifiedInputDir = dfs.makeQualified(inputDir);

    String inputFileName = "src/test/resource/input/wordcount/apache_projects.dat";
    dfs.copyFromLocalFile(new Path(inputFileName), qualifiedInputDir);
    ToolRunner.run(UTIL.getConfiguration(), new WordCount(), new String[] { inputDirName, outputDirName });

    Path outputDir = new Path(outputDirName);
    Path qualifiedOutputDir = dfs.makeQualified(outputDir);

    String localOutputDir = "src/test/resource/output/wordcount/" + prefix;
    dfs.copyToLocalFile(qualifiedOutputDir, new Path(localOutputDir));

    File outputFile = new File(localOutputDir + "/part-00000");
    File expectedFile = new File("src/test/resource/exp/wordcount/apache_projects.exp");
    boolean isEqual = FileUtils.contentEquals(outputFile, expectedFile);
    Assert.assertTrue(isEqual);//  w  w  w .jav a  2 s  .com
}

---

From source file:pl.edu.icm.coansys.richimporttsv.jobs.mapreduce.TestRichImportTsv.java

License:Apache License

@Test
public void testDirRichImportTsv() throws Exception {
    String tableInitName = getCurrentDateAppended("testDirRichImportTsv");
    String inputFileName = "InputFile.dat";
    String outputDirName = getCurrentDateAppended("richtsv-output");
    FileSystem dfs = UTIL.getDFSCluster().getFileSystem();

    Path qualifiedOutputDir = dfs.makeQualified(new Path(outputDirName));
    assertFalse(dfs.exists(qualifiedOutputDir));

    // Prepare the arguments required for the test.
    String[] args = new String[] { "-Dimporttsv.record.separator=#", "-Dimporttsv.separator=$",
            "-Dimporttsv.columns=HBASE_ROW_KEY," + S_COLUMN_FAMILY + ":" + S_COLUMN_QUALIFIER,
            "-Dimporttsv.bulk.output=" + outputDirName, tableInitName, inputFileName };

    doMROnTableTest(inputFileName, S_COLUMN_FAMILY, tableInitName, "KEY1$VALUE\n1#KEY2$VALUE2#\nKEY3$VALUE3",
            args);/* w  w w  .j a  v a  2 s.c  om*/

    assertTrue(dfs.exists(qualifiedOutputDir));
}

---

From source file:pl.edu.icm.coansys.richimporttsv.jobs.mapreduce.TestRichImportTsv.java

License:Apache License

public void testExportImport() throws Exception {

    String tableInitName = getCurrentDateAppended("testExportImport");
    createAndPopulateDefaultTable(tableInitName, TEST_ROW_COUNT);

    FileSystem dfs = UTIL.getDFSCluster().getFileSystem();
    Path qualifiedTempDir = dfs.makeQualified(new Path("export-import-temp-dir"));
    Assert.assertFalse(dfs.exists(qualifiedTempDir));

    Job jobExport = Export.createSubmittableJob(UTIL.getConfiguration(),
            new String[] { tableInitName, qualifiedTempDir.toString() });
    jobExport.waitForCompletion(true);/* ww w  .  j a v  a  2  s. com*/

    Assert.assertTrue(dfs.exists(qualifiedTempDir));

    final String tableImportName = tableInitName + "Import";
    HTable htableImport = UTIL.createTable(Bytes.toBytes(tableImportName), B_COLUMN_FAMILY);

    Job jobImport = Import.createSubmittableJob(UTIL.getConfiguration(),
            new String[] { tableImportName, qualifiedTempDir.toString() });
    jobImport.waitForCompletion(true);
    Assert.assertEquals(TEST_ROW_COUNT, (long) UTIL.countRows(htableImport));

    dropTable(tableInitName);
    dropTable(tableImportName);
}

---

From source file:skewtune.mapreduce.STJobTracker.java

License:Apache License

/**
 * @throws LoginException/*w  w  w. jav  a  2  s . co m*/
 * @see org.apache.hadoop.mapreduce.protocol.ClientProtocol#getStagingAreaDir()
 */
public String getStagingAreaDir() throws IOException {
    try {
        final String user = UserGroupInformation.getCurrentUser().getShortUserName();
        return mrOwner.doAs(new PrivilegedExceptionAction<String>() {
            @Override
            public String run() throws Exception {
                Path stagingRootDir = new Path(
                        conf.get(JTConfig.JT_STAGING_AREA_ROOT, "/tmp/hadoop/mapred/staging"));
                FileSystem fs = stagingRootDir.getFileSystem(conf);
                return fs.makeQualified(new Path(stagingRootDir, user + "/.staging")).toString();
            }
        });
    } catch (InterruptedException ie) {
        throw new IOException(ie);
    }
}

---

From source file:tajo.engine.planner.physical.TestBSTIndexExec.java

License:Apache License

@Before
public void setup() throws Exception {
    this.randomValues = new HashMap<Integer, Integer>();
    this.conf = new TajoConf();
    TajoTestingUtility util = new TajoTestingUtility();
    util.startMiniZKCluster();/*from   w  w  w  .  j a v  a  2 s  . c o m*/
    util.startCatalogCluster();
    catalog = util.getMiniCatalogCluster().getCatalog();

    util.initTestDir();
    File testDir = util.getTestDir("TestPhysicalPlanner");
    FileSystem fs = FileSystem.getLocal(conf);
    sm = StorageManager.get(conf, fs.makeQualified(new Path(testDir.toURI())));

    idxPath = new Path(URI.create(testDir.toURI() + "/test.idx"));

    Schema schema = new Schema();
    schema.addColumn("managerId", DataType.INT);
    schema.addColumn("empId", DataType.INT);
    schema.addColumn("deptName", DataType.STRING);

    this.idxSchema = new Schema();
    idxSchema.addColumn("managerId", DataType.INT);
    SortSpec[] sortKeys = new SortSpec[1];
    sortKeys[0] = new SortSpec(idxSchema.getColumn("managerId"), true, false);
    this.comp = new TupleComparator(idxSchema, sortKeys);

    this.writer = new BSTIndex(conf).getIndexWriter(idxPath, BSTIndex.TWO_LEVEL_INDEX, this.idxSchema,
            this.comp);
    writer.setLoadNum(100);
    writer.open();
    long offset;

    TableMeta employeeMeta = TCatUtil.newTableMeta(schema, StoreType.CSV);
    sm.initTableBase(employeeMeta, "employee");
    FileAppender appender = (FileAppender) sm.getAppender(employeeMeta, "employee", "employee");
    Tuple tuple = new VTuple(employeeMeta.getSchema().getColumnNum());
    for (int i = 0; i < 10000; i++) {

        Tuple key = new VTuple(this.idxSchema.getColumnNum());
        int rndKey = rnd.nextInt(250);
        if (this.randomValues.containsKey(rndKey)) {
            int t = this.randomValues.remove(rndKey) + 1;
            this.randomValues.put(rndKey, t);
        } else {
            this.randomValues.put(rndKey, 1);
        }

        key.put(new Datum[] { DatumFactory.createInt(rndKey) });
        tuple.put(new Datum[] { DatumFactory.createInt(rndKey), DatumFactory.createInt(rnd.nextInt(10)),
                DatumFactory.createString("dept_" + rnd.nextInt(10)) });
        offset = appender.getOffset();
        appender.addTuple(tuple);
        writer.write(key, offset);
    }
    appender.flush();
    appender.close();
    writer.close();

    TableDesc desc = new TableDescImpl("employee", employeeMeta, sm.getTablePath("employee"));
    catalog.addTable(desc);

    analyzer = new QueryAnalyzer(catalog);
    planner = new LogicalPlanner(catalog);
}

---

From source file:tajo.storage.TestRawFile.java

License:Apache License

@Test
public void testRawFile() throws IOException {
    Schema schema = new Schema();
    schema.addColumn("id", DataType.INT);
    schema.addColumn("age", DataType.LONG);
    schema.addColumn("description", DataType.STRING);
    schema.addColumn("null_one", DataType.LONG);
    schema.addColumn("array", DataType.ARRAY);

    TableMeta meta = TCatUtil.newTableMeta(schema, StoreType.RAW);

    FileSystem fs = FileSystem.getLocal(conf);
    Path path = fs.makeQualified(new Path(TEST_PATH, "raw.dat"));

    RawFile.Appender appender = new RawFile.Appender(conf, meta, path);
    appender.init();/*from  ww  w.j av  a2  s .c  om*/
    int tupleNum = 10000;
    VTuple vTuple;
    Datum stringDatum = DatumFactory.createString("abcdefghijklmnopqrstuvwxyz");
    ArrayDatum array = new ArrayDatum(
            new Datum[] { DatumFactory.createLong(1234), DatumFactory.createLong(4567) });

    long startAppend = System.currentTimeMillis();
    for (int i = 0; i < tupleNum; i++) {
        vTuple = new VTuple(5);
        vTuple.put(0, DatumFactory.createInt(i + 1));
        vTuple.put(1, DatumFactory.createLong(25l));
        vTuple.put(2, stringDatum);
        vTuple.put(3, DatumFactory.createNullDatum());
        vTuple.put(4, array);
        appender.addTuple(vTuple);
    }
    long endAppend = System.currentTimeMillis();
    appender.close();

    //TableStat stat = appender.getStats();
    //assertEquals(tupleNum, stat.getNumRows().longValue());

    RawFile.Scanner scanner = new RawFile.Scanner(conf, meta, path);
    scanner.init();
    int tupleCnt = 0;
    long startScan = System.currentTimeMillis();
    while (scanner.next() != null) {
        tupleCnt++;
    }
    long endScan = System.currentTimeMillis();
    scanner.close();

    assertEquals(tupleNum, tupleCnt);

    System.out.println("Append time: " + (endAppend - startAppend) + " msc");
    System.out.println("Scan time: " + (endScan - startScan) + " msc");
}

---

From source file:tajo.storage.TestRawFile.java

License:Apache License

@Test
public void testVariousTypes() throws IOException {
    Schema schema = new Schema();
    schema.addColumn("col1", DataType.BOOLEAN);
    schema.addColumn("col2", DataType.BYTE);
    schema.addColumn("col3", DataType.CHAR);
    schema.addColumn("col4", DataType.SHORT);
    schema.addColumn("col5", DataType.INT);
    schema.addColumn("col6", DataType.LONG);
    schema.addColumn("col7", DataType.FLOAT);
    schema.addColumn("col8", DataType.DOUBLE);
    schema.addColumn("col9", DataType.STRING);
    schema.addColumn("col10", DataType.BYTES);
    schema.addColumn("col11", DataType.IPv4);

    Options options = new Options();
    TableMeta meta = TCatUtil.newTableMeta(schema, StoreType.RAW, options);

    FileSystem fs = FileSystem.getLocal(conf);
    Path path = fs.makeQualified(new Path(TEST_PATH, "raw.dat"));

    RawFile.Appender appender = new RawFile.Appender(conf, meta, path);
    appender.init();//from   w  w  w.ja v  a 2 s.  co m

    Tuple tuple = new VTuple(11);
    tuple.put(new Datum[] { DatumFactory.createBool(true), DatumFactory.createByte((byte) 0x99),
            DatumFactory.createChar('7'), DatumFactory.createShort((short) 17), DatumFactory.createInt(59),
            DatumFactory.createLong(23l), DatumFactory.createFloat(77.9f), DatumFactory.createDouble(271.9f),
            DatumFactory.createString("hyunsik"), DatumFactory.createBytes("hyunsik".getBytes()),
            DatumFactory.createIPv4("192.168.0.1") });
    appender.addTuple(tuple);
    appender.flush();
    appender.close();

    RawFile.Scanner scanner = new RawFile.Scanner(conf, meta, path);
    scanner.init();
    Tuple retrieved = scanner.next();
    for (int i = 0; i < tuple.size(); i++) {
        assertEquals(tuple.get(i), retrieved.get(i));
    }
    scanner.close();
}

---

From source file:uk.ac.ebi.pride.spectracluster.hadoop.example.BaileyBorweinPlouffe.java

License:Apache License

/** Run a map/reduce job to compute Pi. */
private static void compute(int startDigit, int nDigits, int nMaps, String workingDir, Configuration conf,
        PrintStream out) throws IOException {
    final String name = startDigit + "_" + nDigits;

    //setup wroking directory
    out.println("Working Directory = " + workingDir);
    out.println();/*from  w w w.ja v  a 2  s  .co  m*/
    final FileSystem fs = FileSystem.get(conf);
    final Path dir = fs.makeQualified(new Path(workingDir));
    if (fs.exists(dir)) {
        throw new IOException("Working directory " + dir + " already exists.  Please remove it first.");
    } else if (!fs.mkdirs(dir)) {
        throw new IOException("Cannot create working directory " + dir);
    }

    out.println("Start Digit      = " + startDigit);
    out.println("Number of Digits = " + nDigits);
    out.println("Number of Maps   = " + nMaps);

    // setup a job
    final Job job = createJob(name, conf);
    final Path hexfile = new Path(dir, "pi_" + name + ".hex");
    FileOutputFormat.setOutputPath(job, new Path(dir, "out"));

    // setup custom properties
    job.getConfiguration().set(WORKING_DIR_PROPERTY, dir.toString());
    job.getConfiguration().set(HEX_FILE_PROPERTY, hexfile.toString());

    job.getConfiguration().setInt(DIGIT_START_PROPERTY, startDigit);
    job.getConfiguration().setInt(DIGIT_SIZE_PROPERTY, nDigits);
    job.getConfiguration().setInt(DIGIT_PARTS_PROPERTY, nMaps);

    // start a map/reduce job
    out.println("\nStarting Job ...");
    final long startTime = System.currentTimeMillis();
    try {
        if (!job.waitForCompletion(true)) {
            out.println("Job failed.");
            System.exit(1);
        }
    } catch (Exception e) {
        throw new RuntimeException(e);
    } finally {
        final double duration = (System.currentTimeMillis() - startTime) / 1000.0;
        out.println("Duration is " + duration + " seconds.");
    }
    out.println("Output file: " + hexfile);
}

---

• «
• 1
• 2
• 3
• 4
• 5
• 6
• 7
• 8
• 9
• 10
• 11
• 12
• 13
• 14
• 15
• 16
• 17
• 18
• 19
• 20
• 21
• 22
• 23
• 24
• 25
• 26
• 27
• 28
• 29
• 30
• »