Example usage for org.apache.hadoop.fs FileSystem makeQualified

Introduction

In this page you can find the example usage for org.apache.hadoop.fs FileSystem makeQualified.

Prototype

public Path makeQualified(Path path) 

Source Link

Document

Qualify a path to one which uses this FileSystem and, if relative, made absolute.

Usage

From source file:org.mrgeo.format.CsvInputFormatTest.java

License:Apache License

@Test
@Category(UnitTest.class)
public void testBadValues() throws Exception {
    // this class and its unit tests are a work in progress.
    FileSystem fs = new RawLocalFileSystem();
    try {//from   ww w. j  av  a 2  s  . co m
        Job j = new Job(new Configuration());
        Configuration c = j.getConfiguration();
        fs.setConf(c);
        Path testFile = new Path(input, "testErrors.csv");
        testFile = fs.makeQualified(testFile);

        FileInputFormat.addInputPath(j, testFile);
        FileSplit split = new FileSplit(testFile, 0, 500, null);
        CsvInputFormat.CsvRecordReader reader = new CsvInputFormat.CsvRecordReader();
        reader.initialize(split, HadoopUtils.createTaskAttemptContext(c, new TaskAttemptID()));
        @SuppressWarnings("unused")
        int l = 0;

        //StringBuffer buf = new StringBuffer();

        // Test specific rows returned to make sure the values are as expected.
        Assert.assertTrue(reader.nextKeyValue());
        Geometry f = reader.getCurrentValue();
        Assert.assertNotNull(f);
        Assert.assertTrue(f.isEmpty());
        Assert.assertEquals("test1,1,1.5,30.0,40.0", f.getAttribute("string1"));
        Assert.assertNull(f.getAttribute("int1"));
        Assert.assertNull(f.getAttribute("double1"));
        Assert.assertNull(f.getAttribute("x"));
        Assert.assertNull(f.getAttribute("y"));
        // Row 2 check
        Assert.assertTrue(reader.nextKeyValue());
        f = reader.getCurrentValue();
        Assert.assertNotNull(f);
        Assert.assertTrue(f.isEmpty());
        Assert.assertEquals("test2", f.getAttribute("string1"));
        Assert.assertEquals(2, Integer.parseInt(f.getAttribute("int1")));
        Assert.assertEquals("", f.getAttribute("double1"));
        Assert.assertEquals("30.abc", f.getAttribute("x"));
        Assert.assertEquals(40.0, Double.parseDouble(f.getAttribute("y")), EPSILON);
        // Row 3 check
        Assert.assertTrue(reader.nextKeyValue());
        f = reader.getCurrentValue();
        Assert.assertNotNull(f);
        Assert.assertTrue(f.isEmpty());
        Assert.assertEquals("test3", f.getAttribute("string1"));
        Assert.assertEquals(3, Integer.parseInt(f.getAttribute("int1")));
        Assert.assertEquals(3.5, Double.parseDouble(f.getAttribute("double1")), EPSILON);
        Assert.assertEquals(30.0, Double.parseDouble(f.getAttribute("x")), EPSILON);
        Assert.assertEquals("40.abc", f.getAttribute("y"));
        // Row 4 check
        Assert.assertTrue(reader.nextKeyValue());
        f = reader.getCurrentValue();
        Assert.assertNotNull(f);
        Assert.assertTrue(f.isEmpty());
        Assert.assertEquals("test4", f.getAttribute("string1"));
        Assert.assertEquals("", f.getAttribute("int1"));
        Assert.assertEquals(4.5, Double.parseDouble(f.getAttribute("double1")), EPSILON);
        Assert.assertEquals(30.0, Double.parseDouble(f.getAttribute("x")), EPSILON);
        Assert.assertNull(f.getAttribute("y"));
        // Row 5 check
        Assert.assertTrue(reader.nextKeyValue());
        f = reader.getCurrentValue();
        Assert.assertNotNull(f);
        Assert.assertTrue(f.isEmpty());
        Assert.assertEquals("test5", f.getAttribute("string1"));
        Assert.assertEquals(5, Integer.parseInt(f.getAttribute("int1")));
        Assert.assertEquals(5.5, Double.parseDouble(f.getAttribute("double1")), EPSILON);
        Assert.assertEquals("", f.getAttribute("x"));
        Assert.assertEquals(40.0, Double.parseDouble(f.getAttribute("y")), EPSILON);
        // Row 6 check
        Assert.assertTrue(reader.nextKeyValue());
        f = reader.getCurrentValue();
        Assert.assertNotNull(f);
        Assert.assertTrue(f.isEmpty());
        Assert.assertEquals("test6", f.getAttribute("string1"));
        Assert.assertEquals("", f.getAttribute("int1"));
        Assert.assertEquals("", f.getAttribute("double1"));
        Assert.assertEquals("", f.getAttribute("x"));
        Assert.assertEquals("", f.getAttribute("y"));

        // end
        Assert.assertFalse(reader.nextKeyValue());
    } catch (Exception e) {
        e.printStackTrace();
        throw e;
    } finally {
        fs.close();
    }
}

---

From source file:org.mrgeo.format.CsvOutputFormatTest.java

License:Apache License

@Test
@Category(UnitTest.class)
public void testBasics() throws Exception {
    // this class and its unit tests are a work in progress.
    FileSystem fs = new RawLocalFileSystem();
    try {//from   w w  w  .  j  a  va  2 s  . com
        String output = TestUtils.composeOutputDir(CsvOutputFormatTest.class);

        Configuration c = new Configuration();
        fs.setConf(c);
        Path testFile = new Path(output, "testBasics.csv");
        testFile = fs.makeQualified(testFile);
        Path columns = new Path(testFile.toString() + ".columns");

        CsvOutputFormat.CsvRecordWriter writer = new CsvOutputFormat.CsvRecordWriter(columns, testFile);

        WritableGeometry f = GeometryFactory.createEmptyGeometry();

        f.setAttribute("string1", "foo");
        f.setAttribute("int1", "1");
        f.setAttribute("double1", "2.0");
        writer.write(new LongWritable(0), f);

        f.setAttribute("string1", "bar");
        f.setAttribute("int1", "3");
        f.setAttribute("double1", "4.0");
        writer.write(new LongWritable(1), f);

        writer.close(null);

        String input = TestUtils.composeInputDir(CsvOutputFormatTest.class);

        File csvBaselineFile = new File(input, "testBasics.csv");
        File csvOutputFile = new File(output, "testBasics.csv");
        TestUtils.compareTextFiles(csvBaselineFile.getAbsoluteFile(), csvOutputFile.getAbsoluteFile());

        File columnsBaselineFile = new File(input, "testBasics.csv.columns");
        File columnsOutputFile = new File(output, "testBasics.csv.columns");

        TestUtils.compareTextFiles(columnsBaselineFile.getAbsoluteFile(), columnsOutputFile.getAbsoluteFile());
    } catch (Exception e) {
        e.printStackTrace();
        throw e;
    } finally {
        fs.close();
    }
}

---

From source file:org.mrgeo.format.OsmContentHandlerTest.java

License:Apache License

@Ignore
@Test/*from w  w  w  .  java 2 s.  com*/
public void OfftestBenchmark() throws Exception {
    // @TODO this class and its unit tests are a work in progress.
    FileSystem fs = new RawLocalFileSystem();
    try {
        String input = TestUtils.composeInputDir(OsmContentHandlerTest.class);

        Configuration c = new Configuration();
        c.set("xml.content.handler", OsmContentHandler.class.getCanonicalName());
        c.set("xml.pattern", "node");
        c.set("xml.root.tag", "osm");

        fs.setConf(c);
        Path testFile = new Path(input, "sample.osm");
        testFile = fs.makeQualified(testFile);

        c.set("xml.pattern", "place");

        FileSplit split = new FileSplit(testFile, 0, 64 * 1048576, null);
        RecordReader<LongWritable, Geometry> reader = new SaxInputFormat<LongWritable, Geometry>()
                .createRecordReader(split, HadoopUtils.createTaskAttemptContext(c, new TaskAttemptID()));

        int l = 0;
        long start = new Date().getTime();
        while (reader.nextKeyValue()) {
            l++;
        }
        long elapsed = new Date().getTime() - start;
        log.debug("ms per record: {} record count: {}", (double) elapsed / (double) l, l);
        Assert.assertEquals(1, l);
    } catch (Exception e) {
        e.printStackTrace();
        throw e;
    } finally {
        fs.close();
    }
}

---

From source file:org.mrgeo.format.OsmContentHandlerTest.java

License:Apache License

@Ignore
@Test/*from   w  ww.j  a  va  2 s  .  c  om*/
public void OfftestBenchmark2() throws Exception {
    // @TODO this class and its unit tests are a work in progress.
    try {

        Configuration c = new Configuration();
        c.set("xml.content.handler", OsmContentHandler.class.getCanonicalName());
        c.set("xml.pattern", "node");
        c.set("xml.root.tag", "osm");

        FileSystem fs = HadoopFileUtils.getFileSystem();
        fs.setConf(c);
        Path testFile = new Path("/user/jason.surratt/", "georgia.osm");
        testFile = fs.makeQualified(testFile);

        c.set("xml.pattern", "place");

        FileSplit split = new FileSplit(testFile, 0, fs.getFileStatus(testFile).getLen(), null);
        RecordReader<LongWritable, Geometry> reader = new SaxInputFormat<LongWritable, Geometry>()
                .createRecordReader(split, HadoopUtils.createTaskAttemptContext(c, new TaskAttemptID()));

        FileOutputStream fos = new FileOutputStream("georgia-points.txt");
        BufferedWriter wr = new BufferedWriter(new OutputStreamWriter(fos));

        Formatter formatter = new Formatter(wr, Locale.US);

        int l = 0;
        long start = new Date().getTime();
        while (reader.nextKeyValue() && l < 10000) {
            l++;
            Geometry f = reader.getCurrentValue();
            if (f instanceof Point) {
                Point p = (Point) f;
                formatter.format("%.7f %.7f\n", p.getX(), p.getY());
            }
        }

        formatter.close();

        long elapsed = new Date().getTime() - start;
        log.debug("ms per record: {} record count: {}", (double) elapsed / (double) l, l);
    } catch (Exception e) {
        e.printStackTrace();
        throw e;
    }
}

---

From source file:org.mrgeo.format.OsmInputFormatTest.java

License:Apache License

@Test
@Category(UnitTest.class)
public void testBasics() throws Exception {
    // this class and its unit tests are a work in progress.
    FileSystem fs = new RawLocalFileSystem();
    try {/*from   ww  w  . ja  v  a2  s. c  o m*/
        String input = TestUtils.composeInputDir(OsmInputFormatTest.class);

        Configuration c = new Configuration();
        fs.setConf(c);
        Path testFile = new Path(input, "sample.osm");
        testFile = fs.makeQualified(testFile);

        c.set("xml.pattern", "place");

        FileSplit split = new FileSplit(testFile, 0, fs.getFileStatus(testFile).getLen(), null);
        OsmInputFormat.OsmRecordReader reader = new OsmInputFormat.OsmRecordReader();
        reader.initialize(split, HadoopUtils.createTaskAttemptContext(c, new TaskAttemptID()));
        int l = 0;
        while (reader.nextKeyValue() && l < 10000) {
            l++;
        }
        Assert.assertEquals(6, l);
    } catch (Exception e) {
        e.printStackTrace();
        throw e;
    } finally {
        fs.close();
    }
}

---

From source file:org.mrgeo.format.PgQueryInputFormatTest.java

License:Apache License

public RecordReader<LongWritable, Geometry> openReader(Path p) throws IOException {
    FileSystem fs = new RawLocalFileSystem();
    try {//w  w  w  .ja v a  2s. com
        Job j = new Job(new Configuration());
        Configuration c = j.getConfiguration();
        fs.setConf(c);
        Path testFile = fs.makeQualified(p);

        c.set("mapred.input.dir", testFile.toString());
        PgQueryInputFormat format = new PgQueryInputFormat();
        HadoopVectorUtils.setupPgQueryInputFormat(j, "anthro", "anthro4server",
                "jdbc:postgresql://localhost:5432/anthro");
        InputSplit split = null;
        try {
            split = format.getSplits(j).get(0);
            return format.createRecordReader(split,
                    HadoopUtils.createTaskAttemptContext(c, new TaskAttemptID()));
        } catch (Exception e) {
            return null;
        }
    } finally {
        fs.close();
    }
}

---

From source file:org.mrgeo.format.ShpInputFormatTest.java

License:Apache License

public RecordReader<LongWritable, Geometry> openReader(Path p) throws IOException, InterruptedException {
    Job j = new Job(new Configuration());
    Configuration c = j.getConfiguration();
    FileSystem fs = new RawLocalFileSystem();
    try {//  w  w  w  .  jav a 2s.com
        fs.setConf(c);
        Path testFile = fs.makeQualified(p);

        c.set("mapred.input.dir", testFile.toString());
        ShpInputFormat format = new ShpInputFormat();
        InputSplit split = format.getSplits(j).get(0);
        return format.createRecordReader(split, HadoopUtils.createTaskAttemptContext(c, new TaskAttemptID()));
    } finally {
        fs.close();
    }
}

---

From source file:org.mrgeo.format.XmlInputFormatTest.java

License:Apache License

@Test
@Category(UnitTest.class)
public void testBasics() throws Exception {
    // this class and its unit tests are a work in progress.
    FileSystem fs = new RawLocalFileSystem();
    try {/*from   w w w .  j a  v  a2  s.  c  o m*/
        String input = TestUtils.composeInputDir(XmlInputFormatTest.class);

        Configuration c = new Configuration();
        fs.setConf(c);
        Path testFile = new Path(input, "testBasics.xml");
        testFile = fs.makeQualified(testFile);

        c.set("xml.pattern", "node");

        FileSplit split = new FileSplit(testFile, 0, 50, null);
        XmlInputFormat.XmlRecordReader reader = new XmlInputFormat.XmlRecordReader();
        reader.initialize(split, HadoopUtils.createTaskAttemptContext(c, new TaskAttemptID()));
        int l = 0;
        while (reader.nextKeyValue()) {
            System.out.printf("k: %s v: %s\n", reader.getCurrentKey(), reader.getCurrentValue());
            l++;
        }

        split = new FileSplit(testFile, 50, fs.getFileStatus(testFile).getLen() - 50, null);
        reader = new XmlInputFormat.XmlRecordReader();
        reader.initialize(split, HadoopUtils.createTaskAttemptContext(c, new TaskAttemptID()));
        while (reader.nextKeyValue()) {
            System.out.printf("k: %s v: %s\n", reader.getCurrentKey(), reader.getCurrentValue());
            l++;
        }

        Assert.assertEquals(3, l);
    } catch (Exception e) {
        e.printStackTrace();
        throw e;
    } finally {
        fs.close();
    }
}

---

From source file:org.mrgeo.hdfs.vector.CsvOutputFormatTest.java

License:Apache License

@Test
@Category(UnitTest.class)
public void testBasics() throws Exception {
    // this class and its unit tests are a work in progress.
    FileSystem fs = new RawLocalFileSystem();
    try {//  w  ww.  j a v  a  2  s .  c  om
        String output = TestUtils.composeOutputDir(CsvOutputFormatTest.class);

        Configuration c = new Configuration();
        fs.setConf(c);
        Path testFile = new Path(output, "testBasics.csv");
        testFile = fs.makeQualified(testFile);
        Path columns = new Path(testFile.toString() + ".columns");

        CsvOutputFormat.CsvRecordWriter writer = new CsvOutputFormat.CsvRecordWriter(columns, testFile);

        WritableGeometry f = GeometryFactory.createEmptyGeometry();

        f.setAttribute("string1", "foo");
        f.setAttribute("int1", "1");
        f.setAttribute("double1", "2.0");
        writer.write(new FeatureIdWritable(0), f);

        f.setAttribute("string1", "bar");
        f.setAttribute("int1", "3");
        f.setAttribute("double1", "4.0");
        writer.write(new FeatureIdWritable(1), f);

        writer.close(null);

        String input = TestUtils.composeInputDir(CsvOutputFormatTest.class);

        File csvBaselineFile = new File(input, "testBasics.csv");
        File csvOutputFile = new File(output, "testBasics.csv");
        TestUtils.compareTextFiles(csvBaselineFile.getAbsoluteFile(), csvOutputFile.getAbsoluteFile());

        File columnsBaselineFile = new File(input, "testBasics.csv.columns");
        File columnsOutputFile = new File(output, "testBasics.csv.columns");

        TestUtils.compareTextFiles(columnsBaselineFile.getAbsoluteFile(), columnsOutputFile.getAbsoluteFile());
    } catch (Exception e) {
        e.printStackTrace();
        throw e;
    } finally {
        fs.close();
    }
}

---

From source file:org.mrgeo.utils.DependencyLoader.java

License:Apache License

@SuppressFBWarnings(value = "PATH_TRAVERSAL_IN", justification = "File used for addinj to HDFS classpath")
public static Set<String> copyDependencies(final Set<String> localDependencies, Configuration conf)
        throws IOException {
    if (conf == null) {
        conf = HadoopUtils.createConfiguration();
    }/*  ww  w . jav  a2 s. c o m*/

    FileSystem fs = HadoopFileUtils.getFileSystem(conf);
    Path hdfsBase = new Path(MrGeoProperties.getInstance()
            .getProperty(MrGeoConstants.MRGEO_HDFS_DISTRIBUTED_CACHE, "/mrgeo/jars"));

    Set<String> deps = new HashSet<>();

    // prime the set with any dependencies already added
    deps.addAll(conf.getStringCollection("mapreduce.job.classpath.files"));
    //    deps.addAll(conf.getStringCollection(MRJobConfig.CLASSPATH_FILES));

    // copy the dependencies to hdfs, if needed
    for (String local : localDependencies) {
        File file = new File(local);

        addFileToClasspath(conf, deps, fs, hdfsBase, file);
    }

    Set<String> qualified = new HashSet<>();

    // fully qualify the dependency
    for (String dep : deps) {
        qualified.add(fs.makeQualified(new Path(dep)).toString());
    }

    return qualified;

}

---

• «
• 1
• 2
• 3
• 4
• 5
• 6
• 7
• 8
• 9
• 10
• 11
• 12
• 13
• 14
• 15
• 16
• 17
• 18
• 19
• 20
• 21
• 22
• 23
• 24
• 25
• 26
• 27
• 28
• 29
• 30
• »