Example usage for org.apache.hadoop.fs FileSystem makeQualified

Introduction

In this page you can find the example usage for org.apache.hadoop.fs FileSystem makeQualified.

Prototype

public Path makeQualified(Path path) 

Source Link

Document

Qualify a path to one which uses this FileSystem and, if relative, made absolute.

Usage

From source file:com.hdfs.concat.crush.integration.CrushMapReduceTest.java

License:Apache License

@Before
@Override//from   www.j  av a 2  s.c o  m
public void setUp() throws Exception {
    super.setUp();

    job = createJobConf();

    job.setBoolean("mapred.output.compress", true);
    job.set("mapred.output.compression.type", CompressionType.BLOCK.name());
    job.set("mapred.output.compression.codec", CustomCompressionCodec.class.getName());

    FileSystem fs = getFileSystem();

    Path homeDirPath = fs.makeQualified(new Path("."));

    homeDir = homeDirPath.toUri().getPath();

    fs.delete(homeDirPath, true);

    defaultCodec = new DefaultCodec();
    defaultCodec.setConf(job);

    customCodec = new CustomCompressionCodec();
    customCodec.setConf(job);
}

---

From source file:com.inmobi.grill.driver.hive.TestHiveDriver.java

License:Apache License

private void validatePersistentResult(GrillResultSet resultSet, String dataFile, String outptuDir,
        boolean formatNulls) throws Exception {
    assertTrue(resultSet instanceof HivePersistentResultSet);
    HivePersistentResultSet persistentResultSet = (HivePersistentResultSet) resultSet;
    String path = persistentResultSet.getOutputPath();
    QueryHandle handle = persistentResultSet.getQueryHandle();

    Path actualPath = new Path(path);
    FileSystem fs = actualPath.getFileSystem(conf);
    assertEquals(actualPath, fs.makeQualified(new Path(outptuDir, handle.toString())));
    List<String> actualRows = new ArrayList<String>();
    for (FileStatus stat : fs.listStatus(actualPath)) {
        FSDataInputStream in = fs.open(stat.getPath());
        BufferedReader br = null;
        try {// w  ww .j  a  v a  2s . co  m
            br = new BufferedReader(new InputStreamReader(in));
            String line = "";

            while ((line = br.readLine()) != null) {
                System.out.println("Actual:" + line);
                actualRows.add(line.trim());
            }
        } finally {
            if (br != null) {
                br.close();
            }
        }
    }

    BufferedReader br = null;
    List<String> expectedRows = new ArrayList<String>();

    try {
        br = new BufferedReader(new FileReader(new File(dataFile)));
        String line = "";
        while ((line = br.readLine()) != null) {
            String row = line.trim();
            if (formatNulls) {
                row += ",-NA-,";
                row += line.trim();
            }
            expectedRows.add(row);
        }
    } finally {
        if (br != null) {
            br.close();
        }
    }
    assertEquals(actualRows, expectedRows);
}

---

From source file:com.m6d.filecrush.crush.integration.CrushMapReduceTest.java

License:Apache License

@Before
@Override/*from  w w  w.j a v  a  2s.c om*/
public void setUp() throws Exception {
    super.setUp();

    job = createJobConf();

    job.setBoolean("mapreduce.output.fileoutputformat.compress", true);
    job.set("mapreduce.output.fileoutputformat.compress.type", CompressionType.BLOCK.name());
    job.set("mapreduce.output.fileoutputformat.compress.codec", CustomCompressionCodec.class.getName());

    FileSystem fs = getFileSystem();

    Path homeDirPath = fs.makeQualified(new Path("."));

    homeDir = homeDirPath.toUri().getPath();

    fs.delete(homeDirPath, true);

    defaultCodec = new DefaultCodec();
    defaultCodec.setConf(job);

    customCodec = new CustomCompressionCodec();
    customCodec.setConf(job);
}

---

From source file:com.moz.fiji.mapreduce.DistributedCacheJars.java

License:Apache License

/**
 * Lists all jars in the specified directory.
 *
 * @param conf Configuration to get FileSystem from
 * @param jarDirectory The directory of jars to get.
 * @return A list of qualified paths to the jars in jarDirectory.
 * @throws IOException if there's a problem.
 */// ww  w .  ja  v a 2  s . c  o  m
public static Collection<Path> listJarFilesFromDirectory(Configuration conf, Path jarDirectory)
        throws IOException {
    LOG.debug("Listing jar files {}/*.jar", jarDirectory);
    final FileSystem fs = jarDirectory.getFileSystem(conf);
    if (!fs.isDirectory(jarDirectory)) {
        throw new IOException("Attempted to add jars from non-directory: " + jarDirectory);
    }
    final List<Path> jarFiles = Lists.newArrayList();
    for (FileStatus status : fs.listStatus(jarDirectory)) {
        if (!status.isDir() && status.getPath().getName().endsWith(".jar")) {
            jarFiles.add(fs.makeQualified(status.getPath()));
        }
    }
    return jarFiles;
}

---

From source file:com.moz.fiji.mapreduce.IntegrationTestFijiTableInputFormat.java

License:Apache License

public Job setupJob(String jobName, Path outputFile, Class<? extends Mapper> mapperClass,
        Class<? extends Reducer> reducerClass, EntityId startKey, EntityId limitKey, FijiRowFilter filter)
        throws Exception {
    final Job job = new Job(createConfiguration());
    final Configuration conf = job.getConfiguration();

    // Get settings for test.
    final FijiDataRequest request = FijiDataRequest.builder()
            .addColumns(ColumnsDef.create().add("info", "name").add("info", "email")).build();

    job.setJarByClass(IntegrationTestFijiTableInputFormat.class);

    // Setup the InputFormat.
    FijiTableInputFormat.configureJob(job, getFooTable().getURI(), request, startKey, limitKey, filter);
    job.setInputFormatClass(HBaseFijiTableInputFormat.class);

    // Duplicate functionality from MapReduceJobBuilder, since we are not using it here:
    final List<Path> jarFiles = Lists.newArrayList();
    final FileSystem fs = FileSystem.getLocal(conf);
    for (String cpEntry : System.getProperty("java.class.path").split(":")) {
        if (cpEntry.endsWith(".jar")) {
            jarFiles.add(fs.makeQualified(new Path(cpEntry)));
        }/*from  ww  w .  ja  v a  2s .  c  o m*/
    }
    DistributedCacheJars.addJarsToDistributedCache(job, jarFiles);

    // Create a test job.
    job.setJobName(jobName);

    // Setup the OutputFormat.
    TextOutputFormat.setOutputPath(job, outputFile.getParent());
    job.setOutputKeyClass(Text.class);
    job.setOutputValueClass(Text.class);
    job.setOutputFormatClass(TextOutputFormat.class);

    // Set the mapper class.
    if (null != mapperClass) {
        job.setMapperClass(mapperClass);
    }
    // Set the reducer class.
    if (null != reducerClass) {
        job.setReducerClass(reducerClass);
    }

    return job;
}

---

From source file:com.mycompany.MyHadoopSamples1.TransposeJob.java

License:Apache License

public static Configuration buildTransposeJobConf(Configuration initialConf, Path matrixInputPath,
        Path matrixOutputPath, int numInputRows) throws IOException {
    JobConf conf = new JobConf(initialConf, TransposeJob.class);
    conf.setJobName("TransposeJob: " + matrixInputPath + " transpose -> " + matrixOutputPath);
    FileSystem fs = FileSystem.get(conf);
    matrixInputPath = fs.makeQualified(matrixInputPath);
    matrixOutputPath = fs.makeQualified(matrixOutputPath);
    conf.setInt(NUM_ROWS_KEY, numInputRows);

    FileInputFormat.addInputPath(conf, matrixInputPath);
    conf.setInputFormat(SequenceFileInputFormat.class);
    FileOutputFormat.setOutputPath(conf, matrixOutputPath);
    System.out.println("OUTPUT --> " + matrixOutputPath.toString());
    conf.setMapperClass(TransposeMapper.class);
    conf.setMapOutputKeyClass(IntWritable.class);
    conf.setMapOutputValueClass(VectorWritable.class);
    conf.setCombinerClass(MergeVectorsCombiner.class);
    conf.setReducerClass(MergeVectorsReducer.class);
    conf.setOutputFormat(SequenceFileOutputFormat.class);
    conf.setOutputKeyClass(IntWritable.class);
    conf.setOutputValueClass(VectorWritable.class);
    return conf;/* w w w .j a va  2 s. c  o m*/
}

---

From source file:com.ngdata.hbaseindexer.mr.HBaseMapReduceIndexerToolGoLiveTest.java

License:Apache License

@BeforeClass
public static void setupBeforeClass() throws Exception {
    MR_TEST_UTIL = new MRTestUtil(HBASE_TEST_UTILITY);
    HBASE_TEST_UTILITY.startMiniCluster();
    MR_TEST_UTIL.startMrCluster();//from w  ww  . j  a v a  2 s .co m

    FileSystem fs = FileSystem.get(HBASE_TEST_UTILITY.getConfiguration());
    int zkClientPort = HBASE_TEST_UTILITY.getZkCluster().getClientPort();

    SOLR_TEST_UTILITY = new SolrTestingUtility(zkClientPort, NetUtils.getFreePort(),
            ImmutableMap.of("solr.hdfs.blockcache.enabled", "false", "solr.directoryFactory",
                    "HdfsDirectoryFactory", "solr.hdfs.home",
                    fs.makeQualified(new Path("/solrdata")).toString()));
    SOLR_TEST_UTILITY.start();

    SOLR_TEST_UTILITY.uploadConfig("config1", new File(MINIMR_CONF_DIR, "conf"));
    SOLR_TEST_UTILITY.createCollection("collection1", "config1", 2);
    SOLR_TEST_UTILITY.createCollection("collection2", "config1", 2);

    COLLECTION1 = new CloudSolrServer(SOLR_TEST_UTILITY.getZkConnectString());
    COLLECTION1.setDefaultCollection("collection1");

    SOLR_ZK = "127.0.0.1:" + zkClientPort + "/solr";
    INDEXER_ZK = "localhost:" + zkClientPort;
    ZooKeeperItf zkItf = ZkUtil.connect(INDEXER_ZK, 15000);
    INDEXER_MODEL = new IndexerModelImpl(zkItf, "/ngdata/hbaseindexer");
    IndexerDefinition indexerDef = new IndexerDefinitionBuilder().name("zkindexerdef")
            .indexerComponentFactory(DefaultIndexerComponentFactory.class.getName())
            .configuration(Resources.toByteArray(
                    Resources.getResource(HBaseMapReduceIndexerToolGoLiveTest.class, "user_indexer.xml")))
            .connectionParams(ImmutableMap.of("solr.zk", SOLR_ZK, "solr.collection", "collection1")).build();

    addAndWaitForIndexer(indexerDef);

    Closer.close(zkItf);

    HTableDescriptor tableDescriptor = new HTableDescriptor(TEST_TABLE_NAME);
    tableDescriptor.addFamily(new HColumnDescriptor(TEST_COLFAM_NAME));
    HBASE_ADMIN = new HBaseAdmin(HBASE_TEST_UTILITY.getConfiguration());
    HBASE_ADMIN.createTable(tableDescriptor,
            new byte[][] { Bytes.toBytes("row0800"), Bytes.toBytes("row1600") });

    RECORD_TABLE = new HTable(HBASE_TEST_UTILITY.getConfiguration(), TEST_TABLE_NAME);

    for (int i = 0; i < RECORD_COUNT; i++) {
        writeHBaseRecord(String.format("row%04d", i), ImmutableMap.of("firstname", String.format("John%04d", i),
                "lastname", String.format("Doe%04d", i)));
    }

}

---

From source file:com.ngdata.hbaseindexer.mr.HBaseMapReduceIndexerToolTest.java

License:Apache License

@Test
public void testIndexer_SingleShard() throws Exception {

    FileSystem fs = FileSystem.get(HBASE_TEST_UTILITY.getConfiguration());
    MR_TEST_UTIL.runTool("--hbase-indexer-file",
            new File(Resources.getResource(getClass(), "user_indexer.xml").toURI()).toString(),
            "--solr-home-dir", MINIMR_CONF_DIR.toString(), "--output-dir",
            fs.makeQualified(new Path("/solroutput")).toString(), "--shards", "1", "--overwrite-output-dir");

    ForkedTestUtils.validateSolrServerDocumentCount(MINIMR_CONF_DIR,
            FileSystem.get(HBASE_TEST_UTILITY.getConfiguration()), new Path("/solroutput", "results"),
            RECORD_COUNT, 1);//from ww  w  .j av a  2 s. c  o m

}

---

From source file:com.ngdata.hbaseindexer.mr.HBaseMapReduceIndexerToolTest.java

License:Apache License

@Test
public void testIndexer_MultipleShards() throws Exception {

    FileSystem fs = FileSystem.get(HBASE_TEST_UTILITY.getConfiguration());
    MR_TEST_UTIL.runTool("--hbase-indexer-file",
            new File(Resources.getResource(getClass(), "user_indexer.xml").toURI()).toString(),
            "--solr-home-dir", MINIMR_CONF_DIR.toString(), "--output-dir",
            fs.makeQualified(new Path("/solroutput")).toString(), "--shards", "3", "--overwrite-output-dir");

    ForkedTestUtils.validateSolrServerDocumentCount(MINIMR_CONF_DIR,
            FileSystem.get(HBASE_TEST_UTILITY.getConfiguration()), new Path("/solroutput", "results"),
            RECORD_COUNT, 3);/*from   w ww .  j av a2s  .c o m*/

}

---

From source file:com.ngdata.hbaseindexer.mr.HBaseMapReduceIndexerToolTest.java

License:Apache License

@Test
public void testIndexer_Morphlines() throws Exception {

    FileSystem fs = FileSystem.get(HBASE_TEST_UTILITY.getConfiguration());
    MR_TEST_UTIL.runTool("--hbase-indexer-file",
            new File(Resources.getResource("morphline_indexer_without_zk.xml").toURI()).toString(),
            "--solr-home-dir", MINIMR_CONF_DIR.toString(), "--output-dir",
            fs.makeQualified(new Path("/solroutput")).toString(), "--shards", "2", "--reducers", "8",
            "--fanout", "2", "--morphline-file",
            new File(Resources.getResource("extractHBaseCellWithoutZk.conf").toURI()).toString(),
            "--overwrite-output-dir", "--hbase-table-name", "record", "--verbose", "--log4j",
            new File(Resources.getResource("log4j-base.properties").toURI()).toString());

    ForkedTestUtils.validateSolrServerDocumentCount(MINIMR_CONF_DIR,
            FileSystem.get(HBASE_TEST_UTILITY.getConfiguration()), new Path("/solroutput", "results"),
            RECORD_COUNT, 2);/* w  ww  . java 2  s.c om*/

}

---

• «
• 1
• 2
• 3
• 4
• 5
• 6
• 7
• 8
• 9
• 10
• 11
• 12
• 13
• 14
• 15
• 16
• 17
• 18
• 19
• 20
• 21
• 22
• 23
• 24
• 25
• 26
• 27
• 28
• 29
• 30
• »