Example usage for org.apache.hadoop.fs FileSystem makeQualified

Introduction

In this page you can find the example usage for org.apache.hadoop.fs FileSystem makeQualified.

Prototype

public Path makeQualified(Path path) 

Source Link

Document

Qualify a path to one which uses this FileSystem and, if relative, made absolute.

Usage

From source file:org.apache.crunch.io.parquet.AvroParquetFileReaderFactory.java

License:Apache License

@Override
@SuppressWarnings("unchecked")
public Iterator<T> read(FileSystem fs, Path path) {
    Path p = fs.makeQualified(path);
    final ParquetReader reader;
    try {/*  ww  w.  j a va  2 s .  co  m*/
        reader = new ParquetReader(p, new CrunchAvroReadSupport(avroType));
    } catch (IOException e) {
        throw new CrunchRuntimeException(e);
    }
    return new AutoClosingIterator<T>(reader, new UnmodifiableIterator<T>() {

        private T next;

        @Override
        public boolean hasNext() {
            if (next != null) {
                return true;
            }
            try {
                next = (T) reader.read();
            } catch (IOException e) {
                throw new CrunchRuntimeException(e);
            }
            return next != null;
        }

        @Override
        public T next() {
            if (hasNext()) {
                T ret = next;
                next = null;
                return ret;
            }
            throw new NoSuchElementException();
        }
    });

}

---

From source file:org.apache.falcon.entity.FileSystemStorage.java

License:Apache License

private void fileSystemEvictor(String feedPath, String retentionLimit, TimeZone timeZone, Path logFilePath)
        throws IOException, ELException, FalconException {
    Path normalizedPath = new Path(feedPath);
    FileSystem fs = HadoopClientFactory.get().createProxiedFileSystem(normalizedPath.toUri());
    feedPath = normalizedPath.toUri().getPath();
    LOG.info("Normalized path: {}", feedPath);

    Pair<Date, Date> range = EvictionHelper.getDateRange(retentionLimit);

    List<Path> toBeDeleted = discoverInstanceToDelete(feedPath, timeZone, range.first, fs);
    if (toBeDeleted.isEmpty()) {
        LOG.info("No instances to delete.");
        return;/*  w w  w. j a va  2s.c om*/
    }

    DateFormat dateFormat = new SimpleDateFormat(FeedHelper.FORMAT);
    dateFormat.setTimeZone(timeZone);
    Path feedBasePath = fs.makeQualified(FeedHelper.getFeedBasePath(feedPath));
    for (Path path : toBeDeleted) {
        deleteInstance(fs, path, feedBasePath);
        Date date = FeedHelper.getDate(feedPath, new Path(path.toUri().getPath()), timeZone);
        instanceDates.append(dateFormat.format(date)).append(',');
        instancePaths.append(path).append(EvictedInstanceSerDe.INSTANCEPATH_SEPARATOR);
    }
}

---

From source file:org.apache.falcon.service.SharedLibraryHostingService.java

License:Apache License

private void pushExtensionArtifactsToCluster(final Cluster cluster, final FileSystem clusterFs)
        throws FalconException {
    if (!Services.get().isRegistered(ExtensionService.SERVICE_NAME)) {
        LOG.info("ExtensionService not registered, return");
        return;/* w  w  w  .  ja  v  a2s .c  om*/
    }

    ExtensionStore store = ExtensionStore.get();
    if (!store.isExtensionStoreInitialized()) {
        LOG.info(
                "Extension store not initialized by Extension service. Make sure Extension service is added in "
                        + "start up properties");
        return;
    }

    final String filterPath = "/apps/falcon/extensions/mirroring/";
    Path extensionStorePath = store.getExtensionStorePath();
    LOG.info("extensionStorePath :{}", extensionStorePath);
    FileSystem falconFileSystem = HadoopClientFactory.get().createFalconFileSystem(extensionStorePath.toUri());
    String nameNode = StringUtils
            .removeEnd(falconFileSystem.getConf().get(HadoopClientFactory.FS_DEFAULT_NAME_KEY), File.separator);

    String clusterStorageUrl = StringUtils.removeEnd(ClusterHelper.getStorageUrl(cluster), File.separator);

    // If default fs for Falcon server is same as cluster fs abort copy
    if (nameNode.equalsIgnoreCase(clusterStorageUrl)) {
        LOG.info("clusterStorageUrl :{} same return", clusterStorageUrl);
        return;
    }

    try {
        RemoteIterator<LocatedFileStatus> fileStatusListIterator = falconFileSystem
                .listFiles(extensionStorePath, true);

        while (fileStatusListIterator.hasNext()) {
            LocatedFileStatus srcfileStatus = fileStatusListIterator.next();
            Path filePath = Path.getPathWithoutSchemeAndAuthority(srcfileStatus.getPath());

            if (filePath != null && filePath.toString().startsWith(filterPath)) {
                /* HiveDR uses filter path as store path in DRStatusStore, so skip it. Copy only the extension
                 artifacts */
                continue;
            }

            if (srcfileStatus.isDirectory()) {
                if (!clusterFs.exists(filePath)) {
                    HadoopClientFactory.mkdirs(clusterFs, filePath, srcfileStatus.getPermission());
                }
            } else {
                if (clusterFs.exists(filePath)) {
                    FileStatus targetfstat = clusterFs.getFileStatus(filePath);
                    if (targetfstat.getLen() == srcfileStatus.getLen()) {
                        continue;
                    }
                }

                Path parentPath = filePath.getParent();
                if (!clusterFs.exists(parentPath)) {
                    FsPermission dirPerm = falconFileSystem.getFileStatus(parentPath).getPermission();
                    HadoopClientFactory.mkdirs(clusterFs, parentPath, dirPerm);
                }

                FileUtil.copy(falconFileSystem, srcfileStatus, clusterFs, filePath, false, true,
                        falconFileSystem.getConf());
                FileUtil.chmod(clusterFs.makeQualified(filePath).toString(),
                        srcfileStatus.getPermission().toString());
            }
        }
    } catch (IOException | InterruptedException e) {
        throw new FalconException("Failed to copy extension artifacts to cluster" + cluster.getName(), e);
    }
}

---

From source file:org.apache.flink.tez.client.TezExecutor.java

License:Apache License

private static void addLocalResource(TezConfiguration tezConf, Path jarPath, DAG dag) {

    try {//from  w  ww .  ja va  2  s  .  co m
        org.apache.hadoop.fs.FileSystem fs = org.apache.hadoop.fs.FileSystem.get(tezConf);

        LOG.info("Jar path received is " + jarPath.toString());

        String jarFile = jarPath.getName();

        Path remoteJarPath = null;

        /*
        if (tezConf.get(TezConfiguration.TEZ_AM_STAGING_DIR) == null) {
           LOG.info("Tez staging directory is null, setting it.");
           Path stagingDir = new Path(fs.getWorkingDirectory(), UUID.randomUUID().toString());
           LOG.info("Setting Tez staging directory to " + stagingDir.toString());
           tezConf.set(TezConfiguration.TEZ_AM_STAGING_DIR, stagingDir.toString());
           LOG.info("Set Tez staging directory to " + stagingDir.toString());
        }
        Path stagingDir = new Path(tezConf.get(TezConfiguration.TEZ_AM_STAGING_DIR));
        LOG.info("Ensuring that Tez staging directory exists");
        TezClientUtils.ensureStagingDirExists(tezConf, stagingDir);
        LOG.info("Tez staging directory exists and is " + stagingDir.toString());
        */

        Path stagingDir = TezCommonUtils.getTezBaseStagingPath(tezConf);
        LOG.info("Tez staging path is " + stagingDir);
        TezClientUtils.ensureStagingDirExists(tezConf, stagingDir);
        LOG.info("Tez staging dir exists");

        remoteJarPath = fs.makeQualified(new Path(stagingDir, jarFile));
        LOG.info("Copying " + jarPath.toString() + " to " + remoteJarPath.toString());
        fs.copyFromLocalFile(jarPath, remoteJarPath);

        FileStatus remoteJarStatus = fs.getFileStatus(remoteJarPath);
        Credentials credentials = new Credentials();
        TokenCache.obtainTokensForNamenodes(credentials, new Path[] { remoteJarPath }, tezConf);

        Map<String, LocalResource> localResources = new TreeMap<String, LocalResource>();
        LocalResource jobJar = LocalResource.newInstance(ConverterUtils.getYarnUrlFromPath(remoteJarPath),
                LocalResourceType.FILE, LocalResourceVisibility.APPLICATION, remoteJarStatus.getLen(),
                remoteJarStatus.getModificationTime());
        localResources.put(jarFile.toString(), jobJar);

        dag.addTaskLocalFiles(localResources);

        LOG.info("Added job jar as local resource.");
    } catch (Exception e) {
        System.out.println(e.getMessage());
        e.printStackTrace();
        System.exit(-1);
    }
}

---

From source file:org.apache.giraph.io.hbase.TestHBaseRootMarkerVertextFormat.java

License:Apache License

@Test
public void testHBaseInputOutput() throws Exception {
    if (System.getProperty("prop.mapred.job.tracker") != null) {
        if (log.isInfoEnabled())
            log.info("testHBaseInputOutput: Ignore this test if not local mode.");
        return;/*from  w ww.  j  av  a 2  s  .co m*/
    }

    File jarTest = new File(System.getProperty("prop.jarLocation"));
    if (!jarTest.exists()) {
        fail("Could not find Giraph jar at " + "location specified by 'prop.jarLocation'. "
                + "Make sure you built the main Giraph artifact?.");
    }

    FileSystem fs = null;
    Path hbaseRootdir = null;
    try {
        MiniHBaseCluster cluster = testUtil.startMiniCluster(1);
        cluster.waitForActiveAndReadyMaster();
        testUtil.startMiniMapReduceCluster();

        // Let's set up the hbase root directory.
        Configuration conf = testUtil.getConfiguration();
        try {
            fs = testUtil.getTestFileSystem();
            String randomStr = UUID.randomUUID().toString();
            String tmpdir = System.getProperty("java.io.tmpdir") + "/" + randomStr + "/";
            hbaseRootdir = fs.makeQualified(new Path(tmpdir));

            conf.set(HConstants.HBASE_DIR, hbaseRootdir.toString());
            fs.mkdirs(hbaseRootdir);
        } catch (IOException ioe) {
            fail("Could not create hbase root directory.");
        }

        //First let's load some data using ImportTsv into our mock table.
        String INPUT_FILE = hbaseRootdir.toString() + "/graph.csv";
        String[] args = new String[] { "-Dimporttsv.columns=HBASE_ROW_KEY,cf:" + QUALIFER,
                "-Dimporttsv.separator=" + "\u002c", TABLE_NAME, INPUT_FILE };

        GenericOptionsParser opts = new GenericOptionsParser(testUtil.getConfiguration(), args);
        args = opts.getRemainingArgs();

        fs = FileSystem.get(conf);
        fs.setConf(conf);
        Path inputPath = fs.makeQualified(new Path(hbaseRootdir, "graph.csv"));
        FSDataOutputStream op = fs.create(inputPath, true);
        String line1 = "0001,0002\n";
        String line2 = "0002,0004\n";
        String line3 = "0003,0005\n";
        String line4 = "0004,-1\n";
        String line5 = "0005,-1\n";
        op.write(line1.getBytes());
        op.write(line2.getBytes());
        op.write(line3.getBytes());
        op.write(line4.getBytes());
        op.write(line5.getBytes());
        op.close();

        final byte[] FAM = Bytes.toBytes(FAMILY);
        final byte[] TAB = Bytes.toBytes(TABLE_NAME);

        HTableDescriptor desc = new HTableDescriptor(TAB);
        desc.addFamily(new HColumnDescriptor(FAM));
        HBaseAdmin hbaseAdmin = new HBaseAdmin(conf);
        if (hbaseAdmin.isTableAvailable(TABLE_NAME)) {
            hbaseAdmin.disableTable(TABLE_NAME);
            hbaseAdmin.deleteTable(TABLE_NAME);
        }
        hbaseAdmin.createTable(desc);

        // Do the import
        Job job = ImportTsv.createSubmittableJob(conf, args);
        job.waitForCompletion(false);
        assertTrue(job.isSuccessful());
        if (log.isInfoEnabled())
            log.info("ImportTsv successful. Running HBase Giraph job.");

        // Now operate over HBase using Vertex I/O formats
        conf.set(TableInputFormat.INPUT_TABLE, TABLE_NAME);
        conf.set(TableOutputFormat.OUTPUT_TABLE, TABLE_NAME);

        GiraphJob giraphJob = new GiraphJob(conf, BspCase.getCallingMethodName());
        GiraphConfiguration giraphConf = giraphJob.getConfiguration();
        setupConfiguration(giraphJob);
        giraphConf.setComputationClass(EdgeNotification.class);
        giraphConf.setVertexInputFormatClass(TableEdgeInputFormat.class);
        giraphConf.setVertexOutputFormatClass(TableEdgeOutputFormat.class);

        assertTrue(giraphJob.run(true));
        if (log.isInfoEnabled())
            log.info("Giraph job successful. Checking output qualifier.");

        // Do a get on row 0002, it should have a parent of 0001
        // if the outputFormat worked.
        HTable table = new HTable(conf, TABLE_NAME);
        Result result = table.get(new Get("0002".getBytes()));
        byte[] parentBytes = result.getValue(FAMILY.getBytes(), OUTPUT_FIELD.getBytes());
        assertNotNull(parentBytes);
        assertTrue(parentBytes.length > 0);
        assertEquals("0001", Bytes.toString(parentBytes));
    } finally {
        testUtil.shutdownMiniMapReduceCluster();
        testUtil.shutdownMiniCluster();
    }
}

---

From source file:org.apache.giraph.ranking.HostRankHBaseTest.java

License:Apache License

public HostRankHBaseTest() {
    super(HostRankHBaseTest.class.getName());

    // Let's set up the hbase root directory.
    Configuration conf = HBaseConfiguration.create();
    try {/*from w  w  w. j a v a  2  s .  co m*/
        FileSystem fs = FileSystem.get(conf);
        String randomStr = UUID.randomUUID().toString();
        String tmpdir = System.getProperty("java.io.tmpdir") + "/" + randomStr + "/";
        hbaseRootdir = fs.makeQualified(new Path(tmpdir));
        conf.set(HConstants.HBASE_DIR, hbaseRootdir.toString());
        fs.mkdirs(hbaseRootdir);
    } catch (IOException ioe) {
        fail("Could not create hbase root directory.");
    }

    // Start the test utility.
    testUtil = new HBaseTestingUtility(conf);
}

---

From source file:org.apache.giraph.ranking.LinkRankHBaseTest.java

License:Apache License

public LinkRankHBaseTest() {
    super(LinkRankHBaseTest.class.getName());

    // Let's set up the hbase root directory.
    Configuration conf = HBaseConfiguration.create();
    try {//from w w  w . ja  va2 s  .c o  m
        FileSystem fs = FileSystem.get(conf);
        String randomStr = UUID.randomUUID().toString();
        String tmpdir = System.getProperty("java.io.tmpdir") + "/" + randomStr + "/";
        hbaseRootdir = fs.makeQualified(new Path(tmpdir));
        conf.set(HConstants.HBASE_DIR, hbaseRootdir.toString());
        fs.mkdirs(hbaseRootdir);
    } catch (IOException ioe) {
        fail("Could not create hbase root directory.");
    }

    // Start the test utility.
    testUtil = new HBaseTestingUtility(conf);
}

---

From source file:org.apache.giraph.ranking.TrustRankHBaseTest.java

License:Apache License

public TrustRankHBaseTest() {
    super(org.apache.giraph.ranking.HostRankHBaseTest.class.getName());

    // Let's set up the hbase root directory.
    Configuration conf = HBaseConfiguration.create();
    try {//  www . j a v  a2s.  co m
        FileSystem fs = FileSystem.get(conf);
        String randomStr = UUID.randomUUID().toString();
        String tmpdir = System.getProperty("java.io.tmpdir") + "/" + randomStr + "/";
        hbaseRootdir = fs.makeQualified(new Path(tmpdir));
        conf.set(HConstants.HBASE_DIR, hbaseRootdir.toString());
        fs.mkdirs(hbaseRootdir);
    } catch (IOException ioe) {
        fail("Could not create hbase root directory.");
    }

    // Start the test utility.
    testUtil = new HBaseTestingUtility(conf);
}

---

From source file:org.apache.gobblin.config.store.hdfs.SimpleHdfsConfigureStoreFactoryTest.java

License:Apache License

@Test
public void testConfiguration() throws Exception {
    FileSystem localFS = FileSystem.getLocal(new Configuration());
    Path testRoot = localFS.makeQualified(new Path("testConfiguration"));
    Path configRoot = localFS.makeQualified(new Path(testRoot, "dir2"));
    Path configStoreRoot = new Path(configRoot, SimpleHadoopFilesystemConfigStore.CONFIG_STORE_NAME);
    Assert.assertTrue(localFS.mkdirs(configStoreRoot));
    try {// ww  w  .  j  a  va 2s  .co  m
        Config confConf1 = ConfigFactory.empty().withValue(SimpleHDFSConfigStoreFactory.DEFAULT_STORE_URI_KEY,
                ConfigValueFactory.fromAnyRef(configRoot.toString()));
        DefaultCapableLocalConfigStoreFactory confFactory = new DefaultCapableLocalConfigStoreFactory(
                confConf1);
        Assert.assertNotNull(confFactory.getDefaultStoreURI());
        Assert.assertEquals(confFactory.getDefaultStoreURI(), configRoot.toUri());
        Assert.assertEquals(confFactory.getPhysicalScheme(), "file");

        // Valid path
        SimpleHadoopFilesystemConfigStore store1 = confFactory.createConfigStore(new URI("default-file:/d"));
        Assert.assertEquals(store1.getStoreURI().getScheme(), confFactory.getScheme());
        Assert.assertEquals(store1.getStoreURI().getAuthority(),
                confFactory.getDefaultStoreURI().getAuthority());
        Assert.assertEquals(store1.getStoreURI().getPath(), confFactory.getDefaultStoreURI().getPath());

        // Invalid path
        Config confConf2 = ConfigFactory.empty().withValue(SimpleHDFSConfigStoreFactory.DEFAULT_STORE_URI_KEY,
                ConfigValueFactory.fromAnyRef(testRoot.toString()));
        try {
            new DefaultCapableLocalConfigStoreFactory(confConf2).getDefaultStoreURI();
            Assert.fail("Exception expected");
        } catch (IllegalArgumentException e) {
            Assert.assertTrue(e.getMessage().contains("is not a config store."));
        }

        // Empty path
        Config confConf3 = ConfigFactory.empty().withValue(SimpleHDFSConfigStoreFactory.DEFAULT_STORE_URI_KEY,
                ConfigValueFactory.fromAnyRef(""));
        try {
            new DefaultCapableLocalConfigStoreFactory(confConf3).getDefaultStoreURI();
            Assert.fail("Exception expected");
        } catch (IllegalArgumentException e) {
            Assert.assertTrue(e.getMessage().contains("Default store URI should be non-empty"));
        }
    } finally {
        localFS.delete(testRoot, true);
    }
}

---

From source file:org.apache.gobblin.data.management.copy.replication.ConfigBasedDataset.java

License:Apache License

private void calculateDatasetURN() {
    EndPoint e = this.copyRoute.getCopyTo();
    if (e instanceof HadoopFsEndPoint) {
        HadoopFsEndPoint copyTo = (HadoopFsEndPoint) e;
        Configuration conf = HadoopUtils.newConfiguration();
        try {//from  w  w  w.ja va 2s.  co  m
            FileSystem copyToFs = FileSystem.get(copyTo.getFsURI(), conf);
            this.datasetURN = copyToFs.makeQualified(copyTo.getDatasetPath()).toString();
        } catch (IOException e1) {
            // ignored
        }
    } else {
        this.datasetURN = e.toString();
    }
}

---

• «
• 1
• 2
• 3
• 4
• 5
• 6
• 7
• 8
• 9
• 10
• 11
• 12
• 13
• 14
• 15
• 16
• 17
• 18
• 19
• 20
• 21
• 22
• 23
• 24
• 25
• 26
• 27
• 28
• 29
• 30
• »