Example usage for org.apache.hadoop.fs FileSystem mkdirs

List of usage examples for org.apache.hadoop.fs FileSystem mkdirs

  1. HOME
  2. Java
  3. org.apache.hadoop
  4. org.apache.hadoop.fs.*
  5. FileSystem
  6. mkdirs

Introduction

In this page you can find the example usage for org.apache.hadoop.fs FileSystem mkdirs.

Prototype

public boolean mkdirs(Path f) throws IOException

Source Link

Document

Call #mkdirs(Path,FsPermission) with default permission.

Usage

From source file:com.inmobi.conduit.distcp.tools.TestGlobbedCopyListing.java

License:Apache License

private static void mkdirs(String path) throws Exception {
    FileSystem fileSystem = null;
    try {//from  w ww  .  j  av a  2s. c  o  m
        fileSystem = cluster.getFileSystem();
        fileSystem.mkdirs(new Path(path));
        recordInExpectedValues(path);
    } finally {
        IOUtils.cleanup(null, fileSystem);
    }
}

From source file:com.inmobi.conduit.distcp.tools.util.TestDistCpUtils.java

License:Apache License

@Test
public void testPreserve() {
    try {/*from w  w w.j  av a2 s  . c  o m*/
        FileSystem fs = FileSystem.get(config);
        EnumSet<FileAttribute> attributes = EnumSet.noneOf(FileAttribute.class);

        Path path = new Path("/tmp/abc");
        Path src = new Path("/tmp/src");
        fs.mkdirs(path);
        fs.mkdirs(src);
        FileStatus srcStatus = fs.getFileStatus(src);

        FsPermission noPerm = new FsPermission((short) 0);
        fs.setPermission(path, noPerm);
        fs.setOwner(path, "nobody", "nobody");

        DistCpUtils.preserve(fs, path, srcStatus, attributes);
        FileStatus target = fs.getFileStatus(path);
        Assert.assertEquals(target.getPermission(), noPerm);
        Assert.assertEquals(target.getOwner(), "nobody");
        Assert.assertEquals(target.getGroup(), "nobody");

        attributes.add(FileAttribute.PERMISSION);
        DistCpUtils.preserve(fs, path, srcStatus, attributes);
        target = fs.getFileStatus(path);
        Assert.assertEquals(target.getPermission(), srcStatus.getPermission());
        Assert.assertEquals(target.getOwner(), "nobody");
        Assert.assertEquals(target.getGroup(), "nobody");

        attributes.add(FileAttribute.GROUP);
        attributes.add(FileAttribute.USER);
        DistCpUtils.preserve(fs, path, srcStatus, attributes);
        target = fs.getFileStatus(path);
        Assert.assertEquals(target.getPermission(), srcStatus.getPermission());
        Assert.assertEquals(target.getOwner(), srcStatus.getOwner());
        Assert.assertEquals(target.getGroup(), srcStatus.getGroup());

        fs.delete(path, true);
        fs.delete(src, true);
    } catch (IOException e) {
        LOG.error("Exception encountered ", e);
        Assert.fail("Preserve test failure");
    }
}

From source file:com.inmobi.conduit.distcp.tools.util.TestDistCpUtils.java

License:Apache License

public static String createTestSetup(String baseDir, FileSystem fs, FsPermission perm) throws IOException {
    String base = getBase(baseDir);
    fs.mkdirs(new Path(base + "/newTest/hello/world1"));
    fs.mkdirs(new Path(base + "/newTest/hello/world2/newworld"));
    fs.mkdirs(new Path(base + "/newTest/hello/world3/oldworld"));
    fs.setPermission(new Path(base + "/newTest"), perm);
    fs.setPermission(new Path(base + "/newTest/hello"), perm);
    fs.setPermission(new Path(base + "/newTest/hello/world1"), perm);
    fs.setPermission(new Path(base + "/newTest/hello/world2"), perm);
    fs.setPermission(new Path(base + "/newTest/hello/world2/newworld"), perm);
    fs.setPermission(new Path(base + "/newTest/hello/world3"), perm);
    fs.setPermission(new Path(base + "/newTest/hello/world3/oldworld"), perm);
    createFile(fs, base + "/newTest/1");
    createFile(fs, base + "/newTest/hello/2");
    createFile(fs, base + "/newTest/hello/world3/oldworld/3");
    createFile(fs, base + "/newTest/hello/world2/4");
    return base;//w  ww  . ja v a 2 s  . c  om
}

From source file:com.inmobi.conduit.local.CopyMapper.java

License:Apache License

@Override
public void map(Text key, FileStatus value, Context context) throws IOException, InterruptedException {
    Path src = value.getPath();// ww  w.ja  va  2 s  . com
    String dest = key.toString();
    String collector = src.getParent().getName();
    String category = src.getParent().getParent().getName();
    Map<Long, Long> received = null;
    if (context.getConfiguration().getBoolean(ConduitConstants.AUDIT_ENABLED_KEY, true)) {
        received = new HashMap<Long, Long>();
    }
    Configuration srcConf = new Configuration();
    srcConf.set(FS_DEFAULT_NAME_KEY, context.getConfiguration().get(SRC_FS_DEFAULT_NAME_KEY));

    FileSystem fs = FileSystem.get(srcConf);
    Path target = getTempPath(context, src, category, collector);
    if (FileUtil.gzip(src, target, srcConf, received)) {
        LOG.info("File " + src + " is empty hence returning without compressing");
        return;
    }
    // move to final destination
    fs.mkdirs(new Path(dest).makeQualified(fs));
    String destnFilename = collector + "-" + src.getName() + ".gz";
    Path destPath = new Path(dest + File.separator + destnFilename);
    LOG.info("Renaming file " + target + " to " + destPath);
    fs.rename(target, destPath);
    if (received != null) {

        for (Entry<Long, Long> entry : received.entrySet()) {
            String counterNameValue = getCounterNameValue(category, destnFilename, entry.getKey(),
                    entry.getValue());
            context.write(NullWritable.get(), new Text(counterNameValue));
        }
    }

}

From source file:com.inmobi.conduit.local.LocalStreamServiceTest.java

License:Apache License

@Test
public void testFileUtil() throws Exception {
    String streamName = "test1";
    Path rootDir = new Path("/tmp/localServiceTest/testcluster2/mergeservice");
    Path dataDir = new Path(rootDir, "data/test1/testcluster2");
    FileSystem fs = dataDir.getFileSystem(new Configuration());
    fs.mkdirs(dataDir);
    String filenameStr = new String(
            streamName + "-" + TestLocalStreamService.getDateAsYYYYMMDDHHmm(new Date()) + "_00001");
    Path src = new Path(dataDir, filenameStr);

    LOG.debug("Creating Test Data with filename [" + filenameStr + "]");
    FSDataOutputStream streamout = fs.create(src);
    String content = "Creating Test data for teststream";
    Message msg = new Message(content.getBytes());
    long currentTimestamp = new Date().getTime();
    AuditUtil.attachHeaders(msg, currentTimestamp);
    byte[] encodeMsg = Base64.encodeBase64(msg.getData().array());
    streamout.write(encodeMsg);/*w w w.ja  va2  s.  c  om*/
    streamout.write("\n".getBytes());
    streamout.write(encodeMsg);
    streamout.write("\n".getBytes());
    long nextMinuteTimeStamp = currentTimestamp + 60000;
    // Genearate a msg with different timestamp.  Default window period is 60sec
    AuditUtil.attachHeaders(msg, nextMinuteTimeStamp);
    encodeMsg = Base64.encodeBase64(msg.getData().array());
    streamout.write(encodeMsg);
    streamout.close();
    Map<Long, Long> received = new HashMap<Long, Long>();
    Path target = new Path(new Path(rootDir, "system/tmp/LocalStreamService_testcluster2_test1@/"
            + "job_local_0001/attempt_local_0001_m_000000_0/"), filenameStr + ".gz");
    FileUtil.gzip(src, target, new Configuration(), received);
    Assert.assertEquals(2, received.size());
    // current timestamp window = currentTimestamp - (currentTimestamp % 60000)
    Assert.assertTrue(2 == received.get(currentTimestamp - (currentTimestamp % 60000)));
    // next timestamp window = nextMinuteTimeStamp - (nextMinuteTimeStamp %60000)
    Assert.assertTrue(1 == received.get(nextMinuteTimeStamp - (nextMinuteTimeStamp % 60000)));
    fs.delete(rootDir, true);
}

From source file:com.inmobi.conduit.purge.DataPurgerServiceTest.java

License:Apache License

private void createTestPurgePartitionFiles(FileSystem fs, Cluster cluster, Calendar date, Table table)
        throws Exception {
    for (String streamname : cluster.getSourceStreams()) {
        String[] files = new String[NUM_OF_FILES];
        String datapath = Cluster.getDateAsYYYYMMDDHHMNPath(date.getTime());
        String commitpath = cluster.getLocalFinalDestDirRoot() + File.separator + streamname + File.separator
                + datapath;// ww  w  .  ja  v a2 s.  c o m
        fs.mkdirs(new Path(commitpath));
        Map<String, String> partSpec = TestHCatUtil.getPartitionMap(date);
        LOG.info("Adding partition " + partSpec + " for stream " + streamname);
        TestHCatUtil.addPartition(table, partSpec);
        for (int j = 0; j < NUM_OF_FILES; ++j) {
            files[j] = new String(cluster.getName() + "-"
                    + TestLocalStreamService.getDateAsYYYYMMDDHHmm(new Date()) + "_" + idFormat.format(j));
            {
                Path path = new Path(commitpath + File.separator + files[j]);
                LOG.info("Creating streams_local File " + path.getName());
                FSDataOutputStream streamout = fs.create(path);
                streamout.writeBytes("Creating Test data for teststream " + path.toString());
                streamout.close();
                Assert.assertTrue(fs.exists(path));
            }
        }
    }
}

From source file:com.inmobi.conduit.purge.DataPurgerServiceTest.java

License:Apache License

private void createTestPurgefiles(FileSystem fs, Cluster cluster, Calendar date, boolean createEmptyDirs)
        throws Exception {
    for (String streamname : cluster.getSourceStreams()) {
        String[] files = new String[NUM_OF_FILES];
        String datapath = Cluster.getDateAsYYYYMMDDHHMNPath(date.getTime());
        String commitpath = cluster.getLocalFinalDestDirRoot() + File.separator + streamname + File.separator
                + datapath;//from   ww  w .j av  a  2s. c  o m
        String mergecommitpath = cluster.getFinalDestDirRoot() + File.separator + streamname + File.separator
                + datapath;

        String trashpath = cluster.getTrashPath() + File.separator + CalendarHelper.getDateAsString(date)
                + File.separator;
        fs.mkdirs(new Path(commitpath));

        for (int j = 0; j < NUM_OF_FILES; ++j) {
            files[j] = new String(cluster.getName() + "-"
                    + TestLocalStreamService.getDateAsYYYYMMDDHHmm(new Date()) + "_" + idFormat.format(j));
            {
                Path path = new Path(commitpath + File.separator + files[j]);
                // LOG.info("Creating streams_local File " + path.getName());
                FSDataOutputStream streamout = fs.create(path);
                streamout.writeBytes("Creating Test data for teststream " + path.toString());
                streamout.close();
                Assert.assertTrue(fs.exists(path));
            }
            {
                Path path = new Path(mergecommitpath + File.separator + files[j]);
                // LOG.info("Creating streams File " + path.getName());
                FSDataOutputStream streamout = fs.create(path);
                streamout.writeBytes("Creating Test data for teststream " + path.toString());
                streamout.close();
                Assert.assertTrue(fs.exists(path));
            }

            {
                if (!createEmptyDirs) {
                    Path path = new Path(trashpath + File.separator
                            + String.valueOf(date.get(Calendar.HOUR_OF_DAY)) + File.separator + files[j]);
                    // LOG.info("Creating trash File " + path.toString());
                    FSDataOutputStream streamout = fs.create(path);
                    streamout.writeBytes("Creating Test trash data for teststream " + path.getName());
                    streamout.close();
                    Assert.assertTrue(fs.exists(path));
                }
            }

        }
        if (createEmptyDirs) {
            Path path = new Path(trashpath);
            if (!fs.exists(path))
                fs.mkdirs(path);
            Assert.assertTrue(fs.exists(path));
        }
    }

}

From source file:com.inmobi.databus.AbstractService.java

License:Apache License

protected Set<Path> publishMissingPaths(FileSystem fs, String destDir, long commitTime, String categoryName)
        throws Exception {
    Set<Path> missingDirectories = new TreeSet<Path>();
    Calendar commitTimeMinutes = new GregorianCalendar();
    commitTimeMinutes.set(Calendar.MILLISECOND, 0);
    commitTimeMinutes.set(Calendar.SECOND, 0);
    commitTime = commitTimeMinutes.getTimeInMillis();
    Long prevRuntime = new Long(-1);
    if (!prevRuntimeForCategory.containsKey(categoryName)) {
        LOG.debug("Calculating Previous Runtime from Directory Listing");
        prevRuntime = getPreviousRuntime(fs, destDir, categoryName);
    } else {//from   www . jav a2 s  . co m
        LOG.debug("Reading Previous Runtime from Cache");
        prevRuntime = prevRuntimeForCategory.get(categoryName);
    }

    if (prevRuntime != -1) {
        if (isMissingPaths(commitTime, prevRuntime)) {
            LOG.debug("Previous Runtime: [" + getLogDateString(prevRuntime) + "]");
            while (isMissingPaths(commitTime, prevRuntime)) {
                String missingPath = Cluster.getDestDir(destDir, categoryName, prevRuntime);
                Path missingDir = new Path(missingPath);
                if (!fs.exists(missingDir)) {
                    LOG.debug("Creating Missing Directory [" + missingPath + "]");
                    missingDirectories.add(new Path(missingPath));
                    fs.mkdirs(missingDir);
                }
                prevRuntime += MILLISECONDS_IN_MINUTE;
            }
        }
        prevRuntimeForCategory.put(categoryName, commitTime);
    }
    return missingDirectories;
}

From source file:com.inmobi.databus.distcp.MergedStreamService.java

License:Apache License

private void commitMirroredConsumerPaths(Map<String, Set<Path>> committedPaths, Path tmp) throws Exception {
    // Map of Stream and clusters where it's mirrored
    Map<String, Set<Cluster>> mirrorStreamConsumers = new HashMap<String, Set<Cluster>>();
    Map<Path, Path> consumerCommitPaths = new LinkedHashMap<Path, Path>();
    // for each stream in committedPaths
    for (String stream : committedPaths.keySet()) {
        // for each cluster
        for (Cluster cluster : getConfig().getClusters().values()) {
            // is this stream to be mirrored on this cluster
            if (cluster.getMirroredStreams().contains(stream)) {
                Set<Cluster> mirrorConsumers = mirrorStreamConsumers.get(stream);
                if (mirrorConsumers == null)
                    mirrorConsumers = new HashSet<Cluster>();
                mirrorConsumers.add(cluster);
                mirrorStreamConsumers.put(stream, mirrorConsumers);
            }//from w ww .j a v  a2  s .c o  m
        }
    } // for each stream

    // Commit paths for each consumer
    for (String stream : committedPaths.keySet()) {
        // consumers for this stream
        Set<Cluster> consumers = mirrorStreamConsumers.get(stream);
        Path tmpConsumerPath;
        if (consumers == null || consumers.size() == 0) {
            LOG.warn(" Consumers is empty for stream [" + stream + "]");
            continue;
        }
        for (Cluster consumer : consumers) {
            // commit paths for this consumer, this stream
            // adding srcCluster avoids two Remote Copiers creating same filename
            String tmpPath = "src_" + getSrcCluster().getName() + "_via_" + getDestCluster().getName()
                    + "_mirrorto_" + consumer.getName() + "_" + stream;
            tmpConsumerPath = new Path(tmp, tmpPath);
            FSDataOutputStream out = getDestFs().create(tmpConsumerPath);
            try {
                for (Path path : committedPaths.get(stream)) {
                    LOG.debug("Writing Mirror Commit Path [" + path.toString() + "]");
                    out.writeBytes(path.toString());
                    out.writeBytes("\n");
                }
            } finally {
                out.close();
            }
            // Two MergedStreamConsumers will write file for same consumer within
            // the same time
            // adding srcCLuster name avoids that conflict
            Path finalMirrorPath = new Path(getDestCluster().getMirrorConsumePath(consumer),
                    tmpPath + "_" + new Long(System.currentTimeMillis()).toString());
            consumerCommitPaths.put(tmpConsumerPath, finalMirrorPath);
        } // for each consumer
    } // for each stream

    if (consumerCommitPaths == null || consumerCommitPaths.size() == 0) {
        LOG.info("consumerCommitPaths is empty for all stream, skipping mirrorCommit");
        missingDirsCommittedPaths.clear();
        return;
    }
    // Do the final mirrorCommit
    LOG.info("Committing [" + consumerCommitPaths.size() + "] paths for " + "mirrored Stream");
    FileSystem fs = FileSystem.get(getDestCluster().getHadoopConf());
    for (Map.Entry<Path, Path> entry : consumerCommitPaths.entrySet()) {
        LOG.info("Renaming [" + entry.getKey() + "] to [" + entry.getValue() + "]");
        fs.mkdirs(entry.getValue().getParent());
        if (fs.rename(entry.getKey(), entry.getValue()) == false) {
            LOG.warn("Failed to Commit for Mirrored Path. Aborting Transaction " + "to avoid DATA LOSS, "
                    + "Partial data replay can happen for merged and mirror stream");
            throw new Exception("Rename failed from [" + entry.getKey() + "] to [" + entry.getValue() + "]");
        }
    }
    missingDirsCommittedPaths.clear();
}

From source file:com.inmobi.databus.local.CopyMapper.java

License:Apache License

@Override
public void map(Text key, Text value, Context context) throws IOException, InterruptedException {
    Path src = new Path(key.toString());
    String dest = value.toString();
    String collector = src.getParent().getName();
    String category = src.getParent().getParent().getName();

    FileSystem fs = FileSystem.get(context.getConfiguration());
    Path target = getTempPath(context, src, category, collector);
    FileUtil.gzip(src, target, context.getConfiguration());
    // move to final destination
    fs.mkdirs(new Path(dest).makeQualified(fs));
    Path destPath = new Path(dest + File.separator + collector + "-" + src.getName() + ".gz");
    LOG.info("Renaming file " + target + " to " + destPath);
    fs.rename(target, destPath);/*from   www .j  a  va 2  s  .c  om*/

}