List of usage examples for org.apache.hadoop.fs FileSystem create
public FSDataOutputStream create(Path f) throws IOException
From source file:com.inmobi.conduit.distcp.tools.TestCopyListing.java
License:Apache License
@Test public void testMultipleSrcToFile() { FileSystem fs = null; try {/* w w w .j av a 2s.c o m*/ fs = FileSystem.get(getConf()); List<Path> srcPaths = new ArrayList<Path>(); srcPaths.add(new Path("/tmp/in/1")); srcPaths.add(new Path("/tmp/in/2")); Path target = new Path("/tmp/out/1"); TestDistCpUtils.createFile(fs, "/tmp/in/1"); TestDistCpUtils.createFile(fs, "/tmp/in/2"); fs.mkdirs(target); DistCpOptions options = new DistCpOptions(srcPaths, target); validatePaths(options); TestDistCpUtils.delete(fs, "/tmp"); //No errors target = new Path("/tmp/out/1"); fs.create(target).close(); options = new DistCpOptions(srcPaths, target); try { validatePaths(options); Assert.fail("Invalid inputs accepted"); } catch (InvalidInputException ignore) { } TestDistCpUtils.delete(fs, "/tmp"); srcPaths.clear(); srcPaths.add(new Path("/tmp/in/1")); fs.mkdirs(new Path("/tmp/in/1")); target = new Path("/tmp/out/1"); fs.create(target).close(); options = new DistCpOptions(srcPaths, target); try { validatePaths(options); Assert.fail("Invalid inputs accepted"); } catch (InvalidInputException ignore) { } TestDistCpUtils.delete(fs, "/tmp"); } catch (IOException e) { LOG.error("Exception encountered ", e); Assert.fail("Test input validation failed"); } finally { TestDistCpUtils.delete(fs, "/tmp"); } }
From source file:com.inmobi.conduit.distcp.tools.TestCopyListing.java
License:Apache License
@Test public void testBuildListing() { FileSystem fs = null; try {/*from w w w. jav a2 s . c o m*/ fs = FileSystem.get(getConf()); List<Path> srcPaths = new ArrayList<Path>(); Path p1 = new Path("/tmp/in/1"); Path p2 = new Path("/tmp/in/2"); Path p3 = new Path("/tmp/in2/2"); Path target = new Path("/tmp/out/1"); srcPaths.add(p1.getParent()); srcPaths.add(p3.getParent()); TestDistCpUtils.createFile(fs, "/tmp/in/1"); TestDistCpUtils.createFile(fs, "/tmp/in/2"); TestDistCpUtils.createFile(fs, "/tmp/in2/2"); fs.mkdirs(target); OutputStream out = fs.create(p1); out.write("ABC".getBytes()); out.close(); out = fs.create(p2); out.write("DEF".getBytes()); out.close(); out = fs.create(p3); out.write("GHIJ".getBytes()); out.close(); Path listingFile = new Path("/tmp/file"); DistCpOptions options = new DistCpOptions(srcPaths, target); options.setSyncFolder(true); CopyListing listing = new SimpleCopyListing(getConf(), CREDENTIALS); try { listing.buildListing(listingFile, options); Assert.fail("Duplicates not detected"); } catch (DuplicateFileException ignore) { } Assert.assertEquals(listing.getBytesToCopy(), 10); Assert.assertEquals(listing.getNumberOfPaths(), 3); TestDistCpUtils.delete(fs, "/tmp"); try { listing.buildListing(listingFile, options); Assert.fail("Invalid input not detected"); } catch (InvalidInputException ignore) { } TestDistCpUtils.delete(fs, "/tmp"); } catch (IOException e) { LOG.error("Exception encountered ", e); Assert.fail("Test build listing failed"); } finally { TestDistCpUtils.delete(fs, "/tmp"); } }
From source file:com.inmobi.conduit.distcp.tools.util.TestDistCpUtils.java
License:Apache License
public static void createFile(FileSystem fs, String filePath) throws IOException { OutputStream out = fs.create(new Path(filePath)); IOUtils.closeStream(out);// w ww.j a va 2 s. c om }
From source file:com.inmobi.conduit.local.LocalStreamServiceTest.java
License:Apache License
@Test public void testFileUtil() throws Exception { String streamName = "test1"; Path rootDir = new Path("/tmp/localServiceTest/testcluster2/mergeservice"); Path dataDir = new Path(rootDir, "data/test1/testcluster2"); FileSystem fs = dataDir.getFileSystem(new Configuration()); fs.mkdirs(dataDir);// ww w . jav a 2 s .c om String filenameStr = new String( streamName + "-" + TestLocalStreamService.getDateAsYYYYMMDDHHmm(new Date()) + "_00001"); Path src = new Path(dataDir, filenameStr); LOG.debug("Creating Test Data with filename [" + filenameStr + "]"); FSDataOutputStream streamout = fs.create(src); String content = "Creating Test data for teststream"; Message msg = new Message(content.getBytes()); long currentTimestamp = new Date().getTime(); AuditUtil.attachHeaders(msg, currentTimestamp); byte[] encodeMsg = Base64.encodeBase64(msg.getData().array()); streamout.write(encodeMsg); streamout.write("\n".getBytes()); streamout.write(encodeMsg); streamout.write("\n".getBytes()); long nextMinuteTimeStamp = currentTimestamp + 60000; // Genearate a msg with different timestamp. Default window period is 60sec AuditUtil.attachHeaders(msg, nextMinuteTimeStamp); encodeMsg = Base64.encodeBase64(msg.getData().array()); streamout.write(encodeMsg); streamout.close(); Map<Long, Long> received = new HashMap<Long, Long>(); Path target = new Path(new Path(rootDir, "system/tmp/LocalStreamService_testcluster2_test1@/" + "job_local_0001/attempt_local_0001_m_000000_0/"), filenameStr + ".gz"); FileUtil.gzip(src, target, new Configuration(), received); Assert.assertEquals(2, received.size()); // current timestamp window = currentTimestamp - (currentTimestamp % 60000) Assert.assertTrue(2 == received.get(currentTimestamp - (currentTimestamp % 60000))); // next timestamp window = nextMinuteTimeStamp - (nextMinuteTimeStamp %60000) Assert.assertTrue(1 == received.get(nextMinuteTimeStamp - (nextMinuteTimeStamp % 60000))); fs.delete(rootDir, true); }
From source file:com.inmobi.conduit.purge.DataPurgerServiceTest.java
License:Apache License
private void createTestPurgePartitionFiles(FileSystem fs, Cluster cluster, Calendar date, Table table) throws Exception { for (String streamname : cluster.getSourceStreams()) { String[] files = new String[NUM_OF_FILES]; String datapath = Cluster.getDateAsYYYYMMDDHHMNPath(date.getTime()); String commitpath = cluster.getLocalFinalDestDirRoot() + File.separator + streamname + File.separator + datapath;//from w w w .j a v a 2 s . co m fs.mkdirs(new Path(commitpath)); Map<String, String> partSpec = TestHCatUtil.getPartitionMap(date); LOG.info("Adding partition " + partSpec + " for stream " + streamname); TestHCatUtil.addPartition(table, partSpec); for (int j = 0; j < NUM_OF_FILES; ++j) { files[j] = new String(cluster.getName() + "-" + TestLocalStreamService.getDateAsYYYYMMDDHHmm(new Date()) + "_" + idFormat.format(j)); { Path path = new Path(commitpath + File.separator + files[j]); LOG.info("Creating streams_local File " + path.getName()); FSDataOutputStream streamout = fs.create(path); streamout.writeBytes("Creating Test data for teststream " + path.toString()); streamout.close(); Assert.assertTrue(fs.exists(path)); } } } }
From source file:com.inmobi.conduit.purge.DataPurgerServiceTest.java
License:Apache License
private void createTestPurgefiles(FileSystem fs, Cluster cluster, Calendar date, boolean createEmptyDirs) throws Exception { for (String streamname : cluster.getSourceStreams()) { String[] files = new String[NUM_OF_FILES]; String datapath = Cluster.getDateAsYYYYMMDDHHMNPath(date.getTime()); String commitpath = cluster.getLocalFinalDestDirRoot() + File.separator + streamname + File.separator + datapath;//from w w w. jav a2 s .c o m String mergecommitpath = cluster.getFinalDestDirRoot() + File.separator + streamname + File.separator + datapath; String trashpath = cluster.getTrashPath() + File.separator + CalendarHelper.getDateAsString(date) + File.separator; fs.mkdirs(new Path(commitpath)); for (int j = 0; j < NUM_OF_FILES; ++j) { files[j] = new String(cluster.getName() + "-" + TestLocalStreamService.getDateAsYYYYMMDDHHmm(new Date()) + "_" + idFormat.format(j)); { Path path = new Path(commitpath + File.separator + files[j]); // LOG.info("Creating streams_local File " + path.getName()); FSDataOutputStream streamout = fs.create(path); streamout.writeBytes("Creating Test data for teststream " + path.toString()); streamout.close(); Assert.assertTrue(fs.exists(path)); } { Path path = new Path(mergecommitpath + File.separator + files[j]); // LOG.info("Creating streams File " + path.getName()); FSDataOutputStream streamout = fs.create(path); streamout.writeBytes("Creating Test data for teststream " + path.toString()); streamout.close(); Assert.assertTrue(fs.exists(path)); } { if (!createEmptyDirs) { Path path = new Path(trashpath + File.separator + String.valueOf(date.get(Calendar.HOUR_OF_DAY)) + File.separator + files[j]); // LOG.info("Creating trash File " + path.toString()); FSDataOutputStream streamout = fs.create(path); streamout.writeBytes("Creating Test trash data for teststream " + path.getName()); streamout.close(); Assert.assertTrue(fs.exists(path)); } } } if (createEmptyDirs) { Path path = new Path(trashpath); if (!fs.exists(path)) fs.mkdirs(path); Assert.assertTrue(fs.exists(path)); } } }
From source file:com.inmobi.conduit.utils.CollapseFilesInDir.java
License:Apache License
public static void main(String[] args) throws Exception { Configuration configuration = new Configuration(); configuration.set("fs.default.name", args[0]); String dir = args[1];//from w ww . j a va 2 s.c o m FileSystem fs = FileSystem.get(configuration); FileStatus[] fileList; try { fileList = fs.listStatus(new Path(dir)); } catch (FileNotFoundException fe) { fileList = null; } if (fileList != null) { if (fileList.length > 1) { Set<Path> sourceFiles = new HashSet<Path>(); Set<String> consumePaths = new HashSet<String>(); //inputPath has have multiple files due to backlog //read all and create a tmp file for (int i = 0; i < fileList.length; i++) { Path consumeFilePath = fileList[i].getPath().makeQualified(fs); sourceFiles.add(consumeFilePath); FSDataInputStream fsDataInputStream = fs.open(consumeFilePath); try { while (fsDataInputStream.available() > 0) { String fileName = fsDataInputStream.readLine(); if (fileName != null) { consumePaths.add(fileName.trim()); System.out.println("Adding [" + fileName + "] to pull"); } } } finally { fsDataInputStream.close(); } } Path finalPath = new Path(dir, new Long(System.currentTimeMillis()).toString()); FSDataOutputStream out = fs.create(finalPath); BufferedWriter writer = new BufferedWriter(new OutputStreamWriter(out)); try { for (String consumePath : consumePaths) { System.out.println("Adding sourceFile [" + consumePath + "] to" + " distcp " + "FinalList"); writer.write(consumePath); writer.write("\n"); } } finally { writer.close(); } LOG.warn("Final File - [" + finalPath + "]"); for (Path deletePath : sourceFiles) { System.out.println("Deleting - [" + deletePath + "]"); fs.delete(deletePath); } } } }
From source file:com.inmobi.databus.distcp.DistcpBaseService.java
License:Apache License
private Path createInputFileForDISCTP(FileSystem fs, String clusterName, Path tmp, Set<String> minFilesSet) throws IOException { if (minFilesSet.size() > 0) { Path tmpPath = null;/*from w ww. j a v a 2 s.c om*/ FSDataOutputStream out = null; try { tmpPath = new Path(tmp, clusterName + new Long(System.currentTimeMillis()).toString()); out = fs.create(tmpPath); for (String minFile : minFilesSet) { out.write(minFile.getBytes()); out.write('\n'); } } finally { if (out != null) { out.close(); } } return tmpPath; } else return null; }
From source file:com.inmobi.databus.local.LocalStreamService.java
License:Apache License
private Map<Path, Path> prepareForCommit(long commitTime, Map<FileStatus, String> fileListing) throws Exception { FileSystem fs = FileSystem.get(cluster.getHadoopConf()); // find final destination paths Map<Path, Path> mvPaths = new LinkedHashMap<Path, Path>(); FileStatus[] categories = fs.listStatus(tmpJobOutputPath); for (FileStatus categoryDir : categories) { String categoryName = categoryDir.getPath().getName(); Path destDir = new Path(cluster.getLocalDestDir(categoryName, commitTime)); FileStatus[] files = fs.listStatus(categoryDir.getPath()); for (FileStatus file : files) { Path destPath = new Path(destDir, file.getPath().getName()); LOG.debug("Moving [" + file.getPath() + "] to [" + destPath + "]"); mvPaths.put(file.getPath(), destPath); }/*from w w w . jav a2 s. c o m*/ publishMissingPaths(fs, cluster.getLocalFinalDestDirRoot(), commitTime, categoryName); } // find input files for consumer Map<Path, Path> consumerCommitPaths = new HashMap<Path, Path>(); for (Cluster clusterEntry : getConfig().getClusters().values()) { Set<String> destStreams = clusterEntry.getDestinationStreams().keySet(); boolean consumeCluster = false; for (String destStream : destStreams) { if (clusterEntry.getPrimaryDestinationStreams().contains(destStream) && cluster.getSourceStreams().contains(destStream)) { consumeCluster = true; } } if (consumeCluster) { Path tmpConsumerPath = new Path(tmpPath, clusterEntry.getName()); boolean isFileOpened = false; FSDataOutputStream out = null; try { for (Path destPath : mvPaths.values()) { String category = getCategoryFromDestPath(destPath); if (clusterEntry.getDestinationStreams().containsKey(category)) { if (!isFileOpened) { out = fs.create(tmpConsumerPath); isFileOpened = true; } out.writeBytes(destPath.toString()); LOG.debug("Adding [" + destPath + "] for consumer [" + clusterEntry.getName() + "] to commit Paths in [" + tmpConsumerPath + "]"); out.writeBytes("\n"); } } } finally { if (isFileOpened) { out.close(); Path finalConsumerPath = new Path(cluster.getConsumePath(clusterEntry), Long.toString(System.currentTimeMillis())); LOG.debug("Moving [" + tmpConsumerPath + "] to [ " + finalConsumerPath + "]"); consumerCommitPaths.put(tmpConsumerPath, finalConsumerPath); } } } } Map<Path, Path> commitPaths = new LinkedHashMap<Path, Path>(); commitPaths.putAll(mvPaths); commitPaths.putAll(consumerCommitPaths); return commitPaths; }
From source file:com.inmobi.databus.local.LocalStreamService.java
License:Apache License
private void createMRInput(Path inputPath, Map<FileStatus, String> fileListing, Set<FileStatus> trashSet, Map<String, FileStatus> checkpointPaths) throws IOException { FileSystem fs = FileSystem.get(cluster.getHadoopConf()); createListing(fs, fs.getFileStatus(cluster.getDataDir()), fileListing, trashSet, checkpointPaths); FSDataOutputStream out = fs.create(inputPath); try {/*from ww w. jav a2s . co m*/ Iterator<Entry<FileStatus, String>> it = fileListing.entrySet().iterator(); while (it.hasNext()) { Entry<FileStatus, String> entry = it.next(); out.writeBytes(entry.getKey().getPath().toString()); out.writeBytes("\t"); out.writeBytes(entry.getValue()); out.writeBytes("\n"); } } finally { out.close(); } }