List of usage examples for org.apache.hadoop.fs Path SEPARATOR
String SEPARATOR
To view the source code for org.apache.hadoop.fs Path SEPARATOR.
Click Source Link
From source file:co.cask.cdap.internal.app.runtime.batch.dataset.partitioned.DynamicPartitioningOutputCommitter.java
License:Apache License
private PartitionKey getPartitionKey(Partitioning partitioning, String relativePath) { List<String> pathParts = Arrays.asList(relativePath.split(Path.SEPARATOR)); if (pathParts.size() != partitioning.getFields().size()) { throw new IllegalArgumentException( String.format("relativePath '%s' does not have same number of components as partitioning '%s", relativePath, partitioning)); }/*w w w . j ava 2 s . c om*/ PartitionKey.Builder builder = PartitionKey.builder(); int i = 0; for (Map.Entry<String, Partitioning.FieldType> entry : partitioning.getFields().entrySet()) { String keyName = entry.getKey(); Comparable keyValue = entry.getValue().parse(pathParts.get(i)); builder.addField(keyName, keyValue); i++; } return builder.build(); }
From source file:co.cask.cdap.internal.app.runtime.batch.DynamicPartitionerWithAvroTest.java
License:Apache License
@Test public void testDynamicPartitionerWithAvro() throws Exception { ApplicationWithPrograms app = deployApp(AppWithMapReduceUsingAvroDynamicPartitioner.class); final GenericRecord record1 = createRecord("bob", 95111); final GenericRecord record2 = createRecord("sally", 98123); final GenericRecord record3 = createRecord("jane", 84125); final GenericRecord record4 = createRecord("john", 84125); final long now = System.currentTimeMillis(); // the four records above will fall into the partitions defined by the following three keys // we have four records, but only 3 have unique zip codes (which is what we partition by) final PartitionKey key1 = PartitionKey.builder().addLongField("time", now).addIntField("zip", 95111) .build();//from w w w .j a v a2 s . co m final PartitionKey key2 = PartitionKey.builder().addLongField("time", now).addIntField("zip", 98123) .build(); final PartitionKey key3 = PartitionKey.builder().addLongField("time", now).addIntField("zip", 84125) .build(); final Set<PartitionKey> expectedKeys = ImmutableSet.of(key1, key2, key3); // write values to the input kvTable final KeyValueTable kvTable = datasetCache.getDataset(INPUT_DATASET); Transactions.createTransactionExecutor(txExecutorFactory, kvTable) .execute(new TransactionExecutor.Subroutine() { @Override public void apply() { // the keys are not used; it matters that they're unique though kvTable.write("1", record1.toString()); kvTable.write("2", record2.toString()); kvTable.write("3", record3.toString()); kvTable.write("4", record4.toString()); } }); // run the partition writer m/r with this output partition time runProgram(app, AppWithMapReduceUsingAvroDynamicPartitioner.DynamicPartitioningMapReduce.class, new BasicArguments(ImmutableMap.of(OUTPUT_PARTITION_KEY, Long.toString(now)))); // this should have created a partition in the pfs final PartitionedFileSet pfs = datasetCache.getDataset(OUTPUT_DATASET); final Location pfsBaseLocation = pfs.getEmbeddedFileSet().getBaseLocation(); Transactions.createTransactionExecutor(txExecutorFactory, (TransactionAware) pfs) .execute(new TransactionExecutor.Subroutine() { @Override public void apply() throws IOException { Map<PartitionKey, PartitionDetail> partitions = new HashMap<>(); for (PartitionDetail partition : pfs.getPartitions(null)) { partitions.put(partition.getPartitionKey(), partition); // check that the mapreduce wrote the output partition metadata to all the output partitions Assert.assertEquals( AppWithMapReduceUsingAvroDynamicPartitioner.DynamicPartitioningMapReduce.METADATA, partition.getMetadata().asMap()); } Assert.assertEquals(3, partitions.size()); Assert.assertEquals(expectedKeys, partitions.keySet()); // Check the relative path of one partition. Also check that its location = pfs base location + relativePath PartitionDetail partition1 = partitions.get(key1); String relativePath = partition1.getRelativePath(); Assert.assertEquals( Long.toString(now) + Path.SEPARATOR + Integer.toString((int) key1.getField("zip")), relativePath); Assert.assertEquals(pfsBaseLocation.append(relativePath), partition1.getLocation()); Assert.assertEquals(ImmutableList.of(record1), readOutput(partition1.getLocation())); Assert.assertEquals(ImmutableList.of(record2), readOutput(partitions.get(key2).getLocation())); Assert.assertEquals(ImmutableList.of(record3, record4), readOutput(partitions.get(key3).getLocation())); } }); }
From source file:com.alibaba.jstorm.hdfs.spout.HdfsSpout.java
License:Apache License
private void markFileAsBad(Path file) { String fileName = file.toString(); String fileNameMinusSuffix = fileName.substring(0, fileName.indexOf(inprogress_suffix)); String originalName = new Path(fileNameMinusSuffix).getName(); Path newFile = new Path(badFilesDirPath + Path.SEPARATOR + originalName); LOG.info("Moving bad file {} to {}. Processed it till offset {}. SpoutID= {}", originalName, newFile, tracker.getCommitPosition(), spoutId); try {// w w w. ja v a 2s . c om if (!hdfs.rename(file, newFile)) { // seems this can fail by returning false or throwing exception throw new IOException("Move failed for bad file: " + file); // convert false ret value to exception } } catch (IOException e) { LOG.warn("Error moving bad file: " + file + " to destination " + newFile + " SpoutId =" + spoutId, e); } closeReaderAndResetTrackers(); }
From source file:com.alibaba.jstorm.hdfs.spout.HdfsSpout.java
License:Apache License
private String getDefaultLockDir(Path sourceDirPath) { return sourceDirPath.toString() + Path.SEPARATOR + Configs.DEFAULT_LOCK_DIR; }
From source file:com.alibaba.jstorm.hdfs.spout.HdfsSpout.java
License:Apache License
/** Returns the corresponding input file in the 'sourceDirPath' for the specified lock file. * If no such file is found then returns null *//* ww w. j a v a 2 s . com*/ private Path getFileForLockFile(Path lockFile, Path sourceDirPath) throws IOException { String lockFileName = lockFile.getName(); Path dataFile = new Path(sourceDirPath + Path.SEPARATOR + lockFileName + inprogress_suffix); if (hdfs.exists(dataFile)) { return dataFile; } dataFile = new Path(sourceDirPath + Path.SEPARATOR + lockFileName); if (hdfs.exists(dataFile)) { return dataFile; } return null; }
From source file:com.alibaba.jstorm.hdfs.spout.HdfsSpout.java
License:Apache License
private Path renameCompletedFile(Path file) throws IOException { String fileName = file.toString(); String fileNameMinusSuffix = fileName.substring(0, fileName.indexOf(inprogress_suffix)); String newName = new Path(fileNameMinusSuffix).getName(); Path newFile = new Path(archiveDirPath + Path.SEPARATOR + newName); LOG.info("Completed consuming file {}", fileNameMinusSuffix); if (!hdfs.rename(file, newFile)) { throw new IOException("Rename failed for file: " + file); }/*from w w w .ja v a2s . c om*/ LOG.debug("Renamed file {} to {} ", file, newFile); return newFile; }
From source file:com.aliyun.odps.volume.VolumeFSUtil.java
License:Apache License
/** * Whether the pathname is valid. Currently prohibits relative paths, names which contain a ":" or * "//", or other non-canonical paths./*from w ww.j a v a 2 s .co m*/ */ public static boolean isValidName(String src) { if (src == null) return false; // Path must be absolute. if (!src.startsWith(Path.SEPARATOR)) { return false; } // Check for "~" ".." "." ":" "/" String[] components = StringUtils.split(src, '/'); for (int i = 0; i < components.length; i++) { String element = components[i]; if (element.equals("~") || element.equals(".") || element.equals("..") || (element.indexOf(":") >= 0) || (element.indexOf("/") >= 0) || (element.indexOf("\\") >= 0) || (element.indexOf("\0") >= 0)) { return false; } // The string may start or end with a /, but not have // "//" in the middle. if (element.isEmpty() && i != components.length - 1 && i != 0) { return false; } } return true; }
From source file:com.cloudera.flume.handlers.hdfs.EscapedCustomDfsSink.java
License:Apache License
public EscapedCustomDfsSink(String path, String filename, OutputFormat o) { this.path = path; this.filename = filename; shouldSub = Event.containsTag(path) || Event.containsTag(filename); this.format = o; absolutePath = path;/* ww w . j a va 2 s. co m*/ if (filename != null && filename.length() > 0) { if (!absolutePath.endsWith(Path.SEPARATOR)) { absolutePath += Path.SEPARATOR; } absolutePath += this.filename; } }
From source file:com.cloudera.flume.handlers.hive.HiveNotifyingDfsSink.java
License:Apache License
public HiveNotifyingDfsSink(String path, String filename, String hivetable, OutputFormat o, HiveDirCreatedHandler handler) { this.dirpath = path; this.filename = filename; this.format = o; absolutePath = path;// w w w . j a v a2s . c o m this.hivetable = hivetable; if (filename != null && filename.length() > 0) { if (!absolutePath.endsWith(Path.SEPARATOR)) { absolutePath += Path.SEPARATOR; } absolutePath += this.filename; } if (!(o instanceof AvroJsonOutputFormat)) { LOG.warn("Currently, hive only supports only AvroJson output format SerDe."); } this.handler = handler; }
From source file:com.cloudera.sqoop.util.AppendUtils.java
License:Apache License
/** * Creates a unique path object inside the sqoop temporary directory. * /*from w w w. j a v a 2 s.c o m*/ * @param tableName * @return a path pointing to the temporary directory */ public static Path getTempAppendDir(String tableName) { String timeId = DATE_FORM.format(new Date(System.currentTimeMillis())); String tempDir = TEMP_IMPORT_ROOT + Path.SEPARATOR + timeId + tableName; return new Path(tempDir); }