List of usage examples for org.apache.hadoop.mapreduce.filecache DistributedCache createSymlink
@Deprecated public static void createSymlink(Configuration conf)
From source file:com.ailk.oci.ocnosql.tools.load.single.SingleColumnImportTsv.java
License:Apache License
/** * Configure a MapReduce Job to perform an incremental load into the given * table. This/*from www . j av a 2 s. co m*/ * <ul> * <li>Inspects the table to configure a total order partitioner</li> * <li>Uploads the partitions file to the cluster and adds it to the DistributedCache</li> * <li>Sets the number of reduce tasks to match the current number of regions</li> * <li>Sets the output key/value class to match HFileOutputFormat's requirements</li> * <li>Sets the reducer up to perform the appropriate sorting (either KeyValueSortReducer or * PutSortReducer)</li> * </ul> * The user should be sure to set the map output value class to either KeyValue or Put before * running this function. */ public static void configureIncrementalLoad(Job job, HTable table) throws IOException { Configuration conf = job.getConfiguration(); Class<? extends Partitioner> topClass; try { topClass = getTotalOrderPartitionerClass(); } catch (ClassNotFoundException e) { throw new IOException("Failed getting TotalOrderPartitioner", e); } //partition job.setPartitionerClass(topClass); //Set the key class for the job output data job.setOutputKeyClass(ImmutableBytesWritable.class); //Set the value class for job outputs job.setOutputValueClass(KeyValue.class); //outputformatHfile job.setOutputFormatClass(HFileOutputFormat2.class); // Based on the configured map output class, set the correct reducer to properly // sort the incoming values. // TODO it would be nice to pick one or the other of these formats. if (KeyValue.class.equals(job.getMapOutputValueClass())) { job.setReducerClass(KeyValueSortReducer.class); } else if (Put.class.equals(job.getMapOutputValueClass())) { job.setReducerClass(SingleColumnReducer.class); } else { LOG.warn("Unknown map output value type:" + job.getMapOutputValueClass()); } LOG.info("Looking up current regions for table " + table); //?regionstarkey List<ImmutableBytesWritable> startKeys = getRegionStartKeys(table); LOG.info("Configuring " + startKeys.size() + " reduce partitions " + "to match current region count"); //?region?reduce? job.setNumReduceTasks(startKeys.size()); Path partitionsPath = new Path(job.getWorkingDirectory(), "partitions_" + UUID.randomUUID()); LOG.info("Writing partition information to " + partitionsPath); FileSystem fs = partitionsPath.getFileSystem(conf); writePartitions(conf, partitionsPath, startKeys); partitionsPath.makeQualified(fs); URI cacheUri; try { // Below we make explicit reference to the bundled TOP. Its cheating. // We are assume the define in the hbase bundled TOP is as it is in // hadoop (whether 0.20 or 0.22, etc.) /* cacheUri = new URI(partitionsPath.toString() + "#" + org.apache.hadoop.hbase.mapreduce.hadoopbackport.TotalOrderPartitioner.DEFAULT_PATH); */ cacheUri = new URI(partitionsPath.toString() + "#" + TotalOrderPartitioner.DEFAULT_PATH); } catch (URISyntaxException e) { throw new IOException(e); } DistributedCache.addCacheFile(cacheUri, conf); DistributedCache.createSymlink(conf); // Set compression algorithms based on column families configureCompression(table, conf); TableMapReduceUtil.addDependencyJars(job); LOG.info("Incremental table output configured."); }
From source file:gobblin.runtime.mapreduce.MRJobLauncher.java
License:Apache License
/** * Add local non-jar files the job depends on to DistributedCache. *//* ww w. j a v a 2s.c o m*/ @SuppressWarnings("deprecation") private void addLocalFiles(Path jobFileDir, String jobFileList, Configuration conf) throws IOException { DistributedCache.createSymlink(conf); for (String jobFile : SPLITTER.split(jobFileList)) { Path srcJobFile = new Path(jobFile); // DistributedCache requires absolute path, so we need to use makeQualified. Path destJobFile = new Path(this.fs.makeQualified(jobFileDir), srcJobFile.getName()); // Copy the file from local file system to HDFS this.fs.copyFromLocalFile(srcJobFile, destJobFile); // Create a URI that is in the form path#symlink URI destFileUri = URI.create(destJobFile.toUri().getPath() + "#" + destJobFile.getName()); LOG.info(String.format("Adding %s to DistributedCache", destFileUri)); // Finally add the file to DistributedCache with a symlink named after the file name DistributedCache.addCacheFile(destFileUri, conf); } }
From source file:gobblin.runtime.mapreduce.MRJobLauncher.java
License:Apache License
/** * Add non-jar files already on HDFS that the job depends on to DistributedCache. *//*from ww w . j av a 2 s . c o m*/ @SuppressWarnings("deprecation") private void addHDFSFiles(String jobFileList, Configuration conf) { DistributedCache.createSymlink(conf); jobFileList = PasswordManager.getInstance(this.jobProps).readPassword(jobFileList); for (String jobFile : SPLITTER.split(jobFileList)) { Path srcJobFile = new Path(jobFile); // Create a URI that is in the form path#symlink URI srcFileUri = URI.create(srcJobFile.toUri().getPath() + "#" + srcJobFile.getName()); LOG.info(String.format("Adding %s to DistributedCache", srcFileUri)); // Finally add the file to DistributedCache with a symlink named after the file name DistributedCache.addCacheFile(srcFileUri, conf); } }
From source file:org.apache.sysml.runtime.matrix.CSVReblockMR.java
License:Apache License
private static JobReturn runCSVReblockJob(MRJobInstruction inst, String[] inputs, InputInfo[] inputInfos, long[] rlens, long[] clens, int[] brlens, int[] bclens, String reblockInstructions, String otherInstructionsInReducer, int numReducers, int replication, byte[] resultIndexes, String[] outputs, OutputInfo[] outputInfos, Path counterFile, String[] smallestFiles) throws Exception { JobConf job;/*from w ww . ja v a 2s . c o m*/ job = new JobConf(ReblockMR.class); job.setJobName("CSV-Reblock-MR"); byte[] realIndexes = new byte[inputs.length]; for (byte b = 0; b < realIndexes.length; b++) realIndexes[b] = b; //set up the input files and their format information MRJobConfiguration.setUpMultipleInputs(job, realIndexes, inputs, inputInfos, brlens, bclens, false, ConvertTarget.CELL); job.setStrings(SMALLEST_FILE_NAME_PER_INPUT, smallestFiles); //set up the dimensions of input matrices MRJobConfiguration.setMatricesDimensions(job, realIndexes, rlens, clens); //set up the block size MRJobConfiguration.setBlocksSizes(job, realIndexes, brlens, bclens); //set up the aggregate instructions that will happen in the combiner and reducer MRJobConfiguration.setCSVReblockInstructions(job, reblockInstructions); //set up the instructions that will happen in the reducer, after the aggregation instrucions MRJobConfiguration.setInstructionsInReducer(job, otherInstructionsInReducer); //set up the replication factor for the results job.setInt(MRConfigurationNames.DFS_REPLICATION, replication); //set up preferred custom serialization framework for binary block format if (MRJobConfiguration.USE_BINARYBLOCK_SERIALIZATION) MRJobConfiguration.addBinaryBlockSerializationFramework(job); //set up custom map/reduce configurations DMLConfig config = ConfigurationManager.getDMLConfig(); MRJobConfiguration.setupCustomMRConfigurations(job, config); //set up what matrices are needed to pass from the mapper to reducer HashSet<Byte> mapoutputIndexes = MRJobConfiguration.setUpOutputIndexesForMapper(job, realIndexes, null, reblockInstructions, null, otherInstructionsInReducer, resultIndexes); MatrixChar_N_ReducerGroups ret = MRJobConfiguration.computeMatrixCharacteristics(job, realIndexes, null, reblockInstructions, null, null, otherInstructionsInReducer, resultIndexes, mapoutputIndexes, false); MatrixCharacteristics[] stats = ret.stats; //set up the number of reducers int numRed = WriteCSVMR.determineNumReducers(rlens, clens, config.getIntValue(DMLConfig.NUM_REDUCERS), ret.numReducerGroups); job.setNumReduceTasks(numRed); // Print the complete instruction //if (LOG.isTraceEnabled()) // inst.printCompelteMRJobInstruction(stats); // Update resultDimsUnknown based on computed "stats" byte[] resultDimsUnknown = new byte[resultIndexes.length]; for (int i = 0; i < resultIndexes.length; i++) { if (stats[i].getRows() == -1 || stats[i].getCols() == -1) { resultDimsUnknown[i] = (byte) 1; } else { resultDimsUnknown[i] = (byte) 0; } } //set up the multiple output files, and their format information MRJobConfiguration.setUpMultipleOutputs(job, resultIndexes, resultDimsUnknown, outputs, outputInfos, true, true); // configure mapper and the mapper output key value pairs job.setMapperClass(CSVReblockMapper.class); job.setMapOutputKeyClass(TaggedFirstSecondIndexes.class); job.setMapOutputValueClass(BlockRow.class); //configure reducer job.setReducerClass(CSVReblockReducer.class); //turn off adaptivemr job.setBoolean("adaptivemr.map.enable", false); //set unique working dir MRJobConfiguration.setUniqueWorkingDir(job); Path cachefile = new Path(counterFile, "part-00000"); DistributedCache.addCacheFile(cachefile.toUri(), job); DistributedCache.createSymlink(job); job.set(ROWID_FILE_NAME, cachefile.toString()); RunningJob runjob = JobClient.runJob(job); MapReduceTool.deleteFileIfExistOnHDFS(counterFile, job); /* Process different counters */ Group group = runjob.getCounters().getGroup(MRJobConfiguration.NUM_NONZERO_CELLS); for (int i = 0; i < resultIndexes.length; i++) { // number of non-zeros stats[i].setNonZeros(group.getCounter(Integer.toString(i))); // System.out.println("result #"+resultIndexes[i]+" ===>\n"+stats[i]); } return new JobReturn(stats, outputInfos, runjob.isSuccessful()); }
From source file:org.apache.sysml.runtime.matrix.mapred.MRJobConfiguration.java
License:Apache License
public static void setupDistCacheInputs(JobConf job, String indices, String pathsString, ArrayList<String> paths) { job.set(DISTCACHE_INPUT_INDICES, indices); job.set(DISTCACHE_INPUT_PATHS, pathsString); Path p = null;/*from w ww. jav a 2s . c o m*/ for (String spath : paths) { p = new Path(spath); DistributedCache.addCacheFile(p.toUri(), job); DistributedCache.createSymlink(job); } }
From source file:org.apache.sysml.runtime.matrix.SortMR.java
License:Apache License
@SuppressWarnings({ "unchecked", "rawtypes" })
public static JobReturn runJob(MRJobInstruction inst, String input, InputInfo inputInfo, long rlen, long clen,
int brlen, int bclen, String combineInst, String sortInst, int numReducers, int replication,
String output, OutputInfo outputInfo, boolean valueIsWeight) throws Exception {
boolean sortIndexes = getSortInstructionType(sortInst) == SortKeys.OperationTypes.Indexes;
String tmpOutput = sortIndexes ? MRJobConfiguration.constructTempOutputFilename() : output;
JobConf job = new JobConf(SortMR.class);
job.setJobName("SortMR");
//setup partition file
String pfname = MRJobConfiguration.setUpSortPartitionFilename(job);
Path partitionFile = new Path(pfname);
URI partitionUri = new URI(partitionFile.toString());
//setup input/output paths
Path inputDir = new Path(input);
inputDir = inputDir.makeQualified(inputDir.getFileSystem(job));
FileInputFormat.setInputPaths(job, inputDir);
Path outpath = new Path(tmpOutput);
FileOutputFormat.setOutputPath(job, outpath);
MapReduceTool.deleteFileIfExistOnHDFS(outpath, job);
//set number of reducers (1 if local mode)
if (!InfrastructureAnalyzer.isLocalMode(job)) {
MRJobConfiguration.setNumReducers(job, numReducers, numReducers);
//ensure partition size <= 10M records to avoid scalability bottlenecks
//on cp-side qpick instructions for quantile/iqm/median (~128MB)
if (!(getSortInstructionType(sortInst) == SortKeys.OperationTypes.Indexes))
job.setNumReduceTasks((int) Math.max(job.getNumReduceTasks(), rlen / 10000000));
} else //in case of local mode
job.setNumReduceTasks(1);//from ww w. ja v a 2s . c o m
//setup input/output format
job.setInputFormat(SamplingSortMRInputFormat.class);
SamplingSortMRInputFormat.setTargetKeyValueClasses(job,
(Class<? extends WritableComparable>) outputInfo.outputKeyClass, outputInfo.outputValueClass);
//setup instructions and meta information
if (combineInst != null && !combineInst.trim().isEmpty())
job.set(COMBINE_INSTRUCTION, combineInst);
job.set(SORT_INSTRUCTION, sortInst);
job.setBoolean(VALUE_IS_WEIGHT, valueIsWeight);
boolean desc = getSortInstructionDescending(sortInst);
job.setBoolean(SORT_DECREASING, desc);
MRJobConfiguration.setBlockSize(job, (byte) 0, brlen, bclen);
MRJobConfiguration.setInputInfo(job, (byte) 0, inputInfo, brlen, bclen, ConvertTarget.CELL);
int partitionWith0 = SamplingSortMRInputFormat.writePartitionFile(job, partitionFile);
//setup mapper/reducer/partitioner/output classes
if (getSortInstructionType(sortInst) == SortKeys.OperationTypes.Indexes) {
MRJobConfiguration.setInputInfo(job, (byte) 0, inputInfo, brlen, bclen, ConvertTarget.CELL);
job.setOutputFormat(OutputInfo.BinaryBlockOutputInfo.outputFormatClass);
job.setMapperClass(IndexSortMapper.class);
job.setReducerClass(IndexSortReducer.class);
job.setMapOutputKeyClass(!desc ? IndexSortComparable.class : IndexSortComparableDesc.class);
job.setMapOutputValueClass(LongWritable.class);
job.setOutputKeyClass(MatrixIndexes.class);
job.setOutputValueClass(MatrixBlock.class);
} else { //default case: SORT w/wo weights
MRJobConfiguration.setInputInfo(job, (byte) 0, inputInfo, brlen, bclen, ConvertTarget.CELL);
job.setOutputFormat(CompactOutputFormat.class);
job.setMapperClass(ValueSortMapper.class);
job.setReducerClass(ValueSortReducer.class);
job.setOutputKeyClass(outputInfo.outputKeyClass); //double
job.setOutputValueClass(outputInfo.outputValueClass); //int
}
job.setPartitionerClass(TotalOrderPartitioner.class);
//setup distributed cache
DistributedCache.addCacheFile(partitionUri, job);
DistributedCache.createSymlink(job);
//setup replication factor
job.setInt(MRConfigurationNames.DFS_REPLICATION, replication);
//set up custom map/reduce configurations
DMLConfig config = ConfigurationManager.getDMLConfig();
MRJobConfiguration.setupCustomMRConfigurations(job, config);
MatrixCharacteristics[] s = new MatrixCharacteristics[1];
s[0] = new MatrixCharacteristics(rlen, clen, brlen, bclen);
// Print the complete instruction
if (LOG.isTraceEnabled())
inst.printCompleteMRJobInstruction(s);
//set unique working dir
MRJobConfiguration.setUniqueWorkingDir(job);
//run mr job
RunningJob runjob = JobClient.runJob(job);
Group group = runjob.getCounters().getGroup(NUM_VALUES_PREFIX);
numReducers = job.getNumReduceTasks();
//process final meta data
long[] counts = new long[numReducers];
long total = 0;
for (int i = 0; i < numReducers; i++) {
counts[i] = group.getCounter(Integer.toString(i));
total += counts[i];
}
//add missing 0s back to the results
long missing0s = 0;
if (total < rlen * clen) {
if (partitionWith0 < 0)
throw new RuntimeException("no partition contains 0, which is wrong!");
missing0s = rlen * clen - total;
counts[partitionWith0] += missing0s;
} else
partitionWith0 = -1;
if (sortIndexes) {
//run builtin job for shifting partially sorted blocks according to global offsets
//we do this in this custom form since it would not fit into the current structure
//of systemml to output two intermediates (partially sorted data, offsets) out of a
//single SortKeys lop
boolean success = runjob.isSuccessful();
if (success) {
success = runStitchupJob(tmpOutput, rlen, clen, brlen, bclen, counts, numReducers, replication,
output);
}
MapReduceTool.deleteFileIfExistOnHDFS(tmpOutput);
MapReduceTool.deleteFileIfExistOnHDFS(pfname);
return new JobReturn(s[0], OutputInfo.BinaryBlockOutputInfo, success);
} else {
MapReduceTool.deleteFileIfExistOnHDFS(pfname);
return new JobReturn(s[0], counts, partitionWith0, missing0s, runjob.isSuccessful());
}
}