List of usage examples for org.apache.hadoop.mapreduce TaskAttemptID forName
public static TaskAttemptID forName(String str) throws IllegalArgumentException
From source file:com.clojurewerkz.cascading.cassandra.hadoop.ColumnFamilyInputFormat.java
License:Apache License
public org.apache.hadoop.mapred.RecordReader<ByteBuffer, SortedMap<ByteBuffer, IColumn>> getRecordReader( org.apache.hadoop.mapred.InputSplit split, JobConf jobConf, final Reporter reporter) throws IOException { TaskAttemptContext tac = new TaskAttemptContext(jobConf, TaskAttemptID.forName(jobConf.get(MAPRED_TASK_ID))) { @Override/* w w w . j ava 2 s . c om*/ public void progress() { reporter.progress(); } }; ColumnFamilyRecordReader recordReader = new ColumnFamilyRecordReader( jobConf.getInt(CASSANDRA_HADOOP_MAX_KEY_SIZE, CASSANDRA_HADOOP_MAX_KEY_SIZE_DEFAULT)); recordReader.initialize((org.apache.hadoop.mapreduce.InputSplit) split, tac); return recordReader; }
From source file:com.google.appengine.tools.mapreduce.AppEngineTaskAttemptContext.java
License:Apache License
/** * Gets the task attempt ID from the given request. * * @param req a servlet request with the job ID stored in the * {@link #TASK_ATTEMPT_ID_PARAMETER_NAME} parameter * @return the job ID/*from ww w .j a v a2 s.c o m*/ */ // VisibleForTesting static TaskAttemptID getTaskAttemptIDFromRequest(HttpServletRequest req) { String jobIdString = req.getParameter(TASK_ATTEMPT_ID_PARAMETER_NAME); if (jobIdString == null) { throw new RuntimeException("Couldn't get Job ID for request. Aborting!"); } return TaskAttemptID.forName(jobIdString); }
From source file:com.google.appengine.tools.mapreduce.ShardState.java
License:Apache License
/** * Gets the task attempt ID corresponding to this ShardState. * @return the task attempt ID corresponding to this ShardState *///from ww w. java 2 s.co m public TaskAttemptID getTaskAttemptID() { Preconditions.checkNotNull(entity.getKey().getName(), "ShardState must be persisted to call getTaskID()"); return TaskAttemptID.forName(entity.getKey().getName()); }
From source file:com.ibm.stocator.fs.common.Utils.java
License:Open Source License
/** * Extract Hadoop Task ID from path/* w w w . ja va 2 s . co m*/ * @param path * @return task id */ public static String extractTaskID(String path) { if (path.contains(HADOOP_ATTEMPT)) { String prf = path.substring(path.indexOf(HADOOP_ATTEMPT)); if (prf.contains("/")) { return TaskAttemptID.forName(prf.substring(0, prf.indexOf("/"))).toString(); } return TaskAttemptID.forName(prf).toString(); } return null; }
From source file:com.ibm.stocator.fs.cos.COSAPIClient.java
License:Apache License
/** * Extracts from the object key an unified object name or name without task ID * * @param objectKey//from ww w . java2 s. c o m * @param isUnifiedObjectKey * @return */ private String extractFromObjectKeyWithTaskID(String objectKey, boolean isUnifiedObjectKey) { Path p = new Path(objectKey); int index = objectKey.indexOf("-" + HADOOP_ATTEMPT); if (index > 0) { String attempt = objectKey.substring(objectKey.lastIndexOf("-") + 1); try { TaskAttemptID.forName(attempt); if (isUnifiedObjectKey) { return p.getParent().toString(); } else { return objectKey.substring(0, index); } } catch (IllegalArgumentException e) { return objectKey; } } else if (isUnifiedObjectKey && objectKey.indexOf(HADOOP_SUCCESS) > 0) { return p.getParent().toString(); } return objectKey; }
From source file:com.ibm.stocator.fs.swift.SwiftAPIClient.java
License:Open Source License
/** * Accepts any object name.//from w ww. j ava 2 s.c om * If object name of the form * a/b/c/gil.data/part-r-00000-48ae3461-203f-4dd3-b141-a45426e2d26c * .csv-attempt_20160317132a_wrong_0000_m_000000_1 * Then a/b/c/gil.data is returned. * Code testing that attempt_20160317132a_wrong_0000_m_000000_1 is valid * task id identifier * * @param objectName * @return unified object name */ private String extractUnifiedObjectName(String objectName) { Path p = new Path(objectName); if (objectName.indexOf("-" + HADOOP_ATTEMPT) > 0) { String attempt = objectName.substring(objectName.lastIndexOf("-") + 1); try { TaskAttemptID.forName(attempt); return p.getParent().toString(); } catch (IllegalArgumentException e) { return objectName; } } else if (objectName.indexOf(HADOOP_SUCCESS) > 0) { return p.getParent().toString(); } return objectName; }
From source file:com.ibm.stocator.fs.swift.SwiftAPIClient.java
License:Open Source License
/** * Accepts any object name.//from ww w .j ava 2 s . c o m * If object name is of the form * a/b/c/m.data/part-r-00000-48ae3461-203f-4dd3-b141-a45426e2d26c * .csv-attempt_20160317132a_wrong_0000_m_000000_1 * Then a/b/c/m.data/part-r-00000-48ae3461-203f-4dd3-b141-a45426e2d26c.csv is returned. * Perform test that attempt_20160317132a_wrong_0000_m_000000_1 is valid * task id identifier * * @param objectName * @return unified object name */ private String nameWithoutTaskID(String objectName) { int index = objectName.indexOf("-" + HADOOP_ATTEMPT); if (index > 0) { String attempt = objectName.substring(objectName.lastIndexOf("-") + 1); try { TaskAttemptID.forName(attempt); return objectName.substring(0, index); } catch (IllegalArgumentException e) { return objectName; } } return objectName; }
From source file:com.streamsets.pipeline.stage.origin.hdfs.cluster.ClusterHdfsSource.java
License:Apache License
private List<Map.Entry> previewTextBatch(FileStatus fileStatus, int batchSize) throws IOException, InterruptedException { TextInputFormat textInputFormat = new TextInputFormat(); InputSplit fileSplit = new FileSplit(fileStatus.getPath(), 0, fileStatus.getLen(), null); TaskAttemptContext taskAttemptContext = new TaskAttemptContextImpl(hadoopConf, TaskAttemptID.forName("attempt_1439420318532_0011_m_000000_0")); RecordReader<LongWritable, Text> recordReader = textInputFormat.createRecordReader(fileSplit, taskAttemptContext);/* ww w.java 2 s . co m*/ recordReader.initialize(fileSplit, taskAttemptContext); boolean hasNext = recordReader.nextKeyValue(); List<Map.Entry> batch = new ArrayList<>(); while (hasNext && batch.size() < batchSize) { batch.add(new Pair(fileStatus.getPath().toUri().getPath() + "::" + recordReader.getCurrentKey(), String.valueOf(recordReader.getCurrentValue()))); hasNext = recordReader.nextKeyValue(); // not like iterator.hasNext, actually advances } return batch; }
From source file:com.toshiba.mwcloud.gs.hadoop.mapred.GSRowRecordWriter.java
License:Apache License
/** * <div lang="ja">//from w w w. j a v a 2 s.c om * * @param confConfiguration * @throws IOExceptionGridDB?????? * </div><div lang="en"> * Constructor * @param conf Configuration object * @throws IOException an exception occurred in GridDB * </div> */ public GSRowRecordWriter(JobConf conf) throws IOException { TaskAttemptContext context = new TaskAttemptContextImpl(conf, TaskAttemptID.forName(conf.get("mapred.task.id"))); writer_ = new GDRecordWriter(context); }
From source file:com.tuplejump.calliope.hadoop.ColumnFamilyInputFormat.java
License:Apache License
public org.apache.hadoop.mapred.RecordReader<ByteBuffer, SortedMap<ByteBuffer, Column>> getRecordReader( org.apache.hadoop.mapred.InputSplit split, JobConf jobConf, final Reporter reporter) throws IOException { TaskAttemptContext tac = HadoopCompat.newMapContext(jobConf, TaskAttemptID.forName(jobConf.get(MAPRED_TASK_ID)), null, null, null, new ReporterWrapper(reporter), null);/*from w ww .java 2s . c o m*/ ColumnFamilyRecordReader recordReader = new ColumnFamilyRecordReader( jobConf.getInt(CASSANDRA_HADOOP_MAX_KEY_SIZE, CASSANDRA_HADOOP_MAX_KEY_SIZE_DEFAULT)); recordReader.initialize((org.apache.hadoop.mapreduce.InputSplit) split, tac); return recordReader; }