Example usage for org.apache.hadoop.mapreduce TaskID getId

Introduction

In this page you can find the example usage for org.apache.hadoop.mapreduce TaskID getId.

Prototype

public int getId() 

Source Link

Document

returns the int which represents the identifier

Usage

From source file:co.nubetech.hiho.mapreduce.lib.output.AppendSequenceFileOutputFormat.java

License:Apache License

public synchronized static String getUniqueFile(TaskAttemptContext context, String name, String extension) {

    TaskID taskId = context.getTaskAttemptID().getTaskID();
    int partition = taskId.getId();
    partition = partition + (int) fileCount;
    StringBuilder result = new StringBuilder();
    result.append(name);/* w  w w.j a  v  a 2s .c  om*/
    result.append('-');
    // result.append(taskId.isMap() ? 'm' : 'r');
    result.append('-');
    result.append(NUMBER_FORMAT.format(partition));
    result.append(extension);
    return result.toString();
}

---

From source file:com.bonc.mr_roamRecognition_hjpt.comm.NewFileOutputFormat.java

License:Apache License

/**
 * Generate a unique filename, based on the task id, name, and extension
 * /*from   ww  w .  j  ava2s .c  om*/
 * @param context
 *            the task that is calling this
 * @param name
 *            the base filename
 * @param extension
 *            the filename extension
 * @return a string like $name-[mrsct]-$id$extension
 */
public synchronized static String getUniqueFile(TaskAttemptContext context, String name, String extension) {
    TaskID taskId = context.getTaskAttemptID().getTaskID();
    int partition = taskId.getId();
    StringBuilder sb = new StringBuilder();
    sb.append(name);
    sb.append(extension);
    String result = sb.toString().replace("{fileCount}", NUMBER_FORMAT.format(partition));
    return result;
}

---

From source file:com.cloudera.dataflow.spark.ShardNameTemplateHelper.java

License:Open Source License

private static String getOutputFile(TaskAttemptContext context) {
    TaskID taskId = context.getTaskAttemptID().getTaskID();
    int partition = taskId.getId();

    String filePrefix = context.getConfiguration().get(OUTPUT_FILE_PREFIX);
    String fileTemplate = context.getConfiguration().get(OUTPUT_FILE_TEMPLATE);
    String fileSuffix = context.getConfiguration().get(OUTPUT_FILE_SUFFIX);
    return filePrefix + replaceShardNumber(fileTemplate, partition) + fileSuffix;
}

---

From source file:com.datasalt.pangool.solr.SolrRecordWriter.java

License:Apache License

private String getOutFileName(TaskAttemptContext context, String prefix) {
    TaskID taskId = context.getTaskAttemptID().getTaskID();
    int partition = taskId.getId();
    NumberFormat nf = NumberFormat.getInstance();
    nf.setMinimumIntegerDigits(5);//from w  ww.  j av  a  2s . com
    nf.setGroupingUsed(false);
    StringBuilder result = new StringBuilder();
    result.append(prefix);
    result.append("-");
    result.append(nf.format(partition));
    return result.toString();
}

---

From source file:com.david.mos.out.FileOutputFormat.java

License:Apache License

/**
 * Generate a unique filename, based on the task id, name, and extension
 * @param context the task that is calling this
 * @param name the base filename/*from  ww w .ja  v a 2  s .com*/
 * @param extension the filename extension
 * @return a string like $name-[mrsct]-$id$extension
 */
public synchronized static String getUniqueFile(TaskAttemptContext context, String name, String extension) {
    TaskID taskId = context.getTaskAttemptID().getTaskID();
    int partition = taskId.getId();
    StringBuilder result = new StringBuilder();
    result.append(name); //TODO enb-r-00000
    System.out.println("###################################" + name);
    //    result.append('-');
    //    result.append(
    //        TaskID.getRepresentingCharacter(taskId.getTaskType()));
    //    result.append('-');
    //    result.append(NUMBER_FORMAT.format(partition));
    //    result.append(extension);
    return result.toString();
}

---

From source file:cz.seznam.euphoria.hadoop.output.TestDataSinkOutputFormat.java

License:Apache License

private TaskAttemptContext mockContext(Configuration conf, int taskId) {
    TaskAttemptContext ret = mock(TaskAttemptContext.class);
    TaskAttemptID mockAttemptId = mock(TaskAttemptID.class);
    TaskID mockTaskId = mock(TaskID.class);
    when(ret.getConfiguration()).thenReturn(conf);
    when(ret.getTaskAttemptID()).thenReturn(mockAttemptId);
    when(mockAttemptId.getTaskID()).thenReturn(mockTaskId);
    when(mockTaskId.getId()).thenReturn(taskId);
    return ret;/*from  ww  w. j av a2  s .  com*/
}

---

From source file:edu.arizona.cs.hadoop.fs.irods.output.HirodsFileOutputFormat.java

License:Apache License

/**
 * Generate a unique filename, based on the task id, name, and extension
 *
 * @param context the task that is calling this
 * @param name the base filename/*from   www .j a va2s  . com*/
 * @param extension the filename extension
 * @return a string like $name-[mr]-$id$extension
 */
public synchronized static String getUniqueFile(TaskAttemptContext context, String name, String extension) {
    TaskID taskId = context.getTaskAttemptID().getTaskID();
    int partition = taskId.getId();
    StringBuilder result = new StringBuilder();
    result.append(name);
    result.append('-');
    result.append(taskId.isMap() ? 'm' : 'r');
    result.append('-');
    result.append(NUMBER_FORMAT.format(partition));
    result.append(extension);
    return result.toString();
}

---

From source file:org.apache.beam.sdk.io.hadoop.format.HDFSSynchronizationTest.java

License:Apache License

private String getTaskIdPath(TaskID taskID) {
    return getFileInJobFolder(String.valueOf(taskID.getId()));
}

---

From source file:org.apache.mnemonic.hadoop.MneDurableOutputSession.java

License:Apache License

protected String getUniqueName(String name, String extension) {
    int partition;

    NumberFormat numberFormat = NumberFormat.getInstance();
    numberFormat.setMinimumIntegerDigits(5);
    numberFormat.setGroupingUsed(false);

    if (null != getTaskAttemptContext()) {
        TaskID taskId = getTaskAttemptContext().getTaskAttemptID().getTaskID();
        partition = taskId.getId();
    } else {//from w  w  w. j a v a 2  s . com
        partition = getConfiguration().getInt(JobContext.TASK_PARTITION, -1);
    }
    if (partition == -1) {
        throw new IllegalArgumentException("This method can only be called from an application");
    }

    String taskType = getConfiguration().getBoolean(JobContext.TASK_ISMAP, JobContext.DEFAULT_TASK_ISMAP) ? "m"
            : "r";

    StringBuilder result = new StringBuilder();
    result.append(name);
    result.append('-');
    result.append(taskType);
    result.append('-');
    result.append(numberFormat.format(partition));
    result.append(extension);
    return result.toString();

}

---

From source file:org.apache.tez.mapreduce.task.impl.YarnOutputFiles.java

License:Apache License

/**
 * Create a local reduce input file name.
 * /*from w  ww  .  j a va  2 s.c om*/
 * @param mapId a map task id
 * @param size the size of the file
 * @return path
 * @throws IOException
 */
public Path getInputFileForWrite(org.apache.hadoop.mapreduce.TaskID mapId, long size) throws IOException {
    return lDirAlloc.getLocalPathForWrite(String.format(Constants.REDUCE_INPUT_FILE_FORMAT_STRING,
            getAttemptOutputDir().toString(), mapId.getId()), size, conf);
}

---

• «
• 1
• 2
• »