Example usage for org.apache.hadoop.mapreduce TaskID forName

Introduction

In this page you can find the example usage for org.apache.hadoop.mapreduce TaskID forName.

Prototype

public static TaskID forName(String str) throws IllegalArgumentException 

Source Link

Document

Construct a TaskID object from given string.

Usage

From source file:org.apache.carbondata.streaming.CarbonStreamRecordWriter.java

License:Apache License

private void initialize(TaskAttemptContext job) throws IOException {
    // set basic information
    hadoopConf = job.getConfiguration();
    if (carbonLoadModel == null) {
        carbonLoadModel = CarbonStreamOutputFormat.getCarbonLoadModel(hadoopConf);
        if (carbonLoadModel == null) {
            throw new IOException(
                    "CarbonStreamRecordWriter require configuration: mapreduce.output.carbon.load.model");
        }/*from   w w  w  .j  a  v  a2s . com*/
    }
    String segmentId = CarbonStreamOutputFormat.getSegmentId(hadoopConf);
    carbonLoadModel.setSegmentId(segmentId);
    carbonTable = carbonLoadModel.getCarbonDataLoadSchema().getCarbonTable();
    long taskNo = TaskID.forName(hadoopConf.get("mapred.tip.id")).getId();
    carbonLoadModel.setTaskNo("" + taskNo);
    configuration = DataLoadProcessBuilder.createConfiguration(carbonLoadModel);
    maxRowNums = hadoopConf.getInt(CarbonStreamOutputFormat.CARBON_STREAM_BLOCKLET_ROW_NUMS,
            CarbonStreamOutputFormat.CARBON_STREAM_BLOCKLET_ROW_NUMS_DEFAULT) - 1;
    maxCacheSize = hadoopConf.getInt(CarbonStreamOutputFormat.CARBON_STREAM_CACHE_SIZE,
            CarbonStreamOutputFormat.CARBON_STREAM_CACHE_SIZE_DEFAULT);

    segmentDir = CarbonTablePath.getSegmentPath(carbonTable.getAbsoluteTableIdentifier().getTablePath(),
            segmentId);
    fileName = CarbonTablePath.getCarbonDataFileName(0, taskNo, 0, 0, "0", segmentId);

    // initialize metadata
    isNoDictionaryDimensionColumn = CarbonDataProcessorUtil
            .getNoDictionaryMapping(configuration.getDataFields());
    dimensionWithComplexCount = configuration.getDimensionCount();
    measureCount = configuration.getMeasureCount();
    dataFields = configuration.getDataFields();
    measureDataTypes = new DataType[measureCount];
    for (int i = 0; i < measureCount; i++) {
        measureDataTypes[i] = dataFields[dimensionWithComplexCount + i].getColumn().getDataType();
    }
}

---

From source file:org.terrier.indexing.AbstractHadoopIndexer.java

License:Mozilla Public License

public void deleteTaskFiles(String path, JobID job) {
    String[] fileNames = Files.list(path);

    if (fileNames == null)
        return;/*from   w  w w . j  a  v  a 2  s  .c  o  m*/

    for (String filename : fileNames) {
        String periodParts[] = filename.split("\\.");

        try {
            TaskID tid = TaskID.forName(periodParts[0]);

            if (tid.getJobID().compareTo(job) == 0) {
                if (!Files.delete(path + "/" + filename))
                    logger.warn("Could not delete temporary map side-effect file " + path + "/" + filename);
            }
        } catch (Exception e) {
        }

        //remove any empty reduce files created as a side effect of using sequencefileoutputformat rather than nulloutputformat
        if (filename.startsWith("part-r-"))
            Files.delete(path + "/" + filename);
    }
}

---