Example usage for org.apache.commons.io FileUtils ONE_MB

Introduction

In this page you can find the example usage for org.apache.commons.io FileUtils ONE_MB.

Prototype

long ONE_MB

To view the source code for org.apache.commons.io FileUtils ONE_MB.

Click Source Link

Document

The number of bytes in a megabyte.

Usage

From source file:com.linkedin.drelephant.tez.heuristics.ReducerMemoryHeuristicTest.java

private Severity analyzeJob(long taskAvgMemMB, long containerMemMB) throws IOException {
    TezCounterData jobCounter = new TezCounterData();
    TezTaskData[] reducers = new TezTaskData[NUMTASKS + 1];

    TezCounterData counter = new TezCounterData();
    counter.set(TezCounterData.CounterName.PHYSICAL_MEMORY_BYTES, taskAvgMemMB * FileUtils.ONE_MB);

    Properties p = new Properties();
    p.setProperty(com.linkedin.drelephant.mapreduce.heuristics.ReducerMemoryHeuristic.REDUCER_MEMORY_CONF,
            Long.toString(containerMemMB));

    int i = 0;//from  ww  w . j ava  2s. co  m
    for (; i < NUMTASKS; i++) {
        reducers[i] = new TezTaskData("task-id-" + i, "task-attempt-id-" + i);
        reducers[i].setTimeAndCounter(new long[5], counter);
    }
    // Non-sampled task, which does not contain time and counter data
    reducers[i] = new TezTaskData("task-id-" + i, "task-attempt-id-" + i);

    TezApplicationData data = new TezApplicationData().setCounters(jobCounter).setReduceTaskData(reducers);
    data.setConf(p);
    HeuristicResult result = _heuristic.apply(data);
    return result.getSeverity();
}

---

From source file:com.linkedin.drelephant.mapreduce.heuristics.GenericMemoryHeuristic.java

private void loadParameters() {
    Map<String, String> paramMap = _heuristicConfData.getParamMap();
    String heuristicName = _heuristicConfData.getHeuristicName();

    double[] confMemRatioLimits = Utils.getParam(paramMap.get(MEM_RATIO_SEVERITY), memRatioLimits.length);
    if (confMemRatioLimits != null) {
        memRatioLimits = confMemRatioLimits;
    }//from  w w w . j a va  2 s .com
    logger.info(heuristicName + " will use " + MEM_RATIO_SEVERITY + " with the following threshold settings: "
            + Arrays.toString(memRatioLimits));

    long containerMemDefaultBytes = CONTAINER_MEMORY_DEFAULT_BYTES;
    if (paramMap.containsKey(CONTAINER_MEM_DEFAULT_MB)) {
        containerMemDefaultBytes = Long.valueOf(paramMap.get(CONTAINER_MEM_DEFAULT_MB)) * FileUtils.ONE_MB;
    }
    logger.info(heuristicName + " will use " + CONTAINER_MEM_DEFAULT_MB
            + " with the following threshold setting: " + containerMemDefaultBytes);

    double[] confMemoryLimits = Utils.getParam(paramMap.get(CONTAINER_MEM_SEVERITY), memoryLimits.length);
    if (confMemoryLimits != null) {
        memoryLimits = confMemoryLimits;
    }
    logger.info(heuristicName + " will use " + CONTAINER_MEM_SEVERITY
            + " with the following threshold settings: " + Arrays.toString(memoryLimits));
    for (int i = 0; i < memoryLimits.length; i++) {
        memoryLimits[i] = memoryLimits[i] * containerMemDefaultBytes;
    }
}

---

From source file:eu.annocultor.reports.parts.ReportCounter.java

@SuppressWarnings("unchecked")
@Override/*from  ww w .  j ava  2s  .  co m*/
public void load() throws IOException {
    XStream xStream = new XStream();
    //BufferedReader reader = new BufferedReader(new FileReader(getFile())); 
    CountingInputStream cis = new CountingInputStream(new FileInputStream(getFile()));
    ObjectInputStream in = xStream.createObjectInputStream(cis);
    try {
        int mb = 1;
        // how ugly but it should throw exception at the end
        while (true) {
            ObjectCountPair<T> ocp = (ObjectCountPair<T>) in.readObject();
            inc(ocp.getObject(), ocp.getCount());
            if (cis.getByteCount() > FileUtils.ONE_MB * 100 * mb) {
                log.info("Passed " + (cis.getByteCount() / FileUtils.ONE_MB) + " MB");
                mb++;
            }
        }
    } catch (EOFException e) {
        // normal end of file
    } catch (ClassNotFoundException e) {
        throw new IOException(e);
    } finally {
        in.close();
        cis.close();
    }
}

---

From source file:com.linkedin.drelephant.spark.TestSparkAggregatedMetrics.java

@Test
public void TestValidExecutorsAndValidEnvironmentData() {
    ApplicationType appType = new ApplicationType("SPARK");
    AggregatorConfigurationData conf = new AggregatorConfigurationData(
            "org.apache.spark.SparkMetricsAggregator", appType, null);
    SparkMetricsAggregator metrics = new SparkMetricsAggregator(conf);

    MockSparkApplicationData appData = new MockSparkApplicationData();
    appData.getExecutorData().setExecutorInfo("1",
            mockExecutorInfo(100 * FileUtils.ONE_MB, 60 * FileUtils.ONE_MB, 1000));
    appData.getExecutorData().setExecutorInfo("2",
            mockExecutorInfo(100 * FileUtils.ONE_MB, 60 * FileUtils.ONE_MB, 1000));

    appData.getEnvironmentData().addSparkProperty(SPARK_EXECUTOR_MEMORY, "1048576000");

    metrics.aggregate(appData);/*from  w ww  .  j  a v  a2  s .c  om*/

    Assert.assertEquals(2000L, metrics.getResult().getResourceUsed());
    Assert.assertEquals(20L, metrics.getResult().getResourceWasted());
    Assert.assertEquals(0L, metrics.getResult().getTotalDelay());
}

---

From source file:com.norconex.committer.core.impl.MultiCommitter.java

@Override
public void add(String reference, InputStream content, Properties metadata) {
    CachedStreamFactory factory = new CachedStreamFactory((int) FileUtils.ONE_MB, (int) FileUtils.ONE_MB);
    CachedInputStream cachedInputStream = factory.newInputStream(content);
    for (int i = 0; i < committers.size(); i++) {
        ICommitter committer = committers.get(i);
        committer.add(reference, cachedInputStream, metadata);
    }/*w  ww  . j  a  v a 2 s . c om*/
}

---

From source file:com.linkedin.drelephant.mapreduce.TaskLevelAggregatedMetrics.java

/**
 * Computes the aggregated metrics -> peakMemory, delay, total task duration, wasted resources and memory usage.
 * @param taskDatas// w  ww.j a v a2  s  .  co  m
 * @param containerSize
 * @param idealStartTime
 */
private void compute(MapReduceTaskData[] taskDatas, long containerSize, long idealStartTime) {

    long peakMemoryNeed = 0;
    long taskFinishTimeMax = 0;
    long taskDurationMax = 0;

    // if there are zero tasks, then nothing to compute.
    if (taskDatas == null || taskDatas.length == 0) {
        return;
    }

    for (MapReduceTaskData taskData : taskDatas) {
        long taskMemory = taskData.getCounters().get(MapReduceCounterData.CounterName.PHYSICAL_MEMORY_BYTES)
                / FileUtils.ONE_MB; // MB
        long taskVM = taskData.getCounters().get(MapReduceCounterData.CounterName.VIRTUAL_MEMORY_BYTES)
                / FileUtils.ONE_MB; // MB
        long taskDuration = taskData.getFinishTimeMs() - taskData.getStartTimeMs(); // Milliseconds
        long taskCost = (containerSize) * (taskDuration / Statistics.SECOND_IN_MS); // MB Seconds

        durations.add(taskDuration);
        finishTimes.add(taskData.getFinishTimeMs());

        //peak Memory usage
        long memoryRequiredForVM = (long) (taskVM / CLUSTER_MEMORY_FACTOR);
        long biggerMemoryRequirement = memoryRequiredForVM > taskMemory ? memoryRequiredForVM : taskMemory;
        peakMemoryNeed = biggerMemoryRequirement > peakMemoryNeed ? biggerMemoryRequirement : peakMemoryNeed;

        if (taskFinishTimeMax < taskData.getFinishTimeMs()) {
            taskFinishTimeMax = taskData.getFinishTimeMs();
        }

        if (taskDurationMax < taskDuration) {
            taskDurationMax = taskDuration;
        }
        _resourceUsed += taskCost;
    }

    // Compute the delay in starting the task.
    _delay = taskFinishTimeMax - (idealStartTime + taskDurationMax);

    // invalid delay
    if (_delay < 0) {
        _delay = 0;
    }

    // wastedResources
    long wastedMemory = containerSize - (long) (peakMemoryNeed * MEMORY_BUFFER); // give a 50% buffer
    if (wastedMemory > 0) {
        for (long duration : durations) {
            _resourceWasted += (wastedMemory) * (duration / Statistics.SECOND_IN_MS); // MB Seconds
        }
    }
}

---

From source file:com.linkedin.drelephant.tez.TezTaskLevelAggregatedMetrics.java

/**
 * Computes the aggregated metrics -> peakMemory, delay, total task duration, wasted resources and memory usage.
 * @param taskDatas/*  ww w.j  ava2  s  .  c  om*/
 * @param containerSize
 * @param idealStartTime
 */
private void compute(TezTaskData[] taskDatas, long containerSize, long idealStartTime) {

    long peakMemoryNeed = 0;
    long taskFinishTimeMax = 0;
    long taskDurationMax = 0;

    // if there are zero tasks, then nothing to compute.
    if (taskDatas == null || taskDatas.length == 0) {
        return;
    }

    for (TezTaskData taskData : taskDatas) {
        if (!taskData.isSampled()) {
            continue;
        }
        long taskMemory = taskData.getCounters().get(TezCounterData.CounterName.PHYSICAL_MEMORY_BYTES)
                / FileUtils.ONE_MB; // MB
        long taskVM = taskData.getCounters().get(TezCounterData.CounterName.VIRTUAL_MEMORY_BYTES)
                / FileUtils.ONE_MB; // MB
        long taskDuration = taskData.getFinishTime() - taskData.getStartTime(); // Milliseconds
        long taskCost = (containerSize) * (taskDuration / Statistics.SECOND_IN_MS); // MB Seconds

        durations.add(taskDuration);
        finishTimes.add(taskData.getFinishTime());

        //peak Memory usage
        long memoryRequiredForVM = (long) (taskVM / CLUSTER_MEMORY_FACTOR);
        long biggerMemoryRequirement = memoryRequiredForVM > taskMemory ? memoryRequiredForVM : taskMemory;
        peakMemoryNeed = biggerMemoryRequirement > peakMemoryNeed ? biggerMemoryRequirement : peakMemoryNeed;

        if (taskFinishTimeMax < taskData.getFinishTime()) {
            taskFinishTimeMax = taskData.getFinishTime();
        }

        if (taskDurationMax < taskDuration) {
            taskDurationMax = taskDuration;
        }
        _resourceUsed += taskCost;
    }

    // Compute the delay in starting the task.
    _delay = taskFinishTimeMax - (idealStartTime + taskDurationMax);

    // invalid delay
    if (_delay < 0) {
        _delay = 0;
    }

    // wastedResources
    long wastedMemory = containerSize - (long) (peakMemoryNeed * MEMORY_BUFFER);
    if (wastedMemory > 0) {
        for (long duration : durations) {
            _resourceWasted += (wastedMemory) * (duration / Statistics.SECOND_IN_MS); // MB Seconds
        }
    }
}

---

From source file:eu.annocultor.data.sources.XmlDataSource.java

@Override
public void feedData(ConverterHandler handler, Path recordSeparatingPath, Path recordIdentifyingPath)
        throws Exception {

    int result = 0;
    if (isMergeSourceFiles()) {
        handler.multiFileStartDocument();
        handler.startElement("", XConverterFactory.MERGED_SOURCES_OUTER_TAG_FILESET,
                XConverterFactory.MERGED_SOURCES_OUTER_TAG_FILESET, null);
    }// w w w  .  j  a  va  2s  . c  om

    int current = 1;
    for (File src : srcFiles) {
        if (isMergeSourceFiles()) {
            handler.startElement("", XConverterFactory.MERGED_SOURCES_OUTER_TAG_FILE,
                    XConverterFactory.MERGED_SOURCES_OUTER_TAG_FILE, new AttributesProxy(src));
        }

        if (result == 0) {
            log.info("File " + (current++) + "/" + srcFiles.size() + " " + src.getName() + " of "
                    + (src.length() / FileUtils.ONE_MB) + " Mb");
            result = parseSourceFile(handler, src, recordSeparatingPath);
        }
        if (isMergeSourceFiles()) {
            handler.endElement("", XConverterFactory.MERGED_SOURCES_OUTER_TAG_FILE,
                    XConverterFactory.MERGED_SOURCES_OUTER_TAG_FILE);
        }
    }
    if (isMergeSourceFiles()) {
        handler.endElement("", XConverterFactory.MERGED_SOURCES_OUTER_TAG_FILESET,
                XConverterFactory.MERGED_SOURCES_OUTER_TAG_FILESET);
        handler.multiFileEndDocument();
    }
    handler.setConversionResult(result == 0 ? ConversionResult.success : ConversionResult.failure);
}

---

From source file:com.linkedin.drelephant.tez.heuristics.GenericMemoryHeuristic.java

public HeuristicResult apply(TezApplicationData data) {
    if (!data.getSucceeded()) {
        return null;
    }//from www .  j  a  v  a  2  s . c o  m
    TezTaskData[] tasks = getTasks(data);

    List<Long> totalPhysicalMemory = new LinkedList<Long>();
    List<Long> totalVirtualMemory = new LinkedList<Long>();
    List<Long> runTime = new LinkedList<Long>();

    for (TezTaskData task : tasks) {

        if (task.isSampled()) {
            totalPhysicalMemory.add(task.getCounters().get(TezCounterData.CounterName.PHYSICAL_MEMORY_BYTES));
            totalVirtualMemory.add(task.getCounters().get(TezCounterData.CounterName.VIRTUAL_MEMORY_BYTES));
            runTime.add(task.getTotalRunTimeMs());
        }

    }

    long averagePMem = Statistics.average(totalPhysicalMemory);
    long averageVMem = Statistics.average(totalVirtualMemory);
    long maxPMem;
    long minPMem;
    try {
        maxPMem = Collections.max(totalPhysicalMemory);
        minPMem = Collections.min(totalPhysicalMemory);

    } catch (Exception exception) {
        maxPMem = 0;
        minPMem = 0;
    }
    long averageRunTime = Statistics.average(runTime);

    String containerSizeStr;

    if (!Strings.isNullOrEmpty(data.getConf().getProperty(_containerMemConf))) {
        containerSizeStr = data.getConf().getProperty(_containerMemConf);
    } else {
        containerSizeStr = getContainerMemDefaultMBytes();
    }

    long containerSize = Long.valueOf(containerSizeStr) * FileUtils.ONE_MB;

    double averageMemMb = (double) ((averagePMem) / FileUtils.ONE_MB);

    double ratio = averageMemMb / ((double) (containerSize / FileUtils.ONE_MB));

    Severity severity;

    if (tasks.length == 0) {
        severity = Severity.NONE;
    } else {
        severity = getMemoryRatioSeverity(ratio);
    }

    HeuristicResult result = new HeuristicResult(_heuristicConfData.getClassName(),
            _heuristicConfData.getHeuristicName(), severity, Utils.getHeuristicScore(severity, tasks.length));

    result.addResultDetail("Number of tasks", Integer.toString(tasks.length));
    result.addResultDetail("Maximum Physical Memory (MB)",
            tasks.length == 0 ? "0" : Long.toString(maxPMem / FileUtils.ONE_MB));
    result.addResultDetail("Minimum Physical memory (MB)",
            tasks.length == 0 ? "0" : Long.toString(minPMem / FileUtils.ONE_MB));
    result.addResultDetail("Average Physical Memory (MB)",
            tasks.length == 0 ? "0" : Long.toString(averagePMem / FileUtils.ONE_MB));
    result.addResultDetail("Average Virtual Memory (MB)",
            tasks.length == 0 ? "0" : Long.toString(averageVMem / FileUtils.ONE_MB));
    result.addResultDetail("Average Task RunTime",
            tasks.length == 0 ? "0" : Statistics.readableTimespan(averageRunTime));
    result.addResultDetail("Requested Container Memory (MB)",
            (tasks.length == 0 || containerSize == 0 || containerSize == -1) ? "0"
                    : String.valueOf(containerSize / FileUtils.ONE_MB));

    return result;

}

---

From source file:com.linkedin.drelephant.mapreduce.heuristics.GenericMemoryHeuristic.java

@Override
public HeuristicResult apply(MapReduceApplicationData data) {

    if (!data.getSucceeded()) {
        return null;
    }/*  ww  w .j  av a2 s. c om*/

    String containerSizeStr = data.getConf().getProperty(_containerMemConf);
    if (containerSizeStr == null) {
        return null;
    }

    long containerMem;
    try {
        containerMem = Long.parseLong(containerSizeStr);
    } catch (NumberFormatException e) {
        // Some job has a string var like "${VAR}" for this config.
        if (containerSizeStr.startsWith("$")) {
            String realContainerConf = containerSizeStr.substring(containerSizeStr.indexOf("{") + 1,
                    containerSizeStr.indexOf("}"));
            containerMem = Long.parseLong(data.getConf().getProperty(realContainerConf));
        } else {
            throw e;
        }
    }
    containerMem *= FileUtils.ONE_MB;

    MapReduceTaskData[] tasks = getTasks(data);
    List<Long> taskPMems = new ArrayList<Long>();
    List<Long> taskVMems = new ArrayList<Long>();
    List<Long> runtimesMs = new ArrayList<Long>();
    long taskPMin = Long.MAX_VALUE;
    long taskPMax = 0;
    for (MapReduceTaskData task : tasks) {
        if (task.isSampled()) {
            runtimesMs.add(task.getTotalRunTimeMs());
            long taskPMem = task.getCounters().get(MapReduceCounterData.CounterName.PHYSICAL_MEMORY_BYTES);
            long taskVMem = task.getCounters().get(MapReduceCounterData.CounterName.VIRTUAL_MEMORY_BYTES);
            taskPMems.add(taskPMem);
            taskPMin = Math.min(taskPMin, taskPMem);
            taskPMax = Math.max(taskPMax, taskPMem);
            taskVMems.add(taskVMem);
        }
    }

    if (taskPMin == Long.MAX_VALUE) {
        taskPMin = 0;
    }

    long taskPMemAvg = Statistics.average(taskPMems);
    long taskVMemAvg = Statistics.average(taskVMems);
    long averageTimeMs = Statistics.average(runtimesMs);

    Severity severity;
    if (tasks.length == 0) {
        severity = Severity.NONE;
    } else {
        severity = getTaskMemoryUtilSeverity(taskPMemAvg, containerMem);
    }

    HeuristicResult result = new HeuristicResult(_heuristicConfData.getClassName(),
            _heuristicConfData.getHeuristicName(), severity, Utils.getHeuristicScore(severity, tasks.length));

    result.addResultDetail("Number of tasks", Integer.toString(tasks.length));
    result.addResultDetail("Avg task runtime", Statistics.readableTimespan(averageTimeMs));
    result.addResultDetail("Avg Physical Memory (MB)", Long.toString(taskPMemAvg / FileUtils.ONE_MB));
    result.addResultDetail("Max Physical Memory (MB)", Long.toString(taskPMax / FileUtils.ONE_MB));
    result.addResultDetail("Min Physical Memory (MB)", Long.toString(taskPMin / FileUtils.ONE_MB));
    result.addResultDetail("Avg Virtual Memory (MB)", Long.toString(taskVMemAvg / FileUtils.ONE_MB));
    result.addResultDetail("Requested Container Memory", FileUtils.byteCountToDisplaySize(containerMem));

    return result;
}

---

• «
• 1
• 2
• 3
• 4
• 5
• 6
• 7
• »