Example usage for org.apache.commons.io FileUtils ONE_KB

Introduction

In this page you can find the example usage for org.apache.commons.io FileUtils ONE_KB.

Prototype

long ONE_KB

To view the source code for org.apache.commons.io FileUtils ONE_KB.

Click Source Link

Document

The number of bytes in a kilobyte.

Usage

From source file:com.uber.hoodie.func.TestBoundedInMemoryQueue.java

/**
 * Test to ensure that we are reading all records from queue iterator when we have multiple producers
 *///from www .j ava2s .co m
@SuppressWarnings("unchecked")
@Test(timeout = 60000)
public void testCompositeProducerRecordReading() throws Exception {
    final int numRecords = 1000;
    final int numProducers = 40;
    final List<List<HoodieRecord>> recs = new ArrayList<>();

    final BoundedInMemoryQueue<HoodieRecord, Tuple2<HoodieRecord, Optional<IndexedRecord>>> queue = new BoundedInMemoryQueue(
            FileUtils.ONE_KB, getTransformFunction(HoodieTestDataGenerator.avroSchema));

    // Record Key to <Producer Index, Rec Index within a producer>
    Map<String, Tuple2<Integer, Integer>> keyToProducerAndIndexMap = new HashMap<>();

    for (int i = 0; i < numProducers; i++) {
        List<HoodieRecord> pRecs = hoodieTestDataGenerator.generateInserts(commitTime, numRecords);
        int j = 0;
        for (HoodieRecord r : pRecs) {
            Assert.assertTrue(!keyToProducerAndIndexMap.containsKey(r.getRecordKey()));
            keyToProducerAndIndexMap.put(r.getRecordKey(), new Tuple2<>(i, j));
            j++;
        }
        recs.add(pRecs);
    }

    List<BoundedInMemoryQueueProducer<HoodieRecord>> producers = new ArrayList<>();
    for (int i = 0; i < recs.size(); i++) {
        final List<HoodieRecord> r = recs.get(i);
        // Alternate between pull and push based iterators
        if (i % 2 == 0) {
            producers.add(new IteratorBasedQueueProducer<>(r.iterator()));
        } else {
            producers.add(new FunctionBasedQueueProducer<HoodieRecord>((buf) -> {
                Iterator<HoodieRecord> itr = r.iterator();
                while (itr.hasNext()) {
                    try {
                        buf.insertRecord(itr.next());
                    } catch (Exception e) {
                        throw new HoodieException(e);
                    }
                }
                return true;
            }));
        }
    }

    final List<Future<Boolean>> futureList = producers.stream().map(producer -> {
        return executorService.submit(() -> {
            producer.produce(queue);
            return true;
        });
    }).collect(Collectors.toList());

    // Close queue
    Future<Boolean> closeFuture = executorService.submit(() -> {
        try {
            for (Future f : futureList) {
                f.get();
            }
            queue.close();
        } catch (Exception e) {
            throw new RuntimeException(e);
        }
        return true;
    });

    // Used to ensure that consumer sees the records generated by a single producer in FIFO order
    Map<Integer, Integer> lastSeenMap = IntStream.range(0, numProducers).boxed()
            .collect(Collectors.toMap(Function.identity(), x -> -1));
    Map<Integer, Integer> countMap = IntStream.range(0, numProducers).boxed()
            .collect(Collectors.toMap(Function.identity(), x -> 0));

    // Read recs and ensure we have covered all producer recs.
    while (queue.iterator().hasNext()) {
        final Tuple2<HoodieRecord, Optional<IndexedRecord>> payload = queue.iterator().next();
        final HoodieRecord rec = payload._1();
        Tuple2<Integer, Integer> producerPos = keyToProducerAndIndexMap.get(rec.getRecordKey());
        Integer lastSeenPos = lastSeenMap.get(producerPos._1());
        countMap.put(producerPos._1(), countMap.get(producerPos._1()) + 1);
        lastSeenMap.put(producerPos._1(), lastSeenPos + 1);
        // Ensure we are seeing the next record generated
        Assert.assertEquals(lastSeenPos + 1, producerPos._2().intValue());
    }

    for (int i = 0; i < numProducers; i++) {
        // Ensure we have seen all the records for each producers
        Assert.assertEquals(Integer.valueOf(numRecords), countMap.get(i));
    }

    //Ensure Close future is done
    closeFuture.get();
}

---

From source file:net.sf.logsaw.core.internal.logresource.simple.SimpleLogResource.java

/**
 * Returns the units of work for parsing the given log file.
 * @return the units of work or <code>IProgressMonitor.UNKNOWN</code>
 *///ww w.  jav a 2 s . c o m
protected int getUnitsOfWork() {
    // If file is less than 1kB, totalWork will be one
    return ((int) Math.floor((double) logFile.length() / FileUtils.ONE_KB)) + 1;
}

---

From source file:com.linkedin.drelephant.util.Utils.java

/**
 * Convert a value in MBSeconds to GBHours
 * @param MBSeconds The value to convert
 * @return A double of the value in GB Hours unit
 *///from www .  ja  v  a  2s  . c om
public static double MBSecondsToGBHours(long MBSeconds) {
    double GBseconds = (double) MBSeconds / (double) FileUtils.ONE_KB;
    double GBHours = GBseconds / Statistics.HOUR;
    return GBHours;
}

---

From source file:edu.kit.dama.mdm.dataworkflow.ExecutionEnvironmentConfiguration.java

/**
 * Set custom properties as object.//from  w  ww.j av a2s .  co m
 *
 * @param pProperties The properties object.
 *
 * @throws IOException If the serialization failed.
 */
public void setPropertiesAsObject(Properties pProperties) throws IOException {
    String serialized = PropertiesUtil.propertiesToString(pProperties);
    if (serialized != null && serialized.length() > 10 * FileUtils.ONE_KB) {
        throw new IOException(
                "Failed to store custom properties from object. Serialized content exceeds max. size of database field (10 KB).");
    }

    this.setCustomProperties(serialized);
}

---

From source file:com.loop81.fxcomparer.FXComparerController.java

/** Convert the given difference to a more readable string using {@link FileUtils#byteCountToDisplaySize(long)}. */
private String convertdifferenceToReadableString(long difference) {
    if (difference < FileUtils.ONE_KB && difference > -1 * FileUtils.ONE_KB) {
        return (difference > 0 ? "+" : "") + FileUtils.byteCountToDisplaySize(difference);
    } else {/*  w ww.  j  ava2  s .  c o  m*/
        return (difference > 0 ? "+" : "-") + FileUtils.byteCountToDisplaySize(Math.abs(difference)) + " ("
                + difference + " " + MessageBundle.getString("general.bytes") + ")";
    }
}

---

From source file:edu.kit.dama.mdm.dataworkflow.DataWorkflowTask.java

/**
 * Set the object-view-map properties object as object.
 *
 * @param pProperties The object-view-map properties object.
 *
 * @throws IOException If the serialization failed.
 *//*  ww w.j ava2  s.c o m*/
public void setObjectViewMapAsObject(Properties pProperties) throws IOException {
    String serialized = PropertiesUtil.propertiesToString(pProperties);
    if (serialized != null && serialized.length() > 10 * FileUtils.ONE_KB) {
        throw new IOException(
                "Failed to store object view map from object. Serialized content exceeds max. size of database field (10 KB).");
    }

    this.setObjectViewMap(serialized);
}

---

From source file:edu.kit.dama.mdm.dataworkflow.DataWorkflowTask.java

/**
 * Set the object-transfer-map properties object as object.
 *
 * @param pProperties The object-transfer-map properties object.
 *
 * @throws IOException If the serialization failed.
 *//*from   ww w  . ja  v  a 2  s. co m*/
public void setObjectTransferMapAsObject(Properties pProperties) throws IOException {
    String serialized = PropertiesUtil.propertiesToString(pProperties);
    if (serialized != null && serialized.length() > 10 * FileUtils.ONE_KB) {
        throw new IOException(
                "Failed to store object transfer map from object. Serialized content exceeds max. size of database field (10 KB).");
    }

    this.setObjectTransferMap(serialized);
}

---

From source file:edu.kit.dama.mdm.dataworkflow.DataWorkflowTask.java

/**
 * Set the execution settings as properties object.
 *
 * @param pProperties The execution settings properties object.
 *
 * @throws IOException If the serialization failed.
 *//*from w  w  w. ja  v a2 s .  co m*/
public void setExecutionSettingsAsObject(Properties pProperties) throws IOException {
    String serialized = PropertiesUtil.propertiesToString(pProperties);
    if (serialized != null && serialized.length() > 10 * FileUtils.ONE_KB) {
        throw new IOException(
                "Failed to store execution settings from object. Serialized content exceeds max. size of database field (10 KB).");
    }
    this.setExecutionSettings(serialized);
}

---

From source file:com.sunchenbin.store.feilong.core.io.FileUtil.java

/**
 * ??./*  w w  w.j  a  v  a  2s .  co m*/
 * 
 * <p>
 * ???? GB MB KB???? Bytes
 * </p>
 * 
 * <p>
 * Common-io 2.4{@link org.apache.commons.io.FileUtils#byteCountToDisplaySize(long)},GB ??1.99G 1G,apache ?
 * </p>
 * 
 * @param fileSize
 *            ? ??byte
 * @return ?byte ?
 * @see #getFileSize(File)
 * @see org.apache.commons.io.FileUtils#ONE_GB
 * @see org.apache.commons.io.FileUtils#ONE_MB
 * @see org.apache.commons.io.FileUtils#ONE_KB
 * 
 * @see org.apache.commons.io.FileUtils#byteCountToDisplaySize(long)
 */
public static String formatSize(long fileSize) {
    String danwei = "";
    long chushu = 1;// 
    if (fileSize >= FileUtils.ONE_GB) {
        danwei = "GB";
        chushu = FileUtils.ONE_GB;
    } else if (fileSize >= FileUtils.ONE_MB) {
        danwei = "MB";
        chushu = FileUtils.ONE_MB;
    } else if (fileSize >= FileUtils.ONE_KB) {
        danwei = "KB";
        chushu = FileUtils.ONE_KB;
    } else {
        return fileSize + "Bytes";
    }
    String yushu = 100 * (fileSize % chushu) / chushu + ""; // ?
    if ("0".equals(yushu)) {
        return fileSize / chushu + danwei;
    }
    return fileSize / chushu + "." + yushu + danwei;
}

---

From source file:org.apache.eagle.topology.extractor.hdfs.HdfsTopologyEntityParser.java

private HdfsServiceTopologyAPIEntity createNamenodeEntity(String url, long updateTime)
        throws ServiceNotResponseException {
    final String urlString = buildFSNamesystemURL(url);
    final Map<String, JMXBean> jmxBeanMap = JMXQueryHelper.query(urlString);
    final JMXBean bean = jmxBeanMap.get(JMX_FS_NAME_SYSTEM_BEAN_NAME);

    if (bean == null || bean.getPropertyMap() == null) {
        throw new ServiceNotResponseException("Invalid JMX format, FSNamesystem bean is null!");
    }//from  w  w w .ja  va  2 s.co m
    final String hostname = (String) bean.getPropertyMap().get(HA_NAME);
    HdfsServiceTopologyAPIEntity namenode = createHdfsServiceEntity(TopologyConstants.NAME_NODE_ROLE, hostname,
            updateTime);
    try {
        final String state = (String) bean.getPropertyMap().get(HA_STATE);
        namenode.setStatus(state);
        final Double configuredCapacityGB = (Double) bean.getPropertyMap().get(CAPACITY_TOTAL_GB);
        namenode.setConfiguredCapacityTB(Double.toString(configuredCapacityGB / FileUtils.ONE_KB));
        final Double capacityUsedGB = (Double) bean.getPropertyMap().get(CAPACITY_USED_GB);
        namenode.setUsedCapacityTB(Double.toString(capacityUsedGB / FileUtils.ONE_KB));
        final Integer blocksTotal = (Integer) bean.getPropertyMap().get(BLOCKS_TOTAL);
        namenode.setNumBlocks(Integer.toString(blocksTotal));
    } catch (RuntimeException ex) {
        LOG.error(ex.getMessage(), ex);
    }
    return namenode;

}

---

• «
• 1
• 2
• 3
• »