Example usage for org.apache.hadoop.io DataOutputBuffer DataOutputBuffer

Introduction

In this page you can find the example usage for org.apache.hadoop.io DataOutputBuffer DataOutputBuffer.

Prototype

public DataOutputBuffer() 

Source Link

Document

Constructs a new empty buffer.

Usage

From source file:CompressionTest.java

License:Open Source License

public static void main(String[] args) throws IOException {
    DataOutputBuffer chunksBuffer = new DataOutputBuffer();
    DataOutputBuffer metasBuffer = new DataOutputBuffer();

    byte[] data = "alskjdflkajsldfkja;s".getBytes();
    chunksBuffer.write(data);/*from w  ww  .  j a va  2s .c o m*/
    System.out.println(chunksBuffer.size());
    System.out.println(chunksBuffer.getLength());
    chunksBuffer.reset();
    chunksBuffer.write(data, 0, 10);
    System.out.println(chunksBuffer.size());
    System.out.println(chunksBuffer.getLength());

}

---

From source file:TestILineIndex.java

License:Open Source License

public void testPersistable() throws IOException {
    ILineIndex ili = new ILineIndex(1, 1);

    DataOutputBuffer dob = new DataOutputBuffer();
    ili.persistent(dob);//from   w ww.  jav a 2 s  .c o  m
    byte[] data = dob.getData();

    DataInputBuffer dib = new DataInputBuffer();
    dib.reset(data, data.length);

    ILineIndex ili2 = new ILineIndex();
    ili2.unpersistent(dib);

    assertEquals(ili.beginline(), ili2.beginline());
    assertEquals(ili.endline(), ili2.endline());
}

---

From source file:TestISegmentIndex.java

License:Open Source License

public void testPersistable() throws IOException {
    IFileInfo fileInfo = TestUtil.genfileinfo(true, 2);
    ISegmentIndex index = new ISegmentIndex(fileInfo);

    index.update(TestUtil.genseginfo(fileInfo, 0, 2));
    index.update(TestUtil.genseginfo(fileInfo, 1, 2));
    index.update(TestUtil.genseginfo(fileInfo, 2, 2));
    index.update(TestUtil.genseginfo(fileInfo, 3, 2));
    index.update(TestUtil.genseginfo(fileInfo, 4, 2));

    DataOutputBuffer dob = new DataOutputBuffer();
    index.persistent(dob);/*w w  w  . j  a v  a2s  .com*/
    byte[] data = dob.getData();

    DataInputBuffer dib = new DataInputBuffer();
    dib.reset(data, data.length);

    ISegmentIndex index2 = new ISegmentIndex(fileInfo);
    index2.unpersistent(dib);

    assertEquals(5, index2.getSegnum());

    assertEquals(0, index2.getSegid(0));
    assertEquals(0, index2.getSegid(3));
    assertEquals(1, index2.getSegid(4));
    assertEquals(1, index2.getSegid(7));
    assertEquals(4, index2.getSegid(19));
    assertEquals(5, index2.getSegid(20));

    assertEquals(0, index2.getSegid(new IFieldValue(0)));
    assertEquals(0, index2.getSegid(new IFieldValue(3)));
    assertEquals(1, index2.getSegid(new IFieldValue(4)));
    assertEquals(1, index2.getSegid(new IFieldValue(7)));
    assertEquals(4, index2.getSegid(new IFieldValue(19)));
    assertEquals(5, index2.getSegid(new IFieldValue(20)));

}

---

From source file:TestIndexValue.java

License:Open Source License

public void testIndexValue() throws IOException {

    IndexValue value = new IndexValue((short) 1, 2);

    DataOutputBuffer dob = new DataOutputBuffer();
    value.write(dob);/*from   w  w w.j  a v  a  2s  .  co  m*/
    byte[] data = dob.getData();
    DataInputBuffer dib = new DataInputBuffer();
    dib.reset(data, data.length);

    IndexValue value2 = new IndexValue((short) 1, 2);
    value2.readFields(dib);

    assertEquals(value.getFileindex(), value2.getFileindex());
    assertEquals(value.getRowid(), value2.getRowid());

}

---

From source file:TestIFieldType.java

License:Open Source License

public void testWritable() throws IOException {
    IFieldType ift = new IFieldType(ConstVar.FieldType_Int, (short) 1);

    DataOutputBuffer dob = new DataOutputBuffer();
    ift.write(dob);/*  ww w  . jav  a  2s .  com*/
    byte[] data = dob.getData();
    DataInputBuffer dib = new DataInputBuffer();
    dib.reset(data, data.length);

    IFieldType ift2 = new IFieldType();
    ift2.readFields(dib);

    assertEquals(ift.getIndex(), ift2.getIndex());
    assertEquals(ift.getLen(), ift2.getLen());
    assertEquals(ift.getType(), ift2.getType());
}

---

From source file:TestIFieldType.java

License:Open Source License

public void testPersistable() throws IOException {
    IFieldType ift = new IFieldType(ConstVar.FieldType_Int, (short) 1);

    DataOutputBuffer dob = new DataOutputBuffer();
    ift.persistent(dob);//from w  ww .j a  v  a  2 s .c om
    byte[] data = dob.getData();
    DataInputBuffer dib = new DataInputBuffer();
    dib.reset(data, data.length);

    IFieldType ift2 = new IFieldType();
    ift2.unpersistent(dib);

    assertEquals(ift.getIndex(), ift2.getIndex());
    assertEquals(ift.getLen(), ift2.getLen());
    assertEquals(ift.getType(), ift2.getType());
}

---

From source file:TestIFieldValue.java

License:Open Source License

public void testWritable() throws IOException {

    IFieldValue ifv = new IFieldValue(100);

    DataOutputBuffer dob = new DataOutputBuffer();
    ifv.write(dob);//from  w w  w  .j a  v a 2s  .  c  om
    byte[] data = dob.getData();
    DataInputBuffer dib = new DataInputBuffer();
    dib.reset(data, data.length);

    IFieldValue ifv2 = new IFieldValue();
    ifv2.readFields(dib);

    assertEquals(ifv.idx(), ifv2.idx());
    assertEquals(ifv.len(), ifv2.len());
    assertEquals(ifv.type(), ifv2.type());
    assertTrue(ifv.compareTo(ifv2) == 0);

}

---

From source file:TestIFieldValue.java

License:Open Source License

public void testPersitable() throws IOException {
    IFieldValue ifv = new IFieldValue(100);

    DataOutputBuffer dob = new DataOutputBuffer();
    ifv.persistent(dob);//from www .j  a v  a  2s .c o  m

    byte[] data = dob.getData();

    DataInputBuffer dib = new DataInputBuffer();
    dib.reset(data, data.length);

    IFieldValue ifv2 = new IFieldValue(ifv.fieldType());
    ifv2.unpersistent(dib);

    assertEquals(ifv.idx(), ifv2.idx());
    assertEquals(ifv.len(), ifv2.len());
    assertEquals(ifv.type(), ifv2.type());
    assertTrue(ifv.compareTo(ifv2) == 0);

}

---

From source file:ApplicationMaster.java

License:Apache License

/**
 * Main run function for the application master
 *
 * @throws YarnException/* ww  w  .j  a v  a  2  s  .c o m*/
 * @throws IOException
 */
@SuppressWarnings({ "unchecked" })
public void run() throws YarnException, IOException {
    LOG.info("Starting ApplicationMaster");
    try {
        publishApplicationAttemptEvent(timelineClient, appAttemptID.toString(), DSEvent.DS_APP_ATTEMPT_START);
    } catch (Exception e) {
        LOG.error("App Attempt start event coud not be pulished for " + appAttemptID.toString(), e);
    }

    // Note: Credentials, Token, UserGroupInformation, DataOutputBuffer class
    // are marked as LimitedPrivate
    Credentials credentials = UserGroupInformation.getCurrentUser().getCredentials();
    DataOutputBuffer dob = new DataOutputBuffer();
    credentials.writeTokenStorageToStream(dob);
    // Now remove the AM->RM token so that containers cannot access it.
    Iterator<Token<?>> iter = credentials.getAllTokens().iterator();
    LOG.info("Executing with tokens:");
    while (iter.hasNext()) {
        Token<?> token = iter.next();
        LOG.info(token);
        if (token.getKind().equals(AMRMTokenIdentifier.KIND_NAME)) {
            iter.remove();
        }
    }
    allTokens = ByteBuffer.wrap(dob.getData(), 0, dob.getLength());

    // Create appSubmitterUgi and add original tokens to it
    String appSubmitterUserName = System.getenv(ApplicationConstants.Environment.USER.name());
    appSubmitterUgi = UserGroupInformation.createRemoteUser(appSubmitterUserName);
    appSubmitterUgi.addCredentials(credentials);

    AMRMClientAsync.CallbackHandler allocListener = new RMCallbackHandler();
    amRMClient = AMRMClientAsync.createAMRMClientAsync(1000, allocListener);
    amRMClient.init(conf);
    amRMClient.start();

    containerListener = createNMCallbackHandler();
    nmClientAsync = new NMClientAsyncImpl(containerListener);
    nmClientAsync.init(conf);
    nmClientAsync.start();

    // Setup local RPC Server to accept status requests directly from clients
    // TODO need to setup a protocol for client to be able to communicate to
    // the RPC server
    // TODO use the rpc port info to register with the RM for the client to
    // send requests to this app master

    // Register self with ResourceManager
    // This will start heartbeating to the RM
    appMasterHostname = NetUtils.getHostname();
    RegisterApplicationMasterResponse response = amRMClient.registerApplicationMaster(appMasterHostname,
            appMasterRpcPort, appMasterTrackingUrl);
    // Dump out information about cluster capability as seen by the
    // resource manager
    int maxMem = response.getMaximumResourceCapability().getMemory();
    LOG.info("Max mem capabililty of resources in this cluster " + maxMem);

    int maxVCores = response.getMaximumResourceCapability().getVirtualCores();
    LOG.info("Max vcores capabililty of resources in this cluster " + maxVCores);

    // A resource ask cannot exceed the max.
    if (containerMemory > maxMem) {
        LOG.info("Container memory specified above max threshold of cluster." + " Using max value."
                + ", specified=" + containerMemory + ", max=" + maxMem);
        containerMemory = maxMem;
    }

    if (containerVirtualCores > maxVCores) {
        LOG.info("Container virtual cores specified above max threshold of cluster." + " Using max value."
                + ", specified=" + containerVirtualCores + ", max=" + maxVCores);
        containerVirtualCores = maxVCores;
    }

    List<Container> previousAMRunningContainers = response.getContainersFromPreviousAttempts();
    LOG.info(appAttemptID + " received " + previousAMRunningContainers.size()
            + " previous attempts' running containers on AM registration.");
    numAllocatedContainers.addAndGet(previousAMRunningContainers.size());

    int numTotalContainersToRequest = numTotalContainers - previousAMRunningContainers.size();
    // Setup ask for containers from RM
    // Send request for containers to RM
    // Until we get our fully allocated quota, we keep on polling RM for
    // containers
    // Keep looping until all the containers are launched and shell script
    // executed on them ( regardless of success/failure).
    for (int i = 0; i < numTotalContainersToRequest; ++i) {
        ContainerRequest containerAsk = setupContainerAskForRM();
        amRMClient.addContainerRequest(containerAsk);
    }
    numRequestedContainers.set(numTotalContainers);
    try {
        publishApplicationAttemptEvent(timelineClient, appAttemptID.toString(), DSEvent.DS_APP_ATTEMPT_END);
    } catch (Exception e) {
        LOG.error("App Attempt start event coud not be pulished for " + appAttemptID.toString(), e);
    }
}

---

From source file:TestIndexKey.java

License:Open Source License

public void testWritable() throws IOException {
    IndexKey key = new IndexKey();
    IFieldValue ifv = new IFieldValue(100);
    key.addfv(ifv);/*from w  w w  . j  ava2s  .  c  o m*/

    DataOutputBuffer dob = new DataOutputBuffer();
    key.write(dob);
    byte[] data = dob.getData();
    DataInputBuffer dib = new DataInputBuffer();
    dib.reset(data, data.length);

    IndexKey key2 = new IndexKey();
    key2.readFields(dib);

    IndexKey key3 = new IndexKey();
    key3.addfv(new IFieldValue(101));

    IndexKey key4 = new IndexKey();
    key4.addfv(new IFieldValue(99));

    assertTrue(key.compareTo(key2) == 0);
    assertTrue(key.compareTo(key3) < 0);
    assertTrue(key.compareTo(key4) > 0);

}

---

• «
• 1
• 2
• 3
• 4
• 5
• 6
• 7
• 8
• 9
• 10
• 11
• 12
• 13
• 14
• 15
• 16
• 17
• 18
• 19
• 20
• 21
• 22
• 23
• 24
• 25
• 26
• 27
• 28
• 29
• »