Example usage for org.apache.hadoop.io DataOutputBuffer getLength

Introduction

In this page you can find the example usage for org.apache.hadoop.io DataOutputBuffer getLength.

Prototype

public int getLength() 

Source Link

Document

Returns the length of the valid data currently in the buffer.

Usage

From source file:org.apache.mrql.FlinkJsonParser.java

License:Apache License

public String slice() {
    if (splitter.hasNext()) {
        DataOutputBuffer b = splitter.next();
        return new String(b.getData(), 0, b.getLength());
    } else//from  www  . ja  v a 2s.  c  o m
        return null;
}

---

From source file:org.apache.nutch.crawl.MapWritable.java

License:Apache License

/**
 * Copy constructor. This constructor makes a deep copy, using serialization /
 * deserialization to break any possible references to contained objects.
 * //from  ww  w  .  j ava  2s . com
 * @param map map to copy from
 */
public MapWritable(MapWritable map) {
    if (map != null) {
        try {
            DataOutputBuffer dob = new DataOutputBuffer();
            map.write(dob);
            DataInputBuffer dib = new DataInputBuffer();
            dib.reset(dob.getData(), dob.getLength());
            readFields(dib);
        } catch (IOException e) {
            throw new IllegalArgumentException(
                    "this map cannot be copied: " + StringUtils.stringifyException(e));
        }
    }
}

---

From source file:org.apache.nutch.crawl.TestMapWritable.java

License:Apache License

/** Utility method for testing writables, from hadoop code */
public void testWritable(Writable before) throws Exception {
    DataOutputBuffer dob = new DataOutputBuffer();
    before.write(dob);//w w w  .  j av  a2  s . c o  m

    DataInputBuffer dib = new DataInputBuffer();
    dib.reset(dob.getData(), dob.getLength());

    Writable after = (Writable) before.getClass().newInstance();
    after.readFields(dib);

    assertEquals(before, after);
}

---

From source file:org.apache.nutch.crawl.TestMapWritable.java

License:Apache License

public void testRecycling() throws Exception {
    Text value = new Text("value");
    Text key1 = new Text("a");
    Text key2 = new Text("b");

    MapWritable writable = new MapWritable();
    writable.put(key1, value);//from   w w w .j ava  2 s.c  om
    assertEquals(writable.get(key1), value);
    assertNull(writable.get(key2));

    DataOutputBuffer dob = new DataOutputBuffer();
    writable.write(dob);
    writable.clear();
    writable.put(key1, value);
    writable.put(key2, value);
    assertEquals(writable.get(key1), value);
    assertEquals(writable.get(key2), value);

    DataInputBuffer dib = new DataInputBuffer();
    dib.reset(dob.getData(), dob.getLength());
    writable.readFields(dib);
    assertEquals(writable.get(key1), value);
    assertNull(writable.get(key2));
}

---

From source file:org.apache.nutch.searcher.TestHitDetails.java

License:Apache License

public void testHitDetails() throws Exception {
    final int length = 4;
    final String[] fields = new String[] { "a", "b", "c", "a" };
    final String[] values = new String[] { "foo1", "bar", "baz", "foo2" };

    HitDetails before = new HitDetails(fields, values);

    DataOutputBuffer dob = new DataOutputBuffer();
    before.write(dob);/*from  ww w  . ja  va 2s.  c om*/

    DataInputBuffer dib = new DataInputBuffer();
    dib.reset(dob.getData(), dob.getLength());

    HitDetails after = HitDetails.read(dib);

    assertEquals(length, after.getLength());
    for (int i = 0; i < 3; i++) {
        assertEquals(fields[i], after.getField(i));
        assertEquals(values[i], after.getValue(i));
        assertEquals(values[i], after.getValue(fields[i]));
    }
    String[] vals = after.getValues("a");
    assertEquals(2, vals.length);
    assertEquals("foo1", vals[0]);
    assertEquals("foo2", vals[1]);
    vals = after.getValues("b");
    assertEquals(1, vals.length);
    assertEquals("bar", vals[0]);
    vals = after.getValues("c");
    assertEquals(1, vals.length);
    assertEquals("baz", vals[0]);
}

---

From source file:org.apache.nutch.util.WritableTestUtils.java

License:Apache License

/** Utility method for testing writables. */
public static Writable writeRead(Writable before, Configuration conf) throws Exception {

    DataOutputBuffer dob = new DataOutputBuffer();
    before.write(dob);/*from  w  w  w.j  a  v a2  s.  c o  m*/

    DataInputBuffer dib = new DataInputBuffer();
    dib.reset(dob.getData(), dob.getLength());

    Writable after = (Writable) before.getClass().newInstance();
    if (conf != null) {
        ((Configurable) after).setConf(conf);
    }
    after.readFields(dib);
    return after;
}

---

From source file:org.apache.orc.impl.TestRecordReaderImpl.java

License:Apache License

@Test
public void testMaxLengthToReader() throws Exception {
    Configuration conf = new Configuration();
    OrcProto.Type rowType = OrcProto.Type.newBuilder().setKind(OrcProto.Type.Kind.STRUCT).build();
    OrcProto.Footer footer = OrcProto.Footer.newBuilder().setHeaderLength(0).setContentLength(0)
            .setNumberOfRows(0).setRowIndexStride(0).addTypes(rowType).build();
    OrcProto.PostScript ps = OrcProto.PostScript.newBuilder().setCompression(OrcProto.CompressionKind.NONE)
            .setFooterLength(footer.getSerializedSize()).setMagic("ORC").addVersion(0).addVersion(11).build();
    DataOutputBuffer buffer = new DataOutputBuffer();
    footer.writeTo(buffer);/*w  w  w .  j  av a2s .  co  m*/
    ps.writeTo(buffer);
    buffer.write(ps.getSerializedSize());
    FileSystem fs = mock(FileSystem.class, settings);
    FSDataInputStream file = new FSDataInputStream(new BufferInStream(buffer.getData(), buffer.getLength()));
    Path p = new Path("/dir/file.orc");
    when(fs.open(p)).thenReturn(file);
    OrcFile.ReaderOptions options = OrcFile.readerOptions(conf);
    options.filesystem(fs);
    options.maxLength(buffer.getLength());
    when(fs.getFileStatus(p)).thenReturn(new FileStatus(10, false, 3, 3000, 0, p));
    Reader reader = OrcFile.createReader(p, options);
}

---

From source file:org.apache.orc.mapred.TestOrcList.java

License:Apache License

static void cloneWritable(Writable source, Writable destination) throws IOException {
    DataOutputBuffer out = new DataOutputBuffer(1024);
    source.write(out);//www .  j  av a 2 s .  c o  m
    out.flush();
    DataInputBuffer in = new DataInputBuffer();
    in.reset(out.getData(), out.getLength());
    destination.readFields(in);
}

---

From source file:org.apache.samza.job.yarn.ContainerUtil.java

License:Apache License

protected void startContainer(Path packagePath, Container container, Map<String, String> env,
        final String cmd) {
    log.info("starting container {} {} {} {}", new Object[] { packagePath, container, env, cmd });

    // set the local package so that the containers and app master are provisioned with it
    LocalResource packageResource = Records.newRecord(LocalResource.class);
    URL packageUrl = ConverterUtils.getYarnUrlFromPath(packagePath);
    FileStatus fileStatus;//from   ww w .j a v  a  2  s.co  m
    try {
        fileStatus = packagePath.getFileSystem(yarnConfiguration).getFileStatus(packagePath);
    } catch (IOException ioe) {
        log.error("IO Exception when accessing the package status from the filesystem", ioe);
        throw new SamzaException("IO Exception when accessing the package status from the filesystem");
    }

    packageResource.setResource(packageUrl);
    packageResource.setSize(fileStatus.getLen());
    packageResource.setTimestamp(fileStatus.getModificationTime());
    packageResource.setType(LocalResourceType.ARCHIVE);
    packageResource.setVisibility(LocalResourceVisibility.APPLICATION);

    ByteBuffer allTokens;
    // copy tokens (copied from dist shell example)
    try {
        Credentials credentials = UserGroupInformation.getCurrentUser().getCredentials();
        DataOutputBuffer dob = new DataOutputBuffer();
        credentials.writeTokenStorageToStream(dob);

        // now remove the AM->RM token so that containers cannot access it
        Iterator iter = credentials.getAllTokens().iterator();
        while (iter.hasNext()) {
            TokenIdentifier token = ((Token) iter.next()).decodeIdentifier();
            if (token.getKind().equals(AMRMTokenIdentifier.KIND_NAME)) {
                iter.remove();
            }
        }
        allTokens = ByteBuffer.wrap(dob.getData(), 0, dob.getLength());

    } catch (IOException ioe) {
        ioe.printStackTrace();
        throw new SamzaException("IO Exception when writing credentials to output buffer");
    }

    ContainerLaunchContext context = Records.newRecord(ContainerLaunchContext.class);
    context.setEnvironment(env);
    context.setTokens(allTokens.duplicate());
    context.setCommands(new ArrayList<String>() {
        {
            add(cmd);
        }
    });
    context.setLocalResources(Collections.singletonMap("__package", packageResource));

    log.debug("setting package to {}", packageResource);
    log.debug("setting context to {}", context);

    StartContainerRequest startContainerRequest = Records.newRecord(StartContainerRequest.class);
    startContainerRequest.setContainerLaunchContext(context);
    try {
        nmClient.startContainer(container, context);
    } catch (YarnException ye) {
        log.error("Received YarnException when starting container: " + container.getId(), ye);
        throw new SamzaException("Received YarnException when starting container: " + container.getId());
    } catch (IOException ioe) {
        log.error("Received IOException when starting container: " + container.getId(), ioe);
        throw new SamzaException("Received IOException when starting container: " + container.getId());
    }
}

---

From source file:org.apache.samza.job.yarn.refactor.YarnContainerRunner.java

License:Apache License

/**
 *    Runs a command as a process on the container. All binaries needed by the physical process are packaged in the URL
 *    specified by packagePath./*  w ww. ja v a2 s  .  com*/
 */
private void startContainer(Path packagePath, Container container, Map<String, String> env, final String cmd)
        throws SamzaContainerLaunchException {
    log.info("starting container {} {} {} {}", new Object[] { packagePath, container, env, cmd });

    // set the local package so that the containers and app master are provisioned with it
    LocalResource packageResource = Records.newRecord(LocalResource.class);
    URL packageUrl = ConverterUtils.getYarnUrlFromPath(packagePath);
    FileStatus fileStatus;
    try {
        fileStatus = packagePath.getFileSystem(yarnConfiguration).getFileStatus(packagePath);
    } catch (IOException ioe) {
        log.error("IO Exception when accessing the package status from the filesystem", ioe);
        throw new SamzaContainerLaunchException(
                "IO Exception when accessing the package status from the filesystem");
    }

    packageResource.setResource(packageUrl);
    packageResource.setSize(fileStatus.getLen());
    packageResource.setTimestamp(fileStatus.getModificationTime());
    packageResource.setType(LocalResourceType.ARCHIVE);
    packageResource.setVisibility(LocalResourceVisibility.APPLICATION);

    ByteBuffer allTokens;
    // copy tokens (copied from dist shell example)
    try {
        Credentials credentials = UserGroupInformation.getCurrentUser().getCredentials();
        DataOutputBuffer dob = new DataOutputBuffer();
        credentials.writeTokenStorageToStream(dob);

        // now remove the AM->RM token so that containers cannot access it
        Iterator iter = credentials.getAllTokens().iterator();
        while (iter.hasNext()) {
            TokenIdentifier token = ((Token) iter.next()).decodeIdentifier();
            if (token.getKind().equals(AMRMTokenIdentifier.KIND_NAME)) {
                iter.remove();
            }
        }
        allTokens = ByteBuffer.wrap(dob.getData(), 0, dob.getLength());

    } catch (IOException ioe) {
        log.error("IOException when writing credentials.", ioe);
        throw new SamzaContainerLaunchException("IO Exception when writing credentials to output buffer");
    }

    ContainerLaunchContext context = Records.newRecord(ContainerLaunchContext.class);
    context.setEnvironment(env);
    context.setTokens(allTokens.duplicate());
    context.setCommands(new ArrayList<String>() {
        {
            add(cmd);
        }
    });
    context.setLocalResources(Collections.singletonMap("__package", packageResource));

    log.debug("setting package to {}", packageResource);
    log.debug("setting context to {}", context);

    StartContainerRequest startContainerRequest = Records.newRecord(StartContainerRequest.class);
    startContainerRequest.setContainerLaunchContext(context);
    try {
        nmClient.startContainer(container, context);
    } catch (YarnException ye) {
        log.error("Received YarnException when starting container: " + container.getId(), ye);
        throw new SamzaContainerLaunchException(
                "Received YarnException when starting container: " + container.getId(), ye);
    } catch (IOException ioe) {
        log.error("Received IOException when starting container: " + container.getId(), ioe);
        throw new SamzaContainerLaunchException(
                "Received IOException when starting container: " + container.getId(), ioe);
    }
}

---

• «
• 1
• 2
• 3
• 4
• 5
• 6
• 7
• 8
• 9
• 10
• 11
• 12
• 13
• 14
• 15
• 16
• 17
• 18
• 19
• 20
• 21
• 22
• »