List of usage examples for org.apache.hadoop.io DataOutputBuffer reset
public DataOutputBuffer reset()
From source file:com.netflix.suro.input.thrift.MessageSetSerDe.java
License:Apache License
@Override public byte[] serialize(TMessageSet payload) { DataOutputBuffer outBuffer = new DataOutputBuffer(); try {/*from www . j ava 2s. c o m*/ outBuffer.reset(); outBuffer.writeUTF(payload.getApp()); outBuffer.writeInt(payload.getNumMessages()); outBuffer.writeByte(payload.getCompression()); outBuffer.writeLong(payload.getCrc()); outBuffer.writeInt(payload.getMessages().length); outBuffer.write(payload.getMessages()); return ByteBuffer.wrap(outBuffer.getData(), 0, outBuffer.getLength()).array(); } catch (Exception e) { throw new RuntimeException("Failed to serialize TMessageSet: " + e.getMessage(), e); } finally { Closeables.closeQuietly(outBuffer); } }
From source file:it.crs4.pydoop.mapreduce.pipes.CommonStub.java
License:Apache License
protected void writeObject(Writable obj, DataOutputStream stream) throws IOException { // For Text and BytesWritable, encode them directly, so that they end up // in C++ as the natural translations. System.err.println("obj: " + obj); DataOutputBuffer buffer = new DataOutputBuffer(); if (obj instanceof Text) { Text t = (Text) obj; int len = t.getLength(); WritableUtils.writeVLong(stream, len); stream.flush();//from w ww . j av a 2 s . c o m stream.write(t.getBytes(), 0, len); stream.flush(); System.err.println("len: " + len); } else if (obj instanceof BytesWritable) { BytesWritable b = (BytesWritable) obj; int len = b.getLength(); WritableUtils.writeVLong(stream, len); stream.write(b.getBytes(), 0, len); System.err.println("len: " + len); } else { buffer.reset(); obj.write(buffer); int length = buffer.getLength(); WritableUtils.writeVInt(stream, length); stream.write(buffer.getData(), 0, length); System.err.println("len: " + length); } stream.flush(); }
From source file:org.apache.accumulo.master.replication.WorkMaker.java
License:Apache License
protected void addWorkRecord(Text file, Value v, Map<String, String> targets, String sourceTableId) { log.info("Adding work records for " + file + " to targets " + targets); try {//from w w w. j a va2s . c o m Mutation m = new Mutation(file); ReplicationTarget target = new ReplicationTarget(); DataOutputBuffer buffer = new DataOutputBuffer(); Text t = new Text(); for (Entry<String, String> entry : targets.entrySet()) { buffer.reset(); // Set up the writable target.setPeerName(entry.getKey()); target.setRemoteIdentifier(entry.getValue()); target.setSourceTableId(sourceTableId); target.write(buffer); // Throw it in a text for the mutation t.set(buffer.getData(), 0, buffer.getLength()); // Add it to the work section WorkSection.add(m, t, v); } try { writer.addMutation(m); } catch (MutationsRejectedException e) { log.warn("Failed to write work mutations for replication, will retry", e); } } catch (IOException e) { log.warn("Failed to serialize data to Text, will retry", e); } finally { try { writer.flush(); } catch (MutationsRejectedException e) { log.warn("Failed to write work mutations for replication, will retry", e); } } }
From source file:org.apache.blur.lucene.serializer.ProtoSerializer.java
License:Apache License
private static void run(Query query) throws IOException { DataOutputBuffer buffer = new DataOutputBuffer(); DataInputBuffer in = new DataInputBuffer(); QueryWritable outQw = new QueryWritable(); QueryWritable inQw = new QueryWritable(); long s = System.nanoTime(); int count = 100000; for (int i = 0; i < count; i++) { outQw.setQuery(query);/* w w w.j a va 2 s .com*/ outQw.write(buffer); in.reset(buffer.getData(), 0, buffer.getLength()); inQw.readFields(in); buffer.reset(); } long e = System.nanoTime(); System.out.println((e - s) / 1000000.0 / (double) count); // System.out.println((e - s) / (double) count); }
From source file:org.apache.hama.bsp.BSPJobClient.java
License:Apache License
private static int writeSplits(BSPJob job, InputSplit[] splits, Path submitSplitFile, int maxTasks) throws IOException { final DataOutputStream out = writeSplitsFileHeader(job.getConfiguration(), submitSplitFile, splits.length); try {//from w w w . j a va 2 s.c o m DataOutputBuffer buffer = new DataOutputBuffer(); RawSplit rawSplit = new RawSplit(); for (InputSplit split : splits) { // set partitionID to rawSplit if (split.getClass().getName().equals(FileSplit.class.getName()) && job.getBoolean("input.has.partitioned", false)) { String[] extractPartitionID = ((FileSplit) split).getPath().getName().split("[-]"); if (extractPartitionID.length > 1) rawSplit.setPartitionID(Integer.parseInt(extractPartitionID[1])); } rawSplit.setClassName(split.getClass().getName()); buffer.reset(); split.write(buffer); rawSplit.setDataLength(split.getLength()); rawSplit.setBytes(buffer.getData(), 0, buffer.getLength()); rawSplit.setLocations(split.getLocations()); rawSplit.write(out); } } finally { out.close(); } return splits.length; }
From source file:org.apache.tez.auxservices.TestShuffleHandler.java
License:Apache License
/** * Validate the ownership of the map-output files being pulled in. The * local-file-system owner of the file should match the user component in the * * @throws Exception exception//from w w w. j a va 2 s . c om */ @Test(timeout = 100000) public void testMapFileAccess() throws IOException { // This will run only in NativeIO is enabled as SecureIOUtils need it assumeTrue(NativeIO.isAvailable()); Configuration conf = new Configuration(); conf.setInt(ShuffleHandler.SHUFFLE_PORT_CONFIG_KEY, 0); conf.setInt(ShuffleHandler.MAX_SHUFFLE_CONNECTIONS, 3); conf.set(CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHENTICATION, "kerberos"); UserGroupInformation.setConfiguration(conf); File absLogDir = new File("target", TestShuffleHandler.class.getSimpleName() + "LocDir").getAbsoluteFile(); conf.set(YarnConfiguration.NM_LOCAL_DIRS, absLogDir.getAbsolutePath()); ApplicationId appId = ApplicationId.newInstance(12345, 1); LOG.info(appId.toString()); String appAttemptId = "attempt_12345_1_m_1_0"; String user = "randomUser"; String reducerId = "0"; List<File> fileMap = new ArrayList<File>(); createShuffleHandlerFiles(absLogDir, user, appId.toString(), appAttemptId, conf, fileMap); ShuffleHandler shuffleHandler = new ShuffleHandler() { @Override protected Shuffle getShuffle(Configuration conf) { // replace the shuffle handler with one stubbed for testing return new Shuffle(conf) { @Override protected void verifyRequest(String appid, ChannelHandlerContext ctx, HttpRequest request, HttpResponse response, URL requestUri) throws IOException { // Do nothing. } }; } }; shuffleHandler.init(conf); try { shuffleHandler.start(); DataOutputBuffer outputBuffer = new DataOutputBuffer(); outputBuffer.reset(); Token<JobTokenIdentifier> jt = new Token<JobTokenIdentifier>("identifier".getBytes(), "password".getBytes(), new Text(user), new Text("shuffleService")); jt.write(outputBuffer); shuffleHandler.initializeApplication(new ApplicationInitializationContext(user, appId, ByteBuffer.wrap(outputBuffer.getData(), 0, outputBuffer.getLength()))); URL url = new URL("http://127.0.0.1:" + shuffleHandler.getConfig().get(ShuffleHandler.SHUFFLE_PORT_CONFIG_KEY) + "/mapOutput?job=job_12345_0001&dag=1&reduce=" + reducerId + "&map=attempt_12345_1_m_1_0"); HttpURLConnection conn = (HttpURLConnection) url.openConnection(); conn.setRequestProperty(ShuffleHeader.HTTP_HEADER_NAME, ShuffleHeader.DEFAULT_HTTP_HEADER_NAME); conn.setRequestProperty(ShuffleHeader.HTTP_HEADER_VERSION, ShuffleHeader.DEFAULT_HTTP_HEADER_VERSION); conn.connect(); byte[] byteArr = new byte[10000]; try { DataInputStream is = new DataInputStream(conn.getInputStream()); is.readFully(byteArr); } catch (EOFException e) { // ignore } // Retrieve file owner name FileInputStream is = new FileInputStream(fileMap.get(0)); String owner = NativeIO.POSIX.getFstat(is.getFD()).getOwner(); is.close(); String message = "Owner '" + owner + "' for path " + fileMap.get(0).getAbsolutePath() + " did not match expected owner '" + user + "'"; Assert.assertTrue((new String(byteArr)).contains(message)); } finally { shuffleHandler.stop(); FileUtil.fullyDelete(absLogDir); } }
From source file:org.apache.tez.auxservices.TestShuffleHandler.java
License:Apache License
@Test public void testRecovery() throws IOException { final String user = "someuser"; final ApplicationId appId = ApplicationId.newInstance(12345, 1); final JobID jobId = JobID.downgrade(TypeConverter.fromYarn(appId)); final File tmpDir = new File(System.getProperty("test.build.data", System.getProperty("java.io.tmpdir")), TestShuffleHandler.class.getName()); Configuration conf = new Configuration(); conf.setInt(ShuffleHandler.SHUFFLE_PORT_CONFIG_KEY, 0); conf.setInt(ShuffleHandler.MAX_SHUFFLE_CONNECTIONS, 3); ShuffleHandler shuffle = new ShuffleHandler(); // emulate aux services startup with recovery enabled shuffle.setRecoveryPath(new Path(tmpDir.toString())); tmpDir.mkdirs();/*from w w w . ja va 2 s . com*/ try { shuffle.init(conf); shuffle.start(); // setup a shuffle token for an application DataOutputBuffer outputBuffer = new DataOutputBuffer(); outputBuffer.reset(); Token<JobTokenIdentifier> jt = new Token<JobTokenIdentifier>("identifier".getBytes(), "password".getBytes(), new Text(user), new Text("shuffleService")); jt.write(outputBuffer); shuffle.initializeApplication(new ApplicationInitializationContext(user, appId, ByteBuffer.wrap(outputBuffer.getData(), 0, outputBuffer.getLength()))); // verify we are authorized to shuffle int rc = getShuffleResponseCode(shuffle, jt); Assert.assertEquals(HttpURLConnection.HTTP_OK, rc); // emulate shuffle handler restart shuffle.close(); shuffle = new ShuffleHandler(); shuffle.setRecoveryPath(new Path(tmpDir.toString())); shuffle.init(conf); shuffle.start(); // verify we are still authorized to shuffle to the old application rc = getShuffleResponseCode(shuffle, jt); Assert.assertEquals(HttpURLConnection.HTTP_OK, rc); // shutdown app and verify access is lost shuffle.stopApplication(new ApplicationTerminationContext(appId)); rc = getShuffleResponseCode(shuffle, jt); Assert.assertEquals(HttpURLConnection.HTTP_UNAUTHORIZED, rc); // emulate shuffle handler restart shuffle.close(); shuffle = new ShuffleHandler(); shuffle.setRecoveryPath(new Path(tmpDir.toString())); shuffle.init(conf); shuffle.start(); // verify we still don't have access rc = getShuffleResponseCode(shuffle, jt); Assert.assertEquals(HttpURLConnection.HTTP_UNAUTHORIZED, rc); } finally { if (shuffle != null) { shuffle.close(); } FileUtil.fullyDelete(tmpDir); } }
From source file:org.apache.tez.auxservices.TestShuffleHandler.java
License:Apache License
@Test public void testRecoveryFromOtherVersions() throws IOException { final String user = "someuser"; final ApplicationId appId = ApplicationId.newInstance(12345, 1); final File tmpDir = new File(System.getProperty("test.build.data", System.getProperty("java.io.tmpdir")), TestShuffleHandler.class.getName()); Configuration conf = new Configuration(); conf.setInt(ShuffleHandler.SHUFFLE_PORT_CONFIG_KEY, 0); conf.setInt(ShuffleHandler.MAX_SHUFFLE_CONNECTIONS, 3); ShuffleHandler shuffle = new ShuffleHandler(); // emulate aux services startup with recovery enabled shuffle.setRecoveryPath(new Path(tmpDir.toString())); tmpDir.mkdirs();//from w w w .j a v a 2 s . c o m try { shuffle.init(conf); shuffle.start(); // setup a shuffle token for an application DataOutputBuffer outputBuffer = new DataOutputBuffer(); outputBuffer.reset(); Token<JobTokenIdentifier> jt = new Token<JobTokenIdentifier>("identifier".getBytes(), "password".getBytes(), new Text(user), new Text("shuffleService")); jt.write(outputBuffer); shuffle.initializeApplication(new ApplicationInitializationContext(user, appId, ByteBuffer.wrap(outputBuffer.getData(), 0, outputBuffer.getLength()))); // verify we are authorized to shuffle int rc = getShuffleResponseCode(shuffle, jt); Assert.assertEquals(HttpURLConnection.HTTP_OK, rc); // emulate shuffle handler restart shuffle.close(); shuffle = new ShuffleHandler(); shuffle.setRecoveryPath(new Path(tmpDir.toString())); shuffle.init(conf); shuffle.start(); // verify we are still authorized to shuffle to the old application rc = getShuffleResponseCode(shuffle, jt); Assert.assertEquals(HttpURLConnection.HTTP_OK, rc); Version version = Version.newInstance(1, 0); Assert.assertEquals(version, shuffle.getCurrentVersion()); // emulate shuffle handler restart with compatible version Version version11 = Version.newInstance(1, 1); // update version info before close shuffle shuffle.storeVersion(version11); Assert.assertEquals(version11, shuffle.loadVersion()); shuffle.close(); shuffle = new ShuffleHandler(); shuffle.setRecoveryPath(new Path(tmpDir.toString())); shuffle.init(conf); shuffle.start(); // shuffle version will be override by CURRENT_VERSION_INFO after restart // successfully. Assert.assertEquals(version, shuffle.loadVersion()); // verify we are still authorized to shuffle to the old application rc = getShuffleResponseCode(shuffle, jt); Assert.assertEquals(HttpURLConnection.HTTP_OK, rc); // emulate shuffle handler restart with incompatible version Version version21 = Version.newInstance(2, 1); shuffle.storeVersion(version21); Assert.assertEquals(version21, shuffle.loadVersion()); shuffle.close(); shuffle = new ShuffleHandler(); shuffle.setRecoveryPath(new Path(tmpDir.toString())); shuffle.init(conf); try { shuffle.start(); Assert.fail("Incompatible version, should expect fail here."); } catch (ServiceStateException e) { Assert.assertTrue("Exception message mismatch", e.getMessage().contains("Incompatible version for state DB schema:")); } } finally { if (shuffle != null) { shuffle.close(); } FileUtil.fullyDelete(tmpDir); } }
From source file:org.apache.tez.auxservices.TestShuffleHandler.java
License:Apache License
@Test(timeout = 100000) public void testGetMapOutputInfo() throws Exception { final ArrayList<Throwable> failures = new ArrayList<Throwable>(1); Configuration conf = new Configuration(); conf.setInt(ShuffleHandler.SHUFFLE_PORT_CONFIG_KEY, 0); conf.setInt(ShuffleHandler.MAX_SHUFFLE_CONNECTIONS, 3); conf.set(CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHENTICATION, "simple"); UserGroupInformation.setConfiguration(conf); File absLogDir = new File("target", TestShuffleHandler.class.getSimpleName() + "LocDir").getAbsoluteFile(); conf.set(YarnConfiguration.NM_LOCAL_DIRS, absLogDir.getAbsolutePath()); ApplicationId appId = ApplicationId.newInstance(12345, 1); String appAttemptId = "attempt_12345_1_m_1_0"; String user = "randomUser"; String reducerId = "0"; List<File> fileMap = new ArrayList<File>(); createShuffleHandlerFiles(absLogDir, user, appId.toString(), appAttemptId, conf, fileMap); ShuffleHandler shuffleHandler = new ShuffleHandler() { @Override//from w ww . ja v a 2s . c om protected Shuffle getShuffle(Configuration conf) { // replace the shuffle handler with one stubbed for testing return new Shuffle(conf) { @Override protected void populateHeaders(List<String> mapIds, String outputBaseStr, String dagId, String user, Range reduceRange, HttpResponse response, boolean keepAliveParam, Map<String, MapOutputInfo> infoMap) throws IOException { // Only set response headers and skip everything else // send some dummy value for content-length super.setResponseHeaders(response, keepAliveParam, 100); } @Override protected void verifyRequest(String appid, ChannelHandlerContext ctx, HttpRequest request, HttpResponse response, URL requestUri) throws IOException { // Do nothing. } @Override protected void sendError(ChannelHandlerContext ctx, String message, HttpResponseStatus status) { if (failures.size() == 0) { failures.add(new Error(message)); ctx.getChannel().close(); } } @Override protected ChannelFuture sendMapOutput(ChannelHandlerContext ctx, Channel ch, String user, String mapId, Range reduceRange, MapOutputInfo info) throws IOException { // send a shuffle header ShuffleHeader header = new ShuffleHeader("attempt_12345_1_m_1_0", 5678, 5678, 1); DataOutputBuffer dob = new DataOutputBuffer(); header.write(dob); return ch.write(wrappedBuffer(dob.getData(), 0, dob.getLength())); } }; } }; shuffleHandler.init(conf); try { shuffleHandler.start(); DataOutputBuffer outputBuffer = new DataOutputBuffer(); outputBuffer.reset(); Token<JobTokenIdentifier> jt = new Token<JobTokenIdentifier>("identifier".getBytes(), "password".getBytes(), new Text(user), new Text("shuffleService")); jt.write(outputBuffer); shuffleHandler.initializeApplication(new ApplicationInitializationContext(user, appId, ByteBuffer.wrap(outputBuffer.getData(), 0, outputBuffer.getLength()))); URL url = new URL("http://127.0.0.1:" + shuffleHandler.getConfig().get(ShuffleHandler.SHUFFLE_PORT_CONFIG_KEY) + "/mapOutput?job=job_12345_0001&dag=1&reduce=" + reducerId + "&map=attempt_12345_1_m_1_0"); HttpURLConnection conn = (HttpURLConnection) url.openConnection(); conn.setRequestProperty(ShuffleHeader.HTTP_HEADER_NAME, ShuffleHeader.DEFAULT_HTTP_HEADER_NAME); conn.setRequestProperty(ShuffleHeader.HTTP_HEADER_VERSION, ShuffleHeader.DEFAULT_HTTP_HEADER_VERSION); conn.connect(); try { DataInputStream is = new DataInputStream(conn.getInputStream()); ShuffleHeader header = new ShuffleHeader(); header.readFields(is); is.close(); } catch (EOFException e) { // ignore } Assert.assertEquals("sendError called due to shuffle error", 0, failures.size()); } finally { shuffleHandler.stop(); FileUtil.fullyDelete(absLogDir); } }
From source file:org.apache.tez.auxservices.TestShuffleHandler.java
License:Apache License
@Test(timeout = 5000) public void testDagDelete() throws Exception { final ArrayList<Throwable> failures = new ArrayList<Throwable>(1); Configuration conf = new Configuration(); conf.setInt(ShuffleHandler.MAX_SHUFFLE_CONNECTIONS, 3); conf.setInt(ShuffleHandler.SHUFFLE_PORT_CONFIG_KEY, 0); conf.set(CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHENTICATION, "simple"); UserGroupInformation.setConfiguration(conf); File absLogDir = new File("target", TestShuffleHandler.class.getSimpleName() + "LocDir").getAbsoluteFile(); conf.set(YarnConfiguration.NM_LOCAL_DIRS, absLogDir.getAbsolutePath()); ApplicationId appId = ApplicationId.newInstance(12345, 1); String appAttemptId = "attempt_12345_1_m_1_0"; String user = "randomUser"; List<File> fileMap = new ArrayList<File>(); createShuffleHandlerFiles(absLogDir, user, appId.toString(), appAttemptId, conf, fileMap); ShuffleHandler shuffleHandler = new ShuffleHandler() { @Override//from w ww. j a v a2s.c o m protected Shuffle getShuffle(Configuration conf) { // replace the shuffle handler with one stubbed for testing return new Shuffle(conf) { @Override protected void sendError(ChannelHandlerContext ctx, String message, HttpResponseStatus status) { if (failures.size() == 0) { failures.add(new Error(message)); ctx.getChannel().close(); } } }; } }; shuffleHandler.init(conf); try { shuffleHandler.start(); DataOutputBuffer outputBuffer = new DataOutputBuffer(); outputBuffer.reset(); Token<JobTokenIdentifier> jt = new Token<JobTokenIdentifier>("identifier".getBytes(), "password".getBytes(), new Text(user), new Text("shuffleService")); jt.write(outputBuffer); shuffleHandler.initializeApplication(new ApplicationInitializationContext(user, appId, ByteBuffer.wrap(outputBuffer.getData(), 0, outputBuffer.getLength()))); URL url = new URL( "http://127.0.0.1:" + shuffleHandler.getConfig().get(ShuffleHandler.SHUFFLE_PORT_CONFIG_KEY) + "/mapOutput?dagAction=delete&job=job_12345_0001&dag=1"); HttpURLConnection conn = (HttpURLConnection) url.openConnection(); conn.setRequestProperty(ShuffleHeader.HTTP_HEADER_NAME, ShuffleHeader.DEFAULT_HTTP_HEADER_NAME); conn.setRequestProperty(ShuffleHeader.HTTP_HEADER_VERSION, ShuffleHeader.DEFAULT_HTTP_HEADER_VERSION); String dagDirStr = StringUtils.join(Path.SEPARATOR, new String[] { absLogDir.getAbsolutePath(), ShuffleHandler.USERCACHE, user, ShuffleHandler.APPCACHE, appId.toString(), "dag_1/" }); File dagDir = new File(dagDirStr); Assert.assertTrue("Dag Directory does not exist!", dagDir.exists()); conn.connect(); try { DataInputStream is = new DataInputStream(conn.getInputStream()); is.close(); Assert.assertFalse("Dag Directory was not deleted!", dagDir.exists()); } catch (EOFException e) { // ignore } Assert.assertEquals("sendError called due to shuffle error", 0, failures.size()); } finally { shuffleHandler.stop(); FileUtil.fullyDelete(absLogDir); } }