List of usage examples for org.apache.hadoop.io DataOutputBuffer DataOutputBuffer
public DataOutputBuffer()
From source file:org.apache.slider.server.appmaster.SliderAppMaster.java
License:Apache License
private ByteBuffer getContainerCredentials() throws IOException { // a delegation token can be retrieved from filesystem since // the login is via a keytab (see above) Credentials credentials = new Credentials(containerCredentials); ByteBuffer tokens = null;// www. j a v a 2 s . c o m Token<? extends TokenIdentifier>[] hdfsTokens = getClusterFS().getFileSystem() .addDelegationTokens(UserGroupInformation.getLoginUser().getShortUserName(), credentials); if (hdfsTokens.length > 0) { DataOutputBuffer dob = new DataOutputBuffer(); credentials.writeTokenStorageToStream(dob); dob.close(); tokens = ByteBuffer.wrap(dob.getData(), 0, dob.getLength()); } return tokens; }
From source file:org.apache.sqoop.mapreduce.mainframe.TestMainframeDatasetInputSplit.java
License:Apache License
@Test public void testWriteRead() { mfDatasetInputSplit.addDataset("dataSet1"); mfDatasetInputSplit.addDataset("dataSet2"); DataOutputBuffer dob = new DataOutputBuffer(); DataInputBuffer dib = new DataInputBuffer(); MainframeDatasetInputSplit mfReader = new MainframeDatasetInputSplit(); try {//from www . j a v a2s.com mfDatasetInputSplit.write(dob); dib.reset(dob.getData(), dob.getLength()); mfReader.readFields(dib); Assert.assertNotNull("MFReader get data from tester", mfReader); Assert.assertEquals(2, mfReader.getLength()); Assert.assertEquals("dataSet1", mfReader.getNextDataset()); Assert.assertEquals("dataSet2", mfReader.getNextDataset()); } catch (IOException ioe) { Assert.fail("No IOException should be thrown!"); } catch (InterruptedException ie) { Assert.fail("No InterruptedException should be thrown!"); } }
From source file:org.apache.tajo.pullserver.PullServerAuxService.java
License:Apache License
/** * Serialize the shuffle port into a ByteBuffer for use later on. * @param port the port to be sent to the ApplciationMaster * @return the serialized form of the port. *///ww w .j a v a 2 s.co m public static ByteBuffer serializeMetaData(int port) throws IOException { //TODO these bytes should be versioned DataOutputBuffer port_dob = new DataOutputBuffer(); port_dob.writeInt(port); return ByteBuffer.wrap(port_dob.getData(), 0, port_dob.getLength()); }
From source file:org.apache.tajo.yarn.command.LaunchCommand.java
License:Apache License
private void setupSecurityTokens(ContainerLaunchContext amContainer, FileSystem fs) throws IOException { if (UserGroupInformation.isSecurityEnabled()) { Credentials credentials = new Credentials(); String tokenRenewer = conf.get(YarnConfiguration.RM_PRINCIPAL); if (tokenRenewer == null || tokenRenewer.length() == 0) { throw new IOException("Can't get Master Kerberos principal for the RM to use as renewer"); }// w w w . j av a 2 s. co m // For now, only getting tokens for the default file-system. final Token<?> tokens[] = fs.addDelegationTokens(tokenRenewer, credentials); if (tokens != null) { for (Token<?> token : tokens) { LOG.info("Got dt for " + fs.getUri() + "; " + token); } } DataOutputBuffer dob = new DataOutputBuffer(); credentials.writeTokenStorageToStream(dob); ByteBuffer fsTokens = ByteBuffer.wrap(dob.getData(), 0, dob.getLength()); amContainer.setTokens(fsTokens); } }
From source file:org.apache.tajo.yarn.container.WorkerContainerTask.java
License:Apache License
@Override public ContainerLaunchContext getLaunchContext(Container container) throws IOException { // create a container launch context ContainerLaunchContext launchContext = Records.newRecord(ContainerLaunchContext.class); UserGroupInformation user = UserGroupInformation.getCurrentUser(); try {//ww w. j a va2 s . c o m Credentials credentials = user.getCredentials(); DataOutputBuffer dob = new DataOutputBuffer(); credentials.writeTokenStorageToStream(dob); ByteBuffer securityTokens = ByteBuffer.wrap(dob.getData(), 0, dob.getLength()); launchContext.setTokens(securityTokens); } catch (IOException e) { LOG.warn("Getting current user info failed when trying to launch the container" + e.getMessage()); } FileSystem fs = FileSystem.get(appContext.getConfiguration()); // Set the local resources Map<String, LocalResource> localResources = new HashMap<String, LocalResource>(); String suffix = "Tajo" + "/" + appContext.getApplicationId().getId(); Path parentPath = new Path(fs.getHomeDirectory(), suffix); // tar ball Path archivePath = new Path(parentPath, System.getenv(Constants.TAJO_ARCHIVE_PATH)); FileStatus archiveFs = fs.getFileStatus(archivePath); LocalResource archiveRsrc = LocalResource.newInstance(ConverterUtils.getYarnUrlFromURI(archivePath.toUri()), LocalResourceType.ARCHIVE, LocalResourceVisibility.APPLICATION, archiveFs.getLen(), archiveFs.getModificationTime()); localResources.put("tajo", archiveRsrc); Configuration tajoWorkerConf = new Configuration(false); tajoWorkerConf.addResource(new Path("conf", "tajo-site.xml")); tajoWorkerConf.set(Constants.TAJO_MASTER_UMBILICAL_RPC_ADDRESS, appContext.getMasterHost() + ":26001"); tajoWorkerConf.set(Constants.CATALOG_ADDRESS, appContext.getMasterHost() + ":26005"); Path dst = new Path(parentPath, container.getId() + Path.SEPARATOR + "worker-conf"); fs.mkdirs(dst); Path confFile = new Path(dst, "tajo-site.xml"); FSDataOutputStream fdos = fs.create(confFile); tajoWorkerConf.writeXml(fdos); fdos.close(); FileStatus scFileStatus = fs.getFileStatus(dst); LocalResource scRsrc = LocalResource.newInstance(ConverterUtils.getYarnUrlFromURI(dst.toUri()), LocalResourceType.FILE, LocalResourceVisibility.APPLICATION, scFileStatus.getLen(), scFileStatus.getModificationTime()); localResources.put("conf", scRsrc); launchContext.setLocalResources(localResources); // Set the environment setupEnv(launchContext); // Set the necessary command to execute on the allocated container Vector<CharSequence> vargs = new Vector<CharSequence>(5); // Set executable command // Set args for the shell command if any vargs.add("${" + Constants.TAJO_HOME + "}/bin/tajo"); vargs.add("--config"); vargs.add("${" + Constants.TAJO_CONF_DIR + "}"); vargs.add("worker"); // Add log redirect params // Add log redirect params vargs.add("1>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/stdout"); vargs.add("2>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/stderr"); // Get final commmand StringBuilder command = new StringBuilder(); for (CharSequence str : vargs) { command.append(str).append(" "); } List<String> commands = new ArrayList<String>(); commands.add(command.toString()); launchContext.setCommands(commands); return launchContext; }
From source file:org.apache.tez.auxservices.ShuffleHandler.java
License:Apache License
/** * A helper function to serialize the JobTokenIdentifier to be sent to the * ShuffleHandler as ServiceData./*from w w w . ja v a2 s .com*/ * @param jobToken the job token to be used for authentication of * shuffle data requests. * @return the serialized version of the jobToken. */ public static ByteBuffer serializeServiceData(Token<JobTokenIdentifier> jobToken) throws IOException { //TODO these bytes should be versioned DataOutputBuffer jobToken_dob = new DataOutputBuffer(); jobToken.write(jobToken_dob); return ByteBuffer.wrap(jobToken_dob.getData(), 0, jobToken_dob.getLength()); }
From source file:org.apache.tez.auxservices.TestShuffleHandler.java
License:Apache License
/** * Verify client prematurely closing a connection. * * @throws Exception exception.//from w ww .j ava 2s . c o m */ @Test(timeout = 10000) public void testClientClosesConnection() throws Exception { final ArrayList<Throwable> failures = new ArrayList<Throwable>(1); Configuration conf = new Configuration(); conf.setInt(ShuffleHandler.SHUFFLE_PORT_CONFIG_KEY, 0); ShuffleHandler shuffleHandler = new ShuffleHandler() { @Override protected Shuffle getShuffle(Configuration conf) { // replace the shuffle handler with one stubbed for testing return new Shuffle(conf) { @Override protected MapOutputInfo getMapOutputInfo(String dagId, String mapId, String jobId, String user) throws IOException { return null; } @Override protected void populateHeaders(List<String> mapIds, String jobId, String dagId, String user, Range reduceRange, HttpResponse response, boolean keepAliveParam, Map<String, MapOutputInfo> infoMap) throws IOException { // Only set response headers and skip everything else // send some dummy value for content-length super.setResponseHeaders(response, keepAliveParam, 100); } @Override protected void verifyRequest(String appid, ChannelHandlerContext ctx, HttpRequest request, HttpResponse response, URL requestUri) throws IOException { } @Override protected ChannelFuture sendMapOutput(ChannelHandlerContext ctx, Channel ch, String user, String mapId, Range reduceRange, MapOutputInfo info) throws IOException { // send a shuffle header and a lot of data down the channel // to trigger a broken pipe ShuffleHeader header = new ShuffleHeader("attempt_12345_1_m_1_0", 5678, 5678, 1); DataOutputBuffer dob = new DataOutputBuffer(); header.write(dob); ch.write(wrappedBuffer(dob.getData(), 0, dob.getLength())); dob = new DataOutputBuffer(); for (int i = 0; i < 100000; ++i) { header.write(dob); } return ch.write(wrappedBuffer(dob.getData(), 0, dob.getLength())); } @Override protected void sendError(ChannelHandlerContext ctx, HttpResponseStatus status) { if (failures.size() == 0) { failures.add(new Error()); ctx.getChannel().close(); } } @Override protected void sendError(ChannelHandlerContext ctx, String message, HttpResponseStatus status) { if (failures.size() == 0) { failures.add(new Error()); ctx.getChannel().close(); } } }; } }; shuffleHandler.init(conf); shuffleHandler.start(); // simulate a reducer that closes early by reading a single shuffle header // then closing the connection URL url = new URL( "http://127.0.0.1:" + shuffleHandler.getConfig().get(ShuffleHandler.SHUFFLE_PORT_CONFIG_KEY) + "/mapOutput?job=job_12345_1&dag=1&reduce=1&map=attempt_12345_1_m_1_0"); HttpURLConnection conn = (HttpURLConnection) url.openConnection(); conn.setRequestProperty(ShuffleHeader.HTTP_HEADER_NAME, ShuffleHeader.DEFAULT_HTTP_HEADER_NAME); conn.setRequestProperty(ShuffleHeader.HTTP_HEADER_VERSION, ShuffleHeader.DEFAULT_HTTP_HEADER_VERSION); conn.connect(); DataInputStream input = new DataInputStream(conn.getInputStream()); Assert.assertEquals(HttpURLConnection.HTTP_OK, conn.getResponseCode()); Assert.assertEquals("close", conn.getHeaderField(HttpHeaders.Names.CONNECTION)); ShuffleHeader header = new ShuffleHeader(); header.readFields(input); input.close(); shuffleHandler.stop(); Assert.assertTrue("sendError called when client closed connection", failures.size() == 0); }
From source file:org.apache.tez.auxservices.TestShuffleHandler.java
License:Apache License
@Test(timeout = 10000) public void testKeepAlive() throws Exception { final ArrayList<Throwable> failures = new ArrayList<Throwable>(1); Configuration conf = new Configuration(); conf.setInt(ShuffleHandler.SHUFFLE_PORT_CONFIG_KEY, 0); conf.setBoolean(ShuffleHandler.SHUFFLE_CONNECTION_KEEP_ALIVE_ENABLED, true); // try setting to -ve keep alive timeout. conf.setInt(ShuffleHandler.SHUFFLE_CONNECTION_KEEP_ALIVE_TIME_OUT, -100); final LastSocketAddress lastSocketAddress = new LastSocketAddress(); ShuffleHandler shuffleHandler = new ShuffleHandler() { @Override/*from w ww .j av a2 s .co m*/ protected Shuffle getShuffle(final Configuration conf) { // replace the shuffle handler with one stubbed for testing return new Shuffle(conf) { @Override protected MapOutputInfo getMapOutputInfo(String dagId, String mapId, String jobId, String user) throws IOException { return null; } @Override protected void verifyRequest(String appid, ChannelHandlerContext ctx, HttpRequest request, HttpResponse response, URL requestUri) throws IOException { } @Override protected void populateHeaders(List<String> mapIds, String jobId, String dagId, String user, Range reduceRange, HttpResponse response, boolean keepAliveParam, Map<String, MapOutputInfo> infoMap) throws IOException { // Send some dummy data (populate content length details) ShuffleHeader header = new ShuffleHeader("attempt_12345_1_m_1_0", 5678, 5678, 1); DataOutputBuffer dob = new DataOutputBuffer(); header.write(dob); dob = new DataOutputBuffer(); for (int i = 0; i < 100000; ++i) { header.write(dob); } long contentLength = dob.getLength(); // for testing purpose; // disable connectinKeepAliveEnabled if keepAliveParam is available if (keepAliveParam) { connectionKeepAliveEnabled = false; } super.setResponseHeaders(response, keepAliveParam, contentLength); } @Override protected ChannelFuture sendMapOutput(ChannelHandlerContext ctx, Channel ch, String user, String mapId, Range reduceRange, MapOutputInfo info) throws IOException { lastSocketAddress.setAddress(ch.getRemoteAddress()); HttpResponse response = new DefaultHttpResponse(HTTP_1_1, OK); // send a shuffle header and a lot of data down the channel // to trigger a broken pipe ShuffleHeader header = new ShuffleHeader("attempt_12345_1_m_1_0", 5678, 5678, 1); DataOutputBuffer dob = new DataOutputBuffer(); header.write(dob); ch.write(wrappedBuffer(dob.getData(), 0, dob.getLength())); dob = new DataOutputBuffer(); for (int i = 0; i < 100000; ++i) { header.write(dob); } return ch.write(wrappedBuffer(dob.getData(), 0, dob.getLength())); } @Override protected void sendError(ChannelHandlerContext ctx, HttpResponseStatus status) { if (failures.size() == 0) { failures.add(new Error()); ctx.getChannel().close(); } } @Override protected void sendError(ChannelHandlerContext ctx, String message, HttpResponseStatus status) { if (failures.size() == 0) { failures.add(new Error()); ctx.getChannel().close(); } } }; } }; shuffleHandler.init(conf); shuffleHandler.start(); String shuffleBaseURL = "http://127.0.0.1:" + shuffleHandler.getConfig().get(ShuffleHandler.SHUFFLE_PORT_CONFIG_KEY); URL url = new URL( shuffleBaseURL + "/mapOutput?job=job_12345_1&dag=1&reduce=1&" + "map=attempt_12345_1_m_1_0"); HttpURLConnection conn = (HttpURLConnection) url.openConnection(); conn.setRequestProperty(ShuffleHeader.HTTP_HEADER_NAME, ShuffleHeader.DEFAULT_HTTP_HEADER_NAME); conn.setRequestProperty(ShuffleHeader.HTTP_HEADER_VERSION, ShuffleHeader.DEFAULT_HTTP_HEADER_VERSION); conn.connect(); DataInputStream input = new DataInputStream(conn.getInputStream()); Assert.assertEquals(HttpHeaders.Values.KEEP_ALIVE, conn.getHeaderField(HttpHeaders.Names.CONNECTION)); Assert.assertEquals("timeout=1", conn.getHeaderField(HttpHeaders.Values.KEEP_ALIVE)); Assert.assertEquals(HttpURLConnection.HTTP_OK, conn.getResponseCode()); ShuffleHeader header = new ShuffleHeader(); header.readFields(input); byte[] buffer = new byte[1024]; while (input.read(buffer) != -1) { } SocketAddress firstAddress = lastSocketAddress.getSocketAddres(); input.close(); // For keepAlive via URL url = new URL(shuffleBaseURL + "/mapOutput?job=job_12345_1&dag=1&reduce=1&" + "map=attempt_12345_1_m_1_0&keepAlive=true"); conn = (HttpURLConnection) url.openConnection(); conn.setRequestProperty(ShuffleHeader.HTTP_HEADER_NAME, ShuffleHeader.DEFAULT_HTTP_HEADER_NAME); conn.setRequestProperty(ShuffleHeader.HTTP_HEADER_VERSION, ShuffleHeader.DEFAULT_HTTP_HEADER_VERSION); conn.connect(); input = new DataInputStream(conn.getInputStream()); Assert.assertEquals(HttpHeaders.Values.KEEP_ALIVE, conn.getHeaderField(HttpHeaders.Names.CONNECTION)); Assert.assertEquals("timeout=1", conn.getHeaderField(HttpHeaders.Values.KEEP_ALIVE)); Assert.assertEquals(HttpURLConnection.HTTP_OK, conn.getResponseCode()); header = new ShuffleHeader(); header.readFields(input); input.close(); SocketAddress secondAddress = lastSocketAddress.getSocketAddres(); Assert.assertNotNull("Initial shuffle address should not be null", firstAddress); Assert.assertNotNull("Keep-Alive shuffle address should not be null", secondAddress); Assert.assertEquals("Initial shuffle address and keep-alive shuffle " + "address should be the same", firstAddress, secondAddress); }
From source file:org.apache.tez.auxservices.TestShuffleHandler.java
License:Apache License
/** * Validate the limit on number of shuffle connections. * * @throws Exception exception/* w w w . ja va 2s . com*/ */ @Test(timeout = 10000) public void testMaxConnections() throws Exception { Configuration conf = new Configuration(); conf.setInt(ShuffleHandler.SHUFFLE_PORT_CONFIG_KEY, 0); conf.setInt(ShuffleHandler.MAX_SHUFFLE_CONNECTIONS, 3); ShuffleHandler shuffleHandler = new ShuffleHandler() { @Override protected Shuffle getShuffle(Configuration conf) { // replace the shuffle handler with one stubbed for testing return new Shuffle(conf) { @Override protected MapOutputInfo getMapOutputInfo(String dagId, String mapId, String jobId, String user) throws IOException { // Do nothing. return null; } @Override protected void populateHeaders(List<String> mapIds, String jobId, String dagId, String user, Range reduceRange, HttpResponse response, boolean keepAliveParam, Map<String, MapOutputInfo> infoMap) throws IOException { // Do nothing. } @Override protected void verifyRequest(String appid, ChannelHandlerContext ctx, HttpRequest request, HttpResponse response, URL requestUri) throws IOException { // Do nothing. } @Override protected ChannelFuture sendMapOutput(ChannelHandlerContext ctx, Channel ch, String user, String mapId, Range reduceRange, MapOutputInfo info) throws IOException { // send a shuffle header and a lot of data down the channel // to trigger a broken pipe ShuffleHeader header = new ShuffleHeader("dummy_header", 5678, 5678, 1); DataOutputBuffer dob = new DataOutputBuffer(); header.write(dob); ch.write(wrappedBuffer(dob.getData(), 0, dob.getLength())); dob = new DataOutputBuffer(); for (int i = 0; i < 100000; ++i) { header.write(dob); } return ch.write(wrappedBuffer(dob.getData(), 0, dob.getLength())); } }; } }; shuffleHandler.init(conf); shuffleHandler.start(); // setup connections int connAttempts = 3; HttpURLConnection conns[] = new HttpURLConnection[connAttempts]; for (int i = 0; i < connAttempts; i++) { String URLstring = "http://127.0.0.1:" + shuffleHandler.getConfig().get(ShuffleHandler.SHUFFLE_PORT_CONFIG_KEY) + "/mapOutput?job=job_12345_1&dag=1&reduce=1&map=attempt_12345_1_m_" + i + "_0"; URL url = new URL(URLstring); conns[i] = (HttpURLConnection) url.openConnection(); conns[i].setRequestProperty(ShuffleHeader.HTTP_HEADER_NAME, ShuffleHeader.DEFAULT_HTTP_HEADER_NAME); conns[i].setRequestProperty(ShuffleHeader.HTTP_HEADER_VERSION, ShuffleHeader.DEFAULT_HTTP_HEADER_VERSION); } // Try to open numerous connections for (int i = 0; i < connAttempts; i++) { conns[i].connect(); } //Ensure first connections are okay conns[0].getInputStream(); int rc = conns[0].getResponseCode(); Assert.assertEquals(HttpURLConnection.HTTP_OK, rc); conns[1].getInputStream(); rc = conns[1].getResponseCode(); Assert.assertEquals(HttpURLConnection.HTTP_OK, rc); // This connection should be closed because it to above the limit try { conns[2].getInputStream(); rc = conns[2].getResponseCode(); Assert.fail("Expected a SocketException"); } catch (SocketException se) { LOG.info("Expected - connection should not be open"); } catch (Exception e) { Assert.fail("Expected a SocketException"); } shuffleHandler.stop(); }
From source file:org.apache.tez.auxservices.TestShuffleHandler.java
License:Apache License
/** * Validate the ownership of the map-output files being pulled in. The * local-file-system owner of the file should match the user component in the * * @throws Exception exception// w w w. j a v a 2 s . com */ @Test(timeout = 100000) public void testMapFileAccess() throws IOException { // This will run only in NativeIO is enabled as SecureIOUtils need it assumeTrue(NativeIO.isAvailable()); Configuration conf = new Configuration(); conf.setInt(ShuffleHandler.SHUFFLE_PORT_CONFIG_KEY, 0); conf.setInt(ShuffleHandler.MAX_SHUFFLE_CONNECTIONS, 3); conf.set(CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHENTICATION, "kerberos"); UserGroupInformation.setConfiguration(conf); File absLogDir = new File("target", TestShuffleHandler.class.getSimpleName() + "LocDir").getAbsoluteFile(); conf.set(YarnConfiguration.NM_LOCAL_DIRS, absLogDir.getAbsolutePath()); ApplicationId appId = ApplicationId.newInstance(12345, 1); LOG.info(appId.toString()); String appAttemptId = "attempt_12345_1_m_1_0"; String user = "randomUser"; String reducerId = "0"; List<File> fileMap = new ArrayList<File>(); createShuffleHandlerFiles(absLogDir, user, appId.toString(), appAttemptId, conf, fileMap); ShuffleHandler shuffleHandler = new ShuffleHandler() { @Override protected Shuffle getShuffle(Configuration conf) { // replace the shuffle handler with one stubbed for testing return new Shuffle(conf) { @Override protected void verifyRequest(String appid, ChannelHandlerContext ctx, HttpRequest request, HttpResponse response, URL requestUri) throws IOException { // Do nothing. } }; } }; shuffleHandler.init(conf); try { shuffleHandler.start(); DataOutputBuffer outputBuffer = new DataOutputBuffer(); outputBuffer.reset(); Token<JobTokenIdentifier> jt = new Token<JobTokenIdentifier>("identifier".getBytes(), "password".getBytes(), new Text(user), new Text("shuffleService")); jt.write(outputBuffer); shuffleHandler.initializeApplication(new ApplicationInitializationContext(user, appId, ByteBuffer.wrap(outputBuffer.getData(), 0, outputBuffer.getLength()))); URL url = new URL("http://127.0.0.1:" + shuffleHandler.getConfig().get(ShuffleHandler.SHUFFLE_PORT_CONFIG_KEY) + "/mapOutput?job=job_12345_0001&dag=1&reduce=" + reducerId + "&map=attempt_12345_1_m_1_0"); HttpURLConnection conn = (HttpURLConnection) url.openConnection(); conn.setRequestProperty(ShuffleHeader.HTTP_HEADER_NAME, ShuffleHeader.DEFAULT_HTTP_HEADER_NAME); conn.setRequestProperty(ShuffleHeader.HTTP_HEADER_VERSION, ShuffleHeader.DEFAULT_HTTP_HEADER_VERSION); conn.connect(); byte[] byteArr = new byte[10000]; try { DataInputStream is = new DataInputStream(conn.getInputStream()); is.readFully(byteArr); } catch (EOFException e) { // ignore } // Retrieve file owner name FileInputStream is = new FileInputStream(fileMap.get(0)); String owner = NativeIO.POSIX.getFstat(is.getFD()).getOwner(); is.close(); String message = "Owner '" + owner + "' for path " + fileMap.get(0).getAbsolutePath() + " did not match expected owner '" + user + "'"; Assert.assertTrue((new String(byteArr)).contains(message)); } finally { shuffleHandler.stop(); FileUtil.fullyDelete(absLogDir); } }