List of usage examples for org.apache.hadoop.fs CommonConfigurationKeys IPC_MAXIMUM_DATA_LENGTH
String IPC_MAXIMUM_DATA_LENGTH
To view the source code for org.apache.hadoop.fs CommonConfigurationKeys IPC_MAXIMUM_DATA_LENGTH.
Click Source Link
From source file:me.haohui.libhdfspp.TestRpcEngine.java
License:Apache License
@BeforeClass public static void setUp() throws IOException { conf = new Configuration(); conf.setInt(CommonConfigurationKeys.IPC_MAXIMUM_DATA_LENGTH, 1024); // Set RPC engine to protobuf RPC engine RPC.setProtocolEngine(conf, TestRpcService.class, ProtobufRpcEngine.class); // Create server side implementation PBServerImpl serverImpl = new PBServerImpl(); BlockingService service = TestRpcServiceProtos.TestProtobufRpcProto .newReflectiveBlockingService(serverImpl); // Get RPC server for server side implementation server = new RPC.Builder(conf).setProtocol(TestRpcService.class).setInstance(service) .setBindAddress(ADDRESS).setPort(PORT).build(); addr = NetUtils.getConnectAddress(server); server.start();// ww w. ja v a2 s .c o m executor.start(); }
From source file:org.apache.tez.dag.api.client.rpc.TestDAGClientAMProtocolBlockingPBServerImpl.java
License:Apache License
@Test(timeout = 5000) @SuppressWarnings("unchecked") public void testSubmitDagInSessionWithLargeDagPlan() throws Exception { int maxIPCMsgSize = 1024; String dagPlanName = "dagplan-name"; File requestFile = tmpFolder.newFile("request-file"); TezConfiguration conf = new TezConfiguration(); conf.setInt(CommonConfigurationKeys.IPC_MAXIMUM_DATA_LENGTH, maxIPCMsgSize); byte[] randomBytes = new byte[2 * maxIPCMsgSize]; (new Random()).nextBytes(randomBytes); UserPayload payload = UserPayload.create(ByteBuffer.wrap(randomBytes)); Vertex vertex = Vertex.create("V", ProcessorDescriptor.create("P").setUserPayload(payload), 1); DAGPlan dagPlan = DAG.create(dagPlanName).addVertex(vertex).createDag(conf, null, null, null, false); String lrName = "localResource"; String scheme = "file"; String host = "localhost"; int port = 80; String path = "/test"; URL lrURL = URL.newInstance(scheme, host, port, path); LocalResource localResource = LocalResource.newInstance(lrURL, LocalResourceType.FILE, LocalResourceVisibility.PUBLIC, 1, 1); Map<String, LocalResource> localResources = new HashMap<>(); localResources.put(lrName, localResource); SubmitDAGRequestProto.Builder requestBuilder = SubmitDAGRequestProto.newBuilder().setDAGPlan(dagPlan) .setAdditionalAmResources(DagTypeConverters.convertFromLocalResources(localResources)); try (FileOutputStream fileOutputStream = new FileOutputStream(requestFile)) { requestBuilder.build().writeTo(fileOutputStream); }/*from w w w.j ava 2 s . c o m*/ DAGClientHandler dagClientHandler = mock(DAGClientHandler.class); ACLManager aclManager = mock(ACLManager.class); DAGClientAMProtocolBlockingPBServerImpl serverImpl = spy( new DAGClientAMProtocolBlockingPBServerImpl(dagClientHandler, FileSystem.get(conf))); when(dagClientHandler.getACLManager()).thenReturn(aclManager); when(dagClientHandler.submitDAG((DAGPlan) any(), (Map<String, LocalResource>) any())).thenReturn("dag-id"); when(aclManager.checkAMModifyAccess((UserGroupInformation) any())).thenReturn(true); requestBuilder.clear().setSerializedRequestPath(requestFile.getAbsolutePath()); serverImpl.submitDAG(null, requestBuilder.build()); ArgumentCaptor<DAGPlan> dagPlanCaptor = ArgumentCaptor.forClass(DAGPlan.class); verify(dagClientHandler).submitDAG(dagPlanCaptor.capture(), localResourcesCaptor.capture()); dagPlan = dagPlanCaptor.getValue(); localResources = localResourcesCaptor.getValue(); assertEquals(dagPlan.getName(), dagPlanName); assertEquals(dagPlan.getVertexCount(), 1); assertTrue(dagPlan.getSerializedSize() > maxIPCMsgSize); assertArrayEquals(randomBytes, dagPlan.getVertex(0).getProcessorDescriptor().getTezUserPayload().getUserPayload().toByteArray()); assertEquals(localResources.size(), 1); assertTrue(localResources.containsKey(lrName)); localResource = localResources.get(lrName); assertEquals(localResource.getType(), LocalResourceType.FILE); assertEquals(localResource.getVisibility(), LocalResourceVisibility.PUBLIC); lrURL = localResource.getResource(); assertEquals(lrURL.getScheme(), scheme); assertEquals(lrURL.getHost(), host); assertEquals(lrURL.getPort(), port); assertEquals(lrURL.getFile(), path); }