List of usage examples for java.io ByteArrayOutputStream reset
public synchronized void reset()
From source file:org.mcisb.subliminal.SubliminalUtils.java
/** * //from w w w . j av a2 s . c o m * @param elementName * @param is * @param onlyValues * @return Collection * @throws XMLStreamException * @throws UnsupportedEncodingException */ private static Collection<String> getElements(final String elementName, final InputStream is, final boolean onlyValues) throws XMLStreamException, UnsupportedEncodingException { final Collection<String> elements = new ArrayList<>(); final ByteArrayOutputStream os = new ByteArrayOutputStream(); final XMLEventReader reader = XMLInputFactory.newInstance() .createXMLEventReader(new InputStreamReader(is, Charset.defaultCharset().name())); final XMLEventWriter writer = XMLOutputFactory.newInstance() .createXMLEventWriter(new OutputStreamWriter(os, Charset.defaultCharset().name())); boolean read = false; String characters = null; while (reader.peek() != null) { final XMLEvent event = (XMLEvent) reader.next(); switch (event.getEventType()) { case XMLStreamConstants.START_DOCUMENT: case XMLStreamConstants.END_DOCUMENT: { // Ignore. break; } case XMLStreamConstants.START_ELEMENT: { read = read || elementName.equals(event.asStartElement().getName().getLocalPart()); if (read && !onlyValues) { writer.add(event); } break; } case XMLStreamConstants.ATTRIBUTE: { if (read && !onlyValues) { writer.add(event); } break; } case XMLStreamConstants.CHARACTERS: { if (read && !onlyValues) { writer.add(event); } characters = event.asCharacters().getData(); break; } case XMLStreamConstants.END_ELEMENT: { if (read && !onlyValues) { writer.add(event); } if (elementName.equals(event.asEndElement().getName().getLocalPart())) { writer.flush(); if (characters != null) { elements.add(characters); } os.reset(); read = false; } break; } default: { // Ignore break; } } } return elements; }
From source file:com.streamsets.pipeline.stage.origin.kafka.TestKafkaSource.java
@Ignore @Test/*w w w. j a v a 2 s . c o m*/ public void testProduceAvroRecordsWithOutSchema() throws Exception { //create Producer and send messages Schema schema = new Schema.Parser().parse(AVRO_SCHEMA); GenericRecord boss = new GenericData.Record(schema); boss.put("name", "boss"); boss.put("age", 60); boss.put("emails", ImmutableList.of("boss@company.com", "boss2@company.com")); boss.put("boss", null); GenericRecord e3 = new GenericData.Record(schema); e3.put("name", "c"); e3.put("age", 50); e3.put("emails", ImmutableList.of("c@company.com", "c2@company.com")); e3.put("boss", boss); GenericRecord e2 = new GenericData.Record(schema); e2.put("name", "b"); e2.put("age", 40); e2.put("emails", ImmutableList.of("b@company.com", "b2@company.com")); e2.put("boss", boss); GenericRecord e1 = new GenericData.Record(schema); e1.put("name", "a"); e1.put("age", 30); e1.put("emails", ImmutableList.of("a@company.com", "a2@company.com")); e1.put("boss", boss); Producer<String, byte[]> producer = createDefaultProducer(); ByteArrayOutputStream baos = new ByteArrayOutputStream(); GenericDatumWriter<GenericRecord> genericDatumWriter = new GenericDatumWriter<>(schema); BinaryEncoder binaryEncoder = EncoderFactory.get().binaryEncoder(baos, null); genericDatumWriter.write(e1, binaryEncoder); binaryEncoder.flush(); producer.send(new KeyedMessage<>(TOPIC13, "0", baos.toByteArray())); baos.reset(); genericDatumWriter.write(e2, binaryEncoder); binaryEncoder.flush(); producer.send(new KeyedMessage<>(TOPIC13, "0", baos.toByteArray())); baos.reset(); genericDatumWriter.write(e3, binaryEncoder); binaryEncoder.flush(); producer.send(new KeyedMessage<>(TOPIC13, "0", baos.toByteArray())); Map<String, String> kafkaConsumerConfigs = new HashMap<>(); sdcKafkaTestUtil.setAutoOffsetReset(kafkaConsumerConfigs); KafkaConfigBean conf = new KafkaConfigBean(); conf.metadataBrokerList = sdcKafkaTestUtil.getMetadataBrokerURI(); conf.topic = TOPIC13; conf.consumerGroup = CONSUMER_GROUP; conf.zookeeperConnect = zkConnect; conf.maxBatchSize = 100; conf.maxWaitTime = 10000; conf.kafkaConsumerConfigs = kafkaConsumerConfigs; conf.produceSingleRecordPerMessage = false; conf.dataFormat = DataFormat.AVRO; conf.dataFormatConfig.charset = "UTF-8"; conf.dataFormatConfig.removeCtrlChars = false; conf.dataFormatConfig.schemaInMessage = false; conf.dataFormatConfig.avroSchema = AVRO_SCHEMA; SourceRunner sourceRunner = new SourceRunner.Builder(StandaloneKafkaSource.class, createSource(conf)) .addOutputLane("lane").build(); sourceRunner.runInit(); StageRunner.Output output = sourceRunner.runProduce(null, 10); String newOffset = output.getNewOffset(); Assert.assertNull(newOffset); List<Record> records = output.getRecords().get("lane"); Assert.assertEquals(0, sourceRunner.getErrorRecords().size()); Assert.assertEquals(3, records.size()); Record e3Record = records.get(2); Assert.assertTrue(e3Record.has("/name")); Assert.assertEquals("c", e3Record.get("/name").getValueAsString()); Assert.assertTrue(e3Record.has("/age")); Assert.assertEquals(50, e3Record.get("/age").getValueAsInteger()); Assert.assertTrue(e3Record.has("/emails")); Assert.assertTrue(e3Record.get("/emails").getValueAsList() instanceof List); List<Field> emails = e3Record.get("/emails").getValueAsList(); Assert.assertEquals(2, emails.size()); Assert.assertEquals("c@company.com", emails.get(0).getValueAsString()); Assert.assertEquals("c2@company.com", emails.get(1).getValueAsString()); Assert.assertTrue(e3Record.has("/boss")); Assert.assertTrue(e3Record.get("/boss").getValueAsMap() instanceof Map); Assert.assertTrue(e3Record.has("/boss/name")); Assert.assertEquals("boss", e3Record.get("/boss/name").getValueAsString()); Assert.assertTrue(e3Record.has("/boss/age")); Assert.assertEquals(60, e3Record.get("/boss/age").getValueAsInteger()); Assert.assertTrue(e3Record.has("/boss/emails")); Assert.assertTrue(e3Record.get("/boss/emails").getValueAsList() instanceof List); emails = e3Record.get("/boss/emails").getValueAsList(); Assert.assertEquals(2, emails.size()); Assert.assertEquals("boss@company.com", emails.get(0).getValueAsString()); Assert.assertEquals("boss2@company.com", emails.get(1).getValueAsString()); Record e2Record = records.get(1); Assert.assertTrue(e2Record.has("/name")); Assert.assertEquals("b", e2Record.get("/name").getValueAsString()); Assert.assertTrue(e2Record.has("/age")); Assert.assertEquals(40, e2Record.get("/age").getValueAsInteger()); Assert.assertTrue(e2Record.has("/emails")); Assert.assertTrue(e2Record.get("/emails").getValueAsList() instanceof List); emails = e2Record.get("/emails").getValueAsList(); Assert.assertEquals(2, emails.size()); Assert.assertEquals("b@company.com", emails.get(0).getValueAsString()); Assert.assertEquals("b2@company.com", emails.get(1).getValueAsString()); Assert.assertTrue(e2Record.has("/boss")); Assert.assertTrue(e2Record.get("/boss").getValueAsMap() instanceof Map); Assert.assertTrue(e2Record.has("/boss/name")); Assert.assertEquals("boss", e2Record.get("/boss/name").getValueAsString()); Assert.assertTrue(e2Record.has("/boss/age")); Assert.assertEquals(60, e2Record.get("/boss/age").getValueAsInteger()); Assert.assertTrue(e2Record.has("/boss/emails")); Assert.assertTrue(e2Record.get("/boss/emails").getValueAsList() instanceof List); emails = e2Record.get("/boss/emails").getValueAsList(); Assert.assertEquals(2, emails.size()); Assert.assertEquals("boss@company.com", emails.get(0).getValueAsString()); Assert.assertEquals("boss2@company.com", emails.get(1).getValueAsString()); Record e1Record = records.get(0); Assert.assertTrue(e1Record.has("/name")); Assert.assertEquals("a", e1Record.get("/name").getValueAsString()); Assert.assertTrue(e1Record.has("/age")); Assert.assertEquals(30, e1Record.get("/age").getValueAsInteger()); Assert.assertTrue(e1Record.has("/emails")); Assert.assertTrue(e1Record.get("/emails").getValueAsList() instanceof List); emails = e1Record.get("/emails").getValueAsList(); Assert.assertEquals(2, emails.size()); Assert.assertEquals("a@company.com", emails.get(0).getValueAsString()); Assert.assertEquals("a2@company.com", emails.get(1).getValueAsString()); Assert.assertTrue(e1Record.has("/boss")); Assert.assertTrue(e1Record.get("/boss").getValueAsMap() instanceof Map); Assert.assertTrue(e1Record.has("/boss/name")); Assert.assertEquals("boss", e1Record.get("/boss/name").getValueAsString()); Assert.assertTrue(e1Record.has("/boss/age")); Assert.assertEquals(60, e1Record.get("/boss/age").getValueAsInteger()); Assert.assertTrue(e1Record.has("/boss/emails")); Assert.assertTrue(e1Record.get("/boss/emails").getValueAsList() instanceof List); emails = e1Record.get("/boss/emails").getValueAsList(); Assert.assertEquals(2, emails.size()); Assert.assertEquals("boss@company.com", emails.get(0).getValueAsString()); Assert.assertEquals("boss2@company.com", emails.get(1).getValueAsString()); }
From source file:org.apache.pig.piggybank.storage.XMLLoaderBufferedPositionedInputStream.java
/** * This is collect the bytes from current position to the ending tag. * This scans for the tags and do the pattern match byte by byte * this must be used along with //from w ww. j a va 2s .c om * XMLLoaderBufferedPositionedInputStream#skipToTag * * @param tagName the end tag to search for * * @param limit the end pointer for the block for this mapper * * @return the byte array containing the documents until the end of tag * * @see loader.XMLLoaderBufferedPositionedInputStream.collectUntilEndTag * */ private byte[] collectUntilEndTag(String tagName, long limit) { //@todo use the charset and get the charset encoding from the xml encoding. byte[] tmp = tagName.getBytes(); ByteArrayOutputStream collectBuf = new ByteArrayOutputStream(1024); // Levels of elements we went inside matched node int depth = 0; //Since skipToTag was called before this function, we know that we are //currently inside the matched tag. Assuming the XML file is well //structured, we read till we encounter the first close tag. Since //the matched element might contain nested element, we keep track of the //current depth and terminate only when we encounter a closing tag at //level zero // A flag to indicate the parsing is currently inside a (start/end) tag boolean insideTag = false; // A flag to indicate that the current tag is a closing (end) tag boolean closingTag = false; // Last byte read int last_b = -1; while (true) { int b = -1; try { b = this.read(); ++bytesRead; // Add one to the bytes read if (b == -1) { collectBuf.reset(); this.setReadable(false); break; } collectBuf.write((byte) (b)); // Check if the start tag has matched except for the last char if (b == '<') { insideTag = true; closingTag = false; } else if (b == '>') { // Detect the pattern /> if (last_b == '/') closingTag = true; insideTag = false; if (closingTag) { if (depth == 0) break; depth--; } } else if (b == '/' && last_b == '<') { // Detected the pattern </ closingTag = true; } else if (insideTag && last_b == '<') { // First character after '<' which is not a '/' depth++; } } catch (IOException e) { this.setReadable(false); return null; } last_b = b; } return collectBuf.toByteArray(); }
From source file:org.dbgl.gui.AddGameWizardDialog.java
protected void doReloadTemplate(int templateIndex) { try {/* w w w .j a va 2 s .c o m*/ ByteArrayOutputStream bos = new ByteArrayOutputStream(); PrintStream ps = new PrintStream(bos); Conf templateConf = new Conf(templatesList.get(templateIndex), profile.getConf().getDbversion(), ps); profile.getConf().reloadTemplate(templateConf); selectSettingsByConfiguration(profile.getConf()); profile.setNativeCommandsList( dbase.readNativeCommandsList(-1, templatesList.get(templateIndex).getId())); if (bos.size() > 0) { GeneralPurposeDialogs.warningMessage(getParent(), bos.toString()); bos.reset(); } } catch (IOException | SQLException e) { GeneralPurposeDialogs.warningMessage(getParent(), e); } }
From source file:com.geoxp.oss.client.OSSClient.java
public static void putSecret(String ossURL, String secretname, byte[] secret, String sshKeyFingerprint) throws OSSException { SSHAgentClient agent = null;//from www . j a v a2 s. co m HttpClient httpclient = null; try { agent = new SSHAgentClient(); List<SSHKey> sshkeys = agent.requestIdentities(); // // If no SSH Key fingerprint was provided, try all SSH keys available in the agent // List<String> fingerprints = new ArrayList<String>(); if (null == sshKeyFingerprint) { for (SSHKey key : sshkeys) { fingerprints.add(key.fingerprint); } } else { fingerprints.add(sshKeyFingerprint.toLowerCase().replaceAll("[^0-9a-f]", "")); } int idx = 0; for (String fingerprint : fingerprints) { idx++; // // Ask the SSH agent for the SSH key blob // byte[] keyblob = null; for (SSHKey key : sshkeys) { if (key.fingerprint.equals(fingerprint)) { keyblob = key.blob; break; } } // // Throw an exception if this condition is encountered as it can only happen if // there was a provided fingerprint which is not in the agent. // if (null == keyblob) { throw new OSSException("SSH Key " + sshKeyFingerprint + " was not found by your SSH agent."); } // // Retrieve OSS RSA key // RSAPublicKey pubkey = getOSSRSA(ossURL); // // Build the token // // <TS> <<WRAPPED_SECRET><ENCRYPTED_WRAPPING_KEY>> <SSH Signing Key Blob> <SSH Signature Blob> // ByteArrayOutputStream token = new ByteArrayOutputStream(); byte[] tsdata = nowBytes(); token.write(CryptoHelper.encodeNetworkString(tsdata)); ByteArrayOutputStream subtoken = new ByteArrayOutputStream(); subtoken.write(CryptoHelper.encodeNetworkString(secretname.getBytes("UTF-8"))); subtoken.write(CryptoHelper.encodeNetworkString(secret)); token.write(CryptoHelper.encodeNetworkString(subtoken.toByteArray())); token.write(CryptoHelper.encodeNetworkString(keyblob)); byte[] sigblob = agent.sign(keyblob, token.toByteArray()); token.write(CryptoHelper.encodeNetworkString(sigblob)); // // Encrypt the token with a random AES256 key // byte[] aeskey = new byte[32]; CryptoHelper.getSecureRandom().nextBytes(aeskey); byte[] wrappedtoken = CryptoHelper.wrapAES(aeskey, token.toByteArray()); // // Encrypt the random key with OSS' RSA key // byte[] sealedaeskey = CryptoHelper.encryptRSA(pubkey, aeskey); // // Create the token // token.reset(); token.write(CryptoHelper.encodeNetworkString(wrappedtoken)); token.write(CryptoHelper.encodeNetworkString(sealedaeskey)); // // Base64 encode the encryptedtoken // String b64token = new String(Base64.encode(token.toByteArray()), "UTF-8"); // // Send request to OSS // URIBuilder builder = new URIBuilder(ossURL + GuiceServletModule.SERVLET_PATH_PUT_SECRET); builder.addParameter("token", b64token); URI uri = builder.build(); String qs = uri.getRawQuery(); HttpPost post = new HttpPost( uri.getScheme() + "://" + uri.getHost() + ":" + uri.getPort() + uri.getPath()); post.setHeader("Content-Type", "application/x-www-form-urlencoded"); post.setEntity(new StringEntity(qs)); httpclient = newHttpClient(); HttpResponse response = httpclient.execute(post); HttpEntity resEntity = response.getEntity(); String content = EntityUtils.toString(resEntity, "UTF-8"); post.reset(); if (HttpServletResponse.SC_OK != response.getStatusLine().getStatusCode()) { // Only throw an exception if this is the last SSH key we could try if (idx == fingerprints.size()) { throw new OSSException("None of the provided keys (" + idx + ") could be used to store the secret. Latest error message was: " + response.getStatusLine().getReasonPhrase()); } else { continue; } } return; } } catch (OSSException osse) { throw osse; } catch (Exception e) { throw new OSSException(e); } finally { if (null != httpclient) { httpclient.getConnectionManager().shutdown(); } if (null != agent) { agent.close(); } } }
From source file:ipc.Server.java
/** * Setup response for the IPC Call.//from w w w . j a v a 2 s .c o m * * @param response buffer to serialize the response into * @param call {@link Call} to which we are setting up the response * @param status {@link Status} of the IPC call * @param rv return value for the IPC Call, if the call was successful * @param errorClass error class, if the the call failed * @param error error message, if the call failed * @throws IOException */ private void setupResponse(ByteArrayOutputStream response, Call call, Status status, Writable rv, String errorClass, String error) throws IOException { response.reset(); DataOutputStream out = new DataOutputStream(response); out.writeInt(call.id); // write call id out.writeInt(status.state); // write status if (status == Status.SUCCESS) { rv.write(out); } else { WritableUtils.writeString(out, errorClass); WritableUtils.writeString(out, error); } wrapWithSasl(response, call); call.setResponse(ByteBuffer.wrap(response.toByteArray())); }
From source file:org.apache.jmeter.protocol.http.proxy.HttpRequestHdr.java
/** * Parses a http header from a stream.// w w w. ja v a 2 s . co m * * @param in * the stream to parse. * @return array of bytes from client. * @throws IOException when reading the input stream fails */ public byte[] parse(InputStream in) throws IOException { boolean inHeaders = true; int readLength = 0; int dataLength = 0; boolean firstLine = true; ByteArrayOutputStream clientRequest = new ByteArrayOutputStream(); ByteArrayOutputStream line = new ByteArrayOutputStream(); int x; while ((inHeaders || readLength < dataLength) && ((x = in.read()) != -1)) { line.write(x); clientRequest.write(x); if (firstLine && !CharUtils.isAscii((char) x)) {// includes \n throw new IllegalArgumentException("Only ASCII supported in headers (perhaps SSL was used?)"); } if (inHeaders && (byte) x == (byte) '\n') { // $NON-NLS-1$ if (line.size() < 3) { inHeaders = false; firstLine = false; // cannot be first line either } final String reqLine = line.toString(); if (firstLine) { parseFirstLine(reqLine); firstLine = false; } else { // parse other header lines, looking for Content-Length final int contentLen = parseLine(reqLine); if (contentLen > 0) { dataLength = contentLen; // Save the last valid content length one } } if (log.isDebugEnabled()) { log.debug("Client Request Line: '" + reqLine.replaceFirst("\r\n$", "<CRLF>") + "'"); } line.reset(); } else if (!inHeaders) { readLength++; } } // Keep the raw post data rawPostData = line.toByteArray(); if (log.isDebugEnabled()) { log.debug("rawPostData in default JRE encoding: " + new String(rawPostData)); // TODO - charset? log.debug("Request: '" + clientRequest.toString().replaceAll("\r\n", "<CRLF>") + "'"); } return clientRequest.toByteArray(); }
From source file:org.dataconservancy.packaging.tool.impl.PackageStateSerializationIT.java
/** * As configured in production, the {@link AnnotationDrivenPackageStateSerializer} should be encoding characters * using UTF-8, no matter what the platform default is. * * @throws Exception//from ww w . j av a 2 s .co m */ @Test public void testUtf8Encoding() throws Exception { PackageState deserializedState = null; ByteArrayOutputStream sink = new ByteArrayOutputStream(); String unicodeString = "S\u00EDSe\u00F1or!"; byte[] unicodeBytes = unicodeString.getBytes(Charset.forName("UTF-8")); // We will not configure this to archive, so that the serialized stream will // not be placed in a zip entry; this makes searching through the sink for a // byte sequence more robust. // TODO this test method should really be somewhere else; or the PackageStateSerializer should expose a setArchive(boolean) method. if (!(underTest instanceof AnnotationDrivenPackageStateSerializer)) { fail("Expected an instance of AnnotationDrivenPackageStateSerializer"); } ((AnnotationDrivenPackageStateSerializer) underTest).setArchive(false); // Package name with a unicode string state.setPackageName(unicodeString); underTest.serialize(state, StreamId.PACKAGE_NAME, sink); assertTrue(contains(unicodeBytes, sink)); deserializedState = new PackageState(); underTest.deserialize(deserializedState, StreamId.PACKAGE_NAME, new ByteArrayInputStream(sink.toByteArray())); assertEquals(unicodeString, deserializedState.getPackageName()); sink.reset(); deserializedState = null; // Package metadata with a unicode string LinkedHashMap<String, List<String>> packageMetadata = new LinkedHashMap<>(); packageMetadata.put("foo", Collections.singletonList(unicodeString)); state.setPackageMetadataList(packageMetadata); underTest.serialize(state, StreamId.PACKAGE_METADATA, sink); assertTrue(contains(unicodeBytes, sink)); deserializedState = new PackageState(); underTest.deserialize(deserializedState, StreamId.PACKAGE_METADATA, new ByteArrayInputStream(sink.toByteArray())); assertEquals(unicodeString, deserializedState.getPackageMetadataList().get("foo").get(0)); sink.reset(); deserializedState = null; // A String user-defined property value with a unicode string Map<URI, List<Property>> userProps = new HashMap<>(); PropertyType type = new PropertyType(); type.setPropertyValueType(PropertyValueType.STRING); Property property = new Property(type); property.setStringValue(unicodeString); userProps.put(URI.create("http://a/uri"), Collections.singletonList(property)); state.setUserSpecifiedProperties(userProps); underTest.serialize(state, StreamId.USER_SPECIFIED_PROPERTIES, sink); assertTrue(contains(unicodeBytes, sink)); deserializedState = new PackageState(); underTest.deserialize(deserializedState, StreamId.USER_SPECIFIED_PROPERTIES, new ByteArrayInputStream(sink.toByteArray())); assertEquals(unicodeString, deserializedState.getUserSpecifiedProperties().get(URI.create("http://a/uri")) .get(0).getStringValue()); sink.reset(); deserializedState = null; // A IPM node with a unicode string Model ipm = ModelFactory.createDefaultModel(); Statement s = ipm.createStatement(ipm.createResource("foo:s"), ipm.createProperty("foo:p"), ipm.createResource(unicodeString)); ipm.add(s); state.setPackageTree(ipm); underTest.serialize(state, StreamId.PACKAGE_TREE, sink); assertTrue(contains(unicodeBytes, sink)); deserializedState = new PackageState(); underTest.deserialize(deserializedState, StreamId.PACKAGE_TREE, new ByteArrayInputStream(sink.toByteArray())); assertTrue(deserializedState.getPackageTree().listObjectsOfProperty(ResourceFactory.createProperty("foo:p")) .next().toString().endsWith(unicodeString)); sink.reset(); deserializedState = null; // A domain object with a unicode string Model objects = ModelFactory.createDefaultModel(); s = objects.createStatement(objects.createResource("bar:s"), objects.createProperty("bar:p"), objects.createResource(unicodeString)); objects.add(s); state.setDomainObjectRDF(objects); underTest.serialize(state, StreamId.DOMAIN_OBJECTS, sink); assertTrue(contains(unicodeBytes, sink)); deserializedState = new PackageState(); underTest.deserialize(deserializedState, StreamId.DOMAIN_OBJECTS, new ByteArrayInputStream(sink.toByteArray())); assertTrue(deserializedState.getDomainObjectRDF() .listObjectsOfProperty(ResourceFactory.createProperty("bar:p")).next().toString() .endsWith(unicodeString)); sink.reset(); deserializedState = null; }
From source file:jproxy.HttpRequestHdr.java
/** * Parses a http header from a stream.//from w w w.j a v a 2 s . co m * * @param in * the stream to parse. * @return array of bytes from client. * @throws IOException when reading the input stream fails */ public byte[] parse(InputStream in) throws IOException { boolean inHeaders = true; int readLength = 0; int dataLength = 0; boolean firstLine = true; ByteArrayOutputStream clientRequest = new ByteArrayOutputStream(); ByteArrayOutputStream line = new ByteArrayOutputStream(); int x; while ((inHeaders || readLength < dataLength) && ((x = in.read()) != -1)) { line.write(x); clientRequest.write(x); if (firstLine && !CharUtils.isAscii((char) x)) {// includes \n throw new IllegalArgumentException("Only ASCII supported in headers (perhaps SSL was used?)"); } if (inHeaders && (byte) x == (byte) '\n') { // $NON-NLS-1$ if (line.size() < 3) { inHeaders = false; firstLine = false; // cannot be first line either } final String reqLine = line.toString(); if (firstLine) { parseFirstLine(reqLine); firstLine = false; } else { // parse other header lines, looking for Content-Length final int contentLen = parseLine(reqLine); if (contentLen > 0) { dataLength = contentLen; // Save the last valid content length one } } if (log.isDebugEnabled()) { log.debug("Client Request Line: '" + reqLine.replaceFirst("\r\n$", "<CRLF>") + "'"); } line.reset(); } else if (!inHeaders) { readLength++; } } // Keep the raw post data rawPostData = line.toByteArray(); if (log.isDebugEnabled()) { log.debug("rawPostData in default JRE encoding: " + new String(rawPostData)); // TODO - charset? log.debug("Request: '" + clientRequest.toString().replaceAll("\r\n", "<CRLF>") + "'"); } rawRequestData = clientRequest.toByteArray(); return clientRequest.toByteArray(); }
From source file:com.streamsets.pipeline.stage.origin.kafka.TestKafkaSource.java
@Test public void testProduceAvroRecordsWithSchema() throws Exception { //create Producer and send messages Schema schema = new Schema.Parser().parse(AVRO_SCHEMA); GenericRecord boss = new GenericData.Record(schema); boss.put("name", "boss"); boss.put("age", 60); boss.put("emails", ImmutableList.of("boss@company.com", "boss2@company.com")); boss.put("boss", null); GenericRecord e3 = new GenericData.Record(schema); e3.put("name", "c"); e3.put("age", 50); e3.put("emails", ImmutableList.of("c@company.com", "c2@company.com")); e3.put("boss", boss); GenericRecord e2 = new GenericData.Record(schema); e2.put("name", "b"); e2.put("age", 40); e2.put("emails", ImmutableList.of("b@company.com", "b2@company.com")); e2.put("boss", boss); GenericRecord e1 = new GenericData.Record(schema); e1.put("name", "a"); e1.put("age", 30); e1.put("emails", ImmutableList.of("a@company.com", "a2@company.com")); e1.put("boss", boss); Properties props = new Properties(); props.put("metadata.broker.list", sdcKafkaTestUtil.getMetadataBrokerURI()); props.put("serializer.class", "kafka.serializer.DefaultEncoder"); props.put("key.serializer.class", "kafka.serializer.StringEncoder"); props.put("request.required.acks", "1"); ProducerConfig config = new ProducerConfig(props); Producer<String, byte[]> producer = new Producer<>(config); ByteArrayOutputStream baos = new ByteArrayOutputStream(); DatumWriter<GenericRecord> datumWriter = new GenericDatumWriter<>(schema); DataFileWriter<GenericRecord> dataFileWriter = new DataFileWriter<>(datumWriter); dataFileWriter.create(schema, baos); dataFileWriter.append(e1);// w w w . ja v a 2 s . c o m dataFileWriter.flush(); dataFileWriter.close(); producer.send(new KeyedMessage<>(TOPIC12, "0", baos.toByteArray())); baos.reset(); dataFileWriter.create(schema, baos); dataFileWriter.append(e2); dataFileWriter.flush(); dataFileWriter.close(); producer.send(new KeyedMessage<>(TOPIC12, "0", baos.toByteArray())); baos.reset(); dataFileWriter.create(schema, baos); dataFileWriter.append(e3); dataFileWriter.flush(); dataFileWriter.close(); producer.send(new KeyedMessage<>(TOPIC12, "0", baos.toByteArray())); Map<String, String> kafkaConsumerConfigs = new HashMap<>(); sdcKafkaTestUtil.setAutoOffsetReset(kafkaConsumerConfigs); KafkaConfigBean conf = new KafkaConfigBean(); conf.metadataBrokerList = sdcKafkaTestUtil.getMetadataBrokerURI(); conf.topic = TOPIC12; conf.consumerGroup = CONSUMER_GROUP; conf.zookeeperConnect = zkConnect; conf.maxBatchSize = 100; conf.maxWaitTime = 10000; conf.kafkaConsumerConfigs = kafkaConsumerConfigs; conf.produceSingleRecordPerMessage = false; conf.dataFormat = DataFormat.AVRO; conf.dataFormatConfig.charset = "UTF-8"; conf.dataFormatConfig.removeCtrlChars = false; conf.dataFormatConfig.schemaInMessage = true; conf.dataFormatConfig.avroSchema = AVRO_SCHEMA; SourceRunner sourceRunner = new SourceRunner.Builder(StandaloneKafkaSource.class, createSource(conf)) .addOutputLane("lane").build(); sourceRunner.runInit(); StageRunner.Output output = sourceRunner.runProduce(null, 10); String newOffset = output.getNewOffset(); Assert.assertNull(newOffset); List<Record> records = output.getRecords().get("lane"); Assert.assertEquals(3, records.size()); Record e3Record = records.get(2); Assert.assertTrue(e3Record.has("/name")); Assert.assertEquals("c", e3Record.get("/name").getValueAsString()); Assert.assertTrue(e3Record.has("/age")); Assert.assertEquals(50, e3Record.get("/age").getValueAsInteger()); Assert.assertTrue(e3Record.has("/emails")); Assert.assertTrue(e3Record.get("/emails").getValueAsList() instanceof List); List<Field> emails = e3Record.get("/emails").getValueAsList(); Assert.assertEquals(2, emails.size()); Assert.assertEquals("c@company.com", emails.get(0).getValueAsString()); Assert.assertEquals("c2@company.com", emails.get(1).getValueAsString()); Assert.assertTrue(e3Record.has("/boss")); Assert.assertTrue(e3Record.get("/boss").getValueAsMap() instanceof Map); Assert.assertTrue(e3Record.has("/boss/name")); Assert.assertEquals("boss", e3Record.get("/boss/name").getValueAsString()); Assert.assertTrue(e3Record.has("/boss/age")); Assert.assertEquals(60, e3Record.get("/boss/age").getValueAsInteger()); Assert.assertTrue(e3Record.has("/boss/emails")); Assert.assertTrue(e3Record.get("/boss/emails").getValueAsList() instanceof List); emails = e3Record.get("/boss/emails").getValueAsList(); Assert.assertEquals(2, emails.size()); Assert.assertEquals("boss@company.com", emails.get(0).getValueAsString()); Assert.assertEquals("boss2@company.com", emails.get(1).getValueAsString()); Record e2Record = records.get(1); Assert.assertTrue(e2Record.has("/name")); Assert.assertEquals("b", e2Record.get("/name").getValueAsString()); Assert.assertTrue(e2Record.has("/age")); Assert.assertEquals(40, e2Record.get("/age").getValueAsInteger()); Assert.assertTrue(e2Record.has("/emails")); Assert.assertTrue(e2Record.get("/emails").getValueAsList() instanceof List); emails = e2Record.get("/emails").getValueAsList(); Assert.assertEquals(2, emails.size()); Assert.assertEquals("b@company.com", emails.get(0).getValueAsString()); Assert.assertEquals("b2@company.com", emails.get(1).getValueAsString()); Assert.assertTrue(e2Record.has("/boss")); Assert.assertTrue(e2Record.get("/boss").getValueAsMap() instanceof Map); Assert.assertTrue(e2Record.has("/boss/name")); Assert.assertEquals("boss", e2Record.get("/boss/name").getValueAsString()); Assert.assertTrue(e2Record.has("/boss/age")); Assert.assertEquals(60, e2Record.get("/boss/age").getValueAsInteger()); Assert.assertTrue(e2Record.has("/boss/emails")); Assert.assertTrue(e2Record.get("/boss/emails").getValueAsList() instanceof List); emails = e2Record.get("/boss/emails").getValueAsList(); Assert.assertEquals(2, emails.size()); Assert.assertEquals("boss@company.com", emails.get(0).getValueAsString()); Assert.assertEquals("boss2@company.com", emails.get(1).getValueAsString()); Record e1Record = records.get(0); Assert.assertTrue(e1Record.has("/name")); Assert.assertEquals("a", e1Record.get("/name").getValueAsString()); Assert.assertTrue(e1Record.has("/age")); Assert.assertEquals(30, e1Record.get("/age").getValueAsInteger()); Assert.assertTrue(e1Record.has("/emails")); Assert.assertTrue(e1Record.get("/emails").getValueAsList() instanceof List); emails = e1Record.get("/emails").getValueAsList(); Assert.assertEquals(2, emails.size()); Assert.assertEquals("a@company.com", emails.get(0).getValueAsString()); Assert.assertEquals("a2@company.com", emails.get(1).getValueAsString()); Assert.assertTrue(e1Record.has("/boss")); Assert.assertTrue(e1Record.get("/boss").getValueAsMap() instanceof Map); Assert.assertTrue(e1Record.has("/boss/name")); Assert.assertEquals("boss", e1Record.get("/boss/name").getValueAsString()); Assert.assertTrue(e1Record.has("/boss/age")); Assert.assertEquals(60, e1Record.get("/boss/age").getValueAsInteger()); Assert.assertTrue(e1Record.has("/boss/emails")); Assert.assertTrue(e1Record.get("/boss/emails").getValueAsList() instanceof List); emails = e1Record.get("/boss/emails").getValueAsList(); Assert.assertEquals(2, emails.size()); Assert.assertEquals("boss@company.com", emails.get(0).getValueAsString()); Assert.assertEquals("boss2@company.com", emails.get(1).getValueAsString()); }