List of usage examples for org.apache.thrift TDeserializer TDeserializer
public TDeserializer(TProtocolFactory protocolFactory)
From source file:com.knewton.mapreduce.util.SerializationUtils.java
License:Apache License
public static TDeserializer getDeserializerFromConf(Configuration conf) { Class<? extends TProtocolFactory> protocolFactoryClass = conf.getClass( PropertyConstants.SERIALIZATION_FACTORY_PARAMETER.txt, SERIALIZATION_FACTORY_PARAMETER_DEFAULT, TProtocolFactory.class); TProtocolFactory protocolFactory = ReflectionUtils.newInstance(protocolFactoryClass, conf); return new TDeserializer(protocolFactory); }
From source file:com.linkedin.pinot.serde.SerDe.java
License:Apache License
public SerDe(TProtocolFactory factory) { _serializer = new TSerializer(factory); _deserializer = new TDeserializer(factory); }
From source file:com.navercorp.pinpoint.web.mapper.AgentStatMapper.java
License:Apache License
private List<AgentStat> readAgentStatThriftDto(byte[] tAgentStatByteArray) throws TException { // CompactProtocol used TDeserializer deserializer = new TDeserializer(factory); TAgentStat tAgentStat = new TAgentStat(); deserializer.deserialize(tAgentStat, tAgentStatByteArray); TJvmGc gc = tAgentStat.getGc();/*from www. j av a 2s . co m*/ if (gc == null) { return Collections.emptyList(); } AgentStatMemoryGcBo.Builder memoryGcBoBuilder = new AgentStatMemoryGcBo.Builder(tAgentStat.getAgentId(), tAgentStat.getStartTimestamp(), tAgentStat.getTimestamp()); memoryGcBoBuilder.gcType(gc.getType().name()); memoryGcBoBuilder.jvmMemoryHeapUsed(gc.getJvmMemoryHeapUsed()); memoryGcBoBuilder.jvmMemoryHeapMax(gc.getJvmMemoryHeapMax()); memoryGcBoBuilder.jvmMemoryNonHeapUsed(gc.getJvmMemoryNonHeapUsed()); memoryGcBoBuilder.jvmMemoryNonHeapMax(gc.getJvmMemoryNonHeapMax()); memoryGcBoBuilder.jvmGcOldCount(gc.getJvmGcOldCount()); memoryGcBoBuilder.jvmGcOldTime(gc.getJvmGcOldTime()); AgentStat agentStat = new AgentStat(); agentStat.setMemoryGc(memoryGcBoBuilder.build()); List<AgentStat> result = new ArrayList<AgentStat>(1); result.add(agentStat); return result; }
From source file:com.pinterest.secor.io.impl.ThriftParquetFileReaderWriterFactoryTest.java
License:Apache License
@Test public void testThriftParquetReadWriteRoundTrip() throws Exception { Map<String, String> classPerTopic = new HashMap<String, String>(); classPerTopic.put("test-pb-topic", UnitTestMessage.class.getName()); Mockito.when(config.getThriftMessageClassPerTopic()).thenReturn(classPerTopic); Mockito.when(config.getFileReaderWriterFactory()) .thenReturn(ThriftParquetFileReaderWriterFactory.class.getName()); Mockito.when(config.getThriftProtocolClass()).thenReturn(TCompactProtocol.class.getName()); LogFilePath tempLogFilePath = new LogFilePath(Files.createTempDir().toString(), "test-pb-topic", new String[] { "part-1" }, 0, 1, 23232, ".log"); FileWriter fileWriter = ReflectionUtil.createFileWriter(config.getFileReaderWriterFactory(), tempLogFilePath, null, config); UnitTestMessage msg1 = new UnitTestMessage().setRequiredField("abc").setTimestamp(1467176315L); UnitTestMessage msg2 = new UnitTestMessage().setRequiredField("XYZ").setTimestamp(1467176344L); TSerializer serializer = new TSerializer(new TCompactProtocol.Factory()); KeyValue kv1 = new KeyValue(23232, serializer.serialize(msg1)); KeyValue kv2 = new KeyValue(23233, serializer.serialize(msg2)); fileWriter.write(kv1);/* w w w . j a va 2s . c o m*/ fileWriter.write(kv2); fileWriter.close(); FileReader fileReader = ReflectionUtil.createFileReader(config.getFileReaderWriterFactory(), tempLogFilePath, null, config); TDeserializer deserializer = new TDeserializer(new TCompactProtocol.Factory()); KeyValue kvout = fileReader.next(); assertEquals(kv1.getOffset(), kvout.getOffset()); assertArrayEquals(kv1.getValue(), kvout.getValue()); UnitTestMessage actual = new UnitTestMessage(); deserializer.deserialize(actual, kvout.getValue()); assertEquals(msg1.getRequiredField(), actual.getRequiredField()); kvout = fileReader.next(); assertEquals(kv2.getOffset(), kvout.getOffset()); assertArrayEquals(kv2.getValue(), kvout.getValue()); actual = new UnitTestMessage(); deserializer.deserialize(actual, kvout.getValue()); assertEquals(msg2.getRequiredField(), actual.getRequiredField()); }
From source file:com.pinterest.secor.parser.ThriftMessageParser.java
License:Apache License
public ThriftMessageParser(SecorConfig config) throws InstantiationException, IllegalAccessException, ClassNotFoundException { super(config); TProtocolFactory protocolFactory = null; String protocolName = mConfig.getThriftProtocolClass(); if (StringUtils.isNotEmpty(protocolName)) { String factoryClassName = protocolName.concat("$Factory"); protocolFactory = ((Class<? extends TProtocolFactory>) Class.forName(factoryClassName)).newInstance(); } else//from w w w .j av a 2 s. c o m protocolFactory = new TBinaryProtocol.Factory(); mDeserializer = new TDeserializer(protocolFactory); mThriftPath = new ThriftPath(mConfig.getMessageTimestampName(), (short) mConfig.getMessageTimestampId()); mTimestampType = mConfig.getMessageTimestampType(); }
From source file:com.twitter.aurora.codec.ThriftBinaryCodec.java
License:Apache License
/** * Decodes a binary-encoded byte array into a target type. * * @param clazz Class to instantiate and deserialize to. * @param buffer Buffer to decode./*from w w w . ja v a 2s.com*/ * @param <T> Target type. * @return A populated message. * @throws CodingException If the message could not be decoded. */ public static <T extends TBase<T, ?>> T decodeNonNull(Class<T> clazz, byte[] buffer) throws CodingException { Preconditions.checkNotNull(clazz); Preconditions.checkNotNull(buffer); try { T t = clazz.newInstance(); new TDeserializer(PROTOCOL_FACTORY).deserialize(t, buffer); return t; } catch (IllegalAccessException e) { throw new CodingException("Failed to access constructor for target type.", e); } catch (InstantiationException e) { throw new CodingException("Failed to instantiate target type.", e); } catch (TException e) { throw new CodingException("Failed to deserialize thrift object.", e); } }
From source file:com.uber.tchannel.messages.ThriftSerializer.java
License:Open Source License
@Override public <T> T decodeBody(ByteBuf arg3, Class<T> bodyType) { try {/*from ww w .java 2 s. c o m*/ // Create a new instance of type 'T' T base = bodyType.newInstance(); // Get byte[] from ByteBuf byte[] payloadBytes = new byte[arg3.readableBytes()]; arg3.readBytes(payloadBytes); // Actually deserialize the payload TDeserializer deserializer = new TDeserializer(new TBinaryProtocol.Factory()); deserializer.deserialize((TBase) base, payloadBytes); return base; } catch (InstantiationException | IllegalAccessException | TException e) { e.printStackTrace(); } return null; }
From source file:dataServer.StorageRESTClientManager.java
public List<RecommendationEvent> /*List<FeedbackEvent>*/ readFromStorage(String startDate, String endDate) { // GetMethod get = new GetMethod("http://" + storageLocation + ":" + storagePort + storageContext + "/query/feedback/default/?startDate=" + startDate + "&endDate=" + endDate); // Default HTTP response and common properties for responses HttpResponse response = null;//from ww w. jav a2 s .c o m ResponseHandler<String> handler = null; int status = 0; String body = null; // Default query for simple events requestUrl = new StringBuilder("http://" + storageLocation + ":" + storagePortRegistryC + storageRegistryContext + "/query/recommendation/default"); params = new LinkedList<>(); params.add(new BasicNameValuePair("startTime", startDate)); params.add(new BasicNameValuePair("endTime", endDate)); queryString = URLEncodedUtils.format(params, "utf-8"); requestUrl.append("?"); requestUrl.append(queryString); try { HttpGet query = new HttpGet(requestUrl.toString()); query.setHeader("Content-type", "application/json"); response = client.execute(query); // Check status code status = response.getStatusLine().getStatusCode(); if (status != 200) { throw new RuntimeException("Failed! HTTP error code: " + status); } // Get body handler = new BasicResponseHandler(); body = handler.handleResponse(response); //System.out.println("RECOMMENDATION.DEFAULT: " + body); // The result is an array of feedback events serialized as JSON using Apache Thrift. // The feedback events can be deserialized into Java objects using Apache Thrift. ObjectMapper mapper = new ObjectMapper(); JsonNode nodeArray = mapper.readTree(body); List<RecommendationEvent> events = new ArrayList<>(); for (JsonNode node : nodeArray) { byte[] bytes = node.toString().getBytes(); TDeserializer deserializer = new TDeserializer(new TJSONProtocol.Factory()); RecommendationEvent event = new RecommendationEvent(); deserializer.deserialize(event, bytes); System.out.println(event.toString()); events.add(event); } return events; } catch (Exception e) { System.out.println(e.getClass().getName() + ": " + e.getMessage()); return null; } //// GetMethod get = new GetMethod("http://" + storageLocation + ":" + storagePort + storageContext + "/query/feedback/default/?startDate=" + startDate + "&endDate=" + endDate); // // Default HTTP response and common properties for responses // HttpResponse response = null; // ResponseHandler<String> handler = null; // int status = 0; // String body = null; // // Default query for simple events // requestUrl = new StringBuilder("http://" + storageLocation + ":" + storagePort + storageContext + "/query/feedback/default"); // // params = new LinkedList<>(); // params.add(new BasicNameValuePair("startTime", startDate)); // params.add(new BasicNameValuePair("endTime", endDate)); // // queryString = URLEncodedUtils.format(params, "utf-8"); // requestUrl.append("?"); // requestUrl.append(queryString); // // try { // HttpGet query = new HttpGet(requestUrl.toString()); // query.setHeader("Content-type", "application/json"); // response = client.execute(query); // // // Check status code // status = response.getStatusLine().getStatusCode(); // if (status != 200) { // throw new RuntimeException("Failed! HTTP error code: " + status); // } // // // Get body // handler = new BasicResponseHandler(); // body = handler.handleResponse(response); // // System.out.println("FEEDBACK.DEFAULT: " + body); // // The result is an array of feedback events serialized as JSON using Apache Thrift. // // The feedback events can be deserialized into Java objects using Apache Thrift. // ObjectMapper mapper = new ObjectMapper(); // JsonNode nodeArray = mapper.readTree(body); // List<FeedbackEvent> events = new ArrayList<>(); // for (JsonNode node : nodeArray) { // byte[] bytes = node.toString().getBytes(); // TDeserializer deserializer = new TDeserializer(new TJSONProtocol.Factory()); // FeedbackEvent event = new FeedbackEvent(); // deserializer.deserialize(event, bytes); // System.out.println(event.toString()); // events.add(event); // } // return events; // } catch (Exception e) { // System.out.println(e.getClass().getName() + ": " + e.getMessage()); // return null; // } }
From source file:deployer.AccumuloEzDeployerStoreTest.java
License:Apache License
private void assertRowAddedInAccumulo(ArtifactType type, DeploymentStatus status) throws Exception { TDeserializer deSerializer = new TDeserializer(new TCompactProtocol.Factory()); Scanner scanner = connector.createScanner("deployments", new Authorizations("U", "FOUO")); Set<String> foundIndexes = Sets.newHashSet(); Set<String> expectedIndexes = Sets.newHashSet( AccumuloEzDeployerStore.ARTIFACT_INDEX_APPLICATION_CQ.toString(), AccumuloEzDeployerStore.ARTIFACT_INDEX_USER_CQ.toString(), AccumuloEzDeployerStore.ARTIFACT_INDEX_SECURITY_CQ.toString(), AccumuloEzDeployerStore.ARTIFACT_INDEX_STATUS_CQ.toString()); for (Map.Entry<Key, Value> i : scanner) { String cf = i.getKey().getColumnFamily().toString(); if (cf.equals(AccumuloEzDeployerStore.ARTIFACT_CF.toString())) { assertEquals(getFqAppId(TestUtils.APP_NAME, TestUtils.SERVICE_NAME), i.getKey().getRow().toString()); assertEquals("application", i.getKey().getColumnFamily().toString()); String cq = i.getKey().getColumnQualifier().toString(); Set<String> valid_cq = Sets.newHashSet("data", "metadata"); if (cq.equals("data")) { DeploymentArtifact deploymentArtifact = new DeploymentArtifact(); deSerializer.deserialize(deploymentArtifact, i.getValue().get()); TestUtils.assertDeploymentArtifact(deploymentArtifact, type); } else if (cq.equals("metadata")) { DeploymentMetadata artifact = new DeploymentMetadata(); deSerializer.deserialize(artifact, i.getValue().get()); assertDeploymentMetadata(artifact, type); } else { printTable();/*w w w .j a v a 2s . c o m*/ assertTrue("CF must be on of these values: " + valid_cq, valid_cq.contains(cq)); } } else if (cf.startsWith(AccumuloEzDeployerStore.ARTIFACT_INDEX_CF.toString())) { String cq = i.getKey().getColumnQualifier().toString(); if (cf.endsWith(AccumuloEzDeployerStore.ARTIFACT_INDEX_APPLICATION_CQ.toString())) { assertEquals(TestUtils.APP_NAME, i.getKey().getRow().toString()); assertEquals(getFqAppId(TestUtils.APP_NAME, TestUtils.SERVICE_NAME), i.getValue().toString()); foundIndexes.add(AccumuloEzDeployerStore.ARTIFACT_INDEX_APPLICATION_CQ.toString()); } else if (cf.endsWith(AccumuloEzDeployerStore.ARTIFACT_INDEX_USER_CQ.toString())) { assertEquals(TestUtils.USER_NAME, i.getKey().getRow().toString()); assertEquals(getFqAppId(TestUtils.APP_NAME, TestUtils.SERVICE_NAME), i.getValue().toString()); foundIndexes.add(AccumuloEzDeployerStore.ARTIFACT_INDEX_USER_CQ.toString()); } else if (cf.endsWith(AccumuloEzDeployerStore.ARTIFACT_INDEX_SECURITY_CQ.toString())) { assertEquals(TestUtils.SECURITY_ID, i.getKey().getRow().toString()); assertEquals(getFqAppId(TestUtils.APP_NAME, TestUtils.SERVICE_NAME), i.getValue().toString()); foundIndexes.add(AccumuloEzDeployerStore.ARTIFACT_INDEX_SECURITY_CQ.toString()); } else if (cf.endsWith(AccumuloEzDeployerStore.ARTIFACT_INDEX_STATUS_CQ.toString())) { assertEquals(status.toString(), i.getKey().getRow().toString()); assertEquals(getFqAppId(TestUtils.APP_NAME, TestUtils.SERVICE_NAME), i.getValue().toString()); foundIndexes.add(AccumuloEzDeployerStore.ARTIFACT_INDEX_STATUS_CQ.toString()); } else { printTable(); fail("Unknown CF/CQ written to accumulo table: " + cf + "/" + cq); } } else if (cf.equals("VERSION")) { String cq = i.getKey().getColumnQualifier().toString(); String rowId = i.getKey().getRow().toString(); assertEquals("VERSION", cq); assertEquals("00000VERSION", rowId); assertEquals("2", i.getValue().toString()); } else { printTable(); fail("Unknown CF written to accumulo table: " + cf); } } assertEquals(expectedIndexes, foundIndexes); }
From source file:edu.gslis.ts.hadoop.ThriftRMScorerHbase.java
License:Apache License
public ThriftRMScorerHbase(String tableName, String topicsFile, String vocabFile, String outputPath, String stoplist, int queryId, int scanSize) throws Exception { this.tableName = tableName; this.queryId = queryId; queries = readEvents(topicsFile, null); vocab = readVocab(vocabFile, null);/*from www . jav a2 s .co m*/ stopper = readStoplist(stoplist, null); this.scanSize = scanSize; config = HBaseConfiguration.create(); int timeout = 60000 * 20; config.set("hbase.rpc.timeout", String.valueOf(timeout)); connection = ConnectionFactory.createConnection(config); table = connection.getTable(TableName.valueOf(tableName)); deserializer = new TDeserializer(new TBinaryProtocol.Factory()); }