List of usage examples for org.apache.thrift TDeserializer TDeserializer
public TDeserializer(TProtocolFactory protocolFactory)
From source file:io.warp10.standalone.StandaloneDeleteHandler.java
License:Apache License
@Override public void handle(String target, Request baseRequest, HttpServletRequest request, HttpServletResponse response) throws IOException, ServletException { if (target.equals(Constants.API_ENDPOINT_DELETE)) { baseRequest.setHandled(true);// w w w.j a v a2s. c o m } else { return; } // // CORS header // response.setHeader("Access-Control-Allow-Origin", "*"); long nano = System.nanoTime(); // // Extract DatalogRequest if specified // String datalogHeader = request.getHeader(Constants.getHeader(Configuration.HTTP_HEADER_DATALOG)); DatalogRequest dr = null; boolean forwarded = false; if (null != datalogHeader) { byte[] bytes = OrderPreservingBase64.decode(datalogHeader.getBytes(Charsets.US_ASCII)); if (null != datalogPSK) { bytes = CryptoUtils.unwrap(datalogPSK, bytes); } if (null == bytes) { response.sendError(HttpServletResponse.SC_INTERNAL_SERVER_ERROR, "Invalid Datalog header."); return; } TDeserializer deser = new TDeserializer(new TCompactProtocol.Factory()); try { dr = new DatalogRequest(); deser.deserialize(dr, bytes); } catch (TException te) { response.sendError(HttpServletResponse.SC_INTERNAL_SERVER_ERROR, te.getMessage()); return; } Map<String, String> labels = new HashMap<String, String>(); labels.put(SensisionConstants.SENSISION_LABEL_ID, new String( OrderPreservingBase64.decode(dr.getId().getBytes(Charsets.US_ASCII)), Charsets.UTF_8)); labels.put(SensisionConstants.SENSISION_LABEL_TYPE, dr.getType()); Sensision.update(SensisionConstants.CLASS_WARP_DATALOG_REQUESTS_RECEIVED, labels, 1); // // Check that the request query string matches the QS in the datalog request // if (!request.getQueryString().equals(dr.getDeleteQueryString())) { response.sendError(HttpServletResponse.SC_INTERNAL_SERVER_ERROR, "Invalid DatalogRequest."); return; } forwarded = true; } // // TODO(hbs): Extract producer/owner from token // String token = null != dr ? dr.getToken() : request.getHeader(Constants.getHeader(Configuration.HTTP_HEADER_TOKENX)); if (null == token) { response.sendError(HttpServletResponse.SC_INTERNAL_SERVER_ERROR, "Missing token."); return; } WriteToken writeToken; try { writeToken = Tokens.extractWriteToken(token); } catch (WarpScriptException ee) { ee.printStackTrace(); response.sendError(HttpServletResponse.SC_INTERNAL_SERVER_ERROR, ee.getMessage()); return; } String application = writeToken.getAppName(); String producer = Tokens.getUUID(writeToken.getProducerId()); String owner = Tokens.getUUID(writeToken.getOwnerId()); // // For delete operations, producer and owner MUST be equal // if (!producer.equals(owner)) { response.sendError(HttpServletResponse.SC_INTERNAL_SERVER_ERROR, "Invalid write token for deletion."); return; } Map<String, String> sensisionLabels = new HashMap<String, String>(); sensisionLabels.put(SensisionConstants.SENSISION_LABEL_PRODUCER, producer); long count = 0; long gts = 0; Throwable t = null; StringBuilder metas = new StringBuilder(); // Boolean indicating whether or not we should continue adding results to 'metas' boolean metasSaturated = false; // // Extract start/end // String startstr = request.getParameter(Constants.HTTP_PARAM_START); String endstr = request.getParameter(Constants.HTTP_PARAM_END); // // Extract selector // String selector = request.getParameter(Constants.HTTP_PARAM_SELECTOR); String minage = request.getParameter(Constants.HTTP_PARAM_MINAGE); if (null != minage) { response.sendError(HttpServletResponse.SC_INTERNAL_SERVER_ERROR, "Standalone version does not support the '" + Constants.HTTP_PARAM_MINAGE + "' parameter in delete requests."); return; } boolean dryrun = null != request.getParameter(Constants.HTTP_PARAM_DRYRUN); File loggingFile = null; PrintWriter loggingWriter = null; // // Open the logging file if logging is enabled // if (null != loggingDir) { long nanos = null != dr ? dr.getTimestamp() : TimeSource.getNanoTime(); StringBuilder sb = new StringBuilder(); sb.append(Long.toHexString(nanos)); sb.insert(0, "0000000000000000", 0, 16 - sb.length()); sb.append("-"); if (null != dr) { sb.append(dr.getId()); } else { sb.append(datalogId); } sb.append("-"); sb.append(dtf.print(nanos / 1000000L)); sb.append(Long.toString(1000000L + (nanos % 1000000L)).substring(1)); sb.append("Z"); if (null == dr) { dr = new DatalogRequest(); dr.setTimestamp(nanos); dr.setType(Constants.DATALOG_DELETE); dr.setId(datalogId); dr.setToken(token); dr.setDeleteQueryString(request.getQueryString()); } if (null != dr && (!forwarded || (forwarded && this.logforwarded))) { // // Serialize the request // TSerializer ser = new TSerializer(new TCompactProtocol.Factory()); byte[] encoded; try { encoded = ser.serialize(dr); } catch (TException te) { response.sendError(HttpServletResponse.SC_INTERNAL_SERVER_ERROR, te.getMessage()); return; } if (null != this.datalogPSK) { encoded = CryptoUtils.wrap(this.datalogPSK, encoded); } encoded = OrderPreservingBase64.encode(encoded); loggingFile = new File(loggingDir, sb.toString()); loggingWriter = new PrintWriter(new FileWriterWithEncoding(loggingFile, Charsets.UTF_8)); // // Write request // loggingWriter.println(new String(encoded, Charsets.US_ASCII)); } } boolean validated = false; try { if (null == producer || null == owner) { response.sendError(HttpServletResponse.SC_FORBIDDEN, "Invalid token."); return; } // // Build extra labels // Map<String, String> extraLabels = new HashMap<String, String>(); // // Only set owner and potentially app, producer may vary // extraLabels.put(Constants.OWNER_LABEL, owner); // FIXME(hbs): remove me if (null != application) { extraLabels.put(Constants.APPLICATION_LABEL, application); sensisionLabels.put(SensisionConstants.SENSISION_LABEL_APPLICATION, application); } boolean hasRange = false; long start = Long.MIN_VALUE; long end = Long.MAX_VALUE; if (null != startstr) { if (null == endstr) { response.sendError(HttpServletResponse.SC_INTERNAL_SERVER_ERROR, "Both " + Constants.HTTP_PARAM_START + " and " + Constants.HTTP_PARAM_END + " should be defined."); return; } if (startstr.contains("T")) { start = fmt.parseDateTime(startstr).getMillis() * Constants.TIME_UNITS_PER_MS; } else { start = Long.valueOf(startstr); } } if (null != endstr) { if (null == startstr) { response.sendError(HttpServletResponse.SC_INTERNAL_SERVER_ERROR, "Both " + Constants.HTTP_PARAM_START + " and " + Constants.HTTP_PARAM_END + " should be defined."); return; } if (endstr.contains("T")) { end = fmt.parseDateTime(endstr).getMillis() * Constants.TIME_UNITS_PER_MS; } else { end = Long.valueOf(endstr); } } if (Long.MIN_VALUE == start && Long.MAX_VALUE == end && null == request.getParameter(Constants.HTTP_PARAM_DELETEALL)) { response.sendError(HttpServletResponse.SC_INTERNAL_SERVER_ERROR, "Parameter " + Constants.HTTP_PARAM_DELETEALL + " should be set when deleting a full range."); return; } if (Long.MIN_VALUE != start || Long.MAX_VALUE != end) { hasRange = true; } if (start > end) { response.sendError(HttpServletResponse.SC_INTERNAL_SERVER_ERROR, "Invalid time range specification."); return; } // // Extract the class and labels selectors // The class selector and label selectors are supposed to have // values which use percent encoding, i.e. explicit percent encoding which // might have been re-encoded using percent encoding when passed as parameter // // Matcher m = EgressFetchHandler.SELECTOR_RE.matcher(selector); if (!m.matches()) { response.sendError(HttpServletResponse.SC_BAD_REQUEST); return; } String classSelector = URLDecoder.decode(m.group(1), "UTF-8"); String labelsSelection = m.group(2); Map<String, String> labelsSelectors; try { labelsSelectors = GTSHelper.parseLabelsSelectors(labelsSelection); } catch (ParseException pe) { response.sendError(HttpServletResponse.SC_INTERNAL_SERVER_ERROR, pe.getMessage()); return; } validated = true; // // Force 'producer'/'owner'/'app' from token // labelsSelectors.putAll(extraLabels); List<Metadata> metadatas = null; List<String> clsSels = new ArrayList<String>(); List<Map<String, String>> lblsSels = new ArrayList<Map<String, String>>(); clsSels.add(classSelector); lblsSels.add(labelsSelectors); metadatas = directoryClient.find(clsSels, lblsSels); response.setStatus(HttpServletResponse.SC_OK); response.setContentType("text/plain"); PrintWriter pw = response.getWriter(); StringBuilder sb = new StringBuilder(); for (Metadata metadata : metadatas) { // // Remove from DB // if (!hasRange) { if (!dryrun) { this.directoryClient.unregister(metadata); } } // // Remove data // long localCount = 0; if (!dryrun) { localCount = this.storeClient.delete(writeToken, metadata, start, end); } count += localCount; sb.setLength(0); GTSHelper.metadataToString(sb, metadata.getName(), metadata.getLabels()); if (metadata.getAttributesSize() > 0) { GTSHelper.labelsToString(sb, metadata.getAttributes()); } else { sb.append("{}"); } pw.write(sb.toString()); pw.write("\r\n"); if (!metasSaturated) { if (gts < MAX_LOGGED_DELETED_GTS) { metas.append(sb); metas.append("\n"); } else { metasSaturated = true; metas.append("..."); metas.append("\n"); } } gts++; // Log detailed metrics for this GTS owner and app Map<String, String> labels = new HashMap<>(); labels.put(SensisionConstants.SENSISION_LABEL_OWNER, metadata.getLabels().get(Constants.OWNER_LABEL)); labels.put(SensisionConstants.SENSISION_LABEL_APPLICATION, metadata.getLabels().get(Constants.APPLICATION_LABEL)); Sensision.update( SensisionConstants.SENSISION_CLASS_CONTINUUM_STANDALONE_DELETE_DATAPOINTS_PEROWNERAPP, labels, localCount); } } catch (Exception e) { t = e; // If we have not yet written anything on the output stream, call sendError if (0 == gts) { response.sendError(HttpServletResponse.SC_INTERNAL_SERVER_ERROR, e.getMessage()); return; } else { throw e; } } finally { if (null != loggingWriter) { Map<String, String> labels = new HashMap<String, String>(); labels.put(SensisionConstants.SENSISION_LABEL_ID, new String( OrderPreservingBase64.decode(dr.getId().getBytes(Charsets.US_ASCII)), Charsets.UTF_8)); labels.put(SensisionConstants.SENSISION_LABEL_TYPE, dr.getType()); Sensision.update(SensisionConstants.CLASS_WARP_DATALOG_REQUESTS_LOGGED, labels, 1); loggingWriter.close(); if (validated) { loggingFile.renameTo(new File(loggingFile.getAbsolutePath() + DatalogForwarder.DATALOG_SUFFIX)); } else { loggingFile.delete(); } } Sensision.update(SensisionConstants.SENSISION_CLASS_CONTINUUM_STANDALONE_DELETE_REQUESTS, sensisionLabels, 1); Sensision.update(SensisionConstants.SENSISION_CLASS_CONTINUUM_STANDALONE_DELETE_GTS, sensisionLabels, gts); Sensision.update(SensisionConstants.SENSISION_CLASS_CONTINUUM_STANDALONE_DELETE_DATAPOINTS, sensisionLabels, count); Sensision.update(SensisionConstants.SENSISION_CLASS_CONTINUUM_STANDALONE_DELETE_TIME_US, sensisionLabels, (System.nanoTime() - nano) / 1000); LoggingEvent event = LogUtil.setLoggingEventAttribute(null, LogUtil.DELETION_TOKEN, token); event = LogUtil.setLoggingEventAttribute(event, LogUtil.DELETION_SELECTOR, selector); event = LogUtil.setLoggingEventAttribute(event, LogUtil.DELETION_START, startstr); event = LogUtil.setLoggingEventAttribute(event, LogUtil.DELETION_END, endstr); event = LogUtil.setLoggingEventAttribute(event, LogUtil.DELETION_METADATA, metas.toString()); event = LogUtil.setLoggingEventAttribute(event, LogUtil.DELETION_COUNT, Long.toString(count)); event = LogUtil.setLoggingEventAttribute(event, LogUtil.DELETION_GTS, Long.toString(gts)); LogUtil.addHttpHeaders(event, request); if (null != t) { LogUtil.setLoggingEventStackTrace(null, LogUtil.STACK_TRACE, t); } LOG.info(LogUtil.serializeLoggingEvent(this.keyStore, event)); } response.setStatus(HttpServletResponse.SC_OK); }
From source file:io.warp10.standalone.StandaloneDirectoryClient.java
License:Apache License
public StandaloneDirectoryClient(DB db, final KeyStore keystore) { this.initNThreads = Integer.parseInt(WarpConfig.getProperties() .getProperty(Configuration.DIRECTORY_INIT_NTHREADS, DIRECTORY_INIT_NTHREADS_DEFAULT)); this.db = db; this.keystore = keystore; this.aesKey = this.keystore.getKey(KeyStore.AES_LEVELDB_METADATA); this.classKey = this.keystore.getKey(KeyStore.SIPHASH_CLASS); this.classLongs = SipHashInline.getKey(this.classKey); this.labelsKey = this.keystore.getKey(KeyStore.SIPHASH_LABELS); this.labelsLongs = SipHashInline.getKey(this.labelsKey); //// w ww.j ava2s. co m // Read metadata from DB // if (null == db) { return; } DBIterator iter = db.iterator(); iter.seek(METADATA_PREFIX); byte[] stop = "N".getBytes(Charsets.US_ASCII); long count = 0; Thread[] initThreads = new Thread[this.initNThreads]; final AtomicBoolean[] stopMarkers = new AtomicBoolean[this.initNThreads]; final LinkedBlockingQueue<Entry<byte[], byte[]>> resultQ = new LinkedBlockingQueue<Entry<byte[], byte[]>>( initThreads.length * 8192); for (int i = 0; i < initThreads.length; i++) { stopMarkers[i] = new AtomicBoolean(false); final AtomicBoolean stopMe = stopMarkers[i]; initThreads[i] = new Thread(new Runnable() { @Override public void run() { byte[] bytes = new byte[16]; AESWrapEngine engine = null; PKCS7Padding padding = null; if (null != keystore.getKey(KeyStore.AES_LEVELDB_METADATA)) { engine = new AESWrapEngine(); CipherParameters params = new KeyParameter(keystore.getKey(KeyStore.AES_LEVELDB_METADATA)); engine.init(false, params); padding = new PKCS7Padding(); } TDeserializer deserializer = new TDeserializer(new TCompactProtocol.Factory()); while (!stopMe.get()) { try { Entry<byte[], byte[]> result = resultQ.poll(100, TimeUnit.MILLISECONDS); if (null == result) { continue; } byte[] key = result.getKey(); byte[] value = result.getValue(); // // Unwrap // byte[] unwrapped = null != engine ? engine.unwrap(value, 0, value.length) : value; // // Unpad // int padcount = null != padding ? padding.padCount(unwrapped) : 0; byte[] unpadded = null != padding ? Arrays.copyOf(unwrapped, unwrapped.length - padcount) : unwrapped; // // Deserialize // Metadata metadata = new Metadata(); deserializer.deserialize(metadata, unpadded); // // Compute classId/labelsId and compare it to the values in the row key // // 128BITS long classId = GTSHelper.classId(classLongs, metadata.getName()); long labelsId = GTSHelper.labelsId(labelsLongs, metadata.getLabels()); ByteBuffer bb = ByteBuffer.wrap(key).order(ByteOrder.BIG_ENDIAN); bb.position(1); long hbClassId = bb.getLong(); long hbLabelsId = bb.getLong(); // If classId/labelsId are incoherent, skip metadata if (classId != hbClassId || labelsId != hbLabelsId) { // FIXME(hbs): LOG System.err.println("Incoherent class/labels Id for " + metadata); continue; } // 128BITS metadata.setClassId(classId); metadata.setLabelsId(labelsId); if (!metadata.isSetAttributes()) { metadata.setAttributes(new HashMap<String, String>()); } // // Internalize Strings // GTSHelper.internalizeStrings(metadata); synchronized (metadatas) { if (!metadatas.containsKey(metadata.getName())) { metadatas.put(metadata.getName(), (Map) new MapMaker().concurrencyLevel(64).makeMap()); } } synchronized (metadatas.get(metadata.getName())) { if (!metadatas.get(metadata.getName()).containsKey(labelsId)) { metadatas.get(metadata.getName()).put(labelsId, metadata); // // Store Metadata under 'id' // // 128BITS GTSHelper.fillGTSIds(bytes, 0, classId, labelsId); BigInteger id = new BigInteger(bytes); metadatasById.put(id, metadata); continue; } } // FIXME(hbs): LOG System.err.println("Duplicate labelsId for classId " + classId + ": " + metadata); continue; } catch (InvalidCipherTextException icte) { throw new RuntimeException(icte); } catch (TException te) { throw new RuntimeException(te); } catch (InterruptedException ie) { } } } }); initThreads[i].setDaemon(true); initThreads[i].setName("[Directory initializer #" + i + "]"); initThreads[i].start(); } try { long nano = System.nanoTime(); while (iter.hasNext()) { Entry<byte[], byte[]> kv = iter.next(); byte[] key = kv.getKey(); if (Bytes.compareTo(key, stop) >= 0) { break; } boolean interrupted = true; while (interrupted) { interrupted = false; try { resultQ.put(kv); count++; if (0 == count % 1000) { Sensision.set(SensisionConstants.SENSISION_CLASS_CONTINUUM_DIRECTORY_GTS, Sensision.EMPTY_LABELS, count); } } catch (InterruptedException ie) { interrupted = true; } } } // // Wait until resultQ is empty // while (!resultQ.isEmpty()) { try { Thread.sleep(100L); } catch (InterruptedException ie) { } } // // Notify the init threads to stop // for (int i = 0; i < initNThreads; i++) { stopMarkers[i].set(true); } nano = System.nanoTime() - nano; System.out.println("Loaded " + count + " GTS in " + (nano / 1000000.0D) + " ms"); } finally { Sensision.set(SensisionConstants.SENSISION_CLASS_CONTINUUM_DIRECTORY_GTS, Sensision.EMPTY_LABELS, count); try { iter.close(); } catch (IOException ioe) { throw new RuntimeException(ioe); } } }
From source file:io.warp10.standalone.StandaloneIngressHandler.java
License:Apache License
@Override public void handle(String target, Request baseRequest, HttpServletRequest request, HttpServletResponse response) throws IOException, ServletException { String token = null;/*from w w w . j a v a 2s . c om*/ if (target.equals(Constants.API_ENDPOINT_UPDATE)) { baseRequest.setHandled(true); } else if (target.startsWith(Constants.API_ENDPOINT_UPDATE + "/")) { baseRequest.setHandled(true); token = target.substring(Constants.API_ENDPOINT_UPDATE.length() + 1); } else if (target.equals(Constants.API_ENDPOINT_META)) { handleMeta(target, baseRequest, request, response); return; } else { return; } try { // // CORS header // response.setHeader("Access-Control-Allow-Origin", "*"); long nano = System.nanoTime(); // // Extract DatalogRequest if specified // String datalogHeader = request.getHeader(Constants.getHeader(Configuration.HTTP_HEADER_DATALOG)); DatalogRequest dr = null; boolean forwarded = false; if (null != datalogHeader) { byte[] bytes = OrderPreservingBase64.decode(datalogHeader.getBytes(Charsets.US_ASCII)); if (null != datalogPSK) { bytes = CryptoUtils.unwrap(datalogPSK, bytes); } if (null == bytes) { throw new IOException("Invalid Datalog header."); } TDeserializer deser = new TDeserializer(new TCompactProtocol.Factory()); try { dr = new DatalogRequest(); deser.deserialize(dr, bytes); } catch (TException te) { throw new IOException(); } token = dr.getToken(); Map<String, String> labels = new HashMap<String, String>(); labels.put(SensisionConstants.SENSISION_LABEL_ID, new String( OrderPreservingBase64.decode(dr.getId().getBytes(Charsets.US_ASCII)), Charsets.UTF_8)); labels.put(SensisionConstants.SENSISION_LABEL_TYPE, dr.getType()); Sensision.update(SensisionConstants.CLASS_WARP_DATALOG_REQUESTS_RECEIVED, labels, 1); forwarded = true; } // // TODO(hbs): Extract producer/owner from token // if (null == token) { token = request.getHeader(Constants.getHeader(Configuration.HTTP_HEADER_TOKENX)); } WriteToken writeToken; try { writeToken = Tokens.extractWriteToken(token); } catch (WarpScriptException ee) { throw new IOException(ee); } String application = writeToken.getAppName(); String producer = Tokens.getUUID(writeToken.getProducerId()); String owner = Tokens.getUUID(writeToken.getOwnerId()); Map<String, String> sensisionLabels = new HashMap<String, String>(); sensisionLabels.put(SensisionConstants.SENSISION_LABEL_PRODUCER, producer); long count = 0; long total = 0; File loggingFile = null; PrintWriter loggingWriter = null; try { if (null == producer || null == owner) { response.sendError(HttpServletResponse.SC_FORBIDDEN, "Invalid token."); return; } // // Build extra labels // Map<String, String> extraLabels = new HashMap<String, String>(); // Add labels from the WriteToken if they exist if (writeToken.getLabelsSize() > 0) { extraLabels.putAll(writeToken.getLabels()); } // Force internal labels extraLabels.put(Constants.PRODUCER_LABEL, producer); extraLabels.put(Constants.OWNER_LABEL, owner); // FIXME(hbs): remove me, apps should be set in all tokens now... if (null != application) { extraLabels.put(Constants.APPLICATION_LABEL, application); sensisionLabels.put(SensisionConstants.SENSISION_LABEL_APPLICATION, application); } else { // remove application label extraLabels.remove(Constants.APPLICATION_LABEL); } // // Determine if content if gzipped // boolean gzipped = false; if (null != request.getHeader("Content-Type") && "application/gzip".equals(request.getHeader("Content-Type"))) { gzipped = true; } BufferedReader br = null; if (gzipped) { GZIPInputStream is = new GZIPInputStream(request.getInputStream()); br = new BufferedReader(new InputStreamReader(is)); } else { br = request.getReader(); } // // Get the present time // Long now = TimeSource.getTime(); // // Check the value of the 'now' header // // The following values are supported: // // A number, which will be interpreted as an absolute time reference, // i.e. a number of time units since the Epoch. // // A number prefixed by '+' or '-' which will be interpreted as a // delta from the present time. // // A '*' which will mean to not set 'now', and to recompute its value // each time it's needed. // String nowstr = null != dr ? dr.getNow() : request.getHeader(Constants.getHeader(Configuration.HTTP_HEADER_NOW_HEADERX)); if (null != nowstr) { if ("*".equals(nowstr)) { now = null; } else if (nowstr.startsWith("+")) { try { long delta = Long.parseLong(nowstr.substring(1)); now = now + delta; } catch (Exception e) { throw new IOException("Invalid base timestamp."); } } else if (nowstr.startsWith("-")) { try { long delta = Long.parseLong(nowstr.substring(1)); now = now - delta; } catch (Exception e) { throw new IOException("Invalid base timestamp."); } } else { try { now = Long.parseLong(nowstr); } catch (Exception e) { throw new IOException("Invalid base timestamp."); } } } // // Open the logging file if logging is enabled // if (null != loggingDir) { long nanos = null != dr ? dr.getTimestamp() : TimeSource.getNanoTime(); StringBuilder sb = new StringBuilder(); sb.append(Long.toHexString(nanos)); sb.insert(0, "0000000000000000", 0, 16 - sb.length()); sb.append("-"); if (null != dr) { sb.append(dr.getId()); } else { sb.append(datalogId); } sb.append("-"); sb.append(dtf.print(nanos / 1000000L)); sb.append(Long.toString(1000000L + (nanos % 1000000L)).substring(1)); sb.append("Z"); if (null == dr) { dr = new DatalogRequest(); dr.setTimestamp(nanos); dr.setType(Constants.DATALOG_UPDATE); dr.setId(datalogId); dr.setToken(token); if (null == now) { // // We MUST force 'now', otherwise forwarded metrics will not have a // coherent time. This alters the semantics slightly but make it // coherent across the board. // now = TimeSource.getTime(); } dr.setNow(Long.toString(now)); } if (null != dr && (!forwarded || (forwarded && this.logforwarded))) { // // Serialize the request // TSerializer ser = new TSerializer(new TCompactProtocol.Factory()); byte[] encoded; try { encoded = ser.serialize(dr); } catch (TException te) { throw new IOException(te); } if (null != this.datalogPSK) { encoded = CryptoUtils.wrap(this.datalogPSK, encoded); } encoded = OrderPreservingBase64.encode(encoded); loggingFile = new File(loggingDir, sb.toString()); loggingWriter = new PrintWriter(new FileWriterWithEncoding(loggingFile, Charsets.UTF_8)); // // Write request // loggingWriter.println(new String(encoded, Charsets.US_ASCII)); } // // Force 'now' // now = Long.parseLong(dr.getNow()); } // // Loop on all lines // GTSEncoder lastencoder = null; GTSEncoder encoder = null; // // Chunk index when archiving // do { String line = br.readLine(); if (null == line) { break; } line = line.trim(); if (0 == line.length()) { continue; } // // Ignore comments // if ('#' == line.charAt(0)) { continue; } // // Check for pushback // TODO(hbs): implement the actual push back if we are over the subscribed limit // if (count % PUSHBACK_CHECK_INTERVAL == 0) { Sensision.update( SensisionConstants.SENSISION_CLASS_CONTINUUM_STANDALONE_UPDATE_DATAPOINTS_RAW, sensisionLabels, count); total += count; count = 0; } count++; try { encoder = GTSHelper.parse(lastencoder, line, extraLabels, now, maxValueSize, false); //nano2 += System.nanoTime() - nano0; } catch (ParseException pe) { Sensision.update(SensisionConstants.SENSISION_CLASS_CONTINUUM_STANDALONE_UPDATE_PARSEERRORS, sensisionLabels, 1); throw new IOException("Parse error at '" + line + "'", pe); } if (encoder != lastencoder || lastencoder.size() > ENCODER_SIZE_THRESHOLD) { // // Check throttling // if (null != lastencoder) { // 128BITS lastencoder.setClassId(GTSHelper.classId(classKeyLongs, lastencoder.getName())); lastencoder.setLabelsId( GTSHelper.labelsId(labelsKeyLongs, lastencoder.getMetadata().getLabels())); ThrottlingManager.checkMADS(lastencoder.getMetadata(), producer, owner, application, lastencoder.getClassId(), lastencoder.getLabelsId()); ThrottlingManager.checkDDP(lastencoder.getMetadata(), producer, owner, application, (int) lastencoder.getCount()); } // // Build metadata object to push // if (encoder != lastencoder) { Metadata metadata = new Metadata(encoder.getMetadata()); metadata.setSource(Configuration.INGRESS_METADATA_SOURCE); //nano6 += System.nanoTime() - nano0; this.directoryClient.register(metadata); //nano5 += System.nanoTime() - nano0; } if (null != lastencoder) { this.storeClient.store(lastencoder); } if (encoder != lastencoder) { lastencoder = encoder; } else { //lastencoder = null // // Allocate a new GTSEncoder and reuse Metadata so we can // correctly handle a continuation line if this is what occurs next // Metadata metadata = lastencoder.getMetadata(); lastencoder = new GTSEncoder(0L); lastencoder.setMetadata(metadata); } } // // Write the line last, so we do not write lines which triggered exceptions // if (null != loggingWriter) { loggingWriter.println(line); } } while (true); br.close(); if (null != lastencoder && lastencoder.size() > 0) { // 128BITS lastencoder.setClassId(GTSHelper.classId(classKeyLongs, lastencoder.getName())); lastencoder .setLabelsId(GTSHelper.labelsId(labelsKeyLongs, lastencoder.getMetadata().getLabels())); ThrottlingManager.checkMADS(lastencoder.getMetadata(), producer, owner, application, lastencoder.getClassId(), lastencoder.getLabelsId()); ThrottlingManager.checkDDP(lastencoder.getMetadata(), producer, owner, application, (int) lastencoder.getCount()); this.storeClient.store(lastencoder); } // // TODO(hbs): should we update the count in Sensision periodically so you can't trick the throttling mechanism? // } catch (WarpException we) { throw new IOException(we); } finally { this.storeClient.store(null); this.directoryClient.register(null); Sensision.update(SensisionConstants.SENSISION_CLASS_CONTINUUM_STANDALONE_UPDATE_DATAPOINTS_RAW, sensisionLabels, count); Sensision.update(SensisionConstants.SENSISION_CLASS_CONTINUUM_STANDALONE_UPDATE_REQUESTS, sensisionLabels, 1); Sensision.update(SensisionConstants.SENSISION_CLASS_CONTINUUM_STANDALONE_UPDATE_TIME_US, sensisionLabels, (System.nanoTime() - nano) / 1000); if (null != loggingWriter) { Map<String, String> labels = new HashMap<String, String>(); labels.put(SensisionConstants.SENSISION_LABEL_ID, new String( OrderPreservingBase64.decode(dr.getId().getBytes(Charsets.US_ASCII)), Charsets.UTF_8)); labels.put(SensisionConstants.SENSISION_LABEL_TYPE, dr.getType()); Sensision.update(SensisionConstants.CLASS_WARP_DATALOG_REQUESTS_LOGGED, labels, 1); loggingWriter.close(); loggingFile.renameTo(new File(loggingFile.getAbsolutePath() + DatalogForwarder.DATALOG_SUFFIX)); } // // Update stats with CDN // String cdn = request.getHeader(Constants.OVH_CDN_GEO_HEADER); if (null != cdn) { sensisionLabels.put(SensisionConstants.SENSISION_LABEL_CDN, cdn); // Per CDN stat is updated at the end, so update with 'total' + 'count' Sensision.update(SensisionConstants.SENSISION_CLASS_CONTINUUM_STANDALONE_UPDATE_DATAPOINTS_RAW, sensisionLabels, count + total); Sensision.update(SensisionConstants.SENSISION_CLASS_CONTINUUM_STANDALONE_UPDATE_REQUESTS, sensisionLabels, 1); Sensision.update(SensisionConstants.SENSISION_CLASS_CONTINUUM_STANDALONE_UPDATE_TIME_US, sensisionLabels, (System.nanoTime() - nano) / 1000); } } response.setStatus(HttpServletResponse.SC_OK); } catch (Exception e) { if (!response.isCommitted()) { response.sendError(HttpServletResponse.SC_INTERNAL_SERVER_ERROR, e.getMessage()); return; } } }
From source file:io.warp10.standalone.StandaloneIngressHandler.java
License:Apache License
/** * Handle Metadata updating//w ww. j a v a 2s. c o m */ public void handleMeta(String target, Request baseRequest, HttpServletRequest request, HttpServletResponse response) throws IOException, ServletException { if (target.equals(Constants.API_ENDPOINT_META)) { baseRequest.setHandled(true); } else { return; } try { // // CORS header // response.setHeader("Access-Control-Allow-Origin", "*"); // // Extract DatalogRequest if specified // String datalogHeader = request.getHeader(Constants.getHeader(Configuration.HTTP_HEADER_DATALOG)); DatalogRequest dr = null; boolean forwarded = false; if (null != datalogHeader) { byte[] bytes = OrderPreservingBase64.decode(datalogHeader.getBytes(Charsets.US_ASCII)); if (null != datalogPSK) { bytes = CryptoUtils.unwrap(datalogPSK, bytes); } if (null == bytes) { throw new IOException("Invalid Datalog header."); } TDeserializer deser = new TDeserializer(new TCompactProtocol.Factory()); try { dr = new DatalogRequest(); deser.deserialize(dr, bytes); } catch (TException te) { throw new IOException(); } Map<String, String> labels = new HashMap<String, String>(); labels.put(SensisionConstants.SENSISION_LABEL_ID, new String( OrderPreservingBase64.decode(dr.getId().getBytes(Charsets.US_ASCII)), Charsets.UTF_8)); labels.put(SensisionConstants.SENSISION_LABEL_TYPE, dr.getType()); Sensision.update(SensisionConstants.CLASS_WARP_DATALOG_REQUESTS_RECEIVED, labels, 1); forwarded = true; } // // Loop over the input lines. // Each has the following format: // // class{labels}{attributes} // String token = null != dr ? dr.getToken() : request.getHeader(Constants.getHeader(Configuration.HTTP_HEADER_TOKENX)); WriteToken wtoken; try { wtoken = Tokens.extractWriteToken(token); } catch (WarpScriptException ee) { throw new IOException(ee); } String application = wtoken.getAppName(); String producer = Tokens.getUUID(wtoken.getProducerId()); String owner = Tokens.getUUID(wtoken.getOwnerId()); if (null == producer || null == owner) { response.sendError(HttpServletResponse.SC_FORBIDDEN, "Invalid token."); return; } // // Determine if content if gzipped // boolean gzipped = false; if (null != request.getHeader("Content-Type") && "application/gzip".equals(request.getHeader("Content-Type"))) { gzipped = true; } BufferedReader br = null; if (gzipped) { GZIPInputStream is = new GZIPInputStream(request.getInputStream()); br = new BufferedReader(new InputStreamReader(is)); } else { br = request.getReader(); } File loggingFile = null; PrintWriter loggingWriter = null; // // Open the logging file if logging is enabled // if (null != loggingDir) { long nanos = null != dr ? dr.getTimestamp() : TimeSource.getNanoTime(); StringBuilder sb = new StringBuilder(); sb.append(Long.toHexString(nanos)); sb.insert(0, "0000000000000000", 0, 16 - sb.length()); sb.append("-"); if (null != dr) { sb.append(dr.getId()); } else { sb.append(datalogId); } sb.append("-"); sb.append(dtf.print(nanos / 1000000L)); sb.append(Long.toString(1000000L + (nanos % 1000000L)).substring(1)); sb.append("Z"); if (null == dr) { dr = new DatalogRequest(); dr.setTimestamp(nanos); dr.setType(Constants.DATALOG_META); dr.setId(datalogId); dr.setToken(token); } if (null != dr && (!forwarded || (forwarded && this.logforwarded))) { // // Serialize the request // TSerializer ser = new TSerializer(new TCompactProtocol.Factory()); byte[] encoded; try { encoded = ser.serialize(dr); } catch (TException te) { throw new IOException(te); } if (null != this.datalogPSK) { encoded = CryptoUtils.wrap(this.datalogPSK, encoded); } encoded = OrderPreservingBase64.encode(encoded); loggingFile = new File(loggingDir, sb.toString()); loggingWriter = new PrintWriter(new FileWriterWithEncoding(loggingFile, Charsets.UTF_8)); // // Write request // loggingWriter.println(new String(encoded, Charsets.US_ASCII)); } } try { // // Loop on all lines // while (true) { String line = br.readLine(); if (null == line) { break; } // Ignore blank lines if ("".equals(line)) { continue; } // Ignore comments if ('#' == line.charAt(0)) { continue; } Metadata metadata = MetadataUtils.parseMetadata(line); // Add labels from the WriteToken if they exist if (wtoken.getLabelsSize() > 0) { metadata.getLabels().putAll(wtoken.getLabels()); } // // Force owner/producer // metadata.getLabels().put(Constants.PRODUCER_LABEL, producer); metadata.getLabels().put(Constants.OWNER_LABEL, owner); if (null != application) { metadata.getLabels().put(Constants.APPLICATION_LABEL, application); } else { // remove application label metadata.getLabels().remove(Constants.APPLICATION_LABEL); } if (!MetadataUtils.validateMetadata(metadata)) { response.sendError(HttpServletResponse.SC_BAD_REQUEST, "Invalid metadata " + line); return; } metadata.setSource(Configuration.INGRESS_METADATA_UPDATE_ENDPOINT); this.directoryClient.register(metadata); // // Write the line last, so we do not write lines which triggered exceptions // if (null != loggingWriter) { loggingWriter.println(line); } } } finally { if (null != loggingWriter) { Map<String, String> labels = new HashMap<String, String>(); labels.put(SensisionConstants.SENSISION_LABEL_ID, new String( OrderPreservingBase64.decode(dr.getId().getBytes(Charsets.US_ASCII)), Charsets.UTF_8)); labels.put(SensisionConstants.SENSISION_LABEL_TYPE, dr.getType()); Sensision.update(SensisionConstants.CLASS_WARP_DATALOG_REQUESTS_LOGGED, labels, 1); loggingWriter.close(); loggingFile.renameTo(new File(loggingFile.getAbsolutePath() + DatalogForwarder.DATALOG_SUFFIX)); } } response.setStatus(HttpServletResponse.SC_OK); } catch (Exception e) { if (!response.isCommitted()) { response.sendError(HttpServletResponse.SC_INTERNAL_SERVER_ERROR, e.getMessage()); return; } } }
From source file:io.warp10.standalone.StandaloneMemoryStore.java
License:Apache License
private void load(String path) throws IOException { long nano = System.nanoTime(); int gts = 0;/*from w w w. j a va 2 s.com*/ long bytes = 0L; Configuration conf = new Configuration(); conf.set("fs.hdfs.impl", org.apache.hadoop.hdfs.DistributedFileSystem.class.getName()); conf.set("fs.file.impl", org.apache.hadoop.fs.LocalFileSystem.class.getName()); BytesWritable key = new BytesWritable(); BytesWritable value = new BytesWritable(); TDeserializer deserializer = new TDeserializer(new TCompactProtocol.Factory()); SequenceFile.Reader.Option optPath = SequenceFile.Reader.file(new Path(path)); SequenceFile.Reader reader = null; try { reader = new SequenceFile.Reader(conf, optPath); System.out.println("Loading '" + path + "' back in memory."); while (reader.next(key, value)) { gts++; GTSWrapper wrapper = new GTSWrapper(); deserializer.deserialize(wrapper, key.copyBytes()); GTSEncoder encoder = new GTSEncoder(0L, null, value.copyBytes()); encoder.setCount(wrapper.getCount()); bytes += value.getLength() + key.getLength(); encoder.safeSetMetadata(wrapper.getMetadata()); store(encoder); if (null != this.directoryClient) { this.directoryClient.register(wrapper.getMetadata()); } } } catch (FileNotFoundException fnfe) { System.err.println("File '" + path + "' was not found, skipping."); return; } catch (IOException ioe) { throw ioe; } catch (Exception e) { throw new IOException(e); } reader.close(); nano = System.nanoTime() - nano; System.out.println("Loaded " + gts + " GTS (" + bytes + " bytes) in " + (nano / 1000000.0D) + " ms."); }
From source file:io.warp10.udf.TUNWRAP.java
License:Apache License
@Override public Object apply(WarpScriptStack stack) throws WarpScriptException { Object top = stack.pop();/*from ww w . j a v a 2s . c om*/ Set<Object> ticks = new HashSet<Object>(); if (top instanceof Collection) { boolean noticks = false; for (Object elt : (Collection) top) { if (!(elt instanceof Long)) { noticks = true; } } if (!noticks) { ticks.addAll((Collection<Object>) top); top = stack.pop(); } } if (!(top instanceof String) && !(top instanceof byte[]) && !(top instanceof List)) { throw new WarpScriptException(getName() + " operates on a string or byte array or a list thereof."); } List<Object> inputs = new ArrayList<Object>(); if (top instanceof String || top instanceof byte[]) { inputs.add(top); } else { for (Object o : (List) top) { if (!(o instanceof String) && !(o instanceof byte[])) { throw new WarpScriptException( getName() + " operates on a string or byte array or a list thereof."); } inputs.add(o); } } List<Object> outputs = new ArrayList<Object>(); for (Object s : inputs) { byte[] bytes = s instanceof String ? OrderPreservingBase64.decode(s.toString().getBytes(Charsets.US_ASCII)) : (byte[]) s; TDeserializer deser = new TDeserializer(new TCompactProtocol.Factory()); try { GTSWrapper wrapper = new GTSWrapper(); deser.deserialize(wrapper, bytes); GTSDecoder decoder = GTSWrapperHelper.fromGTSWrapperToGTSDecoder(wrapper); GeoTimeSerie[] series = new GeoTimeSerie[4]; for (int i = 0; i < series.length; i++) { // Use a heuristic and consider a hint of 25% of the wrapper count series[i] = new GeoTimeSerie(wrapper.getLastbucket(), (int) wrapper.getBucketcount(), wrapper.getBucketspan(), (int) wrapper.getCount()); series[i].setMetadata(decoder.getMetadata()); } GeoTimeSerie gts = null; while (decoder.next()) { long timestamp = decoder.getTimestamp(); if (!ticks.isEmpty() && !ticks.contains(timestamp)) { continue; } long location = decoder.getLocation(); long elevation = decoder.getElevation(); Object value = decoder.getValue(); if (value instanceof Long) { gts = series[0]; } else if (value instanceof Boolean) { gts = series[1]; } else if (value instanceof String) { gts = series[2]; } else { gts = series[3]; } GTSHelper.setValue(gts, timestamp, location, elevation, value, false); } Map<String, GeoTimeSerie> typedSeries = new HashMap<String, GeoTimeSerie>(); // // Shrink the series // for (int i = 0; i < series.length; i++) { GTSHelper.shrink(series[i]); } typedSeries.put("LONG", series[0]); typedSeries.put("BOOLEAN", series[1]); typedSeries.put("STRING", series[2]); typedSeries.put("DOUBLE", series[3]); outputs.add(typedSeries); } catch (TException te) { throw new WarpScriptException(getName() + " failed to unwrap GTS."); } } if (!(top instanceof List)) { stack.push(outputs.get(0)); } else { stack.push(outputs); } return stack; }
From source file:it.polimi.hegira.adapters.cassandra.Cassandra.java
License:Apache License
@Override protected AbstractDatabase fromMyModel(Metamodel mm) { log.debug(Thread.currentThread().getName() + " Cassandra consumer started "); //Thrift Deserializer TDeserializer deserializer = new TDeserializer(new TBinaryProtocol.Factory()); //retrieve thread number int thread_id = 0; if (TWTs_NO != 0) { thread_id = (int) (Thread.currentThread().getId() % TWTs_NO); }//from w w w. ja va 2s . c o m //instantiate the cassandra transformer //the consistency level is not needed. Entity inserted with eventual consistency CassandraTransformer transformer = new CassandraTransformer(); //instantiate the TableManager TablesManager tablesManager = TablesManager.getTablesManager(); while (true) { log.debug(Thread.currentThread().getName() + " Extracting from the taskQueue" + thread_id + " TWTs_NO: " + TWTs_NO); try { //extract from the task queue Delivery delivery = taskQueues.get(thread_id).getConsumer().nextDelivery(); if (delivery != null) { //deserialize and retrieve the metamodel Metamodel metaModel = new Metamodel(); deserializer.deserialize(metaModel, delivery.getBody()); //retrieve the Cassandra Model CassandraModel cassandraModel = transformer.fromMyModel(metaModel); //retrieve the table and tries perform the insert try { tablesManager.getTable(cassandraModel.getTable()).insert(cassandraModel); } catch (ConnectException ex) { log.error(Thread.currentThread().getName() + " - Not able to connect to Cassandra", ex); //nack taskQueues.get(thread_id).sendNack(delivery); log.info("Sending Nack!! for entity(/ies)"); } catch (ClassNotFoundException ex) { log.error(Thread.currentThread().getName() + " - Error in during the insertion -", ex); //nack taskQueues.get(thread_id).sendNack(delivery); log.info("Sending Nack!! for entity(/ies)"); } //send ack taskQueues.get(thread_id).sendAck(delivery); } else { log.debug(Thread.currentThread().getName() + " - The queue " + TaskQueue.getDefaultTaskQueueName() + " is empty"); return null; } } catch (ShutdownSignalException | ConsumerCancelledException | InterruptedException ex) { log.error(Thread.currentThread().getName() + " - Cannot read next delivery from the queue " + TaskQueue.getDefaultTaskQueueName(), ex); } catch (TException ex) { log.error(Thread.currentThread().getName() + " - Error deserializing message ", ex); } catch (QueueException ex) { log.error(Thread.currentThread().getName() + " - Error sending an acknowledgment to the queue " + TaskQueue.getDefaultTaskQueueName(), ex); } } }
From source file:it.polimi.hegira.adapters.cassandra.CassandraAdapterTest.java
License:Apache License
/** * This test assumes the read consistency is set to eventual */// ww w . j ava 2s.com @Test public void toMyModelTest() { TSerializer serializer = new TSerializer(new TBinaryProtocol.Factory()); TDeserializer deserializer = new TDeserializer(new TBinaryProtocol.Factory()); // // Connects to the db and creates two tables with some sample data for testing purposes // The table will be dropped at the end of the test // String hostname = "127.0.0.1"; Cluster.Builder clusterBuilder = Cluster.builder().addContactPoint(hostname); Cluster cluster = clusterBuilder.build(); session = cluster.connect("test"); String firstTableName = "players"; String secondTableName = "users"; session.execute("CREATE TABLE IF NOT EXISTS " + firstTableName + " ( " + primaryKeyName + " varchar PRIMARY KEY," + "goal int," + "teams list<varchar> );"); session.execute("CREATE TABLE IF NOT EXISTS " + secondTableName + " ( " + primaryKeyName + " varchar PRIMARY KEY," + "age int," + "contacts map<varchar,varchar> );"); List<String> fakeList1 = new ArrayList<String>(); fakeList1.add("Real Madrid"); fakeList1.add("Napoli"); List<String> fakeList2 = new ArrayList<String>(); fakeList2.add("Manchester United"); fakeList2.add("Real Madrid"); Map<String, String> fakeMap1 = new HashMap<String, String>(); fakeMap1.put("Andrea", "andrea@gmail.com"); fakeMap1.put("Andre", "andre@gmail.com"); Map<String, String> fakeMap2 = new HashMap<String, String>(); fakeMap2.put("Luca", "luca@gmail.com"); fakeMap2.put("leo", "leo@gmail.com"); Statement insert1 = QueryBuilder.insertInto(firstTableName).values(new String[] { "id", "goal", "teams" }, new Object[] { "Callejon", 9, fakeList1 }); session.execute(insert1); Statement insert2 = QueryBuilder.insertInto(firstTableName).values(new String[] { "id", "goal", "teams" }, new Object[] { "Ronaldo", 30, fakeList2 }); session.execute(insert2); Statement insert3 = QueryBuilder.insertInto(secondTableName) .values(new String[] { "id", "age", "contacts" }, new Object[] { "Andrea", 22, fakeMap1 }); session.execute(insert3); Statement insert4 = QueryBuilder.insertInto(secondTableName) .values(new String[] { "id", "age", "contacts" }, new Object[] { "Leo", 1, fakeMap2 }); session.execute(insert4); //create the serialized column coresponding to the one read /*Column col=new Column(); col.setColumnName("goal"); try { col.setColumnValue(DefaultSerializer.serialize(9)); } catch (IOException e1) { // TODO Auto-generated catch block e1.printStackTrace(); } col.setColumnValueType("Integer"); col.setIndexable(false); List<Column> colList=new ArrayList<Column>(); colList.add(col); Map<String,List<Column>> columns=new HashMap<String,List<Column>>(); columns.put("test", colList); Metamodel metaColumn=new Metamodel("@players#Callejon", "Callejon", columns);*/ ArgumentCaptor<byte[]> serializedRow = ArgumentCaptor.forClass(byte[].class); reader.toMyModel(null); /*try { Mockito.verify(mockedTaskQueue).publish(serializer.serialize(metaColumn)); } catch (QueueException e) { // TODO Auto-generated catch block e.printStackTrace(); } catch (TException e) { // TODO Auto-generated catch block e.printStackTrace(); }*/ try { Mockito.verify(mockedTaskQueue, Mockito.times(4)).publish(serializedRow.capture()); } catch (QueueException e) { // TODO Auto-generated catch block e.printStackTrace(); } // //check user 1 // Metamodel resultUser1 = new Metamodel(); try { deserializer.deserialize(resultUser1, serializedRow.getAllValues().get(0)); } catch (TException e) { // TODO Auto-generated catch block e.printStackTrace(); } assertEquals("Leo", resultUser1.getRowKey()); assertEquals("@users#Leo", resultUser1.getPartitionGroup()); assertEquals(resultUser1.getColumns().get("users").size(), 2); assertEquals(1, resultUser1.getColumnFamilies().size()); assertEquals("users", resultUser1.getColumnFamilies().get(0)); Column userAge = resultUser1.getColumns().get("users").get(0); Column contactsUser = resultUser1.getColumns().get("users").get(1); assertEquals("age", userAge.getColumnName()); assertEquals("contacts", contactsUser.getColumnName()); try { assertEquals(1, DefaultSerializer.deserialize(userAge.getColumnValue())); assertEquals(fakeMap2, DefaultSerializer.deserialize(contactsUser.getColumnValue())); } catch (ClassNotFoundException e) { // TODO Auto-generated catch block e.printStackTrace(); } catch (IOException e) { // TODO Auto-generated catch block e.printStackTrace(); } assertEquals("Integer", userAge.getColumnValueType()); assertEquals("Map<String,String>", contactsUser.getColumnValueType()); assertEquals(false, userAge.isIndexable()); assertEquals(false, contactsUser.isIndexable()); // //check user 2 // Metamodel resultUser2 = new Metamodel(); try { deserializer.deserialize(resultUser2, serializedRow.getAllValues().get(1)); } catch (TException e) { // TODO Auto-generated catch block e.printStackTrace(); } assertEquals("Andrea", resultUser2.getRowKey()); assertEquals("@users#Andrea", resultUser2.getPartitionGroup()); assertEquals(resultUser2.getColumns().get("users").size(), 2); Column userAge2 = resultUser2.getColumns().get("users").get(0); Column contactsUser2 = resultUser2.getColumns().get("users").get(1); assertEquals("age", userAge2.getColumnName()); assertEquals("contacts", contactsUser2.getColumnName()); try { assertEquals(22, DefaultSerializer.deserialize(userAge2.getColumnValue())); assertEquals(fakeMap1, DefaultSerializer.deserialize(contactsUser2.getColumnValue())); } catch (ClassNotFoundException e) { // TODO Auto-generated catch block e.printStackTrace(); } catch (IOException e) { // TODO Auto-generated catch block e.printStackTrace(); } assertEquals("Integer", userAge2.getColumnValueType()); assertEquals("Map<String,String>", contactsUser2.getColumnValueType()); assertEquals(false, userAge2.isIndexable()); assertEquals(false, contactsUser2.isIndexable()); // //check players row 1 // Metamodel resultPlayer1 = new Metamodel(); try { deserializer.deserialize(resultPlayer1, serializedRow.getAllValues().get(3)); } catch (TException e) { // TODO Auto-generated catch block e.printStackTrace(); } assertEquals("Callejon", resultPlayer1.getRowKey()); assertEquals("@players#Callejon", resultPlayer1.getPartitionGroup()); assertEquals(resultPlayer1.getColumns().get("players").size(), 2); Column resultCol = resultPlayer1.getColumns().get("players").get(0); Column listResult = resultPlayer1.getColumns().get("players").get(1); assertEquals("goal", resultCol.getColumnName()); assertEquals("teams", listResult.getColumnName()); try { assertEquals(9, DefaultSerializer.deserialize(resultCol.getColumnValue())); assertEquals(fakeList1, DefaultSerializer.deserialize(listResult.getColumnValue())); } catch (ClassNotFoundException e) { // TODO Auto-generated catch block e.printStackTrace(); } catch (IOException e) { // TODO Auto-generated catch block e.printStackTrace(); } assertEquals("Integer", resultCol.getColumnValueType()); assertEquals("List<String>", listResult.getColumnValueType()); assertEquals(false, resultCol.isIndexable()); assertEquals(false, listResult.isIndexable()); // // check players row 2 // Metamodel resultPlayer2 = new Metamodel(); try { deserializer.deserialize(resultPlayer2, serializedRow.getAllValues().get(2)); } catch (TException e) { // TODO Auto-generated catch block e.printStackTrace(); } assertEquals("Ronaldo", resultPlayer2.getRowKey()); assertEquals("@players#Ronaldo", resultPlayer2.getPartitionGroup()); assertEquals(resultPlayer2.getColumns().get("players").size(), 2); Column resultCol2 = resultPlayer2.getColumns().get("players").get(0); Column listResult2 = resultPlayer2.getColumns().get("players").get(1); assertEquals("goal", resultCol2.getColumnName()); assertEquals("teams", listResult2.getColumnName()); try { assertEquals(30, DefaultSerializer.deserialize(resultCol2.getColumnValue())); assertEquals(fakeList2, DefaultSerializer.deserialize(listResult2.getColumnValue())); } catch (ClassNotFoundException e) { // TODO Auto-generated catch block e.printStackTrace(); } catch (IOException e) { // TODO Auto-generated catch block e.printStackTrace(); } assertEquals("Integer", resultCol2.getColumnValueType()); assertEquals("List<String>", listResult2.getColumnValueType()); assertEquals(false, resultCol2.isIndexable()); assertEquals(false, listResult2.isIndexable()); // // drops the table // session.execute("DROP TABLE " + firstTableName); session.execute("DROP TABLE " + secondTableName); }
From source file:it.polimi.hegira.adapters.cassandra.CassandraAdapterTest.java
License:Apache License
@Test public void fromMyModel() { writer.fromMyModel(null);/*from w w w . ja v a2 s . c o m*/ ArgumentCaptor<byte[]> serializedRow = ArgumentCaptor.forClass(byte[].class); TDeserializer deserializer = new TDeserializer(new TBinaryProtocol.Factory()); reader.toMyModel(null); try { Mockito.verify(mockedTaskQueue, Mockito.times(5)).publish(serializedRow.capture()); } catch (QueueException e) { // TODO Auto-generated catch block e.printStackTrace(); } Metamodel result = new Metamodel(); try { deserializer.deserialize(result, serializedRow.getAllValues().get(4)); } catch (TException e) { // TODO Auto-generated catch block e.printStackTrace(); } assertEquals("ronaldo", result.getRowKey()); assertEquals("@players#ronaldo", result.getPartitionGroup()); assertEquals(result.getColumns().get("players").size(), 2); Column resultCol = result.getColumns().get("players").get(0); Column set = result.getColumns().get("players").get(1); assertEquals("goal", resultCol.getColumnName()); assertEquals("mates", set.getColumnName()); try { assertEquals(30, DefaultSerializer.deserialize(resultCol.getColumnValue())); assertEquals(fakeSet, DefaultSerializer.deserialize(set.getColumnValue())); } catch (ClassNotFoundException e) { // TODO Auto-generated catch block e.printStackTrace(); } catch (IOException e) { // TODO Auto-generated catch block e.printStackTrace(); } assertEquals("Integer", resultCol.getColumnValueType()); assertEquals("Set<String>", set.getColumnValueType()); assertEquals(false, resultCol.isIndexable()); assertEquals(false, set.isIndexable()); session.execute("DROP TABLE players"); }
From source file:it.polimi.hegira.adapters.datastore.Datastore.java
License:Apache License
@Override protected AbstractDatabase fromMyModel(Metamodel mm) { // TWC//from w ww. j a v a 2s. c om //log.debug(Thread.currentThread().getName()+" Hi I'm the GAE consumer!"); List<Entity> batch = new ArrayList<Entity>(); TDeserializer deserializer = new TDeserializer(new TBinaryProtocol.Factory()); long k = 0; int thread_id = 0; if (TWTs_NO != 0) thread_id = (int) (Thread.currentThread().getId() % TWTs_NO); while (true) { log.debug(Thread.currentThread().getName() + " Extracting from the taskQueue" + thread_id + " TWTs_NO: " + TWTs_NO); try { Delivery delivery = taskQueues.get(thread_id).getConsumer().nextDelivery(2000); if (delivery != null) { Metamodel myModel = new Metamodel(); deserializer.deserialize(myModel, delivery.getBody()); DatastoreTransformer dt = new DatastoreTransformer(connectionList.get(thread_id).ds); DatastoreModel fromMyModel = dt.fromMyModel(myModel); batch.add(fromMyModel.getEntity()); batch.add(fromMyModel.getFictitiousEntity()); taskQueues.get(thread_id).sendAck(delivery.getEnvelope().getDeliveryTag()); k++; if (k % 100 == 0) { putBatch(batch); log.debug(Thread.currentThread().getName() + " ===>100 entities. putting normal batch"); batch = new ArrayList<Entity>(); } else { if (k > 0) { //log.debug(Thread.currentThread().getName()+" ===>Nothing in the queue for me!"); putBatch(batch); log.debug(Thread.currentThread().getName() + " ===>less than 100 entities. putting short batch"); batch = new ArrayList<Entity>(); k = 0; } } } } catch (ShutdownSignalException | ConsumerCancelledException | InterruptedException e) { log.error("Error consuming from the queue " + TaskQueue.getDefaultTaskQueueName(), e); } catch (TException e) { log.error("Errore deserializing", e); } catch (QueueException e) { log.error("Couldn't send the ack to the queue " + TaskQueue.getDefaultTaskQueueName(), e); } } }