Example usage for org.apache.thrift TDeserializer TDeserializer

List of usage examples for org.apache.thrift TDeserializer TDeserializer

Introduction

In this page you can find the example usage for org.apache.thrift TDeserializer TDeserializer.

Prototype

public TDeserializer(TProtocolFactory protocolFactory) 

Source Link

Document

Create a new TDeserializer.

Usage

From source file:io.warp10.script.functions.GEOUNPACK.java

License:Apache License

@Override
public Object apply(WarpScriptStack stack) throws WarpScriptException {

    Object o = stack.pop();/* www  . j  a  v a  2  s  . c  o m*/

    if (!(o instanceof String)) {
        throw new WarpScriptException(getName() + " expects a packed shape on top of the stack.");
    }

    byte[] serialized = OrderPreservingBase64.decode(o.toString().getBytes(Charsets.US_ASCII));

    TDeserializer deserializer = new TDeserializer(new TCompactProtocol.Factory());

    GTSWrapper wrapper = new GTSWrapper();

    try {
        deserializer.deserialize(wrapper, serialized);
    } catch (TException te) {
        throw new WarpScriptException(te);
    }

    GTSDecoder decoder = GTSWrapperHelper.fromGTSWrapperToGTSDecoder(wrapper);

    long[] cells = new long[(int) wrapper.getCount()];

    int idx = 0;

    while (idx < cells.length && decoder.next()) {
        long cell = decoder.getTimestamp();
        Object value = decoder.getValue();

        if (!Boolean.TRUE.equals(value)) {
            throw new WarpScriptException(getName() + " invalid GeoXPShape.");
        }

        cells[idx++] = cell;
    }

    if (idx != cells.length) {
        throw new WarpScriptException(getName() + " invalid GeoXPShape.");
    }

    GeoXPShape shape = GeoXPLib.fromCells(cells, false);

    stack.push(shape);

    return stack;
}

From source file:io.warp10.script.functions.TOGTS.java

License:Apache License

@Override
public Object apply(WarpScriptStack stack) throws WarpScriptException {
    Object top = stack.pop();//from  ww  w . j ava  2  s.  c  om

    if (!(top instanceof String) && !(top instanceof byte[]) && !(top instanceof GTSEncoder)) {
        throw new WarpScriptException(getName() + " operates on a string, byte array or encoder.");
    }

    Map<String, GeoTimeSerie> series = new HashMap<String, GeoTimeSerie>();

    GTSDecoder decoder;

    if (top instanceof GTSEncoder) {
        decoder = ((GTSEncoder) top).getUnsafeDecoder(false);
    } else {
        try {
            byte[] bytes = top instanceof String
                    ? OrderPreservingBase64.decode(top.toString().getBytes(Charsets.US_ASCII))
                    : (byte[]) top;

            TDeserializer deser = new TDeserializer(new TCompactProtocol.Factory());

            GTSWrapper wrapper = new GTSWrapper();

            deser.deserialize(wrapper, bytes);

            decoder = GTSWrapperHelper.fromGTSWrapperToGTSDecoder(wrapper);
        } catch (TException te) {
            throw new WarpScriptException(getName() + " failed to unwrap encoder.", te);
        }
    }

    GeoTimeSerie gts;

    while (decoder.next()) {
        Object value = decoder.getValue();

        String type = "DOUBLE";

        if (value instanceof String) {
            type = "STRING";
        } else if (value instanceof Boolean) {
            type = "BOOLEAN";
        } else if (value instanceof Long) {
            type = "LONG";
        } else {
            type = "DOUBLE";
        }

        gts = series.get(type);
        if (null == gts) {
            gts = new GeoTimeSerie();
            gts.setMetadata(decoder.getMetadata());
            series.put(type, gts);
        }

        GTSHelper.setValue(gts, decoder.getTimestamp(), decoder.getLocation(), decoder.getElevation(), value,
                false);
    }

    stack.push(series);

    return stack;
}

From source file:io.warp10.script.functions.UNSECURE.java

License:Apache License

@Override
public Object apply(WarpScriptStack stack) throws WarpScriptException {

    Object o = stack.pop();//w ww.  j a  va  2 s  .com

    if (this.checkkey && null == stack.getAttribute(WarpScriptStack.ATTRIBUTE_SECURE_KEY)) {
        throw new WarpScriptException("You need to set the secure key first.");
    }

    if (!(o instanceof String)) {
        throw new WarpScriptException(getName() + " operates on a string.");
    }

    // Retrieve raw bytes
    byte[] raw = OrderPreservingBase64.decode(o.toString().getBytes(Charsets.US_ASCII));

    // Unwrap

    synchronized (SECURE.class) {
        if (null == aesKey) {
            try {
                aesKey = WarpDist.getKeyStore().getKey(KeyStore.AES_SECURESCRIPTS);
            } catch (Throwable t) {
                // Catch NoClassDefFoundError
            }
        }
    }

    if (null == aesKey) {
        throw new WarpScriptException("Missing secure script encryption key.");
    }

    byte[] unwrapped = CryptoUtils.unwrap(aesKey, raw);

    // Deserialize
    TDeserializer deserializer = new TDeserializer(new TCompactProtocol.Factory());

    SecureScript sscript = new SecureScript();

    try {
        deserializer.deserialize(sscript, unwrapped);
    } catch (TException te) {
        throw new WarpScriptException("Unable to unsecure script.");
    }

    if (this.checkkey) {
        if (!stack.getAttribute(WarpScriptStack.ATTRIBUTE_SECURE_KEY).toString().equals(sscript.getKey())) {
            throw new WarpScriptException("Invalid secure key.");
        }
    }

    // Decompress script content if needed

    if (sscript.isCompressed()) {
        ByteArrayOutputStream baos = new ByteArrayOutputStream();
        ByteArrayInputStream bais = new ByteArrayInputStream(sscript.getScript());

        try {
            GZIPInputStream gzipin = new GZIPInputStream(bais);

            byte[] buf = new byte[128];

            while (true) {
                int len = gzipin.read(buf);
                if (len < 0) {
                    break;
                }
                baos.write(buf, 0, len);
            }

            sscript.setCompressed(false);
            sscript.setScript(baos.toByteArray());
        } catch (IOException ioe) {
            throw new WarpScriptException("Unable to unsecure script.");
        }
    }

    // Convert bytes to String
    String script = new String(sscript.getScript(), Charsets.UTF_8);

    stack.push(script);

    return stack;
}

From source file:io.warp10.script.functions.UNWRAP.java

License:Apache License

@Override
public Object apply(WarpScriptStack stack) throws WarpScriptException {
    Object top = stack.pop();/*from w w w . j  av a 2 s.  c  o m*/

    if (!(top instanceof String) && !(top instanceof byte[]) && !(top instanceof List)) {
        throw new WarpScriptException(getName() + " operates on a string or byte array or a list thereof.");
    }

    List<Object> inputs = new ArrayList<Object>();

    if (top instanceof String || top instanceof byte[]) {
        inputs.add(top);
    } else {
        for (Object o : (List) top) {
            if (!(o instanceof String) && !(o instanceof byte[])) {
                throw new WarpScriptException(
                        getName() + " operates on a string or byte array or a list thereof.");
            }
            inputs.add(o);
        }
    }

    List<Object> outputs = new ArrayList<Object>();

    for (Object s : inputs) {
        byte[] bytes = s instanceof String
                ? OrderPreservingBase64.decode(s.toString().getBytes(Charsets.US_ASCII))
                : (byte[]) s;

        TDeserializer deser = new TDeserializer(new TCompactProtocol.Factory());

        try {
            GTSWrapper wrapper = new GTSWrapper();

            deser.deserialize(wrapper, bytes);

            GeoTimeSerie gts = GTSWrapperHelper.fromGTSWrapperToGTS(wrapper, this.empty);

            outputs.add(gts);
        } catch (TException te) {
            throw new WarpScriptException(getName() + " failed to unwrap GTS.");
        }
    }

    if (!(top instanceof List)) {
        stack.push(outputs.get(0));
    } else {
        stack.push(outputs);
    }

    return stack;
}

From source file:io.warp10.script.functions.UNWRAPENCODER.java

License:Apache License

@Override
public Object apply(WarpScriptStack stack) throws WarpScriptException {
    Object top = stack.pop();// w  ww .  j a  va2s.com

    if (!(top instanceof String) && !(top instanceof byte[])) {
        throw new WarpScriptException(getName() + " operates on a string or byte array.");
    }

    byte[] bytes = top instanceof String
            ? OrderPreservingBase64.decode(top.toString().getBytes(Charsets.US_ASCII))
            : (byte[]) top;

    TDeserializer deser = new TDeserializer(new TCompactProtocol.Factory());

    try {
        GTSWrapper wrapper = new GTSWrapper();

        deser.deserialize(wrapper, bytes);

        GTSDecoder decoder = GTSWrapperHelper.fromGTSWrapperToGTSDecoder(wrapper);

        decoder.next();

        stack.push(decoder.getEncoder(true));
    } catch (TException te) {
        throw new WarpScriptException(getName() + " failed to unwrap encoder.", te);
    } catch (IOException ioe) {
        throw new WarpScriptException(getName() + " failed to unwrap encoder.", ioe);
    }

    return stack;
}

From source file:io.warp10.script.functions.UNWRAPSIZE.java

License:Apache License

@Override
public Object apply(WarpScriptStack stack) throws WarpScriptException {
    Object top = stack.pop();/* w  w w.  j  a va  2s. com*/

    if (!(top instanceof String) && !(top instanceof byte[]) && !(top instanceof List)) {
        throw new WarpScriptException(getName() + " operates on a string or byte array or a list thereof.");
    }

    List<Object> inputs = new ArrayList<Object>();

    if (top instanceof String || top instanceof byte[]) {
        inputs.add(top);
    } else {
        for (Object o : (List) top) {
            if (!(o instanceof String) && !(o instanceof byte[])) {
                throw new WarpScriptException(
                        getName() + " operates on a string or byte array or a list thereof.");
            }
            inputs.add(o);
        }
    }

    List<Object> outputs = new ArrayList<Object>();

    for (Object s : inputs) {
        byte[] bytes = s instanceof String
                ? OrderPreservingBase64.decode(s.toString().getBytes(Charsets.US_ASCII))
                : (byte[]) s;

        TDeserializer deser = new TDeserializer(new TCompactProtocol.Factory());

        try {
            GTSWrapper wrapper = new GTSWrapper();

            deser.deserialize(wrapper, bytes);

            outputs.add(wrapper.getCount());
        } catch (TException te) {
            throw new WarpScriptException(getName() + " failed to unwrap GTS.");
        }
    }

    if (!(top instanceof List)) {
        stack.push(outputs.get(0));
    } else {
        stack.push(outputs);
    }

    return stack;
}

From source file:io.warp10.script.HyperLogLogPlus.java

License:Apache License

public static HyperLogLogPlus fromBytes(byte[] bytes) throws IOException, ClassNotFoundException {

    TDeserializer deserializer = new TDeserializer(new TCompactProtocol.Factory());

    HyperLogLogPlusParameters params = new HyperLogLogPlusParameters();

    try {/*from  w  w w.j  av  a2s . c  o  m*/
        deserializer.deserialize(params, bytes);
    } catch (TException te) {
        throw new IOException(te);
    }

    HyperLogLogPlus hllp = new HyperLogLogPlus();

    hllp.setInitTime(params.getInitTime());

    if (params.isSetKey()) {
        hllp.setKey(params.getKey());
    }

    // Read p
    hllp.p = params.getP();
    hllp.m = 1 << hllp.p;
    // Read p'
    hllp.pprime = params.getPprime();
    hllp.mprime = 1 << hllp.pprime;

    hllp._64minusp = 64 - hllp.p;
    hllp.pmask = (1L << hllp._64minusp) - 1;
    hllp._64minuspprime = 64 - hllp.pprime;
    hllp.ptopprimemask = ((1L << hllp._64minusp) - 1) ^ ((1L << hllp._64minuspprime) - 1);
    hllp.pprimemask = ((1L << hllp._64minuspprime) - 1);

    // Read the current mode
    hllp.format = params.isSparse() ? Format.SPARSE : Format.NORMAL;

    if (Format.SPARSE == hllp.format) {
        hllp.sparse_list_len = params.getSparseListLen();
        hllp.sparse_list = params.getSparseList();
        // Allocate tmp_set
        hllp.tmp_set = new int[(int) Math.ceil((hllp.m * 6) / 8)];
        hllp.tmp_set_idx = 0;
    } else {
        // Read the registers
        if (params.isGzipped()) {
            ByteArrayInputStream bais = new ByteArrayInputStream(params.getRegisters());
            GZIPInputStream gzis = new GZIPInputStream(bais);
            ByteArrayOutputStream baos = new ByteArrayOutputStream();
            byte[] buf = new byte[1024];
            while (true) {
                int len = gzis.read(buf);

                if (len < 0) {
                    break;
                }
                baos.write(buf, 0, len);
            }
            gzis.close();
            hllp.M = baos.toByteArray();
        } else {
            hllp.M = params.getRegisters();
        }
    }

    return hllp;
}

From source file:io.warp10.script.ScriptRunnerConsumerFactory.java

License:Apache License

@Override
public Runnable getConsumer(final KafkaSynchronizedConsumerPool pool,
        final KafkaStream<byte[], byte[]> stream) {

    return new Runnable() {
        @Override//from  w ww .j a  va  2s  .  c  o  m
        public void run() {
            ConsumerIterator<byte[], byte[]> iter = stream.iterator();

            // Iterate on the messages
            TDeserializer deserializer = new TDeserializer(new TCompactProtocol.Factory());

            KafkaOffsetCounters counters = pool.getCounters();

            try {
                while (iter.hasNext()) {
                    //
                    // Since the call to 'next' may block, we need to first
                    // check that there is a message available
                    //

                    boolean nonEmpty = iter.nonEmpty();

                    if (nonEmpty) {
                        MessageAndMetadata<byte[], byte[]> msg = iter.next();
                        counters.count(msg.partition(), msg.offset());

                        byte[] data = msg.message();

                        Sensision.update(SensisionConstants.SENSISION_CLASS_WARP_RUNNER_KAFKA_IN_MESSAGES,
                                Sensision.EMPTY_LABELS, 1);
                        Sensision.update(SensisionConstants.SENSISION_CLASS_WARP_RUNNER_KAFKA_IN_BYTES,
                                Sensision.EMPTY_LABELS, data.length);

                        if (null != runner.KAFKA_MAC) {
                            data = CryptoUtils.removeMAC(runner.KAFKA_MAC, data);
                        }

                        // Skip data whose MAC was not verified successfully
                        if (null == data) {
                            Sensision.update(
                                    SensisionConstants.SENSISION_CLASS_WARP_RUNNER_KAFKA_IN_INVALIDMACS,
                                    Sensision.EMPTY_LABELS, 1);
                            continue;
                        }

                        // Unwrap data if need be
                        if (null != runner.KAFKA_AES) {
                            data = CryptoUtils.unwrap(runner.KAFKA_AES, data);
                        }

                        // Skip data that was not unwrapped successfuly
                        if (null == data) {
                            Sensision.update(
                                    SensisionConstants.SENSISION_CLASS_WARP_RUNNER_KAFKA_IN_INVALIDCIPHERS,
                                    Sensision.EMPTY_LABELS, 1);
                            continue;
                        }

                        final RunRequest request = new RunRequest();

                        deserializer.deserialize(request, data);

                        //
                        // Check if running is overdue
                        //

                        long now = System.currentTimeMillis();

                        if (request.getScheduledAt() + request.getPeriodicity() >= now) {
                            continue;
                        }

                        //
                        // Decompress script if it is compressed
                        //

                        if (request.isCompressed()) {
                            ByteArrayOutputStream out = new ByteArrayOutputStream();
                            GZIPInputStream in = new GZIPInputStream(
                                    new ByteArrayInputStream(request.getContent()));

                            byte[] buf = new byte[8192];

                            while (true) {
                                int len = in.read(buf);

                                if (len <= 0) {
                                    break;
                                }

                                out.write(buf, 0, len);
                            }

                            in.close();
                            out.close();

                            request.setContent(out.toByteArray());
                        }

                        //
                        // Submit script for execution, do up to 3 attempts
                        //

                        int attempts = 3;

                        while (attempts > 0) {
                            try {
                                runner.executor.submit(new Runnable() {
                                    @Override
                                    public void run() {

                                        Sensision.update(
                                                SensisionConstants.SENSISION_CLASS_EINSTEIN_RUN_CURRENT,
                                                Sensision.EMPTY_LABELS, 1);

                                        Map<String, String> labels = new HashMap<String, String>();
                                        labels.put(SensisionConstants.SENSISION_LABEL_PATH, request.getPath());

                                        Sensision.update(SensisionConstants.SENSISION_CLASS_EINSTEIN_RUN_COUNT,
                                                labels, 1);

                                        long nano = System.nanoTime();

                                        HttpURLConnection conn = null;

                                        long ops = 0;
                                        long fetched = 0;
                                        long elapsed = 0;

                                        try {
                                            conn = (HttpURLConnection) new URL(runner.endpoint)
                                                    .openConnection();

                                            conn.setDoOutput(true);
                                            conn.setChunkedStreamingMode(8192);
                                            conn.setDoInput(true);
                                            conn.setRequestMethod("POST");

                                            conn.connect();

                                            OutputStream out = conn.getOutputStream();

                                            //
                                            // Push the script parameters
                                            //

                                            out.write(Long.toString(request.getPeriodicity())
                                                    .getBytes(Charsets.UTF_8));
                                            out.write(' ');
                                            out.write('\'');
                                            out.write(URLEncoder.encode(Constants.RUNNER_PERIODICITY, "UTF-8")
                                                    .replaceAll("\\+", "%20").getBytes(Charsets.US_ASCII));
                                            out.write('\'');
                                            out.write(' ');
                                            out.write(WarpScriptLib.STORE.getBytes(Charsets.UTF_8));
                                            out.write('\n');

                                            out.write('\'');
                                            out.write(URLEncoder.encode(request.getPath(), "UTF-8")
                                                    .replaceAll("\\+", "%20").getBytes(Charsets.US_ASCII));
                                            out.write('\'');
                                            out.write(' ');
                                            out.write('\'');
                                            out.write(URLEncoder.encode(Constants.RUNNER_PATH, "UTF-8")
                                                    .replaceAll("\\+", "%20").getBytes(Charsets.US_ASCII));
                                            out.write('\'');
                                            out.write(' ');
                                            out.write(WarpScriptLib.STORE.getBytes(Charsets.UTF_8));
                                            out.write('\n');

                                            out.write(Long.toString(request.getScheduledAt())
                                                    .getBytes(Charsets.UTF_8));
                                            out.write(' ');
                                            out.write('\'');
                                            out.write(URLEncoder.encode(Constants.RUNNER_SCHEDULEDAT, "UTF-8")
                                                    .replaceAll("\\+", "%20").getBytes(Charsets.US_ASCII));
                                            out.write('\'');
                                            out.write(' ');
                                            out.write(WarpScriptLib.STORE.getBytes(Charsets.UTF_8));
                                            out.write('\n');

                                            byte[] data = request.getContent();

                                            if (request.isCompressed()) {
                                                ByteArrayInputStream bais = new ByteArrayInputStream(data);
                                                GZIPInputStream gzis = new GZIPInputStream(bais);
                                                byte[] buf = new byte[1024];

                                                while (true) {
                                                    int len = bais.read(buf);

                                                    if (len < 0) {
                                                        break;
                                                    }

                                                    out.write(buf, 0, len);
                                                }

                                                gzis.close();
                                            } else {
                                                out.write(data, 0, data.length);
                                            }

                                            // Add a 'CLEAR' at the end of the script so we don't return anything
                                            out.write(runner.CLEAR);

                                            out.close();

                                            if (200 != conn.getResponseCode()) {
                                                Sensision.update(
                                                        SensisionConstants.SENSISION_CLASS_EINSTEIN_RUN_FAILURES,
                                                        labels, 1);
                                            }

                                            String header = conn.getRequestProperty(
                                                    Constants.getHeader(Configuration.HTTP_HEADER_ELAPSEDX));
                                            if (null != header) {
                                                try {
                                                    elapsed = Long.parseLong(header);
                                                } catch (Exception e) {
                                                }
                                            }
                                            header = conn.getRequestProperty(
                                                    Constants.getHeader(Configuration.HTTP_HEADER_OPSX));
                                            if (null != header) {
                                                try {
                                                    ops = Long.parseLong(header);
                                                } catch (Exception e) {
                                                }
                                            }
                                            header = conn.getRequestProperty(
                                                    Constants.getHeader(Configuration.HTTP_HEADER_FETCHEDX));
                                            if (null != header) {
                                                try {
                                                    fetched = Long.parseLong(header);
                                                } catch (Exception e) {
                                                }
                                            }

                                        } catch (Exception e) {
                                            Sensision.update(
                                                    SensisionConstants.SENSISION_CLASS_EINSTEIN_RUN_FAILURES,
                                                    labels, 1);
                                        } finally {
                                            nano = System.nanoTime() - nano;
                                            Sensision.update(
                                                    SensisionConstants.SENSISION_CLASS_EINSTEIN_RUN_TIME_US,
                                                    labels, (long) (nano / 1000L));
                                            Sensision.update(
                                                    SensisionConstants.SENSISION_CLASS_EINSTEIN_RUN_ELAPSED,
                                                    labels, elapsed);
                                            Sensision.update(
                                                    SensisionConstants.SENSISION_CLASS_EINSTEIN_RUN_FETCHED,
                                                    labels, fetched);
                                            Sensision.update(
                                                    SensisionConstants.SENSISION_CLASS_EINSTEIN_RUN_OPS, labels,
                                                    ops);
                                            Sensision.update(
                                                    SensisionConstants.SENSISION_CLASS_EINSTEIN_RUN_CURRENT,
                                                    Sensision.EMPTY_LABELS, -1);
                                            if (null != conn) {
                                                conn.disconnect();
                                            }
                                        }
                                    }
                                });
                                break;
                            } catch (RejectedExecutionException ree) {
                                // Reschedule script immediately
                                Sensision.update(SensisionConstants.SENSISION_CLASS_WARP_RUNNER_REJECTIONS,
                                        Sensision.EMPTY_LABELS, 1);
                                attempts--;
                            }
                        }

                        if (0 == attempts) {
                            Sensision.update(SensisionConstants.SENSISION_CLASS_WARP_RUNNER_FAILURES,
                                    Sensision.EMPTY_LABELS, 1);
                        }
                    }
                }
            } catch (Throwable t) {
                t.printStackTrace(System.err);
            } finally {
                // Set abort to true in case we exit the 'run' method
                pool.getAbort().set(true);
            }
        }
    };
}

From source file:io.warp10.standalone.Migrate.java

License:Apache License

public static void main(String[] args) throws Exception {
    ////from   w  w  w . j av  a  2 s  .  c om
    // Initialize keys
    //

    byte[] metaKey = Hex.decode("1111111111111111111111111111111111111111111111111111111111111111");
    byte[] classIdKey = Hex.decode("88888888888888888888888888888888");
    byte[] labelsIdKey = Hex.decode("99999999999999999999999999999999");

    String INDB = "/var/tmp/continuum-orig";
    String OUTDB = "/var/tmp/continuum-converted";

    //
    // Open source/target DBs
    //

    Options options = new Options();
    options.createIfMissing(true);
    options.cacheSize(100000000L);
    options.compressionType(CompressionType.SNAPPY);

    DB indb;

    try {
        indb = JniDBFactory.factory.open(new File(INDB), options);
    } catch (UnsatisfiedLinkError ule) {
        System.out.println("WARNING: falling back to pure java implementation of LevelDB.");
        indb = Iq80DBFactory.factory.open(new File(INDB), options);
    }

    DB outdb;

    try {
        outdb = JniDBFactory.factory.open(new File(OUTDB), options);
    } catch (UnsatisfiedLinkError ule) {
        System.out.println("WARNING: falling back to pure java implementation of LevelDB.");
        outdb = Iq80DBFactory.factory.open(new File(OUTDB), options);
    }

    //
    // Read/Update metadata
    //

    DBIterator iter = indb.iterator();

    // Seek start of metadata
    iter.seek("M".getBytes("UTF-8"));

    Map<BigInteger, byte[]> metadatas = new HashMap<BigInteger, byte[]>();

    TDeserializer deserializer = new TDeserializer(new TCompactProtocol.Factory());
    TSerializer serializer = new TSerializer(new TCompactProtocol.Factory());

    long nmeta = 0;

    long nano = System.nanoTime();

    while (iter.hasNext()) {
        Entry<byte[], byte[]> entry = iter.next();

        // Exit when done with metadata
        if (entry.getKey()[0] != 'M') {
            break;
        }

        byte[] clslblids = Arrays.copyOfRange(entry.getKey(), 1, 17);

        BigInteger bi = new BigInteger(clslblids);

        Metadata metadata = new Metadata();

        deserializer.deserialize(metadata, CryptoUtils.unwrap(metaKey, entry.getValue()));

        //
        // Compute new class/labels id
        //

        metadata.setClassId(GTSHelper.classId(classIdKey, metadata.getName()));
        metadata.setLabelsId(GTSHelper.labelsId(labelsIdKey, metadata.getLabels()));

        byte[] value = CryptoUtils.wrap(metaKey, serializer.serialize(metadata));
        byte[] key = new byte[17];

        ByteBuffer bb = ByteBuffer.wrap(key).order(ByteOrder.BIG_ENDIAN);
        bb.put((byte) 'M');
        bb.putLong(metadata.getClassId());
        bb.putLong(metadata.getLabelsId());

        outdb.put(key, value);

        metadatas.put(bi, Arrays.copyOfRange(key, 1, 17));
        nmeta++;
    }

    System.out
            .println("Updated " + nmeta + " metadatas in " + ((System.nanoTime() - nano) / 1000000.0D) + " ms");

    //
    // Read/Store readings
    //

    iter.seek("R".getBytes("UTF-8"));

    long ndata = 0;

    nano = System.nanoTime();

    while (iter.hasNext()) {
        Entry<byte[], byte[]> entry = iter.next();

        if (entry.getKey()[0] != 'R') {
            break;
        }

        byte[] clslblids = Arrays.copyOfRange(entry.getKey(), 1, 17);

        BigInteger bi = new BigInteger(clslblids);

        if (!metadatas.containsKey(bi)) {
            System.out.println("No metadata found for " + new String(Hex.encode(entry.getKey())));
            continue;
        }

        byte[] newkey = Arrays.copyOf(entry.getKey(), entry.getKey().length);
        System.arraycopy(metadatas.get(bi), 0, newkey, 1, 16);

        outdb.put(newkey, entry.getValue());

        ndata++;
    }

    System.out
            .println("Updated " + ndata + " readings in " + ((System.nanoTime() - nano) / 1000000.0D) + " ms");

    indb.close();
    outdb.close();
}

From source file:io.warp10.standalone.StandaloneChunkedMemoryStore.java

License:Apache License

private void load(String path) throws IOException {

    long nano = System.nanoTime();
    int gts = 0;/*from  w  ww.j  a  va 2  s.c  o m*/
    long bytes = 0L;

    Configuration conf = new Configuration();

    conf.set("fs.hdfs.impl", org.apache.hadoop.hdfs.DistributedFileSystem.class.getName());
    conf.set("fs.file.impl", org.apache.hadoop.fs.LocalFileSystem.class.getName());

    BytesWritable key = new BytesWritable();
    BytesWritable value = new BytesWritable();

    TDeserializer deserializer = new TDeserializer(new TCompactProtocol.Factory());

    SequenceFile.Reader.Option optPath = SequenceFile.Reader.file(new Path(path));

    SequenceFile.Reader reader = null;

    boolean failsafe = "true".equals(
            properties.getProperty(io.warp10.continuum.Configuration.STANDALONE_MEMORY_STORE_LOAD_FAILSAFE));

    try {
        reader = new SequenceFile.Reader(conf, optPath);

        System.out.println("Loading '" + path + "' back in memory.");

        while (reader.next(key, value)) {
            gts++;
            GTSWrapper wrapper = new GTSWrapper();
            deserializer.deserialize(wrapper, key.copyBytes());
            GTSEncoder encoder = new GTSEncoder(0L, null, value.copyBytes());
            encoder.setCount(wrapper.getCount());

            bytes += value.getLength() + key.getLength();
            encoder.safeSetMetadata(wrapper.getMetadata());
            store(encoder);
            if (null != this.directoryClient) {
                this.directoryClient.register(wrapper.getMetadata());
            }
        }
    } catch (FileNotFoundException fnfe) {
        System.err.println("File '" + path + "' was not found, skipping.");
        return;
    } catch (IOException ioe) {
        if (!failsafe) {
            throw ioe;
        } else {
            System.err.println("Ignoring exception " + ioe.getMessage() + ".");
        }
    } catch (Exception e) {
        if (!failsafe) {
            throw new IOException(e);
        } else {
            System.err.println("Ignoring exception " + e.getMessage() + ".");
        }
    }

    reader.close();

    nano = System.nanoTime() - nano;

    System.out.println("Loaded " + gts + " GTS (" + bytes + " bytes) in " + (nano / 1000000.0D) + " ms.");
}