Example usage for org.json.simple JSONObject containsKey

List of usage examples for org.json.simple JSONObject containsKey

  1. HOME
  2. Java
  3. org.json
  4. org.json.simple.*
  5. JSONObject
  6. containsKey

Introduction

In this page you can find the example usage for org.json.simple JSONObject containsKey.

Prototype

boolean containsKey(Object key);

Source Link

Document

Returns true if this map contains a mapping for the specified key.

Usage

From source file:org.apache.metron.alerts.adapters.HbaseWhiteAndBlacklistAdapter.java

@SuppressWarnings("unchecked")
public Map<String, JSONObject> alert(JSONObject raw_message) {

    Map<String, JSONObject> alerts = new HashMap<String, JSONObject>();

    JSONObject content = (JSONObject) raw_message.get("message");
    JSONObject enrichment = null;/*from  w w  w. j  av  a  2  s . c  o m*/

    if (raw_message.containsKey("enrichment"))
        enrichment = (JSONObject) raw_message.get("enrichment");

    if (!content.containsKey("ip_src_addr") || !content.containsKey("ip_dst_addr")) {

        int alert_type = 0;

        JSONObject alert = new JSONObject();

        alert.put("title", "IP Check Error Type: : " + alert_type);
        alert.put("priority", "1");
        alert.put("type", "error");
        alert.put("designated_host", "Uknown");
        alert.put("source", "NA");
        alert.put("dest", "NA");
        alert.put("body", "Source or destination IP is missing");

        String alert_id = UUID.randomUUID().toString();

        alert.put("reference_id", alert_id);
        alerts.put(alert_id, alert);

        if (enrichment != null)
            alert.put("enrichment", enrichment);

        LOG.trace("[Metron] Returning alert: " + alerts);

        return alerts;

    }

    String source_ip = content.get("ip_src_addr").toString();
    String dst_ip = content.get("ip_dst_addr").toString();

    if (source_ip == null && dst_ip == null) {

        int alert_type = 1;

        JSONObject alert = new JSONObject();

        alert.put("title", "IP Check Error Type: : " + alert_type);
        alert.put("priority", "1");
        alert.put("type", "error");
        alert.put("designated_host", "Uknown");
        alert.put("source", source_ip);
        alert.put("dest", dst_ip);
        alert.put("body",
                "This communication does not contain a source or destination IP string. Communication between two IPs: "
                        + source_ip + " -> " + dst_ip);

        String alert_id = generateAlertId(source_ip, dst_ip, alert_type);

        alert.put("reference_id", alert_id);
        alerts.put(alert_id, alert);
        if (enrichment != null)
            alert.put("enrichment", enrichment);

        LOG.trace("[Metron] Returning alert: " + alerts);

        return alerts;

    }

    if (!ipvalidator.isValidInet4Address(source_ip) && !ipvalidator.isValidInet4Address(dst_ip)) {
        int alert_type = 2;

        JSONObject alert = new JSONObject();

        alert.put("title", "IP Check Error Type: : " + alert_type);
        alert.put("priority", "1");
        alert.put("type", "error");
        alert.put("designated_host", "Uknown");
        alert.put("source", source_ip);
        alert.put("dest", dst_ip);
        alert.put("content",
                "This communication contains souce and destination IP strings, but these strings are not valid. Communication between two IPs: "
                        + source_ip + " -> " + dst_ip);

        String alert_id = generateAlertId(source_ip, dst_ip, alert_type);

        alert.put("reference_id", alert_id);
        alerts.put(alert_id, alert);
        if (enrichment != null)
            alert.put("enrichment", enrichment);

        LOG.trace("[Metron] Returning alert: " + alerts);

        return alerts;

    }

    String designated_host = null;

    if (loaded_whitelist.contains(source_ip))
        designated_host = source_ip;
    else if (loaded_whitelist.contains(dst_ip))
        designated_host = dst_ip;

    if (designated_host == null) {
        int alert_type = 3;

        JSONObject alert = new JSONObject();

        alert.put("title", "IP Check Error Type: : " + alert_type);
        alert.put("priority", "1");
        alert.put("type", "error");
        alert.put("designated_host", "Uknown");
        alert.put("source", source_ip);
        alert.put("dest", dst_ip);
        alert.put("content",
                "This communication does not contain a source or a destination IP that is in the white list. Communication between two IPs: "
                        + source_ip + " -> " + dst_ip);

        String alert_id = generateAlertId(source_ip, dst_ip, alert_type);

        alert.put("reference_id", alert_id);
        alerts.put(alert_id, alert);
        if (enrichment != null)
            alert.put("enrichment", enrichment);

        LOG.trace("[Metron] Returning alert: " + alerts);

        return alerts;

    }

    if (source_ip.equals(designated_host) && !ipvalidator.isValidInet4Address(dst_ip)) {
        int alert_type = 4;

        JSONObject alert = new JSONObject();

        alert.put("title", "IP Check Error Type: : " + alert_type);
        alert.put("priority", "1");
        alert.put("type", "error");
        alert.put("designated_host", designated_host);
        alert.put("source", source_ip);
        alert.put("dest", dst_ip);
        alert.put("content",
                "This communication contains an IP that is not valid. Communication between two IPs: "
                        + source_ip + " -> " + dst_ip);

        String alert_id = generateAlertId(source_ip, dst_ip, alert_type);

        alert.put("reference_id", alert_id);
        alerts.put(alert_id, alert);
        if (enrichment != null)
            alert.put("enrichment", enrichment);

    }

    if (dst_ip.equals(designated_host) && !ipvalidator.isValidInet4Address(source_ip)) {
        int alert_type = 5;

        JSONObject alert = new JSONObject();

        alert.put("title", "IP Check Error Type: : " + alert_type);
        alert.put("priority", "1");
        alert.put("type", "error");
        alert.put("designated_host", designated_host);
        alert.put("source", source_ip);
        alert.put("dest", dst_ip);
        alert.put("content", "This communication contains IP that is not valid. Communication between two IPs: "
                + source_ip + " -> " + dst_ip);

        String alert_id = generateAlertId(source_ip, dst_ip, alert_type);

        alert.put("reference_id", alert_id);
        alerts.put(alert_id, alert);
        if (enrichment != null)
            alert.put("enrichment", enrichment);

    }

    if (loaded_blacklist.contains(source_ip)) {
        int alert_type = 6;

        JSONObject alert = new JSONObject();

        alert.put("title", "IP Check Error Type: : " + alert_type);
        alert.put("priority", "1");
        alert.put("type", "error");
        alert.put("designated_host", designated_host);
        alert.put("source", source_ip);
        alert.put("dest", dst_ip);
        alert.put("content",
                "This communication contains IP that is black listed. Communication between two IPs: "
                        + source_ip + " -> " + dst_ip);

        String alert_id = generateAlertId(source_ip, dst_ip, alert_type);

        alert.put("reference_id", alert_id);
        alerts.put(alert_id, alert);
        if (enrichment != null)
            alert.put("enrichment", enrichment);

    }

    if (loaded_blacklist.contains(dst_ip)) {
        int alert_type = 7;

        JSONObject alert = new JSONObject();

        alert.put("title", "IP Check Error Type: : " + alert_type);
        alert.put("priority", "1");
        alert.put("type", "error");
        alert.put("designated_host", designated_host);
        alert.put("source", source_ip);
        alert.put("dest", dst_ip);
        alert.put("content",
                "This communication contains IP that is black listed. Communication between two IPs: "
                        + source_ip + " -> " + dst_ip);

        String alert_id = generateAlertId(source_ip, dst_ip, alert_type);

        alert.put("reference_id", alert_id);
        alerts.put(alert_id, alert);
        if (enrichment != null)
            alert.put("enrichment", enrichment);

    }

    if (alerts.isEmpty())
        return null;
    else
        return alerts;
}

From source file:org.apache.metron.alerts.adapters.KeywordsAlertAdapter.java

@Override
public Map<String, JSONObject> alert(JSONObject raw_message) {

    Map<String, JSONObject> alerts = new HashMap<String, JSONObject>();
    JSONObject content = (JSONObject) raw_message.get("message");

    JSONObject enrichment = null;//  ww w. ja v a2 s  . c  o m
    if (raw_message.containsKey("enrichment"))
        enrichment = (JSONObject) raw_message.get("enrichment");

    for (String keyword : keywordList) {
        if (content.toString().contains(keyword)) {

            //check it doesn't have an "exception" keyword in it
            for (String exception : keywordExceptionList) {
                if (content.toString().contains(exception)) {
                    LOG.info("[Metron] KeywordAlertsAdapter: Omitting alert due to exclusion: " + exception);
                    return null;
                }
            }

            LOG.info("[Metron] KeywordAlertsAdapter: Found match for " + keyword);
            JSONObject alert = new JSONObject();

            String source = "unknown";
            String dest = "unknown";
            String host = "unknown";

            if (content.containsKey("ip_src_addr")) {
                source = content.get("ip_src_addr").toString();

                if (RangeChecker.checkRange(loaded_whitelist, source))
                    host = source;
            }

            if (content.containsKey("ip_dst_addr")) {
                dest = content.get("ip_dst_addr").toString();

                if (RangeChecker.checkRange(loaded_whitelist, dest))
                    host = dest;
            }

            alert.put("designated_host", host);
            alert.put("description", content.get("original_string").toString());
            alert.put("priority", "MED");

            String alert_id = generateAlertId(source, dest, 0);

            alert.put("alert_id", alert_id);
            alerts.put(alert_id, alert);

            alert.put("enrichment", enrichment);

            return alerts;
        }
    }

    return null;
}

From source file:org.apache.metron.alerts.adapters.ThreatAlertsAdapter.java

@SuppressWarnings("unchecked")
@Override/*www.  j av  a2  s . c o  m*/
public Map<String, JSONObject> alert(JSONObject raw_message) {

    System.out.println("LOOKING FOR ENRICHMENT TAG: " + enrichment_tag);

    Map<String, JSONObject> alerts = new HashMap<String, JSONObject>();
    JSONObject content = (JSONObject) raw_message.get("message");

    JSONObject enrichment = null;

    if (raw_message.containsKey("enrichment"))
        enrichment = (JSONObject) raw_message.get("enrichment");
    else
        return null;

    if (enrichment.containsKey(enrichment_tag)) {

        System.out.println("FOUND TAG: " + enrichment_tag);

        JSONObject threat = (JSONObject) enrichment.get(enrichment_tag);

        int cnt = 0;
        Object enriched_key = null;

        for (Object key : threat.keySet()) {
            JSONObject tmp = (JSONObject) threat.get(key);
            cnt = cnt + tmp.size();
            if (tmp.size() > 0)
                enriched_key = key;
        }

        if (cnt == 0) {
            System.out.println("TAG HAS NO ELEMENTS");
            return null;
        }

        JSONObject alert = new JSONObject();

        String source = "unknown";
        String dest = "unknown";
        String host = "unknown";

        if (content.containsKey("ip_src_addr")) {
            source = content.get("ip_src_addr").toString();

            if (RangeChecker.checkRange(loaded_whitelist, source))
                host = source;
        }

        if (content.containsKey("ip_dst_addr")) {
            dest = content.get("ip_dst_addr").toString();

            if (RangeChecker.checkRange(loaded_whitelist, dest))
                host = dest;
        }

        JSONObject threatQualifier = (JSONObject) threat.get(enriched_key);

        alert.put("designated_host", host);
        String description =

                new StringBuilder().append("Threat Intelligence match for ")
                        .append(content.get(enriched_key).toString()).append(" from source: ")
                        .append(threatQualifier.keySet().iterator().next().toString()).toString();
        alert.put("description", description);
        alert.put("priority", "MED");

        String alert_id = generateAlertId(source, dest, 0);

        alert.put("alert_id", alert_id);
        alerts.put(alert_id, alert);

        alert.put("enrichment", enrichment);

        return alerts;
    } else {
        System.out.println("DID NOT FIND TAG: " + enrichment_tag);
        return null;
    }

}

From source file:org.apache.metron.alerts.TelemetryAlertsBolt.java

@SuppressWarnings("unchecked")
public void execute(Tuple tuple) {

    LOG.trace("[Metron] Starting to process message for alerts");
    JSONObject original_message = null;
    String key = null;/*www . ja  v  a2s.  c  o m*/

    try {

        key = tuple.getStringByField("key");
        original_message = (JSONObject) tuple.getValueByField("message");

        if (original_message == null || original_message.isEmpty())
            throw new Exception("Could not parse message from byte stream");

        if (key == null)
            throw new Exception("Key is not valid");

        LOG.trace("[Metron] Received tuple: " + original_message);

        JSONObject alerts_tag = new JSONObject();
        Map<String, JSONObject> alerts_list = _adapter.alert(original_message);
        JSONArray uuid_list = new JSONArray();

        if (alerts_list == null || alerts_list.isEmpty()) {
            System.out.println("[Metron] No alerts detected in: " + original_message);
            _collector.ack(tuple);
            _collector.emit("message", new Values(key, original_message));
        } else {
            for (String alert : alerts_list.keySet()) {
                uuid_list.add(alert);

                LOG.trace("[Metron] Checking alerts cache: " + alert);

                if (cache.getIfPresent(alert) == null) {
                    System.out.println("[Metron]: Alert not found in cache: " + alert);

                    JSONObject global_alert = new JSONObject();
                    global_alert.putAll(_identifier);
                    global_alert.putAll(alerts_list.get(alert));
                    global_alert.put("timestamp", System.currentTimeMillis());
                    _collector.emit("alert", new Values(global_alert));

                    cache.put(alert, "");

                } else
                    LOG.trace("Alert located in cache: " + alert);

                LOG.debug("[Metron] Alerts are: " + alerts_list);

                if (original_message.containsKey("alerts")) {
                    JSONArray already_triggered = (JSONArray) original_message.get("alerts");

                    uuid_list.addAll(already_triggered);
                    LOG.trace("[Metron] Messages already had alerts...tagging more");
                }

                original_message.put("alerts", uuid_list);

                LOG.debug("[Metron] Detected alerts: " + alerts_tag);

                _collector.ack(tuple);
                _collector.emit("message", new Values(key, original_message));

            }

            /*
             * if (metricConfiguration != null) { emitCounter.inc();
             * ackCounter.inc(); }
             */
        }

    } catch (Exception e) {
        e.printStackTrace();
        LOG.error("Failed to tag message :" + original_message);
        e.printStackTrace();
        _collector.fail(tuple);

        /*
         * if (metricConfiguration != null) { failCounter.inc(); }
         */

        JSONObject error = ErrorUtils.generateErrorMessage("Alerts problem: " + original_message, e);
        _collector.emit("error", new Values(error));
    }
}

From source file:org.apache.metron.enrichment.bolt.GenericEnrichmentBolt.java

@SuppressWarnings("unchecked")
@Override//from   www.ja  v a 2  s  .  com
public void execute(Tuple tuple) {
    String key = tuple.getStringByField("key");
    JSONObject rawMessage = (JSONObject) tuple.getValueByField("message");
    String subGroup = "";

    JSONObject enrichedMessage = new JSONObject();
    enrichedMessage.put("adapter." + adapter.getClass().getSimpleName().toLowerCase() + ".begin.ts",
            "" + System.currentTimeMillis());
    try {
        if (rawMessage == null || rawMessage.isEmpty())
            throw new Exception("Could not parse binary stream to JSON");
        if (key == null)
            throw new Exception("Key is not valid");
        String sourceType = null;
        if (rawMessage.containsKey(Constants.SENSOR_TYPE)) {
            sourceType = rawMessage.get(Constants.SENSOR_TYPE).toString();
        } else {
            throw new RuntimeException(
                    "Source type is missing from enrichment fragment: " + rawMessage.toJSONString());
        }
        boolean error = false;
        String prefix = null;
        for (Object o : rawMessage.keySet()) {
            String field = (String) o;
            Object value = rawMessage.get(field);
            if (field.equals(Constants.SENSOR_TYPE)) {
                enrichedMessage.put(Constants.SENSOR_TYPE, value);
            } else {
                JSONObject enrichedField = new JSONObject();
                if (value != null) {
                    SensorEnrichmentConfig config = getConfigurations().getSensorEnrichmentConfig(sourceType);
                    if (config == null) {
                        LOG.error("Unable to find SensorEnrichmentConfig for sourceType: " + sourceType);
                        error = true;
                        continue;
                    }
                    config.getConfiguration().putIfAbsent(STELLAR_CONTEXT_CONF, stellarContext);
                    CacheKey cacheKey = new CacheKey(field, value, config);
                    try {
                        adapter.logAccess(cacheKey);
                        prefix = adapter.getOutputPrefix(cacheKey);
                        subGroup = adapter.getStreamSubGroup(enrichmentType, field);
                        enrichedField = cache.getUnchecked(cacheKey);
                        if (enrichedField == null)
                            throw new Exception("[Metron] Could not enrich string: " + value);
                    } catch (Exception e) {
                        LOG.error(e.getMessage(), e);
                        error = true;
                        MetronError metronError = new MetronError()
                                .withErrorType(Constants.ErrorType.ENRICHMENT_ERROR).withThrowable(e)
                                .withErrorFields(new HashSet() {
                                    {
                                        add(field);
                                    }
                                }).addRawMessage(rawMessage);
                        ErrorUtils.handleError(collector, metronError);
                        continue;
                    }
                }
                if (!enrichedField.isEmpty()) {
                    for (Object enrichedKey : enrichedField.keySet()) {
                        if (!StringUtils.isEmpty(prefix)) {
                            enrichedMessage.put(field + "." + enrichedKey, enrichedField.get(enrichedKey));
                        } else {
                            enrichedMessage.put(enrichedKey, enrichedField.get(enrichedKey));
                        }
                    }
                }
            }
        }

        enrichedMessage.put("adapter." + adapter.getClass().getSimpleName().toLowerCase() + ".end.ts",
                "" + System.currentTimeMillis());
        if (error) {
            throw new Exception("Unable to enrich " + rawMessage + " check logs for specifics.");
        }
        if (!enrichedMessage.isEmpty()) {
            collector.emit(enrichmentType, new Values(key, enrichedMessage, subGroup));
        }
    } catch (Exception e) {
        handleError(key, rawMessage, subGroup, enrichedMessage, e);
    }
}

From source file:org.apache.metron.enrichment.common.GenericEnrichmentBolt.java

@SuppressWarnings("unchecked")
public void execute(Tuple tuple) {

    LOG.trace("[Metron] Starting enrichment");

    JSONObject in_json = null;
    String key = null;//from  w  ww .  j  av  a  2s .c om

    try {

        key = tuple.getStringByField("key");
        in_json = (JSONObject) tuple.getValueByField("message");

        if (in_json == null || in_json.isEmpty())
            throw new Exception("Could not parse binary stream to JSON");

        if (key == null)
            throw new Exception("Key is not valid");

        LOG.trace("[Metron] Received tuple: " + in_json);

        JSONObject message = (JSONObject) in_json.get("message");

        if (message == null || message.isEmpty())
            throw new Exception("Could not extract message from JSON: " + in_json);

        LOG.trace("[Metron] Extracted message: " + message);

        for (String jsonkey : _jsonKeys) {
            LOG.trace("[Metron] Processing:" + jsonkey + " within:" + message);

            String jsonvalue = (String) message.get(jsonkey);
            LOG.trace("[Metron] Processing: " + jsonkey + " -> " + jsonvalue);

            if (null == jsonvalue) {
                LOG.trace("[Metron] Key " + jsonkey + "not present in message " + message);
                continue;
            }

            // If the field is empty, no need to enrich
            if (jsonvalue.length() == 0) {
                continue;
            }

            JSONObject enrichment = cache.getUnchecked(jsonvalue);
            LOG.trace("[Metron] Enriched: " + jsonkey + " -> " + enrichment);

            if (enrichment == null)
                throw new Exception("[Metron] Could not enrich string: " + jsonvalue);

            if (!in_json.containsKey("enrichment")) {
                in_json.put("enrichment", new JSONObject());
                LOG.trace("[Metron] Starting a string of enrichments");
            }

            JSONObject enr1 = (JSONObject) in_json.get("enrichment");

            if (enr1 == null)
                throw new Exception("Internal enrichment is empty");

            if (!enr1.containsKey(_enrichment_tag)) {
                enr1.put(_enrichment_tag, new JSONObject());
                LOG.trace("[Metron] Starting a new enrichment");
            }

            LOG.trace("[Metron] ENR1 is: " + enr1);

            JSONObject enr2 = (JSONObject) enr1.get(_enrichment_tag);
            enr2.put(jsonkey, enrichment);

            LOG.trace("[Metron] ENR2 is: " + enr2);

            enr1.put(_enrichment_tag, enr2);
            in_json.put("enrichment", enr1);
        }

        LOG.debug("[Metron] Generated combined enrichment: " + in_json);

        _collector.emit("message", new Values(key, in_json));
        _collector.ack(tuple);

        if (_reporter != null) {
            emitCounter.inc();
            ackCounter.inc();
        }
    } catch (Exception e) {

        LOG.error("[Metron] Unable to enrich message: " + in_json);
        _collector.fail(tuple);

        if (_reporter != null) {
            failCounter.inc();
        }

        JSONObject error = ErrorGenerator.generateErrorMessage("Enrichment problem: " + in_json, e);
        _collector.emit("error", new Values(error));
    }

}

From source file:org.apache.metron.parsers.bolt.ParserBolt.java

@SuppressWarnings("unchecked")
@Override//from  ww w .  j  a  v a2  s .c o m
public void execute(Tuple tuple) {
    byte[] originalMessage = (byte[]) messageGetStrategy.get(tuple);
    SensorParserConfig sensorParserConfig = getSensorParserConfig();
    try {
        //we want to ack the tuple in the situation where we have are not doing a bulk write
        //otherwise we want to defer to the writerComponent who will ack on bulk commit.
        boolean ackTuple = !writer.handleAck();
        int numWritten = 0;
        if (sensorParserConfig != null) {
            List<FieldValidator> fieldValidations = getConfigurations().getFieldValidations();
            Optional<List<JSONObject>> messages = parser.parseOptional(originalMessage);
            for (JSONObject message : messages.orElse(Collections.emptyList())) {
                message.put(Constants.SENSOR_TYPE, getSensorType());
                for (FieldTransformer handler : sensorParserConfig.getFieldTransformations()) {
                    if (handler != null) {
                        handler.transformAndUpdate(message, sensorParserConfig.getParserConfig(),
                                stellarContext);
                    }
                }
                if (!message.containsKey(Constants.GUID)) {
                    message.put(Constants.GUID, UUID.randomUUID().toString());
                }
                if (parser.validate(message) && (filter == null || filter.emitTuple(message, stellarContext))) {
                    numWritten++;
                    List<FieldValidator> failedValidators = getFailedValidators(message, fieldValidations);
                    if (failedValidators.size() > 0) {
                        MetronError error = new MetronError().withErrorType(Constants.ErrorType.PARSER_INVALID)
                                .withSensorType(getSensorType()).addRawMessage(message);
                        Set<String> errorFields = failedValidators.stream()
                                .flatMap(fieldValidator -> fieldValidator.getInput().stream())
                                .collect(Collectors.toSet());
                        if (!errorFields.isEmpty()) {
                            error.withErrorFields(errorFields);
                        }
                        ErrorUtils.handleError(collector, error);
                    } else {
                        writer.write(getSensorType(), tuple, message, getConfigurations(), messageGetStrategy);
                    }
                }
            }
        }
        //if we are supposed to ack the tuple OR if we've never passed this tuple to the bulk writer
        //(meaning that none of the messages are valid either globally or locally)
        //then we want to handle the ack ourselves.
        if (ackTuple || numWritten == 0) {
            collector.ack(tuple);
        }
    } catch (Throwable ex) {
        MetronError error = new MetronError().withErrorType(Constants.ErrorType.PARSER_ERROR).withThrowable(ex)
                .withSensorType(getSensorType()).addRawMessage(originalMessage);
        ErrorUtils.handleError(collector, error);
        collector.ack(tuple);
    }
}

From source file:org.apache.metron.parsers.json.JSONMapParser.java

/**
 * Take raw data and convert it to a list of messages.
 *
 * @return If null is returned, this is treated as an empty list.
 *//*from w ww .  ja  va  2  s.c  o  m*/
@Override
@SuppressWarnings("unchecked")
public List<JSONObject> parse(byte[] rawMessage) {
    try {
        String originalString = new String(rawMessage);
        List<Map<String, Object>> messages = new ArrayList<>();

        // if configured, wrap the json in an entity and array
        if (wrapJson) {
            originalString = wrapMessageJson(originalString);
        }

        if (!StringUtils.isEmpty(jsonpQuery)) {
            Object parsedObject = JsonPath.parse(originalString).read(jsonpQuery, typeRef);
            if (parsedObject != null) {
                messages.addAll((List<Map<String, Object>>) parsedObject);
            }
        } else {
            messages.add(JSONUtils.INSTANCE.load(originalString, JSONUtils.MAP_SUPPLIER));
        }

        ArrayList<JSONObject> parsedMessages = new ArrayList<>();
        for (Map<String, Object> rawMessageMap : messages) {
            JSONObject originalJsonObject = new JSONObject(rawMessageMap);
            JSONObject ret = normalizeJson(rawMessageMap);
            // the original string is the original for THIS sub message
            ret.put("original_string", originalJsonObject.toJSONString());
            if (!ret.containsKey("timestamp")) {
                ret.put("timestamp", System.currentTimeMillis());
            }
            parsedMessages.add(ret);
        }
        return Collections.unmodifiableList(parsedMessages);
    } catch (Throwable e) {
        String message = "Unable to parse " + new String(rawMessage) + ": " + e.getMessage();
        LOG.error(message, e);
        throw new IllegalStateException(message, e);
    }
}

From source file:org.apache.metron.parsers.ParserRunnerImpl.java

/**
 * Post-processes parsed messages by:/*  w  ww .jav a2 s .  co  m*/
 * <ul>
 *   <li>Applying field transformations defined in the sensor parser config</li>
 *   <li>Filtering messages using the configured MessageFilter class</li>
 *   <li>Validating messages using the MessageParser validate method</li>
 * </ul>
 * If a message is successfully processed a message is returned in a ProcessResult.  If a message fails
 * validation, a MetronError object is created and returned in a ProcessResult.  If a message is
 * filtered out an empty Optional is returned.
 *
 * @param sensorType Sensor type of the message
 * @param message Message parsed by the MessageParser
 * @param rawMessage Raw message including metadata
 * @param parser MessageParser for the sensor type
 * @param parserConfigurations Parser configurations
 */
@SuppressWarnings("unchecked")
protected Optional<ProcessResult> processMessage(String sensorType, JSONObject message, RawMessage rawMessage,
        MessageParser<JSONObject> parser, ParserConfigurations parserConfigurations) {
    Optional<ProcessResult> processResult = Optional.empty();
    SensorParserConfig sensorParserConfig = parserConfigurations.getSensorParserConfig(sensorType);
    sensorParserConfig.getRawMessageStrategy().mergeMetadata(message, rawMessage.getMetadata(),
            sensorParserConfig.getMergeMetadata(), sensorParserConfig.getRawMessageStrategyConfig());
    message.put(Constants.SENSOR_TYPE, sensorType);
    applyFieldTransformations(message, rawMessage, sensorParserConfig);
    if (!message.containsKey(Constants.GUID)) {
        message.put(Constants.GUID, UUID.randomUUID().toString());
    }
    MessageFilter<JSONObject> filter = sensorToParserComponentMap.get(sensorType).getFilter();
    if (filter == null || filter.emit(message, stellarContext)) {
        boolean isInvalid = !parser.validate(message);
        List<FieldValidator> failedValidators = null;
        if (!isInvalid) {
            failedValidators = getFailedValidators(message, parserConfigurations);
            isInvalid = !failedValidators.isEmpty();
        }
        if (isInvalid) {
            MetronError error = new MetronError().withErrorType(Constants.ErrorType.PARSER_INVALID)
                    .withSensorType(Collections.singleton(sensorType)).addRawMessage(message);
            Set<String> errorFields = failedValidators == null ? null
                    : failedValidators.stream().flatMap(fieldValidator -> fieldValidator.getInput().stream())
                            .collect(Collectors.toSet());
            if (errorFields != null && !errorFields.isEmpty()) {
                error.withErrorFields(errorFields);
            }
            processResult = Optional.of(new ProcessResult(error));
        } else {
            processResult = Optional.of(new ProcessResult(message));
        }
    }
    return processResult;
}

From source file:org.apache.metron.parsers.syslog.BaseSyslogParser.java

@Override
public boolean validate(JSONObject message) {
    if (!(message.containsKey("original_string"))) {
        LOG.trace("[Metron] Message does not have original_string: {}", message);
        return false;
    } else if (!(message.containsKey("timestamp"))) {
        LOG.trace("[Metron] Message does not have timestamp: {}", message);
        return false;
    } else {//from   w ww  .  j  a v  a2 s .  c om
        LOG.trace("[Metron] Message conforms to schema: {}", message);
        return true;
    }
}