Example usage for com.amazonaws.services.cloudwatch.model StandardUnit BytesSecond

Introduction

In this page you can find the example usage for com.amazonaws.services.cloudwatch.model StandardUnit BytesSecond.

Prototype

StandardUnit BytesSecond

To view the source code for com.amazonaws.services.cloudwatch.model StandardUnit BytesSecond.

Click Source Link

Usage

From source file:uk.gov.gchq.gaffer.performancetesting.aws.PublishAccumuloMetricsToCloudWatch.java

License:Apache License

private List<MetricDatum> gatherMetrics() throws TException {
    final String awsEmrJobFlowId = AwsEmrUtils.getJobFlowId();

    final MasterMonitorInfo stats = this.client.getMasterStats(Tracer.traceInfo(), this.context.rpcCreds());
    LOGGER.trace(stats.toString());//from ww w.ja v a  2 s. c om

    final Date now = new Date();
    final List<MetricDatum> metrics = new ArrayList<>();
    final Map<Pair<String, List<Dimension>>, Double> metricsToCache = new HashMap<>();

    // Master Metrics
    final List<Dimension> masterDimensions = new ArrayList<>();
    if (awsEmrJobFlowId != null) {
        masterDimensions.add(new Dimension().withName("EmrJobFlowId").withValue(awsEmrJobFlowId));
    }
    masterDimensions.add(new Dimension().withName("InstanceName").withValue(this.instanceName));

    metrics.addAll(this.generateCountMetricForEachDimensionCombination("TabletServerCount",
            (double) stats.getTServerInfoSize(), masterDimensions, now));
    metrics.addAll(this.generateCountMetricForEachDimensionCombination("BadTabletServerCount",
            (double) stats.getBadTServersSize(), masterDimensions, now));
    metrics.addAll(this.generateCountMetricForEachDimensionCombination("DeadTabletServerCount",
            (double) stats.getDeadTabletServersSize(), masterDimensions, now));
    metrics.addAll(this.generateCountMetricForEachDimensionCombination("ServersShuttingDownCount",
            (double) stats.getServersShuttingDownSize(), masterDimensions, now));
    metrics.addAll(this.generateCountMetricForEachDimensionCombination("TableCount",
            (double) stats.getTableMapSize(), masterDimensions, now));

    // Tablet Server Metrics
    for (final TabletServerStatus tserver : stats.getTServerInfo()) {
        // Trimming off the port from the tablet server name as (at the moment) we only deploy a single tablet server per YARN node
        // This makes it easier to locate the metrics in CloudWatch
        // @TODO: Make this optional via a command line flag?
        String tabletServerHostName = tserver.getName();
        if (tabletServerHostName.contains(":")) {
            tabletServerHostName = tabletServerHostName.substring(0, tabletServerHostName.lastIndexOf(':'));
        }

        final List<Dimension> tserverDimensions = new ArrayList<>();
        if (awsEmrJobFlowId != null) {
            tserverDimensions.add(new Dimension().withName("EmrJobFlowId").withValue(awsEmrJobFlowId));
        }
        tserverDimensions.add(new Dimension().withName("InstanceName").withValue(this.instanceName));
        tserverDimensions.add(new Dimension().withName("TabletServerName").withValue(tabletServerHostName));

        metrics.addAll(this.generateCountMetricForEachDimensionCombination("DataCacheRequests",
                (double) tserver.getDataCacheRequest(), tserverDimensions, now));
        metrics.addAll(this.generateCountMetricForEachDimensionCombination("DataCacheHits",
                (double) tserver.getDataCacheHits(), tserverDimensions, now));
        metrics.addAll(this.generateCountMetricForEachDimensionCombination("IndexCacheRequests",
                (double) tserver.getIndexCacheRequest(), tserverDimensions, now));
        metrics.addAll(this.generateCountMetricForEachDimensionCombination("IndexCacheHits",
                (double) tserver.getIndexCacheHits(), tserverDimensions, now));
        metrics.addAll(this.generateCountMetricForEachDimensionCombination("Lookups",
                (double) tserver.getLookups(), tserverDimensions, now));
        metrics.addAll(this.generateCountMetricForEachDimensionCombination("OsLoad", tserver.getOsLoad(),
                tserverDimensions, now));

        // Table Metrics
        for (final Map.Entry<String, TableInfo> tableEntry : tserver.getTableMap().entrySet()) {
            final String tableId = tableEntry.getKey();
            final TableInfo info = tableEntry.getValue();

            final String tableName = this.getTableNameForId(tableId);
            if (tableName != null) {
                final List<Dimension> dimensions = new ArrayList<>();
                if (awsEmrJobFlowId != null) {
                    dimensions.add(new Dimension().withName("EmrJobFlowId").withValue(awsEmrJobFlowId));
                }
                dimensions.add(new Dimension().withName("InstanceName").withValue(this.instanceName));
                dimensions.add(new Dimension().withName("TableName").withValue(tableName));
                dimensions.add(new Dimension().withName("TabletServerName").withValue(tabletServerHostName));

                metrics.addAll(this.generateCountMetricForEachDimensionCombination("TabletCount",
                        (double) info.getTablets(), dimensions, now));
                metrics.addAll(this.generateCountMetricForEachDimensionCombination("OnlineTabletCount",
                        (double) info.getOnlineTablets(), dimensions, now));
                metrics.addAll(this.generateCountMetricForEachDimensionCombination("RecordCount",
                        (double) info.getRecs(), dimensions, now));
                metrics.addAll(this.generateCountMetricForEachDimensionCombination("RecordsInMemoryCount",
                        (double) info.getRecsInMemory(), dimensions, now));
                if (info.getScans() != null) {
                    metrics.addAll(this.generateCountMetricForEachDimensionCombination("ScansRunning",
                            (double) info.getScans().getRunning(), dimensions, now));
                    metrics.addAll(this.generateCountMetricForEachDimensionCombination("ScansQueued",
                            (double) info.getScans().getQueued(), dimensions, now));
                }
                if (info.getMinors() != null) {
                    metrics.addAll(this.generateCountMetricForEachDimensionCombination("MinorCompactionCount",
                            (double) info.getMinors().getRunning(), dimensions, now));
                    metrics.addAll(
                            this.generateCountMetricForEachDimensionCombination("MinorCompactionQueuedCount",
                                    (double) info.getMinors().getQueued(), dimensions, now));
                }
                if (info.getMajors() != null) {
                    metrics.addAll(this.generateCountMetricForEachDimensionCombination("MajorCompactionCount",
                            (double) info.getMajors().getRunning(), dimensions, now));
                    metrics.addAll(
                            this.generateCountMetricForEachDimensionCombination("MajorCompactionQueuedCount",
                                    (double) info.getMajors().getQueued(), dimensions, now));
                }

                // The 'IngestRate' metric reported by Accumulo updates every ~5 seconds, so generate our own
                // version of the metric for our interval using 'RecordCount'. Note that our version is affected by
                // MinC + MajC that reduce the number of records so -ve rates are possible!
                final Pair<String, List<Dimension>> metricCacheKey = new Pair<>("RecordCount", dimensions);
                if (this.previousMetrics != null && this.previousMetrics.containsKey(metricCacheKey)) {
                    final double change = (double) info.getRecs() - this.previousMetrics.get(metricCacheKey);
                    final double rate = change / ((now.getTime() - this.previousCaptureDate.getTime()) / 1000);
                    metrics.addAll(
                            this.generateMetricForEachDimensionCombination(
                                    new MetricDatum().withMetricName("CalculatedIngestRate").withValue(rate)
                                            .withUnit(StandardUnit.CountSecond).withTimestamp(now),
                                    dimensions));
                }
                metricsToCache.put(metricCacheKey, (double) info.getRecs());

                metrics.addAll(this.generateMetricForEachDimensionCombination(
                        new MetricDatum().withMetricName("IngestRate").withValue(info.getIngestRate())
                                .withUnit(StandardUnit.CountSecond).withTimestamp(now),
                        dimensions));

                metrics.addAll(this.generateMetricForEachDimensionCombination(
                        new MetricDatum().withMetricName("IngestByteRate").withValue(info.getIngestByteRate())
                                .withUnit(StandardUnit.BytesSecond).withTimestamp(now),
                        dimensions));

                metrics.addAll(this.generateMetricForEachDimensionCombination(
                        new MetricDatum().withMetricName("QueryRate").withValue(info.getQueryRate())
                                .withUnit(StandardUnit.CountSecond).withTimestamp(now),
                        dimensions));

                metrics.addAll(this.generateMetricForEachDimensionCombination(
                        new MetricDatum().withMetricName("QueryByteRate").withValue(info.getQueryByteRate())
                                .withUnit(StandardUnit.BytesSecond).withTimestamp(now),
                        dimensions));

                metrics.addAll(this.generateMetricForEachDimensionCombination(
                        new MetricDatum().withMetricName("ScanRate").withValue(info.getScanRate())
                                .withUnit(StandardUnit.CountSecond).withTimestamp(now),
                        dimensions));
            }
        }
    }

    this.previousCaptureDate = now;
    this.previousMetrics = metricsToCache;

    return metrics;
}

---