Java tutorial
/******************************************************************************* * Copyright 2012 Persistent Systems Ltd. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. ******************************************************************************/ package com.persistent.cloudninja.scheduler; import java.io.BufferedReader; import java.io.IOException; import java.net.URISyntaxException; import java.security.InvalidKeyException; import java.text.ParseException; import java.text.SimpleDateFormat; import java.util.ArrayList; import java.util.Date; import java.util.HashMap; import java.util.List; import java.util.Map; import org.apache.log4j.Logger; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; import org.springframework.util.StopWatch; import com.persistent.cloudninja.dao.StorageAnalyticsLogsSummaryDao; import com.persistent.cloudninja.dao.StorageBandwidthBatchDao; import com.persistent.cloudninja.dao.TaskCompletionDao; import com.persistent.cloudninja.domainobject.StorageAnalyticsLogsSummaryEntity; import com.persistent.cloudninja.domainobject.StorageAnalyticsLogsSummaryId; import com.persistent.cloudninja.domainobject.StorageBandwidthBatchEntity; import com.persistent.cloudninja.utils.SchedulerSettings; import com.persistent.cloudninja.utils.StorageClientUtility; /** * Processor for tenant storage bandwidth calculation task. It processes the * messages generated by generator. */ @Component public class TenantStorageBWProcessor extends TaskActivityBase { private static final Logger LOGGER = Logger.getLogger(TenantStorageBWProcessor.class); private static final int OPERATION_TYPE_INDEX = 2; private static final int REQUEST_STATUS_INDEX = 3; private static final int HTTP_STATUS_CODE_INDEX = 4; private static final int E2E_LATENCY_INDEX = 5; private static final int SERVER_LATENCY_INDEX = 6; private static final int SERVICE_TYPE_INDEX = 10; private static final int REQUESTED_OBJECT_KEY_INDEX = 12; private static final int REQUESTED_OBJECT_KEY_INDEX_BEFORE = 3; private static final int REQUESTED_PACKET_SIZE_INDEX = 18; private static final int RESPONSE_PACKET_SIZE_INDEX = 20; private static final int REQUESTED_OBJECT_KEY_SIZE = 2; private static final int HTTP_SUCCESS_CODE_MAX = 299; private static final int HTTP_SUCCESS_CODE_MIN = 200; private static final long BYTES_TO_KB_CONSTANT = 1024; @Autowired private StorageAnalyticsLogsSummaryDao storageAnalyticsLogsSummaryDao; @Autowired private TaskCompletionDao taskCompletionDao; @Autowired private StorageUtility storageUtility; @Autowired private StorageBandwidthBatchDao storageBandwidthBatchDao; public TenantStorageBWProcessor() throws InvalidKeyException, URISyntaxException { super(new TenantStorageBWQueue(SchedulerSettings.StorageBandwidthQueue, StorageClientUtility.getCloudStorageAccount().createCloudQueueClient())); } /** * Calculates storage bandwidth of tenant. */ @Override public boolean execute() { StopWatch watch = new StopWatch(); boolean retVal = true; List<StorageBandwidthBatchEntity> batchesToProcess = null; try { LOGGER.debug("In TenantStorageBWProcessor"); TenantStorageBWQueue queue = (TenantStorageBWQueue) getWorkQueue(); batchesToProcess = queue.dequeue(); if (batchesToProcess == null) { retVal = false; } else { watch.start(); processBatches(batchesToProcess); watch.stop(); taskCompletionDao.updateTaskCompletionDetails(watch.getTotalTimeSeconds(), "GenerateStorageBandwidth", "Batch Size = " + batchesToProcess.size()); } } catch (Exception e) { retVal = false; LOGGER.error(e.getMessage(), e); } return retVal; } /** * Process batches to get blob URI and number * of lines processed for further processing of logs. * * @param batchesToProcess List of StorageBandwidthBatchEntity * @throws IOException * @throws ParseException */ private void processBatches(List<StorageBandwidthBatchEntity> batchesToProcess) throws IOException, ParseException { int batchSize = batchesToProcess.size(); String uri = null; int numberOfLines = 0; BufferedReader logs = null; for (int batchIndex = 0; batchIndex < batchSize; batchIndex++) { StorageBandwidthBatchEntity entity = batchesToProcess.get(batchIndex); uri = entity.getLogUri(); numberOfLines = entity.getLastLineProcessed(); Date endtime = storageUtility.getLastModifiedDateFromAnalyticsBlob(uri); logs = storageUtility.getReaderStreamForAnalyticsBlob(uri); int linesProcessed = processLogs(logs, numberOfLines); numberOfLines = numberOfLines + linesProcessed; batchesToProcess.get(batchIndex).setLastLineProcessed(numberOfLines); batchesToProcess.get(batchIndex).setLastUpdatedTime(endtime); } //Update into Database table if (batchesToProcess.size() != 0) { storageBandwidthBatchDao.updateStorageBandwidthBatch(batchesToProcess); } } /** * Process Logs and make entries into database. * * @param logs The logs to be processed * @param processedLines Number of already processed lines for the particular blob * @return linesProcessed The number of lines processed. * @throws IOException * @throws ParseException */ private int processLogs(BufferedReader logs, int processedLines) throws IOException, ParseException { Map<String, StorageAnalyticsLogsSummaryEntity> storageAnalyticsLogsSummaryEntityMap = new HashMap<String, StorageAnalyticsLogsSummaryEntity>(); int linesProcessed = 0; for (int index = 0; index < processedLines; index++) { logs.readLine(); } String lineFromLog = null; while ((lineFromLog = logs.readLine()) != null) { //separating the line using character " to get the requested object key String[] columnsAfterSplittingLine = lineFromLog.split("\""); //only if the requested object key exist check for the tenant Id and then parse the specific log if (columnsAfterSplittingLine.length > REQUESTED_OBJECT_KEY_INDEX_BEFORE) { /* * 0 = empty * 1 = service name (eg. cloudninjaforjavastorage) * 2 = container name (eg. tnts-tenantId) * 3 = blob name */ //Splitting to get the tenant ID String[] requestedObjectKey = columnsAfterSplittingLine[REQUESTED_OBJECT_KEY_INDEX_BEFORE] .split("/"); //Checking for the correct line that is to be parsed if ((requestedObjectKey.length > REQUESTED_OBJECT_KEY_SIZE) && (requestedObjectKey[2].startsWith("tnts-") || requestedObjectKey[2].startsWith("tntp-"))) { StringBuffer buffer = new StringBuffer(); //appending the parsed logs excluding the Request URL. for (int index = 0; index < columnsAfterSplittingLine.length; index++) { if (index != 1) { buffer.append(columnsAfterSplittingLine[index].toString()); } } String[] logEntry = buffer.toString().split(";"); String key = constructKey(logEntry); StorageAnalyticsLogsSummaryEntity value = new StorageAnalyticsLogsSummaryEntity(); if (storageAnalyticsLogsSummaryEntityMap.containsKey(key)) { value = constructValueIfKeyExists(logEntry); StorageAnalyticsLogsSummaryEntity oldValue = storageAnalyticsLogsSummaryEntityMap.get(key); oldValue.getId().setSnapshotTime(value.getId().getSnapshotTime()); oldValue.setE2eLatency(oldValue.getE2eLatency() + value.getE2eLatency()); oldValue.setServerLatency(oldValue.getServerLatency() + value.getServerLatency()); oldValue.setRequestPacketSize( oldValue.getRequestPacketSize() + value.getRequestPacketSize()); oldValue.setResponsePacketSize( oldValue.getResponsePacketSize() + value.getResponsePacketSize()); oldValue.setCount(oldValue.getCount() + 1); storageAnalyticsLogsSummaryEntityMap.put(key, oldValue); } else { value = constructValueIfKeyNotExist(logEntry); storageAnalyticsLogsSummaryEntityMap.put(key, value); } } } linesProcessed++; } List<StorageAnalyticsLogsSummaryEntity> storageAnalyticsLogsSummaryEntities = new ArrayList<StorageAnalyticsLogsSummaryEntity>(); for (String key : storageAnalyticsLogsSummaryEntityMap.keySet()) { storageAnalyticsLogsSummaryEntities.add(storageAnalyticsLogsSummaryEntityMap.get(key)); } //Updating StorageAnalyticsLogsSummary table in database if (storageAnalyticsLogsSummaryEntities.size() != 0) { storageAnalyticsLogsSummaryDao.updateStorageAnalyticsLogsSummary(storageAnalyticsLogsSummaryEntities); } return linesProcessed; } /** * Creates new object of StorageAnalyticsLogsSummaryEntity from parsed log, * as the log entry is not present in the Log summary map. * * @param parsedLineFormLogs * @return storageAnalyticsLogsSummaryEntity * @throws ParseException */ private StorageAnalyticsLogsSummaryEntity constructValueIfKeyNotExist(String[] parsedLineFormLogs) throws ParseException { StorageAnalyticsLogsSummaryEntity storageAnalyticsLogsSummaryEntity = new StorageAnalyticsLogsSummaryEntity(); StorageAnalyticsLogsSummaryId id = new StorageAnalyticsLogsSummaryId(); String[] requestedObjectKey = parsedLineFormLogs[REQUESTED_OBJECT_KEY_INDEX].split("/")[2].split("-"); long requestedPacketSize = 0; long responsePacketSize = 0; SimpleDateFormat dateFormatter = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss.SSS"); Date currentDate = new Date(); String time = dateFormatter.format(currentDate); id.setSnapshotTime(dateFormatter.parse(time)); id.setHttpStatusCode(parsedLineFormLogs[HTTP_STATUS_CODE_INDEX]); id.setOperationType(parsedLineFormLogs[OPERATION_TYPE_INDEX]); id.setRequestStatus(parsedLineFormLogs[REQUEST_STATUS_INDEX]); id.setServiceType(parsedLineFormLogs[SERVICE_TYPE_INDEX]); id.setTenant(requestedObjectKey[1]); storageAnalyticsLogsSummaryEntity.setId(id); storageAnalyticsLogsSummaryEntity.setE2eLatency(Float.parseFloat(parsedLineFormLogs[E2E_LATENCY_INDEX])); storageAnalyticsLogsSummaryEntity .setServerLatency(Float.parseFloat(parsedLineFormLogs[SERVER_LATENCY_INDEX])); try { requestedPacketSize = Long.parseLong(parsedLineFormLogs[REQUESTED_PACKET_SIZE_INDEX]); // convert bytes to kilobytes requestedPacketSize = requestedPacketSize / BYTES_TO_KB_CONSTANT; } catch (NumberFormatException e) { requestedPacketSize = 0; } storageAnalyticsLogsSummaryEntity.setRequestPacketSize(requestedPacketSize); try { responsePacketSize = Long.parseLong(parsedLineFormLogs[RESPONSE_PACKET_SIZE_INDEX]); // convert bytes to kilobytes responsePacketSize = responsePacketSize / BYTES_TO_KB_CONSTANT; } catch (NumberFormatException e) { responsePacketSize = 0; } storageAnalyticsLogsSummaryEntity.setResponsePacketSize(responsePacketSize); storageAnalyticsLogsSummaryEntity.setCount(1); int httpStatusCode = Integer.parseInt(parsedLineFormLogs[HTTP_STATUS_CODE_INDEX]); if ((httpStatusCode <= HTTP_SUCCESS_CODE_MAX) && (httpStatusCode >= HTTP_SUCCESS_CODE_MIN)) { storageAnalyticsLogsSummaryEntity.setBillable(true); } else { storageAnalyticsLogsSummaryEntity.setBillable(false); } return storageAnalyticsLogsSummaryEntity; } /** * Creates object of StorageAnalyticsLogsSummaryEntity to * add its attributes to already existing Log summary map entry. * * @param parsedLineFormLogs * @return storageAnalyticsLogsSummaryEntity * @throws ParseException */ private StorageAnalyticsLogsSummaryEntity constructValueIfKeyExists(String[] parsedLineFormLogs) throws ParseException { StorageAnalyticsLogsSummaryEntity storageAnalyticsLogsSummaryEntity = new StorageAnalyticsLogsSummaryEntity(); StorageAnalyticsLogsSummaryId id = new StorageAnalyticsLogsSummaryId(); long requestedPacketSize = 0; long responsePacketSize = 0; SimpleDateFormat dateFormatter = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss.SSS"); Date currentDate = new Date(); String time = dateFormatter.format(currentDate); id.setSnapshotTime(dateFormatter.parse(time)); storageAnalyticsLogsSummaryEntity.setId(id); storageAnalyticsLogsSummaryEntity.setE2eLatency(Float.parseFloat(parsedLineFormLogs[E2E_LATENCY_INDEX])); storageAnalyticsLogsSummaryEntity .setServerLatency(Float.parseFloat(parsedLineFormLogs[SERVER_LATENCY_INDEX])); try { requestedPacketSize = Long.parseLong(parsedLineFormLogs[REQUESTED_PACKET_SIZE_INDEX]); // convert bytes to kilobytes requestedPacketSize = requestedPacketSize / BYTES_TO_KB_CONSTANT; } catch (NumberFormatException e) { requestedPacketSize = 0; } storageAnalyticsLogsSummaryEntity.setRequestPacketSize(requestedPacketSize); try { responsePacketSize = Long.parseLong(parsedLineFormLogs[RESPONSE_PACKET_SIZE_INDEX]); // convert bytes to kilobytes responsePacketSize = responsePacketSize / BYTES_TO_KB_CONSTANT; } catch (NumberFormatException e) { responsePacketSize = 0; } storageAnalyticsLogsSummaryEntity.setResponsePacketSize(responsePacketSize); return storageAnalyticsLogsSummaryEntity; } /** * Constructs unique key for a map to identify the same kind of log messages. * @param parsedLineFormLogs * @return mapKey String, containing tenant Id, * operation type, request status, HTTP status code and service type. */ private String constructKey(String[] parsedLineFormLogs) { StringBuffer mapKey = new StringBuffer(); String[] requestedObjectKey = parsedLineFormLogs[REQUESTED_OBJECT_KEY_INDEX].split("/")[2].split("-"); mapKey.append(requestedObjectKey[1]); mapKey.append(parsedLineFormLogs[OPERATION_TYPE_INDEX]); mapKey.append(parsedLineFormLogs[REQUEST_STATUS_INDEX]); mapKey.append(parsedLineFormLogs[HTTP_STATUS_CODE_INDEX]); mapKey.append(parsedLineFormLogs[SERVICE_TYPE_INDEX]); return mapKey.toString(); } }