List of usage examples for com.google.api.client.googleapis.extensions.appengine.auth.oauth2 AppIdentityCredential AppIdentityCredential
protected AppIdentityCredential(Builder builder)
From source file:com.google.appengine.tools.cloudstorage.oauth.OauthRawGcsService.java
License:Open Source License
OauthRawGcsService(OAuthURLFetchService urlfetch, ImmutableSet<HTTPHeader> headers) {
this.urlfetch = checkNotNull(urlfetch, "Null urlfetch");
this.headers = checkNotNull(headers, "Null headers");
AppIdentityCredential cred = new AppIdentityCredential(OAUTH_SCOPES);
storage = new Storage.Builder(new UrlFetchTransport(), new JacksonFactory(), cred)
.setApplicationName(SystemProperty.applicationId.get()).build();
}
From source file:com.l2bq.datastore.analysis.builtin.BuiltinDatastoreToBigqueryIngesterTask.java
License:Apache License
public void doGet(HttpServletRequest req, HttpServletResponse resp) throws IOException { resp.setContentType("application/json"); String timestampStr = req.getParameter(AnalysisConstants.TIMESTAMP_PARAM); long timestamp = 0; if (AnalysisUtility.areParametersValid(timestampStr)) { try {//from w w w . ja v a 2s. co m timestamp = Long.parseLong(timestampStr); } catch (Exception e) { // leave it at default value } } if (timestamp == 0) { resp.getWriter().write( AnalysisUtility.failureJson("Missing required param: " + AnalysisConstants.TIMESTAMP_PARAM)); resp.setStatus(HttpServletResponse.SC_BAD_REQUEST); return; } String builtinDatastoreExportConfig = req.getParameter(AnalysisConstants.BUILTIN_DATASTORE_EXPORT_CONFIG); if (!AnalysisUtility.areParametersValid(builtinDatastoreExportConfig)) { resp.getWriter().write(AnalysisUtility .failureJson("Missing required param: " + AnalysisConstants.BUILTIN_DATASTORE_EXPORT_CONFIG)); resp.setStatus(HttpServletResponse.SC_BAD_REQUEST); return; } // Instantiate the export config BuiltinDatastoreExportConfiguration exporterConfig = AnalysisUtility .instantiateExportConfig(builtinDatastoreExportConfig); String keyOfCompletedBackup = checkAndGetCompletedBackup( AnalysisUtility.getPreBackupName(timestamp, exporterConfig.getBackupNamePrefix())); if (keyOfCompletedBackup == null) { resp.getWriter().println( AnalysisUtility.successJson("backup incomplete, retrying in " + MILLIS_TO_ENQUEUE + " millis")); enqueueTask(AnalysisUtility.getRequestBaseName(req), exporterConfig, timestamp, MILLIS_TO_ENQUEUE); } else { resp.getWriter().println(AnalysisUtility.successJson("backup complete, starting bigquery ingestion")); AppIdentityCredential credential = new AppIdentityCredential(AnalysisConstants.SCOPES); HttpRequestFactory requestFactory = HTTP_TRANSPORT.createRequestFactory(credential); Bigquery bigquery = Bigquery.builder(HTTP_TRANSPORT, JSON_FACTORY).setHttpRequestInitializer(credential) .setApplicationName("l2bq Logs").build(); String datatableSuffix = ""; if (exporterConfig.appendTimestampToDatatables()) { datatableSuffix = Long.toString(timestamp); } else { datatableSuffix = ""; } if (!exporterConfig.appendTimestampToDatatables()) { // we aren't appending the timestamps so delete the old tables if they exist for (String kind : exporterConfig.getEntityKindsToExport()) { boolean found = true; Table t; try { t = bigquery.tables().get(exporterConfig.getBigqueryProjectId(), exporterConfig.getBigqueryDatasetId(), kind).execute(); } catch (IOException e) { // table not found so don't need to do anything found = false; } if (found) { bigquery.tables().delete(exporterConfig.getBigqueryProjectId(), exporterConfig.getBigqueryDatasetId(), kind).execute(); } } } // now create the ingestion for (String kind : exporterConfig.getEntityKindsToExport()) { Job job = new Job(); JobConfiguration config = new JobConfiguration(); JobConfigurationLoad loadConfig = new JobConfigurationLoad(); String uri = "gs://" + exporterConfig.getBucketName() + "/" + keyOfCompletedBackup + "." + kind + ".backup_info"; loadConfig.setSourceUris(Arrays.asList(uri)); loadConfig.set("sourceFormat", "DATASTORE_BACKUP"); loadConfig.set("allowQuotedNewlines", true); TableReference table = new TableReference(); table.setProjectId(exporterConfig.getBigqueryProjectId()); table.setDatasetId(exporterConfig.getBigqueryDatasetId()); table.setTableId(kind + datatableSuffix); loadConfig.setDestinationTable(table); config.setLoad(loadConfig); job.setConfiguration(config); Insert insert = bigquery.jobs().insert(exporterConfig.getBigqueryProjectId(), job); // TODO(frew): Not sure this is necessary, but monkey-see'ing the example code insert.setProjectId(exporterConfig.getBigqueryProjectId()); JobReference ref = insert.execute().getJobReference(); Queue taskQueue = QueueFactory.getQueue(exporterConfig.getQueueName()); taskQueue.add(Builder .withUrl(AnalysisUtility.getRequestBaseName(req) + "/deleteCompletedCloudStorageFilesTask") .method(Method.GET).param(AnalysisConstants.BIGQUERY_JOB_ID_PARAM, ref.getJobId()) .param(AnalysisConstants.QUEUE_NAME_PARAM, exporterConfig.getQueueName()) .param(AnalysisConstants.BIGQUERY_PROJECT_ID_PARAM, exporterConfig.getBigqueryProjectId())); } } }
From source file:com.l2bq.logging.analysis.BigqueryStatusServlet.java
License:Apache License
public void doGet(HttpServletRequest req, HttpServletResponse resp) throws IOException { resp.setContentType("text/plain"); AppIdentityCredential credential = new AppIdentityCredential(AnalysisConstants.SCOPES); HttpRequestFactory requestFactory = HTTP_TRANSPORT.createRequestFactory(credential); Bigquery bigquery = Bigquery.builder(HTTP_TRANSPORT, JSON_FACTORY).setHttpRequestInitializer(credential) .setApplicationName("Streak Logs").build(); Bigquery.Projects.List projectRequest = bigquery.projects().list(); ProjectList projectResponse = projectRequest.execute(); resp.getWriter().println("Available Projects:" + projectResponse.toPrettyString()); if (projectResponse.getProjects() != null) { for (Projects project : projectResponse.getProjects()) { Bigquery.Jobs.List jobsRequest = bigquery.jobs().list(project.getId()); JobList jobsResponse = jobsRequest.execute(); List<JobList.Jobs> jobs = jobsResponse.getJobs(); resp.getWriter().println("=== Recent jobs for " + project.getId() + " ==="); if (jobs != null) { for (JobList.Jobs job : jobs) { resp.getWriter().println("Job " + job.getId() + ":"); resp.getWriter().println(job.toPrettyString()); String jobId = job.getJobReference().getJobId(); Bigquery.Jobs.Get jobRequest = bigquery.jobs().get(project.getId(), jobId); Job jobResponse = jobRequest.execute(); resp.getWriter().println("Full job description:"); resp.getWriter().println(jobResponse.toPrettyString()); }/* w ww .ja v a 2 s. c om*/ } } } }
From source file:com.l2bq.logging.analysis.LoadCloudStorageToBigqueryTask.java
License:Apache License
public void doGet(HttpServletRequest req, HttpServletResponse resp) throws IOException { resp.setContentType("text/plain"); String queueName = AnalysisUtility.extractParameterOrThrow(req, AnalysisConstants.QUEUE_NAME_PARAM); String bigqueryProjectId = AnalysisUtility.extractParameterOrThrow(req, AnalysisConstants.BIGQUERY_PROJECT_ID_PARAM); String bigqueryDatasetId = AnalysisUtility.extractParameterOrThrow(req, AnalysisConstants.BIGQUERY_DATASET_ID_PARAM); String bigqueryTableId = AnalysisUtility.extractParameterOrThrow(req, AnalysisConstants.BIGQUERY_TABLE_ID_PARAM); bigqueryTableId = bigqueryTableId.replace(":", "_").replace("-", "_"); MemcacheService memcache = MemcacheServiceFactory.getMemcacheService(AnalysisConstants.MEMCACHE_NAMESPACE); Long nextBigQueryJobTime = (Long) memcache.increment(AnalysisConstants.LAST_BIGQUERY_JOB_TIME, AnalysisConstants.LOAD_DELAY_MS, System.currentTimeMillis()); long currentTime = System.currentTimeMillis(); // The task queue has waited a long time to run us. Go ahead and reset the last job time // to prevent a race. if (currentTime > nextBigQueryJobTime + AnalysisConstants.LOAD_DELAY_MS / 2) { memcache.put(AnalysisConstants.LAST_BIGQUERY_JOB_TIME, currentTime); nextBigQueryJobTime = currentTime + AnalysisConstants.LOAD_DELAY_MS; }//from ww w . j a va2s . c o m if (currentTime < nextBigQueryJobTime) { memcache.increment(AnalysisConstants.LAST_BIGQUERY_JOB_TIME, -AnalysisConstants.LOAD_DELAY_MS); Queue taskQueue = QueueFactory.getQueue(queueName); taskQueue.add(Builder.withUrl( AnalysisUtility.getRequestBaseName(req) + "/loadCloudStorageToBigquery?" + req.getQueryString()) .method(Method.GET).etaMillis(nextBigQueryJobTime)); resp.getWriter().println("Rate limiting BigQuery load job - will retry at " + nextBigQueryJobTime); return; } String bucketName = AnalysisUtility.extractParameterOrThrow(req, AnalysisConstants.BUCKET_NAME_PARAM); AppIdentityCredential credential = new AppIdentityCredential(AnalysisConstants.SCOPES); HttpRequestFactory requestFactory = HTTP_TRANSPORT.createRequestFactory(credential); List<String> urisToProcess = new ArrayList<String>(); String schemaBaseUri; String startMsStr = req.getParameter(AnalysisConstants.START_MS_PARAM); // Logs if (AnalysisUtility.areParametersValid(startMsStr)) { long startMs = Long.parseLong(startMsStr); String endMsStr = AnalysisUtility.extractParameterOrThrow(req, AnalysisConstants.END_MS_PARAM); long endMs = Long.parseLong(endMsStr); String exporterSetClassStr = AnalysisUtility.extractParameterOrThrow(req, AnalysisConstants.BIGQUERY_FIELD_EXPORTER_SET_PARAM); BigqueryFieldExporterSet exporterSet = AnalysisUtility.instantiateExporterSet(exporterSetClassStr); String schemaHash = AnalysisUtility.computeSchemaHash(exporterSet); AnalysisUtility.fetchCloudStorageLogUris(bucketName, String.format("%s_%s", exporterSet.getPrefix(), schemaHash), startMs, endMs, requestFactory, urisToProcess, false); schemaBaseUri = urisToProcess.get(0); // Datastore } else { String cloudStoragePathBase = req.getParameter(AnalysisConstants.CLOUD_STORAGE_PATH_BASE_PARAM); String cloudStoragePathBaseEnd = cloudStoragePathBase.substring(0, cloudStoragePathBase.length() - 1) + (char) (cloudStoragePathBase.charAt(cloudStoragePathBase.length() - 1) + 1); AnalysisUtility.fetchCloudStorageUris(bucketName, cloudStoragePathBase, cloudStoragePathBaseEnd, requestFactory, urisToProcess, false); schemaBaseUri = "gs://" + bucketName + "/" + cloudStoragePathBase; } resp.getWriter().println("Got " + urisToProcess.size() + " uris to process"); if (urisToProcess.isEmpty()) { return; } for (String uri : urisToProcess) { resp.getWriter().println("URI: " + uri); } Bigquery bigquery = Bigquery.builder(HTTP_TRANSPORT, JSON_FACTORY).setHttpRequestInitializer(credential) .setApplicationName("l2bq Logs").build(); Job job = new Job(); JobConfiguration config = new JobConfiguration(); JobConfigurationLoad loadConfig = new JobConfigurationLoad(); // To save your logs from File Storage to Big Query, // you must set the fully-qualified URIs that point to your data on Google Cloud Storage // More details on the following link // https://developers.google.com/resources/api-libraries/documentation/bigquery/v2/java/latest/com/google/api/services/bigquery/model/JobConfigurationLoad.html#setSourceUris(java.util.List) loadConfig.setSourceUris(urisToProcess); loadConfig.set("allowQuotedNewlines", true); // You don't need to set schema every time, but you should set schema at the first time. TableSchema schema = new TableSchema(); loadSchema(schemaBaseUri, schema); loadConfig.setSchema(schema); TableReference table = new TableReference(); table.setProjectId(bigqueryProjectId); table.setDatasetId(bigqueryDatasetId); table.setTableId(bigqueryTableId); loadConfig.setDestinationTable(table); config.setLoad(loadConfig); job.setConfiguration(config); Insert insert = bigquery.jobs().insert(bigqueryProjectId, job); // TODO(frew): Not sure this is necessary, but monkey-see'ing the example code insert.setProjectId(bigqueryProjectId); JobReference ref = insert.execute().getJobReference(); resp.getWriter().println("Successfully started job " + ref); String shouldDelete = req.getParameter(AnalysisConstants.DELETE_FROM_CLOUD_STORAGE_PARAM); if (AnalysisUtility.areParametersValid(shouldDelete)) { Queue taskQueue = QueueFactory.getQueue(queueName); taskQueue.add(Builder .withUrl(AnalysisUtility.getRequestBaseName(req) + "/deleteCompletedCloudStorageFilesTask") .method(Method.GET).param(AnalysisConstants.BIGQUERY_JOB_ID_PARAM, ref.getJobId()) .param(AnalysisConstants.QUEUE_NAME_PARAM, queueName) .param(AnalysisConstants.BIGQUERY_PROJECT_ID_PARAM, bigqueryProjectId)); } }
From source file:com.l2bq.logging.analysis.LogExportCronTask.java
License:Apache License
public void doGet(HttpServletRequest req, HttpServletResponse resp) throws IOException { resp.setContentType("text/plain"); String msPerTableStr = req.getParameter(AnalysisConstants.MS_PER_TABLE_PARAM); long msPerTable = 1000 * 60 * 60 * 24; if (AnalysisUtility.areParametersValid(msPerTableStr)) { msPerTable = Long.parseLong(msPerTableStr); }//from ww w .jav a 2 s. c om String msPerFileStr = req.getParameter(AnalysisConstants.MS_PER_FILE_PARAM); long msPerFile = 1000 * 60 * 2; if (AnalysisUtility.areParametersValid(msPerFileStr)) { msPerFile = Long.parseLong(msPerFileStr); } if (msPerTable % msPerFile != 0) { throw new InvalidTaskParameterException("The " + AnalysisConstants.MS_PER_FILE_PARAM + " parameter must divide the " + AnalysisConstants.MS_PER_TABLE_PARAM + " parameter."); } String endMsStr = req.getParameter(AnalysisConstants.END_MS_PARAM); long endMs = System.currentTimeMillis(); if (AnalysisUtility.areParametersValid(endMsStr)) { endMs = Long.parseLong(endMsStr); } // By default look back a ways, but safely under the limit of 1000 files // per listing that Cloud Storage imposes String startMsStr = req.getParameter(AnalysisConstants.START_MS_PARAM); // For testing long startMs = endMs - msPerFile * 10; if (AnalysisUtility.areParametersValid(startMsStr)) { startMs = Long.parseLong(startMsStr); } String logLevel = req.getParameter(AnalysisConstants.LOG_LEVEL_PARAM); if (!AnalysisUtility.areParametersValid(logLevel)) { logLevel = getDefaultLogLevel(); } String deleteFromCloudStorage = req.getParameter(AnalysisConstants.DELETE_FROM_CLOUD_STORAGE_PARAM); // Verify that log level is one of the enum values or ALL if (!"ALL".equals(logLevel)) { LogLevel.valueOf(logLevel); } String bucketName = req.getParameter(AnalysisConstants.BUCKET_NAME_PARAM); if (!AnalysisUtility.areParametersValid(bucketName)) { bucketName = getDefaultBucketName(); } String bigqueryProjectId = req.getParameter(AnalysisConstants.BIGQUERY_PROJECT_ID_PARAM); if (!AnalysisUtility.areParametersValid(bigqueryProjectId)) { bigqueryProjectId = getDefaultBigqueryProjectId(); } String bigqueryDatasetId = req.getParameter(AnalysisConstants.BIGQUERY_DATASET_ID_PARAM); if (!AnalysisUtility.areParametersValid(bigqueryDatasetId)) { bigqueryDatasetId = getDefaultBigqueryDatasetId(); } String bigqueryFieldExporterSetBag = req .getParameter(AnalysisConstants.BIGQUERY_FIELD_EXPORTER_SET_BAG_PARAM); if (!AnalysisUtility.areParametersValid(bigqueryFieldExporterSetBag)) { bigqueryFieldExporterSetBag = this.getDefaultBigqueryFieldExporterSetBag(); } BigqueryFieldExporterSetBag exporterSetBag = AnalysisUtility .instantiateExporterSetBag(bigqueryFieldExporterSetBag); // If there is no bigqueryFieldExporterSetBag, we have to add default ExporterSet List to ExporterSetBag if (exporterSetBag.getExporterSetList() == null || exporterSetBag.getExporterSetList().size() <= 0) { exporterSetBag.setExporterSetList( Arrays.asList(AnalysisUtility.instantiateExporterSet(getDefaultBigqueryFieldExporterSet()))); } // Now, we have ExporterSetBag to support multiple export sets. for (BigqueryFieldExporterSet exporterSet : exporterSetBag.getExporterSetList()) { String schemaHash = AnalysisUtility.computeSchemaHash(exporterSet); String queueName = req.getParameter(AnalysisConstants.QUEUE_NAME_PARAM); if (!AnalysisUtility.areParametersValid(queueName)) { queueName = getDefaultQueueName(); } AppIdentityCredential credential = new AppIdentityCredential(AnalysisConstants.SCOPES); HttpRequestFactory requestFactory = HTTP_TRANSPORT.createRequestFactory(credential); List<String> urisToProcess = new ArrayList<String>(); AnalysisUtility.fetchCloudStorageLogUris(bucketName, String.format("%s_%s", exporterSet.getPrefix(), schemaHash), startMs, endMs, requestFactory, urisToProcess, true); long lastEndMsSeen = startMs - startMs % msPerFile; for (String uri : urisToProcess) { long uriEndMs = AnalysisUtility.getEndMsFromKey(uri); if (uriEndMs > lastEndMsSeen) { lastEndMsSeen = uriEndMs; } } List<String> fieldNames = new ArrayList<String>(); List<String> fieldTypes = new ArrayList<String>(); AnalysisUtility.populateSchema(exporterSet, fieldNames, fieldTypes); FileService fileService = FileServiceFactory.getFileService(); Queue taskQueue = QueueFactory.getQueue(queueName); int taskCount = 0; for (long currentStartMs = lastEndMsSeen; currentStartMs + msPerFile <= endMs; currentStartMs += msPerFile) { long tableStartMs = currentStartMs - currentStartMs % msPerTable; long tableEndMs = tableStartMs + msPerTable; String tableName = exporterSet.getPrefix(); String schemaKey = AnalysisUtility.createSchemaKey( String.format("%s_%s", exporterSet.getPrefix(), schemaHash), currentStartMs, currentStartMs + msPerFile); AnalysisUtility.writeSchema(fileService, bucketName, schemaKey, fieldNames, fieldTypes); TaskOptions taskOptions = Builder .withUrl(AnalysisUtility.getRequestBaseName(req) + "/storeLogsInCloudStorage") .method(Method.GET).param(AnalysisConstants.START_MS_PARAM, "" + currentStartMs) .param(AnalysisConstants.END_MS_PARAM, "" + (currentStartMs + msPerFile)) .param(AnalysisConstants.BUCKET_NAME_PARAM, bucketName) .param(AnalysisConstants.BIGQUERY_PROJECT_ID_PARAM, bigqueryProjectId) .param(AnalysisConstants.BIGQUERY_DATASET_ID_PARAM, bigqueryDatasetId) .param(AnalysisConstants.BIGQUERY_FIELD_EXPORTER_SET_PARAM, exporterSet.getClass().getName()) // .param(AnalysisConstants.BIGQUERY_FIELD_EXPORTER_SET_BAG_PARAM, bigqueryFieldExporterSetBag) .param(AnalysisConstants.QUEUE_NAME_PARAM, queueName) .param(AnalysisConstants.BIGQUERY_TABLE_ID_PARAM, tableName) .param(AnalysisConstants.LOG_LEVEL_PARAM, logLevel); if (AnalysisUtility.areParametersValid(deleteFromCloudStorage)) { taskOptions.param(AnalysisConstants.DELETE_FROM_CLOUD_STORAGE_PARAM, deleteFromCloudStorage); } taskQueue.add(taskOptions); taskCount += 1; } resp.getWriter().println("Successfully started " + taskCount + " tasks"); } }
From source file:com.streak.datastore.analysis.builtin.BuiltinDatastoreToBigqueryIngesterTask.java
License:Apache License
public void doGet(HttpServletRequest req, HttpServletResponse resp) throws IOException { resp.setContentType("application/json"); String timestampStr = req.getParameter(AnalysisConstants.TIMESTAMP_PARAM); long timestamp = 0; if (AnalysisUtility.areParametersValid(timestampStr)) { try {//from www . j a va 2s . c om timestamp = Long.parseLong(timestampStr); } catch (Exception e) { // leave it at default value } } if (timestamp == 0) { log.warning("Missing required param: " + AnalysisConstants.TIMESTAMP_PARAM); resp.setStatus(HttpServletResponse.SC_BAD_REQUEST); return; } String builtinDatastoreExportConfig = req.getParameter(AnalysisConstants.BUILTIN_DATASTORE_EXPORT_CONFIG); if (!AnalysisUtility.areParametersValid(builtinDatastoreExportConfig)) { log.warning("Missing required param: " + AnalysisConstants.BUILTIN_DATASTORE_EXPORT_CONFIG); resp.setStatus(HttpServletResponse.SC_BAD_REQUEST); return; } // Instantiate the export config BuiltinDatastoreExportConfiguration exporterConfig = AnalysisUtility .instantiateDatastoreExportConfig(builtinDatastoreExportConfig); String gsHandleOfBackup = checkAndGetCompletedBackupGSHandle( AnalysisUtility.getPreBackupName(timestamp, exporterConfig.getBackupNamePrefix())); if (gsHandleOfBackup == null) { log.warning("gsHandleOfBackup: null"); resp.getWriter().println( AnalysisUtility.successJson("backup incomplete, retrying in " + MILLIS_TO_ENQUEUE + " millis")); enqueueTask(AnalysisUtility.getRequestBaseName(req), exporterConfig, timestamp, MILLIS_TO_ENQUEUE); return; } log.warning("backup complete, starting bigquery ingestion"); log.warning("gsHandleOfBackup: " + gsHandleOfBackup); AppIdentityCredential credential = new AppIdentityCredential(AnalysisConstants.SCOPES); Bigquery bigquery = new Bigquery.Builder(HTTP_TRANSPORT, JSON_FACTORY, credential) .setApplicationName("Streak Logs").build(); String datatableSuffix = ""; if (exporterConfig.appendTimestampToDatatables()) { datatableSuffix = Long.toString(timestamp); } else { datatableSuffix = ""; } if (!exporterConfig.appendTimestampToDatatables()) { // we aren't appending the timestamps so delete the old tables if // they exist for (String kind : exporterConfig.getEntityKindsToExport()) { boolean found = true; try { bigquery.tables() .get(exporterConfig.getBigqueryProjectId(), exporterConfig.getBigqueryDatasetId(), kind) .execute(); } catch (IOException e) { // table not found so don't need to do anything found = false; } if (found) { bigquery.tables().delete(exporterConfig.getBigqueryProjectId(), exporterConfig.getBigqueryDatasetId(), kind).execute(); } } } // now create the ingestion for (String kind : exporterConfig.getEntityKindsToExport()) { String gsUrl = convertHandleToUrl(gsHandleOfBackup, kind); log.warning("gsUrl: " + gsUrl); Job job = new Job(); JobConfiguration config = new JobConfiguration(); JobConfigurationLoad loadConfig = new JobConfigurationLoad(); loadConfig.setSourceUris(Arrays.asList(gsUrl)); loadConfig.set("sourceFormat", "DATASTORE_BACKUP"); loadConfig.set("allowQuotedNewlines", true); TableReference table = new TableReference(); table.setProjectId(exporterConfig.getBigqueryProjectId()); table.setDatasetId(exporterConfig.getBigqueryDatasetId()); table.setTableId(kind + datatableSuffix); loadConfig.setDestinationTable(table); config.setLoad(loadConfig); job.setConfiguration(config); Insert insert = bigquery.jobs().insert(exporterConfig.getBigqueryProjectId(), job); JobReference jr = insert.execute().getJobReference(); log.warning("Uri: " + gsUrl + ", JobId: " + jr.getJobId()); } }
From source file:com.streak.logging.analysis.BigqueryStatusServlet.java
License:Apache License
public void doGet(HttpServletRequest req, HttpServletResponse resp) throws IOException { resp.setContentType("application/json"); AppIdentityCredential credential = new AppIdentityCredential(AnalysisConstants.SCOPES); Bigquery bigquery = new Bigquery.Builder(HTTP_TRANSPORT, JSON_FACTORY, credential) .setApplicationName("Streak Logs").build(); String jobId = req.getParameter(AnalysisConstants.JOB_ID_PARAM); Object retVal = null;/*from w ww. j a v a2 s.c o m*/ if (jobId == null) { retVal = listAllJobs(resp, bigquery); } else { retVal = listJob(resp, bigquery, jobId); } resp.getWriter().println(new Gson().toJson(retVal)); }
From source file:com.streak.logging.analysis.LoadCloudStorageToBigqueryTask.java
License:Apache License
public void doGet(HttpServletRequest req, HttpServletResponse resp) { resp.setContentType("text/plain"); String queueName = AnalysisUtility.extractParameterOrThrow(req, AnalysisConstants.QUEUE_NAME_PARAM); String bigqueryProjectId = AnalysisUtility.extractParameterOrThrow(req, AnalysisConstants.BIGQUERY_PROJECT_ID_PARAM); String bigqueryDatasetId = AnalysisUtility.extractParameterOrThrow(req, AnalysisConstants.BIGQUERY_DATASET_ID_PARAM); String bigqueryTableId = AnalysisUtility.extractParameterOrThrow(req, AnalysisConstants.BIGQUERY_TABLE_ID_PARAM); String schemaHash = AnalysisUtility.extractParameterOrThrow(req, AnalysisConstants.SCHEMA_HASH_PARAM); MemcacheService memcache = MemcacheServiceFactory.getMemcacheService(AnalysisConstants.MEMCACHE_NAMESPACE); Long nextBigQueryJobTime = (Long) memcache.increment(AnalysisConstants.LAST_BIGQUERY_JOB_TIME, AnalysisConstants.LOAD_DELAY_MS, System.currentTimeMillis()); long currentTime = System.currentTimeMillis(); // The task queue has waited a long time to run us. Go ahead and reset // the last job time // to prevent a race. if (currentTime > nextBigQueryJobTime + AnalysisConstants.LOAD_DELAY_MS / 2) { memcache.put(AnalysisConstants.LAST_BIGQUERY_JOB_TIME, currentTime); nextBigQueryJobTime = currentTime + AnalysisConstants.LOAD_DELAY_MS; }// w w w . j av a2 s. com if (currentTime < nextBigQueryJobTime) { memcache.increment(AnalysisConstants.LAST_BIGQUERY_JOB_TIME, -AnalysisConstants.LOAD_DELAY_MS); Queue taskQueue = QueueFactory.getQueue(queueName); taskQueue.add(Builder.withUrl( AnalysisUtility.getRequestBaseName(req) + "/loadCloudStorageToBigquery?" + req.getQueryString()) .method(Method.GET).etaMillis(nextBigQueryJobTime)); log.info("Rate limiting BigQuery load job - will retry at " + nextBigQueryJobTime); return; } String bucketName = AnalysisUtility.extractParameterOrThrow(req, AnalysisConstants.BUCKET_NAME_PARAM); AppIdentityCredential credential = new AppIdentityCredential(AnalysisConstants.SCOPES); HttpRequestFactory requestFactory = HTTP_TRANSPORT.createRequestFactory(credential); List<String> urisToProcess = new ArrayList<String>(); String startMsStr = AnalysisUtility.extractParameterOrThrow(req, AnalysisConstants.START_MS_PARAM); long startMs = Long.parseLong(startMsStr); String endMsStr = AnalysisUtility.extractParameterOrThrow(req, AnalysisConstants.END_MS_PARAM); long endMs = Long.parseLong(endMsStr); try { urisToProcess = AnalysisUtility.fetchCloudStorageLogUris(bucketName, schemaHash, startMs, endMs, requestFactory); log.info("Got " + urisToProcess.size() + " uris to process"); if (urisToProcess.isEmpty()) { log.info("No URI's to process"); return; } for (String uri : urisToProcess) { log.info("URI: " + uri); } Bigquery bigquery = new Bigquery.Builder(HTTP_TRANSPORT, JSON_FACTORY, credential) .setApplicationName(SystemProperty.applicationId.get()).build(); Job job = new Job(); JobConfiguration config = new JobConfiguration(); JobConfigurationLoad loadConfig = new JobConfigurationLoad(); loadConfig.setSourceUris(urisToProcess); loadConfig.set("allowQuotedNewlines", true); loadConfig.setSourceFormat("NEWLINE_DELIMITED_JSON"); ObjectMapper mapper = new ObjectMapper(); List<TableFieldSchema> schemaFields = mapper.readValue( loadSchemaString("gs://" + bucketName + "/" + schemaHash + ".schema.json"), new TypeReference<List<TableFieldSchema>>() { }); TableSchema tableSchema = new TableSchema().setFields(schemaFields); loadConfig.setSchema(tableSchema); TableReference table = new TableReference(); table.setProjectId(bigqueryProjectId); table.setDatasetId(bigqueryDatasetId); table.setTableId(bigqueryTableId); loadConfig.setDestinationTable(table); config.setLoad(loadConfig); job.setConfiguration(config); Insert insert = bigquery.jobs().insert(bigqueryProjectId, job); // TODO(frew): Not sure this is necessary, but monkey-see'ing the // example code insert.setProjectId(bigqueryProjectId); JobReference ref = insert.execute().getJobReference(); log.info("Successfully started job " + ref); String shouldDeleteString = req.getParameter(AnalysisConstants.DELETE_FROM_CLOUD_STORAGE_PARAM); boolean shouldDelete = (AnalysisUtility.areParametersValid(shouldDeleteString) && shouldDeleteString.equals("true")) || System.getProperty("bigquerylogging.default.deleteFromCloudStorage").equals("true"); if (shouldDelete) { Queue taskQueue = QueueFactory.getQueue(queueName); taskQueue.add(Builder .withUrl(AnalysisUtility.getRequestBaseName(req) + "/deleteCompletedCloudStorageFilesTask") .method(Method.GET).param(AnalysisConstants.BIGQUERY_JOB_ID_PARAM, ref.getJobId()) .param(AnalysisConstants.QUEUE_NAME_PARAM, queueName) .param(AnalysisConstants.BIGQUERY_PROJECT_ID_PARAM, bigqueryProjectId)); } } catch (IOException e) { log.warning("An error occurred while loading cloud storage to Bigquery. Retrying."); Queue taskQueue = QueueFactory.getQueue(queueName); taskQueue.add(Builder.withUrl(req.getRequestURL().toString())); } }
From source file:com.streak.logging.analysis.LogExportCronTask.java
License:Apache License
public void doGet(HttpServletRequest req, HttpServletResponse resp) { resp.setContentType("text/plain"); String msPerTableStr = req.getParameter(AnalysisConstants.MS_PER_TABLE_PARAM); long msPerTable = 1000 * 60 * 60 * 24; if (AnalysisUtility.areParametersValid(msPerTableStr)) { msPerTable = Long.parseLong(msPerTableStr); }/*from w w w .j av a 2 s . co m*/ String msPerFileStr = req.getParameter(AnalysisConstants.MS_PER_FILE_PARAM); long msPerFile = 1000 * 60 * 2; if (AnalysisUtility.areParametersValid(msPerFileStr)) { msPerFile = Long.parseLong(msPerFileStr); } if (msPerTable % msPerFile != 0) { throw new InvalidTaskParameterException("The " + AnalysisConstants.MS_PER_FILE_PARAM + " parameter must divide the " + AnalysisConstants.MS_PER_TABLE_PARAM + " parameter."); } String endMsStr = req.getParameter(AnalysisConstants.END_MS_PARAM); long endMs = System.currentTimeMillis(); if (AnalysisUtility.areParametersValid(endMsStr)) { endMs = Long.parseLong(endMsStr); } // By default look back a ways, but safely under the limit of 1000 files // per listing that Cloud Storage imposes String startMsStr = req.getParameter(AnalysisConstants.START_MS_PARAM); // For testing long startMs = endMs - msPerFile * 10; if (AnalysisUtility.areParametersValid(startMsStr)) { startMs = Long.parseLong(startMsStr); } String logLevel = req.getParameter(AnalysisConstants.LOG_LEVEL_PARAM); if (!AnalysisUtility.areParametersValid(logLevel)) { logLevel = getDefaultLogLevel(); } String deleteFromCloudStorage = req.getParameter(AnalysisConstants.DELETE_FROM_CLOUD_STORAGE_PARAM); // Verify that log level is one of the enum values or ALL if (!"ALL".equals(logLevel)) { LogLevel.valueOf(logLevel); } String bucketName = req.getParameter(AnalysisConstants.BUCKET_NAME_PARAM); if (!AnalysisUtility.areParametersValid(bucketName)) { bucketName = getDefaultBucketName(); } String bigqueryProjectId = req.getParameter(AnalysisConstants.BIGQUERY_PROJECT_ID_PARAM); if (!AnalysisUtility.areParametersValid(bigqueryProjectId)) { bigqueryProjectId = getDefaultBigqueryProjectId(); } String bigqueryDatasetId = req.getParameter(AnalysisConstants.BIGQUERY_DATASET_ID_PARAM); if (!AnalysisUtility.areParametersValid(bigqueryDatasetId)) { bigqueryDatasetId = getDefaultBigqueryDatasetId(); } String bigqueryFieldExporterSet = req.getParameter(AnalysisConstants.BIGQUERY_FIELD_EXPORTER_SET_PARAM); if (!AnalysisUtility.areParametersValid(bigqueryFieldExporterSet)) { bigqueryFieldExporterSet = getDefaultBigqueryFieldExporterSet(); } // Instantiate the exporter set to detect errors before we spawn a bunch // of tasks. BigqueryFieldExporterSet exporterSet = AnalysisUtility.instantiateExporterSet(bigqueryFieldExporterSet); String schemaHash = AnalysisUtility.computeSchemaHash(exporterSet); String queueName = req.getParameter(AnalysisConstants.QUEUE_NAME_PARAM); if (!AnalysisUtility.areParametersValid(queueName)) { queueName = getDefaultQueueName(); } String appVersionsToExport = req.getParameter(AnalysisConstants.APPLICATION_VERSIONS_TO_EXPORT_PARAM); if (!AnalysisUtility.areParametersValid(appVersionsToExport)) { appVersionsToExport = getDefaultAppVersionsToExport(); } AppIdentityCredential credential = new AppIdentityCredential(AnalysisConstants.SCOPES); HttpRequestFactory requestFactory = HTTP_TRANSPORT.createRequestFactory(credential); try { List<String> urisToProcess = AnalysisUtility.fetchCloudStorageLogUris(bucketName, schemaHash, startMs, endMs, requestFactory); long lastEndMsSeen = startMs - startMs % msPerFile; for (String uri : urisToProcess) { long uriEndMs = AnalysisUtility.getEndMsFromKey(uri); if (uriEndMs > lastEndMsSeen) { lastEndMsSeen = uriEndMs; } } List<String> fieldNames = new ArrayList<String>(); List<String> fieldTypes = new ArrayList<String>(); List<BigqueryFieldExporter> fieldExporters = new ArrayList<BigqueryFieldExporter>(); AnalysisUtility.populateSchema(exporterSet, fieldNames, fieldTypes, fieldExporters); FileService fileService = FileServiceFactory.getFileService(); String schemaKey = AnalysisUtility.createSchemaKey(schemaHash); AnalysisUtility.writeSchema(fileService, bucketName, schemaKey, fieldNames, fieldTypes, fieldExporters); Queue taskQueue = QueueFactory.getQueue(queueName); int taskCount = 0; for (long currentStartMs = lastEndMsSeen; currentStartMs + msPerFile <= endMs; currentStartMs += msPerFile) { long tableStartMs = currentStartMs - currentStartMs % msPerTable; long tableEndMs = tableStartMs + msPerTable; String tableName = AnalysisUtility.createLogKey(schemaHash, tableStartMs, tableEndMs); TaskOptions taskOptions = Builder .withUrl(AnalysisUtility.getRequestBaseName(req) + "/storeLogsInCloudStorage") .method(Method.GET).param(AnalysisConstants.START_MS_PARAM, "" + currentStartMs) .param(AnalysisConstants.END_MS_PARAM, "" + (currentStartMs + msPerFile)) .param(AnalysisConstants.BUCKET_NAME_PARAM, bucketName) .param(AnalysisConstants.BIGQUERY_PROJECT_ID_PARAM, bigqueryProjectId) .param(AnalysisConstants.BIGQUERY_DATASET_ID_PARAM, bigqueryDatasetId) .param(AnalysisConstants.BIGQUERY_FIELD_EXPORTER_SET_PARAM, bigqueryFieldExporterSet) .param(AnalysisConstants.QUEUE_NAME_PARAM, queueName) .param(AnalysisConstants.APPLICATION_VERSIONS_TO_EXPORT_PARAM, appVersionsToExport) .param(AnalysisConstants.BIGQUERY_TABLE_ID_PARAM, tableName) .param(AnalysisConstants.SCHEMA_HASH_PARAM, schemaHash) .param(AnalysisConstants.LOG_LEVEL_PARAM, logLevel); if (AnalysisUtility.areParametersValid(deleteFromCloudStorage)) { taskOptions.param(AnalysisConstants.DELETE_FROM_CLOUD_STORAGE_PARAM, deleteFromCloudStorage); } taskQueue.add(taskOptions); taskCount += 1; } // write to response as well as logs (for feedback when running from browser) resp.getWriter().println("Successfully started " + taskCount + " tasks"); log.info("Successfully started " + taskCount + " tasks"); } catch (IOException e) { log.warning("Error while running cron job."); } }
From source file:foo.domaintest.bigquery.BigQueryModule.java
License:Open Source License
/** Provides the authenticated Bigquery API object. */ @Provides/*w w w .ja v a 2 s . c o m*/ @Singleton Bigquery provideBigquery() { return new Bigquery.Builder(new UrlFetchTransport(), new JacksonFactory(), new AppIdentityCredential(BigqueryScopes.all())).setApplicationName(BigQueryModule.class.getName()) .build(); }