Java tutorial
/* * * Copyright 2016 Netflix, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. * */ package com.netflix.genie.web.controllers; import com.fasterxml.jackson.databind.JsonNode; import com.google.common.collect.Lists; import com.google.common.collect.Sets; import com.netflix.genie.common.dto.Application; import com.netflix.genie.common.dto.ApplicationStatus; import com.netflix.genie.common.dto.Cluster; import com.netflix.genie.common.dto.ClusterCriteria; import com.netflix.genie.common.dto.ClusterStatus; import com.netflix.genie.common.dto.Command; import com.netflix.genie.common.dto.CommandStatus; import com.netflix.genie.common.dto.JobExecution; import com.netflix.genie.common.dto.JobRequest; import com.netflix.genie.common.dto.JobStatus; import com.netflix.genie.common.dto.JobStatusMessages; import com.netflix.genie.common.internal.util.GenieHostInfo; import com.netflix.genie.common.util.GenieObjectMapper; import com.netflix.genie.web.properties.FileCacheProperties; import com.netflix.genie.web.properties.JobsLocationsProperties; import io.restassured.RestAssured; import io.restassured.http.ContentType; import io.restassured.specification.RequestSpecification; import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.SystemUtils; import org.hamcrest.Matchers; import org.junit.After; import org.junit.Assert; import org.junit.Assume; import org.junit.Before; import org.junit.Rule; import org.junit.Test; import org.junit.rules.TemporaryFolder; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.core.io.DefaultResourceLoader; import org.springframework.core.io.Resource; import org.springframework.core.io.ResourceLoader; import org.springframework.hateoas.MediaTypes; import org.springframework.http.HttpHeaders; import org.springframework.http.HttpStatus; import org.springframework.http.MediaType; import org.springframework.mock.web.MockMultipartFile; import org.springframework.restdocs.headers.HeaderDocumentation; import org.springframework.restdocs.payload.PayloadDocumentation; import org.springframework.restdocs.request.RequestDocumentation; import org.springframework.restdocs.restassured3.RestAssuredRestDocumentation; import org.springframework.restdocs.restassured3.RestDocumentationFilter; import javax.annotation.Nullable; import java.io.File; import java.io.FileInputStream; import java.io.InputStream; import java.net.URI; import java.net.URISyntaxException; import java.nio.charset.Charset; import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; import java.time.Instant; import java.util.ArrayList; import java.util.Arrays; import java.util.List; import java.util.Set; import java.util.UUID; /** * Integration tests for Jobs REST API. * * @author amsharma * @author tgianos * @since 3.0.0 */ public class JobRestControllerIntegrationTest extends RestControllerIntegrationTestBase { private static final Logger LOG = LoggerFactory.getLogger(JobRestControllerIntegrationTest.class); private static final long SLEEP_TIME = 1000L; private static final String SCHEDULER_JOB_NAME_KEY = "schedulerJobName"; private static final String SCHEDULER_RUN_ID_KEY = "schedulerRunId"; private static final String COMMAND_ARGS_PATH = "commandArgs"; private static final String STATUS_MESSAGE_PATH = "statusMsg"; private static final String CLUSTER_NAME_PATH = "clusterName"; private static final String COMMAND_NAME_PATH = "commandName"; private static final String ARCHIVE_LOCATION_PATH = "archiveLocation"; private static final String STARTED_PATH = "started"; private static final String FINISHED_PATH = "finished"; private static final String CLUSTER_CRITERIAS_PATH = "clusterCriterias"; private static final String COMMAND_CRITERIA_PATH = "commandCriteria"; private static final String GROUP_PATH = "group"; private static final String DISABLE_LOG_ARCHIVAL_PATH = "disableLogArchival"; private static final String EMAIL_PATH = "email"; private static final String CPU_PATH = "cpu"; private static final String MEMORY_PATH = "memory"; private static final String APPLICATIONS_PATH = "applications"; private static final String HOST_NAME_PATH = "hostName"; private static final String PROCESS_ID_PATH = "processId"; private static final String CHECK_DELAY_PATH = "checkDelay"; private static final String EXIT_CODE_PATH = "exitCode"; private static final String CLIENT_HOST_PATH = "clientHost"; private static final String USER_AGENT_PATH = "userAgent"; private static final String NUM_ATTACHMENTS_PATH = "numAttachments"; private static final String TOTAL_SIZE_ATTACHMENTS_PATH = "totalSizeOfAttachments"; private static final String STD_OUT_SIZE_PATH = "stdOutSize"; private static final String STD_ERR_SIZE_PATH = "stdErrSize"; private static final String JOBS_LIST_PATH = EMBEDDED_PATH + ".jobSearchResultList"; private static final String GROUPING_PATH = "grouping"; private static final String GROUPING_INSTANCE_PATH = "groupingInstance"; private static final String JOB_COMMAND_LINK_PATH = "_links.command.href"; private static final String JOB_CLUSTER_LINK_PATH = "_links.cluster.href"; private static final String JOB_APPLICATIONS_LINK_PATH = "_links.applications.href"; private static final long CHECK_DELAY = 500L; private static final String BASE_DIR = "com/netflix/genie/web/controllers/JobRestControllerIntegrationTests/"; private static final String FILE_DELIMITER = "/"; private static final String LOCALHOST_CLUSTER_TAG = "localhost"; private static final String BASH_COMMAND_TAG = "bash"; private static final String JOB_NAME = "List * ... Directories bash job"; private static final String JOB_USER = "genie"; private static final String JOB_VERSION = "1.0"; private static final String JOB_DESCRIPTION = "Genie 3 Test Job"; private static final String JOB_STATUS_MSG = JobStatusMessages.JOB_FINISHED_SUCCESSFULLY; private static final String APP1_ID = "app1"; private static final String APP1_NAME = "Application 1"; private static final String APP1_USER = "genie"; private static final String APP1_VERSION = "1.0"; private static final String APP2_ID = "app2"; private static final String APP2_NAME = "Application 2"; private static final String CMD1_ID = "cmd1"; private static final String CMD1_NAME = "Unix Bash command"; private static final String CMD1_USER = "genie"; private static final String CMD1_VERSION = "1.0"; private static final String CMD1_EXECUTABLE = "/bin/bash"; private static final String CMD1_TAGS = BASH_COMMAND_TAG + "," + "genie.id:" + CMD1_ID + "," + "genie.name:" + CMD1_NAME; private static final String CLUSTER1_ID = "cluster1"; private static final String CLUSTER1_NAME = "Local laptop"; private static final String CLUSTER1_USER = "genie"; private static final String CLUSTER1_VERSION = "1.0"; private static final String CLUSTER1_TAGS = "genie.id:" + CLUSTER1_ID + "," + "genie.name:" + CLUSTER1_NAME + "," + LOCALHOST_CLUSTER_TAG; private static final String JOB_TAG_1 = "aTag"; private static final String JOB_TAG_2 = "zTag"; private static final Set<String> JOB_TAGS = Sets.newHashSet(JOB_TAG_1, JOB_TAG_2); private static final String JOB_GROUPING = UUID.randomUUID().toString(); private static final String JOB_GROUPING_INSTANCE = UUID.randomUUID().toString(); // This file is not UTF-8 encoded. It is uploaded to test server behavior // related to charset headers private static final String GB18030_TXT = "GB18030.txt"; /** * A temporary directory to use that will be cleaned up automatically at the end of testing. */ @Rule public TemporaryFolder temporaryFolder = new TemporaryFolder(); private ResourceLoader resourceLoader; private JsonNode metadata; private String schedulerJobName; private String schedulerRunId; @Autowired private GenieHostInfo genieHostInfo; @Autowired private Resource jobDirResource; @Autowired private FileCacheProperties fileCacheProperties; @Autowired private JobsLocationsProperties jobsLocationsProperties; /** * {@inheritDoc} */ @Before @Override public void setup() throws Exception { super.setup(); // Re-point archives this.jobsLocationsProperties.setArchives(this.temporaryFolder.newFolder().toURI().toString()); this.schedulerJobName = UUID.randomUUID().toString(); this.schedulerRunId = UUID.randomUUID().toString(); this.metadata = GenieObjectMapper.getMapper().readTree("{\"" + SCHEDULER_JOB_NAME_KEY + "\":\"" + this.schedulerJobName + "\", \"" + SCHEDULER_RUN_ID_KEY + "\":\"" + this.schedulerRunId + "\"}"); this.resourceLoader = new DefaultResourceLoader(); this.createAnApplication(APP1_ID, APP1_NAME); this.createAnApplication(APP2_ID, APP2_NAME); this.createAllClusters(); this.createAllCommands(); this.linkAllEntities(); } /** * {@inheritDoc} */ @After @Override public void cleanup() throws Exception { super.cleanup(); } /** * Test the job submit method for success. * * @throws Exception If there is a problem. */ @Test public void testSubmitJobMethodSuccess() throws Exception { this.submitAndCheckJob(1, true); } /** * Test to make sure command args are limitted to 10,000 characters. * * @throws Exception On error */ @Test public void testForTooManyCommandArgs() throws Exception { final JobRequest tooManyCommandArguments = new JobRequest.Builder(JOB_NAME, JOB_USER, JOB_VERSION, Lists.newArrayList(new ClusterCriteria(Sets.newHashSet(LOCALHOST_CLUSTER_TAG))), Sets.newHashSet(BASH_COMMAND_TAG)).withCommandArgs(StringUtils.leftPad("bad", 10_001)).build(); RestAssured.given(this.getRequestSpecification()).contentType(MediaType.APPLICATION_JSON_VALUE) .body(GenieObjectMapper.getMapper().writeValueAsBytes(tooManyCommandArguments)).when() .port(this.port).post(JOBS_API).then() .statusCode(Matchers.is(HttpStatus.PRECONDITION_FAILED.value())); } private void submitAndCheckJob(final int documentationId, final boolean archiveJob) throws Exception { Assume.assumeTrue(SystemUtils.IS_OS_UNIX); final List<String> commandArgs = Lists.newArrayList("-c", "'echo hello world'"); final String clusterTag = LOCALHOST_CLUSTER_TAG; final List<ClusterCriteria> clusterCriteriaList = Lists .newArrayList(new ClusterCriteria(Sets.newHashSet(clusterTag))); final String setUpFile = this.resourceLoader.getResource(BASE_DIR + "job" + FILE_DELIMITER + "jobsetupfile") .getFile().getAbsolutePath(); final String configFile1 = this.resourceLoader.getResource(BASE_DIR + "job" + FILE_DELIMITER + "config1") .getFile().getAbsolutePath(); final Set<String> configs = Sets.newHashSet(configFile1); final String depFile1 = this.resourceLoader.getResource(BASE_DIR + "job" + FILE_DELIMITER + "dep1") .getFile().getAbsolutePath(); final Set<String> dependencies = Sets.newHashSet(depFile1); final String commandTag = BASH_COMMAND_TAG; final Set<String> commandCriteria = Sets.newHashSet(commandTag); final JobRequest jobRequest = new JobRequest.Builder(JOB_NAME, JOB_USER, JOB_VERSION, clusterCriteriaList, commandCriteria).withCommandArgs(commandArgs).withDisableLogArchival(!archiveJob) .withSetupFile(setUpFile).withConfigs(configs).withDependencies(dependencies) .withDescription(JOB_DESCRIPTION).withMetadata(this.metadata).withTags(JOB_TAGS) .withGrouping(JOB_GROUPING).withGroupingInstance(JOB_GROUPING_INSTANCE).build(); final String id = this.submitJob(documentationId, jobRequest, null); this.waitForDone(id); this.checkJobStatus(documentationId, id); this.checkJob(documentationId, id, commandArgs, archiveJob); if (archiveJob) { this.checkJobOutput(documentationId, id); } this.checkJobRequest(documentationId, id, commandArgs, setUpFile, clusterTag, commandTag, configFile1, depFile1, archiveJob); this.checkJobExecution(documentationId, id); this.checkJobMetadata(documentationId, id); this.checkJobCluster(documentationId, id); this.checkJobCommand(documentationId, id); this.checkJobApplications(documentationId, id); this.checkFindJobs(documentationId, id, JOB_USER); this.checkJobArchive(id, archiveJob); Assert.assertThat(this.jobRepository.count(), Matchers.is(1L)); // Check if the cluster setup file is cached final String clusterSetUpFilePath = this.resourceLoader .getResource(BASE_DIR + CMD1_ID + FILE_DELIMITER + "setupfile").getFile().getAbsolutePath(); Assert.assertTrue(Files.exists(Paths.get(new URI(this.fileCacheProperties.getLocation()).getPath(), UUID.nameUUIDFromBytes(clusterSetUpFilePath.getBytes(Charset.forName("UTF-8"))).toString()))); // Test for conflicts this.testForConflicts(id, commandArgs, clusterCriteriaList, commandCriteria); } private String submitJob(final int documentationId, final JobRequest jobRequest, @Nullable final List<MockMultipartFile> attachments) throws Exception { if (attachments != null) { final RestDocumentationFilter createResultFilter = RestAssuredRestDocumentation.document( "{class-name}/" + documentationId + "/submitJobWithAttachments/", HeaderDocumentation.requestHeaders(HeaderDocumentation.headerWithName(HttpHeaders.CONTENT_TYPE) .description(MediaType.MULTIPART_FORM_DATA_VALUE)), // Request headers RequestDocumentation.requestParts( RequestDocumentation.partWithName("request").description( "The job request JSON. Content type must be application/json for part"), RequestDocumentation.partWithName("attachment").description( "An attachment file. There can be multiple. Type should be octet-stream")), // Request parts Snippets.LOCATION_HEADER // Response Headers ); final RequestSpecification jobRequestSpecification = RestAssured.given(this.getRequestSpecification()) .filter(createResultFilter).contentType(MediaType.MULTIPART_FORM_DATA_VALUE) .multiPart("request", GenieObjectMapper.getMapper().writeValueAsString(jobRequest), MediaType.APPLICATION_JSON_VALUE); for (final MockMultipartFile attachment : attachments) { jobRequestSpecification.multiPart("attachment", attachment.getOriginalFilename(), attachment.getBytes(), MediaType.APPLICATION_OCTET_STREAM_VALUE); } return this.getIdFromLocation(jobRequestSpecification.when().port(this.port).post(JOBS_API).then() .statusCode(Matchers.is(HttpStatus.ACCEPTED.value())) .header(HttpHeaders.LOCATION, Matchers.notNullValue()).extract().header(HttpHeaders.LOCATION)); } else { // Use regular POST final RestDocumentationFilter createResultFilter = RestAssuredRestDocumentation.document( "{class-name}/" + documentationId + "/submitJobWithoutAttachments/", Snippets.CONTENT_TYPE_HEADER, // Request headers Snippets.getJobRequestRequestPayload(), // Request Fields Snippets.LOCATION_HEADER // Response Headers ); return this.getIdFromLocation(RestAssured.given(this.getRequestSpecification()) .filter(createResultFilter).contentType(MediaType.APPLICATION_JSON_VALUE) .body(GenieObjectMapper.getMapper().writeValueAsBytes(jobRequest)).when().port(this.port) .post(JOBS_API).then().statusCode(Matchers.is(HttpStatus.ACCEPTED.value())) .header(HttpHeaders.LOCATION, Matchers.notNullValue()).extract().header(HttpHeaders.LOCATION)); } } private void checkJobStatus(final int documentationId, final String id) { final RestDocumentationFilter getResultFilter = RestAssuredRestDocumentation.document( "{class-name}/" + documentationId + "/getJobStatus/", Snippets.ID_PATH_PARAM, // Path parameters Snippets.JSON_CONTENT_TYPE_HEADER, // Response Headers PayloadDocumentation.responseFields(PayloadDocumentation.fieldWithPath("status") .description("The job status. One of: " + Arrays.toString(JobStatus.values())) .attributes(Snippets.EMPTY_CONSTRAINTS)) // Response fields ); RestAssured.given(this.getRequestSpecification()).filter(getResultFilter).when().port(this.port) .get(JOBS_API + "/{id}/status", id).then() .contentType(Matchers.containsString(MediaType.APPLICATION_JSON_VALUE)) .body(STATUS_PATH, Matchers.is(JobStatus.SUCCEEDED.toString())); } private void checkJob(final int documentationId, final String id, final List<String> commandArgs, final boolean archiveJob) { final RestDocumentationFilter getResultFilter = RestAssuredRestDocumentation.document( "{class-name}/" + documentationId + "/getJob/", Snippets.ID_PATH_PARAM, // Path parameters Snippets.HAL_CONTENT_TYPE_HEADER, // Response Headers Snippets.getJobResponsePayload(), // Response fields Snippets.JOB_LINKS // Links ); RestAssured.given(this.getRequestSpecification()).filter(getResultFilter).when().port(this.port) .get(JOBS_API + "/{id}", id).then().statusCode(Matchers.is(HttpStatus.OK.value())) .contentType(Matchers.is(MediaTypes.HAL_JSON_UTF8_VALUE)).body(ID_PATH, Matchers.is(id)) .body(CREATED_PATH, Matchers.notNullValue()).body(UPDATED_PATH, Matchers.notNullValue()) .body(VERSION_PATH, Matchers.is(JOB_VERSION)).body(USER_PATH, Matchers.is(JOB_USER)) .body(NAME_PATH, Matchers.is(JOB_NAME)).body(DESCRIPTION_PATH, Matchers.is(JOB_DESCRIPTION)) .body(METADATA_PATH + "." + SCHEDULER_JOB_NAME_KEY, Matchers.is(this.schedulerJobName)) .body(METADATA_PATH + "." + SCHEDULER_RUN_ID_KEY, Matchers.is(this.schedulerRunId)) .body(COMMAND_ARGS_PATH, Matchers.is(StringUtils.join(commandArgs, StringUtils.SPACE))) .body(STATUS_PATH, Matchers.is(JobStatus.SUCCEEDED.toString())) .body(STATUS_MESSAGE_PATH, Matchers.is(JOB_STATUS_MSG)) .body(STARTED_PATH, Matchers.not(Instant.EPOCH)).body(FINISHED_PATH, Matchers.notNullValue()) // TODO: Flipped during V4 migration to always be on to replecate expected behavior of V3 until clients // can be migrated // .body(ARCHIVE_LOCATION_PATH, archiveJob ? Matchers.notNullValue() : Matchers.isEmptyOrNullString()) .body(ARCHIVE_LOCATION_PATH, Matchers.notNullValue()) .body(CLUSTER_NAME_PATH, Matchers.is(CLUSTER1_NAME)).body(COMMAND_NAME_PATH, Matchers.is(CMD1_NAME)) .body(TAGS_PATH, Matchers.contains(JOB_TAG_1, JOB_TAG_2)) .body(GROUPING_PATH, Matchers.is(JOB_GROUPING)) .body(GROUPING_INSTANCE_PATH, Matchers.is(JOB_GROUPING_INSTANCE)) .body(LINKS_PATH + ".keySet().size()", Matchers.is(9)) .body(LINKS_PATH, Matchers.hasKey(SELF_LINK_KEY)).body(LINKS_PATH, Matchers.hasKey("request")) .body(LINKS_PATH, Matchers.hasKey("execution")).body(LINKS_PATH, Matchers.hasKey("output")) .body(LINKS_PATH, Matchers.hasKey("status")).body(LINKS_PATH, Matchers.hasKey("cluster")) .body(LINKS_PATH, Matchers.hasKey("command")).body(LINKS_PATH, Matchers.hasKey("applications")) .body(LINKS_PATH, Matchers.hasKey("metadata")) .body(JOB_CLUSTER_LINK_PATH, EntityLinkMatcher.matchUri(JOBS_API, "cluster", null, id)) .body(JOB_COMMAND_LINK_PATH, EntityLinkMatcher.matchUri(JOBS_API, "command", null, id)) .body(JOB_APPLICATIONS_LINK_PATH, EntityLinkMatcher.matchUri(JOBS_API, APPLICATIONS_LINK_KEY, null, id)); } private void checkJobOutput(final int documentationId, final String id) throws Exception { // Check getting a directory as json final RestDocumentationFilter jsonResultFilter = RestAssuredRestDocumentation.document( "{class-name}/" + documentationId + "/getJobOutput/json/", Snippets.ID_PATH_PARAM.and(RequestDocumentation.parameterWithName("filePath") .description("The path to the directory to get").optional()), // Path parameters HeaderDocumentation.requestHeaders(HeaderDocumentation.headerWithName(HttpHeaders.ACCEPT) .description(MediaType.APPLICATION_JSON_VALUE).optional()), // Request header HeaderDocumentation.responseHeaders(HeaderDocumentation.headerWithName(HttpHeaders.CONTENT_TYPE) .description(MediaType.APPLICATION_JSON_VALUE)), // Response Headers Snippets.OUTPUT_DIRECTORY_FIELDS); RestAssured.given(this.getRequestSpecification()).filter(jsonResultFilter) .accept(MediaType.APPLICATION_JSON_VALUE).when().port(this.port) .get(JOBS_API + "/{id}/output/{filePath}", id, "").then() .statusCode(Matchers.is(HttpStatus.OK.value())) .contentType(Matchers.equalToIgnoringCase(MediaType.APPLICATION_JSON_UTF8_VALUE)) .body("parent", Matchers.isEmptyOrNullString()).body("directories[0].name", Matchers.is("genie/")) .body("files[0].name", Matchers.is("config1")).body("files[1].name", Matchers.is("dep1")) .body("files[2].name", Matchers.is("jobsetupfile")).body("files[3].name", Matchers.is("run")) .body("files[4].name", Matchers.is("stderr")).body("files[5].name", Matchers.is("stdout")); // Check getting a directory as HTML final RestDocumentationFilter htmlResultFilter = RestAssuredRestDocumentation.document( "{class-name}/" + documentationId + "/getJobOutput/html/", Snippets.ID_PATH_PARAM.and(RequestDocumentation.parameterWithName("filePath") .description("The path to the directory to get").optional()), // Path parameters HeaderDocumentation.requestHeaders( HeaderDocumentation.headerWithName(HttpHeaders.ACCEPT).description(MediaType.TEXT_HTML)), // Request header HeaderDocumentation.responseHeaders(HeaderDocumentation.headerWithName(HttpHeaders.CONTENT_TYPE) .description(MediaType.TEXT_HTML)) // Response Headers ); RestAssured.given(this.getRequestSpecification()).filter(htmlResultFilter).accept(MediaType.TEXT_HTML_VALUE) .when().port(this.port).get(JOBS_API + "/{id}/output/{filePath}", id, "").then() .statusCode(Matchers.is(HttpStatus.OK.value())) .contentType(Matchers.containsString(MediaType.TEXT_HTML_VALUE)); // Check getting a file final RestDocumentationFilter fileResultFilter = RestAssuredRestDocumentation.document( "{class-name}/" + documentationId + "/getJobOutput/file/", Snippets.ID_PATH_PARAM.and(RequestDocumentation.parameterWithName("filePath") .description("The path to the file to get").optional()), // Path parameters HeaderDocumentation.requestHeaders(HeaderDocumentation.headerWithName(HttpHeaders.ACCEPT) .description(MediaType.ALL_VALUE).optional()), // Request header HeaderDocumentation.responseHeaders(HeaderDocumentation.headerWithName(HttpHeaders.CONTENT_TYPE) .description("The content type of the file being returned").optional()) // Response Headers ); // check that the generated run file is correct final String runShFileName = SystemUtils.IS_OS_LINUX ? "linux-runsh.txt" : "non-linux-runsh.txt"; final String runShFile = this.resourceLoader.getResource(BASE_DIR + runShFileName).getFile() .getAbsolutePath(); final String runFileContents = new String(Files.readAllBytes(Paths.get(runShFile)), StandardCharsets.UTF_8); final String jobWorkingDir = this.jobDirResource.getFile().getCanonicalPath() + FILE_DELIMITER + id; final String expectedRunScriptContent = this.getExpectedRunContents(runFileContents, jobWorkingDir, id); RestAssured.given(this.getRequestSpecification()).filter(fileResultFilter).when().port(this.port) .get(JOBS_API + "/{id}/output/{filePath}", id, "run").then() .statusCode(Matchers.is(HttpStatus.OK.value())).body(Matchers.is(expectedRunScriptContent)); } private void checkJobRequest(final int documentationId, final String id, final List<String> commandArgs, final String setupFile, final String clusterTag, final String commandTag, final String configFile1, final String depFile1, final boolean archiveJob) { final RestDocumentationFilter getResultFilter = RestAssuredRestDocumentation.document( "{class-name}/" + documentationId + "/getJobRequest/", Snippets.ID_PATH_PARAM, // Path parameters Snippets.HAL_CONTENT_TYPE_HEADER, // Response Headers Snippets.getJobRequestResponsePayload(), // Response fields Snippets.JOB_REQUEST_LINKS // Links ); RestAssured.given(this.getRequestSpecification()).filter(getResultFilter).when().port(this.port) .get(JOBS_API + "/{id}/request", id).then().statusCode(Matchers.is(HttpStatus.OK.value())) .contentType(Matchers.is(MediaTypes.HAL_JSON_UTF8_VALUE)).body(ID_PATH, Matchers.is(id)) .body(CREATED_PATH, Matchers.notNullValue()).body(UPDATED_PATH, Matchers.notNullValue()) .body(NAME_PATH, Matchers.is(JOB_NAME)).body(VERSION_PATH, Matchers.is(JOB_VERSION)) .body(USER_PATH, Matchers.is(JOB_USER)).body(DESCRIPTION_PATH, Matchers.is(JOB_DESCRIPTION)) .body(METADATA_PATH + "." + SCHEDULER_JOB_NAME_KEY, Matchers.is(this.schedulerJobName)) .body(METADATA_PATH + "." + SCHEDULER_RUN_ID_KEY, Matchers.is(this.schedulerRunId)) .body(COMMAND_ARGS_PATH, Matchers.is(StringUtils.join(commandArgs, StringUtils.SPACE))) .body(SETUP_FILE_PATH, Matchers.is(setupFile)).body(CLUSTER_CRITERIAS_PATH, Matchers.hasSize(1)) .body(CLUSTER_CRITERIAS_PATH + "[0].tags", Matchers.hasSize(1)) .body(CLUSTER_CRITERIAS_PATH + "[0].tags[0]", Matchers.is(clusterTag)) .body(COMMAND_CRITERIA_PATH, Matchers.hasSize(1)) .body(COMMAND_CRITERIA_PATH + "[0]", Matchers.is(commandTag)).body(GROUP_PATH, Matchers.nullValue()) .body(DISABLE_LOG_ARCHIVAL_PATH, Matchers.is(!archiveJob)).body(CONFIGS_PATH, Matchers.hasSize(1)) .body(CONFIGS_PATH + "[0]", Matchers.is(configFile1)).body(DEPENDENCIES_PATH, Matchers.hasSize(1)) .body(DEPENDENCIES_PATH + "[0]", Matchers.is(depFile1)).body(EMAIL_PATH, Matchers.nullValue()) .body(CPU_PATH, Matchers.nullValue()).body(MEMORY_PATH, Matchers.nullValue()) .body(APPLICATIONS_PATH, Matchers.empty()).body(TAGS_PATH, Matchers.contains(JOB_TAG_1, JOB_TAG_2)) .body(GROUPING_PATH, Matchers.is(JOB_GROUPING)) .body(GROUPING_INSTANCE_PATH, Matchers.is(JOB_GROUPING_INSTANCE)); } private void checkJobExecution(final int documentationId, final String id) { final RestDocumentationFilter getResultFilter = RestAssuredRestDocumentation.document( "{class-name}/" + documentationId + "/getJobExecution/", Snippets.ID_PATH_PARAM, // Path parameters Snippets.HAL_CONTENT_TYPE_HEADER, // Response Headers Snippets.getJobExecutionResponsePayload(), // Response fields Snippets.JOB_EXECUTION_LINKS // Links ); RestAssured.given(this.getRequestSpecification()).filter(getResultFilter).when().port(this.port) .get(JOBS_API + "/{id}/execution", id).then().statusCode(Matchers.is(HttpStatus.OK.value())) .contentType(Matchers.is(MediaTypes.HAL_JSON_UTF8_VALUE)).body(ID_PATH, Matchers.is(id)) .body(CREATED_PATH, Matchers.notNullValue()).body(UPDATED_PATH, Matchers.notNullValue()) .body(HOST_NAME_PATH, Matchers.is(this.genieHostInfo.getHostname())) .body(PROCESS_ID_PATH, Matchers.notNullValue()) .body(CHECK_DELAY_PATH, Matchers.is((int) CHECK_DELAY)) .body(EXIT_CODE_PATH, Matchers.is(JobExecution.SUCCESS_EXIT_CODE)); } private void checkJobMetadata(final int documentationId, final String id) { final RestDocumentationFilter getResultFilter = RestAssuredRestDocumentation.document( "{class-name}/" + documentationId + "/getJobMetadata/", Snippets.ID_PATH_PARAM, // Path parameters Snippets.HAL_CONTENT_TYPE_HEADER, // Response Headers Snippets.getJobMetadataResponsePayload(), // Response fields Snippets.JOB_METADATA_LINKS // Links ); RestAssured.given(this.getRequestSpecification()).filter(getResultFilter).when().port(this.port) .get(JOBS_API + "/{id}/metadata", id).then().statusCode(Matchers.is(HttpStatus.OK.value())) .contentType(Matchers.is(MediaTypes.HAL_JSON_UTF8_VALUE)).body(ID_PATH, Matchers.is(id)) .body(CREATED_PATH, Matchers.notNullValue()).body(UPDATED_PATH, Matchers.notNullValue()) .body(CLIENT_HOST_PATH, Matchers.notNullValue()).body(USER_AGENT_PATH, Matchers.notNullValue()) .body(NUM_ATTACHMENTS_PATH, Matchers.notNullValue()) .body(TOTAL_SIZE_ATTACHMENTS_PATH, Matchers.notNullValue()) .body(STD_OUT_SIZE_PATH, Matchers.notNullValue()).body(STD_ERR_SIZE_PATH, Matchers.notNullValue()); } private void checkJobCluster(final int documentationId, final String id) { final RestDocumentationFilter getResultFilter = RestAssuredRestDocumentation.document( "{class-name}/" + documentationId + "/getJobCluster/", Snippets.ID_PATH_PARAM, // Path parameters Snippets.HAL_CONTENT_TYPE_HEADER, // Response Headers Snippets.getClusterResponsePayload(), // Response fields Snippets.CLUSTER_LINKS // Links ); RestAssured.given(this.getRequestSpecification()).filter(getResultFilter).when().port(this.port) .get(JOBS_API + "/{id}/cluster", id).then().statusCode(Matchers.is(HttpStatus.OK.value())) .contentType(Matchers.is(MediaTypes.HAL_JSON_UTF8_VALUE)).body(ID_PATH, Matchers.is(CLUSTER1_ID)) .body(CREATED_PATH, Matchers.notNullValue()).body(UPDATED_PATH, Matchers.notNullValue()) .body(NAME_PATH, Matchers.is(CLUSTER1_NAME)).body(USER_PATH, Matchers.is(CLUSTER1_USER)) .body(VERSION_PATH, Matchers.is(CLUSTER1_VERSION)); } private void checkJobCommand(final int documentationId, final String id) { final RestDocumentationFilter getResultFilter = RestAssuredRestDocumentation.document( "{class-name}/" + documentationId + "/getJobCommand/", Snippets.ID_PATH_PARAM, // Path parameters Snippets.HAL_CONTENT_TYPE_HEADER, // Response Headers Snippets.getCommandResponsePayload(), // Response fields Snippets.COMMAND_LINKS // Links ); RestAssured.given(this.getRequestSpecification()).filter(getResultFilter).when().port(this.port) .get(JOBS_API + "/{id}/command", id).then().statusCode(Matchers.is(HttpStatus.OK.value())) .contentType(Matchers.is(MediaTypes.HAL_JSON_UTF8_VALUE)).body(ID_PATH, Matchers.is(CMD1_ID)) .body(CREATED_PATH, Matchers.notNullValue()).body(UPDATED_PATH, Matchers.notNullValue()) .body(NAME_PATH, Matchers.is(CMD1_NAME)).body(USER_PATH, Matchers.is(CMD1_USER)) .body(VERSION_PATH, Matchers.is(CMD1_VERSION)).body("executable", Matchers.is(CMD1_EXECUTABLE)); } private void checkJobApplications(final int documentationId, final String id) { final RestDocumentationFilter getResultFilter = RestAssuredRestDocumentation.document( "{class-name}/" + documentationId + "/getJobApplications/", Snippets.ID_PATH_PARAM, // Path parameters Snippets.HAL_CONTENT_TYPE_HEADER, // Response Headers PayloadDocumentation.responseFields(PayloadDocumentation.subsectionWithPath("[]") .description("The applications for the job").attributes(Snippets.EMPTY_CONSTRAINTS)) // Response fields ); RestAssured.given(this.getRequestSpecification()).filter(getResultFilter).when().port(this.port) .get(JOBS_API + "/{id}/applications", id).then().statusCode(Matchers.is(HttpStatus.OK.value())) .contentType(Matchers.is(MediaTypes.HAL_JSON_UTF8_VALUE)).body("$", Matchers.hasSize(2)) .body("[0].id", Matchers.is(APP1_ID)).body("[1].id", Matchers.is(APP2_ID)); } private void checkFindJobs(final int documentationId, final String id, final String user) { final RestDocumentationFilter findResultFilter = RestAssuredRestDocumentation.document( "{class-name}/" + documentationId + "/findJobs/", Snippets.JOB_SEARCH_QUERY_PARAMETERS, // Request query parameters Snippets.HAL_CONTENT_TYPE_HEADER, // Response headers Snippets.JOB_SEARCH_RESULT_FIELDS, // Result fields Snippets.SEARCH_LINKS // HAL Links ); RestAssured.given(this.getRequestSpecification()).filter(findResultFilter).param("user", user).when() .port(this.port).get(JOBS_API).then().statusCode(Matchers.is(HttpStatus.OK.value())) .contentType(Matchers.is(MediaTypes.HAL_JSON_UTF8_VALUE)).body(JOBS_LIST_PATH, Matchers.hasSize(1)) .body(JOBS_LIST_PATH + "[0].id", Matchers.is(id)); } private void testForConflicts(final String id, final List<String> commandArgs, final List<ClusterCriteria> clusterCriteriaList, final Set<String> commandCriteria) throws Exception { final JobRequest jobConflictRequest = new JobRequest.Builder(JOB_NAME, JOB_USER, JOB_VERSION, clusterCriteriaList, commandCriteria).withId(id).withCommandArgs(commandArgs) .withDisableLogArchival(true).build(); RestAssured.given(this.getRequestSpecification()).contentType(MediaType.APPLICATION_JSON_VALUE) .body(GenieObjectMapper.getMapper().writeValueAsBytes(jobConflictRequest)).when().port(this.port) .post(JOBS_API).then().statusCode(Matchers.is(HttpStatus.CONFLICT.value())); } private void checkJobArchive(final String id, final boolean jobShouldBeArchived) throws URISyntaxException { final Path archiveDirectory = Paths.get(new URI(this.jobsLocationsProperties.getArchives())).resolve(id); // TODO: This is flipped during V4 migration and should be changed back once clients are fixed // if (jobShouldBeArchived) { Assert.assertTrue(Files.exists(archiveDirectory)); Assert.assertTrue(Files.isDirectory(archiveDirectory)); // } else { // Assert.assertFalse(Files.exists(archiveDirectory)); // } } /** * Test the job submit method for success twice to validate the file cache use. * * @throws Exception If there is a problem. */ @Test public void testSubmitJobMethodTwiceSuccess() throws Exception { submitAndCheckJob(2, true); cleanup(); setup(); submitAndCheckJob(3, false); } /** * Test to make sure we can submit a job with attachments. * * @throws Exception on any error */ @Test public void canSubmitJobWithAttachments() throws Exception { final List<String> commandArgs = Lists.newArrayList("-c", "'echo hello world'"); final List<ClusterCriteria> clusterCriteriaList = Lists .newArrayList(new ClusterCriteria(Sets.newHashSet(LOCALHOST_CLUSTER_TAG))); final String setUpFile = this.resourceLoader.getResource(BASE_DIR + "job" + FILE_DELIMITER + "jobsetupfile") .getFile().getAbsolutePath(); final File attachment1File = this.resourceLoader.getResource(BASE_DIR + "job/query.sql").getFile(); final MockMultipartFile attachment1; try (InputStream is = new FileInputStream(attachment1File)) { attachment1 = new MockMultipartFile("attachment", attachment1File.getName(), MediaType.APPLICATION_OCTET_STREAM_VALUE, is); } final File attachment2File = this.resourceLoader.getResource(BASE_DIR + "job/query2.sql").getFile(); final MockMultipartFile attachment2; try (InputStream is = new FileInputStream(attachment2File)) { attachment2 = new MockMultipartFile("attachment", attachment2File.getName(), MediaType.APPLICATION_OCTET_STREAM_VALUE, is); } final Set<String> commandCriteria = Sets.newHashSet(BASH_COMMAND_TAG); final JobRequest jobRequest = new JobRequest.Builder(JOB_NAME, JOB_USER, JOB_VERSION, clusterCriteriaList, commandCriteria).withCommandArgs(commandArgs).withDisableLogArchival(true).withSetupFile(setUpFile) .withDescription(JOB_DESCRIPTION).build(); this.waitForDone(this.submitJob(4, jobRequest, Lists.newArrayList(attachment1, attachment2))); } /** * Test the job submit method for incorrect cluster resolved. * * @throws Exception If there is a problem. */ @Test public void testSubmitJobMethodMissingCluster() throws Exception { Assume.assumeTrue(SystemUtils.IS_OS_UNIX); final List<String> commandArgs = Lists.newArrayList("-c", "'echo hello world'"); final List<ClusterCriteria> clusterCriteriaList = new ArrayList<>(); final Set<String> clusterTags = Sets.newHashSet("undefined"); final ClusterCriteria clusterCriteria = new ClusterCriteria(clusterTags); clusterCriteriaList.add(clusterCriteria); final String jobId = UUID.randomUUID().toString(); final Set<String> commandCriteria = Sets.newHashSet(BASH_COMMAND_TAG); final JobRequest jobRequest = new JobRequest.Builder(JOB_NAME, JOB_USER, JOB_VERSION, clusterCriteriaList, commandCriteria).withId(jobId).withCommandArgs(commandArgs).withDisableLogArchival(true).build(); RestAssured.given(this.getRequestSpecification()).contentType(MediaType.APPLICATION_JSON_VALUE) .body(GenieObjectMapper.getMapper().writeValueAsBytes(jobRequest)).when().port(this.port) .post(JOBS_API).then().statusCode(Matchers.is(HttpStatus.PRECONDITION_FAILED.value())); Assert.assertThat(this.getStatus(jobId), Matchers.is("{\"status\":\"FAILED\"}")); } /** * Test the job submit method for incorrect cluster criteria. * * @throws Exception If there is a problem. */ @Test public void testSubmitJobMethodInvalidClusterCriteria() throws Exception { Assume.assumeTrue(SystemUtils.IS_OS_UNIX); final List<String> commandArgs = Lists.newArrayList("-c", "'echo hello world'"); final List<ClusterCriteria> clusterCriteriaList = Lists .newArrayList(new ClusterCriteria(Sets.newHashSet(" ", "", null))); final String jobId = UUID.randomUUID().toString(); final Set<String> commandCriteria = Sets.newHashSet("bash"); final JobRequest jobRequest = new JobRequest.Builder(JOB_NAME, JOB_USER, JOB_VERSION, clusterCriteriaList, commandCriteria).withId(jobId).withCommandArgs(commandArgs).withDisableLogArchival(true).build(); RestAssured.given(this.getRequestSpecification()).contentType(MediaType.APPLICATION_JSON_VALUE) .body(GenieObjectMapper.getMapper().writeValueAsBytes(jobRequest)).when().port(this.port) .post(JOBS_API).then().statusCode(Matchers.is(HttpStatus.PRECONDITION_FAILED.value())); RestAssured.given(this.getRequestSpecification()).when().port(this.port) .get(JOBS_API + "/{id}/status", jobId).then().statusCode(Matchers.is(HttpStatus.NOT_FOUND.value())); } /** * Test the job submit method for incorrect cluster criteria. * * @throws Exception If there is a problem. */ @Test public void testSubmitJobMethodInvalidCommandCriteria() throws Exception { Assume.assumeTrue(SystemUtils.IS_OS_UNIX); final List<String> commandArgs = Lists.newArrayList("-c", "'echo hello world'"); final List<ClusterCriteria> clusterCriteriaList = Lists .newArrayList(new ClusterCriteria(Sets.newHashSet("ok"))); final String jobId = UUID.randomUUID().toString(); final Set<String> commandCriteria = Sets.newHashSet(" ", "", null); final JobRequest jobRequest = new JobRequest.Builder(JOB_NAME, JOB_USER, JOB_VERSION, clusterCriteriaList, commandCriteria).withId(jobId).withCommandArgs(commandArgs).withDisableLogArchival(true).build(); RestAssured.given(this.getRequestSpecification()).contentType(MediaType.APPLICATION_JSON_VALUE) .body(GenieObjectMapper.getMapper().writeValueAsBytes(jobRequest)).when().port(this.port) .post(JOBS_API).then().statusCode(Matchers.is(HttpStatus.PRECONDITION_FAILED.value())); RestAssured.given(this.getRequestSpecification()).when().port(this.port) .get(JOBS_API + "/{id}/status", jobId).then().statusCode(Matchers.is(HttpStatus.NOT_FOUND.value())); } /** * Test the job submit method for incorrect command resolved. * * @throws Exception If there is a problem. */ @Test public void testSubmitJobMethodMissingCommand() throws Exception { Assume.assumeTrue(SystemUtils.IS_OS_UNIX); final List<String> commandArgs = Lists.newArrayList("-c", "'echo hello world'"); final List<ClusterCriteria> clusterCriteriaList = new ArrayList<>(); final Set<String> clusterTags = Sets.newHashSet(LOCALHOST_CLUSTER_TAG); final ClusterCriteria clusterCriteria = new ClusterCriteria(clusterTags); clusterCriteriaList.add(clusterCriteria); final String jobId = UUID.randomUUID().toString(); final Set<String> commandCriteria = Sets.newHashSet("undefined"); final JobRequest jobRequest = new JobRequest.Builder(JOB_NAME, JOB_USER, JOB_VERSION, clusterCriteriaList, commandCriteria).withId(jobId).withCommandArgs(commandArgs).withDisableLogArchival(true).build(); RestAssured.given(this.getRequestSpecification()).contentType(MediaType.APPLICATION_JSON_VALUE) .body(GenieObjectMapper.getMapper().writeValueAsBytes(jobRequest)).when().port(this.port) .post(JOBS_API).then().statusCode(Matchers.is(HttpStatus.PRECONDITION_FAILED.value())); Assert.assertThat(this.getStatus(jobId), Matchers.is("{\"status\":\"FAILED\"}")); } /** * Test the job submit method for when the job is killed by sending a DELETE HTTP call. * * @throws Exception If there is a problem. */ @Test public void testSubmitJobMethodKill() throws Exception { Assume.assumeTrue(SystemUtils.IS_OS_UNIX); final List<String> commandArgs = Lists.newArrayList("-c", "'sleep 60'"); final List<ClusterCriteria> clusterCriteriaList = new ArrayList<>(); final Set<String> clusterTags = Sets.newHashSet(LOCALHOST_CLUSTER_TAG); final ClusterCriteria clusterCriteria = new ClusterCriteria(clusterTags); clusterCriteriaList.add(clusterCriteria); final Set<String> commandCriteria = Sets.newHashSet(BASH_COMMAND_TAG); final JobRequest jobRequest = new JobRequest.Builder(JOB_NAME, JOB_USER, JOB_VERSION, clusterCriteriaList, commandCriteria).withCommandArgs(commandArgs).withDisableLogArchival(true).build(); final String jobId = this.getIdFromLocation(RestAssured.given(this.getRequestSpecification()) .contentType(MediaType.APPLICATION_JSON_VALUE) .body(GenieObjectMapper.getMapper().writeValueAsBytes(jobRequest)).when().port(this.port) .post(JOBS_API).then().statusCode(Matchers.is(HttpStatus.ACCEPTED.value())) .header(HttpHeaders.LOCATION, Matchers.notNullValue()).extract().header(HttpHeaders.LOCATION)); this.waitForRunning(jobId); // Make sure we can get output for a running job RestAssured.given(this.getRequestSpecification()).accept(MediaType.APPLICATION_JSON_VALUE).when() .port(this.port).get(JOBS_API + "/{id}/output/{filePath}", jobId, "").then() .statusCode(Matchers.is(HttpStatus.OK.value())) .contentType(Matchers.equalToIgnoringCase(MediaType.APPLICATION_JSON_UTF8_VALUE)) .body("parent", Matchers.isEmptyOrNullString()).body("directories[0].name", Matchers.is("genie/")) .body("files[0].name", Matchers.is("run")).body("files[1].name", Matchers.is("stderr")) .body("files[2].name", Matchers.is("stdout")); RestAssured.given(this.getRequestSpecification()).when().port(this.port) .get(JOBS_API + "/{id}/output/{filePath}", jobId, "stdout").then() .statusCode(Matchers.is(HttpStatus.OK.value())) .contentType(Matchers.containsString(ContentType.TEXT.toString())); // Let it run for a couple of seconds Thread.sleep(2000); // Send a kill request to the job. final RestDocumentationFilter killResultFilter = RestAssuredRestDocumentation .document("{class-name}/killJob/", Snippets.ID_PATH_PARAM); RestAssured.given(this.getRequestSpecification()).filter(killResultFilter).when().port(this.port) .delete(JOBS_API + "/{id}", jobId).then().statusCode(Matchers.is(HttpStatus.ACCEPTED.value())); this.waitForDone(jobId); RestAssured.given(this.getRequestSpecification()).when().port(this.port).get(JOBS_API + "/{id}", jobId) .then().statusCode(Matchers.is(HttpStatus.OK.value())) .contentType(Matchers.is(MediaTypes.HAL_JSON_UTF8_VALUE)).body(ID_PATH, Matchers.is(jobId)) .body(STATUS_PATH, Matchers.is(JobStatus.KILLED.toString())) .body(STATUS_MESSAGE_PATH, Matchers.is(JobStatusMessages.JOB_KILLED_BY_USER)); // Kill the job again to make sure it doesn't cause a problem. RestAssured.given(this.getRequestSpecification()).filter(killResultFilter).when().port(this.port) .delete(JOBS_API + "/{id}", jobId).then().statusCode(Matchers.is(HttpStatus.ACCEPTED.value())); } /** * Test the job submit method for when the job is killed as it times out. * * @throws Exception If there is a problem. */ @Test public void testSubmitJobMethodKillOnTimeout() throws Exception { Assume.assumeTrue(SystemUtils.IS_OS_UNIX); final List<String> commandArgs = Lists.newArrayList("-c", "'sleep 60'"); final List<ClusterCriteria> clusterCriteriaList = new ArrayList<>(); final Set<String> clusterTags = Sets.newHashSet(LOCALHOST_CLUSTER_TAG); final ClusterCriteria clusterCriteria = new ClusterCriteria(clusterTags); clusterCriteriaList.add(clusterCriteria); final Set<String> commandCriteria = Sets.newHashSet(BASH_COMMAND_TAG); final JobRequest jobRequest = new JobRequest.Builder(JOB_NAME, JOB_USER, JOB_VERSION, clusterCriteriaList, commandCriteria).withCommandArgs(commandArgs).withTimeout(5).withDisableLogArchival(true).build(); final String id = this.getIdFromLocation(RestAssured.given(this.getRequestSpecification()) .contentType(MediaType.APPLICATION_JSON_VALUE) .body(GenieObjectMapper.getMapper().writeValueAsBytes(jobRequest)).when().port(this.port) .post(JOBS_API).then().statusCode(Matchers.is(HttpStatus.ACCEPTED.value())) .header(HttpHeaders.LOCATION, Matchers.notNullValue()).extract().header(HttpHeaders.LOCATION)); this.waitForDone(id); RestAssured.given(this.getRequestSpecification()).when().port(this.port).get(JOBS_API + "/{id}", id).then() .statusCode(Matchers.is(HttpStatus.OK.value())) .contentType(Matchers.is(MediaTypes.HAL_JSON_UTF8_VALUE)).body(ID_PATH, Matchers.is(id)) .body(STATUS_PATH, Matchers.is(JobStatus.KILLED.toString())) .body(STATUS_MESSAGE_PATH, Matchers.is("Job exceeded timeout.")); } /** * Test the job submit method for when the job fails. * * @throws Exception If there is a problem. */ @Test public void testSubmitJobMethodFailure() throws Exception { Assume.assumeTrue(SystemUtils.IS_OS_UNIX); final List<String> commandArgs = Lists.newArrayList("-c", "'exit 1'"); final List<ClusterCriteria> clusterCriteriaList = new ArrayList<>(); final Set<String> clusterTags = Sets.newHashSet(LOCALHOST_CLUSTER_TAG); final ClusterCriteria clusterCriteria = new ClusterCriteria(clusterTags); clusterCriteriaList.add(clusterCriteria); final Set<String> commandCriteria = Sets.newHashSet(BASH_COMMAND_TAG); final JobRequest jobRequest = new JobRequest.Builder(JOB_NAME, JOB_USER, JOB_VERSION, clusterCriteriaList, commandCriteria).withCommandArgs(commandArgs).withDisableLogArchival(true).build(); final String id = this.getIdFromLocation(RestAssured.given(this.getRequestSpecification()) .contentType(MediaType.APPLICATION_JSON_VALUE) .body(GenieObjectMapper.getMapper().writeValueAsBytes(jobRequest)).when().port(this.port) .post(JOBS_API).then().statusCode(Matchers.is(HttpStatus.ACCEPTED.value())) .header(HttpHeaders.LOCATION, Matchers.notNullValue()).extract().header(HttpHeaders.LOCATION)); this.waitForDone(id); Assert.assertEquals(this.getStatus(id), "{\"status\":\"FAILED\"}"); RestAssured.given(this.getRequestSpecification()).when().port(this.port).get(JOBS_API + "/{id}", id).then() .statusCode(Matchers.is(HttpStatus.OK.value())) .contentType(Matchers.is(MediaTypes.HAL_JSON_UTF8_VALUE)).body(ID_PATH, Matchers.is(id)) .body(STATUS_PATH, Matchers.is(JobStatus.FAILED.toString())) .body(STATUS_MESSAGE_PATH, Matchers.is(JobStatusMessages.JOB_FAILED)); } /** * Test the response content types to ensure UTF-8. * * @throws Exception If there is a problem. */ @Test public void testResponseContentType() throws Exception { Assume.assumeTrue(SystemUtils.IS_OS_UNIX); final List<String> commandArgs = Lists.newArrayList("-c", "'echo hello'"); final JobRequest jobRequest = new JobRequest.Builder(JOB_NAME, JOB_USER, JOB_VERSION, Lists.newArrayList(new ClusterCriteria(Sets.newHashSet("localhost"))), Sets.newHashSet("bash")) .withCommandArgs(commandArgs).build(); final String jobId = this.getIdFromLocation(RestAssured.given(this.getRequestSpecification()) .contentType(MediaType.APPLICATION_JSON_VALUE) .body(GenieObjectMapper.getMapper().writeValueAsBytes(jobRequest)).when().port(this.port) .post(JOBS_API).then().statusCode(Matchers.is(HttpStatus.ACCEPTED.value())) .header(HttpHeaders.LOCATION, Matchers.notNullValue()).extract().header(HttpHeaders.LOCATION)); this.waitForDone(jobId); RestAssured.given(this.getRequestSpecification()).when().port(this.port) .get(JOBS_API + "/" + jobId + "/output/genie/logs/env.log").then() .statusCode(Matchers.is(HttpStatus.OK.value())) .contentType(Matchers.containsString(MediaType.TEXT_PLAIN_VALUE)) .contentType(Matchers.containsString("UTF-8")); RestAssured.given(this.getRequestSpecification()).when().port(this.port) .get(JOBS_API + "/" + jobId + "/output/genie/logs/genie.log").then() .statusCode(Matchers.is(HttpStatus.OK.value())) .contentType(Matchers.containsString(MediaType.TEXT_PLAIN_VALUE)) .contentType(Matchers.containsString("UTF-8")); RestAssured.given(this.getRequestSpecification()).when().port(this.port) .get(JOBS_API + "/" + jobId + "/output/genie/genie.done").then() .statusCode(Matchers.is(HttpStatus.OK.value())) .contentType(Matchers.containsString(MediaType.TEXT_PLAIN_VALUE)) .contentType(Matchers.containsString("UTF-8")); RestAssured.given(this.getRequestSpecification()).accept(MediaType.ALL_VALUE).when().port(this.port) .get(JOBS_API + "/" + jobId + "/output/stdout").then() .statusCode(Matchers.is(HttpStatus.OK.value())) .contentType(Matchers.containsString(MediaType.TEXT_PLAIN_VALUE)) .contentType(Matchers.containsString("UTF-8")); RestAssured.given(this.getRequestSpecification()).accept(MediaType.ALL_VALUE).when().port(this.port) .get(JOBS_API + "/" + jobId + "/output/stderr").then() .statusCode(Matchers.is(HttpStatus.OK.value())) .contentType(Matchers.containsString(MediaType.TEXT_PLAIN_VALUE)) .contentType(Matchers.containsString("UTF-8")); // Verify the file is served as UTF-8 even if it's not RestAssured.given(this.getRequestSpecification()).accept(MediaType.ALL_VALUE).when().port(this.port) .get(JOBS_API + "/" + jobId + "/output/genie/command/" + CMD1_ID + "/config/" + GB18030_TXT).then() .statusCode(Matchers.is(HttpStatus.OK.value())) .contentType(Matchers.containsString(MediaType.TEXT_PLAIN_VALUE)) .contentType(Matchers.containsString("UTF-8")); } private String getExpectedRunContents(final String runFileContents, final String jobWorkingDir, final String jobId) { return runFileContents.replace("TEST_GENIE_JOB_WORKING_DIR_PLACEHOLDER", jobWorkingDir) .replace("JOB_ID_PLACEHOLDER", jobId).replace("COMMAND_ID_PLACEHOLDER", CMD1_ID) .replace("COMMAND_NAME_PLACEHOLDER", CMD1_NAME).replace("COMMAND_TAGS_PLACEHOLDER", CMD1_TAGS) .replace("CLUSTER_ID_PLACEHOLDER", CLUSTER1_ID).replace("CLUSTER_NAME_PLACEHOLDER", CLUSTER1_NAME) .replace("CLUSTER_TAGS_PLACEHOLDER", CLUSTER1_TAGS) .replace("JOB_TAGS_PLACEHOLDER", JOB_TAG_1 + "," + JOB_TAG_2) .replace("JOB_GROUPING_PLACEHOLDER", JOB_GROUPING) .replace("JOB_GROUPING_INSTANCE_PLACEHOLDER", JOB_GROUPING_INSTANCE); } private String getStatus(final String jobId) { return RestAssured.given(this.getRequestSpecification()).when().port(this.port) .get(JOBS_API + "/{id}/status", jobId).asString(); } private void waitForDone(final String jobId) throws Exception { int counter = 0; while (true) { final String statusString = this.getStatus(jobId); if (statusString.contains("INIT") || statusString.contains("RUNNING")) { LOG.info("Iteration {} sleeping for {} ms", counter, SLEEP_TIME); Thread.sleep(SLEEP_TIME); counter++; } else { break; } } } private void waitForRunning(final String jobId) throws Exception { int counter = 0; while (true) { final String statusString = this.getStatus(jobId); if (statusString.contains("INIT")) { LOG.info("Iteration {} sleeping for {} ms", counter, SLEEP_TIME); Thread.sleep(SLEEP_TIME); counter++; } else { break; } } } private void linkAllEntities() throws Exception { final List<String> apps = new ArrayList<>(); apps.add(APP1_ID); apps.add(APP2_ID); RestAssured.given(this.getRequestSpecification()).contentType(MediaType.APPLICATION_JSON_VALUE) .body(GenieObjectMapper.getMapper().writeValueAsBytes(apps)).when().port(this.port) .post(COMMANDS_API + FILE_DELIMITER + CMD1_ID + FILE_DELIMITER + APPLICATIONS_LINK_KEY).then() .statusCode(Matchers.is(HttpStatus.NO_CONTENT.value())); final List<String> cmds = Lists.newArrayList(CMD1_ID); RestAssured.given(this.getRequestSpecification()).contentType(MediaType.APPLICATION_JSON_VALUE) .body(GenieObjectMapper.getMapper().writeValueAsBytes(cmds)).when().port(this.port) .post(CLUSTERS_API + FILE_DELIMITER + CLUSTER1_ID + FILE_DELIMITER + COMMANDS_LINK_KEY).then() .statusCode(Matchers.is(HttpStatus.NO_CONTENT.value())); } private void createAnApplication(final String id, final String appName) throws Exception { final String setUpFile = this.resourceLoader.getResource(BASE_DIR + id + FILE_DELIMITER + "setupfile") .getFile().getAbsolutePath(); final String depFile1 = this.resourceLoader.getResource(BASE_DIR + id + FILE_DELIMITER + "dep1").getFile() .getAbsolutePath(); final String depFile2 = this.resourceLoader.getResource(BASE_DIR + id + FILE_DELIMITER + "dep2").getFile() .getAbsolutePath(); final Set<String> app1Dependencies = Sets.newHashSet(depFile1, depFile2); final String configFile1 = this.resourceLoader.getResource(BASE_DIR + id + FILE_DELIMITER + "config1") .getFile().getAbsolutePath(); final String configFile2 = this.resourceLoader.getResource(BASE_DIR + id + FILE_DELIMITER + "config2") .getFile().getAbsolutePath(); final Set<String> app1Configs = Sets.newHashSet(configFile1, configFile2); final Application app = new Application.Builder(appName, APP1_USER, APP1_VERSION, ApplicationStatus.ACTIVE) .withId(id).withSetupFile(setUpFile).withConfigs(app1Configs).withDependencies(app1Dependencies) .build(); RestAssured.given(this.getRequestSpecification()).contentType(MediaType.APPLICATION_JSON_VALUE) .body(GenieObjectMapper.getMapper().writeValueAsBytes(app)).when().port(this.port) .post(APPLICATIONS_API).then().statusCode(Matchers.is(HttpStatus.CREATED.value())) .header(HttpHeaders.LOCATION, Matchers.notNullValue()); } private void createAllClusters() throws Exception { final String setUpFile = this.resourceLoader .getResource(BASE_DIR + CLUSTER1_ID + FILE_DELIMITER + "setupfile").getFile().getAbsolutePath(); final String configFile1 = this.resourceLoader .getResource(BASE_DIR + CLUSTER1_ID + FILE_DELIMITER + "config1").getFile().getAbsolutePath(); final String configFile2 = this.resourceLoader .getResource(BASE_DIR + CLUSTER1_ID + FILE_DELIMITER + "config2").getFile().getAbsolutePath(); final Set<String> configs = Sets.newHashSet(configFile1, configFile2); final String depFile1 = this.resourceLoader.getResource(BASE_DIR + CLUSTER1_ID + FILE_DELIMITER + "dep1") .getFile().getAbsolutePath(); final String depFile2 = this.resourceLoader.getResource(BASE_DIR + CLUSTER1_ID + FILE_DELIMITER + "dep2") .getFile().getAbsolutePath(); final Set<String> clusterDependencies = Sets.newHashSet(depFile1, depFile2); final Set<String> tags = Sets.newHashSet(LOCALHOST_CLUSTER_TAG); final Cluster cluster = new Cluster.Builder(CLUSTER1_NAME, CLUSTER1_USER, CLUSTER1_VERSION, ClusterStatus.UP).withId(CLUSTER1_ID).withSetupFile(setUpFile).withConfigs(configs) .withDependencies(clusterDependencies).withTags(tags).build(); RestAssured.given(this.getRequestSpecification()).contentType(MediaType.APPLICATION_JSON_VALUE) .body(GenieObjectMapper.getMapper().writeValueAsBytes(cluster)).when().port(this.port) .post(CLUSTERS_API).then().statusCode(Matchers.is(HttpStatus.CREATED.value())) .header(HttpHeaders.LOCATION, Matchers.notNullValue()); } private void createAllCommands() throws Exception { final String setUpFile = this.resourceLoader.getResource(BASE_DIR + CMD1_ID + FILE_DELIMITER + "setupfile") .getFile().getAbsolutePath(); final String configFile1 = this.resourceLoader.getResource(BASE_DIR + CMD1_ID + FILE_DELIMITER + "config1") .getFile().getAbsolutePath(); final String configFile2 = this.resourceLoader.getResource(BASE_DIR + CMD1_ID + FILE_DELIMITER + "config2") .getFile().getAbsolutePath(); final String configFile3 = this.resourceLoader .getResource(BASE_DIR + CMD1_ID + FILE_DELIMITER + GB18030_TXT).getFile().getAbsolutePath(); final Set<String> configs = Sets.newHashSet(configFile1, configFile2, configFile3); final String depFile1 = this.resourceLoader.getResource(BASE_DIR + CLUSTER1_ID + FILE_DELIMITER + "dep1") .getFile().getAbsolutePath(); final String depFile2 = this.resourceLoader.getResource(BASE_DIR + CLUSTER1_ID + FILE_DELIMITER + "dep2") .getFile().getAbsolutePath(); final Set<String> commandDependencies = Sets.newHashSet(depFile1, depFile2); final Set<String> tags = Sets.newHashSet(BASH_COMMAND_TAG); final Command cmd = new Command.Builder(CMD1_NAME, CMD1_USER, CMD1_VERSION, CommandStatus.ACTIVE, CMD1_EXECUTABLE, CHECK_DELAY).withId(CMD1_ID).withSetupFile(setUpFile).withConfigs(configs) .withDependencies(commandDependencies).withTags(tags).build(); RestAssured.given(this.getRequestSpecification()).contentType(MediaType.APPLICATION_JSON_VALUE) .body(GenieObjectMapper.getMapper().writeValueAsBytes(cmd)).when().port(this.port) .post(COMMANDS_API).then().statusCode(Matchers.is(HttpStatus.CREATED.value())) .header(HttpHeaders.LOCATION, Matchers.notNullValue()); } }