Java tutorial
/* (c) 2015 Open Source Geospatial Foundation - all rights reserved * This code is licensed under the GPL 2.0 license, available at the root * application directory. */ package org.geoserver.restupload; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNotEquals; import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertTrue; import java.io.File; import java.io.FileInputStream; import java.io.IOException; import java.io.InputStream; import java.util.Arrays; import java.util.Date; import java.util.Random; import org.apache.commons.io.FilenameUtils; import org.apache.commons.io.IOUtils; import org.apache.commons.lang.ArrayUtils; import org.geoserver.catalog.MetadataMap; import org.geoserver.catalog.impl.ModificationProxy; import org.geoserver.catalog.rest.CatalogRESTTestSupport; import org.geoserver.config.GeoServerInfo; import org.geoserver.config.impl.SettingsInfoImpl; import org.geoserver.platform.GeoServerExtensions; import org.geoserver.platform.GeoServerResourceLoader; import org.geoserver.platform.resource.Resource; import org.geoserver.rest.util.RESTUtils; import org.junit.Before; import org.junit.Test; import org.restlet.data.Status; import org.springframework.mock.web.MockHttpServletRequest; import org.springframework.mock.web.MockHttpServletResponse; /** * Test class for checking REST resumable upload * * @author Nicola Lagomarsini * */ public class ResumableUploadTest extends CatalogRESTTestSupport { /** Resource used for storing temporary uploads */ private Resource tmpUploadFolder; /** Size for partial uploads */ private long partialSize = 50; /** Relative path of the file to upload */ private String fileName = "/relative/resumableUploadTest.shp"; /** Root folder */ private String root; @Before public void before() throws Exception { GeoServerResourceLoader loader = GeoServerExtensions.bean(GeoServerResourceLoader.class); tmpUploadFolder = loader.get("tmp/upload"); // Selection of the root directory File rootFile = getRootDirectory(); root = rootFile.getAbsolutePath(); // Setting of the global configuration GeoServerInfo global = getGeoServer().getGlobal(); // Selections of the SettingsInfo associated to the GlobalSettings SettingsInfoImpl info = (SettingsInfoImpl) ModificationProxy.unwrap(global.getSettings()); // If no metadata map is present, then a new one is added if (info.getMetadata() == null) { info.setMetadata(new MetadataMap()); } // Selection of the metadata MetadataMap map = info.getMetadata(); // Addition of the key associated to the root directory map.put(RESTUtils.ROOT_KEY, root); // Insertion of the settings inside the global ones global.setSettings(info); // Save settings getGeoServer().save(global); } @Test public void testPostRequest() throws Exception { String uploadId = sendPostRequest(); assertNotNull(uploadId); File uploadedFile = getTempPath(uploadId); assertTrue(uploadedFile.exists()); assertEquals(0, uploadedFile.length()); } @Test public void testSuccessivePostRequest() throws Exception { String uploadId = sendPostRequest(); String secondUploadId = sendPostRequest(); assertNotNull(secondUploadId); assertNotEquals(uploadId, secondUploadId); } @Test public void testUploadFull() throws Exception { String uploadId = sendPostRequest(); File uploadedFile = getTempPath(uploadId); assertTrue(uploadedFile.exists()); MockHttpServletRequest request = createRequest("/rest/resumableupload/" + uploadId); request.setMethod("PUT"); request.setContentType("application/octet-stream"); byte[] bigFile = generateFileAsBytes(); request.setContent(bigFile); request.addHeader("Content-type", "application/octet-stream"); request.addHeader("Content-Length", String.valueOf(bigFile.length)); MockHttpServletResponse response = dispatch(request); assertEquals(Status.SUCCESS_OK.getCode(), response.getStatus()); assertFalse(uploadedFile.exists()); File destinationFile = new File(FilenameUtils.concat(root, fileName.replaceAll("^/", ""))); assertTrue(destinationFile.exists()); assertEquals(bigFile.length, destinationFile.length()); // Check uploaded file byte by byte boolean checkBytes = Arrays.equals(bigFile, toBytes(new FileInputStream(destinationFile))); assertTrue(checkBytes); // Check response content String restUrl = response.getContentAsString(); assertEquals(fileName.replaceAll("^/", ""), restUrl); } @Test public void testUploadFullWithoutRestDir() throws Exception { //Set ROOT_KEY to null GeoServerInfo global = getGeoServer().getGlobal(); SettingsInfoImpl info = (SettingsInfoImpl) ModificationProxy.unwrap(global.getSettings()); MetadataMap map = info.getMetadata(); map.remove(RESTUtils.ROOT_KEY); global.setSettings(info); getGeoServer().save(global); //Set root GeoServerResourceLoader loader = GeoServerExtensions.bean(GeoServerResourceLoader.class); Resource data = loader.get("data"); root = data.dir().getAbsolutePath(); testUploadFull(); before(); } @Test public void testPartialUpload() throws Exception { String uploadId = sendPostRequest(); MockHttpServletRequest request = createRequest("/rest/resumableupload/" + uploadId); request.setMethod("PUT"); request.setContentType("application/octet-stream"); byte[] bigFile = generateFileAsBytes(); byte[] partialFile = ArrayUtils.subarray(bigFile, 0, (int) partialSize); request.setContent(partialFile); request.addHeader("Content-type", "application/octet-stream"); request.addHeader("Content-Length", String.valueOf(bigFile.length)); MockHttpServletResponse response = dispatch(request); assertEquals(ResumableUploadCatalogResource.RESUME_INCOMPLETE.getCode(), response.getStatus()); assertEquals(null, response.getHeader("Content-Length")); assertEquals("0-" + (partialSize - 1), response.getHeader("Range")); File uploadedFile = getTempPath(uploadId); assertTrue(uploadedFile.exists()); assertEquals(partialSize, uploadedFile.length()); boolean checkBytes = Arrays.equals(partialFile, toBytes(new FileInputStream(uploadedFile))); assertTrue(checkBytes); } @Test public void testUploadPartialResume() throws Exception { String uploadId = sendPostRequest(); byte[] bigFile = generateFileAsBytes(); byte[] partialFile1 = ArrayUtils.subarray(bigFile, 0, (int) partialSize); // First upload MockHttpServletRequest request = createRequest("/rest/resumableupload/" + uploadId); request.setMethod("PUT"); request.setContentType("application/octet-stream"); request.setContent(partialFile1); request.addHeader("Content-type", "application/octet-stream"); request.addHeader("Content-Length", String.valueOf(bigFile.length)); dispatch(request); // Resume upload request = createRequest("/rest/resumableupload/" + uploadId); request.setMethod("PUT"); request.setContentType("application/octet-stream"); byte[] partialFile2 = ArrayUtils.subarray(bigFile, (int) partialSize, (int) partialSize * 2); request.setContent(partialFile2); request.addHeader("Content-type", "application/octet-stream"); request.addHeader("Content-Length", String.valueOf(partialFile2.length)); request.addHeader("Content-Range", "bytes " + partialSize + "-" + partialSize * 2 + "/" + bigFile.length); MockHttpServletResponse response = dispatch(request); assertEquals(ResumableUploadCatalogResource.RESUME_INCOMPLETE.getCode(), response.getStatus()); assertEquals(null, response.getHeader("Content-Length")); assertEquals("0-" + (partialSize * 2 - 1), response.getHeader("Range")); File uploadedFile = getTempPath(uploadId); assertTrue(uploadedFile.exists()); assertEquals(partialSize * 2, uploadedFile.length()); // Check uploaded file byte by byte boolean checkBytes = Arrays.equals(ArrayUtils.addAll(partialFile1, partialFile2), toBytes(new FileInputStream(uploadedFile))); assertTrue(checkBytes); } @Test public void testUploadFullResume() throws Exception { String uploadId = sendPostRequest(); byte[] bigFile = generateFileAsBytes(); byte[] partialFile1 = ArrayUtils.subarray(bigFile, 0, (int) partialSize); // First upload MockHttpServletRequest request = createRequest("/rest/resumableupload/" + uploadId); request.setMethod("PUT"); request.setContentType("application/octet-stream"); request.setContent(partialFile1); request.addHeader("Content-type", "application/octet-stream"); request.addHeader("Content-Length", String.valueOf(bigFile.length)); dispatch(request); // Resume upload request = createRequest("/rest/resumableupload/" + uploadId); request.setMethod("PUT"); request.setContentType("application/octet-stream"); byte[] partialFile2 = ArrayUtils.subarray(bigFile, (int) partialSize, bigFile.length); request.setContent(partialFile2); request.addHeader("Content-type", "application/octet-stream"); request.addHeader("Content-Length", String.valueOf(partialFile2.length)); request.addHeader("Content-Range", "bytes " + partialSize + "-" + bigFile.length + "/" + bigFile.length); MockHttpServletResponse response = dispatch(request); assertEquals(Status.SUCCESS_OK.getCode(), response.getStatus()); File uploadedFile = getTempPath(uploadId); assertFalse(uploadedFile.exists()); File destinationFile = new File(FilenameUtils.concat(root, fileName.replaceAll("^/", ""))); assertTrue(destinationFile.exists()); assertEquals(bigFile.length, destinationFile.length()); // Check uploaded file byte by byte boolean checkBytes = Arrays.equals(bigFile, toBytes(new FileInputStream(destinationFile))); assertTrue(checkBytes); // Check response content String restUrl = response.getContentAsString(); assertEquals(fileName.replaceAll("^/", ""), restUrl); } @Test public void testPartialCleanup() throws Exception { // Change cleanup expirationDelay ResumableUploadResourceCleaner cleaner = (ResumableUploadResourceCleaner) applicationContext .getBean("resumableUploadStorageCleaner"); cleaner.setExpirationDelay(1000); // Upload file String uploadId = sendPostRequest(); byte[] bigFile = generateFileAsBytes(); byte[] partialFile = ArrayUtils.subarray(bigFile, 0, (int) partialSize); MockHttpServletRequest request = createRequest("/rest/resumableupload/" + uploadId); request.setMethod("PUT"); request.setContentType("application/octet-stream"); request.setContent(partialFile); request.addHeader("Content-type", "application/octet-stream"); request.addHeader("Content-Length", String.valueOf(bigFile.length)); dispatch(request); File uploadedFile = getTempPath(uploadId); assertTrue(uploadedFile.exists()); // Wait to cleanup, max 2 minutes long startTime = new Date().getTime(); while (uploadedFile.exists() && (new Date().getTime() - startTime) < 120000) { Thread.sleep(1000); } assertTrue(!uploadedFile.exists()); cleaner.setExpirationDelay(300000); } @Test public void testSidecarCleanup() throws Exception { // Change cleanup expirationDelay ResumableUploadResourceCleaner cleaner = (ResumableUploadResourceCleaner) applicationContext .getBean("resumableUploadStorageCleaner"); cleaner.setExpirationDelay(1000); // Upload file String uploadId = sendPostRequest(); byte[] bigFile = generateFileAsBytes(); MockHttpServletRequest request = createRequest("/rest/resumableupload/" + uploadId); request.setMethod("PUT"); request.setContentType("application/octet-stream"); request.setContent(bigFile); request.addHeader("Content-type", "application/octet-stream"); request.addHeader("Content-Length", String.valueOf(bigFile.length)); dispatch(request); File uploadedFile = getTempPath(uploadId); assertFalse(uploadedFile.exists()); File sidecarFile = new File( FilenameUtils.concat(tmpUploadFolder.dir().getCanonicalPath(), uploadId + ".sidecar")); assertTrue(sidecarFile.exists()); // Wait to cleanup, max 2 minutes long startTime = new Date().getTime(); while (sidecarFile.exists() && (new Date().getTime() - startTime) < 120000) { Thread.sleep(1000); } assertFalse(sidecarFile.exists()); // Test GET after sidecar cleanup MockHttpServletResponse response = getAsServletResponse("/rest/resumableupload/" + uploadId, "text/plain"); assertEquals(Status.CLIENT_ERROR_NOT_FOUND.getCode(), response.getStatus()); cleaner.setExpirationDelay(300000); } @Test public void testGetAfterPartial() throws Exception { String uploadId = sendPostRequest(); byte[] bigFile = generateFileAsBytes(); byte[] partialFile = ArrayUtils.subarray(bigFile, 0, (int) partialSize); // First upload MockHttpServletRequest request = createRequest("/rest/resumableupload/" + uploadId); request.setMethod("PUT"); request.setContentType("application/octet-stream"); request.setContent(partialFile); request.addHeader("Content-type", "application/octet-stream"); request.addHeader("Content-Length", String.valueOf(bigFile.length)); dispatch(request); File uploadedFile = getTempPath(uploadId); assertTrue(uploadedFile.exists()); MockHttpServletResponse response = getAsServletResponse("/rest/resumableupload/" + uploadId, "text/plain"); assertEquals(ResumableUploadCatalogResource.RESUME_INCOMPLETE.getCode(), response.getStatus()); assertEquals(null, response.getHeader("Content-Length")); assertEquals("0-" + (partialSize - 1), response.getHeader("Range")); } @Test public void testGetAfterFull() throws Exception { String uploadId = sendPostRequest(); File uploadedFile = getTempPath(uploadId); assertTrue(uploadedFile.exists()); MockHttpServletRequest request = createRequest("/rest/resumableupload/" + uploadId); request.setMethod("PUT"); request.setContentType("application/octet-stream"); byte[] bigFile = generateFileAsBytes(); request.setContent(bigFile); request.addHeader("Content-type", "application/octet-stream"); request.addHeader("Content-Length", String.valueOf(bigFile.length)); MockHttpServletResponse response = dispatch(request); assertEquals(Status.SUCCESS_OK.getCode(), response.getStatus()); File sidecarFile = new File( FilenameUtils.concat(tmpUploadFolder.dir().getCanonicalPath(), uploadId + ".sidecar")); assertTrue(sidecarFile.exists()); } private byte[] generateFileAsBytes() throws IOException { byte[] b = new byte[200]; new Random().nextBytes(b); return b; } private byte[] toBytes(InputStream in) throws IOException { return IOUtils.toByteArray(in); } private File getRootDirectory() throws IOException { File dataDirectoryRoot = getTestData().getDataDirectoryRoot(); File newroot = new File(dataDirectoryRoot, "RESUMABLE_UPLOADED"); newroot.mkdirs(); return newroot; } private File getTempPath(String uploadId) throws IOException { String tempPath = FilenameUtils.removeExtension(fileName) + "_" + uploadId + "." + FilenameUtils.getExtension(fileName); tempPath = tempPath.replaceAll("^/", ""); tempPath = FilenameUtils.concat(tmpUploadFolder.dir().getCanonicalPath(), tempPath); return new File(tempPath); } private String sendPostRequest() throws Exception { MockHttpServletRequest request = createRequest("/rest/resumableupload/"); request.setMethod("POST"); request.setContentType("text/plain"); request.setContent(fileName.getBytes("UTF-8")); request.addHeader("Content-type", "text/plain"); MockHttpServletResponse response = dispatch(request); assertEquals(Status.SUCCESS_CREATED.getCode(), response.getStatus()); String responseBody = response.getContentAsString(); String url = responseBody.split("\\r?\\n")[1]; String uploadId = FilenameUtils.getBaseName(url); return uploadId; } }