Java tutorial
/* * R Service Bus * * Copyright (c) Copyright of Open Analytics NV, 2010-2015 * * =========================================================================== * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU Affero General Public License as published by * the Free Software Foundation, either version 3 of the License, or * (at your option) any later version. * * This program is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU Affero General Public License for more details. * * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see <http://www.gnu.org/licenses/>. */ package eu.openanalytics.rsb.component; import java.io.File; import java.io.FileInputStream; import java.io.FileOutputStream; import java.io.FileWriter; import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; import java.io.StringReader; import java.net.URI; import java.net.URISyntaxException; import java.util.HashMap; import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Map.Entry; import java.util.Set; import java.util.UUID; import java.util.regex.Matcher; import java.util.regex.Pattern; import javax.annotation.Resource; import javax.ws.rs.Consumes; import javax.ws.rs.GET; import javax.ws.rs.HeaderParam; import javax.ws.rs.POST; import javax.ws.rs.PUT; import javax.ws.rs.Path; import javax.ws.rs.PathParam; import javax.ws.rs.Produces; import javax.ws.rs.QueryParam; import javax.ws.rs.core.Context; import javax.ws.rs.core.HttpHeaders; import javax.ws.rs.core.Response; import javax.ws.rs.core.Response.ResponseBuilder; import javax.ws.rs.core.Response.Status; import javax.ws.rs.core.StreamingOutput; import javax.ws.rs.core.UriInfo; import org.apache.commons.codec.digest.DigestUtils; import org.apache.commons.compress.archivers.tar.TarArchiveEntry; import org.apache.commons.compress.archivers.tar.TarArchiveInputStream; import org.apache.commons.compress.compressors.gzip.GzipCompressorInputStream; import org.apache.commons.io.FileUtils; import org.apache.commons.io.IOUtils; import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.Validate; import org.apache.commons.lang3.tuple.Pair; import org.springframework.beans.BeansException; import org.springframework.context.ApplicationContext; import org.springframework.context.ApplicationContextAware; import org.springframework.context.ConfigurableApplicationContext; import org.springframework.stereotype.Component; import eu.openanalytics.rsb.Constants; import eu.openanalytics.rsb.Util; import eu.openanalytics.rsb.config.Configuration; import eu.openanalytics.rsb.config.Configuration.CatalogSection; import eu.openanalytics.rsb.config.ConfigurationFactory; import eu.openanalytics.rsb.config.PersistedConfiguration; import eu.openanalytics.rsb.data.CatalogManager.PutCatalogFileResult; import eu.openanalytics.rsb.rest.types.Catalog; import eu.openanalytics.rsb.rest.types.CatalogDirectory; import eu.openanalytics.rsb.rest.types.CatalogFileType; import eu.openanalytics.rsb.rest.types.RServiPoolType; import eu.openanalytics.rsb.rest.types.RServiPools; import eu.openanalytics.rsb.rservi.RServiPackageManager; /** * @author "OpenAnalytics <rsb.development@openanalytics.eu>" */ @Component("adminResource") @Path("/" + Constants.ADMIN_PATH) public class AdminResource extends AbstractResource implements ApplicationContextAware { private static final String CATALOG_SUBPATH = "catalog"; private static final String SYSTEM_SUBPATH = "system"; public static final String ADMIN_SYSTEM_PATH = Constants.ADMIN_PATH + "/" + SYSTEM_SUBPATH; public static final String ADMIN_CATALOG_PATH = Constants.ADMIN_PATH + "/" + CATALOG_SUBPATH; private static final Pattern TAR_CATALOG_FILE_PATTERN = Pattern.compile(".*/inst/rsb/catalog/(.*)"); private ConfigurableApplicationContext applicationContext; @Resource private RServiPackageManager rServiPackageManager; // exposed for unit testing public void setrServiPackageManager(final RServiPackageManager rServiPackageManager) { this.rServiPackageManager = rServiPackageManager; } @Override public void setApplicationContext(final ApplicationContext applicationContext) throws BeansException { this.applicationContext = (ConfigurableApplicationContext) applicationContext; } @Path("/" + SYSTEM_SUBPATH + "/configuration") @GET @Produces(Constants.JSON_CONTENT_TYPE) public Response getSystemConfiguration() { return Response.ok(Util.toJson(new PersistedConfiguration(getConfiguration()))).build(); } @Path("/" + SYSTEM_SUBPATH + "/configuration") @PUT @Consumes(Constants.JSON_CONTENT_TYPE) public Response putSystemConfiguration(final InputStream in) throws IOException, URISyntaxException { Validate.notNull(getConfiguration().getConfigurationUrl(), "Transient configuration can't be PUT over the API"); final Configuration newConfiguration = ConfigurationFactory.loadJsonConfiguration(in); final String reformattedNewConfiguration = Util.toJson(new PersistedConfiguration(newConfiguration)); final File newConfigurationFile = new File(getConfiguration().getConfigurationUrl().toURI()); final FileWriter fw = new FileWriter(newConfigurationFile); IOUtils.copy(new StringReader(reformattedNewConfiguration), fw); IOUtils.closeQuietly(fw); getLogger().warn("Configuration stored in: " + newConfigurationFile + ". System needs restart in order to activate this new configuration!"); return Response.noContent().build(); } @Path("/" + SYSTEM_SUBPATH + "/restart") @POST public Response restart() { applicationContext.close(); applicationContext.refresh(); return Response.ok("RESTARTED").build(); } @Path("/" + SYSTEM_SUBPATH + "/rservi_pools") @GET @Produces({ Constants.RSB_XML_CONTENT_TYPE, Constants.RSB_JSON_CONTENT_TYPE }) public RServiPools getRServiPools() { final Map<URI, Set<String>> pools = new HashMap<URI, Set<String>>(); addToPools(getConfiguration().getDefaultRserviPoolUri(), null, pools); final Map<String, Set<URI>> applicationSpecificRserviPoolUris = getConfiguration() .getApplicationSpecificRserviPoolUris(); if (applicationSpecificRserviPoolUris != null) { for (final Entry<String, Set<URI>> pool : applicationSpecificRserviPoolUris.entrySet()) { for (final URI uri : pool.getValue()) { addToPools(uri, pool.getKey(), pools); } } } final RServiPools result = Util.REST_OBJECT_FACTORY.createRServiPools(); for (final Entry<URI, Set<String>> pool : pools.entrySet()) { final RServiPoolType rServiPool = Util.REST_OBJECT_FACTORY.createRServiPoolType(); rServiPool.setPoolUri(pool.getKey().toString()); rServiPool.setApplicationNames(StringUtils.join(pool.getValue(), ',')); rServiPool.setDefault(pool.getKey().equals(getConfiguration().getDefaultRserviPoolUri())); result.getContents().add(rServiPool); } return result; } @Path("/" + SYSTEM_SUBPATH + "/r_packages") @POST @Consumes({ Constants.GZIP_CONTENT_TYPE }) public void installRPackage(@QueryParam("rServiPoolUri") final String rServiPoolUri, @QueryParam("sha1hexsum") final String sha1HexSum, @QueryParam("packageName") final String packageName, final InputStream input) throws Exception { Validate.notBlank(rServiPoolUri, "missing query param: rServiPoolUri"); Validate.notBlank(sha1HexSum, "missing query param: sha1hexsum"); // store the package and tar files in temporary files final File tempDirectory = new File(FileUtils.getTempDirectory(), UUID.randomUUID().toString()); FileUtils.forceMkdir(tempDirectory); final File packageSourceFile = new File(tempDirectory, packageName); try { final FileOutputStream output = new FileOutputStream(packageSourceFile); IOUtils.copyLarge(input, output); IOUtils.closeQuietly(output); // validate the checksum final FileInputStream packageSourceInputStream = new FileInputStream(packageSourceFile); final String calculatedSha1HexSum = DigestUtils.sha1Hex(packageSourceInputStream); IOUtils.closeQuietly(packageSourceInputStream); Validate.isTrue(calculatedSha1HexSum.equals(sha1HexSum), "Invalid SHA-1 HEX checksum"); // upload to RServi rServiPackageManager.install(packageSourceFile, rServiPoolUri); // extract catalog files from $PKG_ROOT/inst/rsb/catalog extractCatalogFiles(packageSourceFile); getLogger().info("Package with checksum " + sha1HexSum + " installed to " + rServiPoolUri); } finally { try { FileUtils.forceDelete(tempDirectory); } catch (final Exception e) { getLogger().warn("Failed to delete temporary directory: " + tempDirectory, e); } } } private void extractCatalogFiles(final File packageSourceFile) throws IOException { final File tempDirectory = packageSourceFile.getParentFile(); // 1) extract TAR final File packageTarFile = File.createTempFile("rsb-install.", ".tar", tempDirectory); final GzipCompressorInputStream gzIn = new GzipCompressorInputStream( new FileInputStream(packageSourceFile)); FileOutputStream output = new FileOutputStream(packageTarFile); IOUtils.copyLarge(gzIn, output); IOUtils.closeQuietly(output); IOUtils.closeQuietly(gzIn); // 2) parse TAR and drop files in catalog final TarArchiveInputStream tarIn = new TarArchiveInputStream(new FileInputStream(packageTarFile)); TarArchiveEntry tarEntry = null; while ((tarEntry = tarIn.getNextTarEntry()) != null) { if (!tarEntry.isFile()) { continue; } final Matcher matcher = TAR_CATALOG_FILE_PATTERN.matcher(tarEntry.getName()); if (matcher.matches()) { final byte[] data = IOUtils.toByteArray(tarIn, tarEntry.getSize()); final String catalogFile = matcher.group(1); final File targetCatalogFile = new File(getConfiguration().getCatalogRootDirectory(), catalogFile); output = new FileOutputStream(targetCatalogFile); IOUtils.write(data, output); IOUtils.closeQuietly(output); getLogger().info("Wrote " + data.length + " bytes in catalog file: " + targetCatalogFile); } } IOUtils.closeQuietly(tarIn); } @Path("/" + CATALOG_SUBPATH) @GET @Produces({ Constants.RSB_XML_CONTENT_TYPE, Constants.RSB_JSON_CONTENT_TYPE }) public Catalog getCatalogIndex( @HeaderParam(Constants.APPLICATION_NAME_HTTP_HEADER) final String applicationName, @Context final HttpHeaders httpHeaders, @Context final UriInfo uriInfo) throws IOException, URISyntaxException { final Catalog result = Util.REST_OBJECT_FACTORY.createCatalog(); for (final Entry<Pair<CatalogSection, File>, List<File>> catalogSectionFiles : getCatalogManager() .getCatalog(applicationName).entrySet()) { final CatalogDirectory catalogDirectory = createCatalogDirectory(catalogSectionFiles.getKey().getLeft(), catalogSectionFiles.getKey().getRight(), catalogSectionFiles.getValue(), httpHeaders, uriInfo); result.getDirectories().add(catalogDirectory); } return result; } @Path("/" + CATALOG_SUBPATH + "/{catalogName}/{fileName}") @GET public Response getCatalogFile(@PathParam("catalogName") final String catalogName, @PathParam("fileName") final String fileName, @HeaderParam(Constants.APPLICATION_NAME_HTTP_HEADER) final String applicationName) throws IOException, URISyntaxException { final File catalogFile = getCatalogManager().getCatalogFile(CatalogSection.valueOf(catalogName), applicationName, fileName); if (!catalogFile.isFile()) { return Response.status(Status.NOT_FOUND).build(); } final ResponseBuilder rb = Response.ok(); rb.type(Util.getContentType(catalogFile)); rb.entity(new StreamingOutput() { @Override public void write(final OutputStream output) throws IOException { final FileInputStream fis = new FileInputStream(catalogFile); IOUtils.copy(fis, output); IOUtils.closeQuietly(fis); IOUtils.closeQuietly(output); } }); return rb.build(); } @Path("/" + CATALOG_SUBPATH + "/{catalogName}/{fileName}") @PUT @Consumes(Constants.TEXT_CONTENT_TYPE) public Response putCatalogFile(@PathParam("catalogName") final String catalogName, @PathParam("fileName") final String fileName, @HeaderParam(Constants.APPLICATION_NAME_HTTP_HEADER) final String applicationName, final InputStream in, @Context final HttpHeaders httpHeaders, @Context final UriInfo uriInfo) throws IOException, URISyntaxException { final CatalogSection catalogSection = CatalogSection.valueOf(catalogName); final Pair<PutCatalogFileResult, File> result = getCatalogManager().putCatalogFile(catalogSection, applicationName, fileName, in); if (result.getLeft() == PutCatalogFileResult.UPDATED) { return Response.noContent().build(); } final URI location = buildCatalogFileUri(catalogSection, result.getRight(), httpHeaders, uriInfo); return Response.created(location).build(); } private void addToPools(final URI rServiPoolUri, final String applicationName, final Map<URI, Set<String>> pools) { Set<String> applicationNames = pools.get(rServiPoolUri); if (applicationNames == null) { applicationNames = new HashSet<String>(); } if (StringUtils.isNotBlank(applicationName)) { applicationNames.add(applicationName); } pools.put(rServiPoolUri, applicationNames); } private CatalogDirectory createCatalogDirectory(final CatalogSection catalogSection, final File catalogSectionDirectory, final List<File> catalogFiles, final HttpHeaders httpHeaders, final UriInfo uriInfo) throws URISyntaxException, IOException { final String catalogTypeAsString = catalogSection.toString(); final CatalogDirectory catalogDirectory = Util.REST_OBJECT_FACTORY.createCatalogDirectory(); catalogDirectory.setType(catalogTypeAsString); catalogDirectory.setPath(catalogSectionDirectory.getCanonicalPath()); for (final File file : catalogFiles) { final URI dataUri = buildCatalogFileUri(catalogSection, file, httpHeaders, uriInfo); final CatalogFileType catalogFile = Util.REST_OBJECT_FACTORY.createCatalogFileType(); catalogFile.setName(file.getName()); catalogFile.setDataUri(dataUri.toString()); catalogDirectory.getFiles().add(catalogFile); } return catalogDirectory; } private URI buildCatalogFileUri(final CatalogSection catalogSection, final File file, final HttpHeaders httpHeaders, final UriInfo uriInfo) throws URISyntaxException { return Util.getUriBuilder(uriInfo, httpHeaders).path(Constants.ADMIN_PATH).path(CATALOG_SUBPATH) .path(catalogSection.toString()).path(file.getName()).build(); } }