Java tutorial
/* * Copyright (c) 2016-2017, Joyent, Inc. All rights reserved. * * This Source Code Form is subject to the terms of the Mozilla Public * License, v. 2.0. If a copy of the MPL was not distributed with this * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ package com.joyent.manta.client.multipart; import com.fasterxml.jackson.annotation.JsonInclude; import com.fasterxml.jackson.annotation.JsonProperty; import com.fasterxml.uuid.Generators; import com.joyent.manta.client.MantaClient; import com.joyent.manta.client.MantaMetadata; import com.joyent.manta.client.MantaObject; import com.joyent.manta.client.MantaObjectMapper; import com.joyent.manta.client.MantaObjectResponse; import com.joyent.manta.client.jobs.MantaJob; import com.joyent.manta.client.jobs.MantaJobBuilder; import com.joyent.manta.client.jobs.MantaJobPhase; import com.joyent.manta.exception.MantaClientHttpResponseException; import com.joyent.manta.exception.MantaException; import com.joyent.manta.exception.MantaIOException; import com.joyent.manta.exception.MantaMultipartException; import com.joyent.manta.http.HttpHelper; import com.joyent.manta.http.MantaHttpHeaders; import com.joyent.manta.util.MantaUtils; import org.apache.commons.lang3.Validate; import org.apache.http.HttpEntity; import org.apache.http.HttpStatus; import org.apache.http.StatusLine; import org.apache.http.client.methods.CloseableHttpResponse; import org.apache.http.client.methods.HttpPut; import org.apache.http.impl.client.CloseableHttpClient; import org.apache.http.protocol.HttpContext; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.io.IOException; import java.io.InputStream; import java.io.Serializable; import java.nio.file.Paths; import java.time.Duration; import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Objects; import java.util.Set; import java.util.UUID; import java.util.concurrent.CopyOnWriteArrayList; import java.util.concurrent.atomic.AtomicInteger; import java.util.function.Function; import java.util.stream.Stream; import java.util.stream.StreamSupport; import static com.joyent.manta.client.MantaClient.SEPARATOR; /** * Class providing a jobs-based implementation multipart uploads to Manta. * * @author <a href="https://github.com/dekobon">Elijah Zupancic</a> * @since 2.5.0 (moved from MantaMultipartManager) */ public class JobsMultipartManager extends AbstractMultipartManager<JobsMultipartUpload, MantaMultipartUploadPart> { /** * Logger instance. */ private static final Logger LOGGER = LoggerFactory.getLogger(JobsMultipartManager.class); /** * We only allow 1k parts for jobs based uploads because it is rather * expensive to do multipart with jobs. */ private static final int MAX_PARTS = 1_000; /** * Temporary storage directory on Manta for multipart data. This is a * randomly chosen UUID used so that we don't have directory naming * conflicts. */ static final String MULTIPART_DIRECTORY = "stor/.multipart-6439b444-9041-11e6-9be2-9f622f483d01"; /** * Metadata file containing information about final multipart file. */ static final String METADATA_FILE = "metadata.json"; /** * Number of seconds to poll Manta to see if a job is complete. */ private static final long DEFAULT_SECONDS_TO_POLL = 5L; /** * Number of times to check to see if a multipart transfer has completed. */ private static final int NUMBER_OF_TIMES_TO_POLL = 20; /** * Reference to {@link MantaClient} Manta client object providing access to * Manta. */ private final MantaClient mantaClient; /** * Reference to the collection of all of the {@link AutoCloseable} objects * that will need to be closed when MantaClient is closed. This reference * should point to the value set on the MantaClient field. */ private final Set<AutoCloseable> danglingStreams; /** * Reference to the Apache HTTP Client context and request creation helper. */ private final HttpHelper httpHelper; /** * Full path on Manta to the upload directory. */ private final String resolvedMultipartUploadDirectory; /** * Format for naming Manta jobs. */ private static final String JOB_NAME_FORMAT = "multipart-%s"; /** * Key name for retrieving job id from metadata. */ static final String JOB_ID_METADATA_KEY = "m-multipart-job-id"; /** * Key name for retrieving upload id from final object's metadata. */ static final String UPLOAD_ID_METADATA_KEY = "m-multipart-upload-id"; /** * Creates a new instance backed by the specified {@link MantaClient}. * @param mantaClient Manta client instance to use to communicate with server */ public JobsMultipartManager(final MantaClient mantaClient) { super(); Validate.notNull(mantaClient, "Manta client object must not be null"); this.mantaClient = mantaClient; this.httpHelper = readFieldFromMantaClient("httpHelper", mantaClient, HttpHelper.class); this.resolvedMultipartUploadDirectory = mantaClient.getContext().getMantaHomeDirectory() + SEPARATOR + MULTIPART_DIRECTORY; @SuppressWarnings("unchecked") final Set<AutoCloseable> dangling = (Set<AutoCloseable>) readFieldFromMantaClient("danglingStreams", mantaClient, Set.class); this.danglingStreams = dangling; } @Override public int getMaxParts() { return MAX_PARTS; } @Override public int getMinimumPartSize() { return 1; } @Override public Stream<MantaMultipartUpload> listInProgress() throws IOException { final List<Exception> exceptions = new CopyOnWriteArrayList<>(); final Stream<MantaObject> multipartDirList; try { multipartDirList = mantaClient.listObjects(this.resolvedMultipartUploadDirectory); // This catches an exception on the initial listObjects call } catch (final MantaClientHttpResponseException e) { if (e.getStatusCode() == HttpStatus.SC_NOT_FOUND) { return Stream.empty(); } else { throw e; } } final Stream<MantaMultipartUpload> stream = multipartDirList.filter(MantaObject::isDirectory) .map(object -> { final String idString = MantaUtils.lastItemInPath(object.getPath()); final UUID id = UUID.fromString(idString); try { MultipartMetadata mantaMetadata = downloadMultipartMetadata(id); MantaMultipartUpload upload = new JobsMultipartUpload(id, mantaMetadata.getPath()); return upload; } catch (MantaClientHttpResponseException e) { if (e.getStatusCode() == HttpStatus.SC_NOT_FOUND) { return null; } else { exceptions.add(e); return null; } } catch (IOException | RuntimeException e) { exceptions.add(e); return null; } }) /* We explicitly filter out items that stopped existing when we * went to get the multipart metadata because we encountered a * race condition. */ .filter(Objects::nonNull).onClose(multipartDirList::close); if (exceptions.isEmpty()) { danglingStreams.add(stream); return stream; } final MantaIOException aggregateException = new MantaIOException( "Problem(s) listing multipart uploads in progress"); MantaUtils.attachExceptionsToContext(aggregateException, exceptions); throw aggregateException; } @Override public JobsMultipartUpload initiateUpload(final String path) throws IOException { return initiateUpload(path, new MantaMetadata(), new MantaHttpHeaders()); } @Override public JobsMultipartUpload initiateUpload(final String path, final MantaMetadata mantaMetadata) throws IOException { return initiateUpload(path, mantaMetadata, new MantaHttpHeaders()); } @Override public JobsMultipartUpload initiateUpload(final String path, final MantaMetadata mantaMetadata, final MantaHttpHeaders httpHeaders) throws IOException { return initiateUpload(path, -1L, mantaMetadata, httpHeaders); } @Override public JobsMultipartUpload initiateUpload(final String path, final Long contentLength, final MantaMetadata mantaMetadata, final MantaHttpHeaders httpHeaders) throws IOException { // contentLength parameter is entirely ignored final MantaMetadata metadata; if (mantaMetadata == null) { metadata = new MantaMetadata(); } else { metadata = mantaMetadata; } final UUID uploadId = Generators.timeBasedGenerator().generate(); if (LOGGER.isDebugEnabled()) { LOGGER.debug("Creating a new multipart upload [{}] for {}", uploadId, path); } final String uploadDir = multipartUploadDir(uploadId); mantaClient.putDirectory(uploadDir, true); final String metadataPath = uploadDir + METADATA_FILE; final MultipartMetadata multipartMetadata = new MultipartMetadata().setPath(path) .setObjectMetadata(metadata); if (httpHeaders != null) { multipartMetadata.setContentType(httpHeaders.getContentType()); } final byte[] metadataBytes = MantaObjectMapper.INSTANCE.writeValueAsBytes(multipartMetadata); LOGGER.debug("Writing metadata to: {}", metadataPath); mantaClient.put(metadataPath, metadataBytes); return new JobsMultipartUpload(uploadId, path); } @Override MantaMultipartUploadPart uploadPart(final JobsMultipartUpload upload, final int partNumber, final HttpEntity entity, final HttpContext context) throws IOException { Validate.notNull(upload, "Multipart upload object must not be null"); Validate.notNull(entity, "Upload entity must not be null"); final String path = multipartPath(upload.getId(), partNumber); final HttpPut put = httpHelper.getRequestFactory().put(path); put.setEntity(entity); final int expectedStatusCode = HttpStatus.SC_NO_CONTENT; final CloseableHttpClient httpClient = httpHelper.getConnectionContext().getHttpClient(); try (CloseableHttpResponse response = httpClient.execute(put, context)) { StatusLine statusLine = response.getStatusLine(); final MantaObjectResponse objectResponse = new MantaObjectResponse(path, new MantaHttpHeaders(response.getAllHeaders())); if (statusLine.getStatusCode() != expectedStatusCode) { String errorMessage = "Manta server responded with an unexpected response code"; // We create the exception below because it will parse the JSON error codes MantaClientHttpResponseException mchre = new MantaClientHttpResponseException(put, response, path); // We chain it to this exception so that it obeys the contract MantaMultipartException e = new MantaMultipartException(errorMessage, mchre); HttpHelper.annotateContextedException(e, put, response); throw e; } return new MantaMultipartUploadPart(objectResponse); } } /** * Retrieves information about a single part of a multipart upload. * * @param upload multipart upload object * @param partNumber part number to identify relative location in final file * @return multipart single part object * @throws IOException thrown if there is a problem connecting to Manta */ @Override public MantaMultipartUploadPart getPart(final JobsMultipartUpload upload, final int partNumber) throws IOException { Validate.notNull(upload, "Multipart upload object must not be null"); final String path = multipartPath(upload.getId(), partNumber); final MantaObjectResponse response = mantaClient.head(path); return new MantaMultipartUploadPart(response); } /** * Retrieves the state of a given Manta multipart upload. * * @param upload multipart upload object * @return enum representing the state / status of the multipart upload * @throws IOException thrown if there is a problem connecting to Manta */ @Override public MantaMultipartStatus getStatus(final JobsMultipartUpload upload) throws IOException { Validate.notNull(upload, "Multipart upload object must not be null"); return getStatus(upload, null); } /** * Retrieves the state of a given Manta multipart upload. * * @param upload multipart upload object * @param jobId Manta job id used to concatenate multipart parts * @return enum representing the state / status of the multipart upload * @throws IOException thrown if there is a problem connecting to Manta */ private MantaMultipartStatus getStatus(final MantaMultipartUpload upload, final UUID jobId) throws IOException { Validate.notNull(upload, "Multipart upload id must not be null"); final String dir = multipartUploadDir(upload.getId()); final MantaObjectResponse response; final MantaJob job; try { response = mantaClient.head(dir); } catch (MantaClientHttpResponseException e) { if (e.getStatusCode() == HttpStatus.SC_NOT_FOUND) { if (jobId == null) { job = findJob(upload); } else { job = mantaClient.getJob(jobId); } if (job == null) { return MantaMultipartStatus.UNKNOWN; } else if (job.getCancelled() != null && job.getCancelled()) { return MantaMultipartStatus.ABORTED; } else if (job.getState().equals("done")) { return MantaMultipartStatus.COMPLETED; } else if (job.getState().equals("running")) { return MantaMultipartStatus.COMMITTING; } else { MantaException mioe = new MantaException("Unexpected job state"); mioe.setContextValue("job_state", job.getState()); mioe.setContextValue("job_id", job.getId().toString()); mioe.setContextValue("multipart_id", upload.getId()); mioe.setContextValue("multipart_upload_dir", dir); throw mioe; } } throw e; } if (!response.isDirectory()) { MantaMultipartException e = new MantaMultipartException( "Remote path was a file and not a directory as expected"); e.setContextValue("multipart_upload_dir", dir); throw e; } if (jobId == null) { job = findJob(upload); } else { job = mantaClient.getJob(jobId); } if (job == null) { return MantaMultipartStatus.CREATED; } /* If we still have the directory associated with the multipart * upload AND we are in the state of Cancelled. */ if (job.getCancelled()) { return MantaMultipartStatus.ABORTING; } final String state = job.getState(); /* If we still have the directory associated with the multipart * upload AND we have the job id, we are in a state where the * job hasn't finished clearing out the data files. */ if (state.equals("done") || state.equals("running") || state.equals("queued")) { return MantaMultipartStatus.COMMITTING; } else { return MantaMultipartStatus.UNKNOWN; } } /** * Lists the parts that have already been uploaded. * * @param upload multipart upload object * @return stream of parts identified by integer part number * @throws IOException thrown if there is a problem connecting to Manta */ @Override public Stream<MantaMultipartUploadPart> listParts(final JobsMultipartUpload upload) throws IOException { Validate.notNull(upload, "Multipart upload object must not be null"); final String dir = multipartUploadDir(upload.getId()); Stream<MantaMultipartUploadPart> stream = mantaClient.listObjects(dir) .filter(value -> !Paths.get(value.getPath()).getFileName().toString().equals(METADATA_FILE)) .map(MantaMultipartUploadPart::new); danglingStreams.add(stream); return stream; } /** * Aborts a multipart transfer. * * @param upload multipart upload object * @throws IOException thrown if there is a problem connecting to Manta */ @Override public void abort(final JobsMultipartUpload upload) throws IOException { Validate.notNull(upload, "Multipart upload object must not be null"); final String dir = multipartUploadDir(upload.getId()); final MantaJob job = findJob(upload); LOGGER.debug("Aborting multipart upload [{}]", upload.getId()); if (job != null && (job.getState().equals("running") || job.getState().equals("queued"))) { LOGGER.debug("Aborting multipart upload [{}] backing job [{}]", upload.getId(), job); mantaClient.cancelJob(job.getId()); } LOGGER.debug("Deleting multipart upload data from: {}", dir); mantaClient.deleteRecursive(dir); } @Override public void complete(final JobsMultipartUpload upload, final Iterable<? extends MantaMultipartUploadTuple> parts) throws IOException { Validate.notNull(upload, "Multipart upload object must not be null"); try (Stream<? extends MantaMultipartUploadTuple> stream = StreamSupport.stream(parts.spliterator(), false)) { complete(upload, stream); } } /** * Completes a multipart transfer by assembling the parts on Manta. * This is an asynchronous operation and you will need to call * {@link #waitForCompletion(MantaMultipartUpload, Duration, int, Function)} * to block until the operation completes. * * @param upload multipart upload object * @param partsStream stream of multipart part objects * @throws IOException thrown if there is a problem connecting to Manta */ @Override public void complete(final JobsMultipartUpload upload, final Stream<? extends MantaMultipartUploadTuple> partsStream) throws IOException { Validate.notNull(upload, "Multipart upload object must not be null"); LOGGER.debug("Completing multipart upload [{}]", upload.getId()); final String uploadDir = multipartUploadDir(upload.getId()); final MultipartMetadata metadata = downloadMultipartMetadata(upload.getId()); final Map<String, MantaMultipartUploadPart> listing = new HashMap<>(); try (Stream<MantaMultipartUploadPart> listStream = listParts(upload).limit(getMaxParts())) { listStream.forEach(p -> listing.put(p.getEtag(), p)); } final String path = metadata.getPath(); final StringBuilder jobExecText = new StringBuilder("set -o pipefail; mget -q "); List<MantaMultipartUploadTuple> missingTuples = new ArrayList<>(); final AtomicInteger count = new AtomicInteger(0); try (Stream<? extends MantaMultipartUploadTuple> distinct = partsStream.sorted().distinct()) { distinct.forEach(part -> { final int i = count.incrementAndGet(); if (i > getMaxParts()) { String msg = String.format( "Too many multipart parts specified [%d]. " + "The maximum number of parts is %d", getMaxParts(), count.get()); throw new IllegalArgumentException(msg); } // Catch and log any gaps in part numbers if (i != part.getPartNumber()) { missingTuples.add(new MantaMultipartUploadTuple(i, "N/A")); } else { final MantaMultipartUploadPart o = listing.get(part.getEtag()); if (o != null) { jobExecText.append(o.getObjectPath()).append(" "); } else { missingTuples.add(part); } } }); } if (!missingTuples.isEmpty()) { final MantaMultipartException e = new MantaMultipartException( "Multipart part(s) specified couldn't be found"); int missingCount = 0; for (MantaMultipartUploadTuple missingPart : missingTuples) { String key = String.format("missing_part_%d", ++missingCount); e.setContextValue(key, missingPart.toString()); } throw e; } final String headerFormat = "\"%s: %s\" "; jobExecText.append("| mput ").append("-H ") .append(String.format(headerFormat, UPLOAD_ID_METADATA_KEY, upload.getId())).append("-H ") .append(String.format(headerFormat, JOB_ID_METADATA_KEY, "$MANTA_JOB_ID")).append("-q "); if (metadata.getContentType() != null) { jobExecText.append("-H 'Content-Type: ").append(metadata.getContentType()).append("' "); } MantaMetadata objectMetadata = metadata.getObjectMetadata(); if (objectMetadata != null) { Set<Map.Entry<String, String>> entries = objectMetadata.entrySet(); for (Map.Entry<String, String> entry : entries) { jobExecText.append("-H '").append(entry.getKey()).append(": ").append(entry.getValue()) .append("' "); } } jobExecText.append(path); final MantaJobPhase concatPhase = new MantaJobPhase().setType("reduce").setExec(jobExecText.toString()); final MantaJobPhase cleanupPhase = new MantaJobPhase().setType("reduce").setExec("mrm -r " + uploadDir); MantaJobBuilder.Run run = mantaClient.jobBuilder().newJob(String.format(JOB_NAME_FORMAT, upload.getId())) .addPhase(concatPhase).addPhase(cleanupPhase).run(); // We write the job id to Metadata object so that we can query it easily writeJobIdToMetadata(upload.getId(), run.getId()); if (LOGGER.isDebugEnabled()) { LOGGER.debug("Created job for concatenating parts: {}", run.getId()); } } /** * Downloads the serialized metadata object from Manta and deserializes it. * * @param id multipart upload id * @return metadata object * @throws IOException thrown if there is a problem connecting to Manta */ protected JobsMultipartManager.MultipartMetadata downloadMultipartMetadata(final UUID id) throws IOException { final String uploadDir = multipartUploadDir(id); final String metadataPath = uploadDir + METADATA_FILE; LOGGER.debug("Reading metadata from: {}", metadataPath); try (InputStream in = mantaClient.getAsInputStream(metadataPath)) { return MantaObjectMapper.INSTANCE.readValue(in, JobsMultipartManager.MultipartMetadata.class); } } /** * Writes the multipart job id to the metadata object's Manta metadata. * * @param uploadId multipart upload id * @param jobId Manta job id used to concatenate multipart parts * @throws IOException thrown if there is a problem connecting to Manta */ protected void writeJobIdToMetadata(final UUID uploadId, final UUID jobId) throws IOException { Validate.notNull(uploadId, "Multipart upload id must not be null"); Validate.notNull(jobId, "Job id must not be null"); final String uploadDir = multipartUploadDir(uploadId); final String metadataPath = uploadDir + METADATA_FILE; LOGGER.debug("Writing job id [{}] to: {}", jobId, metadataPath); MantaMetadata metadata = new MantaMetadata(); metadata.put(JOB_ID_METADATA_KEY, jobId.toString()); mantaClient.putMetadata(metadataPath, metadata); } /** * Writes the multipart job id to the metadata object's Manta metadata. * * @param uploadId multipart upload id * @throws IOException thrown if there is a problem connecting to Manta * @return Manta job id used to concatenate multipart parts */ protected UUID getJobIdFromMetadata(final UUID uploadId) throws IOException { Validate.notNull(uploadId, "Multipart upload id must not be null"); final String uploadDir = multipartUploadDir(uploadId); final String metadataPath = uploadDir + METADATA_FILE; try { MantaObjectResponse response = mantaClient.head(metadataPath); String uuidAsString = response.getMetadata().get(JOB_ID_METADATA_KEY); if (uuidAsString == null) { return null; } return UUID.fromString(uuidAsString); } catch (MantaClientHttpResponseException e) { if (e.getStatusCode() == HttpStatus.SC_NOT_FOUND) { return null; } throw e; } } /** * Waits for a multipart upload to complete. Polling every 5 seconds. * * @param <R> Return type for executeWhenTimesToPollExceeded * @param upload multipart upload object * @param executeWhenTimesToPollExceeded lambda executed when timesToPoll has been exceeded * @return null when under poll timeout, otherwise returns return value of executeWhenTimesToPollExceeded * @throws IOException thrown if there is a problem connecting to Manta */ public <R> R waitForCompletion(final MantaMultipartUpload upload, final Function<UUID, R> executeWhenTimesToPollExceeded) throws IOException { Validate.notNull(upload, "Multipart upload object must not be null"); return waitForCompletion(upload, Duration.ofSeconds(DEFAULT_SECONDS_TO_POLL), NUMBER_OF_TIMES_TO_POLL, executeWhenTimesToPollExceeded); } /** * Waits for a multipart upload to complete. Polling for set interval. * * @param <R> Return type for executeWhenTimesToPollExceeded * @param upload multipart upload object * @param pingInterval interval to poll * @param timesToPoll number of times to poll Manta to check for completion * @param executeWhenTimesToPollExceeded lambda executed when timesToPoll has been exceeded * @return null when under poll timeout, otherwise returns return value of executeWhenTimesToPollExceeded * @throws IOException thrown if there is a problem connecting to Manta */ public <R> R waitForCompletion(final MantaMultipartUpload upload, final Duration pingInterval, final int timesToPoll, final Function<UUID, R> executeWhenTimesToPollExceeded) throws IOException { if (timesToPoll <= 0) { String msg = String.format( "times to poll should be set to a value greater than 1. " + "Actual value: %d", timesToPoll); throw new IllegalArgumentException(msg); } final String dir = multipartUploadDir(upload.getId()); final MantaJob job = findJob(upload); if (job == null) { String msg = "Unable for find job associated with multipart upload. " + "Was complete() run for upload or was it run so long ago " + "that we no longer have a record for it?"; MantaMultipartException e = new MantaMultipartException(msg); e.setContextValue("upload_id", upload.getId().toString()); e.setContextValue("upload_directory", dir); e.setContextValue("job_id", job.getId().toString()); throw e; } final long waitMillis = pingInterval.toMillis(); int timesPolled; /* We ping the upload directory and wait for it to be deleted because * there is the chance for a race condition when the job attempts to * delete the upload directory, but isn't finished. */ for (timesPolled = 0; timesPolled < timesToPoll; timesPolled++) { try { final MantaMultipartStatus status = getStatus(upload, job.getId()); // We do a check preemptively because we shouldn't sleep unless we need to if (status.equals(MantaMultipartStatus.COMPLETED)) { return null; } if (status.equals(MantaMultipartStatus.ABORTED)) { String msg = "Manta job backing multipart upload was aborted. " + "This upload was unable to be completed."; MantaMultipartException e = new MantaMultipartException(msg); e.setContextValue("upload_id", upload.getId().toString()); e.setContextValue("upload_directory", dir); e.setContextValue("job_id", job.getId().toString()); throw e; } if (status.equals(MantaMultipartStatus.UNKNOWN)) { String msg = "Manta job backing multipart upload was is in " + "a unknown state. Typically this means that we " + "are unable to get the status of the job backing " + "the multipart upload."; MantaMultipartException e = new MantaMultipartException(msg); e.setContextValue("upload_id", upload.getId().toString()); e.setContextValue("upload_directory", dir); e.setContextValue("job_id", job.getId().toString()); throw e; } // Don't bother to sleep if we won't be doing a check if (timesPolled < timesToPoll + 1) { if (LOGGER.isDebugEnabled()) { LOGGER.debug("Waiting for [{}] ms for upload [{}] to complete " + "(try {} of {})", waitMillis, upload.getId(), timesPolled + 1, timesToPoll); } Thread.sleep(waitMillis); } } catch (InterruptedException e) { /* We assume the client has written logic for when the polling operation * doesn't complete within the time period as expected and we also make * the assumption that that behavior would be acceptable when the thread * has been interrupted. */ return executeWhenTimesToPollExceeded.apply(upload.getId()); } } if (timesPolled >= timesToPoll) { return executeWhenTimesToPollExceeded.apply(upload.getId()); } return null; } /** * Builds the full remote path for a part of a multipart upload. * * @param id multipart upload id * @param partNumber part number to identify relative location in final file * @return temporary path on Manta to store part */ String multipartPath(final UUID id, final int partNumber) { validatePartNumber(partNumber); final String dir = multipartUploadDir(id); return String.format("%s%d", dir, partNumber); } /** * Finds the directory in which to upload parts into. * * @param id multipart transaction id * @return temporary Manta directory in which to upload parts */ String multipartUploadDir(final UUID id) { Validate.notNull(id, "Multipart transaction id must not be null"); return this.resolvedMultipartUploadDirectory + SEPARATOR + id.toString() + SEPARATOR; } /** * Returns the Manta job used to concatenate multiple file parts. * * @param upload multipart upload object * @return Manta job object or null if not found * @throws IOException thrown if there is a problem connecting to Manta */ MantaJob findJob(final MantaMultipartUpload upload) throws IOException { final UUID jobId = getJobIdFromMetadata(upload.getId()); if (jobId == null) { LOGGER.debug("Unable to get job id from metadata directory. Now trying job listing."); try (Stream<MantaJob> jobs = mantaClient .getJobsByName(String.format(JOB_NAME_FORMAT, upload.getId()))) { return jobs.findFirst().orElse(null); } } return mantaClient.getJob(jobId); } /** * Inner class used only with the jobs-based multipart implementation for * storing header and metadata information. */ @JsonInclude static class MultipartMetadata implements Serializable { private static final long serialVersionUID = -4410867990710890357L; /** * Path to final object on Manta. */ @JsonProperty private String path; /** * Metadata of final object. */ @JsonProperty("object_metadata") private HashMap<String, String> objectMetadata; /** * HTTP content type to write to the final object. */ @JsonProperty("content_type") private String contentType; /** * Creates a new instance. */ MultipartMetadata() { } String getPath() { return path; } /** * Sets the path to the final object on Manta. * * @param path remote Manta path * @return this instance */ MultipartMetadata setPath(final String path) { this.path = path; return this; } /** * Gets the metadata associated with the final Manta object. * * @return new instance of {@link MantaMetadata} with data populated */ MantaMetadata getObjectMetadata() { if (this.objectMetadata == null) { return null; } return new MantaMetadata(this.objectMetadata); } /** * Sets the metadata to be written to the final object on Manta. * * @param objectMetadata metadata to write * @return this instance */ MultipartMetadata setObjectMetadata(final MantaMetadata objectMetadata) { if (objectMetadata != null) { this.objectMetadata = new HashMap<>(objectMetadata); } else { this.objectMetadata = null; } return this; } String getContentType() { return contentType; } /** * Sets http headers to write to the final object on Manta. Actually, * we only consume Content-Type for now. * * @param contentType HTTP content type to set for the object * @return this instance */ @SuppressWarnings("UnusedReturnValue") MultipartMetadata setContentType(final String contentType) { this.contentType = contentType; return this; } } }