Java tutorial
/* * Copyright 2014 Carsten Rambow, elomagic. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package de.elomagic.carafile.client; import java.io.BufferedInputStream; import java.io.BufferedOutputStream; import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; import java.io.FileNotFoundException; import java.io.IOException; import java.io.InputStream; import java.io.InputStreamReader; import java.io.OutputStream; import java.net.URI; import java.nio.charset.Charset; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.StandardOpenOption; import java.security.DigestInputStream; import java.security.DigestOutputStream; import java.security.GeneralSecurityException; import java.security.MessageDigest; import java.util.HashMap; import java.util.HashSet; import java.util.Map; import java.util.Set; import org.apache.commons.codec.binary.Hex; import org.apache.commons.codec.digest.DigestUtils; import org.apache.http.HttpHeaders; import org.apache.http.HttpHost; import org.apache.http.HttpResponse; import org.apache.http.HttpStatus; import org.apache.http.client.HttpResponseException; import org.apache.http.client.fluent.Executor; import org.apache.http.client.fluent.Request; import org.apache.http.client.fluent.Response; import org.apache.http.entity.ContentType; import org.apache.http.impl.client.BasicCookieStore; import org.apache.log4j.Logger; import de.elomagic.carafile.share.ChunkData; import de.elomagic.carafile.share.JsonUtil; import de.elomagic.carafile.share.MetaData; import de.elomagic.carafile.share.PeerData; import de.elomagic.carafile.share.PeerDataSet; import de.elomagic.carafile.share.RegistryStatus; /** * Client implementation for the CaraFile network. * * @author carsten.rambow */ public class CaraFileClient { private static final Logger LOG = Logger.getLogger(CaraFileClient.class); private final BasicCookieStore store = new BasicCookieStore(); private PeerChunkSelector peerChunkSelector = new StandardPeerChunkSelector(); private PeerSelector peerSelector = new StandardPeerSelector(); private Executor executor; private HttpHost proxyHost; private URI registryURI; private RequestPasswordListener passwordListener; private String authUserName; private char[] authPassword; private String authDomain; /** * Creates an instance with default {@link StandardPeerChunkSelector} and {@link StandardPeerSelector}. */ public CaraFileClient() { } /** * Creates an instance with given {@link PeerChunkSelector} and default {@link StandardPeerSelector}. * * @param peerChunkSelector Selector */ public CaraFileClient(final PeerChunkSelector peerChunkSelector) { this.peerChunkSelector = peerChunkSelector; } /** * Tries to find a file in the registry with the given file identifier. * * @param fileId Identifier of the file * @return Returns the {@link MetaData} of the file or null when not found. * @throws IOException Thrown when unable to call REST service */ public MetaData findFile(final String fileId) throws IOException { if (registryURI == null) { throw new IllegalArgumentException("Parameter 'registryURI' must not be null!"); } if (fileId == null) { throw new IllegalArgumentException("Parameter 'fileId' must not be null!"); } URI uri = CaraFileUtils.buildURI(registryURI, "registry", "findFile", fileId); HttpResponse response = executeRequest(Request.Get(uri)).returnResponse(); if (response.getStatusLine().getStatusCode() == HttpStatus.SC_OK) { return getMetaDataFromResponse(response); } return null; } /** * Deletes a file at the registry. * * @param fileId File identifier * @param beQuite If true then no FileNotFoundException will be thrown when file already doesn't exists * @throws IOException Thrown when unable to call REST services */ public void deleteFile(final String fileId, final boolean beQuite) throws IOException { if (registryURI == null) { throw new IllegalArgumentException("Parameter 'registryURI' must not be null!"); } if (fileId == null) { throw new IllegalArgumentException("Parameter 'fileId' must not be null!"); } URI uri = CaraFileUtils.buildURI(registryURI, "registry", "deleteFile", fileId); HttpResponse response = executeRequest(Request.Delete(uri)).returnResponse(); int statusCode = response.getStatusLine().getStatusCode(); if (statusCode == HttpStatus.SC_OK || (statusCode == HttpStatus.SC_NOT_FOUND && beQuite)) { } else if (statusCode == HttpStatus.SC_NOT_FOUND) { throw new FileNotFoundException("File Id " + fileId + " not found."); } else { throw new IOException(statusCode + " " + response.getStatusLine().getReasonPhrase()); } } /** * Uploads data via an {@link InputStream} (Single chunk upload). * <p/> * Single chunk upload means that the complete file will be upload in one step. * * @param in The input stream. It's not recommended to use a buffered stream. * @param filename Name of the file * @param contentLength Length of the content in bytes * @return Returns the {@link MetaData} of the uploaded stream * @throws IOException Thrown when unable to call REST services * @see CaraFileClient#uploadFile(java.net.URI, java.nio.file.Path, java.lang.String) */ public MetaData uploadFile(final InputStream in, final String filename, final long contentLength) throws IOException { if (registryURI == null) { throw new IllegalArgumentException("Parameter 'registryURI' must not be null!"); } if (in == null) { throw new IllegalArgumentException("Parameter 'in' must not be null!"); } URI peerURI = peerSelector.getURI(downloadPeerSet(), -1); if (peerURI == null) { throw new IOException("No peer for upload available"); } URI uri = CaraFileUtils.buildURI(peerURI, "peer", "seedFile", filename); MessageDigest messageDigest = DigestUtils.getSha1Digest(); try (BufferedInputStream bis = new BufferedInputStream(in); DigestInputStream dis = new DigestInputStream(bis, messageDigest)) { HttpResponse response = executeRequest( Request.Post(uri).bodyStream(dis, ContentType.APPLICATION_OCTET_STREAM)).returnResponse(); int statusCode = response.getStatusLine().getStatusCode(); if (statusCode != HttpStatus.SC_OK) { throw new HttpResponseException(statusCode, "Unable to upload file: " + response.getStatusLine().getReasonPhrase()); } MetaData md = getMetaDataFromResponse(response); if (!Hex.encodeHexString(messageDigest.digest()).equals(md.getId())) { throw new IOException("Peer response invalid SHA1 of file"); } return md; } } /** * Uploads a file (Multi chunk upload). * <p/> * Multi chunk upload means that the file will be devived in one or more chunks and each chunk can be downloaded to a different peer. * * @param path Must be a file and not a directory. File will not be deleted * @param filename Name of the file. If null then name of the parameter path will be used * @return Returns the {@link MetaData} of the uploaded stream * @throws IOException Thrown when unable to call REST services * @throws java.security.GeneralSecurityException Thrown when unable to determine SHA-1 of the file * @see CaraFileClient#uploadFile(java.net.URI, java.io.InputStream, java.lang.String, long) */ public MetaData uploadFile(final Path path, final String filename) throws IOException, GeneralSecurityException { if (registryURI == null) { throw new IllegalArgumentException("Parameter 'registryURI' must not be null!"); } if (path == null) { throw new IllegalArgumentException("Parameter 'path' must not be null!"); } if (Files.notExists(path)) { throw new FileNotFoundException("File \"" + path + "\" doesn't exists!"); } if (Files.isDirectory(path)) { throw new IOException("Parameter 'path' is not a file!"); } String fn = filename == null ? path.getFileName().toString() : filename; MetaData md = CaraFileUtils.createMetaData(path, fn); md.setRegistryURI(registryURI); String json = JsonUtil.write(md); LOG.debug("Register " + md.getId() + " file at " + registryURI.toString()); URI uri = CaraFileUtils.buildURI(registryURI, "registry", "register"); HttpResponse response = executeRequest(Request.Post(uri).bodyString(json, ContentType.APPLICATION_JSON)) .returnResponse(); if (response.getStatusLine().getStatusCode() != HttpStatus.SC_OK) { throw new IOException("Unable to register file. " + response.getStatusLine().getReasonPhrase()); } Set<PeerData> peerDataSet = downloadPeerSet(); byte[] buffer = new byte[md.getChunkSize()]; try (InputStream in = Files.newInputStream(path, StandardOpenOption.READ); BufferedInputStream bis = new BufferedInputStream(in, md.getChunkSize())) { int bytesRead; int chunkIndex = 0; while ((bytesRead = bis.read(buffer)) > 0) { String chunkId = md.getChunk(chunkIndex).getId(); URI peerURI = peerSelector.getURI(peerDataSet, chunkIndex); URI seedChunkUri = CaraFileUtils.buildURI(peerURI, "peer", "seedChunk", chunkId); LOG.debug("Uploading chunk " + chunkId + " to peer " + seedChunkUri.toString() + ";Index=" + chunkIndex + ";Length=" + bytesRead); response = executeRequest(Request.Post(seedChunkUri).bodyStream( new ByteArrayInputStream(buffer, 0, bytesRead), ContentType.APPLICATION_OCTET_STREAM)) .returnResponse(); if (response.getStatusLine().getStatusCode() != HttpStatus.SC_OK) { throw new IOException("Unable to upload file. " + response.getStatusLine().getStatusCode() + " " + response.getStatusLine().getReasonPhrase()); } chunkIndex++; } } return md; } /** * Downloads a file into a {@link OutputStream}. * * @param fileId Identifier of the file * @param out Output stream where the file will be written in * @throws IOException Thrown when unable to call REST services */ public void downloadFile(final String fileId, final OutputStream out) throws IOException { if (registryURI == null) { throw new IllegalArgumentException("Parameter 'registryURI' must not be null!"); } if (fileId == null) { throw new IllegalArgumentException("Parameter 'fileId' must not be null!"); } if (out == null) { throw new IllegalArgumentException("Parameter 'out' must not be null!"); } MetaData md = findFile(fileId); if (md == null) { throw new IOException("File id " + fileId + " doesn't exists at registry " + registryURI.toString()); } downloadFile(md, out); } /** * Downloads a file into a {@link OutputStream}. * * @param md {@link MetaData} of the file. * @param out The output stream. It's not recommended to use a buffered stream. * @throws IOException Thrown when unable to write file into the output stream or the SHA-1 validation failed. */ public void downloadFile(final MetaData md, final OutputStream out) throws IOException { if (md == null) { throw new IllegalArgumentException("Parameter 'md' must not be null!"); } if (out == null) { throw new IllegalArgumentException("Parameter 'out' must not be null!"); } Map<String, Path> downloadedChunks = new HashMap<>(); Set<String> chunksToDownload = new HashSet<>(); for (ChunkData chunkData : md.getChunks()) { chunksToDownload.add(chunkData.getId()); } try { while (!chunksToDownload.isEmpty()) { PeerChunk pc = peerChunkSelector.getNext(md, chunksToDownload); if (pc == null || pc.getPeerURI() == null) { throw new IOException("No peer found or selected for download"); } Path chunkFile = Files.createTempFile("fs_", ".tmp"); try (OutputStream chunkOut = Files.newOutputStream(chunkFile, StandardOpenOption.APPEND)) { downloadShunk(pc, md, chunkOut); downloadedChunks.put(pc.getChunkId(), chunkFile); chunksToDownload.remove(pc.getChunkId()); chunkOut.flush(); } catch (Exception ex) { Files.deleteIfExists(chunkFile); throw ex; } } MessageDigest messageDigest = DigestUtils.getSha1Digest(); // Write chunk on correct order to file. try (DigestOutputStream dos = new DigestOutputStream(out, messageDigest); BufferedOutputStream bos = new BufferedOutputStream(dos, md.getChunkSize())) { for (ChunkData chunk : md.getChunks()) { Path chunkPath = downloadedChunks.get(chunk.getId()); Files.copy(chunkPath, bos); } } String sha1 = Hex.encodeHexString(messageDigest.digest()); if (!sha1.equalsIgnoreCase(md.getId())) { throw new IOException( "SHA1 validation of file failed. Expected " + md.getId() + " but was " + sha1); } } finally { for (Path path : downloadedChunks.values()) { try { Files.deleteIfExists(path); } catch (IOException ex) { LOG.error("Unable to delete chunk " + path.toString() + "; " + ex.getMessage(), ex); } } } } /** * Don't call this method! * <p/> * This method will be called usually by the server and this class. * * @param sp * @param md * @param out * @throws IOException */ public void downloadShunk(final PeerChunk sp, final MetaData md, final OutputStream out) throws IOException { if (out == null) { throw new IllegalArgumentException("Parameter 'out' must not be null!"); } URI uri = CaraFileUtils.buildURI(sp.getPeerURI(), "peer", "leechChunk", sp.getChunkId()); HttpResponse response = executeRequest( Request.Get(uri).addHeader(HttpHeaders.ACCEPT, ContentType.APPLICATION_OCTET_STREAM.toString())) .returnResponse(); ByteArrayOutputStream baos = new ByteArrayOutputStream(); response.getEntity().writeTo(baos); String sha1 = DigestUtils.sha1Hex(baos.toByteArray()); if (!sha1.equalsIgnoreCase(sp.getChunkId())) { throw new IOException("SHA1 validation failed. Expected " + sp.getChunkId() + " but was " + sha1); } out.write(baos.toByteArray()); } /** * Returns status of the registry. * * @return Status * @throws IOException Thrown when unable to request status of the registry */ public RegistryStatus getRegistryStatus() throws IOException { if (registryURI == null) { throw new IllegalArgumentException("Parameter 'registryURI' must not be null!"); } LOG.debug("Getting registry status"); URI uri = CaraFileUtils.buildURI(registryURI, "registry", "status"); HttpResponse response = executeRequest(Request.Get(uri)).returnResponse(); Charset charset = ContentType.getOrDefault(response.getEntity()).getCharset(); RegistryStatus status = JsonUtil.read(new InputStreamReader(response.getEntity().getContent(), charset), RegistryStatus.class); LOG.debug("Registry status responsed"); return status; } private Set<PeerData> downloadPeerSet() throws IOException { if (registryURI == null) { throw new IllegalArgumentException("Parameter 'registryURI' must not be null!"); } LOG.debug("Download set of peers"); URI uri = CaraFileUtils.buildURI(registryURI, "registry", "listPeers"); HttpResponse response = executeRequest(Request.Get(uri)).returnResponse(); if (response.getStatusLine().getStatusCode() == HttpStatus.SC_OK) { Charset charset = ContentType.getOrDefault(response.getEntity()).getCharset(); Set<PeerData> peerSet = JsonUtil.read(new InputStreamReader(response.getEntity().getContent(), charset), PeerDataSet.class); LOG.debug("Registry response set of " + peerSet.size() + " peer(s)"); return peerSet; } throw new IOException("HTTP responce code " + response.getStatusLine().getStatusCode() + " " + response.getStatusLine().getReasonPhrase()); } public URI getRegistryURI() { return registryURI; } public CaraFileClient setRegistryURI(final URI registryURI) { this.registryURI = registryURI; return this; } /** * Set Http proxy. * * @param proxyHost Proxy host * @return Returns itself */ public CaraFileClient setProxyHost(final HttpHost proxyHost) { this.proxyHost = proxyHost; return this; } public CaraFileClient auth(final String username, final char[] password) { authUserName = username; authPassword = password; return this; } public CaraFileClient auth(final String username, final char[] password, final String domain) { authUserName = username; authPassword = password; authDomain = domain; return this; } /** * Set a different {@link PeerChunkSelector}. * * @param peerChunkSelector The selector * @return Returns itself */ public CaraFileClient setPeerChunkSelector(final PeerChunkSelector peerChunkSelector) { this.peerChunkSelector = peerChunkSelector; return this; } /** * Set a different {@link PeerSelector}. * * @param peerSelector selector * @return Returns itself */ public CaraFileClient setPeerSelector(final PeerSelector peerSelector) { this.peerSelector = peerSelector; return this; } public void setRequestPasswordListener(final RequestPasswordListener passwordListener) { this.passwordListener = passwordListener; } MetaData getMetaDataFromResponse(final HttpResponse response) throws IOException { Charset charset = ContentType.getOrDefault(response.getEntity()).getCharset(); return JsonUtil.readFromReader(new InputStreamReader(response.getEntity().getContent(), charset)); } Response executeRequest(final Request request) throws IOException { if (executor == null) { executor = Executor.newInstance(); } executor.cookieStore(store); if (authUserName != null) { char[] p = authPassword; p = p == null && passwordListener != null ? passwordListener.getPassword(this) : p; if (authDomain == null) { executor.auth(authUserName, new String(p)); } else { executor.auth(authUserName, new String(p), null, authDomain); } } if (proxyHost != null) { request.viaProxy(proxyHost); } request.addHeader(HttpHeaders.ACCEPT, ContentType.APPLICATION_JSON.toString()); return executor.execute(request); } }