Java tutorial
/** * Copyright (C) 2011-2012 Turn, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package com.p2p.peercds.common; import static com.p2p.peercds.common.Constants.BUCKET_NAME; import static com.p2p.peercds.common.Constants.BYTE_ENCODING; import static com.p2p.peercds.common.Constants.CLOUD_KEY; import static com.p2p.peercds.common.Constants.PIECE_LENGTH; import static com.p2p.peercds.common.Constants.SPECIAL_TO_NSP_MAP; import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; import java.io.File; import java.io.FileInputStream; import java.io.IOException; import java.io.OutputStream; import java.io.UnsupportedEncodingException; import java.net.URI; import java.net.URISyntaxException; import java.nio.ByteBuffer; import java.nio.channels.FileChannel; import java.security.MessageDigest; import java.util.ArrayList; import java.util.Arrays; import java.util.Date; import java.util.HashMap; import java.util.HashSet; import java.util.LinkedList; import java.util.List; import java.util.Map; import java.util.Set; import java.util.TreeMap; import java.util.concurrent.Callable; import java.util.concurrent.ExecutionException; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.Future; import org.apache.commons.codec.binary.Hex; import org.apache.commons.codec.digest.DigestUtils; import org.apache.commons.io.FileUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import com.google.common.base.Strings; import com.p2p.peercds.bcodec.BDecoder; import com.p2p.peercds.bcodec.BEValue; import com.p2p.peercds.bcodec.BEncoder; /** * A torrent file tracked by the controller's BitTorrent tracker. * * <p> * This class represents an active torrent on the tracker. The torrent * information is kept in-memory, and is created from the byte blob one would * usually find in a <tt>.torrent</tt> file. * </p> * * <p> * Each torrent also keeps a repository of the peers seeding and leeching this * torrent from the tracker. * </p> * * @author mpetazzoni * @see <a href="http://wiki.theory.org/BitTorrentSpecification#Metainfo_File_Structure">Torrent meta-info file structure specification</a> */ public class Torrent { private static final Logger logger = LoggerFactory.getLogger(Torrent.class); /** * * @author dgiffin * @author mpetazzoni */ public static class TorrentFile { public final File file; public final long size; public TorrentFile(File file, long size) { this.file = file; this.size = size; } } protected final byte[] encoded; protected final byte[] encoded_info; protected final Map<String, BEValue> decoded; protected final Map<String, BEValue> decoded_info; protected final String cloudKey; private final byte[] info_hash; private final String hex_info_hash; private final List<List<URI>> trackers; private final Set<URI> allTrackers; private final Date creationDate; private final String comment; private final String createdBy; private final String name; private final long size; protected final List<TorrentFile> files; private final boolean seeder; /** * Create a new torrent from meta-info binary data. * * Parses the meta-info data (which should be B-encoded as described in the * BitTorrent specification) and create a Torrent object from it. * * @param torrent The meta-info byte data. * @param seeder Whether we'll be seeding for this torrent or not. * @throws IOException When the info dictionary can't be read or * encoded and hashed back to create the torrent's SHA-1 hash. */ public Torrent(byte[] torrent, boolean seeder) throws IOException { this.encoded = torrent; this.seeder = seeder; this.decoded = BDecoder.bdecode(new ByteArrayInputStream(this.encoded)).getMap(); this.decoded_info = this.decoded.get("info").getMap(); ByteArrayOutputStream baos = new ByteArrayOutputStream(); BEncoder.bencode(this.decoded_info, baos); this.encoded_info = baos.toByteArray(); this.info_hash = Torrent.hash(this.encoded_info); this.hex_info_hash = Torrent.byteArrayToHexString(this.info_hash); /** * Parses the announce information from the decoded meta-info * structure. * * <p> * If the torrent doesn't define an announce-list, use the mandatory * announce field value as the single tracker in a single announce * tier. Otherwise, the announce-list must be parsed and the trackers * from each tier extracted. * </p> * * @see <a href="http://bittorrent.org/beps/bep_0012.html">BitTorrent BEP#0012 "Multitracker Metadata Extension"</a> */ try { this.trackers = new ArrayList<List<URI>>(); this.allTrackers = new HashSet<URI>(); if (this.decoded.containsKey("announce-list")) { List<BEValue> tiers = this.decoded.get("announce-list").getList(); for (BEValue tv : tiers) { List<BEValue> trackers = tv.getList(); if (trackers.isEmpty()) { continue; } List<URI> tier = new ArrayList<URI>(); for (BEValue tracker : trackers) { URI uri = new URI(tracker.getString()); // Make sure we're not adding duplicate trackers. if (!this.allTrackers.contains(uri)) { tier.add(uri); this.allTrackers.add(uri); } } // Only add the tier if it's not empty. if (!tier.isEmpty()) { this.trackers.add(tier); } } } else if (this.decoded.containsKey("announce")) { URI tracker = new URI(this.decoded.get("announce").getString()); this.allTrackers.add(tracker); // Build a single-tier announce list. List<URI> tier = new ArrayList<URI>(); tier.add(tracker); this.trackers.add(tier); } } catch (URISyntaxException use) { throw new IOException(use); } this.cloudKey = this.decoded.containsKey(CLOUD_KEY) ? this.decoded.get(CLOUD_KEY).getString() : ""; this.creationDate = this.decoded.containsKey("creation date") ? new Date(this.decoded.get("creation date").getLong() * 1000) : null; this.comment = this.decoded.containsKey("comment") ? this.decoded.get("comment").getString() : null; this.createdBy = this.decoded.containsKey("created by") ? this.decoded.get("created by").getString() : null; this.name = this.decoded_info.get("name").getString(); this.files = new LinkedList<TorrentFile>(); // Parse multi-file torrent file information structure. if (this.decoded_info.containsKey("files")) { for (BEValue file : this.decoded_info.get("files").getList()) { Map<String, BEValue> fileInfo = file.getMap(); StringBuilder path = new StringBuilder(); for (BEValue pathElement : fileInfo.get("path").getList()) { path.append(File.separator).append(pathElement.getString()); } this.files.add( new TorrentFile(new File(this.name, path.toString()), fileInfo.get("length").getLong())); } } else { // For single-file torrents, the name of the torrent is // directly the name of the file. this.files.add(new TorrentFile(new File(this.name), this.decoded_info.get("length").getLong())); } // Calculate the total size of this torrent from its files' sizes. long size = 0; for (TorrentFile file : this.files) { size += file.size; } this.size = size; logger.info("{}-file torrent information:", this.isMultifile() ? "Multi" : "Single"); logger.info(" Torrent name: {}", this.name); logger.info(" Announced at:" + (this.trackers.size() == 0 ? " Seems to be trackerless" : "")); for (int i = 0; i < this.trackers.size(); i++) { List<URI> tier = this.trackers.get(i); for (int j = 0; j < tier.size(); j++) { logger.info(" {}{}", (j == 0 ? String.format("%2d. ", i + 1) : " "), tier.get(j)); } } if (this.creationDate != null) { logger.info(" Created on..: {}", this.creationDate); } if (!Strings.isNullOrEmpty(cloudKey)) logger.info("Seems to be a peercds supported torrent. Cloud-key for this torrent is: " + cloudKey); else logger.info( " As cloud-key is not present, this torrent seems to be created by non-peercds torrent client, all the cloud based optimizations will not be available in this activity "); if (this.comment != null) { logger.info(" Comment.....: {}", this.comment); } if (this.createdBy != null) { logger.info(" Created by..: {}", this.createdBy); } if (this.isMultifile()) { logger.info(" Found {} file(s) in multi-file torrent structure.", this.files.size()); int i = 0; for (TorrentFile file : this.files) { logger.debug(" {}. {} ({} byte(s))", new Object[] { String.format("%2d", ++i), file.file.getPath(), String.format("%,d", file.size) }); } } logger.info(" Pieces......: {} piece(s) ({} byte(s)/piece)", (this.size / this.decoded_info.get("piece length").getInt()) + 1, this.decoded_info.get("piece length").getInt()); logger.info(" Total size..: {} byte(s)", String.format("%,d", this.size)); } /** * Get this torrent's name. * * <p> * For a single-file torrent, this is usually the name of the file. For a * multi-file torrent, this is usually the name of a top-level directory * containing those files. * </p> */ public String getName() { return this.name; } /** * Get this torrent's comment string. */ public String getComment() { return this.comment; } /** * Get this torrent's creator (user, software, whatever...). */ public String getCreatedBy() { return this.createdBy; } /** * Get the total size of this torrent. */ public long getSize() { return this.size; } /** * Get the file names from this torrent. * * @return The list of relative filenames of all the files described in * this torrent. */ public List<String> getFilenames() { List<String> filenames = new LinkedList<String>(); for (TorrentFile file : this.files) { filenames.add(file.file.getPath()); } return filenames; } /** * Tells whether this torrent is multi-file or not. */ protected boolean isMultifile() { return this.files.size() > 1; } /** * Return the hash of the B-encoded meta-info structure of this torrent. */ public byte[] getInfoHash() { return this.info_hash; } /** * Get this torrent's info hash (as an hexadecimal-coded string). */ public String getHexInfoHash() { return this.hex_info_hash; } /** * Return a human-readable representation of this torrent object. * * <p> * The torrent's name is used. * </p> */ public String toString() { return this.getName(); } /** * Return the B-encoded meta-info of this torrent. */ public byte[] getEncoded() { return this.encoded; } /** * Return the trackers for this torrent. */ public List<List<URI>> getAnnounceList() { return this.trackers; } /** * Returns the number of trackers for this torrent. */ public int getTrackerCount() { return this.allTrackers.size(); } /** * Tells whether we were an initial seeder for this torrent. */ public boolean isSeeder() { return this.seeder; } /** * Save this torrent meta-info structure into a .torrent file. * * @param output The stream to write to. * @throws IOException If an I/O error occurs while writing the file. */ public void save(OutputStream output) throws IOException { output.write(this.getEncoded()); logger.info("Torrent.Save(): Flushing file"); output.flush(); } public static byte[] hash(byte[] data) { return DigestUtils.sha1(data); } /** * Convert a byte string to a string containing an hexadecimal * representation of the original data. * * @param bytes The byte array to convert. */ public static String byteArrayToHexString(byte[] bytes) { return new String(Hex.encodeHex(bytes, false)); } /** * Return an hexadecimal representation of the bytes contained in the * given string, following the default, expected byte encoding. * * @param input The input string. */ public static String toHexString(String input) { try { byte[] bytes = input.getBytes(BYTE_ENCODING); return Torrent.byteArrayToHexString(bytes); } catch (UnsupportedEncodingException uee) { return null; } } /** * Determine how many threads to use for the piece hashing. * * <p> * If the environment variable TTORRENT_HASHING_THREADS is set to an * integer value greater than 0, its value will be used. Otherwise, it * defaults to the number of processors detected by the Java Runtime. * </p> * * @return How many threads to use for concurrent piece hashing. */ protected static int getHashingThreadsCount() { String threads = System.getenv("TTORRENT_HASHING_THREADS"); if (threads != null) { try { int count = Integer.parseInt(threads); if (count > 0) { return count; } } catch (NumberFormatException nfe) { // Pass } } return Runtime.getRuntime().availableProcessors(); } /** Torrent loading ---------------------------------------------------- */ /** * Load a torrent from the given torrent file. * * <p> * This method assumes we are not a seeder and that local data needs to be * validated. * </p> * * @param torrent The abstract {@link File} object representing the * <tt>.torrent</tt> file to load. * @throws IOException When the torrent file cannot be read. */ public static Torrent load(File torrent) throws IOException { return Torrent.load(torrent, false); } /** * Load a torrent from the given torrent file. * * @param torrent The abstract {@link File} object representing the * <tt>.torrent</tt> file to load. * @param seeder Whether we are a seeder for this torrent or not (disables * local data validation). * @throws IOException When the torrent file cannot be read. */ public static Torrent load(File torrent, boolean seeder) throws IOException { byte[] data = FileUtils.readFileToByteArray(torrent); return new Torrent(data, seeder); } /** Torrent creation --------------------------------------------------- */ /** * Create a {@link Torrent} object for a file. * * <p> * Hash the given file to create the {@link Torrent} object representing * the Torrent metainfo about this file, needed for announcing and/or * sharing said file. * </p> * * @param source The file to use in the torrent. * @param announce The announce URI that will be used for this torrent. * @param createdBy The creator's name, or any string identifying the * torrent's creator. */ public static Torrent create(File source, URI announce, String createdBy) throws InterruptedException, IOException { return Torrent.create(source, null, announce, null, createdBy); } /** * Create a {@link Torrent} object for a set of files. * * <p> * Hash the given files to create the multi-file {@link Torrent} object * representing the Torrent meta-info about them, needed for announcing * and/or sharing these files. Since we created the torrent, we're * considering we'll be a full initial seeder for it. * </p> * * @param parent The parent directory or location of the torrent files, * also used as the torrent's name. * @param files The files to add into this torrent. * @param announce The announce URI that will be used for this torrent. * @param createdBy The creator's name, or any string identifying the * torrent's creator. */ public static Torrent create(File parent, List<File> files, URI announce, String createdBy) throws InterruptedException, IOException { return Torrent.create(parent, files, announce, null, createdBy); } /** * Create a {@link Torrent} object for a file. * * <p> * Hash the given file to create the {@link Torrent} object representing * the Torrent metainfo about this file, needed for announcing and/or * sharing said file. * </p> * * @param source The file to use in the torrent. * @param announceList The announce URIs organized as tiers that will * be used for this torrent * @param createdBy The creator's name, or any string identifying the * torrent's creator. */ public static Torrent create(File source, List<List<URI>> announceList, String createdBy) throws InterruptedException, IOException { return Torrent.create(source, null, null, announceList, createdBy); } /** * Create a {@link Torrent} object for a set of files. * * <p> * Hash the given files to create the multi-file {@link Torrent} object * representing the Torrent meta-info about them, needed for announcing * and/or sharing these files. Since we created the torrent, we're * considering we'll be a full initial seeder for it. * </p> * * @param source The parent directory or location of the torrent files, * also used as the torrent's name. * @param files The files to add into this torrent. * @param announceList The announce URIs organized as tiers that will * be used for this torrent * @param createdBy The creator's name, or any string identifying the * torrent's creator. */ public static Torrent create(File source, List<File> files, List<List<URI>> announceList, String createdBy) throws InterruptedException, IOException { return Torrent.create(source, files, null, announceList, createdBy); } /** * Helper method to create a {@link Torrent} object for a set of files. * * <p> * Hash the given files to create the multi-file {@link Torrent} object * representing the Torrent meta-info about them, needed for announcing * and/or sharing these files. Since we created the torrent, we're * considering we'll be a full initial seeder for it. * </p> * * @param parent The parent directory or location of the torrent files, * also used as the torrent's name. * @param files The files to add into this torrent. * @param announce The announce URI that will be used for this torrent. * @param announceList The announce URIs organized as tiers that will * be used for this torrent * @param createdBy The creator's name, or any string identifying the * torrent's creator. */ private static Torrent create(File parent, List<File> files, URI announce, List<List<URI>> announceList, String createdBy) throws InterruptedException, IOException { MessageDigest md = DigestUtils.getSha1Digest(); if (files == null || files.isEmpty()) { logger.info("Creating single-file torrent for {}...", parent.getName()); } else { logger.info("Creating {}-file torrent {}...", files.size(), parent.getName()); } Map<String, BEValue> torrent = new HashMap<String, BEValue>(); if (announce != null) { torrent.put("announce", new BEValue(announce.toString())); } if (announceList != null) { List<BEValue> tiers = new LinkedList<BEValue>(); for (List<URI> trackers : announceList) { List<BEValue> tierInfo = new LinkedList<BEValue>(); for (URI trackerURI : trackers) { tierInfo.add(new BEValue(trackerURI.toString())); } tiers.add(new BEValue(tierInfo)); } torrent.put("announce-list", new BEValue(tiers)); } torrent.put("creation date", new BEValue(new Date().getTime() / 1000)); torrent.put("created by", new BEValue(createdBy)); Map<String, BEValue> info = new HashMap<String, BEValue>(); info.put("name", new BEValue(parent.getName())); info.put("piece length", new BEValue(PIECE_LENGTH)); long size = 0; int numFiles = 0; if (files == null || files.isEmpty()) { info.put("length", new BEValue(parent.length())); size = parent.length(); numFiles++; info.put("pieces", new BEValue(Torrent.hashFile(parent), BYTE_ENCODING)); } else { List<BEValue> fileInfo = new LinkedList<BEValue>(); List<File> updatedFilesList = new ArrayList<File>(); updateFileInfo(files, parent, parent, fileInfo, updatedFilesList); logger.info("Number of files in this multi-file torrent: " + updatedFilesList.size()); for (File file : updatedFilesList) size = size + file.length(); logger.info("Number of bytes in this multi-file torrent: " + size); numFiles = updatedFilesList.size(); info.put("files", new BEValue(fileInfo)); BEValue piecesValue = new BEValue(Torrent.hashFiles(updatedFilesList), BYTE_ENCODING); info.put("pieces", piecesValue); } TreeMap<String, BEValue> sortInfoMap = new TreeMap<String, BEValue>(); sortInfoMap.putAll(info); info = new HashMap<String, BEValue>(); info.putAll(sortInfoMap); sortInfoMap = null; torrent.put("info", new BEValue(info)); md.update(torrent.get("info").getMap().get("pieces").getBytes()); byte[] digest = md.digest(); byte[] transformedDigest = new byte[digest.length]; int i = 0; for (byte bt : digest) { short s = (short) (bt & 0xFF); if (s >= 127) { s = (short) (s - 127); if (s < 32) s = (short) (s + 32); } else if (s < 32) s = (short) (s + 32); transformedDigest[i++] = (byte) s; } for (byte b : transformedDigest) logger.debug(new Byte(b).toString()); logger.info("digest str " + new String(digest)); logger.info("transformed digest str " + new String(transformedDigest)); logger.info("Replacing special characters in the cloud key with regular characters"); String cloudKeyForFile = new String(transformedDigest, "UTF-8"); for (String nsChar : SPECIAL_TO_NSP_MAP.keySet()) { if (cloudKeyForFile.contains(nsChar)) cloudKeyForFile = cloudKeyForFile.replaceAll(java.util.regex.Pattern.quote(nsChar), SPECIAL_TO_NSP_MAP.get(nsChar)); } logger.info("Sanitized cloud Key: " + cloudKeyForFile); CloudUploadProgressListener listener = new CloudUploadProgressListener(size, numFiles); boolean success = false; try { success = CloudHelper.uploadTorrent(BUCKET_NAME, cloudKeyForFile.trim(), parent, listener); } catch (S3FetchException e) { // TODO Auto-generated catch block e.printStackTrace(); } if (!success) logger.info("File won't be uploaded to cloud as file is present in swarm and uploaded to cloud"); else logger.info("New file has been introduced in the swarm and has been uploaded to the cloud"); torrent.put(CLOUD_KEY, new BEValue(cloudKeyForFile)); TreeMap<String, BEValue> sortTorrentMap = new TreeMap<String, BEValue>(); sortTorrentMap.putAll(torrent); torrent = new HashMap<String, BEValue>(); torrent.putAll(sortTorrentMap); sortTorrentMap = null; ByteArrayOutputStream baos = new ByteArrayOutputStream(); BEncoder.bencode(new BEValue(torrent), baos); return new Torrent(baos.toByteArray(), true); } private static void updateFileInfo(List<File> filesList, File parent, File actualParent, List<BEValue> fileInfo, List<File> files) throws UnsupportedEncodingException { for (File file : filesList) { if (!file.isDirectory()) { Map<String, BEValue> fileMap = new HashMap<String, BEValue>(); fileMap.put("length", new BEValue(file.length())); LinkedList<BEValue> filePath = new LinkedList<BEValue>(); File actual = file; while (file != null) { if (file.equals(actualParent)) { break; } filePath.addFirst(new BEValue(file.getName())); file = file.getParentFile(); } fileMap.put("path", new BEValue(filePath)); fileInfo.add(new BEValue(fileMap)); logger.info("adding file: " + actual.getName() + " to the files list"); files.add(actual); } else { logger.info("Making recursive call"); updateFileInfo(Arrays.asList(file.listFiles(Constants.hiddenFilesFilter)), file, actualParent, fileInfo, files); } } } /** * A {@link Callable} to hash a data chunk. * * @author mpetazzoni */ private static class CallableChunkHasher implements Callable<String> { private final MessageDigest md; private final ByteBuffer data; CallableChunkHasher(ByteBuffer buffer) { this.md = DigestUtils.getSha1Digest(); this.data = ByteBuffer.allocate(buffer.remaining()); buffer.mark(); this.data.put(buffer); this.data.clear(); buffer.reset(); } @Override public String call() throws UnsupportedEncodingException { this.md.reset(); this.md.update(this.data.array()); return new String(md.digest(), BYTE_ENCODING); } } /** * Return the concatenation of the SHA-1 hashes of a file's pieces. * * <p> * Hashes the given file piece by piece using the default Torrent piece * length (see {@link #PIECE_LENGTH}) and returns the concatenation of * these hashes, as a string. * </p> * * <p> * This is used for creating Torrent meta-info structures from a file. * </p> * * @param file The file to hash. */ private static String hashFile(File file) throws InterruptedException, IOException { return Torrent.hashFiles(Arrays.asList(new File[] { file })); } private static String hashFiles(List<File> files) throws InterruptedException, IOException { int threads = getHashingThreadsCount(); ExecutorService executor = Executors.newFixedThreadPool(threads); ByteBuffer buffer = ByteBuffer.allocate(PIECE_LENGTH); List<Future<String>> results = new LinkedList<Future<String>>(); StringBuilder hashes = new StringBuilder(); long length = 0L; int pieces = 0; long start = System.nanoTime(); for (File file : files) { logger.info("Hashing data from {} with {} threads ({} pieces)...", new Object[] { file.getName(), threads, (int) (Math.ceil((double) file.length() / PIECE_LENGTH)) }); length += file.length(); FileInputStream fis = new FileInputStream(file); FileChannel channel = fis.getChannel(); int step = 10; try { while (channel.read(buffer) > 0) { if (buffer.remaining() == 0) { buffer.clear(); results.add(executor.submit(new CallableChunkHasher(buffer))); } if (results.size() >= threads) { pieces += accumulateHashes(hashes, results); } if (channel.position() / (double) channel.size() * 100f > step) { logger.info(" ... {}% complete", step); step += 10; } } } finally { channel.close(); fis.close(); } } // Hash the last bit, if any if (buffer.position() > 0) { buffer.limit(buffer.position()); buffer.position(0); results.add(executor.submit(new CallableChunkHasher(buffer))); } pieces += accumulateHashes(hashes, results); // Request orderly executor shutdown and wait for hashing tasks to // complete. executor.shutdown(); while (!executor.isTerminated()) { Thread.sleep(10); } long elapsed = System.nanoTime() - start; int expectedPieces = (int) (Math.ceil((double) length / PIECE_LENGTH)); logger.info("Hashed {} file(s) ({} bytes) in {} pieces ({} expected) in {}ms.", new Object[] { files.size(), length, pieces, expectedPieces, String.format("%.1f", elapsed / 1e6), }); return hashes.toString(); } /** * Accumulate the piece hashes into a given {@link StringBuilder}. * * @param hashes The {@link StringBuilder} to append hashes to. * @param results The list of {@link Future}s that will yield the piece * hashes. */ private static int accumulateHashes(StringBuilder hashes, List<Future<String>> results) throws InterruptedException, IOException { try { int pieces = results.size(); for (Future<String> chunk : results) { hashes.append(chunk.get()); } results.clear(); return pieces; } catch (ExecutionException ee) { throw new IOException("Error while hashing the torrent data!", ee); } } }