List of usage examples for com.amazonaws.services.glacier TreeHashGenerator calculateTreeHash
public static String calculateTreeHash(List<byte[]> checksums) throws AmazonClientException
From source file:baldrickv.s3streamingtool.S3StreamingTool.java
License:Open Source License
public static void main(String args[]) throws Exception { BasicParser p = new BasicParser(); Options o = getOptions();// w w w . ja v a 2 s . c o m CommandLine cl = p.parse(o, args); if (cl.hasOption('h')) { HelpFormatter hf = new HelpFormatter(); hf.setWidth(80); StringBuilder sb = new StringBuilder(); sb.append("\n"); sb.append("Upload:\n"); sb.append(" -u -r creds -s 50M -b my_bucket -f hda1.dump -t 10\n"); sb.append("Download:\n"); sb.append(" -d -r creds -s 50M -b my_bucket -f hda1.dump -t 10\n"); sb.append("Upload encrypted:\n"); sb.append(" -u -r creds -z -k secret_key -s 50M -b my_bucket -f hda1.dump -t 10\n"); sb.append("Download encrypted:\n"); sb.append(" -d -r creds -z -k secret_key -s 50M -b my_bucket -f hda1.dump -t 10\n"); sb.append("Cleanup in-progress multipart uploads\n"); sb.append(" -c -r creds -b my_bucket\n"); System.out.println(sb.toString()); hf.printHelp("See above", o); return; } int n = 0; if (cl.hasOption('d')) n++; if (cl.hasOption('u')) n++; if (cl.hasOption('c')) n++; if (cl.hasOption('m')) n++; if (n != 1) { System.err.println("Must specify at exactly one of -d, -u, -c or -m"); System.exit(-1); } if (cl.hasOption('m')) { //InputStream in = new java.io.BufferedInputStream(System.in,1024*1024*2); InputStream in = System.in; System.out.println(TreeHashGenerator.calculateTreeHash(in)); return; } require(cl, 'b'); if (cl.hasOption('d') || cl.hasOption('u')) { require(cl, 'f'); } if (cl.hasOption('z')) { require(cl, 'k'); } AWSCredentials creds = null; if (cl.hasOption('r')) { creds = Utils.loadAWSCredentails(cl.getOptionValue('r')); } else { if (cl.hasOption('i') && cl.hasOption('e')) { creds = new BasicAWSCredentials(cl.getOptionValue('i'), cl.getOptionValue('e')); } else { System.out.println("Must specify either credential file (-r) or AWS key ID and secret (-i and -e)"); System.exit(-1); } } S3StreamConfig config = new S3StreamConfig(); config.setEncryption(false); if (cl.hasOption('z')) { config.setEncryption(true); config.setSecretKey(Utils.loadSecretKey(cl.getOptionValue("k"))); } if (cl.hasOption("encryption-mode")) { config.setEncryptionMode(cl.getOptionValue("encryption-mode")); } config.setS3Bucket(cl.getOptionValue("bucket")); if (cl.hasOption("file")) { config.setS3File(cl.getOptionValue("file")); } if (cl.hasOption("threads")) { config.setIOThreads(Integer.parseInt(cl.getOptionValue("threads"))); } if (cl.hasOption("blocksize")) { String s = cl.getOptionValue("blocksize"); s = s.toUpperCase(); int multi = 1; int end = 0; while ((end < s.length()) && (s.charAt(end) >= '0') && (s.charAt(end) <= '9')) { end++; } int size = Integer.parseInt(s.substring(0, end)); if (end < s.length()) { String m = s.substring(end); if (m.equals("K")) multi = 1024; else if (m.equals("M")) multi = 1048576; else if (m.equals("G")) multi = 1024 * 1024 * 1024; else if (m.equals("KB")) multi = 1024; else if (m.equals("MB")) multi = 1048576; else if (m.equals("GB")) multi = 1024 * 1024 * 1024; else { System.out.println("Unknown suffix on block size. Only K,M and G understood."); System.exit(-1); } } size *= multi; config.setBlockSize(size); } Logger.getLogger("").setLevel(Level.FINE); S3StreamingDownload.log.setLevel(Level.FINE); S3StreamingUpload.log.setLevel(Level.FINE); config.setS3Client(new AmazonS3Client(creds)); config.setGlacierClient(new AmazonGlacierClient(creds)); config.getGlacierClient().setEndpoint("glacier.us-west-2.amazonaws.com"); if (cl.hasOption("glacier")) { config.setGlacier(true); config.setStorageInterface(new StorageGlacier(config.getGlacierClient())); } else { config.setStorageInterface(new StorageS3(config.getS3Client())); } if (cl.hasOption("bwlimit")) { config.setMaxBytesPerSecond(Double.parseDouble(cl.getOptionValue("bwlimit"))); } if (cl.hasOption('c')) { if (config.getGlacier()) { GlacierCleanupMultipart.cleanup(config); } else { S3CleanupMultipart.cleanup(config); } return; } if (cl.hasOption('d')) { config.setOutputStream(System.out); S3StreamingDownload.download(config); return; } if (cl.hasOption('u')) { config.setInputStream(System.in); S3StreamingUpload.upload(config); return; } }
From source file:baldrickv.s3streamingtool.S3StreamingUploadThread.java
License:Open Source License
public void run() { while (true) { try {// www .j a v a 2s. c o m DataBlock db = queue.take(); int block_no = db.getBlockNumber(); String hash = TreeHashGenerator.calculateTreeHash(new ByteArrayInputStream(db.getData())); String part = config.getStorageInterface().uploadPart(config.getS3Bucket(), config.getS3File(), upload_id, hash, config.getBlockSize(), block_no, db.getData(), config.getMaxBytesPerSecond()); weaver.addBlock(block_no, part); read_allow.release(); } catch (InterruptedException e) { return; } } }
From source file:com.brianmcmichael.sagu.SAGU.java
License:Open Source License
@Override public void actionPerformed(ActionEvent e) { String accessString = getAccessKey(); String secretString = getSecretKey(); String vaultString = getVaultName(); int regionInt = getServerRegion(); if (e.getSource() == newVaultButton && checkAWSFields()) { AmazonGlacierClient newVaultClient = makeClient(accessString, secretString, regionInt); AddVaultFrame avf = new AddVaultFrame(newVaultClient, regionInt); avf.setVisible(true);//w ww .j av a 2 s . c om } if (e.getSource() == vaultSelector) { if (vaultSelector.getSelectedItem() != null) { if (vaultSelector.getSelectedIndex() == 0) { vaultField.setText(""); } else { vaultField.setText(vaultSelector.getSelectedItem().toString()); } } } if (e.getSource() == loginButton) { repopulateVaults(accessString, secretString); } if (e.getSource() == exitApplicationMnu) { System.exit(0); } if (e.getSource() == updateMnu || e.getSource() == checkUpdateButton) { JHyperlinkLabel.OpenURI(URL_STRING); } if (e.getSource() == saveFileMnu) { FileDialog fd = new FileDialog(new Frame(), "Save...", FileDialog.SAVE); fd.setFile("Glacier.txt"); fd.setDirectory(appProperties.getDir()); fd.setLocation(50, 50); fd.setVisible(true); String filePath = "" + fd.getDirectory() + System.getProperty("file.separator") + fd.getFile(); File outFile = new File(filePath); if (!outFile.equals("") && !outFile.equals("null")) { try { FileReader fr = new FileReader(getLogFile(0, appProperties)); BufferedReader br = new BufferedReader(fr); FileWriter saveFile = new FileWriter(outFile.toString()); int count = 0; boolean moreLines = true; String ln1; String ln2; String ln3; while (moreLines) { ln1 = br.readLine(); ln2 = br.readLine(); ln3 = br.readLine(); if (ln1 == null) { ln1 = ""; } if (ln2 == null) { ln2 = ""; } if (ln3 == null) { ln3 = ""; } saveFile.write(ln1); saveFile.write("\r\n"); saveFile.write(ln2); saveFile.write("\r\n"); saveFile.write(ln3); saveFile.write("\r\n"); count++; if (ln3.equals("")) { moreLines = false; br.close(); saveFile.close(); JOptionPane.showMessageDialog(null, "Successfully exported " + count + " archive records to " + outFile.toString(), "Export", JOptionPane.INFORMATION_MESSAGE); } } } catch (FileNotFoundException e1) { JOptionPane.showMessageDialog(null, "Unable to locate Glacier.log", "Error", JOptionPane.ERROR_MESSAGE); e1.printStackTrace(); } catch (IOException e1) { e1.printStackTrace(); } } } if (e.getSource() == viewLog || e.getSource() == logButton) { File f = getLogFile(logTypes.getSelectedIndex(), appProperties); if (f.exists()) { JHyperlinkLabel.OpenURI("" + f.toURI()); } else { JOptionPane.showMessageDialog(null, "Log file " + f.getName() + " does not exist.", "Error", JOptionPane.ERROR_MESSAGE); } } if (e.getSource() == deleteArchiveMnu) { if (checkAllFields()) { AmazonGlacierClient newDeleteClient = makeClient(accessString, secretString, regionInt); DeleteArchiveFrame daf = new DeleteArchiveFrame(newDeleteClient, vaultString, regionInt); daf.setVisible(true); } } if (e.getSource() == inventoryRequestButton) { if (checkAllFields()) { AmazonGlacierClient newInventoryClient = makeClient(accessString, secretString, regionInt); InventoryRequest ir = new InventoryRequest(newInventoryClient, vaultString, regionInt); ir.setVisible(true); } } if (e.getSource() == downloadRequestButton || e.getSource() == downloadFileMnu) { if (checkAllFields()) { AmazonGlacierClient newDownloadClient = makeClient(accessString, secretString, regionInt); BasicAWSCredentials credentials = new BasicAWSCredentials(accessString, secretString); AmazonDownloadRequest adr = new AmazonDownloadRequest(newDownloadClient, vaultString, regionInt, credentials); adr.setVisible(true); } } if (e.getSource() == aboutMnu) { JOptionPane.showMessageDialog(null, format(ABOUT_PATTERN, versionNumber), "About", JOptionPane.INFORMATION_MESSAGE); } if (e.getSource() == clearButton) { ddText.setText(""); uploadButton.setText("Select Files"); multiFiles = null; } if (e.getSource() == locationChoice) { repopulateVaults(accessString, secretString); } if (e.getSource() == selectFileButton) { int returnVal = fc.showOpenDialog(SAGU.this); if (returnVal == JFileChooser.APPROVE_OPTION) { if (fc.getSelectedFile().isFile()) { File[] thisFile = new File[1]; thisFile[0] = fc.getSelectedFile(); try { ddText.append(thisFile[0].getCanonicalPath() + "\n"); } catch (java.io.IOException f) { } if (multiFiles != null) { multiFiles = SAGUUtils.concatFileArrays(multiFiles, thisFile); } else { multiFiles = thisFile; } } else { JOptionPane.showMessageDialog(null, NO_DIRECTORIES_ERROR, "Error", JOptionPane.ERROR_MESSAGE); } } } if (e.getSource() == uploadButton) { if ((checkAllFields()) && (checkForFile())) { SwingWorker<Object, Void> uploadWorker = new SwingWorker<Object, Void>() { @Override protected Object doInBackground() throws Exception { String accessString = getAccessKey(); String secretString = getSecretKey(); String vaultName = getVaultName(); File[] uploadFileBatch = multiFiles; // work out exactly how much we are going to upload // so we can support a second total upload progress bar long totalSize = 0; long uploadedSize = 0; for (File f : uploadFileBatch) { totalSize += f.length(); } int locInt = getServerRegion(); multiFiles = null; clearFile(); UploadWindow uw = new UploadWindow(); if (uploadFileBatch.length > 0) { ArrayList<String> uploadList = new ArrayList<String>(); for (int i = 0; i < uploadFileBatch.length; i++) { try { Thread.sleep(100L); // why? } catch (InterruptedException e1) { e1.printStackTrace(); } ClientConfiguration config = new ClientConfiguration(); config.setSocketTimeout(SOCKET_TIMEOUT); config.setMaxErrorRetry(MAX_RETRIES); BasicAWSCredentials credentials = new BasicAWSCredentials(accessString, secretString); client = new AmazonGlacierClient(credentials, config); final Endpoint endpoint = Endpoint.getByIndex(locInt); client.setEndpoint(endpoint.getGlacierEndpoint()); String locationUpped = endpoint.name(); String thisFile = uploadFileBatch[i].getCanonicalPath(); final String description = SAGUUtils.pathToDescription(thisFile); try { ArchiveTransferManager atm = new ArchiveTransferManager(client, credentials); String fileLength = Long.toString(uploadFileBatch[i].length()); uw.setTitle("(" + (i + 1) + "/" + uploadFileBatch.length + ")" + " Uploading: " + thisFile); UploadResult result = atm.upload(null, vaultName, description, uploadFileBatch[i], new OneFileProgressListener(uw, uploadFileBatch[i].length())); uw.addToFinishedFiles(thisFile + "\n"); uploadedSize += uploadFileBatch[i].length(); int percentage = (int) (((double) uploadedSize / totalSize) * 100); uw.updateAllFilesProgress(percentage); final LogWriter logWriter; // write to file if (logCheckMenuItem.isSelected()) { String treeHash = TreeHashGenerator.calculateTreeHash(uploadFileBatch[i]); try { logWriter = new LogWriter(appProperties); try { String thisResult = result.getArchiveId(); logWriter.logUploadedFile(vaultName, locationUpped, thisFile, fileLength, treeHash, thisResult); uploadList.add("Successfully uploaded " + thisFile + " to vault " + vaultName + " at " + locationUpped + ". Bytes: " + fileLength + ". ArchiveID Logged.\n"); } catch (IOException c) { JOptionPane.showMessageDialog(null, LOG_WRITE_ERROR, "IO Error", JOptionPane.ERROR_MESSAGE); uw.dispose(); System.exit(1); } } catch (IOException ex) { JOptionPane.showMessageDialog(null, LOG_CREATION_ERROR, "IO Error", JOptionPane.ERROR_MESSAGE); uw.dispose(); System.exit(1); } } else { JOptionPane.showMessageDialog(null, "Upload Complete!\nArchive ID: " + result.getArchiveId() + "\nIt may take some time for Amazon to update the inventory.", "Uploaded", JOptionPane.INFORMATION_MESSAGE); multiFiles = null; uw.dispose(); } clearFile(); } catch (Exception h) { if (logCheckMenuItem.isSelected()) { writeToErrorLog(h, thisFile); } JOptionPane.showMessageDialog(null, "" + h, "Error", JOptionPane.ERROR_MESSAGE); uw.dispose(); } } StringBuilder sb = new StringBuilder(); for (int j = 0; j < uploadFileBatch.length; j++) { sb.append(uploadList.get(j)); } uw.dispose(); // Move the actual results string to a JTextArea JTextArea uploadCompleteMsg = new JTextArea("Upload Complete! \n" + sb); uploadCompleteMsg.setLineWrap(true); uploadCompleteMsg.setWrapStyleWord(true); uploadCompleteMsg.setEditable(false); // Put the JTextArea in a JScollPane and present that in the JOptionPane JScrollPane uploadCompleteScroll = new JScrollPane(uploadCompleteMsg); uploadCompleteScroll.setPreferredSize(new Dimension(500, 400)); JOptionPane.showMessageDialog(null, uploadCompleteScroll, "Uploaded", JOptionPane.INFORMATION_MESSAGE); // Close the JProgressBar multiFiles = null; clearFile(); } else { JOptionPane.showMessageDialog(null, "This wasn't supposed to happen.", "Bug!", JOptionPane.ERROR_MESSAGE); uw.dispose(); } return null; } private void writeToErrorLog(Exception h, String thisFile) { String thisError = h.toString(); Writer errorOutputLog = null; try { errorOutputLog = new BufferedWriter(new FileWriter(getLogFile(4, appProperties), true)); } catch (Exception badLogCreate) { JOptionPane.showMessageDialog(null, LOG_CREATION_ERROR, "IO Error", JOptionPane.ERROR_MESSAGE); System.exit(1); } try { Date d = new Date(); errorOutputLog.write(System.getProperty("line.separator")); errorOutputLog.write("" + d.toString() + ": \"" + thisFile + "\" *ERROR* " + thisError); errorOutputLog.write(System.getProperty("line.separator")); } catch (Exception badLogWrite) { JOptionPane.showMessageDialog(null, LOG_WRITE_ERROR, "IO Error", JOptionPane.ERROR_MESSAGE); System.exit(1); } } }; uploadWorker.execute(); } } }
From source file:com.brianmcmichael.sagu.SimpleGlacierUploader.java
License:Open Source License
@Override public void actionPerformed(ActionEvent e) { String accessString = getAccessField(); String secretString = getSecretField(); String vaultString = getVaultField(); int regionInt = getServerRegion(); if (e.getSource() == newVaultButton && checkAWSFields()) { AmazonGlacierClient newVaultClient = makeClient(accessString, secretString, regionInt); AddVaultFrame avf = new AddVaultFrame(newVaultClient, regionInt); avf.setVisible(true);// w ww . ja v a2 s . c om } if (e.getSource() == vaultSelector) { if (vaultSelector.getSelectedItem() != null) { if (vaultSelector.getSelectedIndex() == 0) { vaultField.setText(""); } else { vaultField.setText(vaultSelector.getSelectedItem().toString()); } } } if (e.getSource() == loginButton) { repopulateVaults(accessString, secretString); } if (e.getSource() == exitApplicationMnu) { System.exit(0); } if (e.getSource() == updateMnu || e.getSource() == checkUpdateButton) { JHyperlinkLabel.OpenURI(URL_STRING); } if (e.getSource() == saveFileMnu) { FileDialog fd = new FileDialog(new Frame(), "Save...", FileDialog.SAVE); fd.setFile("Glacier.txt"); fd.setDirectory(curDir); fd.setLocation(50, 50); fd.setVisible(true); String filePath = "" + fd.getDirectory() + System.getProperty("file.separator") + fd.getFile(); File outFile = new File(filePath); if (!outFile.equals("") && !outFile.equals("null")) { try { FileReader fr = new FileReader(getLogFilenamePath(0)); BufferedReader br = new BufferedReader(fr); FileWriter saveFile = new FileWriter(outFile.toString()); int count = 0; boolean moreLines = true; String ln1; String ln2; String ln3; while (moreLines) { ln1 = br.readLine(); ln2 = br.readLine(); ln3 = br.readLine(); if (ln1 == null) { ln1 = ""; } if (ln2 == null) { ln2 = ""; } if (ln3 == null) { ln3 = ""; } saveFile.write(ln1); saveFile.write("\r\n"); saveFile.write(ln2); saveFile.write("\r\n"); saveFile.write(ln3); saveFile.write("\r\n"); count++; if (ln3.equals("")) { moreLines = false; br.close(); saveFile.close(); JOptionPane.showMessageDialog(null, "Successfully exported " + count + " archive records to " + outFile.toString(), "Export", JOptionPane.INFORMATION_MESSAGE); } } } catch (FileNotFoundException e1) { JOptionPane.showMessageDialog(null, "Unable to locate Glacier.log", "Error", JOptionPane.ERROR_MESSAGE); e1.printStackTrace(); } catch (IOException e1) { e1.printStackTrace(); } } } if (e.getSource() == viewLog || e.getSource() == logButton) { File f = SimpleGlacierUploader.getLogFilenamePath(getLogFileType()); if (f.exists()) { JHyperlinkLabel.OpenURI("" + f.toURI()); } else { JOptionPane.showMessageDialog(null, "Log file " + f.getName() + " does not exist.", "Error", JOptionPane.ERROR_MESSAGE); } } if (e.getSource() == deleteArchiveMnu) { if (checkAllFields()) { AmazonGlacierClient newDeleteClient = makeClient(accessString, secretString, regionInt); DeleteArchiveFrame daf = new DeleteArchiveFrame(newDeleteClient, vaultString, regionInt); daf.setVisible(true); } } if (e.getSource() == inventoryRequestButton) { if (checkAllFields()) { AmazonGlacierClient newInventoryClient = makeClient(accessString, secretString, regionInt); InventoryRequest ir = new InventoryRequest(newInventoryClient, vaultString, regionInt); ir.setVisible(true); } } if (e.getSource() == downloadRequestButton || e.getSource() == downloadFileMnu) { if (checkAllFields()) { AmazonGlacierClient newDownloadClient = makeClient(accessString, secretString, regionInt); BasicAWSCredentials credentials = new BasicAWSCredentials(accessString, secretString); AmazonDownloadRequest adr = new AmazonDownloadRequest(newDownloadClient, vaultString, regionInt, credentials); adr.setVisible(true); } } if (e.getSource() == aboutMnu) { JOptionPane.showMessageDialog(null, ABOUT_WINDOW_STRING, "About", JOptionPane.INFORMATION_MESSAGE); } if (e.getSource() == clearButton) { ddText.setText(""); uploadButton.setText("Select Files"); multiFiles = null; } if (e.getSource() == locationChoice) { repopulateVaults(accessString, secretString); } if (e.getSource() == selectFileButton) { int returnVal = fc.showOpenDialog(SimpleGlacierUploader.this); if (returnVal == JFileChooser.APPROVE_OPTION) { if (fc.getSelectedFile().isFile()) { File[] thisFile = new File[1]; thisFile[0] = fc.getSelectedFile(); try { ddText.append(thisFile[0].getCanonicalPath() + "\n"); } catch (java.io.IOException f) { } if (multiFiles != null) { multiFiles = concatFileArray(multiFiles, thisFile); } else { multiFiles = thisFile; } } else { JOptionPane.showMessageDialog(null, NO_DIRECTORIES_ERROR, "Error", JOptionPane.ERROR_MESSAGE); } } } if (e.getSource() == uploadButton) { if ((checkAllFields()) && (checkForFile())) { SaveCurrentProperties(accessString, secretString, vaultString, locationChoice.getSelectedIndex()); SwingWorker uploadWorker = new SwingWorker() { @Override protected Object doInBackground() throws Exception { String accessString = getAccessField(); String secretString = getSecretField(); String vaultName = getVaultField(); File[] uploadFileBatch = multiFiles; // work out exactly how much we are going to upload // so we can support a second total upload progress bar long totalSize = 0; long uploadedSize = 0; for (File f : uploadFileBatch) { totalSize += f.length(); } int locInt = locationChoice.getSelectedIndex(); multiFiles = null; clearFile(); UploadWindow uw = new UploadWindow(); if (uploadFileBatch.length > 0) { ArrayList<String> uploadList = new ArrayList<String>(); for (int i = 0; i < uploadFileBatch.length; i++) { // Save Current Settings to properties try { Thread.sleep(100L); } catch (InterruptedException e1) { e1.printStackTrace(); } ClientConfiguration config = new ClientConfiguration(); config.setSocketTimeout(SOCKET_TIMEOUT); config.setMaxErrorRetry(MAX_RETRIES); BasicAWSCredentials credentials = new BasicAWSCredentials(accessString, secretString); client = new AmazonGlacierClient(credentials, config); final Endpoint endpoint = Endpoint.getByIndex(locInt); client.setEndpoint(endpoint.getGlacierEndpoint()); String locationUpped = endpoint.name(); String thisFile = uploadFileBatch[i].getCanonicalPath(); String cleanFile = regexClean(thisFile); try { ArchiveTransferManager atm = new ArchiveTransferManager(client, credentials); String fileLength = Long.toString(uploadFileBatch[i].length()); uw.setTitle("(" + (i + 1) + "/" + uploadFileBatch.length + ")" + " Uploading: " + thisFile); UploadResult result = atm.upload(vaultName, cleanFile, uploadFileBatch[i]); uw.addToLog("Done: " + thisFile + "\n"); uploadedSize += uploadFileBatch[i].length(); int percentage = (int) (((double) uploadedSize / totalSize) * 100); uw.updateProgress(percentage); Writer plainOutputLog = null; Writer plainOutputTxt = null; Writer plainOutputCsv = null; Writer plainOutputYaml = null; // write to file if (logCheckMenuItem.isSelected()) { String treeHash = TreeHashGenerator.calculateTreeHash(uploadFileBatch[i]); try { plainOutputLog = new BufferedWriter( new FileWriter(getLogFilenamePath(0), true)); plainOutputTxt = new BufferedWriter( new FileWriter(getLogFilenamePath(1), true)); plainOutputCsv = new BufferedWriter( new FileWriter(getLogFilenamePath(2), true)); plainOutputYaml = new BufferedWriter( new FileWriter(getLogFilenamePath(3), true)); } catch (IOException ex) { JOptionPane.showMessageDialog(null, LOG_CREATION_ERROR, "IO Error", JOptionPane.ERROR_MESSAGE); uw.dispose(); System.exit(1); } try { Date d = new Date(); String thisResult = result.getArchiveId(); plainOutputLog.write(System.getProperty("line.separator")); plainOutputLog.write(" | ArchiveID: " + thisResult + " "); plainOutputLog.write(System.getProperty("line.separator")); plainOutputLog.write(" | File: " + thisFile + " "); plainOutputLog.write(" | Bytes: " + fileLength + " "); plainOutputLog.write(" | Vault: " + vaultName + " "); plainOutputLog.write(" | Location: " + locationUpped + " "); plainOutputLog.write(" | Date: " + d.toString() + " "); plainOutputLog.write(" | Hash: " + treeHash + " "); plainOutputLog.write(System.getProperty("line.separator")); plainOutputLog.close(); plainOutputTxt.write(System.getProperty("line.separator")); plainOutputTxt.write(" | ArchiveID: " + thisResult + " "); plainOutputTxt.write(System.getProperty("line.separator")); plainOutputTxt.write(" | File: " + thisFile + " "); plainOutputTxt.write(" | Bytes: " + fileLength + " "); plainOutputTxt.write(" | Vault: " + vaultName + " "); plainOutputTxt.write(" | Location: " + locationUpped + " "); plainOutputTxt.write(" | Date: " + d.toString() + " "); plainOutputTxt.write(" | Hash: " + treeHash + " "); plainOutputTxt.write(System.getProperty("line.separator")); plainOutputTxt.close(); plainOutputCsv.write("" + thisResult + ","); plainOutputCsv.write("" + thisFile + ","); plainOutputCsv.write("" + fileLength + ","); plainOutputCsv.write("" + vaultName + ","); plainOutputCsv.write("" + locationUpped + ","); plainOutputCsv.write("" + d.toString() + ","); plainOutputCsv.write("" + treeHash + ","); plainOutputCsv.write(System.getProperty("line.separator")); plainOutputCsv.close(); plainOutputYaml.write(System.getProperty("line.separator")); plainOutputYaml.write("- ArchiveID: \"" + thisResult + "\"" + System.getProperty("line.separator")); plainOutputYaml.write(" File: \"" + thisFile + "\"" + System.getProperty("line.separator")); plainOutputYaml.write(" Bytes: \"" + fileLength + "\"" + System.getProperty("line.separator")); plainOutputYaml.write(" Vault: \"" + vaultName + "\"" + System.getProperty("line.separator")); plainOutputYaml.write(" Location: \"" + locationUpped + "\"" + System.getProperty("line.separator")); plainOutputYaml.write(" Date: \"" + d.toString() + "\"" + System.getProperty("line.separator")); plainOutputYaml.write(" Hash: \"" + treeHash + "\"" + System.getProperty("line.separator")); plainOutputYaml.close(); uploadList.add("Successfully uploaded " + thisFile + " to vault " + vaultName + " at " + locationUpped + ". Bytes: " + fileLength + ". ArchiveID Logged.\n"); } catch (IOException c) { JOptionPane.showMessageDialog(null, LOG_WRITE_ERROR, "IO Error", JOptionPane.ERROR_MESSAGE); uw.dispose(); System.exit(1); } } else { JOptionPane.showMessageDialog(null, "Upload Complete!\nArchive ID: " + result.getArchiveId() + "\nIt may take some time for Amazon to update the inventory.", "Uploaded", JOptionPane.INFORMATION_MESSAGE); multiFiles = null; uw.dispose(); } clearFile(); } catch (Exception h) { if (logCheckMenuItem.isSelected()) { writeToErrorLog(h, thisFile); } JOptionPane.showMessageDialog(null, "" + h, "Error", JOptionPane.ERROR_MESSAGE); uw.dispose(); } } StringBuilder sb = new StringBuilder(); for (int j = 0; j < uploadFileBatch.length; j++) { sb.append(uploadList.get(j)); } uw.dispose(); // Move the actual results string to a JTextArea JTextArea uploadCompleteMsg = new JTextArea("Upload Complete! \n" + sb); uploadCompleteMsg.setLineWrap(true); uploadCompleteMsg.setWrapStyleWord(true); uploadCompleteMsg.setEditable(false); // Put the JTextArea in a JScollPane and present that in the JOptionPane JScrollPane uploadCompleteScroll = new JScrollPane(uploadCompleteMsg); uploadCompleteScroll.setPreferredSize(new Dimension(500, 400)); JOptionPane.showMessageDialog(null, uploadCompleteScroll, "Uploaded", JOptionPane.INFORMATION_MESSAGE); // Close the JProgressBar multiFiles = null; clearFile(); } else { JOptionPane.showMessageDialog(null, "This wasn't supposed to happen.", "Bug!", JOptionPane.ERROR_MESSAGE); uw.dispose(); } return null; } private void writeToErrorLog(Exception h, String thisFile) { String thisError = h.toString(); Writer errorOutputLog = null; try { errorOutputLog = new BufferedWriter(new FileWriter(getLogFilenamePath(4), true)); } catch (Exception badLogCreate) { JOptionPane.showMessageDialog(null, LOG_CREATION_ERROR, "IO Error", JOptionPane.ERROR_MESSAGE); System.exit(1); } try { Date d = new Date(); errorOutputLog.write(System.getProperty("line.separator")); errorOutputLog.write("" + d.toString() + ": \"" + thisFile + "\" *ERROR* " + thisError); errorOutputLog.write(System.getProperty("line.separator")); } catch (Exception badLogWrite) { JOptionPane.showMessageDialog(null, LOG_WRITE_ERROR, "IO Error", JOptionPane.ERROR_MESSAGE); System.exit(1); } } }; uploadWorker.execute(); } } }
From source file:com.brianmcmichael.SimpleGlacierUploader.SimpleGlacierUploader.java
License:Open Source License
@Override public void actionPerformed(ActionEvent e) { String accessString = getAccessField(); String secretString = getSecretField(); String vaultString = getVaultField(); int regionInt = getServerRegion(); if (e.getSource() == newVaultButton && checkAWSFields()) { AmazonGlacierClient newVaultClient = new AmazonGlacierClient(); newVaultClient = makeClient(accessString, secretString, regionInt); AddVaultFrame avf = new AddVaultFrame(newVaultClient, regionInt); avf.setVisible(true);//from w w w . ja va2 s . com } if (e.getSource() == vaultSelector) { if (vaultSelector.getSelectedItem() != null) { if (vaultSelector.getSelectedIndex() == 0) { vaultField.setText(""); } else { vaultField.setText(vaultSelector.getSelectedItem().toString()); } } else { } } if (e.getSource() == loginButton) { repopulateVaults(accessString, secretString, regionInt); } if (e.getSource() == exitApplicationMnu) { System.exit(0); } if (e.getSource() == updateMnu || e.getSource() == checkUpdateButton) { JHyperlinkLabel.OpenURI(URL_STRING); } if (e.getSource() == saveFileMnu) { FileDialog fd = new FileDialog(new Frame(), "Save...", FileDialog.SAVE); fd.setFile("Glacier.txt"); fd.setDirectory(curDir); fd.setLocation(50, 50); fd.setVisible(true); // fd.show(); String filePath = "" + fd.getDirectory() + System.getProperty("file.separator") + fd.getFile(); File outFile = new File(filePath); if ((outFile.equals("") == false) && (outFile.equals("null") == false) && ((outFile == null) == false)) { try { FileReader fr = new FileReader(getLogFilenamePath(0)); BufferedReader br = new BufferedReader(fr); FileWriter saveFile = new FileWriter(outFile.toString()); int count = 0; boolean moreLines = true; String ln1 = ""; String ln2 = ""; String ln3 = ""; while (moreLines == true) { ln1 = br.readLine(); ln2 = br.readLine(); ln3 = br.readLine(); if (ln1 == null) { ln1 = ""; } if (ln2 == null) { ln2 = ""; } if (ln3 == null) { ln3 = ""; } saveFile.write(ln1); saveFile.write("\r\n"); saveFile.write(ln2); saveFile.write("\r\n"); saveFile.write(ln3); saveFile.write("\r\n"); count++; if (ln3.equals("")) { moreLines = false; br.close(); saveFile.close(); JOptionPane.showMessageDialog(null, "Successfully exported " + count + " archive records to " + outFile.toString(), "Export", JOptionPane.INFORMATION_MESSAGE); } } } catch (FileNotFoundException e1) { JOptionPane.showMessageDialog(null, "Unable to locate Glacier.log", "Error", JOptionPane.ERROR_MESSAGE); e1.printStackTrace(); } catch (IOException e1) { e1.printStackTrace(); } } } if (e.getSource() == viewLog || e.getSource() == logButton) { JHyperlinkLabel.OpenURI("" + SimpleGlacierUploader.getLogFilenamePath(getLogFileType()).toURI()); } if (e.getSource() == deleteArchiveMnu) { if (checkAllFields()) { AmazonGlacierClient newDeleteClient = new AmazonGlacierClient(); newDeleteClient = makeClient(accessString, secretString, regionInt); DeleteArchiveFrame daf = new DeleteArchiveFrame(newDeleteClient, vaultString, regionInt); daf.setVisible(true); } } if (e.getSource() == inventoryRequestButton) { if (checkAllFields()) { AmazonGlacierClient newInventoryClient = new AmazonGlacierClient(); newInventoryClient = makeClient(accessString, secretString, regionInt); InventoryRequest ir = new InventoryRequest(newInventoryClient, vaultString, regionInt); ir.setVisible(true); } } if (e.getSource() == downloadRequestButton || e.getSource() == downloadFileMnu) { if (checkAllFields()) { AmazonGlacierClient newDownloadClient = new AmazonGlacierClient(); newDownloadClient = makeClient(accessString, secretString, regionInt); BasicAWSCredentials credentials = new BasicAWSCredentials(accessString, secretString); AmazonDownloadRequest adr = new AmazonDownloadRequest(newDownloadClient, vaultString, regionInt, credentials); adr.setVisible(true); } } if (e.getSource() == aboutMnu) { JOptionPane.showMessageDialog(null, ABOUT_WINDOW_STRING, "About", JOptionPane.INFORMATION_MESSAGE); } if (e.getSource() == clearButton) { ddText.setText(""); uploadButton.setText("Select Files"); multiFiles = null; } if (e.getSource() == locationChoice) { repopulateVaults(accessString, secretString, regionInt); } if (e.getSource() == selectFileButton) { int returnVal = fc.showOpenDialog(SimpleGlacierUploader.this); if (returnVal == JFileChooser.APPROVE_OPTION) { if (fc.getSelectedFile().isFile()) { /* * Removed for v. 0.72 if (fc.getSelectedFile().length() > * max_file_size == true) { * JOptionPane.showMessageDialog(null, * FILE_TOO_BIG_ERROR,"Error",JOptionPane.ERROR_MESSAGE); * try { //ddText.setForeground(rc); ddText.append( * "Unable to upload: " + * fc.getSelectedFile().getCanonicalPath() + "\n" ); } // * end try catch( java.io.IOException g ) {} } else { */ File[] thisFile = new File[1]; thisFile[0] = fc.getSelectedFile(); try { ddText.append(thisFile[0].getCanonicalPath() + "\n"); } // end try catch (java.io.IOException f) { } if (multiFiles != null) { multiFiles = concatFileArray(multiFiles, thisFile); } else { multiFiles = thisFile; } // } } else { JOptionPane.showMessageDialog(null, NO_DIRECTORIES_ERROR, "Error", JOptionPane.ERROR_MESSAGE); } } else { } } if (e.getSource() == uploadButton) { if ((checkAllFields()) && (checkForFile())) { SaveCurrentProperties(accessString, secretString, vaultString, locationChoice.getSelectedIndex()); SwingWorker uploadWorker = new SwingWorker() { @Override protected Object doInBackground() throws Exception { String accessString = getAccessField(); String secretString = getSecretField(); // String vaultString = getVaultField(); String vaultName = getVaultField(); File[] uploadFileBatch = multiFiles; // work out exactly how much we are going to upload // so we can support a second total upload progress bar long totalSize = 0; long uploadedSize = 0; for (File f : uploadFileBatch) { totalSize += f.length(); } int locInt = locationChoice.getSelectedIndex(); multiFiles = null; clearFile(); UploadWindow uw = new UploadWindow(); if (uploadFileBatch.length > 0) { ArrayList<String> uploadList = new ArrayList<String>(); for (int i = 0; i < uploadFileBatch.length; i++) { // Save Current Settings to properties try { Thread.sleep(100L); } catch (InterruptedException e1) { e1.printStackTrace(); } ClientConfiguration config = new ClientConfiguration(); config.setSocketTimeout(SOCKET_TIMEOUT); config.setMaxErrorRetry(MAX_RETRIES); BasicAWSCredentials credentials = new BasicAWSCredentials(accessString, secretString); client = new AmazonGlacierClient(credentials, config); Endpoints ep = new Endpoints(locInt); // String endpointUrl = ep.Endpoint(locInt); client.setEndpoint(ep.Endpoint()); String locationUpped = ep.Location(); String thisFile = uploadFileBatch[i].getCanonicalPath(); String cleanFile = regexClean(thisFile); // char emptyChar = 0xFFFA; // String thisCleanFile = // thisFile.valueOf(emptyChar).replaceAll("\\p{C}", // "?"); try { ArchiveTransferManager atm = new ArchiveTransferManager(client, credentials); String fileLength = Long.toString(uploadFileBatch[i].length()); uw.setTitle("(" + (i + 1) + "/" + uploadFileBatch.length + ")" + " Uploading: " + thisFile); UploadResult result = atm.upload(vaultName, cleanFile, uploadFileBatch[i]); uw.addToLog("Done: " + thisFile + "\n"); uploadedSize += uploadFileBatch[i].length(); int percentage = (int) (((double) uploadedSize / totalSize) * 100); uw.updateProgress(percentage); Writer plainOutputLog = null; Writer plainOutputTxt = null; Writer plainOutputCsv = null; // write to file if (logCheckMenuItem.isSelected()) { String treeHash = TreeHashGenerator.calculateTreeHash(uploadFileBatch[i]); try { plainOutputLog = new BufferedWriter( new FileWriter(getLogFilenamePath(0), true)); plainOutputTxt = new BufferedWriter( new FileWriter(getLogFilenamePath(1), true)); plainOutputCsv = new BufferedWriter( new FileWriter(getLogFilenamePath(2), true)); } catch (IOException ex) { JOptionPane.showMessageDialog(null, LOG_CREATION_ERROR, "IO Error", JOptionPane.ERROR_MESSAGE); uw.dispose(); System.exit(1); } try { Date d = new Date(); String thisResult = result.getArchiveId(); plainOutputLog.write(System.getProperty("line.separator")); plainOutputLog.write(" | ArchiveID: " + thisResult + " "); plainOutputLog.write(System.getProperty("line.separator")); plainOutputLog.write(" | File: " + thisFile + " "); plainOutputLog.write(" | Bytes: " + fileLength + " "); plainOutputLog.write(" | Vault: " + vaultName + " "); plainOutputLog.write(" | Location: " + locationUpped + " "); plainOutputLog.write(" | Date: " + d.toString() + " "); plainOutputLog.write(" | Hash: " + treeHash + " "); plainOutputLog.write(System.getProperty("line.separator")); plainOutputLog.close(); plainOutputTxt.write(System.getProperty("line.separator")); plainOutputTxt.write(" | ArchiveID: " + thisResult + " "); plainOutputTxt.write(System.getProperty("line.separator")); plainOutputTxt.write(" | File: " + thisFile + " "); plainOutputTxt.write(" | Bytes: " + fileLength + " "); plainOutputTxt.write(" | Vault: " + vaultName + " "); plainOutputTxt.write(" | Location: " + locationUpped + " "); plainOutputTxt.write(" | Date: " + d.toString() + " "); plainOutputTxt.write(" | Hash: " + treeHash + " "); plainOutputTxt.write(System.getProperty("line.separator")); plainOutputTxt.close(); plainOutputCsv.write("" + thisResult + ","); plainOutputCsv.write("" + thisFile + ","); plainOutputCsv.write("" + fileLength + ","); plainOutputCsv.write("" + vaultName + ","); plainOutputCsv.write("" + locationUpped + ","); plainOutputCsv.write("" + d.toString() + ","); plainOutputCsv.write("" + treeHash + ","); plainOutputCsv.write(System.getProperty("line.separator")); plainOutputCsv.close(); uploadList.add("Successfully uploaded " + thisFile + " to vault " + vaultName + " at " + locationUpped + ". Bytes: " + fileLength + ". ArchiveID Logged.\n"); } // v0.4 logging code // output.writeUTF("ArchiveID: " + // thisResult + " "); // output.writeUTF(" | File: " + // thisFile + " "); // output.writeUTF(" | Vault: " // +vaultName + " "); // output.writeUTF(" | Location: " + // locationUpped + " "); // output.writeUTF(" | Date: "+d.toString()+"\n\n"); catch (IOException c) { JOptionPane.showMessageDialog(null, LOG_WRITE_ERROR, "IO Error", JOptionPane.ERROR_MESSAGE); uw.dispose(); System.exit(1); } } else { JOptionPane.showMessageDialog(null, "Upload Complete!\nArchive ID: " + result.getArchiveId() + "\nIt may take some time for Amazon to update the inventory.", "Uploaded", JOptionPane.INFORMATION_MESSAGE); multiFiles = null; uw.dispose(); } clearFile(); } catch (Exception h) { if (logCheckMenuItem.isSelected()) { writeToErrorLog(h, thisFile); } JOptionPane.showMessageDialog(null, "" + h, "Error", JOptionPane.ERROR_MESSAGE); uw.dispose(); } } StringBuilder sb = new StringBuilder(); for (int j = 0; j < uploadFileBatch.length; j++) { sb.append(uploadList.get(j)); } uw.dispose(); JOptionPane.showMessageDialog(null, "Upload Complete! \n" + sb, "Uploaded", JOptionPane.INFORMATION_MESSAGE); // Close the JProgressBar multiFiles = null; clearFile(); } else { JOptionPane.showMessageDialog(null, "This wasn't supposed to happen.", "Bug!", JOptionPane.ERROR_MESSAGE); uw.dispose(); } return null; } private void writeToErrorLog(Exception h, String thisFile) { String thisError = h.toString(); Writer errorOutputLog = null; try { errorOutputLog = new BufferedWriter(new FileWriter(getLogFilenamePath(3), true)); } catch (Exception badLogCreate) { JOptionPane.showMessageDialog(null, LOG_CREATION_ERROR, "IO Error", JOptionPane.ERROR_MESSAGE); System.exit(1); } try { Date d = new Date(); errorOutputLog.write(System.getProperty("line.separator")); errorOutputLog.write("" + d.toString() + ": \"" + thisFile + "\" *ERROR* " + thisError); errorOutputLog.write(System.getProperty("line.separator")); } catch (Exception badLogWrite) { JOptionPane.showMessageDialog(null, LOG_WRITE_ERROR, "IO Error", JOptionPane.ERROR_MESSAGE); System.exit(1); } } }; uploadWorker.execute(); } } else { } }
From source file:com.vrane.metaGlacier.Archive.java
private String uploadParts(String uploadId) throws AmazonServiceException, NoSuchAlgorithmException, AmazonClientException, IOException { int filePosition = 0; long currentPosition = 0; byte[] buffer = new byte[(int) granularity]; List<byte[]> binaryChecksums = new LinkedList<>(); final File file = new File(localPath); String contentRange;/* w w w . jav a 2 s . c o m*/ int read = 0; try (FileInputStream fileToUpload = new FileInputStream(file)) { while (currentPosition < file.length()) { read = fileToUpload.read(buffer, filePosition, buffer.length); if (read == -1) { break; } if (Thread.currentThread().isInterrupted()) { LGR.warning("upload job is interrupted."); return null; } LGR.log(Level.FINE, "reading position {0} for file {1}", new Object[] { currentPosition, localPath }); byte[] bytesRead = Arrays.copyOf(buffer, read); contentRange = String.format("bytes %s-%s/*", currentPosition, currentPosition + read - 1); String checksum = TreeHashGenerator.calculateTreeHash(new ByteArrayInputStream(bytesRead)); byte[] binaryChecksum = BinaryUtils.fromHex(checksum); binaryChecksums.add(binaryChecksum); //Upload part. UploadMultipartPartRequest partRequest = new UploadMultipartPartRequest().withVaultName(vaultName) .withBody(new ByteArrayInputStream(bytesRead)).withChecksum(checksum) .withRange(contentRange).withUploadId(uploadId); try { GlacierFrame.getClient(region).uploadMultipartPart(partRequest); } catch (RequestTimeoutException e) { LGR.log(Level.SEVERE, "Request time out at {0}. Retrying in {1} s", new Object[] { HumanBytes.convert(currentPosition), RETRY_SEC }); LGR.log(Level.SEVERE, null, e); try { Thread.sleep(RETRY_SEC * 1000); } catch (InterruptedException ex) { LGR.log(Level.SEVERE, null, ex); return null; } try { GlacierFrame.getClient(region).uploadMultipartPart(partRequest); } catch (RequestTimeoutException ex) { LGR.log(Level.SEVERE, null, ex); LGR.severe("2nd time out. Giving up"); JOptionPane.showMessageDialog(null, "Error uploading"); return null; } } catch (Exception e) { LGR.log(Level.SEVERE, null, e); LGR.severe("Unanticipated error. Giving up."); return null; } if (Thread.currentThread().isInterrupted()) { LGR.warning("upload job is interrupted."); return null; } currentPosition = currentPosition + read; progress_reporter.setFilePosition(currentPosition); } } String checksum = TreeHashGenerator.calculateTreeHash(binaryChecksums); return checksum; }
From source file:de.kopis.glacier.commands.TreeHashArchiveCommand.java
License:Open Source License
public void calculateTreeHash(File file) { if (file.exists()) { log.info(TreeHashGenerator.calculateTreeHash(file)); } else {//from ww w . j av a 2s . co m log.error(String.format("File '%s' not found", file.getAbsolutePath())); } }
From source file:de.kopis.glacier.commands.UploadMultipartArchiveCommand.java
License:Open Source License
private String uploadParts(String uploadId, File file, final String vaultName, final Long partSize) throws AmazonServiceException, NoSuchAlgorithmException, AmazonClientException, IOException { FileInputStream fileToUpload = null; String checksum = ""; try {//from ww w. java 2 s. co m long currentPosition = 0; List<byte[]> binaryChecksums = new LinkedList<byte[]>(); fileToUpload = new FileInputStream(file); int counter = 1; int total = (int) Math.ceil(file.length() / (double) partSize); while (currentPosition < file.length()) { long length = partSize; if (currentPosition + partSize > file.length()) { length = file.length() - currentPosition; } Exception failedException = null; boolean completed = false; int tries = 0; while (!completed && tries < 5) { tries++; InputStream inputSubStream = newInputSubstream(file, currentPosition, length); inputSubStream.mark(-1); checksum = TreeHashGenerator.calculateTreeHash(inputSubStream); byte[] binaryChecksum = BinaryUtils.fromHex(checksum); inputSubStream.reset(); String range = "bytes " + currentPosition + "-" + (currentPosition + length - 1) + "/*"; UploadMultipartPartRequest req = new UploadMultipartPartRequest().withChecksum(checksum) .withBody(inputSubStream).withRange(range).withUploadId(uploadId) .withVaultName(vaultName); try { UploadMultipartPartResult partResult = client.uploadMultipartPart(req); log.info(String.format("Part %d/%d (%s) uploaded, checksum: %s", counter, total, range, partResult.getChecksum())); completed = true; binaryChecksums.add(binaryChecksum); } catch (Exception e) { failedException = e; } finally { if (inputSubStream != null) { try { inputSubStream.close(); } catch (IOException ex) { log.debug("Ignore failure in closing the Closeable", ex); } } } } if (!completed && failedException != null) throw new AmazonClientException("Failed operation", failedException); currentPosition += partSize; ++counter; } checksum = TreeHashGenerator.calculateTreeHash(binaryChecksums); } finally { if (fileToUpload != null) { fileToUpload.close(); } } return checksum; }
From source file:maebackup.MaeBackup.java
License:Open Source License
public static void upload(String lrzname) { try {// ww w . j a v a2 s. co m System.out.println("Uploading to Glacier..."); ClientConfiguration config = new ClientConfiguration(); config.setProtocol(Protocol.HTTPS); AmazonGlacierClient client = new AmazonGlacierClient(credentials, config); client.setEndpoint(endpoint); File file = new File(lrzname); String archiveid = ""; if (file.length() < 5 * 1024 * 1024) { System.out.println("File is small, uploading as single chunk"); String treehash = TreeHashGenerator.calculateTreeHash(file); InputStream is = new FileInputStream(file); byte[] buffer = new byte[(int) file.length()]; int bytes = is.read(buffer); if (bytes != file.length()) throw new RuntimeException("Only read " + bytes + " of " + file.length() + " byte file when preparing for upload."); InputStream bais = new ByteArrayInputStream(buffer); UploadArchiveRequest request = new UploadArchiveRequest(vaultname, lrzname, treehash, bais); UploadArchiveResult result = client.uploadArchive(request); archiveid = result.getArchiveId(); } else { long chunks = file.length() / chunksize; while (chunks > 10000) { chunksize <<= 1; chunks = file.length() / chunksize; } String chunksizestr = new Integer(chunksize).toString(); System.out.println( "Starting multipart upload: " + chunks + " full chunks of " + chunksizestr + " bytes"); InitiateMultipartUploadResult imures = client.initiateMultipartUpload( new InitiateMultipartUploadRequest(vaultname, lrzname, chunksizestr)); String uploadid = imures.getUploadId(); RandomAccessFile raf = new RandomAccessFile(file, "r"); byte[] buffer = new byte[chunksize]; for (long x = 0; x < chunks; x++) { try { System.out.println("Uploading chunk " + x + "/" + chunks); raf.seek(x * chunksize); raf.read(buffer); String parthash = TreeHashGenerator.calculateTreeHash(new ByteArrayInputStream(buffer)); String range = "bytes " + (x * chunksize) + "-" + ((x + 1) * chunksize - 1) + "/*"; client.uploadMultipartPart(new UploadMultipartPartRequest(vaultname, uploadid, parthash, range, new ByteArrayInputStream(buffer))); } catch (Exception e) { e.printStackTrace(); System.err.println("Error uploading chunk " + x + ", retrying..."); x--; } } if (file.length() > chunks * chunksize) { do { try { System.out.println("Uploading final partial chunk"); raf.seek(chunks * chunksize); int bytes = raf.read(buffer); String parthash = TreeHashGenerator .calculateTreeHash(new ByteArrayInputStream(buffer, 0, bytes)); String range = "bytes " + (chunks * chunksize) + "-" + (file.length() - 1) + "/*"; client.uploadMultipartPart(new UploadMultipartPartRequest(vaultname, uploadid, parthash, range, new ByteArrayInputStream(buffer, 0, bytes))); } catch (Exception e) { e.printStackTrace(); System.err.println("Error uploading final chunk, retrying..."); continue; } } while (false); } System.out.println("Completing upload"); String treehash = TreeHashGenerator.calculateTreeHash(file); CompleteMultipartUploadResult result = client .completeMultipartUpload(new CompleteMultipartUploadRequest(vaultname, uploadid, new Long(file.length()).toString(), treehash)); archiveid = result.getArchiveId(); } System.out.println("Uploaded " + lrzname + " to Glacier as ID " + archiveid); File listfile = new File(cachedir, "archives.lst"); FileWriter fw = new FileWriter(listfile, true); fw.write(archiveid + " " + lrzname + "\n"); fw.close(); } catch (Exception e) { throw new RuntimeException(e); } }
From source file:maebackup.MaeBackup.java
License:Open Source License
public static void download(String filename, String jobid) { try {/* ww w . j a v a 2 s . c o m*/ System.out.println("Starting download..."); ClientConfiguration config = new ClientConfiguration(); config.setProtocol(Protocol.HTTPS); AmazonGlacierClient client = new AmazonGlacierClient(credentials, config); client.setEndpoint(endpoint); if (jobid == null || jobid == "") { String archiveid; // Yes, this will screw up on actual 138-character file names, but... yeah. if (filename.length() == 138) { archiveid = filename; } else { File listfile = new File(cachedir, "archives.lst"); Map<File, String> filemap = loadHashes(listfile); archiveid = filemap.get(filename); if (archiveid == null) { System.err.println("Error: Could not find archive ID for file " + filename); System.exit(1); return; } } InitiateJobResult result = client.initiateJob(new InitiateJobRequest(vaultname, new JobParameters().withType("archive-retrieval").withArchiveId(archiveid))); jobid = result.getJobId(); System.out.println("Started download job as ID " + jobid); } else { DescribeJobResult djres = client.describeJob(new DescribeJobRequest(vaultname, jobid)); if (!djres.getStatusCode().equals("Succeeded")) { System.out.println("Job is not listed as Succeeded. It is: " + djres.getStatusCode()); System.out.println(djres.getStatusMessage()); System.exit(2); } long size = djres.getArchiveSizeInBytes(); long chunks = size / chunksize; while (chunks > 10000) { chunksize <<= 1; chunks = size / chunksize; } RandomAccessFile raf = new RandomAccessFile(filename, "w"); raf.setLength(size); byte[] buffer = new byte[chunksize]; for (int x = 0; x < chunks; x++) { try { System.out.println("Downloading chunk " + x + " of " + chunks); String range = "bytes " + (x * chunksize) + "-" + ((x + 1) * chunksize - 1) + "/*"; GetJobOutputResult gjores = client .getJobOutput(new GetJobOutputRequest(vaultname, jobid, range)); gjores.getBody().read(buffer); MessageDigest md = MessageDigest.getInstance("SHA-256"); md.update(buffer, 0, chunksize); byte[] hash = md.digest(); StringBuffer sb = new StringBuffer(); for (byte b : hash) { sb.append(String.format("%02x", b)); } if (!sb.toString().equalsIgnoreCase(gjores.getChecksum())) { System.err.println("Error: Chunk " + x + " does not match SHA-256. Retrying."); x--; continue; } raf.seek(x * chunksize); raf.write(buffer); } catch (Exception e) { System.err.println("Error: Exception while downloading chunk " + x + ". Retrying."); x--; } } if (size > chunks * chunksize) { do { try { System.out.println("Downloading final partial chunk"); String range = "bytes " + (chunks * chunksize) + "-" + (size - 1) + "/*"; GetJobOutputResult gjores = client .getJobOutput(new GetJobOutputRequest(vaultname, jobid, range)); int bytes = gjores.getBody().read(buffer); MessageDigest md = MessageDigest.getInstance("SHA-256"); md.update(buffer, 0, bytes); byte[] hash = md.digest(); StringBuffer sb = new StringBuffer(); for (byte b : hash) { sb.append(String.format("%02x", b)); } if (!sb.toString().equalsIgnoreCase(gjores.getChecksum())) { System.err.println("Error: Final chunk does not match SHA-256. Retrying."); continue; } raf.seek(chunks * chunksize); raf.write(buffer, 0, bytes); } catch (Exception e) { System.err.println("Error: Exception while downloading final chunk. Retrying."); continue; } } while (false); } raf.close(); String treehash = TreeHashGenerator.calculateTreeHash(new File(filename)); if (!treehash.equalsIgnoreCase(djres.getSHA256TreeHash())) { System.err.println("Error: File failed final tree hash check."); System.exit(3); } System.out.println("Download complete."); } } catch (Exception e) { throw new RuntimeException(e); } }