List of usage examples for java.net URLConnection getContentLength
public int getContentLength()
From source file:org.grycap.gpf4med.util.URLUtils.java
/** * Reads data from a URL and writes them to a local file. * @param source Source URL./*ww w. ja v a2 s . c o m*/ * @param destination Destination file. * @throws IOException If an input/output error occurs. */ public static void download(final URL source, final File destination) throws IOException { checkArgument(source != null, "Uninitialized source"); checkArgument(destination != null, "Uninitialized destination"); final URLConnection conn = source.openConnection(); checkState(conn != null, "Cannot open connection to: " + source.toString()); final String contentType = conn.getContentType(); final int contentLength = conn.getContentLength(); checkState(StringUtils.isNotEmpty(contentType), "Cannot determine the content type of: " + source.toString()); if (contentType.startsWith("text/") || contentLength == -1) { URLUtils.downloadText(source, destination); } else { URLUtils.downloadBinary(conn, destination); } }
From source file:org.apache.niolex.commons.net.DownloadUtil.java
/** * Download the file pointed by the url and return the content as byte array. * * @param strUrl//from www.j a va 2 s. c o m * The Url to be downloaded. * @param connectTimeout * Connect timeout in milliseconds. * @param readTimeout * Read timeout in milliseconds. * @param maxFileSize * Max file size in BYTE. * @param useCache Whether we use thread local cache or not. * @return The file content as byte array. * @throws NetException */ public static byte[] downloadFile(String strUrl, int connectTimeout, int readTimeout, int maxFileSize, Boolean useCache) throws NetException { LOG.debug("Start to download file [{}], C{}R{}M{}.", strUrl, connectTimeout, readTimeout, maxFileSize); InputStream in = null; try { URL url = new URL(strUrl); // We use Java URL to download file. URLConnection ucon = url.openConnection(); ucon.setConnectTimeout(connectTimeout); ucon.setReadTimeout(readTimeout); ucon.setDoOutput(false); ucon.setDoInput(true); ucon.connect(); final int contentLength = ucon.getContentLength(); validateContentLength(strUrl, contentLength, maxFileSize); if (ucon instanceof HttpURLConnection) { validateHttpCode(strUrl, (HttpURLConnection) ucon); } in = ucon.getInputStream(); // Get the input stream. byte[] ret = null; // Create the byte array buffer according to the strategy. if (contentLength > 0) { ret = commonDownload(contentLength, in); } else { ret = unusualDownload(strUrl, in, maxFileSize, useCache); } LOG.debug("Succeeded to download file [{}] size {}.", strUrl, ret.length); return ret; } catch (NetException e) { LOG.info(e.getMessage()); throw e; } catch (Exception e) { String msg = "Failed to download file " + strUrl + " msg=" + e.toString(); LOG.warn(msg); throw new NetException(NetException.ExCode.IOEXCEPTION, msg, e); } finally { // Close the input stream. StreamUtil.closeStream(in); } }
From source file:net.sf.firemox.tools.Picture.java
/** * Download a file from the specified URL to the specified local file. * //from w w w . j a v a2 s . co m * @param localFile * is the new card's picture to try first * @param remoteFile * is the URL where this picture will be downloaded in case of the * specified card name has not been found locally. * @param listener * the component waiting for this picture. * @since 0.83 Empty file are deleted to force file to be downloaded. */ public static synchronized void download(String localFile, URL remoteFile, MonitoredCheckContent listener) { BufferedOutputStream out = null; BufferedInputStream in = null; File toDownload = new File(localFile); if (toDownload.exists() && toDownload.length() == 0 && toDownload.canWrite()) { toDownload.delete(); } if (!toDownload.exists() || (toDownload.length() == 0 && toDownload.canWrite())) { // the file has to be downloaded try { if ("file".equals(remoteFile.getProtocol())) { File localRemoteFile = MToolKit .getFile(remoteFile.toString().substring(7).replaceAll("%20", " "), false); int contentLength = (int) localRemoteFile.length(); Log.info("Copying from " + localRemoteFile.getAbsolutePath()); LoaderConsole.beginTask( LanguageManager.getString("downloading") + " " + localRemoteFile.getAbsolutePath() + "(" + FileUtils.byteCountToDisplaySize(contentLength) + ")"); // Copy file in = new BufferedInputStream(new FileInputStream(localRemoteFile)); byte[] buf = new byte[2048]; int currentLength = 0; boolean succeed = false; for (int bufferLen = in.read(buf); bufferLen >= 0; bufferLen = in.read(buf)) { if (!succeed) { toDownload.getParentFile().mkdirs(); out = new BufferedOutputStream(new FileOutputStream(localFile)); succeed = true; } currentLength += bufferLen; if (out != null) { out.write(buf, 0, bufferLen); } if (listener != null) { listener.updateProgress(contentLength, currentLength); } } // Step 3: close streams IOUtils.closeQuietly(in); IOUtils.closeQuietly(out); in = null; out = null; return; } // Testing mode? if (!MagicUIComponents.isUILoaded()) { return; } // Step 1: open streams final URLConnection connection = MToolKit.getHttpConnection(remoteFile); int contentLength = connection.getContentLength(); in = new BufferedInputStream(connection.getInputStream()); Log.info("Download from " + remoteFile + "(" + FileUtils.byteCountToDisplaySize(contentLength) + ")"); LoaderConsole.beginTask(LanguageManager.getString("downloading") + " " + remoteFile + "(" + FileUtils.byteCountToDisplaySize(contentLength) + ")"); // Step 2: read and write until done byte[] buf = new byte[2048]; int currentLength = 0; boolean succeed = false; for (int bufferLen = in.read(buf); bufferLen >= 0; bufferLen = in.read(buf)) { if (!succeed) { toDownload.getParentFile().mkdirs(); out = new BufferedOutputStream(new FileOutputStream(localFile)); succeed = true; } currentLength += bufferLen; if (out != null) { out.write(buf, 0, bufferLen); } if (listener != null) { listener.updateProgress(contentLength, currentLength); } } // Step 3: close streams IOUtils.closeQuietly(in); IOUtils.closeQuietly(out); in = null; out = null; return; } catch (IOException e1) { if (MToolKit.getFile(localFile) != null) { MToolKit.getFile(localFile).delete(); } if (remoteFile.getFile().equals(remoteFile.getFile().toLowerCase())) { Log.fatal("could not load picture " + localFile + " from URL " + remoteFile + ", " + e1.getMessage()); } String tmpRemote = remoteFile.toString().toLowerCase(); try { download(localFile, new URL(tmpRemote), listener); } catch (MalformedURLException e) { Log.fatal("could not load picture " + localFile + " from URL " + tmpRemote + ", " + e.getMessage()); } } } }
From source file:com.waku.mmdataextract.ComprehensiveSearch.java
public static void saveImage(String imgSrc, String toFileName) { String toFile = "output/images/" + toFileName; if (new File(toFile).exists()) { logger.info("File already saved ->" + toFile); return;// w ww . j a v a 2s . c o m } URL u = null; URLConnection uc = null; InputStream raw = null; InputStream in = null; FileOutputStream out = null; try { int endIndex = imgSrc.lastIndexOf("/") + 1; String encodeFileName = URLEncoder.encode(imgSrc.substring(endIndex), "UTF-8").replaceAll("[+]", "%20"); u = new URL("http://shouji.gd.chinamobile.com" + imgSrc.substring(0, endIndex) + encodeFileName); uc = u.openConnection(); String contentType = uc.getContentType(); int contentLength = uc.getContentLength(); if (contentType.startsWith("text/") || contentLength == -1) { logger.error("This is not a binary file. -> " + imgSrc); } raw = uc.getInputStream(); in = new BufferedInputStream(raw); byte[] data = new byte[contentLength]; int bytesRead = 0; int offset = 0; while (offset < contentLength) { bytesRead = in.read(data, offset, data.length - offset); if (bytesRead == -1) break; offset += bytesRead; } if (offset != contentLength) { logger.error("Only read " + offset + " bytes; Expected " + contentLength + " bytes"); } out = new FileOutputStream(toFile); out.write(data); out.flush(); logger.info("Saved file " + u.toString() + " to " + toFile); } catch (Exception e) { e.printStackTrace(); } finally { try { in.close(); } catch (Exception e) { } try { out.close(); } catch (Exception e) { } } }
From source file:com.google.dart.tools.update.core.internal.UpdateUtils.java
/** * Download a URL to a local file, notifying the given monitor along the way. *///from www . j a va2 s . c o m public static void downloadFile(URL downloadUrl, File destFile, String taskName, IProgressMonitor monitor) throws IOException { URLConnection connection = downloadUrl.openConnection(); FileOutputStream out = new FileOutputStream(destFile); int length = connection.getContentLength(); monitor.beginTask(taskName, length); copyStream(connection.getInputStream(), out, monitor, length); monitor.done(); }
From source file:sce.RESTKBJob.java
public static ByteBuffer getAsByteArray(URL url) throws IOException { URLConnection connection = url.openConnection(); ByteArrayOutputStream tmpOut; try (InputStream in = connection.getInputStream()) { int contentLength = connection.getContentLength(); if (contentLength != -1) { tmpOut = new ByteArrayOutputStream(contentLength); } else {// w w w. ja v a 2 s . com tmpOut = new ByteArrayOutputStream(16384); // Pick some appropriate size } byte[] buf = new byte[512]; while (true) { int len = in.read(buf); if (len == -1) { break; } tmpOut.write(buf, 0, len); } } tmpOut.close(); byte[] array = tmpOut.toByteArray(); return ByteBuffer.wrap(array); }
From source file:tkwatch.Utilities.java
/** * Returns the content of a URL as a <code>String</code> instance. This was * used only during development. Retained for completeness. * // w w w . j av a 2 s. co m * @param newUrlName * The name of the URL to visit. * @return The contents of the URL as a <code>String</code> or * <code>null</code> if there's a problem. */ public static final String getHtmlContent(String newUrlName) { int urlContentLength = 0; byte[] inBuffer = null; try { URL url = new URL(newUrlName); URLConnection urlConnection = url.openConnection(); urlContentLength = urlConnection.getContentLength(); if (urlContentLength == Constants.UNKNOWN_LENGTH) inBuffer = new byte[Constants.MAX_URL_CONTENT]; else inBuffer = new byte[urlContentLength]; DataInputStream in = new DataInputStream(new BufferedInputStream(urlConnection.getInputStream())); in.readFully(inBuffer); in.close(); System.out.println("Utilities.getHtmlContent() reports inBuffer length is " + inBuffer.length); } catch (MalformedURLException mue) { return null; } catch (IOException ioe) { return null; } catch (Exception e) { e.printStackTrace(); Utilities.errorMessage(e.getMessage()); return null; } String resultString = new String(inBuffer); System.out.println(resultString.toString()); StringBuffer result = new StringBuffer(); for (int i = 0; i < resultString.length(); i++) { char c = resultString.charAt(i); // Only interested in ASCII if (c < Constants.LF || c > Constants.DEL) continue; if (c == '"') { // Assumes the first character won't be " char previous = resultString.charAt(i - 1); // Escape the \ in \" to deal with "\"" if (previous == '\\') result.append('\\'); // Escape double quotes to allow sane handling of Java strings. result.append('\\'); } // Eliminate single quotes to avoid problems storing strings in SQL. // Eliminate | to allow for alternate string delimiter. if (c != '\'' && c != '|') result.append(c); } return result.toString(); }
From source file:org.apache.brooklyn.util.http.HttpTool.java
/** * Connects to the given url and returns the connection. * Caller should {@code connection.getInputStream().close()} the result of this * (especially if they are making heavy use of this method). *//* www . jav a 2 s.co m*/ public static URLConnection connectToUrl(String u) throws Exception { final URL url = new URL(u); final AtomicReference<Exception> exception = new AtomicReference<Exception>(); // sometimes openConnection hangs, so run in background Future<URLConnection> f = executor.submit(new Callable<URLConnection>() { public URLConnection call() { try { HttpsURLConnection.setDefaultHostnameVerifier(new HostnameVerifier() { @Override public boolean verify(String s, SSLSession sslSession) { return true; } }); URLConnection connection = url.openConnection(); TrustingSslSocketFactory.configure(connection); connection.connect(); connection.getContentLength(); // Make sure the connection is made. return connection; } catch (Exception e) { exception.set(e); LOG.debug("Error connecting to url " + url + " (propagating): " + e, e); } return null; } }); try { URLConnection result = null; try { result = f.get(60, TimeUnit.SECONDS); } catch (InterruptedException e) { throw e; } catch (Exception e) { LOG.debug("Error connecting to url " + url + ", probably timed out (rethrowing): " + e); throw new IllegalStateException( "Connect to URL not complete within 60 seconds, for url " + url + ": " + e); } if (exception.get() != null) { LOG.debug("Error connecting to url " + url + ", thread caller of " + exception, new Throwable("source of rethrown error " + exception)); throw exception.get(); } else { return result; } } finally { f.cancel(true); } }
From source file:sce.RESTAppMetricJob.java
public static ByteBuffer getAsByteArray(URL url) throws IOException { URLConnection connection = url.openConnection(); ByteArrayOutputStream tmpOut; try (InputStream in = connection.getInputStream()) { int contentLength = connection.getContentLength(); if (contentLength != -1) { tmpOut = new ByteArrayOutputStream(contentLength); } else {//w w w . j ava 2 s .c o m tmpOut = new ByteArrayOutputStream(16384); // pick some appropriate size } byte[] buf = new byte[512]; while (true) { int len = in.read(buf); if (len == -1) { break; } tmpOut.write(buf, 0, len); } } tmpOut.close(); byte[] array = tmpOut.toByteArray(); return ByteBuffer.wrap(array); }
From source file:com.alvermont.javascript.tools.shell.ShellMain.java
/** * Read file or url specified by <tt>path</tt>. * @return file or url content as <tt>byte[]</tt> or as <tt>String</tt> if * <tt>convertToString</tt> is true. */// w w w. j a v a 2s .co m private static Object readFileOrUrl(String path, boolean convertToString) { URL url = null; // Assume path is URL if it contains dot and there are at least // 2 characters in the protocol part. The later allows under Windows // to interpret paths with driver letter as file, not URL. if (path.indexOf(':') >= 2) { try { url = new URL(path); } catch (MalformedURLException ex) { log.debug("MalformedURLException in readFileOrUrl", ex); } } InputStream is = null; int capacityHint = 0; if (url == null) { final File file = new File(path); capacityHint = (int) file.length(); try { is = new FileInputStream(file); } catch (IOException ex) { Context.reportError(ToolErrorReporter.getMessage("msg.couldnt.open", path)); return null; } } else { try { final URLConnection uc = url.openConnection(); is = uc.getInputStream(); capacityHint = uc.getContentLength(); // Ignore insane values for Content-Length if (capacityHint > (1 << 20)) { capacityHint = -1; } } catch (IOException ex) { Context.reportError( ToolErrorReporter.getMessage("msg.couldnt.open.url", url.toString(), ex.toString())); return null; } } if (capacityHint <= 0) { capacityHint = 4096; } byte[] data; try { try { data = Kit.readStream(is, capacityHint); } finally { is.close(); } } catch (IOException ex) { Context.reportError(ex.toString()); return null; } Object result; if (!convertToString) { result = data; } else { // Convert to String using the default encoding // XXX: Use 'charset=' argument of Content-Type if URL? result = new String(data); } return result; }