List of usage examples for org.apache.commons.io FileUtils byteCountToDisplaySize
public static String byteCountToDisplaySize(long size)
From source file:org.polymap.p4.data.importer.raster.RasterImporter.java
@Override public void createResultViewer(Composite parent, IPanelToolkit tk) { this.toolkit = tk; if (exception != null) { tk.createFlowText(parent, "\nUnable to read the data.\n\n**Reason**: " + exception.getMessage()); } else {/* ww w.ja v a2 s . c o m*/ try { StringWriter out = new StringWriter(1024); new MarkdownBuilder(out).locale.put(RWT.getLocale()) .paragraph("## Format: {0}", grid.getFormat().getName()).paragraph(p -> { p.add(" * Size: {0}", FileUtils.byteCountToDisplaySize(main.length())).newline(1); p.add(" * CRS: {0}", CRS.toSRS(grid.getCoordinateReferenceSystem())).newline(1); GeneralEnvelope env = grid.getOriginalEnvelope(); p.add(" * Envelope: {0}...{1} / {2}...{3}", env.getMinimum(0), env.getMaximum(0), env.getMinimum(1), env.getMaximum(1)).newline(2); }).paragraph(p -> { p.add("## Coverages").newline(2); try { for (String name : grid.getGridCoverageNames()) { p.add(" * {0}", name).newline(1); } } catch (Exception e) { log.warn("", e); p.add("Error: " + e.getLocalizedMessage()).newline(1); } }).paragraph(p -> { try { if (grid.getMetadataNames() != null) { p.add("## Metadata").newline(2); for (String name : grid.getMetadataNames()) { p.add(" * {0}", name).newline(1); } } } catch (IOException e) { log.warn("", e); } }); //.paragraph( "*Grid support is in **BETA** stage. The data is imported and accessed as is. No error checking, validation or optimization is done.*" ); tk.createFlowText(parent, out.toString()); } catch (Exception e) { log.warn("", e); tk.createFlowText(parent, "\nUnable to read the data.\n\n**Reason**: " + e.getMessage()); site.ok.set(false); exception = e; } } }
From source file:org.p_vcd.model.MyUtil.java
public static String getSizeToText(long numBytes) { return FileUtils.byteCountToDisplaySize(numBytes); }
From source file:org.p_vcd.ui.VcdDialog.java
private void updateReferenceDbDetail() { try {//w w w . j a v a2 s . co m List<VideoDatabase> selectedRefDbs = getSelectedRefDbs(); if (selectedRefDbs.size() == 0) { lbl_refDbTitle.setText(""); lbl_refDbFiles.setText(""); lbl_refDbMetadata.setText(""); return; } lbl_refDbTitle.setText("Files in " + selectedRefDbs.size() + " selected databases:"); int contFiles = 0; long contSize = 0; double contTime = 0; StringBuffer sb = new StringBuffer("<html>"); for (VideoDatabase dbr : selectedRefDbs) { for (PVCDObj p : dbr.getFileList()) { sb.append(p.getFilePath()).append("<br>"); } contFiles += dbr.getFileList().size(); contSize += dbr.getTotalBytesVideos(); contTime += dbr.getTotalSeconds(); } String txtSize = FileUtils.byteCountToDisplaySize(contSize); String txtTime = MyUtil.getSecondsToHHMMSS(contTime); lbl_refDbFiles.setText(sb.toString()); lbl_refDbMetadata .setText(contFiles + " files, " + txtSize + " total size, " + txtTime + " total length."); } catch (Exception e) { e.printStackTrace(); } }
From source file:org.rm3l.ddwrt.tiles.services.vpn.OpenVPNClientTile.java
@Nullable @Override/*from w w w.j a v a2 s . c o m*/ protected Loader<NVRAMInfo> getLoader(int id, Bundle args) { return new AsyncTaskLoader<NVRAMInfo>(this.mParentFragmentActivity) { @Nullable @Override public NVRAMInfo loadInBackground() { try { Log.d(LOG_TAG, "Init background loader for " + OpenVPNClientTile.class + ": routerInfo=" + mRouter + " / this.mAutoRefreshToggle= " + mAutoRefreshToggle + " / nbRunsLoader=" + nbRunsLoader); if (nbRunsLoader > 0 && !mAutoRefreshToggle) { //Skip run Log.d(LOG_TAG, "Skip loader run"); return new NVRAMInfo().setException(new DDWRTTileAutoRefreshNotAllowedException()); } nbRunsLoader++; mNvramInfo = null; @NotNull final NVRAMInfo nvramInfo = new NVRAMInfo(); NVRAMInfo nvramInfoTmp = null; try { nvramInfoTmp = SSHUtils.getNVRamInfoFromRouter(mRouter, mGlobalPreferences, //Status: {1,0} OPENVPNCL_ENABLE, //Server IP Name OPENVPNCL_REMOTEIP, //Port (from 1 to 65535): default=1194 OPENVPNCL_REMOTEPORT, //Tunnel Device: {"tun", "tap"} OPENVPNCL_TUNTAP, //Tunnel Protocol: { "udp", "tcp-client"} OPENVPNCL_PROTO, //Encryption Cipher: //{aes-512-cbc, aes-256-cbc, aes-192-cbc, aes-128-cbc, bf-cbc, none} OPENVPNCL_CIPHER, //Hash Algorithm //{sha512, sha256, sha1, md5, md4, none} OPENVPNCL_AUTH, //TLS Cipher //{TLS-DHE-RSA-WITH-AES-256-GCM-SHA384, //TLS-DHE-RSA-WITH-AES-256-CBC-SHA256, //TLS-DHE-RSA-WITH-AES-128-CBC-SHA, //TLS-RSA-WITH-AES-256-GCM-SHA384, //TLS-RSA-WITH-AES-256-CBC-SHA256, //TLS-RSA-WITH-AES-128-CBC-SHA, //TLS-RSA-WITH-RC4-128-MD5, //0} //Advanced Options: {1,0} OPENVPNCL_ADV, OPENVPNCL_TLSCIP, //LZO Compression: {yes, adaptive, no, off} OPENVPNCL_LZO, //NAT: {1, 0} OPENVPNCL_NAT, //Firewall Protection (enabled if openvpncl_nat==1): {1,0} OPENVPNCL_SEC, //Bridge TAP to br0 (enabled if openvpncl_nat==0)"{1,0} OPENVPNCL_BRIDGE, //IP Address OPENVPNCL_IP, //Subnet Mask OPENVPNCL_MASK, //Tunnel MTU Setting: (from 1 to 65535) OPENVPNCL_MTU, //Tunnel UDP Fragment (num max chars=5): if nothing => Disabled OPENVPNCL_FRAGMENT, //Tunnel UDP MSS-Fix: {1,0} OPENVPNCL_MSSFIX, //nsCertType verification: {1,0} OPENVPNCL_CERTTYPE, //TLS Auth Key OPENVPNCL_TLSAUTH, //Additional Config OPENVPNCL_CONFIG, //Policy based Routing OPENVPNCL_ROUTE, //PKCS12 Key OPENVPNCL_PKCS_12, //Static Key OPENVPNCL_STATIC, //CA Cert OPENVPNCL_CA, //Public Client Cert OPENVPNCL_CLIENT, //Private Client Key OPENVPNCL_KEY); } finally { if (nvramInfoTmp != null) { nvramInfo.putAll(nvramInfoTmp); } final String[] devDeviceLine = SSHUtils.getManualProperty(mRouter, mGlobalPreferences, "cat /tmp/openvpncl/openvpn.conf | grep \"dev \""); String openvpnclIface = null; if (devDeviceLine != null && devDeviceLine.length > 0) { openvpnclIface = devDeviceLine[0].replace("dev ", "").trim(); } if (!Strings.isNullOrEmpty(openvpnclIface)) { nvramInfo.setProperty(OPENVPNCL__DEV, openvpnclIface); //noinspection ConstantConditions final Map<WirelessIfaceTile.IfaceStatsType, Long> ifaceRxAndTxRates = getIfaceRxAndTxRates( openvpnclIface); final Long rxBps = ifaceRxAndTxRates.get(RX_BYTES); final Long txBps = ifaceRxAndTxRates.get(TX_BYTES); if (rxBps != null) { nvramInfo.setProperty(OPENVPNCL__DEV_RX_RATE, rxBps + " B/s (" + FileUtils.byteCountToDisplaySize(rxBps) + "/s)"); } if (txBps != null) { nvramInfo.setProperty(OPENVPNCL__DEV_TX_RATE, txBps + " B/s (" + FileUtils.byteCountToDisplaySize(txBps) + "/s)"); } //Packet Info final String sysClassNetStatsFolder = String.format(CAT_SYS_CLASS_NET_S_STATISTICS, openvpnclIface); try { final String[] packetsInfo = SSHUtils.getManualProperty(mRouter, mGlobalPreferences, String.format("%s/rx_packets", sysClassNetStatsFolder), String.format("%s/rx_errors", sysClassNetStatsFolder), String.format("%s/tx_packets", sysClassNetStatsFolder), String.format("%s/tx_errors", sysClassNetStatsFolder)); if (packetsInfo != null) { final long rxErrors = Long.parseLong(packetsInfo[1]); nvramInfo.setProperty(OPENVPNCL__DEV_RX_PACKETS, String.format("%s (%s)", packetsInfo[0], rxErrors <= 0 ? "no error" : (rxErrors + String.format("error%s", rxErrors > 1 ? "s" : "")))); final long txErrors = Long.parseLong(packetsInfo[3]); nvramInfo.setProperty(OPENVPNCL__DEV_TX_PACKETS, String.format("%s (%s)", packetsInfo[2], txErrors <= 0 ? "no error" : (txErrors + String.format(" error%s", txErrors > 1 ? "s" : "")))); } } catch (final Exception e) { e.printStackTrace(); //No worries } } } if (nvramInfo.isEmpty()) { throw new DDWRTNoDataException("No Data!"); } return nvramInfo; } catch (@NotNull final Exception e) { e.printStackTrace(); return new NVRAMInfo().setException(e); } } @NotNull private Map<WirelessIfaceTile.IfaceStatsType, Long> getIfaceRxAndTxRates( @NotNull final String phyIface) { final Map<WirelessIfaceTile.IfaceStatsType, Long> result = Maps.newHashMapWithExpectedSize(2); final String sysClassNetStatsFolder = String.format(CAT_SYS_CLASS_NET_S_STATISTICS, phyIface); final String rxBytesCmd = String.format("%s/rx_bytes", sysClassNetStatsFolder); final String txBytesCmd = String.format("%s/tx_bytes", sysClassNetStatsFolder); try { final long[] bytesBeforeAndAfter = parseFloatDataFromOutput(SSHUtils.getManualProperty(mRouter, mGlobalPreferences, rxBytesCmd, txBytesCmd, "sleep 1", rxBytesCmd, txBytesCmd)); if (bytesBeforeAndAfter.length >= 4) { result.put(RX_BYTES, Math.abs(bytesBeforeAndAfter[1] - bytesBeforeAndAfter[0])); result.put(TX_BYTES, Math.abs(bytesBeforeAndAfter[3] - bytesBeforeAndAfter[2])); } } catch (Exception e) { e.printStackTrace(); } return result; } @NotNull private long[] parseFloatDataFromOutput(@Nullable final String[] output) { if (output == null || output.length == 0) { throw new IllegalArgumentException("Output null or empty"); } final long[] result = new long[output.length]; for (int i = 0; i < output.length; i++) { result[i] = Long.parseLong(output[i]); } return result; } }; }
From source file:org.rm3l.ddwrt.tiles.status.wireless.WirelessIfaceTile.java
@Nullable @Override/* w w w . j a v a2s.com*/ protected Loader<NVRAMInfo> getLoader(int id, Bundle args) { return new AsyncTaskLoader<NVRAMInfo>(this.mParentFragmentActivity) { @Nullable @Override public NVRAMInfo loadInBackground() { try { Log.d(LOG_TAG, "Init background loader for " + WirelessIfaceTile.class + ": routerInfo=" + mRouter + " / this.mAutoRefreshToggle= " + mAutoRefreshToggle + " / nbRunsLoader=" + nbRunsLoader); if (nbRunsLoader > 0 && !mAutoRefreshToggle) { //Skip run Log.d(LOG_TAG, "Skip loader run"); return new NVRAMInfo().setException(new DDWRTTileAutoRefreshNotAllowedException()); } nbRunsLoader++; final NVRAMInfo nvramInfo = getIfaceNvramInfo(iface); if (parentIface != null && !parentIface.isEmpty()) { nvramInfo.putAll(getIfaceNvramInfo(parentIface)); } if (nvramInfo.isEmpty()) { throw new DDWRTNoDataException("No Data!"); } return nvramInfo; } catch (@NotNull final Exception e) { e.printStackTrace(); return new NVRAMInfo().setException(e); } } @NotNull private NVRAMInfo getIfaceNvramInfo(@NotNull String wlIface) throws Exception { if (DDWRTCompanionConstants.TEST_MODE) { return new NVRAMInfo() .setProperty(WIRELESS_IFACE + (wlIface.equals(parentIface) ? "_parent" : ""), wlIface) .setProperty(wlIface + "_radio", String.valueOf(new Random().nextInt())) .setProperty(wlIface + "_mode", "Mode") .setProperty(wlIface + "_hwaddr", "hw:ad:dr:rd:da:wh") .setProperty(wlIface + "_ifname", wlIface + " ifname") .setProperty(wlIface + "_net_mode", "ng-only").setProperty(wlIface + "_ssid", "SSID") .setProperty(wlIface + "_channel", String.valueOf(new Random().nextInt(100))) .setProperty(wlIface + "_txpwr", String.valueOf(new Random().nextInt(100))) .setProperty(wlIface + "_rate", String.valueOf(new Random().nextInt(100))) .setProperty(wlIface + "_akm", "psk psk2"); } final NVRAMInfo nvramInfo = new NVRAMInfo(); NVRAMInfo nvramInfoTmp = null; try { nvramInfoTmp = SSHUtils.getNVRamInfoFromRouter(mRouter, mGlobalPreferences, wlIface + "_radio", wlIface + "_mode", wlIface + "_hwaddr", wlIface + "_ifname", wlIface + "_net_mode", wlIface + "_ssid", wlIface + "_channel", wlIface + "_txpwr", wlIface + "_rate", wlIface + "_akm", wlIface + "_wpa_psk"); } finally { if (nvramInfoTmp != null) { nvramInfo.putAll(nvramInfoTmp); } nvramInfo.setProperty(WIRELESS_IFACE + (wlIface.equals(parentIface) ? "_parent" : ""), wlIface); //Set Temp, RX and TX Network Bandwidths on Physical Interface final String phyIface = nvramInfo.getProperty(wlIface + "_ifname"); if (!isNullOrEmpty(phyIface)) { //Set temperature //noinspection ConstantConditions final Map<TemperatureUnit, String> ifaceTemperatures = getIfaceTemperature(phyIface); final String celsius = ifaceTemperatures.get(CELSIUS); final String fahrenheit = ifaceTemperatures.get(FAHRENHEIT); if (!(isNullOrEmpty(celsius) || isNullOrEmpty(fahrenheit))) { nvramInfo.setProperty(wlIface + "_temperature", celsius + DEGREE_SYMBOL + CELSIUS + " (" + fahrenheit + DEGREE_SYMBOL + FAHRENHEIT + ")"); } //RX and TX Bytes //noinspection ConstantConditions final Map<IfaceStatsType, Long> ifaceRxAndTxRates = getIfaceRxAndTxRates(phyIface); final Long rxBps = ifaceRxAndTxRates.get(IfaceStatsType.RX_BYTES); final Long txBps = ifaceRxAndTxRates.get(IfaceStatsType.TX_BYTES); if (rxBps != null) { nvramInfo.setProperty(wlIface + "_rx_rate_human_readable", rxBps + " B/s (" + FileUtils.byteCountToDisplaySize(rxBps) + "/s)"); } if (txBps != null) { nvramInfo.setProperty(wlIface + "_tx_rate_human_readable", txBps + " B/s (" + FileUtils.byteCountToDisplaySize(txBps) + "/s)"); } //Packet Info final String sysClassNetStatsFolder = String.format(CAT_SYS_CLASS_NET_S_STATISTICS, phyIface); try { final String[] packetsInfo = SSHUtils.getManualProperty(mRouter, mGlobalPreferences, String.format("%s/rx_packets", sysClassNetStatsFolder), String.format("%s/rx_errors", sysClassNetStatsFolder), String.format("%s/tx_packets", sysClassNetStatsFolder), String.format("%s/tx_errors", sysClassNetStatsFolder)); if (packetsInfo != null) { final long rxErrors = Long.parseLong(packetsInfo[1]); nvramInfo.setProperty(wlIface + "_rx_packets", String.format("%s (%s)", packetsInfo[0], rxErrors <= 0 ? "no error" : (rxErrors + String.format("error%s", rxErrors > 1 ? "s" : "")))); final long txErrors = Long.parseLong(packetsInfo[3]); nvramInfo.setProperty(wlIface + "_tx_packets", String.format("%s (%s)", packetsInfo[2], txErrors <= 0 ? "no error" : (txErrors + String.format(" error%s", txErrors > 1 ? "s" : "")))); } } catch (final Exception e) { e.printStackTrace(); //No worries } } } return nvramInfo; } @NotNull private Map<TemperatureUnit, String> getIfaceTemperature(@NotNull final String phyIface) { final Map<TemperatureUnit, String> result = Maps.newHashMapWithExpectedSize(2); final String phyIfaceVarNameInRouter = WirelessIfaceTile.class.getSimpleName() + "TemperatureCelsius"; try { final String[] temperatures = SSHUtils.getManualProperty(mRouter, mGlobalPreferences, String.format( "%s=$(echo $((`wl -i %s phy_tempsense | awk {' print $1 '}`/2+20))); echo \"C:$%s\"; echo \"F:$(($%s*9/5+32))\"", phyIfaceVarNameInRouter, phyIface, phyIfaceVarNameInRouter, phyIfaceVarNameInRouter)); if (temperatures != null && temperatures.length >= 2) { for (final String temperature : temperatures) { if (temperature == null) { continue; } if (temperature.startsWith("C:")) { result.put(CELSIUS, temperature.replaceAll("C:", "").trim()); } else if (temperature.startsWith("F:")) { result.put(FAHRENHEIT, temperature.replaceAll("F:", "").trim()); } } } } catch (Exception e) { e.printStackTrace(); } return result; } @NotNull private Map<IfaceStatsType, Long> getIfaceRxAndTxRates(@NotNull final String phyIface) { final Map<IfaceStatsType, Long> result = Maps.newHashMapWithExpectedSize(2); final String sysClassNetStatsFolder = String.format(CAT_SYS_CLASS_NET_S_STATISTICS, phyIface); final String rxBytesCmd = String.format("%s/rx_bytes", sysClassNetStatsFolder); final String txBytesCmd = String.format("%s/tx_bytes", sysClassNetStatsFolder); try { final long[] bytesBeforeAndAfter = parseFloatDataFromOutput(SSHUtils.getManualProperty(mRouter, mGlobalPreferences, rxBytesCmd, txBytesCmd, "sleep 1", rxBytesCmd, txBytesCmd)); if (bytesBeforeAndAfter.length >= 4) { result.put(IfaceStatsType.RX_BYTES, Math.abs(bytesBeforeAndAfter[1] - bytesBeforeAndAfter[0])); result.put(IfaceStatsType.TX_BYTES, Math.abs(bytesBeforeAndAfter[3] - bytesBeforeAndAfter[2])); } } catch (Exception e) { e.printStackTrace(); } return result; } @NotNull private long[] parseFloatDataFromOutput(@Nullable final String[] output) { if (output == null || output.length == 0) { throw new IllegalArgumentException("Output null or empty"); } final long[] result = new long[output.length]; for (int i = 0; i < output.length; i++) { result[i] = Long.parseLong(output[i]); } return result; } }; }
From source file:org.rm3l.ddwrt.utils.Utils.java
public static String toHumanReadableByteCount(final long sizeInBytes) { return FileUtils.byteCountToDisplaySize(sizeInBytes); }
From source file:org.runmyprocess.sec.FTP.java
private void listFiles(JSONObject jsonObject) throws Exception { // get all files from server and store them in an array of // FTPFiles//w w w. j a v a2 s . com JSONArray fileInfo = new JSONArray(); LOG.log("LIST directories: " + jsonObject.getString("path"), Level.INFO); String[] directories = this.client.listNames(jsonObject.getString("path")); LOG.log("LIST files: " + jsonObject.getString("path"), Level.INFO); FTPFile[] files = this.client.listFiles(jsonObject.getString("path")); LOG.log("LIST DONE", Level.INFO); for (FTPFile file : files) { if (file.getType() == FTPFile.FILE_TYPE) { JSONObject fileObject = new JSONObject(); fileObject.put("Name", file.getName()); fileObject.put("Size", FileUtils.byteCountToDisplaySize(file.getSize())); fileObject.put("Timestamp", file.getTimestamp().getTime().toString()); fileInfo.add(fileObject); } } response.setStatus(200);//sets the return status to 200 JSONObject resp = new JSONObject(); resp.put("files", fileInfo);//sends the info inside an object resp.put("directories", directories); response.setData(resp); }
From source file:org.sakaiproject.archive.tool.ArchiveAction.java
/** * build the context for batch archive confirm *//*from www . j a v a 2 s .c om*/ public String buildDownloadContext(VelocityPortlet portlet, Context context, RunData rundata, SessionState state) { context.put("tlang", rb); buildMenu(context); //get list of existing archives Collection<File> files = Collections.<File>emptySet(); File archiveBaseDir = new File( serverConfigurationService.getString("archive.storage.path", "sakai/archive")); if (archiveBaseDir.exists() && archiveBaseDir.isDirectory()) { files = FileUtils.listFiles(archiveBaseDir, new SuffixFileFilter(".zip"), null); } List<SparseFile> zips = new ArrayList<SparseFile>(); SimpleDateFormat dateFormatIn = new SimpleDateFormat("yyyyMMddHHmmss"); SimpleDateFormat dateFormatOut = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); Calendar calendar = Calendar.getInstance(); //porcess the list. also get the hash for the file if it exists for (File f : files) { String absolutePath = f.getAbsolutePath(); SparseFile sf = new SparseFile(); sf.setFilename(f.getName()); sf.setAbsolutePath(absolutePath); sf.setSize(FileUtils.byteCountToDisplaySize(f.length())); //get the datetime string, its the last part of the file name, convert back to a date that we can display String dateTimeStr = StringUtils.substringAfterLast(StringUtils.removeEnd(f.getName(), ".zip"), "-"); try { Date date = dateFormatIn.parse(dateTimeStr); sf.setDateCreated(dateFormatOut.format(date)); } catch (ParseException pe) { //ignore, just don't set the date } //get siteId, first part of name String siteId = StringUtils.substringBeforeLast(f.getName(), "-"); sf.setSiteId(siteId); //try to get site title if the site still exists try { Site site = siteService.getSite(siteId); sf.setSiteTitle(site.getTitle()); } catch (IdUnusedException e) { //ignore, no site available } //get the hash. need to read it from the file. Same filename but diff extension String hashFilePath = StringUtils.removeEnd(absolutePath, ".zip"); hashFilePath = hashFilePath + ".sha1"; File hashFile = new File(hashFilePath); try { String hash = FileUtils.readFileToString(hashFile); sf.setHash(hash); } catch (IOException e) { //ignore, dont use the hash } zips.add(sf); } context.put("archives", zips); return "-download"; }
From source file:org.schedulesdirect.grabber.Grabber.java
private void updateZip(NetworkEpgClient clnt) throws IOException, JSONException, JsonParseException { Set<String> completedListings = new HashSet<String>(); LOG.debug(String.format("Using %d worker threads", globalOpts.getMaxThreads())); pool = createThreadPoolExecutor();// w w w .j av a 2 s. c om start = System.currentTimeMillis(); File dest = grabOpts.getTarget(); cachedSeriesIds = new HashSet<String>(); boolean rmDest = false; if (dest.exists()) { ZipEpgClient zipClnt = null; try { zipClnt = new ZipEpgClient(dest); if (!zipClnt.getUserStatus().getLastServerRefresh() .before(clnt.getUserStatus().getLastServerRefresh())) { LOG.info( "Current cache file contains latest data from Schedules Direct server; use --force-download to force a new download from server."); boolean force = grabOpts.isForce(); if (!force) return; else LOG.warn("Forcing an update of data with the server due to user request!"); } } catch (Exception e) { if (grabOpts.isKeep()) { LOG.error("Existing cache is invalid, keeping by user request!", e); return; } else { LOG.warn("Existing cache is invalid, deleting it; use --keep-bad-cache to keep existing cache!", e); rmDest = true; } } finally { if (zipClnt != null) try { zipClnt.close(); } catch (IOException e) { } if (rmDest && !dest.delete()) throw new IOException("Unable to delete " + dest); } } freshZip = !dest.exists(); try (FileSystem vfs = FileSystems.newFileSystem(new URI(String.format("jar:%s", dest.toURI())), Collections.singletonMap("create", "true"))) { if (freshZip) { Path target = vfs.getPath(ZipEpgClient.ZIP_VER_FILE); Files.write(target, Integer.toString(ZipEpgClient.ZIP_VER).getBytes(ZipEpgClient.ZIP_CHARSET)); } ProgramCache progCache = ProgramCache.get(vfs); Path lineups = vfs.getPath("lineups.txt"); Files.deleteIfExists(lineups); Path scheds = vfs.getPath("/schedules/"); if (!Files.isDirectory(scheds)) Files.createDirectory(scheds); Path maps = vfs.getPath("/maps/"); PathUtils.removeDirectory(maps); Files.createDirectory(maps); Path progs = vfs.getPath("/programs/"); if (!Files.isDirectory(progs)) Files.createDirectory(progs); Path logos = vfs.getPath("/logos/"); if (!Files.isDirectory(logos)) Files.createDirectory(logos); Path md5s = vfs.getPath("/md5s/"); if (!Files.isDirectory(md5s)) Files.createDirectory(md5s); Path cache = vfs.getPath(LOGO_CACHE); if (Files.exists(cache)) { String cacheData = new String(Files.readAllBytes(cache), ZipEpgClient.ZIP_CHARSET); logoCache = Config.get().getObjectMapper().readValue(cacheData, JSONObject.class); } else logoCache = new JSONObject(); Path seriesInfo = vfs.getPath("/seriesInfo/"); if (!Files.isDirectory(seriesInfo)) Files.createDirectories(seriesInfo); loadSeriesInfoIds(seriesInfo); missingSeriesIds = Collections.synchronizedSet(new HashSet<String>()); loadRetryIds(vfs.getPath(SERIES_INFO_DATA)); JSONObject resp = Config.get().getObjectMapper().readValue( factory.get(DefaultJsonRequest.Action.GET, RestNouns.LINEUPS, clnt.getHash(), clnt.getUserAgent(), globalOpts.getUrl().toString()).submitForJson(null), JSONObject.class); if (!JsonResponseUtils.isErrorResponse(resp)) Files.write(lineups, resp.toString(3).getBytes(ZipEpgClient.ZIP_CHARSET)); else LOG.error("Received error response when requesting lineup data!"); for (Lineup l : clnt.getLineups()) { buildStationList(); JSONObject o = Config.get().getObjectMapper() .readValue( factory.get(DefaultJsonRequest.Action.GET, l.getUri(), clnt.getHash(), clnt.getUserAgent(), globalOpts.getUrl().toString()).submitForJson(null), JSONObject.class); Files.write(vfs.getPath("/maps", ZipEpgClient.scrubFileName(String.format("%s.txt", l.getId()))), o.toString(3).getBytes(ZipEpgClient.ZIP_CHARSET)); JSONArray stations = o.getJSONArray("stations"); JSONArray ids = new JSONArray(); for (int i = 0; i < stations.length(); ++i) { JSONObject obj = stations.getJSONObject(i); String sid = obj.getString("stationID"); if (stationList != null && !stationList.contains(sid)) LOG.debug(String.format("Skipped %s; not listed in station file", sid)); else if (completedListings.add(sid)) { ids.put(sid); if (!grabOpts.isNoLogos()) { if (logoCacheInvalid(obj)) pool.execute(new LogoTask(obj, vfs, logoCache)); else if (LOG.isDebugEnabled()) LOG.debug(String.format("Skipped logo for %s; already cached!", obj.optString("callsign", null))); } else if (!logosWarned) { logosWarned = true; LOG.warn("Logo downloads disabled by user request!"); } } else LOG.debug(String.format("Skipped %s; already downloaded.", sid)); //pool.setMaximumPoolSize(5); // Processing these new schedules takes all kinds of memory! if (ids.length() == grabOpts.getMaxSchedChunk()) { pool.execute(new ScheduleTask(ids, vfs, clnt, progCache, factory)); ids = new JSONArray(); } } if (ids.length() > 0) pool.execute(new ScheduleTask(ids, vfs, clnt, progCache, factory)); } pool.shutdown(); try { LOG.debug("Waiting for SchedLogoExecutor to terminate..."); if (pool.awaitTermination(15, TimeUnit.MINUTES)) LOG.debug("SchedLogoExecutor: Terminated successfully."); else { failedTask = true; LOG.warn( "SchedLogoExecutor: Termination timed out; some tasks probably didn't finish properly!"); } } catch (InterruptedException e) { failedTask = true; LOG.warn( "SchedLogoExecutor: Termination interrupted); some tasks probably didn't finish properly!"); } Files.write(cache, logoCache.toString(3).getBytes(ZipEpgClient.ZIP_CHARSET), StandardOpenOption.TRUNCATE_EXISTING, StandardOpenOption.WRITE, StandardOpenOption.CREATE); ScheduleTask.commit(vfs); pool = createThreadPoolExecutor(); //pool.setMaximumPoolSize(5); // Again, we've got memory problems String[] dirtyPrograms = progCache.getDirtyIds(); progCache.markAllClean(); progCache = null; LOG.info(String.format("Identified %d program ids requiring an update!", dirtyPrograms.length)); Collection<String> progIds = new ArrayList<String>(); for (String progId : dirtyPrograms) { progIds.add(progId); if (progIds.size() == grabOpts.getMaxProgChunk()) { pool.execute(new ProgramTask(progIds, vfs, clnt, factory, missingSeriesIds, "programs", null, false)); progIds.clear(); } } if (progIds.size() > 0) pool.execute( new ProgramTask(progIds, vfs, clnt, factory, missingSeriesIds, "programs", null, false)); pool.shutdown(); try { LOG.debug("Waiting for ProgramExecutor to terminate..."); if (pool.awaitTermination(15, TimeUnit.MINUTES)) { LOG.debug("ProgramExecutor: Terminated successfully."); Iterator<String> itr = missingSeriesIds.iterator(); while (itr.hasNext()) { String id = itr.next(); if (cachedSeriesIds.contains(id)) itr.remove(); } if (missingSeriesIds.size() > 0) { LOG.info(String.format("Grabbing %d series info programs!", missingSeriesIds.size())); Set<String> retrySet = new HashSet<>(); try { new ProgramTask(missingSeriesIds, vfs, clnt, factory, missingSeriesIds, "seriesInfo", retrySet, true).run(); } catch (RuntimeException e) { LOG.error("SeriesInfo task failed!", e); Grabber.failedTask = true; } Path seriesInfoData = vfs.getPath(SERIES_INFO_DATA); if (retrySet.size() > 0) { StringBuilder sb = new StringBuilder(); for (String id : retrySet) sb.append(id + "\n"); Files.write(seriesInfoData, sb.toString().getBytes(ZipEpgClient.ZIP_CHARSET), StandardOpenOption.WRITE, StandardOpenOption.TRUNCATE_EXISTING, StandardOpenOption.CREATE); } else if (Files.exists(seriesInfoData)) Files.delete(seriesInfoData); } } else { failedTask = true; LOG.warn("ProgramExecutor: Termination timed out; some tasks probably didn't finish properly!"); } } catch (InterruptedException e) { failedTask = true; LOG.warn("ProgramExecutor: Termination interrupted); some tasks probably didn't finish properly!"); } String userData = clnt.getUserStatus().toJson(); if (failedTask) { LOG.error("One or more tasks failed! Resetting last data refresh timestamp to zero."); SimpleDateFormat fmt = Config.get().getDateTimeFormat(); String exp = fmt.format(new Date(0L)); JSONObject o = Config.get().getObjectMapper().readValue(userData, JSONObject.class); o.put("lastDataUpdate", exp); userData = o.toString(2); } Path p = vfs.getPath(USER_DATA); Files.write(p, userData.getBytes(ZipEpgClient.ZIP_CHARSET), StandardOpenOption.WRITE, StandardOpenOption.TRUNCATE_EXISTING, StandardOpenOption.CREATE); removeIgnoredStations(vfs); } catch (URISyntaxException e1) { throw new RuntimeException(e1); } finally { Runtime rt = Runtime.getRuntime(); LOG.info(String.format("MemStats:%n\tFREE: %s%n\tUSED: %s%n\t MAX: %s", FileUtils.byteCountToDisplaySize(rt.freeMemory()), FileUtils.byteCountToDisplaySize(rt.totalMemory()), FileUtils.byteCountToDisplaySize(rt.maxMemory()))); } }
From source file:org.sonar.batch.bootstrap.DryRunDatabase.java
private void downloadDatabase(String projectKey, File toFile) { try {/* ww w .j a va 2s . co m*/ server.download("/batch_bootstrap/db?project=" + projectKey, toFile); if (LOG.isDebugEnabled()) { LOG.debug("Dry Run database size: {}", FileUtils.byteCountToDisplaySize(FileUtils.sizeOf(toFile))); } } catch (SonarException e) { Throwable rootCause = Throwables.getRootCause(e); if ((rootCause instanceof HttpException) && (((HttpException) rootCause).getResponseCode() == 401)) { throw new SonarException(String.format("You don't have access rights to project [%s]", projectKey), e); } throw e; } }