List of usage examples for java.util.zip ZipInputStream ZipInputStream
public ZipInputStream(InputStream in, Charset charset)
From source file:io.gromit.geolite2.geonames.SubdivisionFinder.java
/** * Read level two.//from www. j av a 2 s . c om * * @param subdivisionTwoLocationUrl the subdivision two location url * @return the subdivision finder */ public SubdivisionFinder readLevelTwo(String subdivisionTwoLocationUrl) { ZipInputStream zipis = null; try { zipis = new ZipInputStream(new URL(subdivisionTwoLocationUrl).openStream(), Charset.forName("UTF-8")); ZipEntry zipEntry = zipis.getNextEntry(); logger.info("reading " + zipEntry.getName()); if (crc2 == zipEntry.getCrc()) { logger.info("skipp, same CRC"); return this; } CsvParserSettings settings = new CsvParserSettings(); settings.setSkipEmptyLines(true); settings.trimValues(true); CsvFormat format = new CsvFormat(); format.setDelimiter('\t'); format.setLineSeparator("\n"); format.setCharToEscapeQuoteEscaping('\0'); format.setQuote('\0'); settings.setFormat(format); CsvParser parser = new CsvParser(settings); List<String[]> lines = parser.parseAll(new InputStreamReader(zipis, "UTF-8")); for (String[] entry : lines) { Subdivision subdivision = new Subdivision(); subdivision.setId(entry[0]); subdivision.setName(entry[1]); subdivision.setGeonameId(NumberUtils.toInt(entry[2])); idTowMap.put(subdivision.getId(), subdivision); geonameIdMap.put(subdivision.getGeonameId(), subdivision); } logger.info("loaded " + lines.size() + " subdivisions level 2"); } catch (Exception e) { logger.error(e.getMessage(), e); } finally { try { zipis.close(); } catch (Exception e) { } ; } return this; }
From source file:com.ery.hadoop.mrddx.file.LineRecordReader.java
void openFile() throws IOException { start = split.getStart();/*from www .java2s. co m*/ end = start + split.getLength(); final Path file = split.getPath(); LOG.info("split.getFileIndex=" + split.getFileIndex() + ",file.path=" + file.toString() + " fileEncodeing=" + fileEncodeing + " " + split.getStart() + ":" + split.getLength()); // open the file and seek to the start of the split FileSystem fs = file.getFileSystem(job); FSDataInputStream fileIn = fs.open(split.getPath()); compressionCodecs = new CompressionCodecFactory(job); codec = compressionCodecs.getCodec(file); if (file.getName().endsWith(".zip")) { LOG.info("use ZipInputStream read file " + split.getPath()); ZipInputStream zin = new ZipInputStream(fileIn, Charset.forName(fileEncodeing)); in = new LineReader(zin, job); filePosition = fileIn; codec = new GzipCodec(); return; } if (isCompressedInput()) { decompressor = CodecPool.getDecompressor(codec); if (codec instanceof SplittableCompressionCodec) { final SplitCompressionInputStream cIn = ((SplittableCompressionCodec) codec).createInputStream( fileIn, decompressor, start, end, SplittableCompressionCodec.READ_MODE.BYBLOCK); // tar.gzTarInputStream // new TarInputStream(codec.createInputStream(fileIn, // decompressor) String filename = file.getName(); if (filename.endsWith(".tar.gz")) { in = new LineReader(new TarInputStream(cIn), job); } else { in = new LineReader(cIn, job); } start = cIn.getAdjustedStart(); end = cIn.getAdjustedEnd(); filePosition = cIn; // take pos from compressed stream } else { String filename = file.getName(); if (filename.endsWith(".tar.gz") || filename.endsWith(".tar")) { in = new LineReader(new TarInputStream(codec.createInputStream(fileIn, decompressor)), job); } else { in = new LineReader(codec.createInputStream(fileIn, decompressor), job); } filePosition = fileIn; } } else { fileIn.seek(start); String filename = file.getName(); if (filename.endsWith(".tar")) { in = new LineReader(new TarInputStream(fileIn), job); } else { in = new LineReader(fileIn, job); } filePosition = fileIn; } // If this is not the first split, we always throw away first record // because we always (except the last split) read one extra line in // next() method. if (start != 0) { start += in.readLine(new Text(), 0, maxBytesToConsume(start)); } this.pos = start; }
From source file:io.sledge.core.impl.extractor.SledgeApplicationPackageExtractor.java
private ZipInputStream getNewUtf8ZipInputStream(ApplicationPackage appPackage) { return new ZipInputStream(new BufferedInputStream(appPackage.getPackageFile()), Charset.forName("UTF-8")); }
From source file:cpcc.vvrte.services.db.DownloadServiceTest.java
@Test public void shouldGetAllVirtualVehicles() throws IOException { byte[] actual = sut.getAllVirtualVehicles(); ByteArrayInputStream bis = new ByteArrayInputStream(actual); ZipInputStream zis = new ZipInputStream(bis, Charset.forName("UTF-8")); ZipEntry entry = zis.getNextEntry(); assertThat(entry).isNotNull();/* ww w.j av a 2 s . c o m*/ assertThat(entry.getName()).isEqualTo(VV_ONE_UUID + "/"); assertThat(entry.getTime()).isEqualTo(CURRENT_TIME); entry = zis.getNextEntry(); assertThat(entry).isNotNull(); assertThat(entry.getName()).isEqualTo(VV_ONE_UUID + "/vv.properties"); assertThat(entry.getTime()).isEqualTo(CURRENT_TIME); Properties actualProps = new Properties(); actualProps.load(zis); assertThat(actualProps.getProperty("api-version")).isEqualTo(Integer.toString(VV_ONE_API_VERSION)); assertThat(actualProps.getProperty("end-time")).isEqualTo(sdf.format(VV_ONE_END_TIME)); assertThat(actualProps.getProperty("name")).isEqualTo(VV_ONE_NAME); assertThat(actualProps.getProperty("start-time")).isEqualTo(VV_ONE_START_TIME); assertThat(actualProps.getProperty("state")).isEqualTo(VV_ONE_STATE.name()); entry = zis.getNextEntry(); assertThat(entry).isNotNull(); assertThat(entry.getName()).isEqualTo(VV_ONE_UUID + "/code.js"); assertThat(entry.getTime()).isEqualTo(CURRENT_TIME); assertThat(IOUtils.toString(zis)).isEqualTo(VV_ONE_CODE); entry = zis.getNextEntry(); assertThat(entry).isNotNull(); assertThat(entry.getName()).isEqualTo(VV_ONE_UUID + "/state-info.txt"); assertThat(entry.getTime()).isEqualTo(CURRENT_TIME); assertThat(IOUtils.toString(zis)).isEqualTo(VV_ONE_STATE_INFO); entry = zis.getNextEntry(); assertThat(entry).isNotNull(); assertThat(entry.getName()).isEqualTo(VV_ONE_UUID + "/storage/"); assertThat(entry.getTime()).isEqualTo(CURRENT_TIME); entry = zis.getNextEntry(); assertThat(entry).isNotNull(); assertThat(entry.getName()).isEqualTo(VV_TWO_UUID + "/"); assertThat(entry.getTime()).isEqualTo(CURRENT_TIME); entry = zis.getNextEntry(); assertThat(entry).isNotNull(); assertThat(entry.getName()).isEqualTo(VV_TWO_UUID + "/vv.properties"); actualProps = new Properties(); actualProps.load(zis); assertThat(actualProps.getProperty("api-version")).isEqualTo(Integer.toString(VV_TWO_API_VERSION)); assertThat(actualProps.getProperty("end-time")).isEqualTo(VV_TWO_END_TIME_STR); assertThat(actualProps.getProperty("name")).isEqualTo(VV_TWO_NAME); assertThat(actualProps.getProperty("start-time")).isEqualTo(sdf.format(VV_TWO_START_TIME)); assertThat(actualProps.getProperty("state")).isEqualTo(VV_TWO_STATE.name()); entry = zis.getNextEntry(); assertThat(entry).isNotNull(); assertThat(entry.getName()).isEqualTo(VV_TWO_UUID + "/code.js"); assertThat(entry.getTime()).isEqualTo(CURRENT_TIME); assertThat(IOUtils.toString(zis)).isEqualTo(VV_TWO_CODE); entry = zis.getNextEntry(); assertThat(entry).isNotNull(); assertThat(entry.getName()).isEqualTo(VV_TWO_UUID + "/continuation.dat"); assertThat(entry.getTime()).isEqualTo(CURRENT_TIME); assertThat(IOUtils.toByteArray(zis)).isEqualTo(VV_TWO_CONTINUATION); entry = zis.getNextEntry(); assertThat(entry).isNotNull(); assertThat(entry.getName()).isEqualTo(VV_TWO_UUID + "/storage/"); assertThat(entry.getTime()).isEqualTo(CURRENT_TIME); entry = zis.getNextEntry(); assertThat(entry).isNotNull(); assertThat(entry.getName()).isEqualTo(VV_TWO_UUID + "/storage/itemOne1.json"); assertThat(entry.getTime()).isEqualTo(ITEM_ONE_1_TIME.getTime()); assertThat(IOUtils.toByteArray(zis)).isEqualTo(ITEM_ONE_1_CONTENT); }