List of usage examples for org.apache.hadoop.io IOUtils copyBytes
public static void copyBytes(InputStream in, OutputStream out, long count, boolean close) throws IOException
From source file:org.sf.xrime.algorithms.partitions.connected.bi.BCCAlgorithm.java
License:Apache License
@Override public void execute() throws ProcessorExecutionException { try {/*from w ww . ja va2s.c o m*/ if (getSource().getPaths() == null || getSource().getPaths().size() == 0 || getDestination().getPaths() == null || getDestination().getPaths().size() == 0) { throw new ProcessorExecutionException("No input and/or output paths specified."); } // The prefix used by temp directories which store intermediate results of each steps. String temp_dir_prefix = getDestination().getPath().getParent().toString() + "/bcc_" + getDestination().getPath().getName() + "_"; // Create the temporary directory manager. SequenceTempDirMgr dirMgr = new SequenceTempDirMgr(temp_dir_prefix, context); // Sequence number begins with zero. dirMgr.setSeqNum(0); Path tmpDir; // 1. Transform input from outgoing adjacency vertexes lists to AdjSetVertex. System.out.println("++++++>" + dirMgr.getSeqNum() + ": Transform input to AdjSetVertex"); Transformer transformer = new OutAdjVertex2AdjSetVertexTransformer(); transformer.setConf(context); transformer.setSrcPath(getSource().getPath()); // Generate temporary directory. tmpDir = dirMgr.getTempDir(); // And use it as the destination directory. transformer.setDestPath(tmpDir); transformer.setMapperNum(getMapperNum()); transformer.setReducerNum(getReducerNum()); transformer.execute(); // 2. Add initial labels (empty) to AdjSetVertex. System.out.println("++++++>" + dirMgr.getSeqNum() + ": Transform input to LabeledAdjSetVertex"); Vertex2LabeledTransformer l_transformer = new Vertex2LabeledTransformer(); l_transformer.setConf(context); l_transformer.setSrcPath(tmpDir); // Generate temporary directory. tmpDir = dirMgr.getTempDir(); // And use it as the destination directory. l_transformer.setDestPath(tmpDir); l_transformer.setMapperNum(getMapperNum()); l_transformer.setReducerNum(getReducerNum()); // Don't forget this. l_transformer.setOutputValueClass(LabeledAdjSetVertex.class); l_transformer.execute(); Graph src; Graph dest; // 3. Generate the spanning tree. // 3.1 Choose root for the spanning tree. // Remember the path. Path path_to_remember = tmpDir; System.out.println("++++++>" + dirMgr.getSeqNum() + ": SpanningTreeRootChoose"); src = new Graph(Graph.defaultGraph()); src.setPath(tmpDir); dest = new Graph(Graph.defaultGraph()); tmpDir = dirMgr.getTempDir(); dest.setPath(tmpDir); GraphAlgorithm choose_root = new SpanningTreeRootChoose(); choose_root.setConf(context); choose_root.setSource(src); choose_root.setDestination(dest); choose_root.setMapperNum(getMapperNum()); choose_root.setReducerNum(getReducerNum()); choose_root.execute(); // 3.2 Read the id of the chosen root vertex. // Determine the file path. Path the_file = new Path(tmpDir.toString() + "/part-00000"); // Get the file system object. FileSystem client = FileSystem.get(context); // Check for existence. if (!client.exists(the_file)) { throw new ProcessorExecutionException("Did not find the chosen vertex in " + the_file.toString()); } // Open the file for read. FSDataInputStream input_stream = client.open(the_file); ByteArrayOutputStream output_stream = new ByteArrayOutputStream(); // Copy to output. IOUtils.copyBytes(input_stream, output_stream, context, false); // Get the line. String the_line = output_stream.toString(); String root_vertex_id = the_line.substring(SpanningTreeRootChoose.SPANNING_TREE_ROOT.length()).trim(); // Close the input and output stream. input_stream.close(); output_stream.close(); System.out.println("++++++> Chosen the root of spanning tree = " + root_vertex_id); // 3.3 Generate the spanning tree root at chosen vertex. while (true) { // 3.3.1 Spanning. System.out.println("++++++>" + dirMgr.getSeqNum() + " Generate the spanning tree rooted at : (" + root_vertex_id + ") from " + tmpDir); src = new Graph(Graph.defaultGraph()); src.setPath(path_to_remember); tmpDir = dirMgr.getTempDir(); dest = new Graph(Graph.defaultGraph()); dest.setPath(tmpDir); // Remember the directory again. path_to_remember = tmpDir; GraphAlgorithm spanning = new SpanningTreeGenerate(); spanning.setConf(context); spanning.setSource(src); spanning.setDestination(dest); spanning.setMapperNum(getMapperNum()); spanning.setReducerNum(getReducerNum()); spanning.setParameter(ConstantLabels.ROOT_ID, root_vertex_id); spanning.execute(); // 3.3.2 Check for convergence. System.out.println("++++++>" + dirMgr.getSeqNum() + " Test spanning convergence"); src = new Graph(Graph.defaultGraph()); src.setPath(tmpDir); tmpDir = dirMgr.getTempDir(); dest = new Graph(Graph.defaultGraph()); dest.setPath(tmpDir); GraphAlgorithm conv_tester = new SpanningConvergenceTest(); conv_tester.setConf(context); conv_tester.setSource(src); conv_tester.setDestination(dest); conv_tester.setMapperNum(getMapperNum()); conv_tester.setReducerNum(getReducerNum()); conv_tester.execute(); // 3.3.3 Check test result. long vertexes_out_of_tree = MRConsoleReader.getMapOutputRecordNum(conv_tester.getFinalStatus()); System.out.println("++++++> number of vertexes out of the spanning tree = " + vertexes_out_of_tree); if (vertexes_out_of_tree == 0) { // Converged. break; } } // Here we should have already got a spanning tree. // 4. From the spanning tree to sets of edges, where each set corresponds to a circle in the graph. System.out.println("++++++> From spanning tree to sets of edges"); src = new Graph(Graph.defaultGraph()); src.setPath(path_to_remember); tmpDir = dirMgr.getTempDir(); dest = new Graph(Graph.defaultGraph()); dest.setPath(tmpDir); GraphAlgorithm tree2set = new Tree2EdgeSet(); tree2set.setConf(context); tree2set.setSource(src); tree2set.setDestination(dest); tree2set.setMapperNum(getMapperNum()); tree2set.setReducerNum(getReducerNum()); tree2set.execute(); long map_input_records_num = -1; long map_output_records_num = -2; Stack<Path> expanding_stack = new Stack<Path>(); // 5. loop of MinorJoin + Join. do { // 5.1 MinorJoin. System.out.println("++++++>" + dirMgr.getSeqNum() + ": EdgeSetMinorJoin"); GraphAlgorithm minorjoin = new EdgeSetMinorJoin(); minorjoin.setConf(context); src = new Graph(Graph.defaultGraph()); // Use the output directory of last step as the input directory of this step. src.setPath(tmpDir); dest = new Graph(Graph.defaultGraph()); // Generate a new temporary directory. tmpDir = dirMgr.getTempDir(); dest.setPath(tmpDir); minorjoin.setSource(src); minorjoin.setDestination(dest); minorjoin.setMapperNum(getMapperNum()); minorjoin.setReducerNum(getReducerNum()); minorjoin.execute(); // Push the minor join result to the stack. This directory will be used in the // future to determine the partition on edge set. expanding_stack.push(tmpDir); // 5.2 Join. System.out.println("++++++>" + dirMgr.getSeqNum() + ": EdgeSetJoin"); GraphAlgorithm join = new EdgeSetJoin(); join.setConf(context); src = new Graph(Graph.defaultGraph()); src.setPath(tmpDir); dest = new Graph(Graph.defaultGraph()); tmpDir = dirMgr.getTempDir(); dest.setPath(tmpDir); join.setSource(src); join.setDestination(dest); join.setMapperNum(getMapperNum()); join.setReducerNum(getReducerNum()); join.execute(); // 5.3 Check for convergence. map_input_records_num = MRConsoleReader.getMapInputRecordNum(join.getFinalStatus()); map_output_records_num = MRConsoleReader.getMapOutputRecordNum(join.getFinalStatus()); System.out.println("++++++> map in/out : " + map_input_records_num + "/" + map_output_records_num); } while (map_input_records_num != map_output_records_num); // 6. Expand, i.e., propagate label to determine the partition on edge set. while (expanding_stack.size() > 0) { // 6.1 Expand System.out.println("++++++>" + dirMgr.getSeqNum() + ": EdgeSetExpand"); GraphAlgorithm expand = new EdgeSetExpand(); expand.setConf(context); src = new Graph(Graph.defaultGraph()); src.addPath(expanding_stack.pop()); src.addPath(tmpDir); dest = new Graph(Graph.defaultGraph()); tmpDir = dirMgr.getTempDir(); dest.setPath(tmpDir); expand.setSource(src); expand.setDestination(dest); expand.setMapperNum(getMapperNum()); expand.setReducerNum(getReducerNum()); expand.execute(); // 6.2 MinorExpand System.out.println("++++++>" + dirMgr.getSeqNum() + ": EdgeSetMinorExpand"); GraphAlgorithm minorexpand = new EdgeSetMinorExpand(); minorexpand.setConf(context); src = new Graph(Graph.defaultGraph()); src.setPath(tmpDir); dest = new Graph(Graph.defaultGraph()); tmpDir = dirMgr.getTempDir(); dest.setPath(tmpDir); minorexpand.setSource(src); minorexpand.setDestination(dest); minorexpand.setMapperNum(getMapperNum()); minorexpand.setReducerNum(getReducerNum()); minorexpand.execute(); } // 7. Summarize sets. System.out.println("++++++>" + dirMgr.getSeqNum() + ": EdgeSetSummarize"); GraphAlgorithm summarize = new EdgeSetSummarize(); summarize.setConf(context); src = new Graph(Graph.defaultGraph()); src.setPath(tmpDir); dest = new Graph(Graph.defaultGraph()); dest.setPath(getDestination().getPath()); summarize.setSource(src); summarize.setDestination(dest); summarize.setMapperNum(getMapperNum()); summarize.setReducerNum(getReducerNum()); summarize.execute(); // Delete all temporary directories. dirMgr.deleteAll(); } catch (IOException e) { throw new ProcessorExecutionException(e); } catch (IllegalAccessException e) { throw new ProcessorExecutionException(e); } }
From source file:org.sf.xrime.algorithms.partitions.connected.strongly.SCCAlgorithm.java
License:Apache License
/** * Choose a pivot vertex from the graph. * @return return the id of the chosen vertex. * @throws ProcessorExecutionException/*w w w .j a v a2 s .c om*/ */ private String choosePivotVertex() throws ProcessorExecutionException { String result = null; Graph src; Graph dest; Path tmpDir; // 3.1. Choose a pivot vertex. System.out.println("##########>" + _dirMgr.getSeqNum() + " Choose the pivot vertex"); src = new Graph(Graph.defaultGraph()); src.setPath(_curr_path); dest = new Graph(Graph.defaultGraph()); try { tmpDir = _dirMgr.getTempDir(); } catch (IOException e) { throw new ProcessorExecutionException(e); } dest.setPath(tmpDir); GraphAlgorithm choose_pivot = new PivotChoose(); choose_pivot.setConf(context); choose_pivot.setSource(src); choose_pivot.setDestination(dest); choose_pivot.setMapperNum(getMapperNum()); choose_pivot.setReducerNum(getReducerNum()); choose_pivot.execute(); // 3.2 Read the vertex. try { // Determine the file path. Path the_file = new Path(tmpDir.toString() + "/part-00000"); // Get the file system object. FileSystem client = FileSystem.get(context); // Check for existence. if (!client.exists(the_file)) { throw new ProcessorExecutionException("Did not find the chosen vertex in " + the_file.toString()); } // Open the file for read. FSDataInputStream input_stream = client.open(the_file); ByteArrayOutputStream output_stream = new ByteArrayOutputStream(); // Copy to output. IOUtils.copyBytes(input_stream, output_stream, context, false); // Get the line. String the_line = output_stream.toString(); result = the_line.substring(PivotChoose.KEY_PIVOT.length()).trim(); // Close the input and output stream. input_stream.close(); output_stream.close(); System.out.println("##########> Chosen pivot id = " + result); } catch (IOException e) { throw new ProcessorExecutionException(e); } return result; }
From source file:org.smartfrog.services.hadoop.operations.utils.DfsUtils.java
License:Open Source License
/** * Copy a file/* www. ja v a 2 s. c o m*/ * * @param srcFS source filesystem * @param src source path * @param dstFS destination filesystem * @param dst destination path * @param overwrite overwrite * @param blocksize block size * @throws SmartFrogRuntimeException for any failure */ public static void copyFile(FileSystem srcFS, Path src, FileSystem dstFS, Path dst, boolean overwrite, int blocksize) throws SmartFrogRuntimeException { assertNotDependent(srcFS, src, dstFS, dst); FileStatus status; URI fsuri = srcFS.getUri(); try { status = srcFS.getFileStatus(src); } catch (FileNotFoundException fe) { throw new SmartFrogRuntimeException(ERROR_MISSING_SOURCE_FILE + src + " in " + fsuri, fe); } catch (IOException e) { throw new SmartFrogRuntimeException(ERROR_NO_STAT + src + " in " + fsuri + " : " + e, e); } if (status.isDir()) { throw new SmartFrogRuntimeException(ERROR_NO_DIRECTORY_COPY + src + " in " + fsuri); } InputStream in = null; OutputStream out = null; try { in = srcFS.open(src); out = dstFS.create(dst, overwrite); } catch (IOException e) { //close the input stream if it is not already in there org.smartfrog.services.filesystem.FileSystem.close(in); org.smartfrog.services.filesystem.FileSystem.close(out); } try { IOUtils.copyBytes(in, out, blocksize, true); } catch (IOException e) { throw new SmartFrogRuntimeException( ERROR_COPY_FAILED + src + " in " + fsuri + " to " + dst + " in " + dstFS.getUri() + " : " + e, e); } }
From source file:org.terrier.structures.indexing.singlepass.hadoop.Inv2DirectMultiReduce.java
License:Mozilla Public License
@SuppressWarnings("unchecked") private static void finish(IndexOnDisk index, Configuration conf, String sourceStructureName, String targetStructureName, boolean[] blocksfields, final int numberOfReducers, final int numberOfReduceTaskLimits) throws IOException, Exception { Iterator<DocumentIndexEntry> diis = (Iterator<DocumentIndexEntry>) index .getIndexStructureInputStream("document"); DocumentIndexBuilder dios = new DocumentIndexBuilder(index, "document-df"); BitIndexPointer pointer = new SimpleBitIndexPointer(); final boolean blocks = blocksfields[0]; final boolean fields = blocksfields[1]; if (numberOfReducers == 1) { String outputPrefix = "-0"; DataInputStream currentStream = new DataInputStream(Files.openFileStream(((IndexOnDisk) index).getPath() + ApplicationSetup.FILE_SEPARATOR + ((IndexOnDisk) index).getPrefix() + "." + targetStructureName + outputPrefix + ".pointers")); logger.info("Adding pointers to the document index"); while (diis.hasNext()) { DocumentIndexEntry die = diis.next(); pointer.readFields(currentStream); DocumentIndexEntry newDIentry = fields ? new FieldDocumentIndexEntry(die) : new BasicDocumentIndexEntry(die); newDIentry.setOffset(pointer); newDIentry.setNumberOfEntries(pointer.getNumberOfEntries()); dios.addEntryToBuffer(newDIentry); }//from w w w . jav a 2 s . c om IndexUtil.close(diis); logger.info("Renaming reducer output as direct file"); Files.delete(((IndexOnDisk) index).getPath() + ApplicationSetup.FILE_SEPARATOR + ((IndexOnDisk) index).getPrefix() + "." + targetStructureName + BitIn.USUAL_EXTENSION); Files.rename( ((IndexOnDisk) index).getPath() + ApplicationSetup.FILE_SEPARATOR + ((IndexOnDisk) index).getPrefix() + "." + targetStructureName + outputPrefix + BitIn.USUAL_EXTENSION, ((IndexOnDisk) index).getPath() + ApplicationSetup.FILE_SEPARATOR + ((IndexOnDisk) index).getPrefix() + "." + targetStructureName + BitIn.USUAL_EXTENSION); currentStream.close(); Files.delete(((IndexOnDisk) index).getPath() + ApplicationSetup.FILE_SEPARATOR + ((IndexOnDisk) index).getPrefix() + "." + targetStructureName + outputPrefix + ".pointers"); } else if (numberOfReducers <= numberOfReduceTaskLimits) { logger.info("Merging direct index pointers from " + numberOfReducers + " reducers"); final int partitionSize = (int) Math.ceil( (double) (index.getCollectionStatistics().getNumberOfDocuments()) / (double) numberOfReducers); for (byte reduce = 0; reduce < numberOfReducers; reduce++) { logger.info("Merging in pointers from reduce task " + reduce); String outputPrefix = "-" + reduce; DataInputStream currentStream = new DataInputStream( Files.openFileStream(((IndexOnDisk) index).getPath() + ApplicationSetup.FILE_SEPARATOR + ((IndexOnDisk) index).getPrefix() + "." + targetStructureName + outputPrefix + ".pointers")); for (int docOffset = 0; docOffset < partitionSize && diis.hasNext(); docOffset++) { DocumentIndexEntry die = diis.next(); pointer.readFields(currentStream); DocumentIndexEntry newDIentry = fields ? new FieldDocumentIndexEntry(die) : new BasicDocumentIndexEntry(die); newDIentry.setOffset(pointer); newDIentry.setFileNumber(reduce); newDIentry.setNumberOfEntries(pointer.getNumberOfEntries()); dios.addEntryToBuffer(newDIentry); } currentStream.close(); Files.delete(((IndexOnDisk) index).getPath() + ApplicationSetup.FILE_SEPARATOR + ((IndexOnDisk) index).getPrefix() + "." + targetStructureName + outputPrefix + ".pointers"); logger.info("Renaming direct file part for reduce task " + reduce); String sourcePartDFfilename = ((IndexOnDisk) index).getPath() + ApplicationSetup.FILE_SEPARATOR + ((IndexOnDisk) index).getPrefix() + "." + targetStructureName + outputPrefix + BitIn.USUAL_EXTENSION; String destPartDFfilename = ((IndexOnDisk) index).getPath() + ApplicationSetup.FILE_SEPARATOR + ((IndexOnDisk) index).getPrefix() + "." + targetStructureName + BitIn.USUAL_EXTENSION + reduce; Files.rename(sourcePartDFfilename, destPartDFfilename); } index.setIndexProperty("index." + targetStructureName + ".data-files", "" + numberOfReducers); index.flush(); IndexUtil.close(diis); } else { logger.info("Merging direct index output from " + numberOfReducers + " reducers"); final int partitionSize = (int) Math.ceil( (double) (index.getCollectionStatistics().getNumberOfDocuments()) / (double) numberOfReducers); final OutputStream DFout = Files.writeFileStream(((IndexOnDisk) index).getPath() + ApplicationSetup.FILE_SEPARATOR + ((IndexOnDisk) index).getPrefix() + "." + targetStructureName + BitIn.USUAL_EXTENSION); long finalFileOffset = 0; for (int reduce = 0; reduce < numberOfReducers; reduce++) { logger.info("Copying document index part for reduce task " + reduce); String outputPrefix = "-" + reduce; DataInputStream currentStream = new DataInputStream( Files.openFileStream(((IndexOnDisk) index).getPath() + ApplicationSetup.FILE_SEPARATOR + ((IndexOnDisk) index).getPrefix() + "." + targetStructureName + outputPrefix + ".pointers")); for (int docOffset = 0; docOffset < partitionSize && diis.hasNext(); docOffset++) { DocumentIndexEntry die = diis.next(); pointer.readFields(currentStream); DocumentIndexEntry newDIentry = fields ? new FieldDocumentIndexEntry(die) : new BasicDocumentIndexEntry(die); newDIentry.setOffset(finalFileOffset + pointer.getOffset(), pointer.getOffsetBits()); newDIentry.setNumberOfEntries(pointer.getNumberOfEntries()); dios.addEntryToBuffer(newDIentry); } currentStream.close(); Files.delete(((IndexOnDisk) index).getPath() + ApplicationSetup.FILE_SEPARATOR + ((IndexOnDisk) index).getPrefix() + "." + targetStructureName + outputPrefix + ".pointers"); logger.info("Copying direct file part for reduce task " + reduce); String partDFfilename = ((IndexOnDisk) index).getPath() + ApplicationSetup.FILE_SEPARATOR + ((IndexOnDisk) index).getPrefix() + "." + targetStructureName + outputPrefix + BitIn.USUAL_EXTENSION; InputStream partDF = Files.openFileStream(partDFfilename); finalFileOffset += Files.length(partDFfilename); IOUtils.copyBytes(partDF, DFout, conf, false); partDF.close(); Files.delete(partDFfilename); } IndexUtil.close(diis); DFout.close(); } dios.close(); Files.copyFile( ((IndexOnDisk) index).getPath() + ApplicationSetup.FILE_SEPARATOR + ((IndexOnDisk) index).getPrefix() + "." + "document.fsarrayfile", ((IndexOnDisk) index).getPath() + ApplicationSetup.FILE_SEPARATOR + ((IndexOnDisk) index).getPrefix() + "." + "document-backup.fsarrayfile"); IndexUtil.renameIndexStructure(index, "document-df", "document"); if (fields) { index.addIndexStructure("document-factory", FieldDocumentIndexEntry.Factory.class.getName(), "java.lang.String", "${index.direct.fields.count}"); } else { index.addIndexStructure("document-factory", BasicDocumentIndexEntry.Factory.class.getName(), "", ""); } String directIndexClass = blocks ? BlockDirectIndex.class.getName() : DirectIndex.class.getName(); String directIndexInputStreamClass = blocks ? BlockDirectIndexInputStream.class.getName() : DirectIndexInputStream.class.getName(); String postingIterator; if (blocks) { postingIterator = fields ? BlockFieldIterablePosting.class.getName() : BlockIterablePosting.class.getName(); } else { postingIterator = fields ? FieldIterablePosting.class.getName() : BasicIterablePosting.class.getName(); } if (fields) { index.setIndexProperty("index." + targetStructureName + ".fields.count", index.getIndexProperty("index." + sourceStructureName + ".fields.count", "0")); index.setIndexProperty("index." + targetStructureName + ".fields.names", index.getIndexProperty("index." + sourceStructureName + ".fields.names", "")); } index.addIndexStructure(targetStructureName, directIndexClass, "org.terrier.structures.IndexOnDisk,java.lang.String,java.lang.Class", "index,structureName," + postingIterator); index.addIndexStructureInputStream(targetStructureName, directIndexInputStreamClass, "org.terrier.structures.IndexOnDisk,java.lang.String,java.lang.Class", "index,structureName," + postingIterator); index.flush(); }
From source file:sharedsidefunctions.DiamondAlignment.java
License:Apache License
public static void align(String diamond, String localDB, String key, String[] arguments, Configuration conf) throws IOException, InterruptedException { ArrayList<String> argumentsList = new ArrayList<String>(Arrays.asList(arguments)); // String qda[] = {"-q", "/tmp/" + key, "-d", localDB, "-a", "/tmp/" + key}; String qda[] = { "-q", "/tmp/" + key, "-d", localDB }; argumentsList.add(0, diamond);//from ww w. j av a 2s .c o m argumentsList.addAll(new ArrayList<String>(Arrays.asList(qda))); // Process p1 = Runtime.getRuntime().exec(argumentsList.toArray(new String[argumentsList.size()])); // p1.waitFor(); String hadoopUser = UserGroupInformation.getCurrentUser().getUserName(); Process p = Runtime.getRuntime().exec(argumentsList.toArray(new String[argumentsList.size()])); FileSystem fs = FileSystem.get(conf); //process stream copied to HDFS stream InputStream in = p.getInputStream(); FSDataOutputStream out = fs.create(new Path("/user/" + hadoopUser + "/Hamond/output/" + key + ".out")); IOUtils.copyBytes(in, out, 4096, true); p.waitFor(); }
From source file:sharedsidefunctions.DiamondView.java
License:Apache License
public static void view(String diamond, String key, Configuration conf) throws IOException, InterruptedException { String hadoopUser = UserGroupInformation.getCurrentUser().getUserName(); String view[] = { diamond, "view", "-a", "/tmp/" + key }; Process p2 = Runtime.getRuntime().exec(view); FileSystem fs = FileSystem.get(conf); //process stream copied to HDFS stream InputStream in = p2.getInputStream(); FSDataOutputStream out = fs.create(new Path("/user/" + hadoopUser + "/Hamond/output/" + key + ".out")); IOUtils.copyBytes(in, out, 4096, true); p2.waitFor();//from w w w . ja va2 s. co m }
From source file:tv.icntv.log.crawl.TestWriterHdfs.java
License:Apache License
public static void test1() throws ClassNotFoundException, IOException { String codecClassName = "org.apache.hadoop.io.compression.GzipCodec"; Class<?> codecClass = Class.forName(codecClassName); Configuration config = new Configuration(); CompressionCodec codec = (CompressionCodec) ReflectionUtils.newInstance(codecClass, config); CompressionOutputStream out = codec.createOutputStream(System.out); IOUtils.copyBytes(new FileInputStream(new File("d:\\11.txt")), System.out, 4096, false); out.close();/*from w w w . j a v a2s .com*/ }
From source file:tv.icntv.log.crawl.TestWriterHdfs.java
License:Apache License
public static void test2() throws ClassNotFoundException, IOException { String codecClassName = "org.apache.hadoop.io.compression.GzipCodec"; Class<?> codecClass = Class.forName(codecClassName); Configuration config = new Configuration(); CompressionCodec codec = (CompressionCodec) ReflectionUtils.newInstance(codecClass, config); CompressionOutputStream out = codec.createOutputStream(System.out); IOUtils.copyBytes(new FileInputStream(new File("d:\\11.txt")), System.out, 4096, false); out.close();/*from www . jav a 2 s . c o m*/ }