List of usage examples for org.apache.hadoop.io BytesWritable get
@Deprecated public byte[] get()
From source file:Relevance.java
License:Apache License
public static byte[] getBytes(BytesWritable bw) { byte[] ret = new byte[bw.getSize()]; System.arraycopy(bw.get(), 0, ret, 0, bw.getSize()); return ret;//from w ww . j av a2 s . c o m }
From source file:edu.uci.ics.hivesterix.serde.lazy.LazySerDe.java
License:Apache License
/** * Deserialize a table record to a Lazy struct. *//* w w w . j a va2 s . com*/ @SuppressWarnings("deprecation") @Override public Object deserialize(Writable field) throws SerDeException { if (byteArrayRef == null) { byteArrayRef = new ByteArrayRef(); } if (field instanceof BytesWritable) { BytesWritable b = (BytesWritable) field; if (b.getSize() == 0) { return null; } // For backward-compatibility with hadoop 0.17 byteArrayRef.setData(b.get()); cachedLazyStruct.init(byteArrayRef.getData(), 0, b.getSize()); } else if (field instanceof Text) { Text t = (Text) field; if (t.getLength() == 0) { return null; } byteArrayRef.setData(t.getBytes()); cachedLazyStruct.init(byteArrayRef.getData(), 0, t.getLength()); } else { throw new SerDeException(getClass().toString() + ": expects either BytesWritable or Text object!"); } return cachedLazyStruct; }
From source file:es.pic.astro.hadoop.io.BinaryOutputFormat.java
License:Apache License
/** * create the final out file, and output row by row. After one row is * appended, a configured row separator is appended * * @param jc/*ww w .jav a2 s. co m*/ * the job configuration file * @param outPath * the final output file to be created * @param valueClass * the value class used for create * @param isCompressed * whether the content is compressed or not * @param tableProperties * the tableProperties of this file's corresponding table * @param progress * progress used for status report * @return the RecordWriter */ @Override public RecordWriter getHiveRecordWriter(JobConf jc, Path outPath, Class<? extends Writable> valueClass, boolean isCompressed, Properties tableProperties, Progressable progress) throws IOException { FileSystem fs = outPath.getFileSystem(jc); final OutputStream outStream = Utilities.createCompressedStream(jc, fs.create(outPath, progress), isCompressed); return new RecordWriter() { @Override public void write(Writable r) throws IOException { if (r instanceof Text) { Text tr = (Text) r; outStream.write(tr.getBytes(), 0, tr.getLength()); } else { // DynamicSerDe always writes out BytesWritable BytesWritable bw = (BytesWritable) r; outStream.write(bw.get(), 0, bw.getSize()); } } @Override public void close(boolean abort) throws IOException { outStream.close(); } }; }
From source file:hydrograph.engine.hive.scheme.ParquetWritableUtilsTest.java
License:Apache License
@Test public void itShouldGetBytesWritable() { try {//from w ww. j a v a 2s .c o m ArrayWritable writable = ParquetWritableUtils.createStruct(obj, (StructObjectInspector) io); BytesWritable bw = (BytesWritable) writable.get()[1]; Assert.assertTrue(writable.get()[1] instanceof BytesWritable); Assert.assertEquals("hive", new String(bw.get())); } catch (Exception e) { throw new RuntimeException(e); } }
From source file:org.altlaw.util.hadoop.SeqFilePrinter.java
License:Apache License
/** Runs the process. Keys are printed to standard output; * information about the sequence file is printed to standard * error. *///from w ww.j a va 2s . c om public void execute() throws Exception { Path path = new Path(inputFile); SequenceFile.Reader reader = new SequenceFile.Reader(setup.getLocalFileSystem(), path, setup.getConf()); try { System.out.println("Key type is " + reader.getKeyClassName()); System.out.println("Value type is " + reader.getValueClassName()); if (reader.isCompressed()) { System.err.println("Values are compressed."); if (reader.isBlockCompressed()) { System.err.println("Records are block-compressed."); } System.err.println("Compression type is " + reader.getCompressionCodec().getClass().getName()); } System.out.println(""); Writable key = (Writable) (reader.getKeyClass().newInstance()); Writable val = (Writable) (reader.getValueClass().newInstance()); String value; while (reader.next(key, val)) { System.out.println("============================================================"); System.out.println("KEY:\t" + key.toString()); if (val instanceof BytesWritable) { BytesWritable v = (BytesWritable) val; value = new String(v.get(), 0, v.getSize()); } else { value = val.toString(); } System.out.println("VALUE:\n" + value); } } finally { reader.close(); } }
From source file:org.cloudata.core.tablet.backup.RestoreBinaryMap.java
License:Apache License
@Override public void map(BytesWritable key, BytesWritable value, OutputCollector<Text, Text> output, Reporter reporter) throws IOException { if (confErr != null) { throw confErr; }//from www . j a v a 2 s . c o m ByteArrayInputStream bin = new ByteArrayInputStream(key.get(), 0, key.getSize()); DataInputStream in = new DataInputStream(bin); Row row = new Row(); row.readFields(in); uploader.put(row); count++; if (count % 10000 == 0) { LOG.info(count + " cells restored"); } in.close(); }
From source file:org.cloudata.core.tablet.backup.RestoreBinaryPartitionMap.java
License:Apache License
@Override public void map(BytesWritable key, BytesWritable value, OutputCollector<Text, Text> output, Reporter reporter) throws IOException { ByteArrayInputStream bin = new ByteArrayInputStream(key.get(), 0, key.getSize()); DataInputStream in = new DataInputStream(bin); Row row = new Row(); row.readFields(in);//from ww w . ja v a 2s.com lastRow = row; in.close(); }
From source file:org.commoncrawl.util.CompressedURLFPListV2.java
License:Open Source License
public static void main(String[] args) { // initialize ... final Configuration conf = new Configuration(); conf.addResource("nutch-default.xml"); conf.addResource("nutch-site.xml"); conf.addResource("core-site.xml"); conf.addResource("hdfs-site.xml"); conf.addResource("mapred-site.xml"); BasicConfigurator.configure();/*from w ww. j a va2s .co m*/ CrawlEnvironment.setHadoopConfig(conf); try { FileSystem fs = CrawlEnvironment.getDefaultFileSystem(); Path testFile = new Path("crawl/linkdb/merged1282844121161/linkData/part-00000"); SequenceFile.Reader reader = new SequenceFile.Reader(fs, testFile, conf); URLFPV2 fp = new URLFPV2(); BytesWritable bytes = new BytesWritable(); while (reader.next(fp, bytes)) { if (bytes.getLength() != 0) { DataInputBuffer inputStream = new DataInputBuffer(); inputStream.reset(bytes.get(), bytes.getLength()); CompressedURLFPListV2.Reader listReader = new CompressedURLFPListV2.Reader(inputStream); while (listReader.hasNext()) { URLFPV2 nextFP = listReader.next(); LOG.info("DH:" + nextFP.getDomainHash() + " UH:" + nextFP.getUrlHash()); } } else { LOG.error("ZERO BYTE LIST!"); } } reader.close(); } catch (IOException e) { LOG.error(CCStringUtils.stringifyException(e)); } if (1 == 1) return; validateDuplicateChecking(); // validateReallyBigList(); validateURLFPSerializationRootDomain(); validateURLFPSerializationSingleSubDomain(); validateURLFPSerializationMultiDomain(); validateURLFPFlagSerializationRootDomain(); validateURLFPFlagSerializationMultipleSubDomains(); validateURLFPFlagSerializationOneSubDomain(); }
From source file:org.platform.modules.hadoop.format.output.CustomOutputFormat.java
License:Apache License
/** * create the final out file, and output row by row. After one row is * appended, a configured row separator is appended * /*from w w w . j ava 2s .co m*/ * @param jc * the job configuration file * @param outPath * the final output file to be created * @param valueClass * the value class used for create * @param isCompressed * whether the content is compressed or not * @param tableProperties * the tableProperties of this file's corresponding table * @param progress * progress used for status report * @return the RecordWriter */ @Override public RecordWriter getHiveRecordWriter(JobConf jc, Path outPath, Class<? extends Writable> valueClass, boolean isCompressed, Properties tableProperties, Progressable progress) throws IOException { int rowSeparator = 0; String rowSeparatorString = tableProperties.getProperty(serdeConstants.LINE_DELIM, "\n"); try { rowSeparator = Byte.parseByte(rowSeparatorString); } catch (NumberFormatException e) { rowSeparator = rowSeparatorString.charAt(0); } final int finalRowSeparator = rowSeparator; FileSystem fs = outPath.getFileSystem(jc); final OutputStream outStream = Utilities.createCompressedStream(jc, fs.create(outPath), isCompressed); return new RecordWriter() { @SuppressWarnings("deprecation") public void write(Writable r) throws IOException { if (r instanceof Text) { Text tr = (Text) r; String strReplace = tr.toString().toLowerCase().replace(":", "::"); Text txtReplace = new Text(); txtReplace.set(strReplace); outStream.write(txtReplace.getBytes(), 0, txtReplace.getLength()); // outStream.write(tr.getBytes(), 0, tr.getLength()); outStream.write(finalRowSeparator); } else { // DynamicSerDe always writes out BytesWritable BytesWritable bw = (BytesWritable) r; outStream.write(bw.get(), 0, bw.getSize()); outStream.write(finalRowSeparator); } } public void close(boolean abort) throws IOException { outStream.close(); } }; }
From source file:org.platform.utils.bigdata.hive.CustomOutputFormat.java
License:Apache License
/** * create the final out file, and output row by row. After one row is * appended, a configured row separator is appended * /*from w w w . java2 s . co m*/ * @param jc * the job configuration file * @param outPath * the final output file to be created * @param valueClass * the value class used for create * @param isCompressed * whether the content is compressed or not * @param tableProperties * the tableProperties of this file's corresponding table * @param progress * progress used for status report * @return the RecordWriter */ @Override public RecordWriter getHiveRecordWriter(JobConf jc, Path outPath, Class<? extends Writable> valueClass, boolean isCompressed, Properties tableProperties, Progressable progress) throws IOException { int rowSeparator = 0; String rowSeparatorString = tableProperties.getProperty(serdeConstants.LINE_DELIM, "\n"); try { rowSeparator = Byte.parseByte(rowSeparatorString); } catch (NumberFormatException e) { rowSeparator = rowSeparatorString.charAt(0); } final int finalRowSeparator = rowSeparator; FileSystem fs = outPath.getFileSystem(jc); final OutputStream outStream = Utilities.createCompressedStream(jc, fs.create(outPath), isCompressed); return new RecordWriter() { @SuppressWarnings("deprecation") @Override public void write(Writable r) throws IOException { if (r instanceof Text) { Text tr = (Text) r; String strReplace = tr.toString().replace(":", "::"); Text txtReplace = new Text(); txtReplace.set(strReplace); outStream.write(txtReplace.getBytes(), 0, txtReplace.getLength()); // outStream.write(tr.getBytes(), 0, tr.getLength()); outStream.write(finalRowSeparator); } else { // DynamicSerDe always writes out BytesWritable BytesWritable bw = (BytesWritable) r; outStream.write(bw.get(), 0, bw.getSize()); outStream.write(finalRowSeparator); } } @Override public void close(boolean abort) throws IOException { outStream.close(); } }; }