List of usage examples for org.apache.hadoop.io.serializer Serializer close
void close() throws IOException;
Close the underlying output stream and clear up any resources.
From source file:com.google.cloud.bigtable.beam.sequencefiles.HadoopSerializationCoder.java
License:Open Source License
/** {@inheritDoc} */ @Override/*from ww w .j a v a 2 s.co m*/ public void encode(T value, OutputStream outStream) throws CoderException, IOException { Serializer<T> serializer = this.serialization.getSerializer(type); serializer.open(new UncloseableOutputStream(outStream)); try { serializer.serialize(value); } finally { serializer.close(); } }
From source file:cz.seznam.euphoria.hadoop.utils.Cloner.java
License:Apache License
/** * Help method retrieving a cloner for given class type from the * given configuration./*from w w w .ja v a 2 s .c om*/ * * @param <T> the type of objects the resulting cloner will be able to handle * * @param what the class for which to retrieve a cloner * @param conf the hadoop configuration defining the serializer/deserializer * to utilize for cloning * * @return a cloner instance able to clone objects of the specified type */ static <T> Cloner<T> get(Class<T> what, Configuration conf) { SerializationFactory factory = new SerializationFactory(conf); Serialization<T> serialization = factory.getSerialization(what); if (serialization == null) { // FIXME: if we cannot (de)serialize just do not clone return t -> t; } Deserializer<T> deserializer = serialization.getDeserializer(what); Serializer<T> serializer = serialization.getSerializer(what); return (T elem) -> { try { ByteArrayOutputStream baos = new ByteArrayOutputStream(); serializer.open(baos); serializer.serialize(elem); serializer.close(); byte[] serialized = baos.toByteArray(); ByteArrayInputStream bais = new ByteArrayInputStream(serialized); deserializer.open(bais); T deserialized = deserializer.deserialize(null); deserializer.close(); return deserialized; } catch (IOException ex) { throw new RuntimeException(ex); } }; }
From source file:org.apache.apex.examples.mroperator.MapOperator.java
License:Apache License
@SuppressWarnings("rawtypes") @Override//from w ww. jav a 2 s. com public Collection<Partition<MapOperator<K1, V1, K2, V2>>> definePartitions( Collection<Partition<MapOperator<K1, V1, K2, V2>>> partitions, PartitioningContext context) { int tempPartitionCount = partitionCount; Collection c = partitions; Collection<Partition<MapOperator<K1, V1, K2, V2>>> operatorPartitions = c; Partition<MapOperator<K1, V1, K2, V2>> template; Iterator<Partition<MapOperator<K1, V1, K2, V2>>> itr = operatorPartitions.iterator(); template = itr.next(); Configuration conf = new Configuration(); SerializationFactory serializationFactory = new SerializationFactory(conf); if (outstream.size() == 0) { InputSplit[] splits; try { splits = getSplits(new JobConf(conf), tempPartitionCount, template.getPartitionedInstance().getDirName()); } catch (Exception e1) { logger.info(" can't get splits {}", e1.getMessage()); throw new RuntimeException(e1); } Collection<Partition<MapOperator<K1, V1, K2, V2>>> operList = new ArrayList<Partition<MapOperator<K1, V1, K2, V2>>>(); itr = operatorPartitions.iterator(); int size = splits.length; Serializer keySerializer = serializationFactory.getSerializer(splits[0].getClass()); while (size > 0 && itr.hasNext()) { Partition<MapOperator<K1, V1, K2, V2>> p = itr.next(); MapOperator<K1, V1, K2, V2> opr = p.getPartitionedInstance(); opr.setInputFormatClass(inputFormatClass); opr.setMapClass(mapClass); opr.setCombineClass(combineClass); opr.setConfigFile(configFile); try { keySerializer.open(opr.getOutstream()); keySerializer.serialize(splits[size - 1]); opr.setInputSplitClass(splits[size - 1].getClass()); } catch (IOException e) { logger.info("error while serializing {}", e.getMessage()); } size--; operList.add(p); } while (size > 0) { MapOperator<K1, V1, K2, V2> opr = new MapOperator<K1, V1, K2, V2>(); opr.setInputFormatClass(inputFormatClass); opr.setMapClass(mapClass); opr.setCombineClass(combineClass); opr.setConfigFile(configFile); try { keySerializer.open(opr.getOutstream()); keySerializer.serialize(splits[size - 1]); opr.setInputSplitClass(splits[size - 1].getClass()); } catch (IOException e) { logger.info("error while serializing {}", e.getMessage()); } size--; operList.add(new DefaultPartition<MapOperator<K1, V1, K2, V2>>(opr)); } try { keySerializer.close(); } catch (IOException e) { throw new RuntimeException(e); } return operList; } return null; }
From source file:org.apache.apex.examples.mroperator.MapOperatorTest.java
License:Apache License
public void testNodeProcessingSchema(MapOperator<LongWritable, Text, Text, IntWritable> oper) throws IOException { CollectorTestSink sortSink = new CollectorTestSink(); oper.output.setSink(sortSink);// w w w. j ava 2s .c o m oper.setMapClass(WordCount.Map.class); oper.setCombineClass(WordCount.Reduce.class); oper.setDirName(testMeta.testDir); oper.setConfigFile(null); oper.setInputFormatClass(TextInputFormat.class); Configuration conf = new Configuration(); JobConf jobConf = new JobConf(conf); FileInputFormat.setInputPaths(jobConf, new Path(testMeta.testDir)); TextInputFormat inputFormat = new TextInputFormat(); inputFormat.configure(jobConf); InputSplit[] splits = inputFormat.getSplits(jobConf, 1); SerializationFactory serializationFactory = new SerializationFactory(conf); Serializer keySerializer = serializationFactory.getSerializer(splits[0].getClass()); keySerializer.open(oper.getOutstream()); keySerializer.serialize(splits[0]); oper.setInputSplitClass(splits[0].getClass()); keySerializer.close(); oper.setup(null); oper.beginWindow(0); oper.emitTuples(); oper.emitTuples(); oper.endWindow(); oper.beginWindow(1); oper.emitTuples(); oper.endWindow(); Assert.assertEquals("number emitted tuples", 3, sortSink.collectedTuples.size()); for (Object o : sortSink.collectedTuples) { LOG.debug(o.toString()); } LOG.debug("Done testing round\n"); oper.teardown(); }
From source file:org.apache.avro.hadoop.io.TestAvroSerialization.java
License:Apache License
private <T, O> O roundTrip(Schema schema, T data, Class<? extends GenericData> modelClass) throws IOException { Job job = new Job(); AvroJob.setMapOutputKeySchema(job, schema); if (modelClass != null) AvroJob.setDataModelClass(job, modelClass); AvroSerialization serialization = ReflectionUtils.newInstance(AvroSerialization.class, job.getConfiguration());/*from ww w . j av a2 s .c om*/ Serializer<AvroKey<T>> serializer = serialization.getSerializer(AvroKey.class); Deserializer<AvroKey<O>> deserializer = serialization.getDeserializer(AvroKey.class); ByteArrayOutputStream baos = new ByteArrayOutputStream(); serializer.open(baos); serializer.serialize(new AvroKey<T>(data)); serializer.close(); ByteArrayInputStream bais = new ByteArrayInputStream(baos.toByteArray()); deserializer.open(bais); AvroKey<O> result = null; result = deserializer.deserialize(result); deserializer.close(); return result.datum(); }
From source file:org.apache.gora.util.IOUtils.java
License:Apache License
/** Serializes the object to the given dataoutput using * available Hadoop serializations/*from ww w. j a v a 2 s . c o m*/ * @throws IOException */ public static <T> void serialize(Configuration conf, DataOutput out, T obj, Class<T> objClass) throws IOException { SerializationFactory serializationFactory = new SerializationFactory(getOrCreateConf(conf)); Serializer<T> serializer = serializationFactory.getSerializer(objClass); ByteBufferOutputStream os = new ByteBufferOutputStream(); try { serializer.open(os); serializer.serialize(obj); int length = 0; List<ByteBuffer> buffers = os.getBufferList(); for (ByteBuffer buffer : buffers) { length += buffer.limit() - buffer.arrayOffset(); } WritableUtils.writeVInt(out, length); for (ByteBuffer buffer : buffers) { byte[] arr = buffer.array(); out.write(arr, buffer.arrayOffset(), buffer.limit()); } } finally { if (serializer != null) serializer.close(); if (os != null) os.close(); } }