List of usage examples for org.apache.hadoop.io FloatWritable set
public void set(float value)
From source file:com.facebook.hive.orc.lazy.LazyFloatTreeReader.java
License:Open Source License
FloatWritable createWritable(Object previous, float value) throws IOException { FloatWritable result = null; if (previous == null) { result = new FloatWritable(); } else {/*from w w w. j av a 2s.c om*/ result = (FloatWritable) previous; } result.set(value); return result; }
From source file:com.jfolson.hive.serde.RTypedBytesWritableInput.java
License:Apache License
public FloatWritable readFloat(FloatWritable fw) throws IOException { if (fw == null) { fw = new FloatWritable(); }//from www . ja v a 2 s .com fw.set(in.readFloat()); return fw; }
From source file:com.linkedin.cubert.io.virtual.VirtualProgressReporter.java
License:Open Source License
/** * Set the progress of the current task. * *Note: Works only when using a Virtual Input Format * * @param value value of the progress must lie within [0.0, 1.0] *///from w w w . j ava 2s . co m public static void setProgress(float value) { if (PhaseContext.isIntialized()) { final Mapper.Context mapContext = PhaseContext.getMapContext(); try { final FloatWritable progress = (FloatWritable) mapContext.getCurrentKey(); progress.set(value); mapContext.nextKeyValue(); } catch (Exception e) { System.err.println("Unable to report progress in Load Cyclic. Exception: " + e); e.printStackTrace(); } } }
From source file:hivemall.classifier.BinaryOnlineClassifierUDTF.java
License:Open Source License
@Override public final void close() throws HiveException { super.close(); if (model != null) { int numForwarded = 0; if (useCovariance()) { final WeightValueWithCovar probe = new WeightValueWithCovar(); final Object[] forwardMapObj = new Object[3]; final FloatWritable fv = new FloatWritable(); final FloatWritable cov = new FloatWritable(); final IMapIterator<Object, IWeightValue> itor = model.entries(); while (itor.next() != -1) { itor.getValue(probe);// w ww . ja v a 2 s .c o m if (!probe.isTouched()) { continue; // skip outputting untouched weights } Object k = itor.getKey(); fv.set(probe.get()); cov.set(probe.getCovariance()); forwardMapObj[0] = k; forwardMapObj[1] = fv; forwardMapObj[2] = cov; forward(forwardMapObj); numForwarded++; } } else { final WeightValue probe = new WeightValue(); final Object[] forwardMapObj = new Object[2]; final FloatWritable fv = new FloatWritable(); final IMapIterator<Object, IWeightValue> itor = model.entries(); while (itor.next() != -1) { itor.getValue(probe); if (!probe.isTouched()) { continue; // skip outputting untouched weights } Object k = itor.getKey(); fv.set(probe.get()); forwardMapObj[0] = k; forwardMapObj[1] = fv; forward(forwardMapObj); numForwarded++; } } int numMixed = model.getNumMixed(); this.model = null; logger.info("Trained a prediction model using " + count + " training examples" + (numMixed > 0 ? "( numMixed: " + numMixed + " )" : "")); logger.info("Forwarded the prediction model of " + numForwarded + " rows"); } }
From source file:hivemall.classifier.KernelExpansionPassiveAggressiveUDTF.java
License:Apache License
@Override public void close() throws HiveException { final IntWritable h = new IntWritable(0); // row[0] final FloatWritable w0 = new FloatWritable(_w0); // row[1] final FloatWritable w1 = new FloatWritable(); // row[2] final FloatWritable w2 = new FloatWritable(); // row[3] final IntWritable hk = new IntWritable(0); // row[4] final FloatWritable w3 = new FloatWritable(); // row[5] final Object[] row = new Object[] { h, w0, null, null, null, null }; forward(row); // 0(f), w0 row[1] = null;// w ww. j a v a2s . c o m row[2] = w1; row[3] = w2; final Int2FloatMap w2map = _w2; for (Int2FloatMap.Entry e : Fastutil.fastIterable(_w1)) { int k = e.getIntKey(); Preconditions.checkArgument(k > 0, HiveException.class); h.set(k); w1.set(e.getFloatValue()); w2.set(w2map.get(k)); forward(row); // h(f), w1, w2 } this._w1 = null; this._w2 = null; row[0] = null; row[2] = null; row[3] = null; row[4] = hk; row[5] = w3; _w3.int2FloatEntrySet(); for (Int2FloatMap.Entry e : Fastutil.fastIterable(_w3)) { int k = e.getIntKey(); Preconditions.checkArgument(k > 0, HiveException.class); hk.set(k); w3.set(e.getFloatValue()); forward(row); // hk(f), w3 } this._w3 = null; }
From source file:hivemall.classifier.multiclass.MulticlassOnlineClassifierUDTF.java
License:Open Source License
@Override public final void close() throws HiveException { super.close(); if (label2model != null) { long numForwarded = 0L; long numMixed = 0L; if (useCovariance()) { final WeightValueWithCovar probe = new WeightValueWithCovar(); final Object[] forwardMapObj = new Object[4]; final FloatWritable fv = new FloatWritable(); final FloatWritable cov = new FloatWritable(); for (Map.Entry<Object, PredictionModel> entry : label2model.entrySet()) { Object label = entry.getKey(); forwardMapObj[0] = label; PredictionModel model = entry.getValue(); numMixed += model.getNumMixed(); IMapIterator<Object, IWeightValue> itor = model.entries(); while (itor.next() != -1) { itor.getValue(probe); if (!probe.isTouched()) { continue; // skip outputting untouched weights }/*w w w. j a v a 2 s . c o m*/ Object k = itor.getKey(); fv.set(probe.get()); cov.set(probe.getCovariance()); forwardMapObj[1] = k; forwardMapObj[2] = fv; forwardMapObj[3] = cov; forward(forwardMapObj); numForwarded++; } } } else { final WeightValue probe = new WeightValue(); final Object[] forwardMapObj = new Object[3]; final FloatWritable fv = new FloatWritable(); for (Map.Entry<Object, PredictionModel> entry : label2model.entrySet()) { Object label = entry.getKey(); forwardMapObj[0] = label; PredictionModel model = entry.getValue(); numMixed += model.getNumMixed(); IMapIterator<Object, IWeightValue> itor = model.entries(); while (itor.next() != -1) { itor.getValue(probe); if (!probe.isTouched()) { continue; // skip outputting untouched weights } Object k = itor.getKey(); fv.set(probe.get()); forwardMapObj[1] = k; forwardMapObj[2] = fv; forward(forwardMapObj); numForwarded++; } } } this.label2model = null; logger.info("Trained a prediction model using " + count + " training examples" + (numMixed > 0 ? "( numMixed: " + numMixed + " )" : "")); logger.info("Forwarded the prediction model of " + numForwarded + " rows"); } }
From source file:hivemall.fm.FactorizationMachineUDTF.java
License:Apache License
private void forwardAsIntFeature(@Nonnull final FactorizationMachineModel model, final int factors) throws HiveException { final IntWritable f_idx = new IntWritable(0); final FloatWritable f_Wi = new FloatWritable(0.f); final FloatWritable[] f_Vi = HiveUtils.newFloatArray(factors, 0.f); final Object[] forwardObjs = new Object[3]; forwardObjs[0] = f_idx;//from w w w.j a va 2 s . c o m forwardObjs[1] = f_Wi; forwardObjs[2] = null; // W0 f_idx.set(0); f_Wi.set(model.getW0()); // V0 is null forward(forwardObjs); // Wi, Vif (i starts from 1..P) forwardObjs[2] = Arrays.asList(f_Vi); for (int i = model.getMinIndex(), maxIdx = model.getMaxIndex(); i <= maxIdx; i++) { final float[] vi = model.getV(i); if (vi == null) { continue; } f_idx.set(i); // set Wi final float w = model.getW(i); f_Wi.set(w); // set Vif for (int f = 0; f < factors; f++) { float v = vi[f]; f_Vi[f].set(v); } forward(forwardObjs); } }
From source file:hivemall.fm.FactorizationMachineUDTF.java
License:Apache License
private void forwardAsStringFeature(@Nonnull final FMStringFeatureMapModel model, final int factors) throws HiveException { final Text feature = new Text(); final FloatWritable f_Wi = new FloatWritable(0.f); final FloatWritable[] f_Vi = HiveUtils.newFloatArray(factors, 0.f); final Object[] forwardObjs = new Object[3]; forwardObjs[0] = feature;/*from w ww. j av a 2s . co m*/ forwardObjs[1] = f_Wi; forwardObjs[2] = null; // W0 feature.set("0"); f_Wi.set(model.getW0()); // V0 is null forward(forwardObjs); // Wi, Vif (i starts from 1..P) forwardObjs[2] = Arrays.asList(f_Vi); final IMapIterator<String, Entry> itor = model.entries(); while (itor.next() != -1) { String i = itor.getKey(); assert (i != null); // set i feature.set(i); Entry entry = itor.getValue(); // set Wi f_Wi.set(entry.W); // set Vif final float[] Vi = entry.Vf; for (int f = 0; f < factors; f++) { float v = Vi[f]; f_Vi[f].set(v); } forward(forwardObjs); } }
From source file:hivemall.GeneralLearnerBaseUDTF.java
License:Apache License
protected void forwardModel() throws HiveException { int numForwarded = 0; if (useCovariance()) { final WeightValueWithCovar probe = new WeightValueWithCovar(); final Object[] forwardMapObj = new Object[3]; final FloatWritable fv = new FloatWritable(); final FloatWritable cov = new FloatWritable(); final IMapIterator<Object, IWeightValue> itor = model.entries(); while (itor.next() != -1) { itor.getValue(probe);//from www . ja v a 2 s. c o m if (!probe.isTouched()) { continue; // skip outputting untouched weights } Object k = itor.getKey(); fv.set(probe.get()); cov.set(probe.getCovariance()); forwardMapObj[0] = k; forwardMapObj[1] = fv; forwardMapObj[2] = cov; forward(forwardMapObj); numForwarded++; } } else { final WeightValue probe = new WeightValue(); final Object[] forwardMapObj = new Object[2]; final FloatWritable fv = new FloatWritable(); final IMapIterator<Object, IWeightValue> itor = model.entries(); while (itor.next() != -1) { itor.getValue(probe); if (!probe.isTouched()) { continue; // skip outputting untouched weights } Object k = itor.getKey(); fv.set(probe.get()); forwardMapObj[0] = k; forwardMapObj[1] = fv; forward(forwardMapObj); numForwarded++; } } long numMixed = model.getNumMixed(); logger.info("Trained a prediction model using " + count + " training examples" + (numMixed > 0 ? "( numMixed: " + numMixed + " )" : "")); logger.info("Forwarded the prediction model of " + numForwarded + " rows"); }
From source file:hivemall.mf.BPRMatrixFactorizationUDTF.java
License:Apache License
@Override public void close() throws HiveException { if (model != null) { if (count == 0) { this.model = null; // help GC return; }/*w w w . j av a 2 s . c o m*/ if (iterations > 1) { runIterativeTraining(iterations); } final IntWritable idx = new IntWritable(); final FloatWritable[] Pu = HiveUtils.newFloatArray(factor, 0.f); final FloatWritable[] Qi = HiveUtils.newFloatArray(factor, 0.f); final FloatWritable Bi = useBiasClause ? new FloatWritable() : null; final Object[] forwardObj = new Object[] { idx, Pu, Qi, Bi }; int numForwarded = 0; for (int i = model.getMinIndex(), maxIdx = model.getMaxIndex(); i <= maxIdx; i++) { idx.set(i); Rating[] userRatings = model.getUserVector(i); if (userRatings == null) { forwardObj[1] = null; } else { forwardObj[1] = Pu; copyTo(userRatings, Pu); } Rating[] itemRatings = model.getItemVector(i); if (itemRatings == null) { forwardObj[2] = null; } else { forwardObj[2] = Qi; copyTo(itemRatings, Qi); } if (useBiasClause) { Bi.set(model.getItemBias(i)); } forward(forwardObj); numForwarded++; } this.model = null; // help GC LOG.info("Forwarded the prediction model of " + numForwarded + " rows. [lastLosses=" + cvState.getCumulativeLoss() + ", #trainingExamples=" + count + "]"); } }