List of usage examples for org.apache.hadoop.io FloatWritable FloatWritable
public FloatWritable(float value)
From source file:edu.umd.cloud9.io.HashMapWritableTest.java
License:Apache License
@Test(expected = IOException.class) public void testTypeSafety() throws IOException { HashMapWritable<Writable, Writable> origMap = new HashMapWritable<Writable, Writable>(); origMap.put(new Text("hi"), new FloatWritable(5.3f)); origMap.put(new Text("there"), new Text("bbb")); ByteArrayOutputStream bytesOut = new ByteArrayOutputStream(); DataOutputStream dataOut = new DataOutputStream(bytesOut); origMap.write(dataOut);/*from w ww.j a v a 2s. c om*/ HashMapWritable<Writable, Writable> map = new HashMapWritable<Writable, Writable>(); map.readFields(new DataInputStream(new ByteArrayInputStream(bytesOut.toByteArray()))); }
From source file:edu.umd.cloud9.io.pair.PairOfWritablesTest.java
License:Apache License
@Test public void testBasic() throws IOException { PairOfWritables<Text, IntWritable> pair1 = new PairOfWritables<Text, IntWritable>(new Text("hi"), new IntWritable(1)); assertEquals(new Text("hi"), pair1.getLeftElement()); assertEquals(new IntWritable(1), pair1.getRightElement()); PairOfWritables<IntWritable, FloatWritable> pair2 = new PairOfWritables<IntWritable, FloatWritable>( new IntWritable(1), new FloatWritable(1.0f)); assertEquals(new IntWritable(1), pair2.getLeftElement()); assertEquals(new FloatWritable(1.0f), pair2.getRightElement()); }
From source file:edu.umd.ujjwalgoel.AnalyzePMI.java
License:Apache License
@SuppressWarnings({ "static-access" })
public static void main(String[] args) {
Options options = new Options();
options.addOption(OptionBuilder.withArgName("path").hasArg().withDescription("input path").create(INPUT));
CommandLine cmdline = null;/*from w w w. ja va 2 s . c o m*/
CommandLineParser parser = new GnuParser();
try {
cmdline = parser.parse(options, args);
} catch (ParseException exp) {
System.err.println("Error parsing command line: " + exp.getMessage());
System.exit(-1);
}
if (!cmdline.hasOption(INPUT)) {
System.out.println("args: " + Arrays.toString(args));
HelpFormatter formatter = new HelpFormatter();
formatter.setWidth(120);
formatter.printHelp(AnalyzePMI.class.getName(), options);
ToolRunner.printGenericCommandUsage(System.out);
System.exit(-1);
}
String inputPath = cmdline.getOptionValue(INPUT);
System.out.println("input path: " + inputPath);
BufferedReader br = null;
int countPairs = 0;
List<PairOfWritables<PairOfStrings, FloatWritable>> pmis = new ArrayList<PairOfWritables<PairOfStrings, FloatWritable>>();
List<PairOfWritables<PairOfStrings, FloatWritable>> cloudPmis = new ArrayList<PairOfWritables<PairOfStrings, FloatWritable>>();
List<PairOfWritables<PairOfStrings, FloatWritable>> lovePmis = new ArrayList<PairOfWritables<PairOfStrings, FloatWritable>>();
PairOfWritables<PairOfStrings, FloatWritable> highestPMI = null;
PairOfWritables<PairOfStrings, FloatWritable> highestCloudPMI = null;
PairOfWritables<PairOfStrings, FloatWritable> highestCloudPMI2 = null;
PairOfWritables<PairOfStrings, FloatWritable> highestCloudPMI3 = null;
PairOfWritables<PairOfStrings, FloatWritable> highestLovePMI = null;
PairOfWritables<PairOfStrings, FloatWritable> highestLovePMI2 = null;
PairOfWritables<PairOfStrings, FloatWritable> highestLovePMI3 = null;
try {
FileSystem fs = FileSystem.get(new Configuration());
FileStatus[] status = fs.listStatus(new Path(inputPath));
//PairOfStrings pair = new PairOfStrings();
for (int i = 0; i < status.length; i++) {
br = new BufferedReader(new InputStreamReader(fs.open(status[i].getPath())));
String line = br.readLine();
while (line != null) {
String[] words = line.split("\\t");
float value = Float.parseFloat(words[1].trim());
String[] wordPair = words[0].replaceAll("\\(", "").replaceAll("\\)", "").split(",");
PairOfStrings pair = new PairOfStrings();
pair.set(wordPair[0].trim(), wordPair[1].trim());
if (wordPair[0].trim().equals("cloud")) {
PairOfWritables<PairOfStrings, FloatWritable> cloudPmi = new PairOfWritables<PairOfStrings, FloatWritable>();
cloudPmi.set(pair, new FloatWritable(value));
cloudPmis.add(cloudPmi);
if ((highestCloudPMI == null)
|| (highestCloudPMI.getRightElement().compareTo(cloudPmi.getRightElement()) < 0)) {
highestCloudPMI = cloudPmi;
} else if ((highestCloudPMI2 == null)
|| (highestCloudPMI2.getRightElement().compareTo(cloudPmi.getRightElement()) < 0)) {
highestCloudPMI2 = cloudPmi;
} else if ((highestCloudPMI3 == null)
|| (highestCloudPMI3.getRightElement().compareTo(cloudPmi.getRightElement()) < 0)) {
highestCloudPMI3 = cloudPmi;
}
}
if (wordPair[0].trim().equals("love")) {
PairOfWritables<PairOfStrings, FloatWritable> lovePmi = new PairOfWritables<PairOfStrings, FloatWritable>();
lovePmi.set(pair, new FloatWritable(value));
lovePmis.add(lovePmi);
if ((highestLovePMI == null)
|| (highestLovePMI.getRightElement().compareTo(lovePmi.getRightElement()) < 0)) {
highestLovePMI = lovePmi;
} else if ((highestLovePMI2 == null)
|| (highestLovePMI2.getRightElement().compareTo(lovePmi.getRightElement()) < 0)) {
highestLovePMI2 = lovePmi;
} else if ((highestLovePMI3 == null)
|| (highestLovePMI3.getRightElement().compareTo(lovePmi.getRightElement()) < 0)) {
highestLovePMI3 = lovePmi;
}
}
PairOfWritables<PairOfStrings, FloatWritable> pmi = new PairOfWritables<PairOfStrings, FloatWritable>();
pmi.set(pair, new FloatWritable(value));
pmis.add(pmi);
if (highestPMI == null) {
highestPMI = pmi;
} else if (highestPMI.getRightElement().compareTo(pmi.getRightElement()) < 0) {
highestPMI = pmi;
}
countPairs++;
line = br.readLine();
}
}
} catch (Exception ex) {
System.out.println("ERROR" + ex.getMessage());
}
/*Collections.sort(pmis, new Comparator<PairOfWritables<PairOfStrings, FloatWritable>>() {
public int compare(PairOfWritables<PairOfStrings, FloatWritable> e1,
PairOfWritables<PairOfStrings, FloatWritable> e2) {
/*if (e2.getRightElement().compareTo(e1.getRightElement()) == 0) {
return e1.getLeftElement().getLeftElement().compareTo(e2.getLeftElement().getLeftElement());
}
return e2.getRightElement().compareTo(e1.getRightElement());
}
});
Collections.sort(cloudPmis, new Comparator<PairOfWritables<PairOfStrings, FloatWritable>>() {
public int compare(PairOfWritables<PairOfStrings, FloatWritable> e1,
PairOfWritables<PairOfStrings, FloatWritable> e2) {
if (e2.getRightElement().compareTo(e1.getRightElement()) == 0) {
return e1.getLeftElement().getLeftElement().compareTo(e2.getLeftElement().getLeftElement());
}
return e2.getRightElement().compareTo(e1.getRightElement());
}
});
Collections.sort(lovePmis, new Comparator<PairOfWritables<PairOfStrings, FloatWritable>>() {
public int compare(PairOfWritables<PairOfStrings, FloatWritable> e1,
PairOfWritables<PairOfStrings, FloatWritable> e2) {
if (e2.getRightElement().compareTo(e1.getRightElement()) == 0) {
return e1.getLeftElement().getLeftElement().compareTo(e2.getLeftElement().getLeftElement());
}
return e2.getRightElement().compareTo(e1.getRightElement());
}
});
PairOfWritables<PairOfStrings, FloatWritable> highestPMI = pmis.get(0);
PairOfWritables<PairOfStrings, FloatWritable> highestCloudPMI = cloudPmis.get(0); PairOfWritables<PairOfStrings, FloatWritable> highestCloudPMI2 = cloudPmis.get(1);
PairOfWritables<PairOfStrings, FloatWritable> highestCloudPMI3 = cloudPmis.get(2);
PairOfWritables<PairOfStrings, FloatWritable> highestLovePMI = lovePmis.get(0); PairOfWritables<PairOfStrings, FloatWritable> highestLovePMI2 = lovePmis.get(1);
PairOfWritables<PairOfStrings, FloatWritable> highestLovePMI3 = lovePmis.get(2);*/
System.out.println("Total Distinct Pairs : " + countPairs);
System.out.println("Pair with highest PMI : (" + highestPMI.getLeftElement().getLeftElement() + ", "
+ highestPMI.getLeftElement().getRightElement());
System.out
.println("Word with highest PMI with Cloud : " + highestCloudPMI.getLeftElement().getRightElement()
+ " with value : " + highestCloudPMI.getRightElement().get());
System.out.println(
"Word with second highest PMI with Cloud : " + highestCloudPMI2.getLeftElement().getRightElement()
+ " with value : " + highestCloudPMI2.getRightElement().get());
System.out.println(
"Word with third highest PMI with Cloud : " + highestCloudPMI3.getLeftElement().getRightElement()
+ " with value : " + highestCloudPMI3.getRightElement().get());
System.out.println("Word with highest PMI with Love : " + highestLovePMI.getLeftElement().getRightElement()
+ " with value : " + highestLovePMI.getRightElement().get());
System.out.println(
"Word with second highest PMI with Love : " + highestLovePMI2.getLeftElement().getRightElement()
+ " with value : " + highestLovePMI2.getRightElement().get());
System.out.println(
"Word with third highest PMI with Love : " + highestLovePMI3.getLeftElement().getRightElement()
+ " with value : " + highestLovePMI3.getRightElement().get());
}
From source file:eu.stratosphere.hadoopcompatibility.datatypes.DefaultStratosphereTypeConverter.java
License:Apache License
@SuppressWarnings("unchecked") private <T> T convert(Record stratosphereType, int pos, Class<T> hadoopType) { if (hadoopType == LongWritable.class) { return (T) new LongWritable((stratosphereType.getField(pos, LongValue.class)).getValue()); }/* w ww . ja v a 2 s . co m*/ if (hadoopType == org.apache.hadoop.io.Text.class) { return (T) new Text((stratosphereType.getField(pos, StringValue.class)).getValue()); } if (hadoopType == org.apache.hadoop.io.IntWritable.class) { return (T) new IntWritable((stratosphereType.getField(pos, IntValue.class)).getValue()); } if (hadoopType == org.apache.hadoop.io.FloatWritable.class) { return (T) new FloatWritable((stratosphereType.getField(pos, FloatValue.class)).getValue()); } if (hadoopType == org.apache.hadoop.io.DoubleWritable.class) { return (T) new DoubleWritable((stratosphereType.getField(pos, DoubleValue.class)).getValue()); } if (hadoopType == org.apache.hadoop.io.BooleanWritable.class) { return (T) new BooleanWritable((stratosphereType.getField(pos, BooleanValue.class)).getValue()); } if (hadoopType == org.apache.hadoop.io.ByteWritable.class) { return (T) new ByteWritable((stratosphereType.getField(pos, ByteValue.class)).getValue()); } throw new RuntimeException("Unable to convert Stratosphere type (" + stratosphereType.getClass().getCanonicalName() + ") to Hadoop."); }
From source file:hivemall.classifier.KernelExpansionPassiveAggressiveUDTF.java
License:Apache License
@Override public void close() throws HiveException { final IntWritable h = new IntWritable(0); // row[0] final FloatWritable w0 = new FloatWritable(_w0); // row[1] final FloatWritable w1 = new FloatWritable(); // row[2] final FloatWritable w2 = new FloatWritable(); // row[3] final IntWritable hk = new IntWritable(0); // row[4] final FloatWritable w3 = new FloatWritable(); // row[5] final Object[] row = new Object[] { h, w0, null, null, null, null }; forward(row); // 0(f), w0 row[1] = null;/*w w w. jav a 2 s. com*/ row[2] = w1; row[3] = w2; final Int2FloatMap w2map = _w2; for (Int2FloatMap.Entry e : Fastutil.fastIterable(_w1)) { int k = e.getIntKey(); Preconditions.checkArgument(k > 0, HiveException.class); h.set(k); w1.set(e.getFloatValue()); w2.set(w2map.get(k)); forward(row); // h(f), w1, w2 } this._w1 = null; this._w2 = null; row[0] = null; row[2] = null; row[3] = null; row[4] = hk; row[5] = w3; _w3.int2FloatEntrySet(); for (Int2FloatMap.Entry e : Fastutil.fastIterable(_w3)) { int k = e.getIntKey(); Preconditions.checkArgument(k > 0, HiveException.class); hk.set(k); w3.set(e.getFloatValue()); forward(row); // hk(f), w3 } this._w3 = null; }
From source file:hivemall.fm.FactorizationMachineUDTF.java
License:Apache License
private void forwardAsIntFeature(@Nonnull final FactorizationMachineModel model, final int factors) throws HiveException { final IntWritable f_idx = new IntWritable(0); final FloatWritable f_Wi = new FloatWritable(0.f); final FloatWritable[] f_Vi = HiveUtils.newFloatArray(factors, 0.f); final Object[] forwardObjs = new Object[3]; forwardObjs[0] = f_idx;/*from w ww . j av a 2 s. co m*/ forwardObjs[1] = f_Wi; forwardObjs[2] = null; // W0 f_idx.set(0); f_Wi.set(model.getW0()); // V0 is null forward(forwardObjs); // Wi, Vif (i starts from 1..P) forwardObjs[2] = Arrays.asList(f_Vi); for (int i = model.getMinIndex(), maxIdx = model.getMaxIndex(); i <= maxIdx; i++) { final float[] vi = model.getV(i); if (vi == null) { continue; } f_idx.set(i); // set Wi final float w = model.getW(i); f_Wi.set(w); // set Vif for (int f = 0; f < factors; f++) { float v = vi[f]; f_Vi[f].set(v); } forward(forwardObjs); } }
From source file:hivemall.fm.FactorizationMachineUDTF.java
License:Apache License
private void forwardAsStringFeature(@Nonnull final FMStringFeatureMapModel model, final int factors) throws HiveException { final Text feature = new Text(); final FloatWritable f_Wi = new FloatWritable(0.f); final FloatWritable[] f_Vi = HiveUtils.newFloatArray(factors, 0.f); final Object[] forwardObjs = new Object[3]; forwardObjs[0] = feature;/*from www. j a va2s. com*/ forwardObjs[1] = f_Wi; forwardObjs[2] = null; // W0 feature.set("0"); f_Wi.set(model.getW0()); // V0 is null forward(forwardObjs); // Wi, Vif (i starts from 1..P) forwardObjs[2] = Arrays.asList(f_Vi); final IMapIterator<String, Entry> itor = model.entries(); while (itor.next() != -1) { String i = itor.getKey(); assert (i != null); // set i feature.set(i); Entry entry = itor.getValue(); // set Wi f_Wi.set(entry.W); // set Vif final float[] Vi = entry.Vf; for (int f = 0; f < factors; f++) { float v = Vi[f]; f_Vi[f].set(v); } forward(forwardObjs); } }
From source file:hivemall.ftvec.ExtractWeightUDF.java
License:Open Source License
private static FloatWritable extractWeights(String ftvec) throws UDFArgumentException { if (ftvec == null) { return null; }/*from w ww . ja va 2 s . c om*/ String[] splits = ftvec.split(":"); if (splits.length != 2) { throw new UDFArgumentException("Unexpected feature vector representation: " + ftvec); } float f = Float.valueOf(splits[1]); return new FloatWritable(f); }
From source file:hivemall.knn.distance.AngularDistanceUDF.java
License:Apache License
@Override public FloatWritable evaluate(DeferredObject[] arguments) throws HiveException { List<String> ftvec1 = HiveUtils.asStringList(arguments[0], arg0ListOI); List<String> ftvec2 = HiveUtils.asStringList(arguments[1], arg1ListOI); float d = 1.f - AngularSimilarityUDF.angularSimilarity(ftvec1, ftvec2); return new FloatWritable(d); }
From source file:hivemall.knn.distance.AngularDistanceUDF.java
License:Apache License
@Deprecated public FloatWritable evaluate(List<String> ftvec1, List<String> ftvec2) { float d = 1.f - AngularSimilarityUDF.angularSimilarity(ftvec1, ftvec2); return new FloatWritable(d); }