List of usage examples for org.apache.hadoop.io IntWritable set
public void set(int value)
From source file:io.aos.hdfs.IntWritableTest.java
License:Apache License
@Test public void walkthroughWithNoArgsConstructor() throws IOException { // vv IntWritableTest IntWritable writable = new IntWritable(); writable.set(163); // ^^ IntWritableTest checkWalkthrough(writable);//from ww w. j ava2 s .c om }
From source file:io.aos.hdfs.MapFileWriteDemo.java
License:Apache License
public static void main(String... args) throws IOException { String uri = args[0];/*from w ww. j a va2 s. c o m*/ Configuration conf = new Configuration(); FileSystem fs = FileSystem.get(URI.create(uri), conf); IntWritable key = new IntWritable(); Text value = new Text(); MapFile.Writer writer = null; try { writer = new MapFile.Writer(conf, fs, uri, key.getClass(), value.getClass()); for (int i = 0; i < 1024; i++) { key.set(i + 1); value.set(DATA[i % DATA.length]); writer.append(key, value); } } finally { IOUtils.closeStream(writer); } }
From source file:io.aos.hdfs.SequenceFileWriteDemo.java
License:Apache License
public static void main(String... args) throws IOException { String uri = args[0];/*w w w . j av a 2 s. c om*/ Configuration conf = new Configuration(); FileSystem fs = FileSystem.get(URI.create(uri), conf); Path path = new Path(uri); IntWritable key = new IntWritable(); Text value = new Text(); SequenceFile.Writer writer = null; try { writer = SequenceFile.createWriter(fs, conf, path, key.getClass(), value.getClass()); for (int i = 0; i < 100; i++) { key.set(100 - i); value.set(DATA[i % DATA.length]); System.out.printf("[%s]\t%s\t%s\n", writer.getLength(), key, value); writer.append(key, value); } } finally { IOUtils.closeStream(writer); } }
From source file:it.crs4.pydoop.mapreduce.pipes.PipeApplicationStub.java
License:Apache License
public void binaryProtocolStub() { try {//ww w . j a v a 2 s . com System.err.println("binaryProtocolStub()"); initSoket(); System.err.println("done with socket()"); // output code WritableUtils.writeVInt(dataOut, 50); IntWritable wt = new IntWritable(); wt.set(123); writeObject(wt, dataOut); writeObject(new Text("value"), dataOut); // PARTITIONED_OUTPUT WritableUtils.writeVInt(dataOut, 51); WritableUtils.writeVInt(dataOut, 0); writeObject(wt, dataOut); writeObject(new Text("value"), dataOut); // STATUS WritableUtils.writeVInt(dataOut, 52); Text.writeString(dataOut, "PROGRESS"); dataOut.flush(); // progress WritableUtils.writeVInt(dataOut, 53); dataOut.writeFloat(0.55f); // register counter WritableUtils.writeVInt(dataOut, 55); // id WritableUtils.writeVInt(dataOut, 0); Text.writeString(dataOut, "group"); Text.writeString(dataOut, "name"); // increment counter WritableUtils.writeVInt(dataOut, 56); WritableUtils.writeVInt(dataOut, 0); WritableUtils.writeVLong(dataOut, 2); // map item int intValue = WritableUtils.readVInt(dataInput); System.out.println("intValue:" + intValue); IntWritable iw = new IntWritable(); readObject(iw, dataInput); System.out.println("key:" + iw.get()); Text txt = new Text(); readObject(txt, dataInput); System.out.println("value:" + txt.toString()); // done // end of session WritableUtils.writeVInt(dataOut, 54); System.out.println("finish"); dataOut.flush(); dataOut.close(); } catch (Exception x) { x.printStackTrace(); } finally { closeSoket(); } }
From source file:libra.preprocess.common.kmerindex.KmerIndexReader.java
License:Apache License
@Override public boolean next(CompressedSequenceWritable key, IntWritable val) throws IOException { KmerIndexBufferEntry entry = this.buffer.poll(); if (entry != null) { key.set(entry.getKey());/*from w w w. j av a2 s .co m*/ val.set(entry.getVal().get()); return true; } fillBuffer(); entry = this.buffer.poll(); if (entry != null) { key.set(entry.getKey()); val.set(entry.getVal().get()); return true; } return false; }
From source file:microbench.WordCountOnHDFSDataLocal.java
License:Apache License
public static void main(String[] args) throws IOException, InterruptedException { try {/*from w ww .ja v a2s.com*/ parseArgs(args); HashMap<String, String> conf = new HashMap<String, String>(); initConf(conf); MPI_D.Init(args, MPI_D.Mode.Common, conf); JobConf jobConf = new JobConf(confPath); if (MPI_D.COMM_BIPARTITE_O != null) { // O communicator int rank = MPI_D.Comm_rank(MPI_D.COMM_BIPARTITE_O); int size = MPI_D.Comm_size(MPI_D.COMM_BIPARTITE_O); if (rank == 0) { DataMPIUtil.printArgs(args); } System.out.println("The O task " + rank + " of " + size + " is working..."); HadoopReader<LongWritable, Text> reader = HadoopIOUtil.getReader(jobConf, inDir, TextInputFormat.class, rank, MPI_D.COMM_BIPARTITE_O); Text word = new Text(); IntWritable one = new IntWritable(1); LongWritable khead = reader.createKey(); Text vhead = reader.createValue(); while (reader.next(khead, vhead)) { StringTokenizer itr = new StringTokenizer(vhead.toString()); while (itr.hasMoreTokens()) { word.set(itr.nextToken()); // send key-value MPI_D.Send(word, one); } } reader.close(); } else if (MPI_D.COMM_BIPARTITE_A != null) { // A communicator int rank = MPI_D.Comm_rank(MPI_D.COMM_BIPARTITE_A); int size = MPI_D.Comm_size(MPI_D.COMM_BIPARTITE_A); System.out.println("The A task " + rank + " of " + size + " is working..."); HadoopWriter<Text, IntWritable> outrw = HadoopIOUtil.getNewWriter(jobConf, outDir, Text.class, IntWritable.class, TextOutputFormat.class, null, rank, MPI_D.COMM_BIPARTITE_A); Text oldKey = null; IntWritable valueData = new IntWritable(); int sum = 0; Object[] keyValue = MPI_D.Recv(); while (keyValue != null) { Text key = (Text) keyValue[0]; IntWritable value = (IntWritable) keyValue[1]; if (oldKey == null) { oldKey = key; sum = value.get(); } else { if (key.equals(oldKey)) { sum += value.get(); } else { valueData.set(sum); outrw.write(oldKey, valueData); oldKey = key; sum = value.get(); } } keyValue = MPI_D.Recv(); } if (oldKey != null) { valueData.set(sum); outrw.write(oldKey, valueData); } outrw.close(); } MPI_D.Finalize(); } catch (MPI_D_Exception e) { e.printStackTrace(); } }
From source file:mlbench.kmeans.KmeansInit.java
License:Apache License
/** * get the input values and choose the K clusters' centers * * @param dataPath// ww w . j a v a 2 s .c o m * @throws MPI_D_Exception * @throws IOException * @throws MPIException */ @SuppressWarnings("deprecation") private static void init(String args[], String dataPath, int kCluster, HashMap<String, String> conf) throws MPI_D_Exception, IOException, MPIException { MPI_D.Init(args, MPI_D.Mode.Common, conf); if (MPI_D.COMM_BIPARTITE_O != null) { rank = MPI_D.Comm_rank(MPI_D.COMM_BIPARTITE_O); size = MPI_D.Comm_size(MPI_D.COMM_BIPARTITE_O); FileSplit[] inputs = DataMPIUtil.HDFSDataLocalLocator.getTaskInputs(MPI_D.COMM_BIPARTITE_O, config, dataPath, rank); // for record the initialized state for (FileSplit path : inputs) { SequenceFileInputFormat f = new SequenceFileInputFormat(); JobConf jobConf = new JobConf(confPath); Reporter r = new KmeansUtils.EmptyReport(); RecordReader<LongWritable, VectorWritable> reader = f.getRecordReader(path, jobConf, r); Random random = new Random(1000); LongWritable k = reader.createKey(); VectorWritable v = reader.createValue(); IntWritable cluster = new IntWritable(); while (reader.next(k, v)) { cluster.set(random.nextInt(kCluster)); MPI_D.Send(cluster, v); } reader.close(); } } else { IntWritable key = null, newKey = null; VectorWritable point = null, newPoint = null; double sum[] = null; int count = 0; Object[] vals = MPI_D.Recv(); while (vals != null) { newKey = (IntWritable) vals[0]; newPoint = (VectorWritable) vals[1]; if (key == null && point == null) { sum = new double[newPoint.get().size()]; } else if (!key.equals(newKey)) { double[] centerVals = new double[sum.length]; for (int i = 0; i < centerVals.length; i++) { centerVals[i] = sum[i] / count; } PointVector oneCenter = new PointVector(Integer.valueOf(key.toString()), centerVals); centers.add(oneCenter); sum = new double[point.get().size()]; count = 0; } key = newKey; point = newPoint; KmeansUtils.accumulate(sum, newPoint.get()); count++; vals = MPI_D.Recv(); } if (newKey != null && newPoint != null) { double[] centerVals = new double[sum.length]; for (int i = 0; i < centerVals.length; i++) { centerVals[i] = sum[i] / count; } PointVector oneCenter = new PointVector(key.get(), centerVals); centers.add(oneCenter); } transfer = new KmeansUtils.CenterTransfer(config, rank, size); transfer.gatherCentersByP2P(centers); if (rank == 0) { OutputStream resOut = KmeansUtils.getOutputStream(outPath, config); DataOutput os = new DataOutputStream(resOut); for (PointVector centerPoint : centers) { os.write((centerPoint.toString() + "\n").getBytes()); } resOut.flush(); resOut.close(); } System.out.println("rank " + rank + " finish"); } MPI_D.Finalize(); }
From source file:mlbench.pagerank.PagerankNaive.java
License:Apache License
@SuppressWarnings({ "rawtypes", "unchecked" })
public static void main(String[] args) throws IOException, InterruptedException {
try {/*ww w. ja v a 2s. c om*/
parseArgs(args);
HashMap<String, String> conf = new HashMap<String, String>();
initConf(conf);
MPI_D.Init(args, MPI_D.Mode.Common, conf);
JobConf jobConf = new JobConf(confPath);
if (MPI_D.COMM_BIPARTITE_O != null) {
// O communicator
int rank = MPI_D.Comm_rank(MPI_D.COMM_BIPARTITE_O);
int size = MPI_D.Comm_size(MPI_D.COMM_BIPARTITE_O);
if (rank == 0) {
LOG.info(PagerankNaive.class.getSimpleName() + " O start.");
}
FileSplit[] inputs1 = DataMPIUtil.HDFSDataLocalLocator.getTaskInputs(MPI_D.COMM_BIPARTITE_O,
jobConf, edgeDir, rank);
FileSplit[] inputs2 = DataMPIUtil.HDFSDataLocalLocator.getTaskInputs(MPI_D.COMM_BIPARTITE_O,
jobConf, vecDir, rank);
FileSplit[] inputs = (FileSplit[]) ArrayUtils.addAll(inputs2, inputs1);
for (int i = 0; i < inputs.length; i++) {
FileSplit fsplit = inputs[i];
LineRecordReader kvrr = new LineRecordReader(jobConf, fsplit);
LongWritable key = kvrr.createKey();
Text value = kvrr.createValue();
{
IntWritable k = new IntWritable();
Text v = new Text();
while (kvrr.next(key, value)) {
String line_text = value.toString();
// ignore comments in edge file
if (line_text.startsWith("#"))
continue;
final String[] line = line_text.split("\t");
if (line.length < 2)
continue;
// vector : ROWID VALUE('vNNNN')
if (line[1].charAt(0) == 'v') {
k.set(Integer.parseInt(line[0]));
v.set(line[1]);
MPI_D.Send(k, v);
} else {
/*
* In other matrix-vector multiplication, we
* output (dst, src) here However, In PageRank,
* the matrix-vector computation formula is M^T
* * v. Therefore, we output (src,dst) here.
*/
int src_id = Integer.parseInt(line[0]);
int dst_id = Integer.parseInt(line[1]);
k.set(src_id);
v.set(line[1]);
MPI_D.Send(k, v);
if (make_symmetric == 1) {
k.set(dst_id);
v.set(line[0]);
MPI_D.Send(k, v);
}
}
}
}
}
} else if (MPI_D.COMM_BIPARTITE_A != null) {
// A communicator
int rank = MPI_D.Comm_rank(MPI_D.COMM_BIPARTITE_A);
if (rank == 0) {
LOG.info(PagerankNaive.class.getSimpleName() + " A start.");
}
HadoopWriter<IntWritable, Text> outrw = HadoopIOUtil.getNewWriter(jobConf, outDir,
IntWritable.class, Text.class, TextOutputFormat.class, null, rank, MPI_D.COMM_BIPARTITE_A);
IntWritable oldKey = null;
int i;
double cur_rank = 0;
ArrayList<Integer> dst_nodes_list = new ArrayList<Integer>();
Object[] keyValue = MPI_D.Recv();
while (keyValue != null) {
IntWritable key = (IntWritable) keyValue[0];
Text value = (Text) keyValue[1];
if (oldKey == null) {
oldKey = key;
}
// A new key arrives
if (!key.equals(oldKey)) {
outrw.write(oldKey, new Text("s" + cur_rank));
int outdeg = dst_nodes_list.size();
if (outdeg > 0) {
cur_rank = cur_rank / (double) outdeg;
}
for (i = 0; i < outdeg; i++) {
outrw.write(new IntWritable(dst_nodes_list.get(i)), new Text("v" + cur_rank));
}
oldKey = key;
cur_rank = 0;
dst_nodes_list = new ArrayList<Integer>();
}
// common record
String line_text = value.toString();
final String[] line = line_text.split("\t");
if (line.length == 1) {
if (line_text.charAt(0) == 'v') { // vector : VALUE
cur_rank = Double.parseDouble(line_text.substring(1));
} else { // edge : ROWID
dst_nodes_list.add(Integer.parseInt(line[0]));
}
}
keyValue = MPI_D.Recv();
}
// write the left part
if (cur_rank != 0) {
outrw.write(oldKey, new Text("s" + cur_rank));
int outdeg = dst_nodes_list.size();
if (outdeg > 0) {
cur_rank = cur_rank / (double) outdeg;
}
for (i = 0; i < outdeg; i++) {
outrw.write(new IntWritable(dst_nodes_list.get(i)), new Text("v" + cur_rank));
}
}
outrw.close();
}
MPI_D.Finalize();
} catch (MPI_D_Exception e) {
e.printStackTrace();
}
}
From source file:nthu.scopelab.tsqr.ssvd.SSVDSolver.java
License:Apache License
/** * run all SSVD jobs.//from w w w . j a v a2s . co m * * @throws IOException * if I/O condition occurs. */ public void run() throws Exception { try { System.out.println("SSVD start!"); FileSystem fs = FileSystem.get(conf); Path qPath = new Path(outputPath, "Q-job"); Path btPath = new Path(outputPath, "Bt-job"); Path yPath = new Path(outputPath, "Y-job"); //tetst phase Path uHatPath = new Path(outputPath, "UHat"); Path svPath = new Path(outputPath, "Sigma"); Path uPath = new Path(outputPath, "U"); Path vPath = new Path(outputPath, "V"); if (overwrite) { fs.delete(outputPath, true); } int[] iseed = { 0, 0, 0, 1 }; double[] x = new double[1]; Dlarnv.dlarnv(2, iseed, 0, 1, x, 0); long seed = (long) (x[0] * (double) Long.MAX_VALUE); long start, end; start = new Date().getTime(); QJob.run(conf, inputPath, qPath.toString(), reduceSchedule, k, p, seed, mis); end = new Date().getTime(); System.out.println("Q-Job done " + Long.toString(end - start)); Logger LOG = LoggerFactory.getLogger(SSVDSolver.class); /* * restrict number of reducers to a reasonable number so we don't have to * run too many additions in the frontend when reconstructing BBt for the * last B' and BB' computations. The user may not realize that and gives a * bit too many (I would be happy i that were ever the case though). */ start = new Date().getTime(); BtJob.run(conf, inputPath, btPath, qPath.toString(), k, p, outerBlockHeight, q <= 0 ? Math.min(1000, reduceTasks) : reduceTasks, q <= 0, reduceSchedule, mis); end = new Date().getTime(); System.out.println("Bt-Job done " + Long.toString(end - start)); // power iterations is unnecessary in application of recommendation system /*for (int i = 0; i < q; i++) { Path btPathGlob = new Path(btPath, BtJob.OUTPUT_BT + "-*"); Path aBtPath = new Path(outputPath, String.format("ABt-job-%d", i + 1)); qPath = new Path(outputPath, String.format("ABtQ-job-%d", i + 1)); ABtDenseOutJob.run(conf, inputPath, btPathGlob, aBtPath,//qPath, //ablockRows, //minSplitSize, k, p, //abtBlockHeight, reduceTasks, //broadcast mis); ToolRunner.run(conf, new QRFirstJob(), new String[]{ "-input", aBtPath.toString(), "-output", qPath.toString(), "-mis",Integer.toString(mis), "-colsize", Integer.toString(k+p), "-reduceSchedule", reduceSchedule}); btPath = new Path(outputPath, String.format("Bt-job-%d", i + 1)); BtJob.run(conf, inputPath, btPath, qPath.toString(), k, p, outerBlockHeight, i == q - 1 ? Math.min(1000, reduceTasks) : reduceTasks, i == q - 1, reduceSchedule, mis); }*/ cmUpperTriangDenseMatrix bbt = loadAndSumUpperTriangMatrices(fs, new Path(btPath, BtJob.OUTPUT_BBT + "-*"), conf); // convert bbt to something our eigensolver could understand assert bbt.numColumns() == k + p; double[][] bbtSquare = new double[k + p][]; for (int i = 0; i < k + p; i++) { bbtSquare[i] = new double[k + p]; } for (int i = 0; i < k + p; i++) { for (int j = i; j < k + p; j++) { bbtSquare[i][j] = bbtSquare[j][i] = bbt.get(i, j); } } svalues = new double[k + p]; // try something else. EigenSolver eigenWrapper = new EigenSolver(bbtSquare); double[] eigenva2 = eigenWrapper.getWR(); for (int i = 0; i < k + p; i++) { svalues[i] = Math.sqrt(eigenva2[i]); // sqrt? } // save/redistribute UHat double[][] uHat = eigenWrapper.getVL(); //double[][] uHat = eigenWrapper.getUHat(); fs.mkdirs(uHatPath); SequenceFile.Writer uHatWriter = SequenceFile.createWriter(fs, conf, uHatPath = new Path(uHatPath, "uhat.seq"), IntWritable.class, VectorWritable.class, CompressionType.BLOCK); int m = uHat.length; IntWritable iw = new IntWritable(); VectorWritable vw = new VectorWritable(); for (int i = 0; i < m; i++) { vw.set(new DenseVector(uHat[i], true)); iw.set(i); uHatWriter.append(iw, vw); } uHatWriter.close(); SequenceFile.Writer svWriter = SequenceFile.createWriter(fs, conf, svPath = new Path(svPath, "svalues.seq"), IntWritable.class, VectorWritable.class, CompressionType.BLOCK); vw.set(new DenseVector(svalues, true)); svWriter.append(iw, vw); svWriter.close(); start = new Date().getTime(); UJob ujob = null; if (computeU) { ujob = new UJob(); ujob.start(conf, new Path(btPath, BtJob.Q_MAT + "-*"), uHatPath, svPath, uPath, k, cUHalfSigma, mis); // actually this is map-only job anyway } VJob vjob = null; if (computeV) { vjob = new VJob(); vjob.start(conf, new Path(btPath, BtJob.OUTPUT_BT + "-*"), uHatPath, svPath, vPath, k, reduceTasks, subRowSize, cVHalfSigma, mis); } if (ujob != null) { ujob.waitForCompletion(); this.uPath = uPath.toString(); } System.out.println("U-Job done "); if (vjob != null) { vjob.waitForCompletion(); this.vPath = vPath.toString(); } end = new Date().getTime(); System.out.println("U-Job+V-Job done " + (end - start)); } catch (InterruptedException exc) { throw new IOException("Interrupted", exc); } catch (ClassNotFoundException exc) { throw new IOException(exc); } }
From source file:org.apache.accumulo.server.tabletserver.log.MultiReaderTest.java
License:Apache License
@Test public void testMultiReader() throws IOException { Path manyMaps = new Path("file://" + root.getRoot().getAbsolutePath() + "/manyMaps"); MultiReader reader = new MultiReader(fs, manyMaps); IntWritable key = new IntWritable(); BytesWritable value = new BytesWritable(); for (int i = 0; i < 1000; i++) { if (i == 10) continue; assertTrue(reader.next(key, value)); assertEquals(i, key.get());/*from ww w .j a v a 2 s .c o m*/ } assertEquals(value.compareTo(new BytesWritable("someValue".getBytes())), 0); assertFalse(reader.next(key, value)); key.set(500); assertTrue(reader.seek(key)); scan(reader, 500); key.set(10); assertFalse(reader.seek(key)); scan(reader, 10); key.set(1000); assertFalse(reader.seek(key)); assertFalse(reader.next(key, value)); key.set(-1); assertFalse(reader.seek(key)); key.set(0); assertTrue(reader.next(key, value)); assertEquals(0, key.get()); reader.close(); fs.deleteRecursively(new Path(manyMaps, "even")); reader = new MultiReader(fs, manyMaps); key.set(501); assertTrue(reader.seek(key)); scanOdd(reader, 501); key.set(1000); assertFalse(reader.seek(key)); assertFalse(reader.next(key, value)); key.set(-1); assertFalse(reader.seek(key)); key.set(1); assertTrue(reader.next(key, value)); assertEquals(1, key.get()); reader.close(); }