List of usage examples for org.apache.hadoop.io SequenceFile.Reader SequenceFile.Reader
public Reader(Configuration conf, Option... opts) throws IOException
From source file:DAAL.KmeansStep1Mapper.java
License:Open Source License
@Override public void map(Object key, Text value, Context context) throws IOException, InterruptedException { /* Read a data set */ String filePath = "/Hadoop/Kmeans/data/" + value; double[] data = new double[nFeatures * nVectorsInBlock]; readData(filePath, nFeatures, nVectorsInBlock, data); DaalContext daalContext = new DaalContext(); HomogenNumericTable ntData = new HomogenNumericTable(daalContext, data, nFeatures, nVectorsInBlock); /* Create an algorithm to compute k-means on local nodes */ DistributedStep1Local kmeansLocal = new DistributedStep1Local(daalContext, Double.class, Method.defaultDense, nClusters); /* Get the centroids table computed in step 2 */ SequenceFile.Reader reader = new SequenceFile.Reader(new Configuration(), SequenceFile.Reader.file(new Path("/Hadoop/Kmeans/initResults/centroids"))); IntWritable step1key = new IntWritable(); WriteableData step1value = new WriteableData(); reader.next(step1key, step1value);//from w ww . j av a 2 s . co m reader.close(); HomogenNumericTable centroids = (HomogenNumericTable) step1value.getObject(daalContext); /* Set the algorithm parameters */ kmeansLocal.input.set(InputId.data, ntData); kmeansLocal.input.set(InputId.inputCentroids, centroids); /* Compute k-means on local nodes */ PartialResult pres = kmeansLocal.compute(); /* Write the data prepended with a data set sequence number. Needed to know the position of the data set in the input data */ context.write(new IntWritable(0), new WriteableData(index, pres)); daalContext.dispose(); index += totalTasks; }
From source file:DAAL.QRStep3Mapper.java
License:Open Source License
@Override public void map(IntWritable step2key, WriteableData step2value, Context context) throws IOException, InterruptedException { DaalContext daalContext = new DaalContext(); SequenceFile.Reader reader = new SequenceFile.Reader(new Configuration(), SequenceFile.Reader.file(new Path("/Hadoop/QR/step1/step1x" + step2value.getId()))); IntWritable step1key = new IntWritable(); WriteableData step1value = new WriteableData(); reader.next(step1key, step1value);/* w w w . j ava 2s. com*/ reader.close(); DataCollection s1 = (DataCollection) step1value.getObject(daalContext); DataCollection s2 = (DataCollection) step2value.getObject(daalContext); /* Create an algorithm to compute QR decomposition on the master node */ DistributedStep3Local qrStep3Local = new DistributedStep3Local(daalContext, Double.class, Method.defaultDense); qrStep3Local.input.set(DistributedStep3LocalInputId.inputOfStep3FromStep1, s1); qrStep3Local.input.set(DistributedStep3LocalInputId.inputOfStep3FromStep2, s2); /* Compute QR decomposition in step 3 */ qrStep3Local.compute(); Result result = qrStep3Local.finalizeCompute(); HomogenNumericTable Qi = (HomogenNumericTable) result.get(ResultId.matrixQ); SequenceFile.Writer writer = SequenceFile.createWriter(new Configuration(), SequenceFile.Writer.file(new Path("/Hadoop/QR/Output/Qx" + step2value.getId())), SequenceFile.Writer.keyClass(IntWritable.class), SequenceFile.Writer.valueClass(WriteableData.class)); writer.append(new IntWritable(0), new WriteableData(step2value.getId(), Qi)); writer.close(); daalContext.dispose(); }
From source file:DAAL.SVDStep3Mapper.java
License:Open Source License
@Override public void map(IntWritable step2key, WriteableData step2value, Context context) throws IOException, InterruptedException { DaalContext daalContext = new DaalContext(); SequenceFile.Reader reader = new SequenceFile.Reader(new Configuration(), SequenceFile.Reader.file(new Path("/Hadoop/SVD/step1/step1x" + step2value.getId()))); IntWritable step1key = new IntWritable(); WriteableData step1value = new WriteableData(); reader.next(step1key, step1value);/*from w w w . ja v a2 s .c o m*/ reader.close(); DataCollection s1 = (DataCollection) step1value.getObject(daalContext); DataCollection s2 = (DataCollection) step2value.getObject(daalContext); /* Create an algorithm to compute SVD on the master node */ DistributedStep3Local svdStep3Local = new DistributedStep3Local(daalContext, Double.class, Method.defaultDense); svdStep3Local.input.set(DistributedStep3LocalInputId.inputOfStep3FromStep1, s1); svdStep3Local.input.set(DistributedStep3LocalInputId.inputOfStep3FromStep2, s2); /* Compute SVD in step 3 */ svdStep3Local.compute(); Result result = svdStep3Local.finalizeCompute(); HomogenNumericTable Ui = (HomogenNumericTable) result.get(ResultId.leftSingularMatrix); SequenceFile.Writer writer = SequenceFile.createWriter(new Configuration(), SequenceFile.Writer.file(new Path("/Hadoop/SVD/Output/Ux" + step2value.getId())), SequenceFile.Writer.keyClass(IntWritable.class), SequenceFile.Writer.valueClass(WriteableData.class)); writer.append(new IntWritable(0), new WriteableData(step2value.getId(), Ui)); writer.close(); daalContext.dispose(); }
From source file:jadoop.util.SingleRecordSplitSequenceFileInputFormat.java
License:Open Source License
@Override public List<InputSplit> getSplits(JobContext job) throws IOException { List<InputSplit> splits = new ArrayList<InputSplit>(); List<FileStatus> files = listStatus(job); for (FileStatus file : files) { Path seqFilePath = file.getPath(); SequenceFile.Reader reader = new SequenceFile.Reader(job.getConfiguration(), Reader.file(seqFilePath)); Text key = new Text(); TextArrayWritable val = new TextArrayWritable(); long prevPos = 0; while (reader.next(key, val)) { long curPos = reader.getPosition(); FileSplit split = new FileSplit(seqFilePath, prevPos, curPos - prevPos, null); splits.add(split);/* w ww . ja va2s . c o m*/ prevPos = curPos; } reader.close(); } return splits; }