List of usage examples for org.apache.hadoop.io ArrayWritable set
public void set(Writable[] values)
From source file:edu.ub.ahstfg.indexer.mapred.IndexRecordReader.java
License:Open Source License
@Override public synchronized boolean next(IntWritable key, ArrayWritable value) throws IOException { if (iMapper >= numMachines) { return false; }// w w w.ja v a2s . c o m key.set(iMapper); int nDocs = docsPerMapper[iMapper]; Writable[] outSet = new Writable[nDocs]; for (int i = 0; i < nDocs; i++) { if (!lineReader.next(lineKey, lineValue)) { return false; } String[] pair = Utils.trimStringArray(lineValue.toString().split("\t")); if (pair.length == 2) { if (!pair[0].equals(FeatureDescriptor.KEY)) { String[] keyTerms = pair[1].split(";"); String[] keywords = keyTerms[0].split(":"); String[] terms = keyTerms[1].split(":"); if (!keywords[0].equals("keywords") || !terms[0].equals("terms")) { continue; } try { outSet[i] = new DocumentDescriptor(pair[0], Utils.stringArray2ShortArray(Utils.trimStringArray(terms[1].split(","))), Utils.stringArray2ShortArray(Utils.trimStringArray(keywords[1].split(",")))); } catch (ArrayIndexOutOfBoundsException ex) { outSet[i] = new DocumentDescriptor(pair[0], Utils.stringArray2ShortArray(Utils.trimStringArray(terms[1].split(","))), new short[0]); } } else { i--; } } } value.set(outSet); iMapper++; return true; }
From source file:edu.ub.ahstfg.io.WritableConverter.java
License:Open Source License
/** * Converts a long LinkedList to ArrayWritable. * @param input Long LinkedList to convert. * @return Converted ArrayWritable./* ww w . j a va2s . c o m*/ */ public static ArrayWritable LinkedListShort2ArrayWritable(LinkedList<Short> input) { ArrayWritable ret = new ArrayWritable(IntWritable.class); Writable[] ws = new Writable[input.size()]; for (int i = 0; i < input.size(); i++) { ws[i] = new IntWritable(input.get(i)); } ret.set(ws); return ret; }
From source file:edu.ub.ahstfg.io.WritableConverter.java
License:Open Source License
/** * Converts String LinkedList to ArrayWritable. * @param input String LinkedList to convert. * @return Converted ArrayWritable./*from w w w .ja v a2s . co m*/ */ public static ArrayWritable LinkedListString2ArrayWritable(LinkedList<String> input) { ArrayWritable ret = new ArrayWritable(Text.class); Writable[] ws = new Writable[input.size()]; for (int i = 0; i < input.size(); i++) { ws[i] = new Text(input.get(i)); } ret.set(ws); return ret; }
From source file:edu.ub.ahstfg.io.WritableConverter.java
License:Open Source License
/** * Converts static long array to ArrayWritable. * @param input Static long array to convert. * @return Converted ArrayWritable./*from ww w .j av a 2 s.co m*/ */ public static ArrayWritable shortArray2ArrayWritable(short[] input) { ArrayWritable ret = new ArrayWritable(IntWritable.class); IntWritable[] t = new IntWritable[input.length]; int i = 0; for (short s : input) { t[i] = new IntWritable((int) s); i++; } ret.set(t); return ret; }
From source file:edu.ub.ahstfg.io.WritableConverter.java
License:Open Source License
/** * Converts a static String array to ArrayWritable. * @param input Static String array to convert. * @return Converted ArrayWritable.//from w ww.j av a 2 s . c o m */ public static ArrayWritable stringArray2ArrayWritable(String[] input) { ArrayWritable ret = new ArrayWritable(Text.class); Text[] t = new Text[input.length]; int i = 0; for (String s : input) { t[i] = new Text(s); i++; } ret.set(t); return ret; }
From source file:edu.umn.cs.spatialHadoop.mapred.SpatialRecordReader.java
License:Open Source License
/** * Reads all shapes left in the current block in one shot. This function * runs a loop where it keeps reading shapes by calling the method * {@link #nextShape(Shape)} until one of the following conditions happen. * 1. The whole file is read. No more records to read. * 2. Number of parsed records reaches the threshold defined by the * configuration parameter spatialHadoop.mapred.MaxShapesPerRead. * To disable this check, set the configuration parameter to -1 * 3. Total size of parsed data from file reaches the threshold defined by * the configuration parameter spatialHadoop.mapred.MaxBytesPerRead. * To disable this check, set the configuration parameter to -1. * /*from w ww .j a va2 s. c o m*/ * @param shapes * @return * @throws IOException */ protected boolean nextShapes(ArrayWritable shapes) throws IOException { // Prepare a vector that will hold all objects in this Vector<Shape> vshapes = new Vector<Shape>(); try { Shape stockObject = (Shape) shapes.getValueClass().newInstance(); // Reached the end of this split if (getFilePosition() >= end) return false; long initialReadPos = getPos(); long readBytes = 0; // Read all shapes in this block while ((maxShapesInOneRead <= 0 || vshapes.size() < maxShapesInOneRead) && (maxBytesInOneRead <= 0 || readBytes < maxBytesInOneRead) && nextShape(stockObject)) { vshapes.add(stockObject.clone()); readBytes = getPos() - initialReadPos; } // Store them in the return value shapes.set(vshapes.toArray(new Shape[vshapes.size()])); return !vshapes.isEmpty(); } catch (InstantiationException e1) { e1.printStackTrace(); } catch (IllegalAccessException e1) { e1.printStackTrace(); } catch (OutOfMemoryError e) { LOG.error("Error reading shapes. Stopped with " + vshapes.size() + " shapes"); throw e; } return false; }
From source file:io.aos.hdfs.ArrayWritableTest.java
License:Apache License
@Test public void test() throws IOException { // vv ArrayWritableTest ArrayWritable writable = new ArrayWritable(Text.class); // ^^ ArrayWritableTest writable.set(new Text[] { new Text("cat"), new Text("dog") }); TextArrayWritable dest = new TextArrayWritable(); WritableUtils.cloneInto(dest, writable); assertThat(dest.get().length, is(2)); // TODO: fix cast, also use single assert assertThat((Text) dest.get()[0], is(new Text("cat"))); assertThat((Text) dest.get()[1], is(new Text("dog"))); Text[] copy = (Text[]) dest.toArray(); assertThat(copy[0], is(new Text("cat"))); assertThat(copy[1], is(new Text("dog"))); }
From source file:mapreduceindexfiles.WholeFileRecordReader.java
@Override public boolean next(Text k, ArrayWritable v) throws IOException { //throw new UnsupportedOperationException("Not supported yet."); //To change body of generated methods, choose Tools | Templates. //if (!processed) { //byte[] contents = new byte[(int) fileSplit.getLength()]; Path file = fileSplit.getPath(); FileSystem fs = file.getFileSystem(conf); FSDataInputStream in = null;/*from ww w .j a v a 2 s .c o m*/ try { in = fs.open(file); String linePosition = Long.toString(in.getPos()); BufferedReader reader = new BufferedReader(new InputStreamReader(in)); //IOUtils.readFully(in, contents, 0, contents.length); String line = ""; while (reader.ready()) { line = reader.readLine(); } Text values[] = new Text[2]; values[0] = new Text(linePosition); values[1] = new Text(line); v.set(values); write = v; } finally { IOUtils.closeStream(in); } //processed = true; return true; // } //return false; }
From source file:org.apache.carbondata.hive.CarbonHiveRecordReader.java
License:Apache License
@Override public boolean next(Void aVoid, ArrayWritable value) throws IOException { if (carbonIterator.hasNext()) { Object obj = readSupport.readRow(carbonIterator.next()); recordReaderCounter++;/*from w ww. j av a 2 s.co m*/ Writable[] objArray = (Writable[]) obj; Writable[] sysArray = new Writable[value.get().length]; if (columnIds != null && columnIds.length > 0 && objArray.length == columnIds.length) { for (int i = 0; i < columnIds.length; i++) { sysArray[columnIds[i]] = objArray[i]; } value.set(sysArray); } else { value.set(objArray); } return true; } else { return false; } }
From source file:org.apache.hama.bsp.TestCheckpoint.java
License:Apache License
public void testPeerRecovery() throws Exception { Configuration config = new Configuration(); config.set(SyncServiceFactory.SYNC_CLIENT_CLASS, TempSyncClient.class.getName()); config.set(Constants.FAULT_TOLERANCE_CLASS, AsyncRcvdMsgCheckpointImpl.class.getName()); config.setBoolean(Constants.CHECKPOINT_ENABLED, true); int port = BSPNetUtils.getFreePort(12502); LOG.info("Got port = " + port); config.set(Constants.PEER_HOST, Constants.DEFAULT_PEER_HOST); config.setInt(Constants.PEER_PORT, port); config.set("bsp.output.dir", "/tmp/hama-test_out"); config.set("bsp.local.dir", "/tmp/hama-test"); FileSystem dfs = FileSystem.get(config); BSPJob job = new BSPJob(new BSPJobID("checkpttest", 1), "/tmp"); TaskAttemptID taskId = new TaskAttemptID(new TaskID(job.getJobID(), 1), 1); TestMessageManager messenger = new TestMessageManager(); PeerSyncClient syncClient = SyncServiceFactory.getPeerSyncClient(config); Text txtMessage = new Text("data"); String writeKey = "job_checkpttest_0001/checkpoint/1/"; Writable[] writableArr = new Writable[2]; writableArr[0] = new LongWritable(3L); writableArr[1] = new LongWritable(5L); ArrayWritable arrWritable = new ArrayWritable(LongWritable.class); arrWritable.set(writableArr); syncClient.storeInformation(writeKey, arrWritable, true, null); String writePath = "checkpoint/job_checkpttest_0001/3/1"; FSDataOutputStream out = dfs.create(new Path(writePath)); for (int i = 0; i < 5; ++i) { out.writeUTF(txtMessage.getClass().getCanonicalName()); txtMessage.write(out);//from www . j av a 2s. c o m } out.close(); @SuppressWarnings("unused") BSPPeer<?, ?, ?, ?, Text> bspTask = new TestBSPPeer(job, config, taskId, new Counters(), 3L, (BSPPeerSyncClient) syncClient, messenger, TaskStatus.State.RECOVERING); BSPMessageBundle<Text> bundleRead = messenger.getLoopbackBundle(); assertEquals(5, bundleRead.size()); String recoveredMsg = bundleRead.iterator().next().toString(); assertEquals(recoveredMsg, "data"); dfs.delete(new Path("checkpoint"), true); }