List of usage examples for org.apache.hadoop.io LongWritable set
public void set(long value)
From source file:org.rassee.omniture.hadoop.mapred.OmnitureDataFileRecordReader.java
License:Open Source License
public boolean next(LongWritable key, Text value) throws IOException { while (pos <= end) { key.set(pos); int newSize = lineReader.readLine(value, maxLineLength, Math.max((int) Math.min(Integer.MAX_VALUE, end - pos), maxLineLength)); pos += newSize;// w w w . j a v a 2 s . co m if (newSize == 0) { return false; } String line = value.toString().replaceAll("\\\\\t", "").replaceAll("\\\\(\n|\r|\r\n)", ""); value.set(line); if (newSize < maxLineLength) { return true; } // line too long. try again LOG.info("Skipped line of size " + newSize + " at pos " + (pos - newSize)); } return false; }
From source file:org.vilcek.hive.kv.KVHiveRecordReader.java
License:Apache License
@Override public boolean next(LongWritable k, MapWritable v) throws IOException { boolean ret = iter.hasNext(); if (ret) {// w w w. j a v a 2 s .c o m current = iter.next(); k.set(cnt); v.clear(); List<String> majorKeysList = current.getKey().getMajorPath(); List<String> minorKeysList = current.getKey().getMinorPath(); for (int i = 0; i < majorKeyLabelsArray.length; i++) { try { String key = majorKeyLabelsArray[i]; String value = majorKeysList.get(i); if (!value.equals(SERIALIZED_NULL)) { v.put(new Text(key), new Text(value)); } } catch (ArrayIndexOutOfBoundsException e) { } } byte[] value = current.getValue().getValue(); if (!value.toString().equals(SERIALIZED_NULL)) { if (Format.AVRO == current.getValue().getFormat() && binding != null) { try { JsonRecord object = binding.toObject(current.getValue()); JsonNode jsonNode = object.getJsonNode(); value = jsonNode.toString().getBytes("UTF8"); } catch (Throwable ignored) { } } if (minorKeysList.isEmpty()) { v.put(new Text("value"), new Text(value)); } else { for (int j = 0; j < minorKeysList.size(); j++) { String key = minorKeysList.get(j); v.put(new Text(key), new Text(value)); } } } cnt++; return ret; } else { return false; } }
From source file:org.vroyer.hive.solr.SolrReader.java
License:Open Source License
@Override public boolean next(LongWritable keyHolder, MapWritable valueHolder) throws IOException { if (StringUtils.isBlank(facetMapping)) { SolrDocument doc = cursor.nextDocument(); if (doc == null) { return false; }/*from w ww .j a v a 2s . c o m*/ keyHolder.set(pos++); Object[] values = new Object[solrColumns.length]; for (int i = 0; i < solrColumns.length; i++) { values[i] = doc.getFieldValue(solrColumns[i]); } setValueHolder(valueHolder, values); } else { FacetEntry facetEntry = cursor.nextFacetEntry(); if (facetEntry == null) { return false; } keyHolder.set(pos++); setValueHolder(valueHolder, new Object[] { facetEntry.getValue(), facetEntry.getCount() }); } return true; }
From source file:protobuf.mapred.ProtobufRecordReader.java
License:Open Source License
public synchronized boolean next(LongWritable key, BytesWritable value) throws IOException { if (recordReader.isBlockConsumed(splitLength)) { LOG.info("Consumed all the split"); key = null;// ww w . j a va 2 s.com value = null; return false; } recordio.RecordReader.Buffer data = recordReader.read(); if (data == null) { LOG.info("get EOF, consumed all the file"); key = null; value = null; return false; } pos = recordReader.getConsumedBytes() + start; key.set(pos); value.set(data.buffer, data.offset, data.length); return true; }
From source file:redpoll.examples.sogou.SogouRecordReader.java
License:Apache License
public synchronized boolean next(LongWritable key, Document value) throws IOException { if (pos < end) { long docPos = in.nextDoc((SogouDocument) value); if (docPos < 0) return false; key.set(start + docPos); pos = start + in.getPosition();/*w w w.j a v a 2 s . com*/ return true; } return false; }
From source file:sg.edu.nus.harmonious.utils.HUtils.java
License:Apache License
private static void testBinarySearch(long[] v, long query[], int[] ans, int orderFlag) { LongWritable[] vals = new LongWritable[v.length]; for (int i = 0; i < v.length; i++) vals[i] = new LongWritable(v[i]); LongWritable k = new LongWritable(); for (int i = 0; i < query.length; i++) { k.set(query[i]); int ret = binarySearch(vals, k, orderFlag); System.out.println((ans[i] == ret) + ": " + ans[i] + ":" + ret); }//from ww w. j a v a 2 s.c om }
From source file:statics.UDAFPercentile.java
License:Apache License
/** * Increment the State object with o as the key, and i as the count. *//*from w w w . j ava 2 s . c o m*/ private static void increment(State s, LongWritable o, long i) { couter++; if (s.counts == null) { s.counts = new HashMap<LongWritable, LongWritable>(); } LongWritable count = s.counts.get(o); if (count == null) { // We have to create a new object, because the object o belongs // to the code that creates it and may get its value changed. LongWritable key = new LongWritable(); key.set(o.get()); s.counts.put(key, new LongWritable(i)); } else { count.set(count.get() + i); } }
From source file:StorageEngineClient.ColumnStorageIRecordReader.java
License:Open Source License
@Override public boolean next(LongWritable key, IRecord value) throws IOException { if (currentrec >= recnum || icdf == null) return false; key.set(currentrec); if (!icdf.next(value)) return false; currentrec++;/*from w ww .java 2 s .c o m*/ return true; }
From source file:StorageEngineClient.FormatStorageIRecordReader.java
License:Open Source License
@Override public boolean next(LongWritable key, IRecord value) throws IOException { if (currentrec >= recnum) return false; key.set(currentrec); if (!isGZ) {//w w w. ja v a2s . co m if (!ifdf.next(value)) { String err = "FSIR error read:\t" + this.file + ":\tcurrentrec\t" + currentrec + "\trecnum\t" + recnum + "\tbeginline\t" + this.beginline + "\r\nvalue" + value.showstr(); throw new IOException(err); } } else { try { int newSize = in.readLine(tValue); if (newSize == 0) { return false; } } catch (Exception e) { return false; } StringTokenizer stk = new StringTokenizer(tValue.toString(), new String(new char[] { '\01' })); int j = 0; while (stk.hasMoreTokens()) { IRecord.IFType type = this.fieldtypes.get(j); String str = stk.nextToken(); if (!str.equals("\\N")) { if (type.type() == ConstVar.FieldType_Byte) value.addFieldValue(new IRecord.IFValue(Byte.valueOf(str), j)); else if (type.type() == ConstVar.FieldType_Short) value.addFieldValue(new IRecord.IFValue(Short.valueOf(str), j)); else if (type.type() == ConstVar.FieldType_Int) value.addFieldValue(new IRecord.IFValue(Integer.valueOf(str), j)); else if (type.type() == ConstVar.FieldType_Long) value.addFieldValue(new IRecord.IFValue(Long.valueOf(str), j)); else if (type.type() == ConstVar.FieldType_Float) value.addFieldValue(new IRecord.IFValue(Float.valueOf(str), j)); else if (type.type() == ConstVar.FieldType_Double) value.addFieldValue(new IRecord.IFValue(Double.valueOf(str), j)); else if (type.type() == ConstVar.FieldType_String) { if (str.equals("\\NN")) { value.addFieldValue(new IRecord.IFValue("", j)); } else { value.addFieldValue(new IRecord.IFValue(str, j)); } } } else { value.setNull(j); } j++; } } currentrec++; return true; }
From source file:StorageEngineClient.MultiColumnStorageRecordReader.java
License:Open Source License
@Override public boolean next(LongWritable key, IRecord value) throws IOException { if (paths == null || paths.length == 0 || icdf == null) return false; key.set(currentrecnum++); if (icdf.next(value)) return true; currentFile++;/*from w w w.j a v a2 s. c o m*/ while (currentFile < paths.length) { icdf.close(); icdf = new IColumnDataFile(conf); icdf.open(paths[currentFile].toString()); if (icdf.next(value)) return true; currentFile++; } return false; }