List of usage examples for org.apache.hadoop.io Text decode
public static String decode(byte[] utf8) throws CharacterCodingException
From source file:com.cqx.mr.MRSearchAuto.java
public void searchHBase(int numOfDays) throws IOException, InterruptedException, ClassNotFoundException { long startTime; long endTime; Configuration conf = HBaseConfiguration.create(); conf.set("hbase.zookeeper.quorum", "node2,node3,node4"); conf.set("fs.default.name", "hdfs://node1"); conf.set("mapred.job.tracker", "node1:54311"); /*//from ww w. j a v a2 s . c o m * ?map */ conf.set("search.license", "C87310"); conf.set("search.color", "10"); conf.set("search.direction", "2"); Job job = new Job(conf, "MRSearchHBase"); System.out.println("search.license: " + conf.get("search.license")); job.setNumReduceTasks(0); job.setJarByClass(MRSearchAuto.class); Scan scan = new Scan(); scan.addFamily(FAMILY_NAME); byte[] startRow = Bytes.toBytes("2011010100000"); byte[] stopRow; switch (numOfDays) { case 1: stopRow = Bytes.toBytes("2011010200000"); break; case 10: stopRow = Bytes.toBytes("2011011100000"); break; case 30: stopRow = Bytes.toBytes("2011020100000"); break; case 365: stopRow = Bytes.toBytes("2012010100000"); break; default: stopRow = Bytes.toBytes("2011010101000"); } // ?key scan.setStartRow(startRow); scan.setStopRow(stopRow); TableMapReduceUtil.initTableMapperJob(TABLE_NAME, scan, SearchMapper.class, ImmutableBytesWritable.class, Text.class, job); Path outPath = new Path("searchresult"); HDFS_File file = new HDFS_File(); file.DelFile(conf, outPath.getName(), true); // FileOutputFormat.setOutputPath(job, outPath);// startTime = System.currentTimeMillis(); job.waitForCompletion(true); endTime = System.currentTimeMillis(); System.out.println("Time used: " + (endTime - startTime)); System.out.println("startRow:" + Text.decode(startRow)); System.out.println("stopRow: " + Text.decode(stopRow)); }
From source file:com.main.MRSearchMain.java
public void searchHBase(int numOfDays) throws IOException, InterruptedException, ClassNotFoundException { long startTime; long endTime; String path = "/home/hadoop/app/hadoop-2.0.0-cdh4.3.0/etc/hadoop/"; Configuration conf = HBaseConfiguration.create(); // conf.set("hbase.zookeeper.quorum", "streamslab.localdomain"); // conf.set("fs.default.name", "hdfs://streamslab.localdomain:8020"); // conf.set("mapred.job.tracker", "hdfs://streamslab.localdomain:50300"); // conf.set("fs.hdfs.impl", // org.apache.hadoop.hdfs.DistributedFileSystem.class.getName()); conf.set("fs.file.impl", org.apache.hadoop.fs.LocalFileSystem.class.getName()); //?,FileSystem? conf.addResource(new Path(path + "core-site.xml")); conf.addResource(new Path(path + "hdfs-site.xml")); conf.addResource(new Path(path + "mapred-site.xml")); /* /* w ww . j av a2 s .co m*/ * ?map */ conf.set("search.license", "C87310"); conf.set("search.color", "10"); conf.set("search.direction", "2"); Job job = new Job(conf, "MRSearchHBase"); System.out.println("search.license: " + conf.get("search.license")); job.setNumReduceTasks(0); job.setJarByClass(MRSearchMain.class); Scan scan = new Scan(); scan.addFamily(FAMILY_NAME); byte[] startRow = Bytes.toBytes("2011010100000"); byte[] stopRow; switch (numOfDays) { case 1: stopRow = Bytes.toBytes("2011010200000"); break; case 10: stopRow = Bytes.toBytes("2011011100000"); break; case 30: stopRow = Bytes.toBytes("2011020100000"); break; case 365: stopRow = Bytes.toBytes("2012010100000"); break; default: stopRow = Bytes.toBytes("2011010101000"); } // ?key scan.setStartRow(startRow); scan.setStopRow(stopRow); TableMapReduceUtil.initTableMapperJob(TABLE_NAME, scan, SearchMapper.class, ImmutableBytesWritable.class, Text.class, job); Path outPath = new Path("searchresult"); LOG.info("outPath:" + outPath.toString()); //hdfs FileSystem file = null; try { file = FileSystem.get(conf); } catch (IOException e) { e.printStackTrace(); } // HDFS_File file = new HDFS_File(); // file.DelFile(conf, outPath.getName(), true); // //"hdfs://streamslab.localdomain:8020/ if (file.exists(outPath)) { file.delete(outPath, true); LOG.info("=====delPath " + outPath.toString() + "====="); } FileOutputFormat.setOutputPath(job, outPath);// startTime = System.currentTimeMillis(); job.waitForCompletion(true); endTime = System.currentTimeMillis(); LOG.info("Time used: " + (endTime - startTime)); LOG.info("startRow:" + Text.decode(startRow)); LOG.info("stopRow: " + Text.decode(stopRow)); }
From source file:com.mr.SearchMapper.java
protected void map(ImmutableBytesWritable key, Result value, Context context) throws InterruptedException, IOException { String string = ""; String tempString;//from w w w. j a va 2 s.c om /**/ for (int i = 0; i < 1; i++) { // /mapfilter tempString = Text.decode(value.getValue(FAMILY_NAME, QUALIFIER_NAME[i])); if (tempString.equals(/* strConditionStrings[i] */"C87310")) { LOG.info("C87310. conf: " + strConditionStrings[0]); if (tempString.equals(strConditionStrings[i])) { string = string + tempString + " "; } else { return; } } else { return; } } word.set(string); context.write(null, word); }
From source file:org.apache.accumulo.core.file.rfile.bcfile.Utils.java
License:Apache License
/** * Read a String as a VInt n, followed by n Bytes in Text format. * * @param in/*from w w w . j a v a2 s. c o m*/ * The input stream. * @return The string */ public static String readString(DataInput in) throws IOException { int length = readVInt(in); if (length == -1) return null; byte[] buffer = new byte[length]; in.readFully(buffer); return Text.decode(buffer); }
From source file:org.mitre.ccv.mapred.io.KmerEntropyPairWritable.java
License:Open Source License
@Override public void readFields(DataInput in) throws IOException { in.readByte();/*from ww w.j a v a 2 s . co m*/ this.value = in.readDouble(); //this.key = in.readUTF(); /** * Adapted from Text */ int length = in.readInt(); byte[] bytes = new byte[length]; in.readFully(bytes, 0, length); this.key = Text.decode(bytes); }
From source file:org.mitre.ccv.mapred.io.KmerEntropyPairWritable.java
License:Open Source License
public void readFields(ByteBuffer in) throws IOException { in.get(); // readByte() this.value = in.getDouble(); int length = in.getInt(); byte[] bytes = new byte[length]; in.get(bytes, 0, length);/*from w w w .j av a2 s .c o m*/ this.key = Text.decode(bytes); }