Example usage for org.apache.hadoop.io Text Text

Introduction

In this page you can find the example usage for org.apache.hadoop.io Text Text.

Prototype

public Text(byte[] utf8)

Source Link

Document

Construct from a byte array.

Usage

From source file:TestStringRelevance.java

License:Apache License

@Override
public void setUp() throws Exception {
    fs.delete(new Path(INPUT), true);
    fs.delete(new Path(QUERY), true);
    fs.delete(new Path(OUTPUT), true);

    inputTap = new Hfs(new SequenceFile(new Fields("str1", "str2")), INPUT);
    TapCollector coll = new TapCollector(inputTap, new JobConf());
    coll.add(tuple1);/*from   ww w .ja va  2s .  co m*/
    coll.add(tuple2);
    coll.add(tuple3);
    coll.add(tuple4);
    coll.add(tuple5);
    coll.add(tuple6);
    coll.add(tuple7);
    coll.add(tuple8);
    coll.add(tuple9);
    coll.close();

    keyTap = new Hfs(new SequenceFile(new Fields("str")), QUERY);
    coll = new TapCollector(keyTap, new JobConf());
    coll.add(new Tuple(new Text("nathan@rapleaf.com")));
    coll.add(new Tuple(new Text("1@gmail.com")));
    coll.add(new Tuple(new Text("2@gmail.com")));
    coll.add(new Tuple(new Text("6@gmail.com")));
    coll.close();

    outputTap = new Hfs(new SequenceFile(new Fields("str1", "str2")), OUTPUT);
}

From source file:ZipFileRecordReader.java

License:Apache License

/**
 * This is where the magic happens, each ZipEntry is decompressed and
 * readied for the Mapper. The contents of each file is held *in memory*
 * in a BytesWritable object./*from   w  ww. j  av  a  2 s . c  o  m*/
 *
 * If the ZipFileInputFormat has been set to Lenient (not the default),
 * certain exceptions will be gracefully ignored to prevent a larger job
 * from failing.
 */
@Override
public boolean nextKeyValue() throws IOException, InterruptedException {
    ZipEntry entry = null;
    try {
        entry = zip.getNextEntry();
    } catch (ZipException e) {
        if (ZipFileInputFormat.getLenient() == false)
            throw e;
    }

    // Sanity check
    if (entry == null) {
        isFinished = true;
        return false;
    }

    // Filename
    currentKey = new Text(entry.getName());

    // Read the file contents
    ByteArrayOutputStream bos = new ByteArrayOutputStream();
    byte[] temp = new byte[8192];
    while (true) {
        int bytesRead = 0;
        try {
            bytesRead = zip.read(temp, 0, 8192);
        } catch (EOFException e) {
            if (ZipFileInputFormat.getLenient() == false)
                throw e;
            return false;
        }
        if (bytesRead > 0)
            bos.write(temp, 0, bytesRead);
        else
            break;
    }
    zip.closeEntry();

    // Uncompressed contents
    currentValue = new BytesWritable(bos.toByteArray());
    return true;
}

From source file:DisplayClustering.java

License:Apache License

protected static void writeSampleData(Path output) throws IOException {
    Configuration conf = new Configuration();
    FileSystem fs = FileSystem.get(output.toUri(), conf);
    SequenceFile.Writer writer = new SequenceFile.Writer(fs, conf, output, Text.class, VectorWritable.class);
    try {//from ww w  . j ava2s.c  o m
        int i = 0;
        for (VectorWritable vw : SAMPLE_DATA) {
            writer.append(new Text("sample_" + i++), vw);
        }
    } finally {
        Closeables.close(writer, false);
    }
}

From source file:TestString.java

License:Apache License

@Test
public void testStringSubstring() throws Exception {
    Text text = new Text("string");
    Text text1 = new Text();
    Text text2 = new Text();

    long start = System.nanoTime();
    for (int i = 0; i < 100000000; i++) {
        String str = text.toString();
        String str1 = str.substring(0, 2);
        String str2 = str.substring(3, str.length());
        text1.set(str1);/*from  w ww.jav a2  s  . c  o m*/
        text2.set(str2);
    }
    long end = System.nanoTime();
    System.out.println("TextStringSubString");
    System.out.println("text1: " + text1.toString());
    System.out.println("text2: " + text2.toString());
    System.out.println("Elapsed Time: " + (end - start) / 1000000000f + " seconds.");
}

From source file:TestString.java

License:Apache License

@Test
public void testTextSubstring() throws Exception {
    Text text = new Text("string");
    Text text1 = new Text();
    Text text2 = new Text();

    long start = System.nanoTime();
    for (int i = 0; i < 100000000; i++) {
        text1.set(text.getBytes(), 0, 2);
        text2.set(text.getBytes(), 3, text.getLength() - 3);
    }/* w  w w . j  a  v  a  2 s . c o m*/
    long end = System.nanoTime();
    System.out.println("TestTextSubString");
    System.out.println("text1: " + text1.toString());
    System.out.println("text2: " + text2.toString());
    System.out.println("Elapsed Time: " + (end - start) / 1000000000f + " seconds.");
}

From source file:Deducer.java

License:Apache License

public void reduce(Text key, Iterator<Text> values, OutputCollector<Text, Text> output, Reporter reporter)
        throws IOException {
    Set<String> attackers = new TreeSet<String>();
    while (values.hasNext()) {
        String valStr = values.next().toString();
        attackers.add(valStr);/*w  w  w.j  av  a  2s  .  c  o m*/
    }
    output.collect(key, new Text(attackers.toString()));
}

From source file:PageRankIterationReducerTest.java

public void test1() {
    Text key = new Text("testKey");
    PageRankFollower pageRankFollower = new PageRankFollower(new Text(key.toString()), 1.0, 0,
            new ArrayList<Text>());
    List<PageRankFollower> values = new ArrayList<PageRankFollower>();
    values.add(pageRankFollower);//ww  w.j  a v a2  s . c  o m

    PageRankFollower result = (new PageRankIterationReducer()).calculatePageRank(key, values);
    System.out.println("result: " + result);

}

From source file:TestBAM.java

License:Open Source License

@Override
protected void map(LongWritable ignored, SAMRecordWritable wrec,
        org.apache.hadoop.mapreduce.Mapper<LongWritable, SAMRecordWritable, Text, SAMRecordWritable>.Context ctx)
        throws InterruptedException, IOException {
    final SAMRecord record = wrec.get();
    int len = record.getInferredInsertSize();//record.getAlignmentEnd() - record.getAlignmentStart();
    int chr = Integer.parseInt(record.getReferenceName());
    if (Math.abs(len) > 1000 && chr == 20) {
        System.out.println(record.toString());
        System.out.println("AbsLen:" + Integer.toString(Math.abs(len)));
        ctx.write(new Text(wrec.get().getReadName()), wrec);
    }/*from  w  w w  .  jav a2s .c  om*/
}

From source file:TaggedMapOutput.java

License:Apache License

public TaggedMapOutput() {
    this.tag = new Text("");
}

From source file:SampleUdf.java

License:Apache License

public Text evaluate(final Text s, Text sleepTime) throws InterruptedException {

    Long time = 180 * 1000L;//from w w w.  j  a  va2  s.c  om

    if (sleepTime != null) {
        time = Long.parseLong(sleepTime.toString()) * 1000L;
    }

    System.out.println("Sleep Time : " + time);

    Thread.sleep(time);

    if (s == null) {
        return null;
    }

    return new Text(s.toString().toLowerCase());
}