Example usage for org.apache.hadoop.io Text Text

Introduction

In this page you can find the example usage for org.apache.hadoop.io Text Text.

Prototype

public Text(byte[] utf8)

Source Link

Document

Construct from a byte array.

Usage

From source file:com.accumulobook.designs.termindex.WikipediaQuery.java

License:Apache License

private Iterable<Entry<Key, Value>> retrieveRecords(Connector conn, Collection<Range> matches)
        throws TableNotFoundException {
    // retrieve original articles
    BatchScanner bscanner = conn.createBatchScanner(WikipediaConstants.ARTICLES_TABLE, auths, 10);
    bscanner.setRanges(matches);//from  ww w . j  a v  a2s . co m

    // fetch only the article contents
    bscanner.fetchColumn(new Text(WikipediaConstants.METADATA_FAMILY),
            new Text(WikipediaConstants.REVISION_QUAL));

    return bscanner;
}

From source file:com.accumulobook.designs.termindex.WikipediaQuery.java

License:Apache License

public void queryDateRange(final Date start, final Date stop) throws TableNotFoundException {

    DateLexicoder dl = new DateLexicoder();

    Scanner scanner = conn.createScanner(WikipediaConstants.INDEX_TABLE, auths);

    // scan over the range of dates specified
    scanner.setRange(new Range(new Text(dl.encode(start)), new Text(dl.encode(stop))));

    // store all article titles returned
    HashSet<Range> matches = new HashSet<>();
    for (Entry<Key, Value> entry : scanner) {
        matches.add(new Range(entry.getKey().getColumnQualifier().toString()));
    }//from   w ww. j av  a2  s.co m

    if (matches.isEmpty()) {
        System.out.println("no results");
        return;
    }

    for (Entry<Key, Value> entry : retrieveRecords(conn, matches)) {
        System.out.println("Title:\t" + entry.getKey().getRow().toString() + "\nRevision:\t"
                + entry.getValue().toString() + "\n");
    }
}

From source file:com.acme.io.JsonStorage.java

License:Apache License

/**
 * Write a tuple to the data store./* w  w  w .ja  v  a  2s  . c o  m*/
 * @param t the tuple to store.
 * @throws IOException if an exception occurs during the write
 */
public void putNext(Tuple t) throws IOException {
    // Build a ByteArrayOutputStream to write the JSON into
    ByteArrayOutputStream baos = new ByteArrayOutputStream(BUF_SIZE);
    // Build the generator
    JsonGenerator json = jsonFactory.createJsonGenerator(baos, JsonEncoding.UTF8);

    // Write the beginning of the top level tuple object
    json.writeStartObject();
    for (int i = 0; i < fields.length; i++) {
        writeField(json, fields[i], t.get(i));
    }
    json.writeEndObject();
    json.close();

    // Hand a null key and our string to Hadoop
    try {
        writer.write(null, new Text(baos.toByteArray()));
    } catch (InterruptedException ie) {
        throw new IOException(ie);
    }
}

From source file:com.alectenharmsel.research.hadoop.MoabLicenseInfoTest.java

@Before
public void setUp() {
    mapDriver = new MapDriver(new MoabLicenseInfo.Map());
    reduceDriver = new ReduceDriver(new MoabLicenseInfo.Reduce());

    key = new Text("cfd_solv_ser-05-11");
    vals = new ArrayList<Text>();
    vals.add(new Text("0,6"));
    vals.add(new Text("0,6"));
    vals.add(new Text("0,6"));
    vals.add(new Text("0,6"));
    vals.add(new Text("2,8"));
    vals.add(new Text("2,8"));
    vals.add(new Text("1,8"));
    vals.add(new Text("0,8"));
    reduceDriver.withInput(key, vals);//from  www  .j av a  2s.c o  m
}

From source file:com.alectenharmsel.research.hadoop.MoabLicenseInfoTest.java

@Test
public void testNoOutputNeeded() throws IOException {
    List<Pair<Text, Text>> res = mapDriver.withInput(new LongWritable(0),
            new Text("05/11 22:58:25  MNodeUpdateResExpression(nyx5624,FALSE,TRUE)")).run();
    Assert.assertTrue(res.isEmpty());/*  w  ww . ja v  a  2 s. c  om*/
}

From source file:com.alectenharmsel.research.hadoop.MoabLicenseInfoTest.java

@Test
public void testLicenseLine() throws IOException {
    mapDriver.withInput(new LongWritable(0), new Text(
            "05/11 22:58:25  INFO:     License cfd_solv_ser        0 of   6 available  (Idle: 33.3%  Active: 66.67%)"))
            .withOutput(new Text("cfd_solv_ser-05-11"), new Text("0,6")).runTest();
}

From source file:com.alectenharmsel.research.MoabLicensesMapper.java

License:Apache License

public void map(LongWritable key, Text contents, Context context) throws IOException, InterruptedException {
    if (contents.toString().contains("License")) {
        String date = "";
        String licenseInfo = "";
        String pkgName = "";
        ArrayList<String> license = new ArrayList<String>();
        String[] blah = contents.toString().split(" ");

        for (String tmp : blah) {
            if (tmp.length() != 0) {
                license.add(tmp);/*ww w .j  a va 2  s  . c om*/
            }
        }

        if (license.size() != 13) {
            return;
        }

        date = license.get(0).replaceAll("/", "-");
        pkgName = license.get(4);
        licenseInfo += license.get(5) + "," + license.get(7);
        context.write(new Text(pkgName + "-" + date), new Text(licenseInfo));
    }
}

From source file:com.alectenharmsel.research.MoabLicensesMapperTest.java

@Test
public void testNoOutputNeeded() throws IOException {
    List<Pair<Text, Text>> res = driver.withInput(new LongWritable(0),
            new Text("05/11 22:58:25  MNodeUpdateResExpression(nyx5624,FALSE,TRUE)")).run();
    Assert.assertTrue(res.isEmpty());//from   www .  j a  va2 s.c o  m
}

From source file:com.alectenharmsel.research.MoabLicensesMapperTest.java

@Test
public void testLicenseLine() throws IOException {
    driver.withInput(new LongWritable(0), new Text(
            "05/11 22:58:25  INFO:     License cfd_solv_ser        0 of   6 available  (Idle: 33.3%  Active: 66.67%)"))
            .withOutput(new Text("cfd_solv_ser-05-11"), new Text("0,6")).runTest();
}

From source file:com.alectenharmsel.research.MoabLicensesReducer.java

License:Apache License

public void reduce(Text key, Iterable<Text> counts, Context context) throws IOException, InterruptedException {
    int sum = 0;/*from ww  w .j a  va  2s .  co  m*/
    int num = 0;
    int total = 0;

    for (Text tmp : counts) {
        String[] split = tmp.toString().split(",");
        sum += Integer.parseInt(split[0]);
        total += Integer.parseInt(split[1]);
        num++;
    }

    double avgAvail = (double) sum / (double) num;
    String avgTotal = "";
    if (total % num == 0) {
        avgTotal = String.valueOf(total / num);
    } else {
        avgTotal = String.valueOf((double) total / (double) num);
    }

    String[] keyArr = key.toString().split("-");
    String keyOut = keyArr[keyArr.length - 2] + "-" + keyArr[keyArr.length - 1];

    keyOut += ",";
    for (int i = 0; i < keyArr.length - 2; i++) {
        if (i > 0) {
            keyOut += "-";
        }
        keyOut += keyArr[i];
    }

    context.write(new Text(keyOut), new Text(avgAvail + "," + avgTotal));
}