Example usage for org.apache.hadoop.io Text append

List of usage examples for org.apache.hadoop.io Text append

  1. HOME
  2. Java
  3. org.apache.hadoop
  4. org.apache.hadoop.io.*
  5. Text
  6. append

Introduction

In this page you can find the example usage for org.apache.hadoop.io Text append.

Prototype

public void append(byte[] utf8, int start, int len)

Source Link

Document

Append a range of bytes to the end of the given text

Usage

From source file:org.apache.accumulo.core.iterators.user.IndexedDocIterator.java

License:Apache License

protected Key buildDocKey() {
    if (log.isTraceEnabled())
        log.trace("building doc key for " + currentPartition + " " + currentDocID);
    int zeroIndex = currentDocID.find("\0");
    if (zeroIndex < 0)
        throw new IllegalArgumentException("bad current docID");
    Text colf = new Text(docColf);
    colf.append(nullByte, 0, 1);
    colf.append(currentDocID.getBytes(), 0, zeroIndex);
    docColfSet = Collections/*from  www  . jav  a2 s  . co m*/
            .singleton((ByteSequence) new ArrayByteSequence(colf.getBytes(), 0, colf.getLength()));
    if (log.isTraceEnabled())
        log.trace(zeroIndex + " " + currentDocID.getLength());
    Text colq = new Text();
    colq.set(currentDocID.getBytes(), zeroIndex + 1, currentDocID.getLength() - zeroIndex - 1);
    Key k = new Key(currentPartition, colf, colq);
    if (log.isTraceEnabled())
        log.trace("built doc key for seek: " + k.toString());
    return k;
}

From source file:org.apache.accumulo.core.iterators.user.IndexedDocIteratorTest.java

License:Apache License

private TreeMap<Key, Value> createSortedMap(float hitRatio, int numRows, int numDocsPerRow,
        Text[] columnFamilies, Text[] otherColumnFamilies, HashSet<Text> docs, Text[] negatedColumns) {
    StringBuilder sb = new StringBuilder();
    Random r = new Random();
    Value v = new Value(new byte[0]);
    TreeMap<Key, Value> map = new TreeMap<>();
    boolean[] negateMask = new boolean[columnFamilies.length];

    for (int i = 0; i < columnFamilies.length; i++) {
        negateMask[i] = false;/*from  w ww .  ja  va2  s  .co  m*/
        if (negatedColumns.length > 0)
            for (Text ng : negatedColumns)
                if (columnFamilies[i].equals(ng))
                    negateMask[i] = true;
    }
    for (int i = 0; i < numRows; i++) {
        Text row = new Text(String.format("%06d", i));
        for (int startDocID = docid; docid - startDocID < numDocsPerRow; docid++) {
            sb.setLength(0);
            sb.append("fake doc contents");
            boolean docHits = true;
            Text doc = new Text("type");
            doc.append(nullByte, 0, 1);
            doc.append(String.format("%010d", docid).getBytes(), 0, 10);
            for (int j = 0; j < columnFamilies.length; j++) {
                if (r.nextFloat() < hitRatio) {
                    Text colq = new Text(columnFamilies[j]);
                    colq.append(nullByte, 0, 1);
                    colq.append(doc.getBytes(), 0, doc.getLength());
                    colq.append(nullByte, 0, 1);
                    colq.append("stuff".getBytes(), 0, "stuff".length());
                    Key k = new Key(row, indexColf, colq);
                    map.put(k, v);
                    sb.append(" ");
                    sb.append(columnFamilies[j]);
                    if (negateMask[j])
                        docHits = false;
                } else {
                    if (!negateMask[j])
                        docHits = false;
                }
            }
            if (docHits) {
                docs.add(doc);
            }
            for (Text cf : otherColumnFamilies) {
                if (r.nextFloat() < hitRatio) {
                    Text colq = new Text(cf);
                    colq.append(nullByte, 0, 1);
                    colq.append(doc.getBytes(), 0, doc.getLength());
                    colq.append(nullByte, 0, 1);
                    colq.append("stuff".getBytes(), 0, "stuff".length());
                    Key k = new Key(row, indexColf, colq);
                    map.put(k, v);
                    sb.append(" ");
                    sb.append(cf);
                }
            }
            sb.append(" docID=").append(doc);
            Key k = new Key(row, docColf, new Text(String.format("%010d", docid).getBytes()));
            map.put(k, new Value(sb.toString().getBytes()));
        }
    }
    return map;
}

From source file:org.apache.accumulo.core.util.shell.commands.GetSplitsCommand.java

License:Apache License

@Override
public int execute(final String fullCommand, final CommandLine cl, final Shell shellState)
        throws IOException, AccumuloException, AccumuloSecurityException, TableNotFoundException {
    final String tableName = OptUtil.getTableOpt(cl, shellState);

    final String outputFile = cl.getOptionValue(outputFileOpt.getOpt());
    final String m = cl.getOptionValue(maxSplitsOpt.getOpt());
    final int maxSplits = m == null ? 0 : Integer.parseInt(m);
    final boolean encode = cl.hasOption(base64Opt.getOpt());
    final boolean verbose = cl.hasOption(verboseOpt.getOpt());

    final PrintLine p = outputFile == null ? new PrintShell(shellState.getReader()) : new PrintFile(outputFile);

    try {//from ww w. j  a v  a2s.  com
        if (!verbose) {
            for (Text row : maxSplits > 0
                    ? shellState.getConnector().tableOperations().listSplits(tableName, maxSplits)
                    : shellState.getConnector().tableOperations().listSplits(tableName)) {
                p.print(encode(encode, row));
            }
        } else {
            String systemTableToCheck = MetadataTable.NAME.equals(tableName) ? RootTable.NAME
                    : MetadataTable.NAME;
            final Scanner scanner = shellState.getConnector().createScanner(systemTableToCheck,
                    Authorizations.EMPTY);
            TabletsSection.TabletColumnFamily.PREV_ROW_COLUMN.fetch(scanner);
            final Text start = new Text(
                    shellState.getConnector().tableOperations().tableIdMap().get(tableName));
            final Text end = new Text(start);
            end.append(new byte[] { '<' }, 0, 1);
            scanner.setRange(new Range(start, end));
            for (Iterator<Entry<Key, Value>> iterator = scanner.iterator(); iterator.hasNext();) {
                final Entry<Key, Value> next = iterator.next();
                if (TabletsSection.TabletColumnFamily.PREV_ROW_COLUMN.hasColumns(next.getKey())) {
                    KeyExtent extent = new KeyExtent(next.getKey().getRow(), next.getValue());
                    final String pr = encode(encode, extent.getPrevEndRow());
                    final String er = encode(encode, extent.getEndRow());
                    final String line = String.format("%-26s (%s, %s%s", obscuredTabletName(extent),
                            pr == null ? "-inf" : pr, er == null ? "+inf" : er,
                            er == null ? ") Default Tablet " : "]");
                    p.print(line);
                }
            }
        }

    } finally {
        p.close();
    }

    return 0;
}

From source file:org.apache.accumulo.core.util.TextUtil.java

License:Apache License

public static Text truncate(Text text, int maxLen) {
    if (text.getLength() > maxLen) {
        Text newText = new Text();
        newText.append(text.getBytes(), 0, maxLen);
        String suffix = "... TRUNCATED";
        newText.append(suffix.getBytes(UTF_8), 0, suffix.length());
        return newText;
    }/*from w w  w . j a  v a  2 s  . c o m*/

    return text;
}

From source file:org.apache.accumulo.examples.dirlist.QueryUtil.java

License:Apache License

/**
 * Given a path, construct an accumulo row prepended with the path's depth for the directory table.
 *
 * @param path// ww  w  . j a va2  s.  c om
 *          the full path of a file or directory
 * @return the accumulo row associated with this path
 */
public static Text getRow(String path) {
    Text row = new Text(String.format("%03d", getDepth(path)));
    row.append(path.getBytes(), 0, path.length());
    return row;
}

From source file:org.apache.accumulo.examples.dirlist.QueryUtil.java

License:Apache License

/**
 * Given a path, construct an accumulo row prepended with the {@link #FORWARD_PREFIX} for the index table.
 *
 * @param path/*from w  ww.  ja va  2s. c  om*/
 *          the full path of a file or directory
 * @return the accumulo row associated with this path
 */
public static Text getForwardIndex(String path) {
    String part = path.substring(path.lastIndexOf("/") + 1);
    if (part.length() == 0)
        return null;
    Text row = new Text(FORWARD_PREFIX);
    row.append(part.getBytes(), 0, part.length());
    return row;
}

From source file:org.apache.accumulo.examples.dirlist.QueryUtil.java

License:Apache License

/**
 * Given a path, construct an accumulo row prepended with the {@link #REVERSE_PREFIX} with the path reversed for the index table.
 *
 * @param path// w  w  w  .j a  va 2 s .  c  o m
 *          the full path of a file or directory
 * @return the accumulo row associated with this path
 */
public static Text getReverseIndex(String path) {
    String part = path.substring(path.lastIndexOf("/") + 1);
    if (part.length() == 0)
        return null;
    byte[] rev = new byte[part.length()];
    int i = part.length() - 1;
    for (byte b : part.getBytes())
        rev[i--] = b;
    Text row = new Text(REVERSE_PREFIX);
    row.append(rev, 0, rev.length);
    return row;
}

From source file:org.apache.accumulo.examples.filedata.ChunkInputStreamIT.java

License:Apache License

static void addData(List<Entry<Key, Value>> data, String row, String cf, int chunkSize, int chunkCount,
        String vis, String value) {
    Text chunkCQ = new Text(FileDataIngest.intToBytes(chunkSize));
    chunkCQ.append(FileDataIngest.intToBytes(chunkCount), 0, 4);
    data.add(new KeyValue(new Key(new Text(row), new Text(cf), chunkCQ, new Text(vis)), value.getBytes()));
}

From source file:org.apache.accumulo.examples.filedata.ChunkInputStreamTest.java

License:Apache License

private static void addData(List<Entry<Key, Value>> data, String row, String cf, int chunkSize, int chunkCount,
        String vis, String value) {
    Text chunkCQ = new Text(FileDataIngest.intToBytes(chunkSize));
    chunkCQ.append(FileDataIngest.intToBytes(chunkCount), 0, 4);
    data.add(new KeyValue(new Key(new Text(row), new Text(cf), chunkCQ, new Text(vis)), value.getBytes()));
}

From source file:org.apache.accumulo.examples.filedata.FileDataIngest.java

License:Apache License

public String insertFileData(String filename, BatchWriter bw) throws MutationsRejectedException, IOException {
    if (chunkSize == 0)
        return "";
    md5digest.reset();/*from  w ww.  java  2  s  . c o m*/
    String uid = hexString(md5digest.digest(filename.getBytes()));

    // read through file once, calculating hashes
    md5digest.reset();
    InputStream fis = null;
    int numRead = 0;
    try {
        fis = new FileInputStream(filename);
        numRead = fis.read(buf);
        while (numRead >= 0) {
            if (numRead > 0) {
                md5digest.update(buf, 0, numRead);
            }
            numRead = fis.read(buf);
        }
    } finally {
        if (fis != null) {
            fis.close();
        }
    }

    String hash = hexString(md5digest.digest());
    Text row = new Text(hash);

    // write info to accumulo
    Mutation m = new Mutation(row);
    m.put(REFS_CF, KeyUtil.buildNullSepText(uid, REFS_ORIG_FILE), cv, new Value(filename.getBytes()));
    String fext = getExt(filename);
    if (fext != null)
        m.put(REFS_CF, KeyUtil.buildNullSepText(uid, REFS_FILE_EXT), cv, new Value(fext.getBytes()));
    bw.addMutation(m);

    // read through file again, writing chunks to accumulo
    int chunkCount = 0;
    try {
        fis = new FileInputStream(filename);
        numRead = fis.read(buf);
        while (numRead >= 0) {
            while (numRead < buf.length) {
                int moreRead = fis.read(buf, numRead, buf.length - numRead);
                if (moreRead > 0)
                    numRead += moreRead;
                else if (moreRead < 0)
                    break;
            }
            m = new Mutation(row);
            Text chunkCQ = new Text(chunkSizeBytes);
            chunkCQ.append(intToBytes(chunkCount), 0, 4);
            m.put(CHUNK_CF, chunkCQ, cv, new Value(buf, 0, numRead));
            bw.addMutation(m);
            if (chunkCount == Integer.MAX_VALUE)
                throw new RuntimeException("too many chunks for file " + filename + ", try raising chunk size");
            chunkCount++;
            numRead = fis.read(buf);
        }
    } finally {
        if (fis != null) {
            fis.close();
        }
    }
    m = new Mutation(row);
    Text chunkCQ = new Text(chunkSizeBytes);
    chunkCQ.append(intToBytes(chunkCount), 0, 4);
    m.put(new Text(CHUNK_CF), chunkCQ, cv, new Value(new byte[0]));
    bw.addMutation(m);
    return hash;
}