Example usage for org.apache.hadoop.io Text equals

List of usage examples for org.apache.hadoop.io Text equals

  1. HOME
  2. Java
  3. org.apache.hadoop
  4. org.apache.hadoop.io.*
  5. Text
  6. equals

Introduction

In this page you can find the example usage for org.apache.hadoop.io Text equals.

Prototype

@Override
public boolean equals(Object o)

Source Link

Document

Returns true iff o is a Text with the same contents.

Usage

From source file:org.apache.accumulo.server.master.tableOps.ImportedTableInfo.java

License:Apache License

@Override
public Repo<Master> call(long tid, Master master) throws Exception {

    Path path = new Path(tableInfo.exportDir, Constants.EXPORT_FILE);

    BatchWriter mbw = null;/*from   w w w. j  a  v a 2 s .com*/
    ZipInputStream zis = null;

    try {
        VolumeManager fs = master.getFileSystem();

        mbw = master.getConnector().createBatchWriter(MetadataTable.NAME, new BatchWriterConfig());

        zis = new ZipInputStream(fs.open(path));

        Map<String, String> fileNameMappings = readMappingFile(fs, tableInfo);

        String bulkDir = new Path(tableInfo.importDir).getName();

        ZipEntry zipEntry;
        while ((zipEntry = zis.getNextEntry()) != null) {
            if (zipEntry.getName().equals(Constants.EXPORT_METADATA_FILE)) {
                DataInputStream in = new DataInputStream(new BufferedInputStream(zis));

                Key key = new Key();
                Value val = new Value();

                Mutation m = null;
                Text currentRow = null;
                int dirCount = 0;

                while (true) {
                    key.readFields(in);
                    val.readFields(in);

                    Text endRow = new KeyExtent(key.getRow(), (Text) null).getEndRow();
                    Text metadataRow = new KeyExtent(new Text(tableInfo.tableId), endRow, null)
                            .getMetadataEntry();

                    Text cq;

                    if (key.getColumnFamily().equals(DataFileColumnFamily.NAME)) {
                        String oldName = new Path(key.getColumnQualifier().toString()).getName();
                        String newName = fileNameMappings.get(oldName);

                        if (newName == null) {
                            throw new ThriftTableOperationException(tableInfo.tableId, tableInfo.tableName,
                                    TableOperation.IMPORT, TableOperationExceptionType.OTHER,
                                    "File " + oldName + " does not exist in import dir");
                        }

                        cq = new Text("/" + bulkDir + "/" + newName);
                    } else {
                        cq = key.getColumnQualifier();
                    }

                    if (m == null) {
                        m = new Mutation(metadataRow);
                        TabletsSection.ServerColumnFamily.DIRECTORY_COLUMN.put(m,
                                new Value(FastFormat.toZeroPaddedString(dirCount++, 8, 16, "/c-".getBytes())));
                        currentRow = metadataRow;
                    }

                    if (!currentRow.equals(metadataRow)) {
                        mbw.addMutation(m);
                        m = new Mutation(metadataRow);
                        TabletsSection.ServerColumnFamily.DIRECTORY_COLUMN.put(m,
                                new Value(FastFormat.toZeroPaddedString(dirCount++, 8, 16, "/c-".getBytes())));
                    }

                    m.put(key.getColumnFamily(), cq, val);

                    if (endRow == null && TabletsSection.TabletColumnFamily.PREV_ROW_COLUMN.hasColumns(key)) {
                        mbw.addMutation(m);
                        break; // its the last column in the last row
                    }
                }

                break;
            }
        }

        return new MoveExportedFiles(tableInfo);
    } catch (IOException ioe) {
        log.warn(ioe.getMessage(), ioe);
        throw new ThriftTableOperationException(tableInfo.tableId, tableInfo.tableName, TableOperation.IMPORT,
                TableOperationExceptionType.OTHER, "Error reading " + path + " " + ioe.getMessage());
    } finally {
        if (zis != null) {
            try {
                zis.close();
            } catch (IOException ioe) {
                log.warn("Failed to close zip file ", ioe);
            }
        }

        if (mbw != null) {
            mbw.close();
        }
    }
}

From source file:org.apache.accumulo.server.tabletserver.ScanRunState.java

License:Apache License

static Value checkTabletMetadata(KeyExtent extent, TServerInstance instance,
        SortedMap<Key, Value> tabletsKeyValues, Text metadataEntry) throws AccumuloException {

    TServerInstance future = null;/*from ww  w .j a  va2  s . c o  m*/
    Value prevEndRow = null;
    Value dir = null;
    Value time = null;
    for (Entry<Key, Value> entry : tabletsKeyValues.entrySet()) {
        Key key = entry.getKey();
        if (!metadataEntry.equals(key.getRow())) {
            log.info("Unexpected row in tablet metadata " + metadataEntry + " " + key.getRow());
            return null;
        }
        Text cf = key.getColumnFamily();
        if (cf.equals(TabletsSection.FutureLocationColumnFamily.NAME)) {
            if (future != null) {
                throw new AccumuloException("Tablet has multiple future locations " + extent);
            }
            future = new TServerInstance(entry.getValue(), key.getColumnQualifier());
        } else if (cf.equals(TabletsSection.CurrentLocationColumnFamily.NAME)) {
            log.info("Tablet seems to be already assigned to "
                    + new TServerInstance(entry.getValue(), key.getColumnQualifier()));
            return null;
        } else if (TabletsSection.TabletColumnFamily.PREV_ROW_COLUMN.hasColumns(key)) {
            prevEndRow = entry.getValue();
        } else if (TabletsSection.ServerColumnFamily.DIRECTORY_COLUMN.hasColumns(key)) {
            dir = entry.getValue();
        } else if (TabletsSection.ServerColumnFamily.TIME_COLUMN.hasColumns(key)) {
            time = entry.getValue();
        }
    }

    if (prevEndRow == null) {
        throw new AccumuloException("Metadata entry does not have prev row (" + metadataEntry + ")");
    } else {
        KeyExtent ke2 = new KeyExtent(metadataEntry, prevEndRow);
        if (!extent.equals(ke2)) {
            log.info("Tablet prev end row mismatch " + extent + " " + ke2.getPrevEndRow());
            return null;
        }
    }

    if (dir == null) {
        throw new AccumuloException("Metadata entry does not have directory (" + metadataEntry + ")");
    }

    if (time == null) {
        throw new AccumuloException("Metadata entry does not have time (" + metadataEntry + ")");
    }

    if (future == null) {
        log.info("The master has not assigned " + extent + " to " + instance);
        return null;
    }

    if (!instance.equals(future)) {
        log.info("Table " + extent + " has been assigned to " + future + " which is not " + instance);
        return null;
    }

    return dir;
}

From source file:org.apache.accumulo.server.test.performance.scan.CollectTabletStats.java

License:Apache License

private static void calcTabletStats(Connector conn, String table, String[] auths, int batchSize, KeyExtent ke,
        String[] columns) throws Exception {

    // long t1 = System.currentTimeMillis();

    Scanner scanner = conn.createScanner(table, new Authorizations(auths));
    scanner.setBatchSize(batchSize);//from   w w  w.j a  v  a  2s.c  o  m
    scanner.setRange(new Range(ke.getPrevEndRow(), false, ke.getEndRow(), true));

    for (String c : columns) {
        scanner.fetchColumnFamily(new Text(c));
    }

    Stat rowLen = new Stat();
    Stat cfLen = new Stat();
    Stat cqLen = new Stat();
    Stat cvLen = new Stat();
    Stat valLen = new Stat();
    Stat colsPerRow = new Stat();

    Text lastRow = null;
    int colsPerRowCount = 0;

    for (Entry<Key, Value> entry : scanner) {

        Key key = entry.getKey();
        Text row = key.getRow();

        if (lastRow == null) {
            lastRow = row;
        }

        if (!lastRow.equals(row)) {
            colsPerRow.addStat(colsPerRowCount);
            lastRow = row;
            colsPerRowCount = 0;
        }

        colsPerRowCount++;

        rowLen.addStat(row.getLength());
        cfLen.addStat(key.getColumnFamilyData().length());
        cqLen.addStat(key.getColumnQualifierData().length());
        cvLen.addStat(key.getColumnVisibilityData().length());
        valLen.addStat(entry.getValue().get().length);
    }

    synchronized (System.out) {
        System.out.println("");
        System.out.println("\tTablet " + ke.getUUID() + " statistics : ");
        printStat("Row length", rowLen);
        printStat("Column family length", cfLen);
        printStat("Column qualifier length", cqLen);
        printStat("Column visibility length", cvLen);
        printStat("Value length", valLen);
        printStat("Columns per row", colsPerRow);
        System.out.println("");
    }

}

From source file:org.apache.accumulo.server.util.CheckForMetadataProblems.java

License:Apache License

public static void checkMetadataAndRootTableEntries(String tableNameToCheck, ClientOpts opts, VolumeManager fs)
        throws Exception {
    System.out.println("Checking table: " + tableNameToCheck);
    Map<String, TreeSet<KeyExtent>> tables = new HashMap<>();

    Scanner scanner;/*w  w  w  .j  a v  a  2s .c  o  m*/

    scanner = opts.getConnector().createScanner(tableNameToCheck, Authorizations.EMPTY);

    scanner.setRange(MetadataSchema.TabletsSection.getRange());
    TabletsSection.TabletColumnFamily.PREV_ROW_COLUMN.fetch(scanner);
    scanner.fetchColumnFamily(TabletsSection.CurrentLocationColumnFamily.NAME);

    Text colf = new Text();
    Text colq = new Text();
    boolean justLoc = false;

    int count = 0;

    for (Entry<Key, Value> entry : scanner) {
        colf = entry.getKey().getColumnFamily(colf);
        colq = entry.getKey().getColumnQualifier(colq);

        count++;

        String tableName = (new KeyExtent(entry.getKey().getRow(), (Text) null)).getTableId();

        TreeSet<KeyExtent> tablets = tables.get(tableName);
        if (tablets == null) {
            Set<Entry<String, TreeSet<KeyExtent>>> es = tables.entrySet();

            for (Entry<String, TreeSet<KeyExtent>> entry2 : es) {
                checkTable(entry2.getKey(), entry2.getValue(), opts);
            }

            tables.clear();

            tablets = new TreeSet<>();
            tables.put(tableName, tablets);
        }

        if (TabletsSection.TabletColumnFamily.PREV_ROW_COLUMN.equals(colf, colq)) {
            KeyExtent tabletKe = new KeyExtent(entry.getKey().getRow(), entry.getValue());
            tablets.add(tabletKe);
            justLoc = false;
        } else if (colf.equals(TabletsSection.CurrentLocationColumnFamily.NAME)) {
            if (justLoc) {
                System.out.println("Problem at key " + entry.getKey());
                sawProblems = true;
            }
            justLoc = true;
        }
    }

    if (count == 0) {
        System.err.println("ERROR : " + tableNameToCheck + " table is empty");
        sawProblems = true;
    }

    Set<Entry<String, TreeSet<KeyExtent>>> es = tables.entrySet();

    for (Entry<String, TreeSet<KeyExtent>> entry : es) {
        checkTable(entry.getKey(), entry.getValue(), opts);
    }

    if (!sawProblems) {
        System.out.println("No problems found");
    }
    // end METADATA table sanity check
}

From source file:org.apache.accumulo.server.util.CountDiskRows.java

License:Apache License

public static void main(String[] args) throws IOException {
    if (args.length < 1) {
        log.error("usage: CountDiskRows tablename");
        return;//from w  w  w  .j a va2  s .  co m
    }

    Configuration conf = CachedConfiguration.getInstance();
    FileSystem fs = FileSystem.get(conf);

    Key key = new Key();
    Value value = new Value();
    int numrows = 0;
    Text prevRow = new Text("");
    Text row = null;

    FileStatus[] tablets = fs.listStatus(new Path(ServerConstants.getTablesDir() + "/" + args[0]));
    for (FileStatus tablet : tablets) {
        FileStatus[] mapfiles = fs.listStatus(tablet.getPath());
        for (FileStatus mapfile : mapfiles) {
            MyMapFile.Reader mfr = new MyMapFile.Reader(fs, mapfile.getPath().toString(), conf);
            while (mfr.next(key, value)) {
                row = key.getRow();
                if (!row.equals(prevRow)) {
                    prevRow = new Text(row);
                    numrows++;
                }
            }
        }
    }

    log.info("files in directory " + args[0] + " have " + numrows);
}

From source file:org.apache.accumulo.server.util.MetadataTable.java

License:Apache License

public static List<LogEntry> getLogEntries(TCredentials credentials, KeyExtent extent)
        throws IOException, KeeperException, InterruptedException {
    log.info("Scanning logging entries for " + extent);
    ArrayList<LogEntry> result = new ArrayList<LogEntry>();
    if (extent.equals(Constants.ROOT_TABLET_EXTENT)) {
        log.info("Getting logs for root tablet from zookeeper");
        getRootLogEntries(result);/*from  w  ww.j  a  v  a2 s  .c  o  m*/
    } else {
        log.info("Scanning metadata for logs used for tablet " + extent);
        Scanner scanner = getTabletLogScanner(credentials, extent);
        Text pattern = extent.getMetadataEntry();
        for (Entry<Key, Value> entry : scanner) {
            Text row = entry.getKey().getRow();
            if (entry.getKey().getColumnFamily().equals(Constants.METADATA_LOG_COLUMN_FAMILY)) {
                if (row.equals(pattern)) {
                    result.add(entryFromKeyValue(entry.getKey(), entry.getValue()));
                }
            }
        }
    }

    Collections.sort(result, new Comparator<LogEntry>() {
        @Override
        public int compare(LogEntry o1, LogEntry o2) {
            long diff = o1.timestamp - o2.timestamp;
            if (diff < 0)
                return -1;
            if (diff > 0)
                return 1;
            return 0;
        }
    });
    log.info("Returning logs " + result + " for extent " + extent);
    return result;
}

From source file:org.apache.accumulo.server.util.MetadataTableUtil.java

License:Apache License

public static List<LogEntry> getLogEntries(ClientContext context, KeyExtent extent)
        throws IOException, KeeperException, InterruptedException {
    log.info("Scanning logging entries for " + extent);
    ArrayList<LogEntry> result = new ArrayList<>();
    if (extent.equals(RootTable.EXTENT)) {
        log.info("Getting logs for root tablet from zookeeper");
        getRootLogEntries(result);//from w w  w  .j a v a  2s.  co m
    } else {
        log.info("Scanning metadata for logs used for tablet " + extent);
        Scanner scanner = getTabletLogScanner(context, extent);
        Text pattern = extent.getMetadataEntry();
        for (Entry<Key, Value> entry : scanner) {
            Text row = entry.getKey().getRow();
            if (entry.getKey().getColumnFamily().equals(LogColumnFamily.NAME)) {
                if (row.equals(pattern)) {
                    result.add(LogEntry.fromKeyValue(entry.getKey(), entry.getValue()));
                }
            }
        }
    }

    log.info("Returning logs " + result + " for extent " + extent);
    return result;
}

From source file:org.apache.accumulo.server.util.TabletIterator.java

License:Apache License

@Override
public boolean hasNext() {
    while (currentTabletKeys == null) {

        currentTabletKeys = scanToPrevEndRow();
        if (currentTabletKeys.size() == 0) {
            break;
        }/*ww w.  j  a  va 2s.  c  o  m*/

        Key prevEndRowKey = currentTabletKeys.lastKey();
        Value prevEndRowValue = currentTabletKeys.get(prevEndRowKey);

        if (!TabletsSection.TabletColumnFamily.PREV_ROW_COLUMN.hasColumns(prevEndRowKey)) {
            log.debug("{}", currentTabletKeys);
            throw new RuntimeException("Unexpected key " + prevEndRowKey);
        }

        Text per = KeyExtent.decodePrevEndRow(prevEndRowValue);
        Text lastEndRow;

        if (lastTablet == null) {
            lastEndRow = null;
        } else {
            lastEndRow = new KeyExtent(lastTablet, (Text) null).getEndRow();

            // do table transition sanity check
            String lastTable = new KeyExtent(lastTablet, (Text) null).getTableId();
            String currentTable = new KeyExtent(prevEndRowKey.getRow(), (Text) null).getTableId();

            if (!lastTable.equals(currentTable) && (per != null || lastEndRow != null)) {
                log.info("Metadata inconsistency on table transition : " + lastTable + " " + currentTable + " "
                        + per + " " + lastEndRow);

                currentTabletKeys = null;
                resetScanner();

                sleepUninterruptibly(250, TimeUnit.MILLISECONDS);

                continue;
            }
        }

        boolean perEqual = (per == null && lastEndRow == null)
                || (per != null && lastEndRow != null && per.equals(lastEndRow));

        if (!perEqual) {

            log.info("Metadata inconsistency : " + per + " != " + lastEndRow + " metadataKey = "
                    + prevEndRowKey);

            currentTabletKeys = null;
            resetScanner();

            sleepUninterruptibly(250, TimeUnit.MILLISECONDS);

            continue;

        }
        // this tablet is good, so set it as the last tablet
        lastTablet = prevEndRowKey.getRow();
    }

    return currentTabletKeys.size() > 0;
}

From source file:org.apache.accumulo.server.util.TabletIterator.java

License:Apache License

private SortedMap<Key, Value> scanToPrevEndRow() {

    Text curMetaDataRow = null;

    TreeMap<Key, Value> tm = new TreeMap<>();

    boolean sawPrevEndRow = false;

    while (true) {
        while (iter.hasNext()) {
            Entry<Key, Value> entry = iter.next();

            if (curMetaDataRow == null) {
                curMetaDataRow = entry.getKey().getRow();
            }/*from   www .  ja  va 2s .c o  m*/

            if (!curMetaDataRow.equals(entry.getKey().getRow())) {
                // tablet must not have a prev end row, try scanning again
                break;
            }

            tm.put(entry.getKey(), entry.getValue());

            if (TabletsSection.TabletColumnFamily.PREV_ROW_COLUMN.hasColumns(entry.getKey())) {
                sawPrevEndRow = true;
                break;
            }
        }

        if (!sawPrevEndRow && tm.size() > 0) {
            log.warn("Metadata problem : tablet " + curMetaDataRow + " has no prev end row");
            resetScanner();
            curMetaDataRow = null;
            tm.clear();
            sleepUninterruptibly(250, TimeUnit.MILLISECONDS);
        } else {
            break;
        }
    }

    return tm;
}

From source file:org.apache.accumulo.test.functional.CloneTestIT.java

License:Apache License

private void checkMetadata(String table, Connector conn) throws Exception {
    Scanner s = conn.createScanner(MetadataTable.NAME, Authorizations.EMPTY);

    s.fetchColumnFamily(MetadataSchema.TabletsSection.DataFileColumnFamily.NAME);
    MetadataSchema.TabletsSection.ServerColumnFamily.DIRECTORY_COLUMN.fetch(s);
    String tableId = conn.tableOperations().tableIdMap().get(table);

    Assert.assertNotNull("Could not get table id for " + table, tableId);

    s.setRange(Range.prefix(tableId));/*from  w w  w .j  ava  2 s .c o  m*/

    Key k;
    Text cf = new Text(), cq = new Text();
    int itemsInspected = 0;
    for (Entry<Key, Value> entry : s) {
        itemsInspected++;
        k = entry.getKey();
        k.getColumnFamily(cf);
        k.getColumnQualifier(cq);

        if (cf.equals(MetadataSchema.TabletsSection.DataFileColumnFamily.NAME)) {
            Path p = new Path(cq.toString());
            FileSystem fs = cluster.getFileSystem();
            Assert.assertTrue("File does not exist: " + p, fs.exists(p));
        } else if (cf
                .equals(MetadataSchema.TabletsSection.ServerColumnFamily.DIRECTORY_COLUMN.getColumnFamily())) {
            Assert.assertEquals("Saw unexpected cq",
                    MetadataSchema.TabletsSection.ServerColumnFamily.DIRECTORY_COLUMN.getColumnQualifier(), cq);
            Path tabletDir = new Path(entry.getValue().toString());
            Path tableDir = tabletDir.getParent();
            Path tablesDir = tableDir.getParent();

            Assert.assertEquals(ServerConstants.TABLE_DIR, tablesDir.getName());
        } else {
            Assert.fail("Got unexpected key-value: " + entry);
            throw new RuntimeException();
        }
    }

    Assert.assertTrue("Expected to find metadata entries", itemsInspected > 0);
}