List of usage examples for org.apache.hadoop.io Text equals
@Override public boolean equals(Object o)
o is a Text with the same contents. From source file:org.apache.accumulo.core.util.TextUtilTest.java
License:Apache License
/** * co/*ww w. j a v a 2 s . c o m*/ */ public void testGetBytes() { String longMessage = "This is some text"; Text longMessageText = new Text(longMessage); String smallerMessage = "a"; Text smallerMessageText = new Text(smallerMessage); Text someText = new Text(longMessage); assertTrue(someText.equals(longMessageText)); someText.set(smallerMessageText); assertTrue(someText.getLength() != someText.getBytes().length); assertTrue(TextUtil.getBytes(someText).length == smallerMessage.length()); assertTrue((new Text(TextUtil.getBytes(someText))).equals(smallerMessageText)); }
From source file:org.apache.accumulo.examples.dirlist.QueryUtil.java
License:Apache License
/** * Returns either the {@link #DIR_COLF} or a decoded string version of the colf. * * @param colf/*from w w w . java 2 s . c o m*/ * the column family */ public static String getType(Text colf) { if (colf.equals(DIR_COLF)) return colf.toString() + ":"; return Long.toString(Ingest.encoder.decode(colf.getBytes())) + ":"; }
From source file:org.apache.accumulo.examples.shard.ContinuousQuery.java
License:Apache License
private static ArrayList<Text[]> findRandomTerms(Scanner scanner, int numTerms) { Text currentRow = null; ArrayList<Text> words = new ArrayList<>(); ArrayList<Text[]> ret = new ArrayList<>(); Random rand = new Random(); for (Entry<Key, Value> entry : scanner) { Key key = entry.getKey(); if (currentRow == null) currentRow = key.getRow();//from w w w . j ava2 s.c o m if (!currentRow.equals(key.getRow())) { selectRandomWords(words, ret, rand, numTerms); words.clear(); currentRow = key.getRow(); } words.add(key.getColumnFamily()); } selectRandomWords(words, ret, rand, numTerms); return ret; }
From source file:org.apache.accumulo.examples.simple.shard.ContinuousQuery.java
License:Apache License
private static ArrayList<Text[]> findRandomTerms(Scanner scanner, int numTerms) { Text currentRow = null; ArrayList<Text> words = new ArrayList<Text>(); ArrayList<Text[]> ret = new ArrayList<Text[]>(); Random rand = new Random(); for (Entry<Key, Value> entry : scanner) { Key key = entry.getKey(); if (currentRow == null) currentRow = key.getRow();// w w w. j ava 2s. com if (!currentRow.equals(key.getRow())) { selectRandomWords(words, ret, rand, numTerms); words.clear(); currentRow = key.getRow(); } words.add(key.getColumnFamily()); } selectRandomWords(words, ret, rand, numTerms); return ret; }
From source file:org.apache.accumulo.gc.GarbageCollectionAlgorithm.java
License:Apache License
private void confirmDeletes(GarbageCollectionEnvironment gce, SortedMap<String, String> candidateMap) throws TableNotFoundException, AccumuloException, AccumuloSecurityException { boolean checkForBulkProcessingFiles = false; Iterator<String> relativePaths = candidateMap.keySet().iterator(); while (!checkForBulkProcessingFiles && relativePaths.hasNext()) checkForBulkProcessingFiles |= relativePaths.next().toLowerCase(Locale.ENGLISH) .contains(Constants.BULK_PREFIX); if (checkForBulkProcessingFiles) { Iterator<String> blipiter = gce.getBlipIterator(); // WARNING: This block is IMPORTANT // You MUST REMOVE candidates that are in the same folder as a bulk // processing flag! while (blipiter.hasNext()) { String blipPath = blipiter.next(); blipPath = makeRelative(blipPath, 2); Iterator<String> tailIter = candidateMap.tailMap(blipPath).keySet().iterator(); int count = 0; while (tailIter.hasNext()) { if (tailIter.next().startsWith(blipPath)) { count++;/*from w w w. j ava2 s. c o m*/ tailIter.remove(); } else { break; } } if (count > 0) log.debug("Folder has bulk processing flag: " + blipPath); } } Iterator<Entry<Key, Value>> iter = gce.getReferenceIterator(); while (iter.hasNext()) { Entry<Key, Value> entry = iter.next(); Key key = entry.getKey(); Text cft = key.getColumnFamily(); if (cft.equals(DataFileColumnFamily.NAME) || cft.equals(ScanFileColumnFamily.NAME)) { String cq = key.getColumnQualifier().toString(); String reference = cq; if (cq.startsWith("/")) { String tableID = new String(KeyExtent.tableOfMetadataRow(key.getRow())); reference = "/" + tableID + cq; } else if (!cq.contains(":") && !cq.startsWith("../")) { throw new RuntimeException("Bad file reference " + cq); } reference = makeRelative(reference, 3); // WARNING: This line is EXTREMELY IMPORTANT. // You MUST REMOVE candidates that are still in use if (candidateMap.remove(reference) != null) log.debug("Candidate was still in use: " + reference); String dir = reference.substring(0, reference.lastIndexOf('/')); if (candidateMap.remove(dir) != null) log.debug("Candidate was still in use: " + reference); } else if (TabletsSection.ServerColumnFamily.DIRECTORY_COLUMN.hasColumns(key)) { String tableID = new String(KeyExtent.tableOfMetadataRow(key.getRow())); String dir = entry.getValue().toString(); if (!dir.contains(":")) { if (!dir.startsWith("/")) throw new RuntimeException("Bad directory " + dir); dir = "/" + tableID + dir; } dir = makeRelative(dir, 2); if (candidateMap.remove(dir) != null) log.debug("Candidate was still in use: " + dir); } else throw new RuntimeException( "Scanner over metadata table returned unexpected column : " + entry.getKey()); } confirmDeletesFromReplication(gce.getReplicationNeededIterator(), candidateMap.entrySet().iterator()); }
From source file:org.apache.accumulo.master.MasterClientServiceHandler.java
License:Apache License
/** * @return return true records are only in place which are fully replicated *///from w ww. j a v a 2 s .c om protected boolean allReferencesReplicated(BatchScanner bs, Text tableId, Set<String> relevantLogs) { Text rowHolder = new Text(), colfHolder = new Text(); for (Entry<Key, Value> entry : bs) { drainLog.trace("Got key {}", entry.getKey().toStringNoTruncate()); entry.getKey().getColumnQualifier(rowHolder); if (tableId.equals(rowHolder)) { entry.getKey().getRow(rowHolder); entry.getKey().getColumnFamily(colfHolder); String file; if (colfHolder.equals(ReplicationSection.COLF)) { file = rowHolder.toString(); file = file.substring(ReplicationSection.getRowPrefix().length()); } else if (colfHolder.equals(OrderSection.NAME)) { file = OrderSection.getFile(entry.getKey(), rowHolder); long timeClosed = OrderSection.getTimeClosed(entry.getKey(), rowHolder); drainLog.trace("Order section: {} and {}", timeClosed, file); } else { file = rowHolder.toString(); } // Skip files that we didn't observe when we started (new files/data) if (!relevantLogs.contains(file)) { drainLog.trace("Found file that we didn't care about {}", file); continue; } else { drainLog.trace("Found file that we *do* care about {}", file); } try { Status stat = Status.parseFrom(entry.getValue().get()); if (!StatusUtil.isFullyReplicated(stat)) { drainLog.trace("{} and {} is not replicated", file, ProtobufUtil.toString(stat)); return false; } drainLog.trace("{} and {} is replicated", file, ProtobufUtil.toString(stat)); } catch (InvalidProtocolBufferException e) { drainLog.trace("Could not parse protobuf for {}", entry.getKey(), e); } } } return true; }
From source file:org.apache.accumulo.master.tableOps.PopulateMetadataTable.java
License:Apache License
@Override public Repo<Master> call(long tid, Master master) throws Exception { Path path = new Path(tableInfo.exportDir, Constants.EXPORT_FILE); BatchWriter mbw = null;/*from w w w .j a v a 2 s .co m*/ ZipInputStream zis = null; try { VolumeManager fs = master.getFileSystem(); mbw = master.getConnector().createBatchWriter(MetadataTable.NAME, new BatchWriterConfig()); zis = new ZipInputStream(fs.open(path)); Map<String, String> fileNameMappings = readMappingFile(fs, tableInfo); log.info("importDir is " + tableInfo.importDir); // This is a directory already prefixed with proper volume information e.g. hdfs://localhost:8020/path/to/accumulo/tables/... final String bulkDir = tableInfo.importDir; final String[] tableDirs = ServerConstants.getTablesDirs(); ZipEntry zipEntry; while ((zipEntry = zis.getNextEntry()) != null) { if (zipEntry.getName().equals(Constants.EXPORT_METADATA_FILE)) { DataInputStream in = new DataInputStream(new BufferedInputStream(zis)); Key key = new Key(); Value val = new Value(); Mutation m = null; Text currentRow = null; int dirCount = 0; while (true) { key.readFields(in); val.readFields(in); Text endRow = new KeyExtent(key.getRow(), (Text) null).getEndRow(); Text metadataRow = new KeyExtent(tableInfo.tableId, endRow, null).getMetadataEntry(); Text cq; if (key.getColumnFamily().equals(DataFileColumnFamily.NAME)) { String oldName = new Path(key.getColumnQualifier().toString()).getName(); String newName = fileNameMappings.get(oldName); if (newName == null) { throw new AcceptableThriftTableOperationException(tableInfo.tableId, tableInfo.tableName, TableOperation.IMPORT, TableOperationExceptionType.OTHER, "File " + oldName + " does not exist in import dir"); } cq = new Text(bulkDir + "/" + newName); } else { cq = key.getColumnQualifier(); } if (m == null) { // Make a unique directory inside the table's dir. Cannot import multiple tables into one table, so don't need to use unique allocator String tabletDir = new String( FastFormat.toZeroPaddedString(dirCount++, 8, 16, Constants.CLONE_PREFIX_BYTES), UTF_8); // Build up a full hdfs://localhost:8020/accumulo/tables/$id/c-XXXXXXX String absolutePath = getClonedTabletDir(master, tableDirs, tabletDir); m = new Mutation(metadataRow); TabletsSection.ServerColumnFamily.DIRECTORY_COLUMN.put(m, new Value(absolutePath.getBytes(UTF_8))); currentRow = metadataRow; } if (!currentRow.equals(metadataRow)) { mbw.addMutation(m); // Make a unique directory inside the table's dir. Cannot import multiple tables into one table, so don't need to use unique allocator String tabletDir = new String( FastFormat.toZeroPaddedString(dirCount++, 8, 16, Constants.CLONE_PREFIX_BYTES), UTF_8); // Build up a full hdfs://localhost:8020/accumulo/tables/$id/c-XXXXXXX String absolutePath = getClonedTabletDir(master, tableDirs, tabletDir); m = new Mutation(metadataRow); TabletsSection.ServerColumnFamily.DIRECTORY_COLUMN.put(m, new Value(absolutePath.getBytes(UTF_8))); } m.put(key.getColumnFamily(), cq, val); if (endRow == null && TabletsSection.TabletColumnFamily.PREV_ROW_COLUMN.hasColumns(key)) { mbw.addMutation(m); break; // its the last column in the last row } } break; } } return new MoveExportedFiles(tableInfo); } catch (IOException ioe) { log.warn("{}", ioe.getMessage(), ioe); throw new AcceptableThriftTableOperationException(tableInfo.tableId, tableInfo.tableName, TableOperation.IMPORT, TableOperationExceptionType.OTHER, "Error reading " + path + " " + ioe.getMessage()); } finally { if (zis != null) { try { zis.close(); } catch (IOException ioe) { log.warn("Failed to close zip file ", ioe); } } if (mbw != null) { mbw.close(); } } }
From source file:org.apache.accumulo.master.tableOps.tableImport.PopulateMetadataTable.java
License:Apache License
@Override public Repo<Master> call(long tid, Master master) throws Exception { Path path = new Path(tableInfo.exportDir, Constants.EXPORT_FILE); BatchWriter mbw = null;//from w w w. j a va 2 s . com ZipInputStream zis = null; try { VolumeManager fs = master.getFileSystem(); mbw = master.getContext().createBatchWriter(MetadataTable.NAME, new BatchWriterConfig()); zis = new ZipInputStream(fs.open(path)); Map<String, String> fileNameMappings = readMappingFile(fs, tableInfo); log.info("importDir is " + tableInfo.importDir); // This is a directory already prefixed with proper volume information e.g. // hdfs://localhost:8020/path/to/accumulo/tables/... final String bulkDir = tableInfo.importDir; final String[] volumes = ServerConstants.getBaseUris(master.getContext()); ZipEntry zipEntry; while ((zipEntry = zis.getNextEntry()) != null) { if (zipEntry.getName().equals(Constants.EXPORT_METADATA_FILE)) { DataInputStream in = new DataInputStream(new BufferedInputStream(zis)); Key key = new Key(); Value val = new Value(); Mutation m = null; Text currentRow = null; int dirCount = 0; while (true) { key.readFields(in); val.readFields(in); Text endRow = new KeyExtent(key.getRow(), (Text) null).getEndRow(); Text metadataRow = new KeyExtent(tableInfo.tableId, endRow, null).getMetadataEntry(); Text cq; if (key.getColumnFamily().equals(DataFileColumnFamily.NAME)) { String oldName = new Path(key.getColumnQualifier().toString()).getName(); String newName = fileNameMappings.get(oldName); if (newName == null) { throw new AcceptableThriftTableOperationException(tableInfo.tableId.canonical(), tableInfo.tableName, TableOperation.IMPORT, TableOperationExceptionType.OTHER, "File " + oldName + " does not exist in import dir"); } cq = new Text(bulkDir + "/" + newName); } else { cq = key.getColumnQualifier(); } if (m == null) { // Make a unique directory inside the table's dir. Cannot import multiple tables into // one table, so don't need to use unique allocator String tabletDir = new String( FastFormat.toZeroPaddedString(dirCount++, 8, 16, Constants.CLONE_PREFIX_BYTES), UTF_8); // Build up a full hdfs://localhost:8020/accumulo/tables/$id/c-XXXXXXX String absolutePath = getClonedTabletDir(master, endRow, volumes, tabletDir); m = new Mutation(metadataRow); TabletsSection.ServerColumnFamily.DIRECTORY_COLUMN.put(m, new Value(absolutePath.getBytes(UTF_8))); currentRow = metadataRow; } if (!currentRow.equals(metadataRow)) { mbw.addMutation(m); // Make a unique directory inside the table's dir. Cannot import multiple tables into // one table, so don't need to use unique allocator String tabletDir = new String( FastFormat.toZeroPaddedString(dirCount++, 8, 16, Constants.CLONE_PREFIX_BYTES), UTF_8); // Build up a full hdfs://localhost:8020/accumulo/tables/$id/c-XXXXXXX String absolutePath = getClonedTabletDir(master, endRow, volumes, tabletDir); m = new Mutation(metadataRow); TabletsSection.ServerColumnFamily.DIRECTORY_COLUMN.put(m, new Value(absolutePath.getBytes(UTF_8))); } m.put(key.getColumnFamily(), cq, val); if (endRow == null && TabletsSection.TabletColumnFamily.PREV_ROW_COLUMN.hasColumns(key)) { mbw.addMutation(m); break; // its the last column in the last row } } break; } } return new MoveExportedFiles(tableInfo); } catch (IOException ioe) { log.warn("{}", ioe.getMessage(), ioe); throw new AcceptableThriftTableOperationException(tableInfo.tableId.canonical(), tableInfo.tableName, TableOperation.IMPORT, TableOperationExceptionType.OTHER, "Error reading " + path + " " + ioe.getMessage()); } finally { if (zis != null) { try { zis.close(); } catch (IOException ioe) { log.warn("Failed to close zip file ", ioe); } } if (mbw != null) { mbw.close(); } } }
From source file:org.apache.accumulo.master.TabletGroupWatcher.java
License:Apache License
private void sendSplitRequest(MergeInfo info, TabletState state, TabletLocationState tls) { // Already split? if (!info.getState().equals(MergeState.SPLITTING)) return;/*from w w w.ja v a 2 s. c o m*/ // Merges don't split if (!info.isDelete()) return; // Online and ready to split? if (!state.equals(TabletState.HOSTED)) return; // Does this extent cover the end points of the delete? KeyExtent range = info.getExtent(); if (tls.extent.overlaps(range)) { for (Text splitPoint : new Text[] { range.getPrevEndRow(), range.getEndRow() }) { if (splitPoint == null) continue; if (!tls.extent.contains(splitPoint)) continue; if (splitPoint.equals(tls.extent.getEndRow())) continue; if (splitPoint.equals(tls.extent.getPrevEndRow())) continue; try { TServerConnection conn; conn = this.master.tserverSet.getConnection(tls.current); if (conn != null) { Master.log.info("Asking " + tls.current + " to split " + tls.extent + " at " + splitPoint); conn.splitTablet(this.master.masterLock, tls.extent, splitPoint); } else { Master.log.warn("Not connected to server " + tls.current); } } catch (NotServingTabletException e) { Master.log.debug("Error asking tablet server to split a tablet: " + e); } catch (Exception e) { Master.log.warn("Error asking tablet server to split a tablet: " + e); } } } }
From source file:org.apache.accumulo.server.constraints.MetadataConstraints.java
License:Apache License
@Override public List<Short> check(Environment env, Mutation mutation) { ArrayList<Short> violations = null; Collection<ColumnUpdate> colUpdates = mutation.getUpdates(); // check the row, it should contains at least one ; or end with < boolean containsSemiC = false; byte[] row = mutation.getRow(); // always allow rows that fall within reserved areas if (row.length > 0 && row[0] == '~') return null; if (row.length > 2 && row[0] == '!' && row[1] == '!' && row[2] == '~') return null; for (byte b : row) { if (b == ';') { containsSemiC = true;//from ww w. jav a 2s. co m } if (b == ';' || b == '<') break; if (!validTableNameChars[0xff & b]) { violations = addIfNotPresent(violations, 4); } } if (!containsSemiC) { // see if last row char is < if (row.length == 0 || row[row.length - 1] != '<') { violations = addIfNotPresent(violations, 4); } } else { if (row.length == 0) { violations = addIfNotPresent(violations, 4); } } if (row.length > 0 && row[0] == '!') { if (row.length < 3 || row[1] != '0' || (row[2] != '<' && row[2] != ';')) { violations = addIfNotPresent(violations, 4); } } // ensure row is not less than Constants.METADATA_TABLE_ID if (new Text(row).compareTo(new Text(MetadataTable.ID)) < 0) { violations = addViolation(violations, 5); } boolean checkedBulk = false; for (ColumnUpdate columnUpdate : colUpdates) { Text columnFamily = new Text(columnUpdate.getColumnFamily()); if (columnUpdate.isDeleted()) { if (!isValidColumn(columnUpdate)) { violations = addViolation(violations, 2); } continue; } if (columnUpdate.getValue().length == 0 && !columnFamily.equals(ScanFileColumnFamily.NAME)) { violations = addViolation(violations, 6); } if (columnFamily.equals(DataFileColumnFamily.NAME)) { try { DataFileValue dfv = new DataFileValue(columnUpdate.getValue()); if (dfv.getSize() < 0 || dfv.getNumEntries() < 0) { violations = addViolation(violations, 1); } } catch (NumberFormatException nfe) { violations = addViolation(violations, 1); } catch (ArrayIndexOutOfBoundsException aiooe) { violations = addViolation(violations, 1); } } else if (columnFamily.equals(ScanFileColumnFamily.NAME)) { } else if (columnFamily.equals(TabletsSection.BulkFileColumnFamily.NAME)) { if (!columnUpdate.isDeleted() && !checkedBulk) { // splits, which also write the time reference, are allowed to write this reference even when // the transaction is not running because the other half of the tablet is holding a reference // to the file. boolean isSplitMutation = false; // When a tablet is assigned, it re-writes the metadata. It should probably only update the location information, // but it writes everything. We allow it to re-write the bulk information if it is setting the location. // See ACCUMULO-1230. boolean isLocationMutation = false; HashSet<Text> dataFiles = new HashSet<>(); HashSet<Text> loadedFiles = new HashSet<>(); String tidString = new String(columnUpdate.getValue(), UTF_8); int otherTidCount = 0; for (ColumnUpdate update : mutation.getUpdates()) { if (new ColumnFQ(update).equals(TabletsSection.ServerColumnFamily.DIRECTORY_COLUMN)) { isSplitMutation = true; } else if (new Text(update.getColumnFamily()) .equals(TabletsSection.CurrentLocationColumnFamily.NAME)) { isLocationMutation = true; } else if (new Text(update.getColumnFamily()).equals(DataFileColumnFamily.NAME)) { dataFiles.add(new Text(update.getColumnQualifier())); } else if (new Text(update.getColumnFamily()) .equals(TabletsSection.BulkFileColumnFamily.NAME)) { loadedFiles.add(new Text(update.getColumnQualifier())); if (!new String(update.getValue(), UTF_8).equals(tidString)) { otherTidCount++; } } } if (!isSplitMutation && !isLocationMutation) { long tid = Long.parseLong(tidString); try { if (otherTidCount > 0 || !dataFiles.equals(loadedFiles) || !getArbitrator().transactionAlive(Constants.BULK_ARBITRATOR_TYPE, tid)) { violations = addViolation(violations, 8); } } catch (Exception ex) { violations = addViolation(violations, 8); } } checkedBulk = true; } } else { if (!isValidColumn(columnUpdate)) { violations = addViolation(violations, 2); } else if (new ColumnFQ(columnUpdate).equals(TabletsSection.TabletColumnFamily.PREV_ROW_COLUMN) && columnUpdate.getValue().length > 0 && (violations == null || !violations.contains((short) 4))) { KeyExtent ke = new KeyExtent(new Text(mutation.getRow()), (Text) null); Text per = KeyExtent.decodePrevEndRow(new Value(columnUpdate.getValue())); boolean prevEndRowLessThanEndRow = per == null || ke.getEndRow() == null || per.compareTo(ke.getEndRow()) < 0; if (!prevEndRowLessThanEndRow) { violations = addViolation(violations, 3); } } else if (new ColumnFQ(columnUpdate).equals(TabletsSection.ServerColumnFamily.LOCK_COLUMN)) { if (zooCache == null) { zooCache = new ZooCache(); } if (zooRoot == null) { zooRoot = ZooUtil.getRoot(HdfsZooInstance.getInstance()); } boolean lockHeld = false; String lockId = new String(columnUpdate.getValue(), UTF_8); try { lockHeld = ZooLock.isLockHeld(zooCache, new ZooUtil.LockID(zooRoot, lockId)); } catch (Exception e) { log.debug("Failed to verify lock was held {} {}", lockId, e.getMessage()); } if (!lockHeld) { violations = addViolation(violations, 7); } } } } if (violations != null) { log.debug("violating metadata mutation : {}", new String(mutation.getRow(), UTF_8)); for (ColumnUpdate update : mutation.getUpdates()) { log.debug(" update: {}:{} value {}", new String(update.getColumnFamily(), UTF_8), new String(update.getColumnQualifier(), UTF_8), (update.isDeleted() ? "[delete]" : new String(update.getValue(), UTF_8))); } } return violations; }