List of usage examples for org.apache.hadoop.fs FileSystem getConf
@Override
public Configuration getConf()
From source file:org.apache.accumulo.test.randomwalk.security.TableOp.java
License:Apache License
@Override public void visit(State state, Environment env, Properties props) throws Exception { Connector conn = env.getInstance().getConnector(WalkingSecurity.get(state, env).getTabUserName(), WalkingSecurity.get(state, env).getTabToken()); String action = props.getProperty("action", "_random"); TablePermission tp;/*from w w w . j a va 2 s . co m*/ if ("_random".equalsIgnoreCase(action)) { Random r = new Random(); tp = TablePermission.values()[r.nextInt(TablePermission.values().length)]; } else { tp = TablePermission.valueOf(action); } boolean tableExists = WalkingSecurity.get(state, env).getTableExists(); String tableName = WalkingSecurity.get(state, env).getTableName(); String namespaceName = WalkingSecurity.get(state, env).getNamespaceName(); switch (tp) { case READ: { boolean canRead = WalkingSecurity.get(state, env) .canScan(WalkingSecurity.get(state, env).getTabCredentials(), tableName, namespaceName); Authorizations auths = WalkingSecurity.get(state, env) .getUserAuthorizations(WalkingSecurity.get(state, env).getTabCredentials()); boolean ambiguousZone = WalkingSecurity.get(state, env).inAmbiguousZone(conn.whoami(), tp); boolean ambiguousAuths = WalkingSecurity.get(state, env).ambiguousAuthorizations(conn.whoami()); Scanner scan = null; try { scan = conn.createScanner(tableName, conn.securityOperations().getUserAuthorizations(conn.whoami())); int seen = 0; Iterator<Entry<Key, Value>> iter = scan.iterator(); while (iter.hasNext()) { Entry<Key, Value> entry = iter.next(); Key k = entry.getKey(); seen++; if (!auths.contains(k.getColumnVisibilityData()) && !ambiguousAuths) throw new AccumuloException( "Got data I should not be capable of seeing: " + k + " table " + tableName); } if (!canRead && !ambiguousZone) throw new AccumuloException( "Was able to read when I shouldn't have had the perm with connection user " + conn.whoami() + " table " + tableName); for (Entry<String, Integer> entry : WalkingSecurity.get(state, env).getAuthsMap().entrySet()) { if (auths.contains(entry.getKey().getBytes(UTF_8))) seen = seen - entry.getValue(); } if (seen != 0 && !ambiguousAuths) throw new AccumuloException("Got mismatched amounts of data"); } catch (TableNotFoundException tnfe) { if (tableExists) throw new AccumuloException("Accumulo and test suite out of sync: table " + tableName, tnfe); return; } catch (AccumuloSecurityException ae) { if (ae.getSecurityErrorCode().equals(SecurityErrorCode.PERMISSION_DENIED)) { if (canRead && !ambiguousZone) throw new AccumuloException( "Table read permission out of sync with Accumulo: table " + tableName, ae); else return; } if (ae.getSecurityErrorCode().equals(SecurityErrorCode.BAD_AUTHORIZATIONS)) { if (ambiguousAuths) return; else throw new AccumuloException("Mismatched authorizations! ", ae); } throw new AccumuloException("Unexpected exception!", ae); } catch (RuntimeException re) { if (re.getCause() instanceof AccumuloSecurityException && ((AccumuloSecurityException) re.getCause()).getSecurityErrorCode() .equals(SecurityErrorCode.PERMISSION_DENIED)) { if (canRead && !ambiguousZone) throw new AccumuloException( "Table read permission out of sync with Accumulo: table " + tableName, re.getCause()); else return; } if (re.getCause() instanceof AccumuloSecurityException && ((AccumuloSecurityException) re.getCause()).getSecurityErrorCode() .equals(SecurityErrorCode.BAD_AUTHORIZATIONS)) { if (ambiguousAuths) return; else throw new AccumuloException("Mismatched authorizations! ", re.getCause()); } throw new AccumuloException("Unexpected exception!", re); } finally { if (scan != null) { scan.close(); scan = null; } } break; } case WRITE: boolean canWrite = WalkingSecurity.get(state, env) .canWrite(WalkingSecurity.get(state, env).getTabCredentials(), tableName, namespaceName); boolean ambiguousZone = WalkingSecurity.get(state, env).inAmbiguousZone(conn.whoami(), tp); String key = WalkingSecurity.get(state, env).getLastKey() + "1"; Mutation m = new Mutation(new Text(key)); for (String s : WalkingSecurity.get(state, env).getAuthsArray()) { m.put(new Text(), new Text(), new ColumnVisibility(s), new Value("value".getBytes(UTF_8))); } BatchWriter writer = null; try { try { writer = conn.createBatchWriter(tableName, new BatchWriterConfig().setMaxMemory(9000l).setMaxWriteThreads(1)); } catch (TableNotFoundException tnfe) { if (tableExists) throw new AccumuloException("Table didn't exist when it should have: " + tableName); return; } boolean works = true; try { writer.addMutation(m); writer.close(); } catch (MutationsRejectedException mre) { // Currently no method for detecting reason for mre. Waiting on ACCUMULO-670 // For now, just wait a second and go again if they can write! if (!canWrite) return; if (ambiguousZone) { Thread.sleep(1000); try { writer = conn.createBatchWriter(tableName, new BatchWriterConfig().setMaxWriteThreads(1)); writer.addMutation(m); writer.close(); writer = null; } catch (MutationsRejectedException mre2) { throw new AccumuloException("Mutation exception!", mre2); } } } if (works) for (String s : WalkingSecurity.get(state, env).getAuthsArray()) WalkingSecurity.get(state, env).increaseAuthMap(s, 1); } finally { if (writer != null) { writer.close(); writer = null; } } break; case BULK_IMPORT: key = WalkingSecurity.get(state, env).getLastKey() + "1"; SortedSet<Key> keys = new TreeSet<>(); for (String s : WalkingSecurity.get(state, env).getAuthsArray()) { Key k = new Key(key, "", "", s); keys.add(k); } Path dir = new Path("/tmp", "bulk_" + UUID.randomUUID().toString()); Path fail = new Path(dir.toString() + "_fail"); FileSystem fs = WalkingSecurity.get(state, env).getFs(); FileSKVWriter f = FileOperations.getInstance().newWriterBuilder() .forFile(dir + "/securityBulk." + RFile.EXTENSION, fs, fs.getConf()) .withTableConfiguration(AccumuloConfiguration.getDefaultConfiguration()).build(); f.startDefaultLocalityGroup(); fs.mkdirs(fail); for (Key k : keys) f.append(k, new Value("Value".getBytes(UTF_8))); f.close(); try { conn.tableOperations().importDirectory(tableName, dir.toString(), fail.toString(), true); } catch (TableNotFoundException tnfe) { if (tableExists) throw new AccumuloException("Table didn't exist when it should have: " + tableName); return; } catch (AccumuloSecurityException ae) { if (ae.getSecurityErrorCode().equals(SecurityErrorCode.PERMISSION_DENIED)) { if (WalkingSecurity.get(state, env).canBulkImport( WalkingSecurity.get(state, env).getTabCredentials(), tableName, namespaceName)) throw new AccumuloException("Bulk Import failed when it should have worked: " + tableName); return; } else if (ae.getSecurityErrorCode().equals(SecurityErrorCode.BAD_CREDENTIALS)) { if (WalkingSecurity.get(state, env).userPassTransient(conn.whoami())) return; } throw new AccumuloException("Unexpected exception!", ae); } for (String s : WalkingSecurity.get(state, env).getAuthsArray()) WalkingSecurity.get(state, env).increaseAuthMap(s, 1); fs.delete(dir, true); fs.delete(fail, true); if (!WalkingSecurity.get(state, env).canBulkImport(WalkingSecurity.get(state, env).getTabCredentials(), tableName, namespaceName)) throw new AccumuloException( "Bulk Import succeeded when it should have failed: " + dir + " table " + tableName); break; case ALTER_TABLE: AlterTable.renameTable(conn, state, env, tableName, tableName + "plus", WalkingSecurity.get(state, env).canAlterTable( WalkingSecurity.get(state, env).getTabCredentials(), tableName, namespaceName), tableExists); break; case GRANT: props.setProperty("task", "grant"); props.setProperty("perm", "random"); props.setProperty("source", "table"); props.setProperty("target", "system"); AlterTablePerm.alter(state, env, props); break; case DROP_TABLE: props.setProperty("source", "table"); DropTable.dropTable(state, env, props); break; } }
From source file:org.apache.accumulo.testing.core.randomwalk.bulk.BulkPlusOne.java
License:Apache License
static void bulkLoadLots(Logger log, State state, RandWalkEnv env, Value value) throws Exception { final Path dir = new Path("/tmp", "bulk_" + UUID.randomUUID().toString()); final Path fail = new Path(dir.toString() + "_fail"); final DefaultConfiguration defaultConfiguration = AccumuloConfiguration.getDefaultConfiguration(); final Random rand = (Random) state.get("rand"); final FileSystem fs = (FileSystem) state.get("fs"); fs.mkdirs(fail);//from w ww . ja v a 2s . c om final int parts = rand.nextInt(10) + 1; TreeSet<Integer> startRows = new TreeSet<>(); startRows.add(0); while (startRows.size() < parts) startRows.add(rand.nextInt(LOTS)); List<String> printRows = new ArrayList<>(startRows.size()); for (Integer row : startRows) printRows.add(String.format(FMT, row)); String markerColumnQualifier = String.format("%07d", counter.incrementAndGet()); log.debug("preparing bulk files with start rows " + printRows + " last row " + String.format(FMT, LOTS - 1) + " marker " + markerColumnQualifier); List<Integer> rows = new ArrayList<>(startRows); rows.add(LOTS); for (int i = 0; i < parts; i++) { String fileName = dir + "/" + String.format("part_%d.", i) + RFile.EXTENSION; FileSKVWriter f = FileOperations.getInstance().newWriterBuilder().forFile(fileName, fs, fs.getConf()) .withTableConfiguration(defaultConfiguration).build(); f.startDefaultLocalityGroup(); int start = rows.get(i); int end = rows.get(i + 1); for (int j = start; j < end; j++) { Text row = new Text(String.format(FMT, j)); for (Column col : COLNAMES) { f.append(new Key(row, col.getColumnFamily(), col.getColumnQualifier()), value); } f.append(new Key(row, MARKER_CF, new Text(markerColumnQualifier)), ONE); } f.close(); } env.getAccumuloConnector().tableOperations().importDirectory(Setup.getTableName(), dir.toString(), fail.toString(), true); fs.delete(dir, true); FileStatus[] failures = fs.listStatus(fail); if (failures != null && failures.length > 0) { state.set("bulkImportSuccess", "false"); throw new Exception(failures.length + " failure files found importing files from " + dir); } fs.delete(fail, true); log.debug("Finished bulk import, start rows " + printRows + " last row " + String.format(FMT, LOTS - 1) + " marker " + markerColumnQualifier); }
From source file:org.apache.accumulo.testing.core.randomwalk.security.TableOp.java
License:Apache License
@Override public void visit(State state, RandWalkEnv env, Properties props) throws Exception { Connector conn = env.getAccumuloInstance().getConnector(WalkingSecurity.get(state, env).getTabUserName(), WalkingSecurity.get(state, env).getTabToken()); String action = props.getProperty("action", "_random"); TablePermission tp;/* w w w .j a v a 2 s. co m*/ if ("_random".equalsIgnoreCase(action)) { Random r = new Random(); tp = TablePermission.values()[r.nextInt(TablePermission.values().length)]; } else { tp = TablePermission.valueOf(action); } boolean tableExists = WalkingSecurity.get(state, env).getTableExists(); String tableName = WalkingSecurity.get(state, env).getTableName(); String namespaceName = WalkingSecurity.get(state, env).getNamespaceName(); switch (tp) { case READ: { boolean canRead = WalkingSecurity.get(state, env) .canScan(WalkingSecurity.get(state, env).getTabCredentials(), tableName, namespaceName); Authorizations auths = WalkingSecurity.get(state, env) .getUserAuthorizations(WalkingSecurity.get(state, env).getTabCredentials()); boolean ambiguousZone = WalkingSecurity.get(state, env).inAmbiguousZone(conn.whoami(), tp); boolean ambiguousAuths = WalkingSecurity.get(state, env).ambiguousAuthorizations(conn.whoami()); Scanner scan = null; try { scan = conn.createScanner(tableName, conn.securityOperations().getUserAuthorizations(conn.whoami())); int seen = 0; Iterator<Entry<Key, Value>> iter = scan.iterator(); while (iter.hasNext()) { Entry<Key, Value> entry = iter.next(); Key k = entry.getKey(); seen++; if (!auths.contains(k.getColumnVisibilityData()) && !ambiguousAuths) throw new AccumuloException( "Got data I should not be capable of seeing: " + k + " table " + tableName); } if (!canRead && !ambiguousZone) throw new AccumuloException( "Was able to read when I shouldn't have had the perm with connection user " + conn.whoami() + " table " + tableName); for (Entry<String, Integer> entry : WalkingSecurity.get(state, env).getAuthsMap().entrySet()) { if (auths.contains(entry.getKey().getBytes(UTF_8))) seen = seen - entry.getValue(); } if (seen != 0 && !ambiguousAuths) throw new AccumuloException("Got mismatched amounts of data"); } catch (TableNotFoundException tnfe) { if (tableExists) throw new AccumuloException("Accumulo and test suite out of sync: table " + tableName, tnfe); return; } catch (AccumuloSecurityException ae) { if (ae.getSecurityErrorCode().equals(SecurityErrorCode.PERMISSION_DENIED)) { if (canRead && !ambiguousZone) throw new AccumuloException( "Table read permission out of sync with Accumulo: table " + tableName, ae); else return; } if (ae.getSecurityErrorCode().equals(SecurityErrorCode.BAD_AUTHORIZATIONS)) { if (ambiguousAuths) return; else throw new AccumuloException("Mismatched authorizations! ", ae); } throw new AccumuloException("Unexpected exception!", ae); } catch (RuntimeException re) { if (re.getCause() instanceof AccumuloSecurityException && ((AccumuloSecurityException) re.getCause()).getSecurityErrorCode() .equals(SecurityErrorCode.PERMISSION_DENIED)) { if (canRead && !ambiguousZone) throw new AccumuloException( "Table read permission out of sync with Accumulo: table " + tableName, re.getCause()); else return; } if (re.getCause() instanceof AccumuloSecurityException && ((AccumuloSecurityException) re.getCause()).getSecurityErrorCode() .equals(SecurityErrorCode.BAD_AUTHORIZATIONS)) { if (ambiguousAuths) return; else throw new AccumuloException("Mismatched authorizations! ", re.getCause()); } throw new AccumuloException("Unexpected exception!", re); } finally { if (scan != null) { scan.close(); scan = null; } } break; } case WRITE: boolean canWrite = WalkingSecurity.get(state, env) .canWrite(WalkingSecurity.get(state, env).getTabCredentials(), tableName, namespaceName); boolean ambiguousZone = WalkingSecurity.get(state, env).inAmbiguousZone(conn.whoami(), tp); String key = WalkingSecurity.get(state, env).getLastKey() + "1"; Mutation m = new Mutation(new Text(key)); for (String s : WalkingSecurity.get(state, env).getAuthsArray()) { m.put(new Text(), new Text(), new ColumnVisibility(s), new Value("value".getBytes(UTF_8))); } BatchWriter writer = null; try { try { writer = conn.createBatchWriter(tableName, new BatchWriterConfig().setMaxMemory(9000l).setMaxWriteThreads(1)); } catch (TableNotFoundException tnfe) { if (tableExists) throw new AccumuloException("Table didn't exist when it should have: " + tableName); return; } boolean works = true; try { writer.addMutation(m); writer.close(); } catch (MutationsRejectedException mre) { // Currently no method for detecting reason for mre. // Waiting on ACCUMULO-670 // For now, just wait a second and go again if they can // write! if (!canWrite) return; if (ambiguousZone) { Thread.sleep(1000); try { writer = conn.createBatchWriter(tableName, new BatchWriterConfig().setMaxWriteThreads(1)); writer.addMutation(m); writer.close(); writer = null; } catch (MutationsRejectedException mre2) { throw new AccumuloException("Mutation exception!", mre2); } } } if (works) for (String s : WalkingSecurity.get(state, env).getAuthsArray()) WalkingSecurity.get(state, env).increaseAuthMap(s, 1); } finally { if (writer != null) { writer.close(); writer = null; } } break; case BULK_IMPORT: key = WalkingSecurity.get(state, env).getLastKey() + "1"; SortedSet<Key> keys = new TreeSet<>(); for (String s : WalkingSecurity.get(state, env).getAuthsArray()) { Key k = new Key(key, "", "", s); keys.add(k); } Path dir = new Path("/tmp", "bulk_" + UUID.randomUUID().toString()); Path fail = new Path(dir.toString() + "_fail"); FileSystem fs = WalkingSecurity.get(state, env).getFs(); FileSKVWriter f = FileOperations.getInstance().newWriterBuilder() .forFile(dir + "/securityBulk." + RFile.EXTENSION, fs, fs.getConf()) .withTableConfiguration(AccumuloConfiguration.getDefaultConfiguration()).build(); f.startDefaultLocalityGroup(); fs.mkdirs(fail); for (Key k : keys) f.append(k, new Value("Value".getBytes(UTF_8))); f.close(); try { conn.tableOperations().importDirectory(tableName, dir.toString(), fail.toString(), true); } catch (TableNotFoundException tnfe) { if (tableExists) throw new AccumuloException("Table didn't exist when it should have: " + tableName); return; } catch (AccumuloSecurityException ae) { if (ae.getSecurityErrorCode().equals(SecurityErrorCode.PERMISSION_DENIED)) { if (WalkingSecurity.get(state, env).canBulkImport( WalkingSecurity.get(state, env).getTabCredentials(), tableName, namespaceName)) throw new AccumuloException("Bulk Import failed when it should have worked: " + tableName); return; } else if (ae.getSecurityErrorCode().equals(SecurityErrorCode.BAD_CREDENTIALS)) { if (WalkingSecurity.get(state, env).userPassTransient(conn.whoami())) return; } throw new AccumuloException("Unexpected exception!", ae); } for (String s : WalkingSecurity.get(state, env).getAuthsArray()) WalkingSecurity.get(state, env).increaseAuthMap(s, 1); fs.delete(dir, true); fs.delete(fail, true); if (!WalkingSecurity.get(state, env).canBulkImport(WalkingSecurity.get(state, env).getTabCredentials(), tableName, namespaceName)) throw new AccumuloException( "Bulk Import succeeded when it should have failed: " + dir + " table " + tableName); break; case ALTER_TABLE: AlterTable.renameTable(conn, state, env, tableName, tableName + "plus", WalkingSecurity.get(state, env).canAlterTable( WalkingSecurity.get(state, env).getTabCredentials(), tableName, namespaceName), tableExists); break; case GRANT: props.setProperty("task", "grant"); props.setProperty("perm", "random"); props.setProperty("source", "table"); props.setProperty("target", "system"); AlterTablePerm.alter(state, env, props); break; case DROP_TABLE: props.setProperty("source", "table"); DropTable.dropTable(state, env, props); break; } }
From source file:org.apache.accumulo.tserver.compaction.MajorCompactionRequest.java
License:Apache License
public FileSKVIterator openReader(FileRef ref) throws IOException { // @TODO verify the file isn't some random file in HDFS // @TODO ensure these files are always closed? FileOperations fileFactory = FileOperations.getInstance(); FileSystem ns = volumeManager.getVolumeByPath(ref.path()).getFileSystem(); FileSKVIterator openReader = fileFactory.newReaderBuilder().forFile(ref.path().toString(), ns, ns.getConf()) .withTableConfiguration(tableConfig).seekToBeginning().build(); return openReader; }
From source file:org.apache.accumulo.tserver.Compactor.java
License:Apache License
@Override public CompactionStats call() throws IOException, CompactionCanceledException { FileSKVWriter mfw = null;/*from w ww.ja va2s . co m*/ CompactionStats majCStats = new CompactionStats(); boolean remove = runningCompactions.add(this); clearStats(); String oldThreadName = Thread.currentThread().getName(); String newThreadName = "MajC compacting " + extent.toString() + " started " + dateFormatter.format(new Date()) + " file: " + outputFile; Thread.currentThread().setName(newThreadName); thread = Thread.currentThread(); try { FileOperations fileFactory = FileOperations.getInstance(); FileSystem ns = this.fs.getFileSystemByPath(outputFile.path()); mfw = fileFactory.openWriter(outputFile.path().toString(), ns, ns.getConf(), acuTableConf); Map<String, Set<ByteSequence>> lGroups; try { lGroups = LocalityGroupUtil.getLocalityGroups(acuTableConf); } catch (LocalityGroupConfigurationError e) { throw new IOException(e); } long t1 = System.currentTimeMillis(); HashSet<ByteSequence> allColumnFamilies = new HashSet<ByteSequence>(); if (mfw.supportsLocalityGroups()) { for (Entry<String, Set<ByteSequence>> entry : lGroups.entrySet()) { setLocalityGroup(entry.getKey()); compactLocalityGroup(entry.getKey(), entry.getValue(), true, mfw, majCStats); allColumnFamilies.addAll(entry.getValue()); } } setLocalityGroup(""); compactLocalityGroup(null, allColumnFamilies, false, mfw, majCStats); long t2 = System.currentTimeMillis(); FileSKVWriter mfwTmp = mfw; mfw = null; // set this to null so we do not try to close it again in finally if the close fails mfwTmp.close(); // if the close fails it will cause the compaction to fail // Verify the file, since hadoop 0.20.2 sometimes lies about the success of close() try { FileSKVIterator openReader = fileFactory.openReader(outputFile.path().toString(), false, ns, ns.getConf(), acuTableConf); openReader.close(); } catch (IOException ex) { log.error("Verification of successful compaction fails!!! " + extent + " " + outputFile, ex); throw ex; } log.debug(String.format("Compaction %s %,d read | %,d written | %,6d entries/sec | %6.3f secs", extent, majCStats.getEntriesRead(), majCStats.getEntriesWritten(), (int) (majCStats.getEntriesRead() / ((t2 - t1) / 1000.0)), (t2 - t1) / 1000.0)); majCStats.setFileSize( fileFactory.getFileSize(outputFile.path().toString(), ns, ns.getConf(), acuTableConf)); return majCStats; } catch (IOException e) { log.error(e, e); throw e; } catch (RuntimeException e) { log.error(e, e); throw e; } finally { Thread.currentThread().setName(oldThreadName); if (remove) { thread = null; runningCompactions.remove(this); } try { if (mfw != null) { // compaction must not have finished successfully, so close its output file try { mfw.close(); } finally { if (!fs.deleteRecursively(outputFile.path())) if (fs.exists(outputFile.path())) log.error("Unable to delete " + outputFile); } } } catch (IOException e) { log.warn(e, e); } catch (RuntimeException exception) { log.warn(exception, exception); } } }
From source file:org.apache.accumulo.tserver.FileManager.java
License:Apache License
private List<FileSKVIterator> reserveReaders(KeyExtent tablet, Collection<String> files, boolean continueOnFailure) throws IOException { if (!tablet.isMeta() && files.size() >= maxOpen) { throw new IllegalArgumentException("requested files exceeds max open"); }/*from w w w . jav a 2s .c om*/ if (files.size() == 0) { return Collections.emptyList(); } List<String> filesToOpen = null; List<FileSKVIterator> filesToClose = Collections.emptyList(); List<FileSKVIterator> reservedFiles = new ArrayList<>(); Map<FileSKVIterator, String> readersReserved = new HashMap<>(); if (!tablet.isMeta()) { filePermits.acquireUninterruptibly(files.size()); } // now that the we are past the semaphore, we have the authority // to open files.size() files // determine what work needs to be done in sync block // but do the work of opening and closing files outside // a synch block synchronized (this) { filesToOpen = takeOpenFiles(files, reservedFiles, readersReserved); int numOpen = countReaders(openFiles); if (filesToOpen.size() + numOpen + reservedReaders.size() > maxOpen) { filesToClose = takeLRUOpenFiles((filesToOpen.size() + numOpen + reservedReaders.size()) - maxOpen); } } // close files before opening files to ensure we stay under resource // limitations closeReaders(filesToClose); // open any files that need to be opened for (String file : filesToOpen) { try { if (!file.contains(":")) throw new IllegalArgumentException("Expected uri, got : " + file); Path path = new Path(file); FileSystem ns = fs.getVolumeByPath(path).getFileSystem(); // log.debug("Opening "+file + " path " + path); FileSKVIterator reader = FileOperations.getInstance().newReaderBuilder() .forFile(path.toString(), ns, ns.getConf()) .withTableConfiguration( context.getServerConfigurationFactory().getTableConfiguration(tablet)) .withBlockCache(dataCache, indexCache).build(); reservedFiles.add(reader); readersReserved.put(reader, file); } catch (Exception e) { ProblemReports.getInstance(context) .report(new ProblemReport(tablet.getTableId(), ProblemType.FILE_READ, file, e)); if (continueOnFailure) { // release the permit for the file that failed to open if (!tablet.isMeta()) { filePermits.release(1); } log.warn("Failed to open file {} {} continuing...", file, e.getMessage()); } else { // close whatever files were opened closeReaders(reservedFiles); if (!tablet.isMeta()) { filePermits.release(files.size()); } log.error("Failed to open file {} {}", file, e.getMessage()); throw new IOException("Failed to open " + file, e); } } } synchronized (this) { // update set of reserved readers reservedReaders.putAll(readersReserved); } return reservedFiles; }
From source file:org.apache.accumulo.tserver.log.LocalWALRecovery.java
License:Apache License
public void recoverLocalWriteAheadLogs(FileSystem fs) throws IOException { for (String directory : options.directories) { File localDirectory = new File(directory); if (!localDirectory.isAbsolute()) { localDirectory = new File(System.getenv("ACCUMULO_HOME"), directory); }/*from w w w . j a va2 s .c o m*/ if (!localDirectory.isDirectory()) { log.warn("Local walog dir " + localDirectory.getAbsolutePath() + " does not exist or is not a directory."); continue; } if (options.destination == null) { // Defer loading the default value until now because it might require talking to zookeeper. options.destination = ServerConstants.getWalDirs()[0]; } log.info("Copying WALs to " + options.destination); for (File file : localDirectory.listFiles()) { String name = file.getName(); try { UUID.fromString(name); } catch (IllegalArgumentException ex) { log.info("Ignoring non-log file " + file.getAbsolutePath()); continue; } LogFileKey key = new LogFileKey(); LogFileValue value = new LogFileValue(); log.info("Openning local log " + file.getAbsolutePath()); Path localWal = new Path(file.toURI()); FileSystem localFs = FileSystem.getLocal(fs.getConf()); Reader reader = new SequenceFile.Reader(localFs, localWal, localFs.getConf()); // Reader reader = new SequenceFile.Reader(localFs.getConf(), SequenceFile.Reader.file(localWal)); Path tmp = new Path(options.destination + "/" + name + ".copy"); FSDataOutputStream writer = fs.create(tmp); while (reader.next(key, value)) { try { key.write(writer); value.write(writer); } catch (EOFException ex) { break; } } writer.close(); reader.close(); fs.rename(tmp, new Path(tmp.getParent(), name)); if (options.deleteLocal) { if (file.delete()) { log.info("Copied and deleted: " + name); } else { log.info("Failed to delete: " + name + " (but it is safe for you to delete it manually)."); } } else { log.info("Safe to delete: " + name); } } } }
From source file:org.apache.accumulo.tserver.log.MultiReader.java
License:Apache License
public MultiReader(VolumeManager fs, Path directory) throws IOException { boolean foundFinish = false; for (FileStatus child : fs.listStatus(directory)) { if (child.getPath().getName().startsWith("_")) continue; if (SortedLogState.isFinished(child.getPath().getName())) { foundFinish = true;// w w w. j a v a 2 s. c o m continue; } FileSystem ns = fs.getVolumeByPath(child.getPath()).getFileSystem(); heap.add(new Index(new Reader(ns.makeQualified(child.getPath()), ns.getConf()))); } if (!foundFinish) throw new IOException( "Sort \"" + SortedLogState.FINISHED.getMarker() + "\" flag not found in " + directory); }
From source file:org.apache.accumulo.tserver.log.MultiReaderTest.java
License:Apache License
@Before public void setUp() throws Exception { root.create();//from w ww. j a va 2s .co m String path = root.getRoot().getAbsolutePath() + "/manyMaps"; fs = VolumeManagerImpl.getLocal(path); Path root = new Path("file://" + path); fs.mkdirs(root); fs.create(new Path(root, "finished")).close(); FileSystem ns = fs.getVolumeByPath(root).getFileSystem(); Writer oddWriter = new Writer(ns.getConf(), ns.makeQualified(new Path(root, "odd")), Writer.keyClass(IntWritable.class), Writer.valueClass(BytesWritable.class)); BytesWritable value = new BytesWritable("someValue".getBytes()); for (int i = 1; i < 1000; i += 2) { oddWriter.append(new IntWritable(i), value); } oddWriter.close(); Writer evenWriter = new Writer(ns.getConf(), ns.makeQualified(new Path(root, "even")), Writer.keyClass(IntWritable.class), Writer.valueClass(BytesWritable.class)); for (int i = 0; i < 1000; i += 2) { if (i == 10) continue; evenWriter.append(new IntWritable(i), value); } evenWriter.close(); }
From source file:org.apache.accumulo.tserver.log.RecoveryLogReader.java
License:Apache License
public RecoveryLogReader(VolumeManager fs, Path directory, LogFileKey start, LogFileKey end) throws IOException { boolean foundFinish = false; for (FileStatus child : fs.listStatus(directory)) { if (child.getPath().getName().startsWith("_")) continue; if (SortedLogState.isFinished(child.getPath().getName())) { foundFinish = true;/*from w w w .ja va2 s . com*/ continue; } FileSystem ns = fs.getVolumeByPath(child.getPath()).getFileSystem(); heap.add(new Index(new Reader(ns.makeQualified(child.getPath()), ns.getConf()))); } if (!foundFinish) throw new IOException( "Sort \"" + SortedLogState.FINISHED.getMarker() + "\" flag not found in " + directory); iter = new SortCheckIterator(new RangeIterator(start, end)); }