List of usage examples for org.apache.hadoop.fs FileSystem getConf
@Override
public Configuration getConf()
From source file:org.apache.accumulo.server.tabletserver.compaction.MajorCompactionRequest.java
License:Apache License
public FileSKVIterator openReader(FileRef ref) throws IOException { // @TODO verify the file isn't some random file in HDFS // @TODO ensure these files are always closed? FileOperations fileFactory = FileOperations.getInstance(); FileSystem ns = volumeManager.getFileSystemByPath(ref.path()); FileSKVIterator openReader = fileFactory.openReader(ref.path().toString(), true, ns, ns.getConf(), tableConfig);//from ww w . ja v a 2s . c om return openReader; }
From source file:org.apache.accumulo.server.tabletserver.Compactor.java
License:Apache License
@Override public CompactionStats call() throws IOException, CompactionCanceledException { FileSKVWriter mfw = null;//from w w w. j ava 2s . c o m CompactionStats majCStats = new CompactionStats(); boolean remove = runningCompactions.add(this); clearStats(); String oldThreadName = Thread.currentThread().getName(); String newThreadName = "MajC compacting " + extent.toString() + " started " + dateFormatter.format(new Date()) + " file: " + outputFile; Thread.currentThread().setName(newThreadName); thread = Thread.currentThread(); try { FileOperations fileFactory = FileOperations.getInstance(); FileSystem ns = this.fs.getFileSystemByPath(outputFile.path()); mfw = fileFactory.openWriter(outputFile.path().toString(), ns, ns.getConf(), acuTableConf); Map<String, Set<ByteSequence>> lGroups; try { lGroups = LocalityGroupUtil.getLocalityGroups(acuTableConf); } catch (LocalityGroupConfigurationError e) { throw new IOException(e); } long t1 = System.currentTimeMillis(); HashSet<ByteSequence> allColumnFamilies = new HashSet<ByteSequence>(); if (mfw.supportsLocalityGroups()) { for (Entry<String, Set<ByteSequence>> entry : lGroups.entrySet()) { setLocalityGroup(entry.getKey()); compactLocalityGroup(entry.getKey(), entry.getValue(), true, mfw, majCStats); allColumnFamilies.addAll(entry.getValue()); } } setLocalityGroup(""); compactLocalityGroup(null, allColumnFamilies, false, mfw, majCStats); long t2 = System.currentTimeMillis(); FileSKVWriter mfwTmp = mfw; mfw = null; // set this to null so we do not try to close it again in finally if the close fails mfwTmp.close(); // if the close fails it will cause the compaction to fail // Verify the file, since hadoop 0.20.2 sometimes lies about the success of close() try { FileSKVIterator openReader = fileFactory.openReader(outputFile.path().toString(), false, ns, ns.getConf(), acuTableConf); openReader.close(); } catch (IOException ex) { log.error("Verification of successful compaction fails!!! " + extent + " " + outputFile, ex); throw ex; } log.debug(String.format("Compaction %s %,d read | %,d written | %,6d entries/sec | %6.3f secs", extent, majCStats.getEntriesRead(), majCStats.getEntriesWritten(), (int) (majCStats.getEntriesRead() / ((t2 - t1) / 1000.0)), (t2 - t1) / 1000.0)); majCStats.setFileSize( fileFactory.getFileSize(outputFile.path().toString(), ns, ns.getConf(), acuTableConf)); return majCStats; } catch (IOException e) { log.error(e, e); throw e; } catch (RuntimeException e) { log.error(e, e); throw e; } finally { Thread.currentThread().setName(oldThreadName); if (remove) { thread = null; runningCompactions.remove(this); } try { if (mfw != null) { // compaction must not have finished successfully, so close its output file try { mfw.close(); } finally { if (!fs.deleteRecursively(outputFile.path())) if (fs.exists(outputFile.path())) log.error("Unable to delete " + outputFile); } } } catch (IOException e) { log.warn(e, e); } } }
From source file:org.apache.accumulo.server.tabletserver.FileManager.java
License:Apache License
private List<FileSKVIterator> reserveReaders(Text table, Collection<String> files, boolean continueOnFailure) throws IOException { if (files.size() >= maxOpen) { throw new IllegalArgumentException("requested files exceeds max open"); }/*from w ww.j a v a 2s. c om*/ if (files.size() == 0) { return Collections.emptyList(); } List<String> filesToOpen = null; List<FileSKVIterator> filesToClose = Collections.emptyList(); List<FileSKVIterator> reservedFiles = new ArrayList<FileSKVIterator>(); Map<FileSKVIterator, String> readersReserved = new HashMap<FileSKVIterator, String>(); filePermits.acquireUninterruptibly(files.size()); // now that the we are past the semaphore, we have the authority // to open files.size() files // determine what work needs to be done in sync block // but do the work of opening and closing files outside // a synch block synchronized (this) { filesToOpen = takeOpenFiles(files, reservedFiles, readersReserved); int numOpen = countReaders(openFiles); if (filesToOpen.size() + numOpen + reservedReaders.size() > maxOpen) { filesToClose = takeLRUOpenFiles((filesToOpen.size() + numOpen + reservedReaders.size()) - maxOpen); } } // close files before opening files to ensure we stay under resource // limitations closeReaders(filesToClose); // open any files that need to be opened for (String file : filesToOpen) { try { if (!file.contains(":")) throw new IllegalArgumentException("Expected uri, got : " + file); Path path = new Path(file); FileSystem ns = fs.getFileSystemByPath(path); //log.debug("Opening "+file + " path " + path); FileSKVIterator reader = FileOperations.getInstance().openReader(path.toString(), false, ns, ns.getConf(), conf.getTableConfiguration(table.toString()), dataCache, indexCache); reservedFiles.add(reader); readersReserved.put(reader, file); } catch (Exception e) { ProblemReports.getInstance() .report(new ProblemReport(table.toString(), ProblemType.FILE_READ, file, e)); if (continueOnFailure) { // release the permit for the file that failed to open filePermits.release(1); log.warn("Failed to open file " + file + " " + e.getMessage() + " continuing..."); } else { // close whatever files were opened closeReaders(reservedFiles); filePermits.release(files.size()); log.error("Failed to open file " + file + " " + e.getMessage()); throw new IOException("Failed to open " + file, e); } } } synchronized (this) { // update set of reserved readers reservedReaders.putAll(readersReserved); } return reservedFiles; }
From source file:org.apache.accumulo.server.tabletserver.log.MultiReader.java
License:Apache License
public MultiReader(VolumeManager fs, Path directory) throws IOException { boolean foundFinish = false; for (FileStatus child : fs.listStatus(directory)) { if (child.getPath().getName().startsWith("_")) continue; if (child.getPath().getName().equals("finished")) { foundFinish = true;// w w w. j a v a2 s . c o m continue; } FileSystem ns = fs.getFileSystemByPath(child.getPath()); heap.add(new Index(new Reader(ns, child.getPath().toString(), ns.getConf()))); } if (!foundFinish) throw new IOException("Sort \"finished\" flag not found in " + directory); }
From source file:org.apache.accumulo.server.tabletserver.log.MultiReaderTest.java
License:Apache License
@Before public void setUp() throws Exception { // quiet log messages about compress.CodecPool Logger.getRootLogger().setLevel(Level.ERROR); fs = VolumeManagerImpl.getLocal();// ww w. jav a 2s .c o m root.create(); String path = root.getRoot().getAbsolutePath(); Path root = new Path("file://" + path + "/manyMaps"); fs.mkdirs(root); fs.create(new Path(root, "finished")).close(); FileSystem ns = fs.getDefaultVolume(); Writer writer = new Writer(ns.getConf(), ns, new Path(root, "odd").toString(), IntWritable.class, BytesWritable.class); BytesWritable value = new BytesWritable("someValue".getBytes()); for (int i = 1; i < 1000; i += 2) { writer.append(new IntWritable(i), value); } writer.close(); writer = new Writer(ns.getConf(), ns, new Path(root, "even").toString(), IntWritable.class, BytesWritable.class); for (int i = 0; i < 1000; i += 2) { if (i == 10) continue; writer.append(new IntWritable(i), value); } writer.close(); }
From source file:org.apache.accumulo.server.tabletserver.log.SortedLogRecoveryTest.java
License:Apache License
private static List<Mutation> recover(Map<String, KeyValue[]> logs, Set<String> files, KeyExtent extent) throws IOException { TemporaryFolder root = new TemporaryFolder(new File(System.getProperty("user.dir") + "/target")); root.create();//from w w w. ja v a 2s . c o m final String workdir = "file://" + root.getRoot().getAbsolutePath() + "/workdir"; VolumeManager fs = VolumeManagerImpl.getLocal(); fs.deleteRecursively(new Path(workdir)); ArrayList<Path> dirs = new ArrayList<Path>(); try { for (Entry<String, KeyValue[]> entry : logs.entrySet()) { String path = workdir + "/" + entry.getKey(); FileSystem ns = fs.getFileSystemByPath(new Path(path)); Writer map = new MapFile.Writer(ns.getConf(), ns, path + "/log1", LogFileKey.class, LogFileValue.class); for (KeyValue lfe : entry.getValue()) { map.append(lfe.key, lfe.value); } map.close(); ns.create(new Path(path, "finished")).close(); dirs.add(new Path(path)); } // Recover SortedLogRecovery recovery = new SortedLogRecovery(fs); CaptureMutations capture = new CaptureMutations(); recovery.recover(extent, dirs, files, capture); return capture.result; } finally { root.delete(); } }
From source file:org.apache.accumulo.server.tabletserver.Tablet.java
License:Apache License
/** * yet another constructor - this one allows us to avoid costly lookups into the Metadata table if we already know the files we need - as at split time *//* w ww . j a v a 2 s . c om*/ private Tablet(final TabletServer tabletServer, final Text location, final KeyExtent extent, final TabletResourceManager trm, final Configuration conf, final VolumeManager fs, final List<LogEntry> logEntries, final SortedMap<FileRef, DataFileValue> datafiles, String time, final TServerInstance lastLocation, Set<FileRef> scanFiles, long initFlushID, long initCompactID) throws IOException { Path locationPath; if (location.find(":") >= 0) { locationPath = new Path(location.toString()); } else { locationPath = fs.getFullPath(FileType.TABLE, extent.getTableId().toString() + location.toString()); } this.location = locationPath.makeQualified(fs.getFileSystemByPath(locationPath)); this.lastLocation = lastLocation; this.tabletDirectory = location.toString(); this.conf = conf; this.acuTableConf = tabletServer.getTableConfiguration(extent); this.fs = fs; this.extent = extent; this.tabletResources = trm; this.lastFlushID = initFlushID; this.lastCompactID = initCompactID; if (extent.isRootTablet()) { long rtime = Long.MIN_VALUE; for (FileRef ref : datafiles.keySet()) { Path path = ref.path(); FileSystem ns = fs.getFileSystemByPath(path); FileSKVIterator reader = FileOperations.getInstance().openReader(path.toString(), true, ns, ns.getConf(), tabletServer.getTableConfiguration(extent)); long maxTime = -1; try { while (reader.hasTop()) { maxTime = Math.max(maxTime, reader.getTopKey().getTimestamp()); reader.next(); } } finally { reader.close(); } if (maxTime > rtime) { time = TabletTime.LOGICAL_TIME_ID + "" + maxTime; rtime = maxTime; } } } this.tabletServer = tabletServer; this.logId = tabletServer.createLogId(extent); this.timer = new TabletStatsKeeper(); setupDefaultSecurityLabels(extent); tabletMemory = new TabletMemory(); tabletTime = TabletTime.getInstance(time); persistedTime = tabletTime.getTime(); acuTableConf.addObserver(configObserver = new ConfigurationObserver() { private void reloadConstraints() { constraintChecker.set(new ConstraintChecker(getTableConfiguration())); } @Override public void propertiesChanged() { reloadConstraints(); try { setupDefaultSecurityLabels(extent); } catch (Exception e) { log.error("Failed to reload default security labels for extent: " + extent.toString()); } } @Override public void propertyChanged(String prop) { if (prop.startsWith(Property.TABLE_CONSTRAINT_PREFIX.getKey())) reloadConstraints(); else if (prop.equals(Property.TABLE_DEFAULT_SCANTIME_VISIBILITY.getKey())) { try { log.info("Default security labels changed for extent: " + extent.toString()); setupDefaultSecurityLabels(extent); } catch (Exception e) { log.error("Failed to reload default security labels for extent: " + extent.toString()); } } } @Override public void sessionExpired() { log.debug("Session expired, no longer updating per table props..."); } }); // Force a load of any per-table properties configObserver.propertiesChanged(); tabletResources.setTablet(this, acuTableConf); if (!logEntries.isEmpty()) { log.info("Starting Write-Ahead Log recovery for " + this.extent); final long[] count = new long[2]; final CommitSession commitSession = tabletMemory.getCommitSession(); count[1] = Long.MIN_VALUE; try { Set<String> absPaths = new HashSet<String>(); for (FileRef ref : datafiles.keySet()) absPaths.add(ref.path().toString()); tabletServer.recover(this.tabletServer.getFileSystem(), this, logEntries, absPaths, new MutationReceiver() { @Override public void receive(Mutation m) { // LogReader.printMutation(m); Collection<ColumnUpdate> muts = m.getUpdates(); for (ColumnUpdate columnUpdate : muts) { if (!columnUpdate.hasTimestamp()) { // if it is not a user set timestamp, it must have been set // by the system count[1] = Math.max(count[1], columnUpdate.getTimestamp()); } } tabletMemory.mutate(commitSession, Collections.singletonList(m)); count[0]++; } }); if (count[1] != Long.MIN_VALUE) { tabletTime.useMaxTimeFromWALog(count[1]); } commitSession.updateMaxCommittedTime(tabletTime.getTime()); tabletMemory.updateMemoryUsageStats(); if (count[0] == 0) { MetadataTableUtil.removeUnusedWALEntries(extent, logEntries, tabletServer.getLock()); logEntries.clear(); } } catch (Throwable t) { if (acuTableConf.getBoolean(Property.TABLE_FAILURES_IGNORE)) { log.warn("Error recovering from log files: ", t); } else { throw new RuntimeException(t); } } // make some closed references that represent the recovered logs currentLogs = new HashSet<DfsLogger>(); for (LogEntry logEntry : logEntries) { for (String log : logEntry.logSet) { String[] parts = log.split("/", 2); Path file = fs.getFullPath(FileType.WAL, parts[1]); currentLogs.add(new DfsLogger(tabletServer.getServerConfig(), logEntry.server, file)); } } log.info("Write-Ahead Log recovery complete for " + this.extent + " (" + count[0] + " mutations applied, " + tabletMemory.getNumEntries() + " entries created)"); } String contextName = acuTableConf.get(Property.TABLE_CLASSPATH); if (contextName != null && !contextName.equals("")) { // initialize context classloader, instead of possibly waiting for it to initialize for a scan // TODO this could hang, causing other tablets to fail to load - ACCUMULO-1292 AccumuloVFSClassLoader.getContextManager().getClassLoader(contextName); } // do this last after tablet is completely setup because it // could cause major compaction to start datafileManager = new DatafileManager(datafiles); computeNumEntries(); datafileManager.removeFilesAfterScan(scanFiles); // look for hints of a failure on the previous tablet server if (!logEntries.isEmpty() || needsMajorCompaction(MajorCompactionReason.NORMAL)) { // look for any temp files hanging around removeOldTemporaryFiles(); } log.log(TLevel.TABLET_HIST, extent + " opened "); }
From source file:org.apache.accumulo.server.tabletserver.Tablet.java
License:Apache License
private Map<FileRef, Pair<Key, Key>> getFirstAndLastKeys(SortedMap<FileRef, DataFileValue> files) throws IOException { FileOperations fileFactory = FileOperations.getInstance(); Map<FileRef, Pair<Key, Key>> falks = new HashMap<FileRef, Pair<Key, Key>>(); for (Entry<FileRef, DataFileValue> entry : files.entrySet()) { FileRef file = entry.getKey();/*ww w . j a v a 2 s. c o m*/ FileSystem ns = fs.getFileSystemByPath(file.path()); FileSKVIterator openReader = fileFactory.openReader(file.path().toString(), true, ns, ns.getConf(), acuTableConf); try { Key first = openReader.getFirstKey(); Key last = openReader.getLastKey(); falks.put(file, new Pair<Key, Key>(first, last)); } finally { openReader.close(); } } return falks; }
From source file:org.apache.accumulo.server.test.randomwalk.bulk.BulkPlusOne.java
License:Apache License
static void bulkLoadLots(Logger log, State state, Value value) throws Exception { final Path dir = new Path("/tmp", "bulk_" + UUID.randomUUID().toString()); final Path fail = new Path(dir.toString() + "_fail"); final DefaultConfiguration defaultConfiguration = AccumuloConfiguration.getDefaultConfiguration(); final Random rand = (Random) state.get("rand"); final FileSystem fs = (FileSystem) state.get("fs"); fs.mkdirs(fail);//from w w w .j a va 2 s. c o m final int parts = rand.nextInt(10) + 1; TreeSet<Integer> startRows = new TreeSet<Integer>(); startRows.add(0); while (startRows.size() < parts) startRows.add(rand.nextInt(LOTS)); List<String> printRows = new ArrayList<String>(startRows.size()); for (Integer row : startRows) printRows.add(String.format(FMT, row)); String markerColumnQualifier = String.format("%07d", counter.incrementAndGet()); log.debug("preparing bulk files with start rows " + printRows + " last row " + String.format(FMT, LOTS - 1) + " marker " + markerColumnQualifier); List<Integer> rows = new ArrayList<Integer>(startRows); rows.add(LOTS); for (int i = 0; i < parts; i++) { String fileName = dir + "/" + String.format("part_%d.", i) + RFile.EXTENSION; FileSKVWriter f = FileOperations.getInstance().openWriter(fileName, fs, fs.getConf(), defaultConfiguration); f.startDefaultLocalityGroup(); int start = rows.get(i); int end = rows.get(i + 1); for (int j = start; j < end; j++) { Text row = new Text(String.format(FMT, j)); for (Column col : COLNAMES) { f.append(new Key(row, col.getColumnFamily(), col.getColumnQualifier()), value); } f.append(new Key(row, MARKER_CF, new Text(markerColumnQualifier)), ONE); } f.close(); } state.getConnector().tableOperations().importDirectory(Setup.getTableName(), dir.toString(), fail.toString(), true); fs.delete(dir, true); FileStatus[] failures = fs.listStatus(fail); if (failures != null && failures.length > 0) { throw new Exception(failures.length + " failure files found importing files from " + dir); } fs.delete(fail, true); log.debug("Finished bulk import, start rows " + printRows + " last row " + String.format(FMT, LOTS - 1) + " marker " + markerColumnQualifier); }
From source file:org.apache.accumulo.server.test.randomwalk.security.TableOp.java
License:Apache License
@Override public void visit(State state, Properties props) throws Exception { boolean userExists = SecurityHelper.getTabUserExists(state); Connector conn;// www . j a va2 s . com try { conn = state.getInstance().getConnector(SecurityHelper.getTabUserName(state), SecurityHelper.getTabUserPass(state)); } catch (AccumuloSecurityException ae) { if (ae.getErrorCode().equals(SecurityErrorCode.BAD_CREDENTIALS)) { if (userExists) throw new AccumuloException("User didn't exist when they should (or worse- password mismatch)", ae); else return; } throw new AccumuloException("Unexpected exception!", ae); } String action = props.getProperty("action", "_random"); TablePermission tp; if ("_random".equalsIgnoreCase(action)) { Random r = new Random(); tp = TablePermission.values()[r.nextInt(TablePermission.values().length)]; } else { tp = TablePermission.valueOf(action); } boolean tableExists = SecurityHelper.getTableExists(state); boolean hasPerm = SecurityHelper.getTabPerm(state, SecurityHelper.getTabUserName(state), tp); String tableName = state.getString("secTableName"); switch (tp) { case READ: Authorizations auths = SecurityHelper.getUserAuths(state, SecurityHelper.getTabUserName(state)); boolean canRead = SecurityHelper.getTabPerm(state, SecurityHelper.getTabUserName(state), TablePermission.READ); try { Scanner scan = conn.createScanner(tableName, conn.securityOperations().getUserAuthorizations(SecurityHelper.getTabUserName(state))); int seen = 0; Iterator<Entry<Key, Value>> iter = scan.iterator(); while (iter.hasNext()) { Entry<Key, Value> entry = iter.next(); Key k = entry.getKey(); seen++; if (!auths.contains(k.getColumnVisibilityData())) throw new AccumuloException( "Got data I should not be capable of seeing: " + k + " table " + tableName); } if (!canRead) throw new AccumuloException( "Was able to read when I shouldn't have had the perm with connection user " + conn.whoami() + " table " + tableName); for (Entry<String, Integer> entry : SecurityHelper.getAuthsMap(state).entrySet()) { if (auths.contains(entry.getKey().getBytes())) seen = seen - entry.getValue(); } if (seen != 0) throw new AccumuloException("Got mismatched amounts of data"); } catch (TableNotFoundException tnfe) { if (tableExists) throw new AccumuloException("Accumulo and test suite out of sync: table " + tableName, tnfe); return; } catch (AccumuloSecurityException ae) { if (ae.getErrorCode().equals(SecurityErrorCode.PERMISSION_DENIED)) { if (canRead) throw new AccumuloException( "Table read permission out of sync with Accumulo: table " + tableName, ae); else return; } throw new AccumuloException("Unexpected exception!", ae); } catch (RuntimeException re) { if (re.getCause() instanceof AccumuloSecurityException && ((AccumuloSecurityException) re.getCause()).getErrorCode() .equals(SecurityErrorCode.PERMISSION_DENIED)) { if (canRead) throw new AccumuloException( "Table read permission out of sync with Accumulo: table " + tableName, re.getCause()); else return; } throw new AccumuloException("Unexpected exception!", re); } break; case WRITE: String key = SecurityHelper.getLastKey(state) + "1"; Mutation m = new Mutation(new Text(key)); for (String s : SecurityHelper.getAuthsArray()) { m.put(new Text(), new Text(), new ColumnVisibility(s), new Value("value".getBytes())); } BatchWriter writer; try { writer = conn.createBatchWriter(tableName, 9000l, 0l, 1); } catch (TableNotFoundException tnfe) { if (tableExists) throw new AccumuloException("Table didn't exist when it should have: " + tableName); return; } boolean works = true; try { writer.addMutation(m); } catch (MutationsRejectedException mre) { throw new AccumuloException("Mutation exception!", mre); } if (works) for (String s : SecurityHelper.getAuthsArray()) SecurityHelper.increaseAuthMap(state, s, 1); break; case BULK_IMPORT: key = SecurityHelper.getLastKey(state) + "1"; SortedSet<Key> keys = new TreeSet<Key>(); for (String s : SecurityHelper.getAuthsArray()) { Key k = new Key(key, "", "", s); keys.add(k); } Path dir = new Path("/tmp", "bulk_" + UUID.randomUUID().toString()); Path fail = new Path(dir.toString() + "_fail"); FileSystem fs = SecurityHelper.getFs(state); FileSKVWriter f = FileOperations.getInstance().openWriter(dir + "/securityBulk." + RFile.EXTENSION, fs, fs.getConf(), AccumuloConfiguration.getDefaultConfiguration()); f.startDefaultLocalityGroup(); fs.mkdirs(fail); for (Key k : keys) f.append(k, new Value("Value".getBytes())); f.close(); try { conn.tableOperations().importDirectory(tableName, dir.toString(), fail.toString(), true); } catch (TableNotFoundException tnfe) { if (tableExists) throw new AccumuloException("Table didn't exist when it should have: " + tableName); return; } catch (AccumuloSecurityException ae) { if (ae.getErrorCode().equals(SecurityErrorCode.PERMISSION_DENIED)) { if (hasPerm) throw new AccumuloException("Bulk Import failed when it should have worked: " + tableName); return; } throw new AccumuloException("Unexpected exception!", ae); } for (String s : SecurityHelper.getAuthsArray()) SecurityHelper.increaseAuthMap(state, s, 1); fs.delete(dir, true); fs.delete(fail, true); if (!hasPerm) throw new AccumuloException( "Bulk Import succeeded when it should have failed: " + dir + " table " + tableName); break; case ALTER_TABLE: AlterTable.renameTable(conn, state, tableName, tableName + "plus", hasPerm, tableExists); break; case GRANT: props.setProperty("task", "grant"); props.setProperty("perm", "random"); props.setProperty("source", "table"); props.setProperty("target", "system"); AlterTablePerm.alter(state, props); break; case DROP_TABLE: props.setProperty("source", "table"); DropTable.dropTable(state, props); break; } }