List of usage examples for org.apache.hadoop.fs Path SEPARATOR
String SEPARATOR
To view the source code for org.apache.hadoop.fs Path SEPARATOR.
Click Source Link
From source file:notifiedEscapedCustomDfs.notifiedEscapedCustomDfsSink.java
License:Apache License
public notifiedEscapedCustomDfsSink(String path, String filename, OutputFormat o) { this.path = path; this.filename = filename; shouldSub = Event.containsTag(path) || Event.containsTag(filename); this.format = o; absolutePath = path;/*from ww w . j a v a 2s .co m*/ if (filename != null && filename.length() > 0) { if (!absolutePath.endsWith(Path.SEPARATOR)) { absolutePath += Path.SEPARATOR; } absolutePath += this.filename; } }
From source file:org.apache.accumulo.master.Master.java
License:Apache License
private void moveRootTabletToRootTable(IZooReaderWriter zoo) throws Exception { String dirZPath = ZooUtil.getRoot(getInstance()) + RootTable.ZROOT_TABLET_PATH; if (!zoo.exists(dirZPath)) { Path oldPath = fs.getFullPath(FileType.TABLE, "/" + MetadataTable.ID + "/root_tablet"); if (fs.exists(oldPath)) { String newPath = fs.choose(Optional.of(RootTable.ID), ServerConstants.getBaseUris()) + Constants.HDFS_TABLES_DIR + Path.SEPARATOR + RootTable.ID; fs.mkdirs(new Path(newPath)); if (!fs.rename(oldPath, new Path(newPath))) { throw new IOException("Failed to move root tablet from " + oldPath + " to " + newPath); }// ww w . j a va2 s .co m log.info("Upgrade renamed " + oldPath + " to " + newPath); } Path location = null; for (String basePath : ServerConstants.getTablesDirs()) { Path path = new Path(basePath + "/" + RootTable.ID + RootTable.ROOT_TABLET_LOCATION); if (fs.exists(path)) { if (location != null) { throw new IllegalStateException( "Root table at multiple locations " + location + " " + path); } location = path; } } if (location == null) throw new IllegalStateException("Failed to find root tablet"); log.info("Upgrade setting root table location in zookeeper " + location); zoo.putPersistentData(dirZPath, location.toString().getBytes(), NodeExistsPolicy.FAIL); } }
From source file:org.apache.accumulo.master.tableOps.ChooseDir.java
License:Apache License
@Override public Repo<Master> call(long tid, Master master) throws Exception { // Constants.DEFAULT_TABLET_LOCATION has a leading slash prepended to it so we don't need to add one here tableInfo.dir = master.getFileSystem().choose(Optional.of(tableInfo.tableId), ServerConstants.getBaseUris()) + Constants.HDFS_TABLES_DIR + Path.SEPARATOR + tableInfo.tableId + Constants.DEFAULT_TABLET_LOCATION; return new CreateDir(tableInfo); }
From source file:org.apache.accumulo.master.tableOps.create.ChooseDir.java
License:Apache License
@Override public Repo<Master> call(long tid, Master master) throws Exception { // Constants.DEFAULT_TABLET_LOCATION has a leading slash prepended to it so we don't need to add // one here//w ww. ja v a2s .c o m VolumeChooserEnvironment chooserEnv = new VolumeChooserEnvironmentImpl(tableInfo.getTableId(), null, master.getContext()); String baseDir = master.getFileSystem().choose(chooserEnv, ServerConstants.getBaseUris(master.getContext())) + Constants.HDFS_TABLES_DIR + Path.SEPARATOR + tableInfo.getTableId(); tableInfo.defaultTabletDir = baseDir + Constants.DEFAULT_TABLET_LOCATION; if (tableInfo.getInitialSplitSize() > 0) { createTableDirectoriesInfo(master, baseDir); } return new CreateDir(tableInfo); }
From source file:org.apache.accumulo.master.TabletGroupWatcher.java
License:Apache License
private void deleteTablets(MergeInfo info) throws AccumuloException { KeyExtent extent = info.getExtent(); String targetSystemTable = extent.isMeta() ? RootTable.NAME : MetadataTable.NAME; Master.log.debug("Deleting tablets for " + extent); char timeType = '\0'; KeyExtent followingTablet = null;//w ww.j a va 2 s. c o m if (extent.getEndRow() != null) { Key nextExtent = new Key(extent.getEndRow()).followingKey(PartialKey.ROW); followingTablet = getHighTablet( new KeyExtent(extent.getTableId(), nextExtent.getRow(), extent.getEndRow())); Master.log.debug("Found following tablet " + followingTablet); } try { Connector conn = this.master.getConnector(); Text start = extent.getPrevEndRow(); if (start == null) { start = new Text(); } Master.log.debug("Making file deletion entries for " + extent); Range deleteRange = new Range(KeyExtent.getMetadataEntry(extent.getTableId(), start), false, KeyExtent.getMetadataEntry(extent.getTableId(), extent.getEndRow()), true); Scanner scanner = conn.createScanner(targetSystemTable, Authorizations.EMPTY); scanner.setRange(deleteRange); TabletsSection.ServerColumnFamily.DIRECTORY_COLUMN.fetch(scanner); TabletsSection.ServerColumnFamily.TIME_COLUMN.fetch(scanner); scanner.fetchColumnFamily(DataFileColumnFamily.NAME); scanner.fetchColumnFamily(TabletsSection.CurrentLocationColumnFamily.NAME); Set<FileRef> datafiles = new TreeSet<>(); for (Entry<Key, Value> entry : scanner) { Key key = entry.getKey(); if (key.compareColumnFamily(DataFileColumnFamily.NAME) == 0) { datafiles.add(new FileRef(this.master.fs, key)); if (datafiles.size() > 1000) { MetadataTableUtil.addDeleteEntries(extent, datafiles, master); datafiles.clear(); } } else if (TabletsSection.ServerColumnFamily.TIME_COLUMN.hasColumns(key)) { timeType = entry.getValue().toString().charAt(0); } else if (key.compareColumnFamily(TabletsSection.CurrentLocationColumnFamily.NAME) == 0) { throw new IllegalStateException("Tablet " + key.getRow() + " is assigned during a merge!"); } else if (TabletsSection.ServerColumnFamily.DIRECTORY_COLUMN.hasColumns(key)) { // ACCUMULO-2974 Need to include the TableID when converting a relative path to an absolute path. // The value has the leading path separator already included so it doesn't need it included. String path = entry.getValue().toString(); if (path.contains(":")) { datafiles.add(new FileRef(path)); } else { datafiles.add(new FileRef(path, this.master.fs.getFullPath(FileType.TABLE, Path.SEPARATOR + extent.getTableId() + path))); } if (datafiles.size() > 1000) { MetadataTableUtil.addDeleteEntries(extent, datafiles, master); datafiles.clear(); } } } MetadataTableUtil.addDeleteEntries(extent, datafiles, master); BatchWriter bw = conn.createBatchWriter(targetSystemTable, new BatchWriterConfig()); try { deleteTablets(info, deleteRange, bw, conn); } finally { bw.close(); } if (followingTablet != null) { Master.log.debug("Updating prevRow of " + followingTablet + " to " + extent.getPrevEndRow()); bw = conn.createBatchWriter(targetSystemTable, new BatchWriterConfig()); try { Mutation m = new Mutation(followingTablet.getMetadataEntry()); TabletsSection.TabletColumnFamily.PREV_ROW_COLUMN.put(m, KeyExtent.encodePrevEndRow(extent.getPrevEndRow())); ChoppedColumnFamily.CHOPPED_COLUMN.putDelete(m); bw.addMutation(m); bw.flush(); } finally { bw.close(); } } else { // Recreate the default tablet to hold the end of the table Master.log.debug("Recreating the last tablet to point to " + extent.getPrevEndRow()); String tdir = master.getFileSystem().choose(Optional.of(extent.getTableId()), ServerConstants.getBaseUris()) + Constants.HDFS_TABLES_DIR + Path.SEPARATOR + extent.getTableId() + Constants.DEFAULT_TABLET_LOCATION; MetadataTableUtil.addTablet(new KeyExtent(extent.getTableId(), null, extent.getPrevEndRow()), tdir, master, timeType, this.master.masterLock); } } catch (RuntimeException | IOException | TableNotFoundException | AccumuloSecurityException ex) { throw new AccumuloException(ex); } }
From source file:org.apache.accumulo.server.fs.VolumeUtil.java
License:Apache License
private static String decommisionedTabletDir(AccumuloServerContext context, ZooLock zooLock, VolumeManager vm, KeyExtent extent, String metaDir) throws IOException { Path dir = new Path(metaDir); if (isActiveVolume(dir)) return metaDir; if (!dir.getParent().getParent().getName().equals(ServerConstants.TABLE_DIR)) { throw new IllegalArgumentException("Unexpected table dir " + dir); }// ww w.j a v a2s. c o m Path newDir = new Path(vm.choose(Optional.of(extent.getTableId()), ServerConstants.getBaseUris()) + Path.SEPARATOR + ServerConstants.TABLE_DIR + Path.SEPARATOR + dir.getParent().getName() + Path.SEPARATOR + dir.getName()); log.info("Updating directory for " + extent + " from " + dir + " to " + newDir); if (extent.isRootTablet()) { // the root tablet is special case, its files need to be copied if its dir is changed // this code needs to be idempotent FileSystem fs1 = vm.getVolumeByPath(dir).getFileSystem(); FileSystem fs2 = vm.getVolumeByPath(newDir).getFileSystem(); if (!same(fs1, dir, fs2, newDir)) { if (fs2.exists(newDir)) { Path newDirBackup = getBackupName(fs2, newDir); // never delete anything because were dealing with the root tablet // one reason this dir may exist is because this method failed previously log.info("renaming " + newDir + " to " + newDirBackup); if (!fs2.rename(newDir, newDirBackup)) { throw new IOException("Failed to rename " + newDir + " to " + newDirBackup); } } // do a lot of logging since this is the root tablet log.info("copying " + dir + " to " + newDir); if (!FileUtil.copy(fs1, dir, fs2, newDir, false, CachedConfiguration.getInstance())) { throw new IOException("Failed to copy " + dir + " to " + newDir); } // only set the new location in zookeeper after a successful copy log.info("setting root tablet location to " + newDir); MetadataTableUtil.setRootTabletDir(newDir.toString()); // rename the old dir to avoid confusion when someone looks at filesystem... its ok if we fail here and this does not happen because the location in // zookeeper is the authority Path dirBackup = getBackupName(fs1, dir); log.info("renaming " + dir + " to " + dirBackup); fs1.rename(dir, dirBackup); } else { log.info("setting root tablet location to " + newDir); MetadataTableUtil.setRootTabletDir(newDir.toString()); } return newDir.toString(); } else { MetadataTableUtil.updateTabletDir(extent, newDir.toString(), context, zooLock); return newDir.toString(); } }
From source file:org.apache.accumulo.server.init.Initialize.java
License:Apache License
private boolean initialize(Opts opts, String instanceNamePath, VolumeManager fs, String rootUser) { UUID uuid = UUID.randomUUID(); // the actual disk locations of the root table and tablets String[] configuredVolumes = VolumeConfiguration.getVolumeUris(SiteConfiguration.getInstance()); final String rootTabletDir = new Path( fs.choose(Optional.<String>empty(), configuredVolumes) + Path.SEPARATOR + ServerConstants.TABLE_DIR + Path.SEPARATOR + RootTable.ID + RootTable.ROOT_TABLET_LOCATION).toString(); try {//from w ww . j a v a2 s. co m initZooKeeper(opts, uuid.toString(), instanceNamePath, rootTabletDir); } catch (Exception e) { log.error("FATAL: Failed to initialize zookeeper", e); return false; } try { initFileSystem(opts, fs, uuid, rootTabletDir); } catch (Exception e) { log.error("FATAL Failed to initialize filesystem", e); if (SiteConfiguration.getInstance().get(Property.INSTANCE_VOLUMES).trim().equals("")) { Configuration fsConf = CachedConfiguration.getInstance(); final String defaultFsUri = "file:///"; String fsDefaultName = fsConf.get("fs.default.name", defaultFsUri), fsDefaultFS = fsConf.get("fs.defaultFS", defaultFsUri); // Try to determine when we couldn't find an appropriate core-site.xml on the classpath if (defaultFsUri.equals(fsDefaultName) && defaultFsUri.equals(fsDefaultFS)) { log.error("FATAL: Default filesystem value ('fs.defaultFS' or 'fs.default.name') of '" + defaultFsUri + "' was found in the Hadoop configuration"); log.error( "FATAL: Please ensure that the Hadoop core-site.xml is on the classpath using 'general.classpaths' in accumulo-site.xml"); } } return false; } final ServerConfigurationFactory confFactory = new ServerConfigurationFactory( HdfsZooInstance.getInstance()); // When we're using Kerberos authentication, we need valid credentials to perform initialization. If the user provided some, use them. // If they did not, fall back to the credentials present in accumulo-site.xml that the servers will use themselves. try { final SiteConfiguration siteConf = confFactory.getSiteConfiguration(); if (siteConf.getBoolean(Property.INSTANCE_RPC_SASL_ENABLED)) { final UserGroupInformation ugi = UserGroupInformation.getCurrentUser(); // We don't have any valid creds to talk to HDFS if (!ugi.hasKerberosCredentials()) { final String accumuloKeytab = siteConf.get(Property.GENERAL_KERBEROS_KEYTAB), accumuloPrincipal = siteConf.get(Property.GENERAL_KERBEROS_PRINCIPAL); // Fail if the site configuration doesn't contain appropriate credentials to login as servers if (StringUtils.isBlank(accumuloKeytab) || StringUtils.isBlank(accumuloPrincipal)) { log.error( "FATAL: No Kerberos credentials provided, and Accumulo is not properly configured for server login"); return false; } log.info("Logging in as " + accumuloPrincipal + " with " + accumuloKeytab); // Login using the keytab as the 'accumulo' user UserGroupInformation.loginUserFromKeytab(accumuloPrincipal, accumuloKeytab); } } } catch (IOException e) { log.error("FATAL: Failed to get the Kerberos user", e); return false; } try { AccumuloServerContext context = new AccumuloServerContext(confFactory); initSecurity(context, opts, uuid.toString(), rootUser); } catch (Exception e) { log.error("FATAL: Failed to initialize security", e); return false; } return true; }
From source file:org.apache.accumulo.server.init.Initialize.java
License:Apache License
private void initFileSystem(Opts opts, VolumeManager fs, UUID uuid, String rootTabletDir) throws IOException { initDirs(fs, uuid, VolumeConfiguration.getVolumeUris(SiteConfiguration.getInstance()), false); // initialize initial system tables config in zookeeper initSystemTablesConfig();/*from ww w. j a va2s. c o m*/ String tableMetadataTabletDir = fs.choose(Optional.<String>empty(), ServerConstants.getBaseUris()) + Constants.HDFS_TABLES_DIR + Path.SEPARATOR + MetadataTable.ID + TABLE_TABLETS_TABLET_DIR; String replicationTableDefaultTabletDir = fs.choose(Optional.<String>empty(), ServerConstants.getBaseUris()) + Constants.HDFS_TABLES_DIR + Path.SEPARATOR + ReplicationTable.ID + Constants.DEFAULT_TABLET_LOCATION; String defaultMetadataTabletDir = fs.choose(Optional.<String>empty(), ServerConstants.getBaseUris()) + Constants.HDFS_TABLES_DIR + Path.SEPARATOR + MetadataTable.ID + Constants.DEFAULT_TABLET_LOCATION; // create table and default tablets directories createDirectories(fs, rootTabletDir, tableMetadataTabletDir, defaultMetadataTabletDir, replicationTableDefaultTabletDir); String ext = FileOperations.getNewFileExtension(AccumuloConfiguration.getDefaultConfiguration()); // populate the metadata tables tablet with info about the replication table's one initial tablet String metadataFileName = tableMetadataTabletDir + Path.SEPARATOR + "0_1." + ext; Tablet replicationTablet = new Tablet(ReplicationTable.ID, replicationTableDefaultTabletDir, null, null); createMetadataFile(fs, metadataFileName, replicationTablet); // populate the root tablet with info about the metadata table's two initial tablets String rootTabletFileName = rootTabletDir + Path.SEPARATOR + "00000_00000." + ext; Text splitPoint = TabletsSection.getRange().getEndKey().getRow(); Tablet tablesTablet = new Tablet(MetadataTable.ID, tableMetadataTabletDir, null, splitPoint, metadataFileName); Tablet defaultTablet = new Tablet(MetadataTable.ID, defaultMetadataTabletDir, splitPoint, null); createMetadataFile(fs, rootTabletFileName, tablesTablet, defaultTablet); }
From source file:org.apache.accumulo.server.util.FileUtil.java
License:Apache License
private static Path createTmpDir(AccumuloConfiguration acuConf, VolumeManager fs) throws IOException { String accumuloDir = fs.choose(Optional.<String>empty(), ServerConstants.getBaseUris()); Path result = null;/*from w w w.j ava2 s .co m*/ while (result == null) { result = new Path(accumuloDir + Path.SEPARATOR + "tmp/idxReduce_" + String.format("%09d", new Random().nextInt(Integer.MAX_VALUE))); try { fs.getFileStatus(result); result = null; continue; } catch (FileNotFoundException fne) { // found an unused temp directory } fs.mkdirs(result); // try to reserve the tmp dir // In some versions of hadoop, two clients concurrently trying to create the same directory might both return true // Creating a file is not subject to this, so create a special file to make sure we solely will use this directory if (!fs.createNewFile(new Path(result, "__reserve"))) result = null; } return result; }
From source file:org.apache.accumulo.server.util.MetadataTableUtil.java
License:Apache License
public static void cloneTable(ClientContext context, String srcTableId, String tableId, VolumeManager volumeManager) throws Exception { Connector conn = context.getConnector(); try (BatchWriter bw = conn.createBatchWriter(MetadataTable.NAME, new BatchWriterConfig())) { while (true) { try { initializeClone(MetadataTable.NAME, srcTableId, tableId, conn, bw); // the following loop looks changes in the file that occurred during the copy.. if files were dereferenced then they could have been GCed while (true) { int rewrites = checkClone(MetadataTable.NAME, srcTableId, tableId, conn, bw); if (rewrites == 0) break; }//from w ww. ja v a 2 s.c om bw.flush(); break; } catch (TabletIterator.TabletDeletedException tde) { // tablets were merged in the src table bw.flush(); // delete what we have cloned and try again deleteTable(tableId, false, context, null); log.debug("Tablets merged in table " + srcTableId + " while attempting to clone, trying again"); sleepUninterruptibly(100, TimeUnit.MILLISECONDS); } } // delete the clone markers and create directory entries Scanner mscanner = conn.createScanner(MetadataTable.NAME, Authorizations.EMPTY); mscanner.setRange(new KeyExtent(tableId, null, null).toMetadataRange()); mscanner.fetchColumnFamily(ClonedColumnFamily.NAME); int dirCount = 0; for (Entry<Key, Value> entry : mscanner) { Key k = entry.getKey(); Mutation m = new Mutation(k.getRow()); m.putDelete(k.getColumnFamily(), k.getColumnQualifier()); String dir = volumeManager.choose(Optional.of(tableId), ServerConstants.getBaseUris()) + Constants.HDFS_TABLES_DIR + Path.SEPARATOR + tableId + Path.SEPARATOR + new String( FastFormat.toZeroPaddedString(dirCount++, 8, 16, Constants.CLONE_PREFIX_BYTES)); TabletsSection.ServerColumnFamily.DIRECTORY_COLUMN.put(m, new Value(dir.getBytes(UTF_8))); bw.addMutation(m); } } }