Example usage for org.apache.hadoop.fs FileSystem getConf

List of usage examples for org.apache.hadoop.fs FileSystem getConf

  1. HOME
  2. Java
  3. org.apache.hadoop
  4. org.apache.hadoop.fs.*
  5. FileSystem
  6. getConf

Introduction

In this page you can find the example usage for org.apache.hadoop.fs FileSystem getConf.

Prototype

@Override
    public Configuration getConf()

Source Link

Usage

From source file:org.apache.accumulo.server.util.FileUtil.java

License:Apache License

public static Collection<String> reduceFiles(AccumuloConfiguration acuConf, Configuration conf,
        VolumeManager fs, Text prevEndRow, Text endRow, Collection<String> mapFiles, int maxFiles, Path tmpDir,
        int pass) throws IOException {
    ArrayList<String> paths = new ArrayList<>(mapFiles);

    if (paths.size() <= maxFiles)
        return paths;

    String newDir = String.format("%s/pass_%04d", tmpDir, pass);

    int start = 0;

    ArrayList<String> outFiles = new ArrayList<>();

    int count = 0;

    while (start < paths.size()) {
        int end = Math.min(maxFiles + start, paths.size());
        List<String> inFiles = paths.subList(start, end);

        start = end;/*from  ww w.ja  v  a 2  s .c o m*/

        String newMapFile = String.format("%s/%04d.%s", newDir, count++, RFile.EXTENSION);

        outFiles.add(newMapFile);
        FileSystem ns = fs.getVolumeByPath(new Path(newMapFile)).getFileSystem();
        FileSKVWriter writer = new RFileOperations().newWriterBuilder()
                .forFile(newMapFile.toString(), ns, ns.getConf()).withTableConfiguration(acuConf).build();
        writer.startDefaultLocalityGroup();
        List<SortedKeyValueIterator<Key, Value>> iters = new ArrayList<>(inFiles.size());

        FileSKVIterator reader = null;
        try {
            for (String s : inFiles) {
                ns = fs.getVolumeByPath(new Path(s)).getFileSystem();
                reader = FileOperations.getInstance().newIndexReaderBuilder().forFile(s, ns, ns.getConf())
                        .withTableConfiguration(acuConf).build();
                iters.add(reader);
            }

            MultiIterator mmfi = new MultiIterator(iters, true);

            while (mmfi.hasTop()) {
                Key key = mmfi.getTopKey();

                boolean gtPrevEndRow = prevEndRow == null || key.compareRow(prevEndRow) > 0;
                boolean lteEndRow = endRow == null || key.compareRow(endRow) <= 0;

                if (gtPrevEndRow && lteEndRow)
                    writer.append(key, new Value(new byte[0]));

                if (!lteEndRow)
                    break;

                mmfi.next();
            }
        } finally {
            try {
                if (reader != null)
                    reader.close();
            } catch (IOException e) {
                log.error("{}", e.getMessage(), e);
            }

            for (SortedKeyValueIterator<Key, Value> r : iters)
                try {
                    if (r != null)
                        ((FileSKVIterator) r).close();
                } catch (IOException e) {
                    // continue closing
                    log.error("{}", e.getMessage(), e);
                }

            try {
                writer.close();
            } catch (IOException e) {
                log.error("{}", e.getMessage(), e);
                throw e;
            }
        }
    }

    return reduceFiles(acuConf, conf, fs, prevEndRow, endRow, outFiles, maxFiles, tmpDir, pass + 1);
}

From source file:org.apache.accumulo.server.util.FileUtil.java

License:Apache License

private static long countIndexEntries(AccumuloConfiguration acuConf, Text prevEndRow, Text endRow,
        Collection<String> mapFiles, boolean useIndex, Configuration conf, VolumeManager fs,
        ArrayList<FileSKVIterator> readers) throws IOException {

    long numKeys = 0;

    // count the total number of index entries
    for (String ref : mapFiles) {
        FileSKVIterator reader = null;//  w  w  w. ja va  2  s.c o  m
        Path path = new Path(ref);
        FileSystem ns = fs.getVolumeByPath(path).getFileSystem();
        try {
            if (useIndex)
                reader = FileOperations.getInstance().newIndexReaderBuilder()
                        .forFile(path.toString(), ns, ns.getConf()).withTableConfiguration(acuConf).build();
            else
                reader = FileOperations.getInstance().newScanReaderBuilder()
                        .forFile(path.toString(), ns, ns.getConf()).withTableConfiguration(acuConf)
                        .overRange(new Range(prevEndRow, false, null, true), LocalityGroupUtil.EMPTY_CF_SET,
                                false)
                        .build();

            while (reader.hasTop()) {
                Key key = reader.getTopKey();
                if (endRow != null && key.compareRow(endRow) > 0)
                    break;
                else if (prevEndRow == null || key.compareRow(prevEndRow) > 0)
                    numKeys++;

                reader.next();
            }
        } finally {
            try {
                if (reader != null)
                    reader.close();
            } catch (IOException e) {
                log.error("{}", e.getMessage(), e);
            }
        }

        if (useIndex)
            readers.add(FileOperations.getInstance().newIndexReaderBuilder()
                    .forFile(path.toString(), ns, ns.getConf()).withTableConfiguration(acuConf).build());
        else
            readers.add(FileOperations.getInstance().newScanReaderBuilder()
                    .forFile(path.toString(), ns, ns.getConf()).withTableConfiguration(acuConf)
                    .overRange(new Range(prevEndRow, false, null, true), LocalityGroupUtil.EMPTY_CF_SET, false)
                    .build());

    }
    return numKeys;
}

From source file:org.apache.accumulo.server.util.FileUtil.java

License:Apache License

public static Map<FileRef, FileInfo> tryToGetFirstAndLastRows(VolumeManager fs, AccumuloConfiguration acuConf,
        Set<FileRef> mapfiles) {

    HashMap<FileRef, FileInfo> mapFilesInfo = new HashMap<>();

    long t1 = System.currentTimeMillis();

    for (FileRef mapfile : mapfiles) {

        FileSKVIterator reader = null;/*from ww w  . ja  va 2 s .com*/
        FileSystem ns = fs.getVolumeByPath(mapfile.path()).getFileSystem();
        try {
            reader = FileOperations.getInstance().newReaderBuilder()
                    .forFile(mapfile.toString(), ns, ns.getConf()).withTableConfiguration(acuConf).build();

            Key firstKey = reader.getFirstKey();
            if (firstKey != null) {
                mapFilesInfo.put(mapfile, new FileInfo(firstKey, reader.getLastKey()));
            }

        } catch (IOException ioe) {
            log.warn("Failed to read map file to determine first and last key : " + mapfile, ioe);
        } finally {
            if (reader != null) {
                try {
                    reader.close();
                } catch (IOException ioe) {
                    log.warn("failed to close " + mapfile, ioe);
                }
            }
        }

    }

    long t2 = System.currentTimeMillis();

    log.debug(String.format("Found first and last keys for %d map files in %6.2f secs", mapfiles.size(),
            (t2 - t1) / 1000.0));

    return mapFilesInfo;
}

From source file:org.apache.accumulo.server.util.FileUtil.java

License:Apache License

public static WritableComparable<Key> findLastKey(VolumeManager fs, AccumuloConfiguration acuConf,
        Collection<FileRef> mapFiles) throws IOException {
    Key lastKey = null;/*from  ww  w . j ava2s  .c  o m*/

    for (FileRef ref : mapFiles) {
        Path path = ref.path();
        FileSystem ns = fs.getVolumeByPath(path).getFileSystem();
        FileSKVIterator reader = FileOperations.getInstance().newReaderBuilder()
                .forFile(path.toString(), ns, ns.getConf()).withTableConfiguration(acuConf).seekToBeginning()
                .build();

        try {
            if (!reader.hasTop())
                // file is empty, so there is no last key
                continue;

            Key key = reader.getLastKey();

            if (lastKey == null || key.compareTo(lastKey) > 0)
                lastKey = key;
        } finally {
            try {
                if (reader != null)
                    reader.close();
            } catch (IOException e) {
                log.error("{}", e.getMessage(), e);
            }
        }
    }

    return lastKey;

}

From source file:org.apache.accumulo.server.util.FileUtil.java

License:Apache License

public static Map<KeyExtent, Long> estimateSizes(AccumuloConfiguration acuConf, Path mapFile, long fileSize,
        List<KeyExtent> extents, Configuration conf, VolumeManager fs) throws IOException {

    long totalIndexEntries = 0;
    Map<KeyExtent, MLong> counts = new TreeMap<>();
    for (KeyExtent keyExtent : extents)
        counts.put(keyExtent, new MLong(0));

    Text row = new Text();
    FileSystem ns = fs.getVolumeByPath(mapFile).getFileSystem();
    FileSKVIterator index = FileOperations.getInstance().newIndexReaderBuilder()
            .forFile(mapFile.toString(), ns, ns.getConf()).withTableConfiguration(acuConf).build();

    try {// w  ww. j a v  a  2 s  .  c  om
        while (index.hasTop()) {
            Key key = index.getTopKey();
            totalIndexEntries++;
            key.getRow(row);

            for (Entry<KeyExtent, MLong> entry : counts.entrySet())
                if (entry.getKey().contains(row))
                    entry.getValue().l++;

            index.next();
        }
    } finally {
        try {
            if (index != null)
                index.close();
        } catch (IOException e) {
            // continue with next file
            log.error("{}", e.getMessage(), e);
        }
    }

    Map<KeyExtent, Long> results = new TreeMap<>();
    for (KeyExtent keyExtent : extents) {
        double numEntries = counts.get(keyExtent).l;
        if (numEntries == 0)
            numEntries = 1;
        long estSize = (long) ((numEntries / totalIndexEntries) * fileSize);
        results.put(keyExtent, estSize);
    }
    return results;
}

From source file:org.apache.accumulo.server.util.Initialize.java

License:Apache License

private static void initFileSystem(Opts opts, VolumeManager fs, UUID uuid) throws IOException {
    FileStatus fstat;// www  .j  ava  2s . c  o m

    // the actual disk locations of the root table and tablets
    final Path rootTablet = new Path(ServerConstants.getRootTabletDir());

    // the actual disk locations of the metadata table and tablets
    final Path[] metadataTableDirs = paths(ServerConstants.getMetadataTableDirs());

    String tableMetadataTabletDir = fs
            .choose(ServerConstants.prefix(ServerConstants.getMetadataTableDirs(), TABLE_TABLETS_TABLET_DIR));
    String defaultMetadataTabletDir = fs.choose(
            ServerConstants.prefix(ServerConstants.getMetadataTableDirs(), Constants.DEFAULT_TABLET_LOCATION));

    fs.mkdirs(new Path(ServerConstants.getDataVersionLocation(), "" + ServerConstants.DATA_VERSION));

    // create an instance id
    fs.mkdirs(ServerConstants.getInstanceIdLocation());
    fs.createNewFile(new Path(ServerConstants.getInstanceIdLocation(), uuid.toString()));

    // initialize initial metadata config in zookeeper
    initMetadataConfig();

    // create metadata table
    for (Path mtd : metadataTableDirs) {
        try {
            fstat = fs.getFileStatus(mtd);
            if (!fstat.isDir()) {
                log.fatal("location " + mtd.toString() + " exists but is not a directory");
                return;
            }
        } catch (FileNotFoundException fnfe) {
            if (!fs.mkdirs(mtd)) {
                log.fatal("unable to create directory " + mtd.toString());
                return;
            }
        }
    }

    // create root table and tablet
    try {
        fstat = fs.getFileStatus(rootTablet);
        if (!fstat.isDir()) {
            log.fatal("location " + rootTablet.toString() + " exists but is not a directory");
            return;
        }
    } catch (FileNotFoundException fnfe) {
        if (!fs.mkdirs(rootTablet)) {
            log.fatal("unable to create directory " + rootTablet.toString());
            return;
        }
    }

    // populate the root tablet with info about the default tablet
    // the root tablet contains the key extent and locations of all the
    // metadata tablets
    String initRootTabFile = rootTablet + "/00000_00000."
            + FileOperations.getNewFileExtension(AccumuloConfiguration.getDefaultConfiguration());
    FileSystem ns = fs.getFileSystemByPath(new Path(initRootTabFile));
    FileSKVWriter mfw = FileOperations.getInstance().openWriter(initRootTabFile, ns, ns.getConf(),
            AccumuloConfiguration.getDefaultConfiguration());
    mfw.startDefaultLocalityGroup();

    Text tableExtent = new Text(KeyExtent.getMetadataEntry(new Text(MetadataTable.ID),
            MetadataSchema.TabletsSection.getRange().getEndKey().getRow()));

    // table tablet's directory
    Key tableDirKey = new Key(tableExtent, TabletsSection.ServerColumnFamily.DIRECTORY_COLUMN.getColumnFamily(),
            TabletsSection.ServerColumnFamily.DIRECTORY_COLUMN.getColumnQualifier(), 0);
    mfw.append(tableDirKey, new Value(tableMetadataTabletDir.getBytes()));

    // table tablet time
    Key tableTimeKey = new Key(tableExtent, TabletsSection.ServerColumnFamily.TIME_COLUMN.getColumnFamily(),
            TabletsSection.ServerColumnFamily.TIME_COLUMN.getColumnQualifier(), 0);
    mfw.append(tableTimeKey, new Value((TabletTime.LOGICAL_TIME_ID + "0").getBytes()));

    // table tablet's prevrow
    Key tablePrevRowKey = new Key(tableExtent,
            TabletsSection.TabletColumnFamily.PREV_ROW_COLUMN.getColumnFamily(),
            TabletsSection.TabletColumnFamily.PREV_ROW_COLUMN.getColumnQualifier(), 0);
    mfw.append(tablePrevRowKey, KeyExtent.encodePrevEndRow(null));

    // ----------] default tablet info
    Text defaultExtent = new Text(KeyExtent.getMetadataEntry(new Text(MetadataTable.ID), null));

    // default's directory
    Key defaultDirKey = new Key(defaultExtent,
            TabletsSection.ServerColumnFamily.DIRECTORY_COLUMN.getColumnFamily(),
            TabletsSection.ServerColumnFamily.DIRECTORY_COLUMN.getColumnQualifier(), 0);
    mfw.append(defaultDirKey, new Value(defaultMetadataTabletDir.getBytes()));

    // default's time
    Key defaultTimeKey = new Key(defaultExtent, TabletsSection.ServerColumnFamily.TIME_COLUMN.getColumnFamily(),
            TabletsSection.ServerColumnFamily.TIME_COLUMN.getColumnQualifier(), 0);
    mfw.append(defaultTimeKey, new Value((TabletTime.LOGICAL_TIME_ID + "0").getBytes()));

    // default's prevrow
    Key defaultPrevRowKey = new Key(defaultExtent,
            TabletsSection.TabletColumnFamily.PREV_ROW_COLUMN.getColumnFamily(),
            TabletsSection.TabletColumnFamily.PREV_ROW_COLUMN.getColumnQualifier(), 0);
    mfw.append(defaultPrevRowKey,
            KeyExtent.encodePrevEndRow(MetadataSchema.TabletsSection.getRange().getEndKey().getRow()));

    mfw.close();

    // create table and default tablets directories
    for (String s : Arrays.asList(tableMetadataTabletDir, defaultMetadataTabletDir)) {
        Path dir = new Path(s);
        try {
            fstat = fs.getFileStatus(dir);
            if (!fstat.isDir()) {
                log.fatal("location " + dir.toString() + " exists but is not a directory");
                return;
            }
        } catch (FileNotFoundException fnfe) {
            try {
                fstat = fs.getFileStatus(dir);
                if (!fstat.isDir()) {
                    log.fatal("location " + dir.toString() + " exists but is not a directory");
                    return;
                }
            } catch (FileNotFoundException fnfe2) {
                // create table info dir
                if (!fs.mkdirs(dir)) {
                    log.fatal("unable to create directory " + dir.toString());
                    return;
                }
            }

            // create default dir
            if (!fs.mkdirs(dir)) {
                log.fatal("unable to create directory " + dir.toString());
                return;
            }
        }
    }
}

From source file:org.apache.accumulo.server.util.OfflineMetadataScanner.java

License:Apache License

private List<SortedKeyValueIterator<Key, Value>> openMapFiles(Collection<String> files, VolumeManager fs,
        AccumuloConfiguration conf) throws IOException {
    List<SortedKeyValueIterator<Key, Value>> readers = new ArrayList<SortedKeyValueIterator<Key, Value>>();
    for (String file : files) {
        FileSystem ns = fs.getFileSystemByPath(new Path(file));
        FileSKVIterator reader = FileOperations.getInstance().openReader(file, true, ns, ns.getConf(), conf);
        readers.add(reader);/*from   w w  w . j a  v a 2s . c o m*/
    }
    return readers;
}

From source file:org.apache.accumulo.start.classloader.vfs.AccumuloVFSClassLoader.java

License:Apache License

public static ClassLoader getClassLoader() throws IOException {
    ReloadingClassLoader localLoader = loader;
    while (null == localLoader) {
        synchronized (lock) {
            if (null == loader) {

                FileSystemManager vfs = generateVfs();

                // Set up the 2nd tier class loader
                if (null == parent) {
                    parent = AccumuloClassLoader.getClassLoader();
                }/*  w  w w  . j  av  a2 s .  c  o m*/

                FileObject[] vfsCP = resolve(vfs,
                        AccumuloClassLoader.getAccumuloString(VFS_CLASSLOADER_SYSTEM_CLASSPATH_PROPERTY, ""));

                if (vfsCP.length == 0) {
                    localLoader = createDynamicClassloader(parent);
                    loader = localLoader;
                    return localLoader.getClassLoader();
                }

                // Create the Accumulo Context ClassLoader using the DEFAULT_CONTEXT
                localLoader = createDynamicClassloader(new VFSClassLoader(vfsCP, vfs, parent));
                loader = localLoader;

                // An HDFS FileSystem and Configuration object were created for each unique HDFS namespace in the call to resolve above.
                // The HDFS Client did us a favor and cached these objects so that the next time someone calls FileSystem.get(uri), they
                // get the cached object. However, these objects were created not with the system VFS classloader, but the classloader above
                // it. We need to override the classloader on the Configuration objects. Ran into an issue were log recovery was being attempted
                // and SequenceFile$Reader was trying to instantiate the key class via WritableName.getClass(String, Configuration)
                for (FileObject fo : vfsCP) {
                    if (fo instanceof HdfsFileObject) {
                        String uri = fo.getName().getRootURI();
                        Configuration c = new Configuration(true);
                        c.set(FileSystem.FS_DEFAULT_NAME_KEY, uri);
                        FileSystem fs = FileSystem.get(c);
                        fs.getConf().setClassLoader(loader.getClassLoader());
                    }
                }

            }
        }
    }

    return localLoader.getClassLoader();
}

From source file:org.apache.accumulo.test.BulkImportMonitoringIT.java

License:Apache License

@Test
public void test() throws Exception {
    getCluster().getClusterControl().start(ServerType.MONITOR);
    final Connector c = getConnector();
    final String tableName = getUniqueNames(1)[0];
    c.tableOperations().create(tableName);
    c.tableOperations().setProperty(tableName, Property.TABLE_MAJC_RATIO.getKey(), "1");
    // splits to slow down bulk import
    SortedSet<Text> splits = new TreeSet<>();
    for (int i = 1; i < 0xf; i++) {
        splits.add(new Text(Integer.toHexString(i)));
    }//from w ww  .  jav a2  s . c om
    c.tableOperations().addSplits(tableName, splits);

    MasterMonitorInfo stats = getCluster().getMasterMonitorInfo();
    assertEquals(1, stats.tServerInfo.size());
    assertEquals(0, stats.bulkImports.size());
    assertEquals(0, stats.tServerInfo.get(0).bulkImports.size());

    log.info("Creating lots of bulk import files");
    final FileSystem fs = getCluster().getFileSystem();
    final Path basePath = getCluster().getTemporaryPath();
    CachedConfiguration.setInstance(fs.getConf());

    final Path base = new Path(basePath, "testBulkLoad" + tableName);
    fs.delete(base, true);
    fs.mkdirs(base);

    ExecutorService es = Executors.newFixedThreadPool(5);
    List<Future<Pair<String, String>>> futures = new ArrayList<>();
    for (int i = 0; i < 10; i++) {
        final int which = i;
        futures.add(es.submit(new Callable<Pair<String, String>>() {
            @Override
            public Pair<String, String> call() throws Exception {
                Path bulkFailures = new Path(base, "failures" + which);
                Path files = new Path(base, "files" + which);
                fs.mkdirs(bulkFailures);
                fs.mkdirs(files);
                for (int i = 0; i < 10; i++) {
                    FileSKVWriter writer = FileOperations.getInstance().newWriterBuilder()
                            .forFile(files.toString() + "/bulk_" + i + "." + RFile.EXTENSION, fs, fs.getConf())
                            .withTableConfiguration(AccumuloConfiguration.getDefaultConfiguration()).build();
                    writer.startDefaultLocalityGroup();
                    for (int j = 0x100; j < 0xfff; j += 3) {
                        writer.append(new Key(Integer.toHexString(j)), new Value(new byte[0]));
                    }
                    writer.close();
                }
                return new Pair<>(files.toString(), bulkFailures.toString());
            }
        }));
    }
    List<Pair<String, String>> dirs = new ArrayList<>();
    for (Future<Pair<String, String>> f : futures) {
        dirs.add(f.get());
    }
    log.info("Importing");
    long now = System.currentTimeMillis();
    List<Future<Object>> errs = new ArrayList<>();
    for (Pair<String, String> entry : dirs) {
        final String dir = entry.getFirst();
        final String err = entry.getSecond();
        errs.add(es.submit(new Callable<Object>() {
            @Override
            public Object call() throws Exception {
                c.tableOperations().importDirectory(tableName, dir, err, false);
                return null;
            }
        }));
    }
    es.shutdown();
    while (!es.isTerminated() && stats.bulkImports.size() + stats.tServerInfo.get(0).bulkImports.size() == 0) {
        es.awaitTermination(10, TimeUnit.MILLISECONDS);
        stats = getCluster().getMasterMonitorInfo();
    }
    log.info(stats.bulkImports.toString());
    assertTrue(stats.bulkImports.size() > 0);
    // look for exception
    for (Future<Object> err : errs) {
        err.get();
    }
    es.awaitTermination(2, TimeUnit.MINUTES);
    assertTrue(es.isTerminated());
    log.info(String.format("Completed in %.2f seconds", (System.currentTimeMillis() - now) / 1000.));
}

From source file:org.apache.accumulo.test.BulkImportSequentialRowsIT.java

License:Apache License

@Test
public void testBulkImportFailure() throws Exception {
    String tableName = getUniqueNames(1)[0];
    TableOperations to = getConnector().tableOperations();
    to.create(tableName);//  ww w .jav a  2s  .c o m
    FileSystem fs = getFileSystem();
    Path rootPath = new Path(fs.makeQualified(getUsableDir()), getClass().getSimpleName());
    log.info("Writing to {}", rootPath);
    if (fs.exists(rootPath)) {
        assertTrue(fs.delete(rootPath, true));
    }
    assertTrue(fs.mkdirs(rootPath));

    Path bulk = new Path(rootPath, "bulk");
    log.info("bulk: {}", bulk);
    assertTrue(fs.mkdirs(bulk));
    Path err = new Path(rootPath, "err");
    log.info("err: {}", err);

    assertTrue(fs.mkdirs(bulk));
    assertTrue(fs.mkdirs(err));

    Path rfile = new Path(bulk, "file.rf");

    log.info("Generating RFile {}", rfile.toUri().toString());

    GenerateSequentialRFile.main(new String[] { "-f", rfile.toUri().toString(), "-nr", Long.toString(NR), "-nv",
            Long.toString(NV) });

    assertTrue("Expected that " + rfile + " exists, but it does not", fs.exists(rfile));

    FsShell fsShell = new FsShell(fs.getConf());
    assertEquals("Failed to chmod " + rootPath, 0,
            fsShell.run(new String[] { "-chmod", "-R", "777", rootPath.toString() }));

    // Add some splits
    to.addSplits(tableName, getSplits());

    // Then import a single rfile to all the tablets, hoping that we get a failure to import because of the balancer moving tablets around
    // and then we get to verify that the bug is actually fixed.
    to.importDirectory(tableName, bulk.toString(), err.toString(), false);

    // The bug is that some tablets don't get imported into.
    assertEquals(NR * NV, Iterables.size(getConnector().createScanner(tableName, Authorizations.EMPTY)));
}