Example usage for org.apache.hadoop.hdfs DFSConfigKeys DFS_HEARTBEAT_INTERVAL_KEY

Introduction

In this page you can find the example usage for org.apache.hadoop.hdfs DFSConfigKeys DFS_HEARTBEAT_INTERVAL_KEY.

Prototype

String DFS_HEARTBEAT_INTERVAL_KEY

To view the source code for org.apache.hadoop.hdfs DFSConfigKeys DFS_HEARTBEAT_INTERVAL_KEY.

Click Source Link

Usage

From source file:backup.integration.MiniClusterTestBase.java

License:Apache License

private Configuration setupConfig(File hdfsDir) throws Exception {
    Configuration conf = new Configuration();
    File backup = new File(tmpHdfs, "backup");
    backup.mkdirs();//from   w  w w  .ja  v  a  2 s  .  com
    conf.set(DFS_BACKUP_NAMENODE_LOCAL_DIR_KEY, backup.getAbsolutePath());
    conf.set(MiniDFSCluster.HDFS_MINIDFS_BASEDIR, hdfsDir.getAbsolutePath());
    conf.set(DFSConfigKeys.DFS_DATANODE_FSDATASET_FACTORY_KEY, BackupFsDatasetSpiFactory.class.getName());
    conf.set(DFSConfigKeys.DFS_DATANODE_PLUGINS_KEY, DataNodeBackupServicePlugin.class.getName());
    conf.set(DFSConfigKeys.DFS_NAMENODE_PLUGINS_KEY, NameNodeBackupServicePlugin.class.getName());

    conf.setInt(BackupConstants.DFS_BACKUP_DATANODE_RPC_PORT_KEY, 0);
    conf.setInt(BackupConstants.DFS_BACKUP_NAMENODE_HTTP_PORT_KEY, 0);

    conf.setLong(DFSConfigKeys.DFS_HEARTBEAT_INTERVAL_KEY, 2);// 3
    conf.setLong(DFSConfigKeys.DFS_NAMENODE_STALE_DATANODE_MINIMUM_INTERVAL_KEY, 2);// 3
    conf.setLong(DFSConfigKeys.DFS_NAMENODE_STALE_DATANODE_INTERVAL_KEY, 6000);// 30000
    conf.setLong(DFSConfigKeys.DFS_NAMENODE_HEARTBEAT_RECHECK_INTERVAL_KEY, 6000);// 5*60*1000

    org.apache.commons.configuration.Configuration configuration = BackupUtil.convert(conf);
    setupBackupStore(configuration);
    @SuppressWarnings("unchecked")
    Iterator<String> keys = configuration.getKeys();
    while (keys.hasNext()) {
        String key = keys.next();
        conf.set(key, configuration.getString(key));
    }

    return conf;
}

---

From source file:com.mellanox.r4h.TestReadWhileWriting.java

License:Apache License

/** Test reading while writing. */
@Test/*from   ww w  .j  a  va  2s  . c  o m*/
public void pipeline_02_03() throws Exception {
    final Configuration conf = new HdfsConfiguration();
    conf.setLong(DFSConfigKeys.DFS_HEARTBEAT_INTERVAL_KEY, 1);

    // create cluster
    final MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).numDataNodes(4).build();
    try {
        //change the lease limits.
        cluster.setLeasePeriod(SOFT_LEASE_LIMIT, HARD_LEASE_LIMIT);

        //wait for the cluster
        cluster.waitActive();
        final FileSystem fs = cluster.getFileSystem();
        final Path p = new Path(DIR, "file1");
        final int half = BLOCK_SIZE / 2;

        //a. On Machine M1, Create file. Write half block of data.
        //   Invoke DFSOutputStream.hflush() on the dfs file handle.
        //   Do not close file yet.
        {
            final FSDataOutputStream out = fs.create(p, true,
                    fs.getConf().getInt(CommonConfigurationKeys.IO_FILE_BUFFER_SIZE_KEY, 4096), (short) 3,
                    BLOCK_SIZE);
            write(out, 0, half);

            //hflush
            ((DFSOutputStream) out.getWrappedStream()).hflush();
        }

        //b. On another machine M2, open file and verify that the half-block
        //   of data can be read successfully.
        checkFile(p, half, conf);
        MiniDFSClusterBridge.getAppendTestUtilLOG().info("leasechecker.interruptAndJoin()");
        ((DistributedFileSystem) fs).dfs.getLeaseRenewer().interruptAndJoin();

        //c. On M1, append another half block of data.  Close file on M1.
        {
            //sleep to let the lease is expired.
            Thread.sleep(2 * SOFT_LEASE_LIMIT);

            final UserGroupInformation current = UserGroupInformation.getCurrentUser();
            final UserGroupInformation ugi = UserGroupInformation
                    .createUserForTesting(current.getShortUserName() + "x", new String[] { "supergroup" });
            final DistributedFileSystem dfs = ugi.doAs(new PrivilegedExceptionAction<DistributedFileSystem>() {
                @Override
                public DistributedFileSystem run() throws Exception {
                    return (DistributedFileSystem) FileSystem.newInstance(conf);
                }
            });
            final FSDataOutputStream out = append(dfs, p);
            write(out, 0, half);
            out.close();
        }

        //d. On M2, open file and read 1 block of data from it. Close file.
        checkFile(p, 2 * half, conf);
    } finally {
        cluster.shutdown();
    }
}

---

From source file:com.uber.hoodie.common.table.log.HoodieLogFormatAppendFailureTest.java

License:Apache License

@BeforeClass
public static void setUpClass() throws IOException {
    // NOTE : The MiniClusterDFS leaves behind the directory under which the cluster was created
    baseDir = new File("/tmp/" + UUID.randomUUID().toString());
    FileUtil.fullyDelete(baseDir);/*from   w w w. java 2s. c om*/
    // Append is not supported in LocalFileSystem. HDFS needs to be setup.
    Configuration conf = new Configuration();
    // lower heartbeat interval for fast recognition of DN
    conf.set(MiniDFSCluster.HDFS_MINIDFS_BASEDIR, baseDir.getAbsolutePath());
    conf.setInt(DFSConfigKeys.DFS_NAMENODE_HEARTBEAT_RECHECK_INTERVAL_KEY, 1000);
    conf.setInt(DFSConfigKeys.DFS_HEARTBEAT_INTERVAL_KEY, 1);
    conf.setInt(DFSConfigKeys.DFS_CLIENT_SOCKET_TIMEOUT_KEY, 3000);
    cluster = new MiniDFSCluster.Builder(conf).checkExitOnShutdown(true).numDataNodes(4).build();
}

---

From source file:mzb.Balancer.java

License:Apache License

/**
 * Balance all namenodes.//from w w  w.  j av  a 2  s.c o m
 * For each iteration,
 * for each namenode,
 * execute a {@link Balancer} to work through all datanodes once.  
 */
static int run(Collection<URI> namenodes, Configuration conf) throws IOException, InterruptedException {
    final long sleeptime = 2000 * conf.getLong(DFSConfigKeys.DFS_HEARTBEAT_INTERVAL_KEY,
            DFSConfigKeys.DFS_HEARTBEAT_INTERVAL_DEFAULT);
    LOG.info("namenodes = " + namenodes);
    //    LOG.info("p         = " + p);

    final Formatter formatter = new Formatter(System.out);
    System.out.println(
            "Time Stamp               Iteration#  Bytes Already Moved  Bytes Left To Move  Bytes Being Moved");

    final List<NameNodeConnector> connectors = new ArrayList<NameNodeConnector>(namenodes.size());
    try {
        for (URI uri : namenodes) {
            connectors.add(new NameNodeConnector(uri, conf));
        }

        boolean done = false;
        for (int iteration = 0; !done; iteration++) {
            done = true;
            Collections.shuffle(connectors);
            for (NameNodeConnector nnc : connectors) {
                final Balancer b = new Balancer(nnc, conf);
                final ReturnStatus r = b.run(iteration, formatter, conf);
                // clean all lists
                b.resetData(conf);
                if (r == ReturnStatus.IN_PROGRESS) {
                    done = false;
                } else if (r != ReturnStatus.SUCCESS) {
                    //must be an error statue, return.
                    return r.code;
                }
            }

            if (!done) {
                Thread.sleep(sleeptime);
            }
        }
    } finally {
        for (NameNodeConnector nnc : connectors) {
            nnc.close();
        }
    }
    return ReturnStatus.SUCCESS.code;
}

---