Example usage for org.apache.hadoop.hdfs DFSConfigKeys DFS_CLIENT_READ_SHORTCIRCUIT_KEY

Introduction

In this page you can find the example usage for org.apache.hadoop.hdfs DFSConfigKeys DFS_CLIENT_READ_SHORTCIRCUIT_KEY.

Prototype

String DFS_CLIENT_READ_SHORTCIRCUIT_KEY

To view the source code for org.apache.hadoop.hdfs DFSConfigKeys DFS_CLIENT_READ_SHORTCIRCUIT_KEY.

Click Source Link

Usage

From source file:com.cloudera.impala.service.JniFrontend.java

License:Apache License

/**
 * Return an empty string if short circuit read is properly enabled. If not, return an
 * error string describing the issues./*from   w  w w  . j  a va2 s  .  c om*/
 */
private String checkShortCircuitRead(Configuration conf) {
    StringBuilder output = new StringBuilder();
    String errorMessage = "ERROR: short-circuit local reads is disabled because\n";
    String prefix = "  - ";
    StringBuilder errorCause = new StringBuilder();

    // dfs.domain.socket.path must be set properly
    String domainSocketPath = conf.getTrimmed(DFSConfigKeys.DFS_DOMAIN_SOCKET_PATH_KEY,
            DFSConfigKeys.DFS_DOMAIN_SOCKET_PATH_DEFAULT);
    if (domainSocketPath.isEmpty()) {
        errorCause.append(prefix);
        errorCause.append(DFSConfigKeys.DFS_DOMAIN_SOCKET_PATH_KEY);
        errorCause.append(" is not configured.\n");
    } else {
        // The socket path parent directory must be readable and executable.
        File socketFile = new File(domainSocketPath);
        File socketDir = socketFile.getParentFile();
        if (socketDir == null || !socketDir.canRead() || !socketDir.canExecute()) {
            errorCause.append(prefix);
            errorCause.append("Impala cannot read or execute the parent directory of ");
            errorCause.append(DFSConfigKeys.DFS_DOMAIN_SOCKET_PATH_KEY);
            errorCause.append("\n");
        }
    }

    // dfs.client.read.shortcircuit must be set to true.
    if (!conf.getBoolean(DFSConfigKeys.DFS_CLIENT_READ_SHORTCIRCUIT_KEY,
            DFSConfigKeys.DFS_CLIENT_READ_SHORTCIRCUIT_DEFAULT)) {
        errorCause.append(prefix);
        errorCause.append(DFSConfigKeys.DFS_CLIENT_READ_SHORTCIRCUIT_KEY);
        errorCause.append(" is not enabled.\n");
    }

    // dfs.client.use.legacy.blockreader.local must be set to false
    if (conf.getBoolean(DFSConfigKeys.DFS_CLIENT_USE_LEGACY_BLOCKREADERLOCAL,
            DFSConfigKeys.DFS_CLIENT_USE_LEGACY_BLOCKREADERLOCAL_DEFAULT)) {
        errorCause.append(prefix);
        errorCause.append(DFSConfigKeys.DFS_CLIENT_USE_LEGACY_BLOCKREADERLOCAL);
        errorCause.append(" should not be enabled.\n");
    }

    if (errorCause.length() > 0) {
        output.append(errorMessage);
        output.append(errorCause);
    }

    return output.toString();
}

---

From source file:com.cloudera.impala.service.JniFrontend.java

License:Apache License

/**
 * Check short circuit read for CDH 4.1.
 * Return an empty string if short circuit read is properly enabled. If not, return an
 * error string describing the issues./*from   w  w  w  .jav a2s . c  o  m*/
 */
private String checkShortCircuitReadCdh41(Configuration conf) {
    StringBuilder output = new StringBuilder();
    String errorMessage = "ERROR: short-circuit local reads is disabled because\n";
    String prefix = "  - ";
    StringBuilder errorCause = new StringBuilder();

    // Client side checks
    // dfs.client.read.shortcircuit must be set to true.
    if (!conf.getBoolean(DFSConfigKeys.DFS_CLIENT_READ_SHORTCIRCUIT_KEY,
            DFSConfigKeys.DFS_CLIENT_READ_SHORTCIRCUIT_DEFAULT)) {
        errorCause.append(prefix);
        errorCause.append(DFSConfigKeys.DFS_CLIENT_READ_SHORTCIRCUIT_KEY);
        errorCause.append(" is not enabled.\n");
    }

    // dfs.client.use.legacy.blockreader.local must be set to true
    if (!conf.getBoolean(DFSConfigKeys.DFS_CLIENT_USE_LEGACY_BLOCKREADERLOCAL,
            DFSConfigKeys.DFS_CLIENT_USE_LEGACY_BLOCKREADERLOCAL_DEFAULT)) {
        errorCause.append(prefix);
        errorCause.append(DFSConfigKeys.DFS_CLIENT_USE_LEGACY_BLOCKREADERLOCAL);
        errorCause.append(" is not enabled.\n");
    }

    // Server side checks
    // Check data node server side configuration by reading the CONF from the data node
    // web UI
    // TODO: disabled for now
    //cdh41ShortCircuitReadDatanodeCheck(errorCause, prefix);

    if (errorCause.length() > 0) {
        output.append(errorMessage);
        output.append(errorCause);
    }

    return output.toString();
}

---

From source file:org.apache.impala.service.JniFrontend.java

License:Apache License

/**
 * Returns an error message if short circuit reads are enabled but misconfigured.
 * Otherwise, returns an empty string,/*from   ww  w  . ja  v a  2  s.  co  m*/
 */
private String checkShortCircuitRead(Configuration conf) {
    if (!conf.getBoolean(DFSConfigKeys.DFS_CLIENT_READ_SHORTCIRCUIT_KEY,
            DFSConfigKeys.DFS_CLIENT_READ_SHORTCIRCUIT_DEFAULT)) {
        LOG.info("Short-circuit reads are not enabled.");
        return "";
    }

    StringBuilder output = new StringBuilder();
    String errorMessage = "Invalid short-circuit reads configuration:\n";
    String prefix = "  - ";
    StringBuilder errorCause = new StringBuilder();

    // dfs.domain.socket.path must be set properly
    String domainSocketPath = conf.getTrimmed(DFSConfigKeys.DFS_DOMAIN_SOCKET_PATH_KEY,
            DFSConfigKeys.DFS_DOMAIN_SOCKET_PATH_DEFAULT);
    if (domainSocketPath.isEmpty()) {
        errorCause.append(prefix);
        errorCause.append(DFSConfigKeys.DFS_DOMAIN_SOCKET_PATH_KEY);
        errorCause.append(" is not configured.\n");
    } else {
        // The socket path parent directory must be readable and executable.
        File socketFile = new File(domainSocketPath);
        File socketDir = socketFile.getParentFile();
        if (socketDir == null || !socketDir.canRead() || !socketDir.canExecute()) {
            errorCause.append(prefix);
            errorCause.append("Impala cannot read or execute the parent directory of ");
            errorCause.append(DFSConfigKeys.DFS_DOMAIN_SOCKET_PATH_KEY);
            errorCause.append("\n");
        }
    }

    // dfs.client.use.legacy.blockreader.local must be set to false
    if (conf.getBoolean(DFSConfigKeys.DFS_CLIENT_USE_LEGACY_BLOCKREADERLOCAL,
            DFSConfigKeys.DFS_CLIENT_USE_LEGACY_BLOCKREADERLOCAL_DEFAULT)) {
        errorCause.append(prefix);
        errorCause.append(DFSConfigKeys.DFS_CLIENT_USE_LEGACY_BLOCKREADERLOCAL);
        errorCause.append(" should not be enabled.\n");
    }

    if (errorCause.length() > 0) {
        output.append(errorMessage);
        output.append(errorCause);
    }

    return output.toString();
}

---

From source file:org.apache.tajo.TajoTestingCluster.java

License:Apache License

/**
 * Start a minidfscluster./*from   w w w  .j  a va 2  s.c o m*/
 * Can only create one.
 * @param servers How many DNs to start.
 * @param dir Where to home your dfs cluster.
 * @param hosts hostnames DNs to run on.
 * @throws Exception
 * @see {@link #shutdownMiniDFSCluster()}
 * @return The mini dfs cluster created.
 * @throws java.io.IOException
 */
public MiniDFSCluster startMiniDFSCluster(int servers, File dir, final String hosts[]) throws IOException {

    conf.set(MiniDFSCluster.HDFS_MINIDFS_BASEDIR, dir.toString());
    conf.setInt(DFSConfigKeys.DFS_REPLICATION_KEY, 1);
    conf.setBoolean(DFSConfigKeys.DFS_CLIENT_READ_SHORTCIRCUIT_KEY, false);
    MiniDFSCluster.Builder builder = new MiniDFSCluster.Builder(new HdfsConfiguration(conf));
    builder.hosts(hosts);
    builder.numDataNodes(servers);
    builder.format(true);
    builder.manageNameDfsDirs(true);
    builder.manageDataDfsDirs(true);
    builder.waitSafeMode(true);
    this.dfsCluster = builder.build();

    // Set this just-started cluser as our filesystem.
    this.defaultFS = this.dfsCluster.getFileSystem();
    this.conf.set(CommonConfigurationKeysPublic.FS_DEFAULT_NAME_KEY, defaultFS.getUri().toString());
    this.conf.setVar(TajoConf.ConfVars.ROOT_DIR, defaultFS.getUri() + "/tajo");
    isDFSRunning = true;
    return this.dfsCluster;
}

---