Example usage for org.apache.hadoop.security UserGroupInformation getCurrentUser

List of usage examples for org.apache.hadoop.security UserGroupInformation getCurrentUser

  1. HOME
  2. Java
  3. org.apache.hadoop
  4. org.apache.hadoop.security.*
  5. UserGroupInformation
  6. getCurrentUser

Introduction

In this page you can find the example usage for org.apache.hadoop.security UserGroupInformation getCurrentUser.

Prototype

@InterfaceAudience.Public
@InterfaceStability.Evolving
public static UserGroupInformation getCurrentUser() throws IOException

Source Link

Document

Return the current user, including any doAs in the current stack.

Usage

From source file:org.apache.tajo.master.QueryMaster.java

License:Apache License

/**
 * It initializes the final output and staging directory and sets
 * them to variables./*from   w w  w. ja  v a  2  s . co  m*/
 */
private void initStagingDir() throws IOException {
    QueryConf conf = getContext().getConf();

    String realUser;
    String currentUser;
    UserGroupInformation ugi;
    ugi = UserGroupInformation.getLoginUser();
    realUser = ugi.getShortUserName();
    currentUser = UserGroupInformation.getCurrentUser().getShortUserName();

    String givenOutputTableName = conf.getOutputTable();
    Path stagingDir;

    // If final output directory is not given by an user,
    // we use the query id as a output directory.
    if (givenOutputTableName.equals("")) {
        this.isCreateTableStmt = false;
        FileSystem defaultFS = FileSystem.get(conf);

        Path homeDirectory = defaultFS.getHomeDirectory();
        if (!defaultFS.exists(homeDirectory)) {
            defaultFS.mkdirs(homeDirectory, new FsPermission(USER_DIR_PERMISSION));
        }

        Path userQueryDir = new Path(homeDirectory, TajoConstants.USER_QUERYDIR_PREFIX);

        if (defaultFS.exists(userQueryDir)) {
            FileStatus fsStatus = defaultFS.getFileStatus(userQueryDir);
            String owner = fsStatus.getOwner();

            if (!(owner.equals(currentUser) || owner.equals(realUser))) {
                throw new IOException("The ownership on the user's query " + "directory " + userQueryDir
                        + " is not as expected. " + "It is owned by " + owner + ". The directory must "
                        + "be owned by the submitter " + currentUser + " or " + "by " + realUser);
            }

            if (!fsStatus.getPermission().equals(USER_DIR_PERMISSION)) {
                LOG.info("Permissions on staging directory " + userQueryDir + " are " + "incorrect: "
                        + fsStatus.getPermission() + ". Fixing permissions " + "to correct value "
                        + USER_DIR_PERMISSION);
                defaultFS.setPermission(userQueryDir, new FsPermission(USER_DIR_PERMISSION));
            }
        } else {
            defaultFS.mkdirs(userQueryDir, new FsPermission(USER_DIR_PERMISSION));
        }

        stagingDir = StorageUtil.concatPath(userQueryDir, queryId.toString());

        if (defaultFS.exists(stagingDir)) {
            throw new IOException("The staging directory " + stagingDir
                    + "already exists. The directory must be unique to each query");
        } else {
            defaultFS.mkdirs(stagingDir, new FsPermission(USER_DIR_PERMISSION));
        }

        // Set the query id to the output table name
        conf.setOutputTable(queryId.toString());

    } else {
        this.isCreateTableStmt = true;
        Path warehouseDir = new Path(conf.getVar(TajoConf.ConfVars.ROOT_DIR), TajoConstants.WAREHOUSE_DIR);
        stagingDir = new Path(warehouseDir, conf.getOutputTable());

        FileSystem fs = warehouseDir.getFileSystem(conf);
        if (fs.exists(stagingDir)) {
            throw new IOException("The staging directory " + stagingDir
                    + " already exists. The directory must be unique to each query");
        } else {
            // TODO - should have appropriate permission
            fs.mkdirs(stagingDir, new FsPermission(USER_DIR_PERMISSION));
        }
    }

    conf.setOutputPath(stagingDir);
    outputPath = stagingDir;
    LOG.info("Initialized Query Staging Dir: " + outputPath);
}

From source file:org.apache.tajo.master.rm.YarnTajoResourceManager.java

License:Apache License

@Override
public void init(Configuration conf) {
    this.conf = conf;
    connectYarnClient();/*from  ww  w  .  ja v  a 2  s.co  m*/

    final YarnConfiguration yarnConf = new YarnConfiguration(conf);
    final YarnRPC rpc = YarnRPC.create(conf);
    final InetSocketAddress rmAddress = conf.getSocketAddr(YarnConfiguration.RM_SCHEDULER_ADDRESS,
            YarnConfiguration.DEFAULT_RM_SCHEDULER_ADDRESS, YarnConfiguration.DEFAULT_RM_SCHEDULER_PORT);

    UserGroupInformation currentUser;
    try {
        currentUser = UserGroupInformation.getCurrentUser();
    } catch (IOException e) {
        throw new YarnRuntimeException(e);
    }

    rmClient = currentUser.doAs(new PrivilegedAction<ApplicationMasterProtocol>() {
        @Override
        public ApplicationMasterProtocol run() {
            return (ApplicationMasterProtocol) rpc.getProxy(ApplicationMasterProtocol.class, rmAddress,
                    yarnConf);
        }
    });
}

From source file:org.apache.tajo.master.rule.FileSystemRule.java

License:Apache License

private void canAccessToPath(FileStatus fsStatus, FsAction action) throws Exception {
    FsPermission permission = fsStatus.getPermission();
    UserGroupInformation userGroupInformation = UserGroupInformation.getCurrentUser();
    String userName = userGroupInformation.getShortUserName();
    List<String> groupList = Arrays.asList(userGroupInformation.getGroupNames());

    if (userName.equals(fsStatus.getOwner())) {
        if (permission.getUserAction().implies(action)) {
            return;
        }/*from   w  w w  .  ja  va  2 s.c  o m*/
    } else if (groupList.contains(fsStatus.getGroup())) {
        if (permission.getGroupAction().implies(action)) {
            return;
        }
    } else {
        if (permission.getOtherAction().implies(action)) {
            return;
        }
    }
    throw new AccessControlException(
            String.format("Permission denied: user=%s, path=\"%s\":%s:%s:%s%s", userName, fsStatus.getPath(),
                    fsStatus.getOwner(), fsStatus.getGroup(), fsStatus.isDirectory() ? "d" : "-", permission));
}

From source file:org.apache.tajo.master.TaskRunnerLauncherImpl.java

License:Apache License

private ContainerLaunchContext createCommonContainerLaunchContext() {
    TajoConf conf = (TajoConf) getConfig();

    ContainerLaunchContext ctx = Records.newRecord(ContainerLaunchContext.class);
    try {/*w w w  .j a  v a 2  s. c  o  m*/
        ctx.setUser(UserGroupInformation.getCurrentUser().getShortUserName());
    } catch (IOException e) {
        e.printStackTrace();
    }

    ////////////////////////////////////////////////////////////////////////////
    // Set the env variables to be setup
    ////////////////////////////////////////////////////////////////////////////
    LOG.info("Set the environment for the application master");

    Map<String, String> environment = new HashMap<String, String>();
    //String initialClassPath = getInitialClasspath(conf);
    environment.put(Environment.SHELL.name(), "/bin/bash");
    environment.put(Environment.JAVA_HOME.name(), System.getenv(Environment.JAVA_HOME.name()));

    // TODO - to be improved with org.apache.tajo.sh shell script
    Properties prop = System.getProperties();
    if (prop.getProperty("tajo.test", "FALSE").equalsIgnoreCase("TRUE")) {
        environment.put(Environment.CLASSPATH.name(), prop.getProperty("java.class.path", null));
    } else {
        // Add AppMaster.jar location to classpath
        // At some point we should not be required to add
        // the hadoop specific classpaths to the env.
        // It should be provided out of the box.
        // For now setting all required classpaths including
        // the classpath to "." for the application jar
        StringBuilder classPathEnv = new StringBuilder("./");
        //for (String c : conf.getStrings(YarnConfiguration.YARN_APPLICATION_CLASSPATH)) {
        for (String c : YarnConfiguration.DEFAULT_YARN_APPLICATION_CLASSPATH) {
            classPathEnv.append(':');
            classPathEnv.append(c.trim());
        }

        classPathEnv.append(":" + System.getenv("TAJO_BASE_CLASSPATH"));
        classPathEnv.append(":./log4j.properties:./*");
        environment.put("HADOOP_HOME", System.getenv("HADOOP_HOME"));
        environment.put(Environment.HADOOP_COMMON_HOME.name(), System.getenv("HADOOP_HOME"));
        environment.put(Environment.HADOOP_HDFS_HOME.name(), System.getenv("HADOOP_HOME"));
        environment.put(Environment.HADOOP_YARN_HOME.name(), System.getenv("HADOOP_HOME"));
        environment.put("TAJO_BASE_CLASSPATH", System.getenv("TAJO_BASE_CLASSPATH"));
        environment.put(Environment.CLASSPATH.name(), classPathEnv.toString());
    }

    ctx.setEnvironment(environment);

    ////////////////////////////////////////////////////////////////////////////
    // Set the local resources
    ////////////////////////////////////////////////////////////////////////////
    Map<String, LocalResource> localResources = new HashMap<String, LocalResource>();
    FileSystem fs = null;

    LOG.info("defaultFS: " + conf.get("fs.default.name"));
    LOG.info("defaultFS: " + conf.get("fs.defaultFS"));
    try {
        fs = FileSystem.get(conf);
    } catch (IOException e) {
        e.printStackTrace();
    }

    FileContext fsCtx = null;
    try {
        fsCtx = FileContext.getFileContext(getConfig());
    } catch (UnsupportedFileSystemException e) {
        e.printStackTrace();
    }

    LOG.info("Writing a QueryConf to HDFS and add to local environment");
    Path queryConfPath = new Path(fs.getHomeDirectory(), QueryConf.FILENAME);
    try {
        writeConf(conf, queryConfPath);

        LocalResource queryConfSrc = createApplicationResource(fsCtx, queryConfPath, LocalResourceType.FILE);
        localResources.put(QueryConf.FILENAME, queryConfSrc);

        ctx.setLocalResources(localResources);
    } catch (IOException e) {
        e.printStackTrace();
    }

    // Add shuffle token
    Map<String, ByteBuffer> serviceData = new HashMap<String, ByteBuffer>();
    try {
        //LOG.info("Putting shuffle token in serviceData");
        serviceData.put(PullServerAuxService.PULLSERVER_SERVICEID, PullServerAuxService.serializeMetaData(0));
    } catch (IOException ioe) {
        LOG.error(ioe);
    }
    ctx.setServiceData(serviceData);

    return ctx;
}

From source file:org.apache.tajo.master.TaskRunnerLauncherImpl.java

License:Apache License

protected ContainerManager getCMProxy(ContainerId containerID, final String containerManagerBindAddr,
        ContainerToken containerToken) throws IOException {
    String[] hosts = containerManagerBindAddr.split(":");
    final InetSocketAddress cmAddr = new InetSocketAddress(hosts[0], Integer.parseInt(hosts[1]));
    UserGroupInformation user = UserGroupInformation.getCurrentUser();

    if (UserGroupInformation.isSecurityEnabled()) {
        Token<ContainerTokenIdentifier> token = ProtoUtils.convertFromProtoFormat(containerToken, cmAddr);
        // the user in createRemoteUser in this context has to be ContainerID
        user = UserGroupInformation.createRemoteUser(containerID.toString());
        user.addToken(token);/*from   w  w w .j av  a 2 s  . com*/
    }

    ContainerManager proxy = user.doAs(new PrivilegedAction<ContainerManager>() {
        @Override
        public ContainerManager run() {
            return (ContainerManager) yarnRPC.getProxy(ContainerManager.class, cmAddr, getConfig());
        }
    });
    return proxy;
}

From source file:org.apache.tajo.master.YarnContainerProxy.java

License:Apache License

protected ContainerManagementProtocol getCMProxy(ContainerId containerID, final String containerManagerBindAddr,
        Token containerToken) throws IOException {
    String[] hosts = containerManagerBindAddr.split(":");
    final InetSocketAddress cmAddr = new InetSocketAddress(hosts[0], Integer.parseInt(hosts[1]));
    UserGroupInformation user = UserGroupInformation.getCurrentUser();

    if (UserGroupInformation.isSecurityEnabled()) {
        org.apache.hadoop.security.token.Token<ContainerTokenIdentifier> token = ConverterUtils
                .convertFromYarn(containerToken, cmAddr);
        // the user in createRemoteUser in this context has to be ContainerID
        user = UserGroupInformation.createRemoteUser(containerID.toString());
        user.addToken(token);/*  w w  w. j a  va2s.  c om*/
    }

    ContainerManagementProtocol proxy = user.doAs(new PrivilegedAction<ContainerManagementProtocol>() {
        @Override
        public ContainerManagementProtocol run() {
            return (ContainerManagementProtocol) yarnRPC.getProxy(ContainerManagementProtocol.class, cmAddr,
                    conf);
        }
    });

    return proxy;
}

From source file:org.apache.tajo.master.YarnContainerProxy.java

License:Apache License

public static ContainerLaunchContext createCommonContainerLaunchContext(Configuration config, String queryId,
        boolean isMaster) {
    TajoConf conf = (TajoConf) config;/* w w  w.  ja v  a2 s .co m*/

    ContainerLaunchContext ctx = Records.newRecord(ContainerLaunchContext.class);

    try {
        ByteBuffer userToken = ByteBuffer
                .wrap(UserGroupInformation.getCurrentUser().getShortUserName().getBytes());
        ctx.setTokens(userToken);
    } catch (IOException e) {
        e.printStackTrace();
    }

    ////////////////////////////////////////////////////////////////////////////
    // Set the env variables to be setup
    ////////////////////////////////////////////////////////////////////////////
    LOG.info("Set the environment for the application master");

    Map<String, String> environment = new HashMap<String, String>();
    //String initialClassPath = getInitialClasspath(conf);
    environment.put(ApplicationConstants.Environment.SHELL.name(), "/bin/bash");
    if (System.getenv(ApplicationConstants.Environment.JAVA_HOME.name()) != null) {
        environment.put(ApplicationConstants.Environment.JAVA_HOME.name(),
                System.getenv(ApplicationConstants.Environment.JAVA_HOME.name()));
    }

    // TODO - to be improved with org.apache.tajo.sh shell script
    Properties prop = System.getProperties();

    if (prop.getProperty("tajo.test", "FALSE").equalsIgnoreCase("TRUE")
            || (System.getenv("tajo.test") != null && System.getenv("tajo.test").equalsIgnoreCase("TRUE"))) {
        LOG.info("tajo.test is TRUE");
        environment.put(ApplicationConstants.Environment.CLASSPATH.name(),
                prop.getProperty("java.class.path", null));
        environment.put("tajo.test", "TRUE");
    } else {
        // Add AppMaster.jar location to classpath
        // At some point we should not be required to add
        // the hadoop specific classpaths to the env.
        // It should be provided out of the box.
        // For now setting all required classpaths including
        // the classpath to "." for the application jar
        StringBuilder classPathEnv = new StringBuilder("./");
        //for (String c : conf.getStrings(YarnConfiguration.YARN_APPLICATION_CLASSPATH)) {
        for (String c : YarnConfiguration.DEFAULT_YARN_APPLICATION_CLASSPATH) {
            classPathEnv.append(':');
            classPathEnv.append(c.trim());
        }

        classPathEnv.append(":" + System.getenv("TAJO_BASE_CLASSPATH"));
        classPathEnv.append(":./log4j.properties:./*");
        if (System.getenv("HADOOP_HOME") != null) {
            environment.put("HADOOP_HOME", System.getenv("HADOOP_HOME"));
            environment.put(ApplicationConstants.Environment.HADOOP_COMMON_HOME.name(),
                    System.getenv("HADOOP_HOME"));
            environment.put(ApplicationConstants.Environment.HADOOP_HDFS_HOME.name(),
                    System.getenv("HADOOP_HOME"));
            environment.put(ApplicationConstants.Environment.HADOOP_YARN_HOME.name(),
                    System.getenv("HADOOP_HOME"));
        }

        if (System.getenv("TAJO_BASE_CLASSPATH") != null) {
            environment.put("TAJO_BASE_CLASSPATH", System.getenv("TAJO_BASE_CLASSPATH"));
        }
        environment.put(ApplicationConstants.Environment.CLASSPATH.name(), classPathEnv.toString());
    }

    ctx.setEnvironment(environment);

    if (LOG.isDebugEnabled()) {
        LOG.debug("=================================================");
        for (Map.Entry<String, String> entry : environment.entrySet()) {
            LOG.debug(entry.getKey() + "=" + entry.getValue());
        }
        LOG.debug("=================================================");
    }
    ////////////////////////////////////////////////////////////////////////////
    // Set the local resources
    ////////////////////////////////////////////////////////////////////////////
    Map<String, LocalResource> localResources = new HashMap<String, LocalResource>();
    LOG.info("defaultFS: " + conf.get(CommonConfigurationKeysPublic.FS_DEFAULT_NAME_KEY));

    try {
        FileSystem fs = FileSystem.get(conf);
        FileContext fsCtx = FileContext.getFileContext(conf);
        Path systemConfPath = TajoConf.getSystemConfPath(conf);
        if (!fs.exists(systemConfPath)) {
            LOG.error("system_conf.xml (" + systemConfPath.toString() + ") Not Found");
        }
        LocalResource systemConfResource = createApplicationResource(fsCtx, systemConfPath,
                LocalResourceType.FILE);
        localResources.put(TajoConstants.SYSTEM_CONF_FILENAME, systemConfResource);
        ctx.setLocalResources(localResources);
    } catch (IOException e) {
        LOG.error(e.getMessage(), e);
    }

    Map<String, ByteBuffer> serviceData = new HashMap<String, ByteBuffer>();
    try {
        serviceData.put(PullServerAuxService.PULLSERVER_SERVICEID, PullServerAuxService.serializeMetaData(0));
    } catch (IOException ioe) {
        LOG.error(ioe);
    }
    ctx.setServiceData(serviceData);

    return ctx;
}

From source file:org.apache.tajo.storage.FileTablespace.java

License:Apache License

public URI prepareStagingSpace(TajoConf conf, String queryId, OverridableConf context, TableMeta meta)
        throws IOException {

    String realUser;//from w  w  w .j  a  va 2  s  . c  om
    String currentUser;
    UserGroupInformation ugi;
    ugi = UserGroupInformation.getLoginUser();
    realUser = ugi.getShortUserName();
    currentUser = UserGroupInformation.getCurrentUser().getShortUserName();

    Path stagingDir = new Path(getStagingUri(context, queryId, meta));

    ////////////////////////////////////////////
    // Create Output Directory
    ////////////////////////////////////////////

    if (fs.exists(stagingDir)) {
        throw new IOException("The staging directory '" + stagingDir + "' already exists");
    }
    fs.mkdirs(stagingDir, new FsPermission(STAGING_DIR_PERMISSION));
    FileStatus fsStatus = fs.getFileStatus(stagingDir);
    String owner = fsStatus.getOwner();

    if (!owner.isEmpty() && !(owner.equals(currentUser) || owner.equals(realUser))) {
        throw new IOException("The ownership on the user's query " + "directory " + stagingDir
                + " is not as expected. " + "It is owned by " + owner + ". The directory must "
                + "be owned by the submitter " + currentUser + " or " + "by " + realUser);
    }

    if (!fsStatus.getPermission().equals(STAGING_DIR_PERMISSION)) {
        LOG.info("Permissions on staging directory " + stagingDir + " are " + "incorrect: "
                + fsStatus.getPermission() + ". Fixing permissions " + "to correct value "
                + STAGING_DIR_PERMISSION);
        fs.setPermission(stagingDir, new FsPermission(STAGING_DIR_PERMISSION));
    }

    Path stagingResultDir = new Path(stagingDir, TajoConstants.RESULT_DIR_NAME);
    fs.mkdirs(stagingResultDir);

    return stagingDir.toUri();
}

From source file:org.apache.tajo.yarn.container.WorkerContainerTask.java

License:Apache License

@Override
public ContainerLaunchContext getLaunchContext(Container container) throws IOException {
    // create a container launch context
    ContainerLaunchContext launchContext = Records.newRecord(ContainerLaunchContext.class);
    UserGroupInformation user = UserGroupInformation.getCurrentUser();
    try {// www  .j  av  a2 s.c  o  m
        Credentials credentials = user.getCredentials();
        DataOutputBuffer dob = new DataOutputBuffer();
        credentials.writeTokenStorageToStream(dob);
        ByteBuffer securityTokens = ByteBuffer.wrap(dob.getData(), 0, dob.getLength());
        launchContext.setTokens(securityTokens);
    } catch (IOException e) {
        LOG.warn("Getting current user info failed when trying to launch the container" + e.getMessage());
    }

    FileSystem fs = FileSystem.get(appContext.getConfiguration());

    // Set the local resources
    Map<String, LocalResource> localResources = new HashMap<String, LocalResource>();

    String suffix = "Tajo" + "/" + appContext.getApplicationId().getId();
    Path parentPath = new Path(fs.getHomeDirectory(), suffix);

    // tar ball
    Path archivePath = new Path(parentPath, System.getenv(Constants.TAJO_ARCHIVE_PATH));
    FileStatus archiveFs = fs.getFileStatus(archivePath);
    LocalResource archiveRsrc = LocalResource.newInstance(ConverterUtils.getYarnUrlFromURI(archivePath.toUri()),
            LocalResourceType.ARCHIVE, LocalResourceVisibility.APPLICATION, archiveFs.getLen(),
            archiveFs.getModificationTime());
    localResources.put("tajo", archiveRsrc);

    Configuration tajoWorkerConf = new Configuration(false);
    tajoWorkerConf.addResource(new Path("conf", "tajo-site.xml"));
    tajoWorkerConf.set(Constants.TAJO_MASTER_UMBILICAL_RPC_ADDRESS, appContext.getMasterHost() + ":26001");
    tajoWorkerConf.set(Constants.CATALOG_ADDRESS, appContext.getMasterHost() + ":26005");
    Path dst = new Path(parentPath, container.getId() + Path.SEPARATOR + "worker-conf");
    fs.mkdirs(dst);
    Path confFile = new Path(dst, "tajo-site.xml");
    FSDataOutputStream fdos = fs.create(confFile);
    tajoWorkerConf.writeXml(fdos);
    fdos.close();
    FileStatus scFileStatus = fs.getFileStatus(dst);
    LocalResource scRsrc = LocalResource.newInstance(ConverterUtils.getYarnUrlFromURI(dst.toUri()),
            LocalResourceType.FILE, LocalResourceVisibility.APPLICATION, scFileStatus.getLen(),
            scFileStatus.getModificationTime());
    localResources.put("conf", scRsrc);
    launchContext.setLocalResources(localResources);

    // Set the environment
    setupEnv(launchContext);

    // Set the necessary command to execute on the allocated container
    Vector<CharSequence> vargs = new Vector<CharSequence>(5);

    // Set executable command
    // Set args for the shell command if any
    vargs.add("${" + Constants.TAJO_HOME + "}/bin/tajo");
    vargs.add("--config");
    vargs.add("${" + Constants.TAJO_CONF_DIR + "}");
    vargs.add("worker");
    // Add log redirect params
    // Add log redirect params
    vargs.add("1>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/stdout");
    vargs.add("2>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/stderr");

    // Get final commmand
    StringBuilder command = new StringBuilder();
    for (CharSequence str : vargs) {
        command.append(str).append(" ");
    }

    List<String> commands = new ArrayList<String>();
    commands.add(command.toString());
    launchContext.setCommands(commands);
    return launchContext;
}

From source file:org.apache.tez.auxservices.TestIndexCache.java

License:Apache License

@Test
public void testLRCPolicy() throws Exception {
    Random r = new Random();
    long seed = r.nextLong();
    r.setSeed(seed);//from  w w w . j a  v a2  s  .co  m
    System.out.println("seed: " + seed);
    fs.delete(p, true);
    conf.setInt(INDEX_CACHE_MB, 1);
    final int partsPerMap = 1000;
    final int bytesPerFile = partsPerMap * 24;
    IndexCache cache = new IndexCache(conf);

    // fill cache
    int totalsize = bytesPerFile;
    for (; totalsize < 1024 * 1024; totalsize += bytesPerFile) {
        Path f = new Path(p, Integer.toString(totalsize, 36));
        writeFile(fs, f, totalsize, partsPerMap);
        TezIndexRecord rec = cache.getIndexInformation(Integer.toString(totalsize, 36), r.nextInt(partsPerMap),
                f, UserGroupInformation.getCurrentUser().getShortUserName());
        checkRecord(rec, totalsize);
    }

    // delete files, ensure cache retains all elem
    for (FileStatus stat : fs.listStatus(p)) {
        fs.delete(stat.getPath(), true);
    }
    for (int i = bytesPerFile; i < 1024 * 1024; i += bytesPerFile) {
        Path f = new Path(p, Integer.toString(i, 36));
        TezIndexRecord rec = cache.getIndexInformation(Integer.toString(i, 36), r.nextInt(partsPerMap), f,
                UserGroupInformation.getCurrentUser().getShortUserName());
        checkRecord(rec, i);
    }

    // push oldest (bytesPerFile) out of cache
    Path f = new Path(p, Integer.toString(totalsize, 36));
    writeFile(fs, f, totalsize, partsPerMap);
    cache.getIndexInformation(Integer.toString(totalsize, 36), r.nextInt(partsPerMap), f,
            UserGroupInformation.getCurrentUser().getShortUserName());
    fs.delete(f, false);

    // oldest fails to read, or error
    boolean fnf = false;
    try {
        cache.getIndexInformation(Integer.toString(bytesPerFile, 36), r.nextInt(partsPerMap),
                new Path(p, Integer.toString(bytesPerFile)),
                UserGroupInformation.getCurrentUser().getShortUserName());
    } catch (IOException e) {
        if (e.getCause() == null || !(e.getCause() instanceof FileNotFoundException)) {
            throw e;
        } else {
            fnf = true;
        }
    }
    if (!fnf)
        fail("Failed to push out last entry");
    // should find all the other entries
    for (int i = bytesPerFile << 1; i < 1024 * 1024; i += bytesPerFile) {
        TezIndexRecord rec = cache.getIndexInformation(Integer.toString(i, 36), r.nextInt(partsPerMap),
                new Path(p, Integer.toString(i, 36)), UserGroupInformation.getCurrentUser().getShortUserName());
        checkRecord(rec, i);
    }
    TezIndexRecord rec = cache.getIndexInformation(Integer.toString(totalsize, 36), r.nextInt(partsPerMap), f,
            UserGroupInformation.getCurrentUser().getShortUserName());

    checkRecord(rec, totalsize);
}