Example usage for org.apache.hadoop.fs Options Options

Introduction

In this page you can find the example usage for org.apache.hadoop.fs Options Options.

Prototype

Options

Source Link

Usage

From source file:finderbots.recommenders.hadoop.ActionSplitterJob.java

License:Apache License

@Override
public int run(String[] args) throws Exception {
    options = new Options();
    CmdLineParser parser = new CmdLineParser(options);
    String s = options.toString();

    try {/*  w ww .  j  a  va 2 s .  c  o  m*/
        parser.parseArgument(args);
    } catch (CmdLineException e) {
        System.err.println(e.getMessage());
        parser.printUsage(System.err);
        return -1;
    }

    this.userIndex = HashBiMap.create();
    this.itemIndex = HashBiMap.create();

    // split into actions and store in subdirs
    // create indexes for users and items
    Path inputPath = new Path(options.getInputDir());
    FileSystem fs = inputPath.getFileSystem(new JobConf());
    Path outputPath = new Path(options.getOutputDir());
    // todo: can put this into m/r if it helps speed up
    split(inputPath, outputPath);// split into actions and store in subdirs

    Path indexesPath = new Path(options.getIndexDir());
    Path userIndexPath = new Path(options.getIndexDir(), options.getUserIndexFile());
    Path itemIndexPath = new Path(options.getIndexDir(), options.getItemIndexFile());
    if (fs.exists(userIndexPath))
        fs.delete(userIndexPath, false);//delete file only!
    if (fs.exists(itemIndexPath))
        fs.delete(itemIndexPath, false);//delete file only!
    // get the size of the matrices and put them where the calling job
    // can find them
    HadoopUtil.writeInt(getNumberOfUsers(), new Path(indexesPath, options.getNumUsersFile()), getConf());
    HadoopUtil.writeInt(getNumberOfItems(), new Path(indexesPath, options.getNumItemsFile()), getConf());
    //write the indexes to tsv files
    saveIndexes(indexesPath);
    return 0;
}

---