Example usage for org.apache.hadoop.mapred JobConf setClassLoader

List of usage examples for org.apache.hadoop.mapred JobConf setClassLoader

  1. HOME
  2. Java
  3. org.apache.hadoop
  4. org.apache.hadoop.mapred.*
  5. JobConf
  6. setClassLoader

Introduction

In this page you can find the example usage for org.apache.hadoop.mapred JobConf setClassLoader.

Prototype

public void setClassLoader(ClassLoader classLoader)

Source Link

Document

Set the class loader that will be used to load the various objects.

Usage

From source file:org.apache.asterix.external.adapter.factory.HDFSAdapterFactory.java

License:Apache License

public static JobConf configureJobConf(Map<String, String> configuration) throws Exception {
    JobConf conf = new JobConf();
    String formatClassName = (String) formatClassNames
            .get(((String) configuration.get(KEY_INPUT_FORMAT)).trim());
    String localShortCircuitSocketPath = (String) configuration.get(KEY_LOCAL_SOCKET_PATH);
    if (formatClassName == null) {
        formatClassName = ((String) configuration.get(KEY_INPUT_FORMAT)).trim();
    }//from  ww  w.  j  a  v  a2s. c  o m
    conf.set(KEY_HADOOP_FILESYSTEM_URI, ((String) configuration.get(KEY_HDFS_URL)).trim());
    conf.set(KEY_HADOOP_FILESYSTEM_CLASS, CLASS_NAME_HDFS_FILESYSTEM);
    conf.setClassLoader(HDFSAdapter.class.getClassLoader());
    conf.set(KEY_HADOOP_INPUT_DIR, ((String) configuration.get(KEY_PATH)).trim());
    conf.set(KEY_HADOOP_INPUT_FORMAT, formatClassName);

    // Enable local short circuit reads if user supplied the parameters
    if (localShortCircuitSocketPath != null) {
        conf.set(KEY_HADOOP_SHORT_CIRCUIT, "true");
        conf.set(KEY_HADOOP_SOCKET_PATH, localShortCircuitSocketPath.trim());
    }
    return conf;
}

From source file:org.apache.asterix.external.util.HDFSUtils.java

License:Apache License

public static JobConf configureHDFSJobConf(Map<String, String> configuration) {
    JobConf conf = new JobConf();

    String localShortCircuitSocketPath = configuration.get(ExternalDataConstants.KEY_LOCAL_SOCKET_PATH);
    String formatClassName = HDFSUtils.getInputFormatClassName(configuration);
    conf.set(ExternalDataConstants.KEY_HADOOP_FILESYSTEM_URI,
            configuration.get(ExternalDataConstants.KEY_HDFS_URL).trim());
    conf.set(ExternalDataConstants.KEY_HADOOP_FILESYSTEM_CLASS,
            ExternalDataConstants.CLASS_NAME_HDFS_FILESYSTEM);
    conf.setClassLoader(HDFSInputStream.class.getClassLoader());
    conf.set(ExternalDataConstants.KEY_HADOOP_INPUT_DIR,
            configuration.get(ExternalDataConstants.KEY_PATH).trim());
    conf.set(ExternalDataConstants.KEY_HADOOP_INPUT_FORMAT, formatClassName);

    // Enable local short circuit reads if user supplied the parameters
    if (localShortCircuitSocketPath != null) {
        conf.set(ExternalDataConstants.KEY_HADOOP_SHORT_CIRCUIT, "true");
        conf.set(ExternalDataConstants.KEY_HADOOP_SOCKET_PATH, localShortCircuitSocketPath.trim());
    }//from   ww  w  . j a  v  a2  s.c  o m
    return conf;
}

From source file:org.apache.asterix.test.runtime.HDFSCluster.java

License:Apache License

private static JobConf configureJobConf() throws Exception {
    JobConf conf = new JobConf();
    String hdfsUrl = "hdfs://127.0.0.1:31888";
    String hdfsPath = "/asterix/extrasmalltweets.txt";
    conf.set("fs.default.name", hdfsUrl);
    conf.set("fs.hdfs.impl", "org.apache.hadoop.hdfs.DistributedFileSystem");
    conf.setClassLoader(GenericAdapter.class.getClassLoader());
    conf.set("mapred.input.dir", hdfsPath);
    conf.set("mapred.input.format.class", "org.apache.hadoop.mapred.TextInputFormat");
    return conf;/*from  www. j a va2s .  com*/
}

From source file:org.apache.hyracks.dataflow.hadoop.HadoopReadOperatorDescriptor.java

License:Apache License

@SuppressWarnings("deprecation")
@Override//w  ww.j a  v a2 s  .  c o  m
public IOperatorNodePushable createPushRuntime(final IHyracksTaskContext ctx,
        final IRecordDescriptorProvider recordDescProvider, final int partition, int nPartitions)
        throws HyracksDataException {
    return new AbstractUnaryOutputSourceOperatorNodePushable() {
        @Override
        public void initialize() throws HyracksDataException {
            try {
                JobConf conf = DatatypeHelper.map2JobConf((HashMap) jobConfMap);
                Thread.currentThread().setContextClassLoader(this.getClass().getClassLoader());
                conf.setClassLoader(this.getClass().getClassLoader());
                RecordReader hadoopRecordReader;
                Object key;
                Object value;
                Object[] splits = inputSplitsProxy.toInputSplits(conf);
                Object inputSplit = splits[partition];

                if (conf.getUseNewMapper()) {
                    JobContext context = new ContextFactory().createJobContext(conf);
                    org.apache.hadoop.mapreduce.InputFormat inputFormat = (org.apache.hadoop.mapreduce.InputFormat) ReflectionUtils
                            .newInstance(context.getInputFormatClass(), conf);
                    TaskAttemptContext taskAttemptContext = new ContextFactory().createContext(jobConf, null);
                    hadoopRecordReader = (RecordReader) inputFormat.createRecordReader(
                            (org.apache.hadoop.mapreduce.InputSplit) inputSplit, taskAttemptContext);
                } else {
                    Class inputFormatClass = conf.getInputFormat().getClass();
                    InputFormat inputFormat = (InputFormat) ReflectionUtils.newInstance(inputFormatClass, conf);
                    hadoopRecordReader = (RecordReader) inputFormat.getRecordReader(
                            (org.apache.hadoop.mapred.InputSplit) inputSplit, conf, createReporter());
                }

                Class inputKeyClass;
                Class inputValueClass;
                if (hadoopRecordReader instanceof SequenceFileRecordReader) {
                    inputKeyClass = ((SequenceFileRecordReader) hadoopRecordReader).getKeyClass();
                    inputValueClass = ((SequenceFileRecordReader) hadoopRecordReader).getValueClass();
                } else {
                    inputKeyClass = hadoopRecordReader.createKey().getClass();
                    inputValueClass = hadoopRecordReader.createValue().getClass();
                }

                key = hadoopRecordReader.createKey();
                value = hadoopRecordReader.createValue();
                FrameTupleAppender appender = new FrameTupleAppender(new VSizeFrame(ctx));
                RecordDescriptor outputRecordDescriptor = DatatypeHelper.createKeyValueRecordDescriptor(
                        (Class<? extends Writable>) hadoopRecordReader.createKey().getClass(),
                        (Class<? extends Writable>) hadoopRecordReader.createValue().getClass());
                int nFields = outputRecordDescriptor.getFieldCount();
                ArrayTupleBuilder tb = new ArrayTupleBuilder(nFields);
                writer.open();
                try {
                    while (hadoopRecordReader.next(key, value)) {
                        tb.reset();
                        switch (nFields) {
                        case 2:
                            tb.addField(outputRecordDescriptor.getFields()[0], key);
                        case 1:
                            tb.addField(outputRecordDescriptor.getFields()[1], value);
                        }
                        FrameUtils.appendToWriter(writer, appender, tb.getFieldEndOffsets(), tb.getByteArray(),
                                0, tb.getSize());
                    }
                    appender.flush(writer, true);
                } catch (Exception e) {
                    writer.fail();
                    throw new HyracksDataException(e);
                } finally {
                    writer.close();
                }
                hadoopRecordReader.close();
            } catch (InstantiationException e) {
                throw new HyracksDataException(e);
            } catch (IllegalAccessException e) {
                throw new HyracksDataException(e);
            } catch (ClassNotFoundException e) {
                throw new HyracksDataException(e);
            } catch (InterruptedException e) {
                throw new HyracksDataException(e);
            } catch (IOException e) {
                throw new HyracksDataException(e);
            }
        }
    };
}

From source file:org.apache.hyracks.hdfs.dataflow.HDFSReadOperatorDescriptor.java

License:Apache License

@Override
public IOperatorNodePushable createPushRuntime(final IHyracksTaskContext ctx,
        IRecordDescriptorProvider recordDescProvider, final int partition, final int nPartitions)
        throws HyracksDataException {
    final InputSplit[] inputSplits = splitsFactory.getSplits();

    return new AbstractUnaryOutputSourceOperatorNodePushable() {
        private String nodeName = ctx.getJobletContext().getApplicationContext().getNodeId();

        @SuppressWarnings("unchecked")
        @Override/*from  w  w w. jav  a 2 s .c o m*/
        public void initialize() throws HyracksDataException {
            ClassLoader ctxCL = Thread.currentThread().getContextClassLoader();
            try {
                writer.open();
                Thread.currentThread().setContextClassLoader(ctx.getJobletContext().getClassLoader());
                JobConf conf = confFactory.getConf();
                conf.setClassLoader(ctx.getJobletContext().getClassLoader());
                IKeyValueParser parser = tupleParserFactory.createKeyValueParser(ctx);
                try {
                    parser.open(writer);
                    InputFormat inputFormat = conf.getInputFormat();
                    for (int i = 0; i < inputSplits.length; i++) {
                        /**
                         * read all the partitions scheduled to the current node
                         */
                        if (scheduledLocations[i].equals(nodeName)) {
                            /**
                             * pick an unread split to read
                             * synchronize among simultaneous partitions in the same machine
                             */
                            synchronized (executed) {
                                if (executed[i] == false) {
                                    executed[i] = true;
                                } else {
                                    continue;
                                }
                            }

                            /**
                             * read the split
                             */
                            RecordReader reader = inputFormat.getRecordReader(inputSplits[i], conf,
                                    Reporter.NULL);
                            Object key = reader.createKey();
                            Object value = reader.createValue();
                            while (reader.next(key, value) == true) {
                                parser.parse(key, value, writer, inputSplits[i].toString());
                            }
                        }
                    }
                } finally {
                    parser.close(writer);
                }
            } catch (Throwable th) {
                writer.fail();
                throw new HyracksDataException(th);
            } finally {
                writer.close();
                Thread.currentThread().setContextClassLoader(ctxCL);
            }
        }
    };
}

From source file:org.mitre.ccv.mapred.CalculateCompositionVectors.java

License:Open Source License

@Override
public int run(String[] args) throws Exception {
    JobConf conf = new JobConf(getConf());
    int start = CalculateKmerCounts.DEFAULT_START;
    int end = CalculateKmerCounts.DEFAULT_END;
    boolean cleanLogs = false;

    // @TODO: use commons getopts
    List<String> other_args = new ArrayList<String>();
    for (int i = 0; i < args.length; ++i) {
        try {/* ww  w .  java 2  s  . c  om*/
            if ("-m".equals(args[i])) {
                conf.setNumMapTasks(Integer.parseInt(args[++i]));
            } else if ("-r".equals(args[i])) {
                conf.setNumReduceTasks(Integer.parseInt(args[++i]));
            } else if ("-s".equals(args[i])) {
                start = Integer.parseInt(args[++i]);
            } else if ("-e".equals(args[i])) {
                end = Integer.parseInt(args[++i]);
            } else if ("-c".equals(args[i])) {
                cleanLogs = true;
            } else if ("-libjars".equals(args[i])) {
                conf.set("tmpjars", FileUtils.validateFiles(args[++i], conf));

                URL[] libjars = FileUtils.getLibJars(conf);
                if (libjars != null && libjars.length > 0) {
                    // Add libjars to client/tasks classpath
                    conf.setClassLoader(new URLClassLoader(libjars, conf.getClassLoader()));
                    // Adds libjars to our classpath
                    Thread.currentThread().setContextClassLoader(
                            new URLClassLoader(libjars, Thread.currentThread().getContextClassLoader()));
                }
            } else {
                other_args.add(args[i]);
            }
        } catch (NumberFormatException except) {
            System.out.println("ERROR: Integer expected instead of " + args[i]);
            return printUsage();
        } catch (ArrayIndexOutOfBoundsException except) {
            System.out.println("ERROR: Required parameter missing from " + args[i - 1]);
            return printUsage();
        }
    }
    // Make sure there are exactly 2 parameters left.
    if (other_args.size() != 2) {
        System.out.println("ERROR: Wrong number of parameters: " + other_args.size() + " instead of 2.");

        return printUsage();
    }
    return initJob(conf, start, end, other_args.get(0), other_args.get(1), cleanLogs);
}

From source file:org.mitre.ccv.mapred.CalculateCosineDistanceMatrix.java

License:Open Source License

@Override
public int run(String[] args) throws Exception {
    JobConf conf = new JobConf(getConf());

    String phylip = null;//from  w w  w . ja va  2s.c  om
    String packedRow = null;
    int fractionDigits = 6;

    //String userJarLocation = "/path/to/jar";
    //conf.setJar(userJarLocation); //were conf is the JobConf object
    ArrayList<String> other_args = new ArrayList<String>();
    for (int i = 0; i < args.length; ++i) {
        try {
            if ("-m".equals(args[i])) {
                conf.setNumMapTasks(Integer.parseInt(args[++i]));
            } else if ("-r".equals(args[i])) {
                conf.setNumReduceTasks(Integer.parseInt(args[++i]));
            } else if ("-D".equals(args[i])) {
                String[] props = args[++i].split("=");
                conf.set(props[0], props[1]);
            } else if ("-libjars".equals(args[i])) {
                conf.set("tmpjars", FileUtils.validateFiles(args[++i], conf));

                URL[] libjars = FileUtils.getLibJars(conf);
                if (libjars != null && libjars.length > 0) {
                    // Add libjars to client/tasks classpath
                    conf.setClassLoader(new URLClassLoader(libjars, conf.getClassLoader()));
                    // Adds libjars to our classpath
                    Thread.currentThread().setContextClassLoader(
                            new URLClassLoader(libjars, Thread.currentThread().getContextClassLoader()));
                }
            } else if ("-phylip".equals(args[i])) {
                phylip = args[++i];
            } else if ("-packedRow".equals(args[i])) {
                packedRow = args[++i];
            } else if ("-digits".equals(args[i])) {
                fractionDigits = Integer.parseInt(args[++i]);
            } else {
                other_args.add(args[i]);
            }
        } catch (NumberFormatException except) {
            System.out.println("ERROR: Integer expected instead of " + args[i]);
            return printUsage();
        } catch (ArrayIndexOutOfBoundsException except) {
            System.out.println("ERROR: Required parameter missing from " + args[i - 1]);
            return printUsage();
        }
    }

    boolean writeMatrix = (phylip != null || packedRow != null) ? true : false;

    // Make sure there are exactly 3 parameters left.
    if ((other_args.size() != 2 && !writeMatrix) || (other_args.size() == 0 && writeMatrix)) {
        System.out.println("ERROR: Wrong number of parameters: " + other_args.size() + " instead of 2.");

        return printUsage();
    }

    int ret = 0;
    if (other_args.size() == 2) {
        ret = this.initJob(conf, other_args.get(0), other_args.get(1));
    }
    // check writing out in Phylip format
    if (ret == 0 && other_args.size() == 1 && phylip != null) {
        printPhylipSquare(conf, other_args.get(0), phylip, fractionDigits);
    } else if (ret == 0 && other_args.size() == 2 && phylip != null) {
        printPhylipSquare(conf, other_args.get(1), phylip, fractionDigits);
    }

    // check writing out in row packed order
    if (ret == 0 && other_args.size() == 1 && packedRow != null) {
        printRowMajorMatrix(conf, other_args.get(0), packedRow, fractionDigits);
    } else if (ret == 0 && other_args.size() == 2 && packedRow != null) {
        printRowMajorMatrix(conf, other_args.get(1), packedRow, fractionDigits);
    }

    return ret;
}

From source file:org.mitre.ccv.mapred.CalculateKmerCounts.java

License:Open Source License

@Override
public int run(String[] args) throws Exception {
    JobConf conf = new JobConf(getConf());
    int start = DEFAULT_START;
    int end = DEFAULT_END;

    // @TODO: use commons getopts
    List<String> other_args = new ArrayList<String>();
    for (int i = 0; i < args.length; ++i) {
        try {// w  ww.  j  a va  2 s .co m
            if ("-m".equals(args[i])) {
                conf.setNumMapTasks(Integer.parseInt(args[++i]));
            } else if ("-r".equals(args[i])) {
                conf.setNumReduceTasks(Integer.parseInt(args[++i]));
            } else if ("-s".equals(args[i])) {
                start = Integer.parseInt(args[++i]);
            } else if ("-e".equals(args[i])) {
                end = Integer.parseInt(args[++i]);
            } else if ("-f".equals(args[i])) {
                conf.get(FAST_MAP, "true");
            } else if ("-libjars".equals(args[i])) {
                conf.set("tmpjars", FileUtils.validateFiles(args[++i], conf));

                URL[] libjars = FileUtils.getLibJars(conf);
                if (libjars != null && libjars.length > 0) {
                    // Add libjars to client/tasks classpath
                    conf.setClassLoader(new URLClassLoader(libjars, conf.getClassLoader()));
                    // Adds libjars to our classpath
                    Thread.currentThread().setContextClassLoader(
                            new URLClassLoader(libjars, Thread.currentThread().getContextClassLoader()));
                }
            } else {
                other_args.add(args[i]);
            }
        } catch (NumberFormatException except) {
            System.out.println("ERROR: Integer expected instead of " + args[i]);
            return printUsage();
        } catch (ArrayIndexOutOfBoundsException except) {
            System.out.println("ERROR: Required parameter missing from " + args[i - 1]);
            return printUsage();
        }
    }
    // Make sure there are exactly 2 parameters left.
    if (other_args.size() != 2) {
        System.out.println("ERROR: Wrong number of parameters: " + other_args.size() + " instead of 2.");

        return printUsage();
    }

    return initJob(conf, start, end, other_args.get(0), other_args.get(1));

}

From source file:org.mitre.ccv.mapred.CalculateKmerPiValues.java

License:Open Source License

@Override
public int run(String[] args) throws Exception {
    JobConf conf = new JobConf(getConf());
    boolean cleanLogs = false;
    Integer start = CalculateKmerCounts.DEFAULT_START;
    Integer end = CalculateKmerCounts.DEFAULT_END;

    List<String> other_args = new ArrayList<String>();
    for (int i = 0; i < args.length; ++i) {
        try {// w w w  .j av  a2 s  .c  om
            if ("-m".equals(args[i])) {
                conf.setNumMapTasks(Integer.parseInt(args[++i]));
            } else if ("-r".equals(args[i])) {
                conf.setNumReduceTasks(Integer.parseInt(args[++i]));
            } else if ("-c".equals(args[i])) {
                cleanLogs = true;
            } else if ("-s".equals(args[i])) {
                start = Integer.parseInt(args[++i]);
            } else if ("-e".equals(args[i])) {
                end = Integer.parseInt(args[++i]);
            } else if ("-libjars".equals(args[i])) {
                conf.set("tmpjars", FileUtils.validateFiles(args[++i], conf));

                URL[] libjars = FileUtils.getLibJars(conf);
                if (libjars != null && libjars.length > 0) {
                    // Add libjars to client/tasks classpath
                    conf.setClassLoader(new URLClassLoader(libjars, conf.getClassLoader()));
                    // Adds libjars to our classpath
                    Thread.currentThread().setContextClassLoader(
                            new URLClassLoader(libjars, Thread.currentThread().getContextClassLoader()));
                }
            } else {
                other_args.add(args[i]);
            }
        } catch (NumberFormatException except) {
            System.out.println("ERROR: Integer expected instead of " + args[i]);
            return printUsage();
        } catch (ArrayIndexOutOfBoundsException except) {
            System.out.println("ERROR: Required parameter missing from " + args[i - 1]);
            return printUsage();
        }
    }
    // Make sure there are exactly 2 parameters left.
    if (other_args.size() != 2) {
        System.out.println("ERROR: Wrong number of parameters: " + other_args.size() + " instead of 2.");
        return printUsage();
    }

    return initJob(conf, start, end, other_args.get(0), other_args.get(1), cleanLogs);
}

From source file:org.mitre.ccv.mapred.CalculateKmerProbabilities.java

License:Open Source License

@Override
public int run(String[] args) throws Exception {
    JobConf conf = new JobConf(getConf());
    boolean cleanLogs = false;
    int start = CalculateKmerCounts.DEFAULT_START;
    int end = CalculateKmerCounts.DEFAULT_END;
    int length = -1;

    // @TODO: use commons getopts
    List<String> other_args = new ArrayList<String>();
    for (int i = 0; i < args.length; ++i) {
        try {//from   ww w . j av a  2s .  c om
            if ("-m".equals(args[i])) {
                conf.setNumMapTasks(Integer.parseInt(args[++i]));
            } else if ("-r".equals(args[i])) {
                conf.setNumReduceTasks(Integer.parseInt(args[++i]));
            } else if ("-s".equals(args[i])) {
                start = Integer.parseInt(args[++i]);
            } else if ("-e".equals(args[i])) {
                end = Integer.parseInt(args[++i]);
            } else if ("-c".equals(args[i])) {
                cleanLogs = true;
            } else if ("-l".equals(args[i])) {
                length = Integer.parseInt(args[++i]);
            } else if ("-libjars".equals(args[i])) {
                conf.set("tmpjars", FileUtils.validateFiles(args[++i], conf));

                URL[] libjars = FileUtils.getLibJars(conf);
                if (libjars != null && libjars.length > 0) {
                    // Add libjars to client/tasks classpath
                    conf.setClassLoader(new URLClassLoader(libjars, conf.getClassLoader()));
                    // Adds libjars to our classpath
                    Thread.currentThread().setContextClassLoader(
                            new URLClassLoader(libjars, Thread.currentThread().getContextClassLoader()));
                }
            } else {
                other_args.add(args[i]);
            }
        } catch (NumberFormatException except) {
            System.out.println("ERROR: Integer expected instead of " + args[i]);
            return printUsage();
        } catch (ArrayIndexOutOfBoundsException except) {
            System.out.println("ERROR: Required parameter missing from " + args[i - 1]);
            return printUsage();
        }
    }
    // Make sure there are exactly 2 parameters left.
    if (other_args.size() != 2) {
        System.out.println("ERROR: Wrong number of parameters: " + other_args.size() + " instead of 2.");
        return printUsage();
    }

    if (length <= 0) {
        System.out.println("ERROR: Requires total length of sequence to be > 0");
        return printUsage();
    }

    //return initJob(conf, inTable, sb.toString().trim(), new Path(other_args.get(1)));
    return initJob(conf, start, end, length, other_args.get(0), other_args.get(1), cleanLogs);

}