Java tutorial
/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. * The ASF licenses this file to You under the Apache License, Version 2.0 * (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.ignite.internal.processors.hadoop.impl.v2; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FSDataInputStream; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.LocalFileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.io.ByteWritable; import org.apache.hadoop.io.Text; import org.apache.hadoop.io.Writable; import org.apache.hadoop.io.serializer.Deserializer; import org.apache.hadoop.io.serializer.Serialization; import org.apache.hadoop.io.serializer.SerializationFactory; import org.apache.hadoop.io.serializer.WritableSerialization; import org.apache.hadoop.mapred.JobConf; import org.apache.hadoop.mapred.JobContextImpl; import org.apache.hadoop.mapred.JobID; import org.apache.hadoop.mapred.TaskAttemptID; import org.apache.hadoop.mapred.TaskID; import org.apache.hadoop.mapreduce.JobContext; import org.apache.hadoop.mapreduce.JobSubmissionFiles; import org.apache.hadoop.mapreduce.MRJobConfig; import org.apache.hadoop.mapreduce.TaskType; import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.util.ReflectionUtils; import org.apache.ignite.IgniteCheckedException; import org.apache.ignite.hadoop.io.BytesWritablePartiallyRawComparator; import org.apache.ignite.hadoop.io.PartiallyRawComparator; import org.apache.ignite.hadoop.io.TextPartiallyRawComparator; import org.apache.ignite.internal.processors.hadoop.HadoopCommonUtils; import org.apache.ignite.internal.processors.hadoop.HadoopExternalSplit; import org.apache.ignite.hadoop.HadoopInputSplit; import org.apache.ignite.internal.processors.hadoop.HadoopJobEx; import org.apache.ignite.internal.processors.hadoop.HadoopJobId; import org.apache.ignite.internal.processors.hadoop.HadoopJobProperty; import org.apache.ignite.internal.processors.hadoop.HadoopPartitioner; import org.apache.ignite.internal.processors.hadoop.HadoopSerialization; import org.apache.ignite.internal.processors.hadoop.HadoopSplitWrapper; import org.apache.ignite.internal.processors.hadoop.HadoopTask; import org.apache.ignite.internal.processors.hadoop.HadoopTaskCancelledException; import org.apache.ignite.internal.processors.hadoop.HadoopTaskContext; import org.apache.ignite.internal.processors.hadoop.HadoopTaskInfo; import org.apache.ignite.internal.processors.hadoop.HadoopTaskType; import org.apache.ignite.internal.processors.hadoop.counter.HadoopCounter; import org.apache.ignite.internal.processors.hadoop.counter.HadoopCounters; import org.apache.ignite.internal.processors.hadoop.counter.HadoopCountersImpl; import org.apache.ignite.internal.processors.hadoop.impl.fs.HadoopLazyConcurrentMap; import org.apache.ignite.internal.processors.hadoop.impl.v1.HadoopV1CleanupTask; import org.apache.ignite.internal.processors.hadoop.impl.v1.HadoopV1MapTask; import org.apache.ignite.internal.processors.hadoop.impl.v1.HadoopV1Partitioner; import org.apache.ignite.internal.processors.hadoop.impl.v1.HadoopV1ReduceTask; import org.apache.ignite.internal.processors.hadoop.impl.v1.HadoopV1SetupTask; import org.apache.ignite.internal.processors.hadoop.io.PartiallyOffheapRawComparatorEx; import org.apache.ignite.internal.processors.igfs.IgfsUtils; import org.apache.ignite.internal.util.typedef.F; import org.apache.ignite.internal.util.typedef.internal.A; import org.jetbrains.annotations.Nullable; import java.io.DataInput; import java.io.File; import java.io.IOException; import java.security.PrivilegedExceptionAction; import java.util.Comparator; import java.util.HashMap; import java.util.Map; import java.util.UUID; import java.util.concurrent.Callable; import static org.apache.ignite.internal.processors.hadoop.impl.HadoopUtils.jobLocalDir; import static org.apache.ignite.internal.processors.hadoop.impl.HadoopUtils.taskLocalDir; import static org.apache.ignite.internal.processors.hadoop.impl.HadoopUtils.transformException; import static org.apache.ignite.internal.processors.hadoop.impl.HadoopUtils.unwrapSplit; import static org.apache.ignite.internal.processors.hadoop.impl.fs.HadoopFileSystemCacheUtils.FsCacheKey; import static org.apache.ignite.internal.processors.hadoop.impl.fs.HadoopFileSystemCacheUtils.createHadoopLazyConcurrentMap; import static org.apache.ignite.internal.processors.hadoop.impl.fs.HadoopFileSystemCacheUtils.fileSystemForMrUserWithCaching; import static org.apache.ignite.internal.processors.hadoop.impl.fs.HadoopParameters.PARAM_IGFS_PREFER_LOCAL_WRITES; /** * Context for task execution. */ public class HadoopV2TaskContext extends HadoopTaskContext { /** */ private static final boolean COMBINE_KEY_GROUPING_SUPPORTED; /** Lazy per-user file system cache used by the Hadoop task. */ private static final HadoopLazyConcurrentMap<FsCacheKey, FileSystem> fsMap = createHadoopLazyConcurrentMap(); /** Default partial comparator mappings. */ private static final Map<String, String> PARTIAL_COMPARATORS = new HashMap<>(); /** * This method is called with reflection upon Job finish with class loader of each task. * This will clean up all the Fs created for specific task. * Each class loader sees uses its own instance of <code>fsMap<code/> since the class loaders * are different. * * @throws IgniteCheckedException On error. */ public static void close() throws IgniteCheckedException { fsMap.close(); } /** Flag is set if new context-object code is used for running the mapper. */ private final boolean useNewMapper; /** Flag is set if new context-object code is used for running the reducer. */ private final boolean useNewReducer; /** Flag is set if new context-object code is used for running the combiner. */ private final boolean useNewCombiner; /** */ private final JobContextImpl jobCtx; /** Set if task is to cancelling. */ private volatile boolean cancelled; /** Current task. */ private volatile HadoopTask task; /** Local node ID */ private final UUID locNodeId; /** Counters for task. */ private final HadoopCounters cntrs = new HadoopCountersImpl(); static { boolean ok; try { JobContext.class.getDeclaredMethod("getCombinerKeyGroupingComparator"); ok = true; } catch (NoSuchMethodException ignore) { ok = false; } COMBINE_KEY_GROUPING_SUPPORTED = ok; PARTIAL_COMPARATORS.put(ByteWritable.class.getName(), BytesWritablePartiallyRawComparator.class.getName()); PARTIAL_COMPARATORS.put(Text.class.getName(), TextPartiallyRawComparator.class.getName()); } /** * @param taskInfo Task info. * @param job Job. * @param jobId Job ID. * @param locNodeId Local node ID. * @param jobConfDataInput DataInput for read JobConf. */ public HadoopV2TaskContext(HadoopTaskInfo taskInfo, HadoopJobEx job, HadoopJobId jobId, @Nullable UUID locNodeId, DataInput jobConfDataInput) throws IgniteCheckedException { super(taskInfo, job); this.locNodeId = locNodeId; // Before create JobConf instance we should set new context class loader. ClassLoader oldLdr = HadoopCommonUtils.setContextClassLoader(getClass().getClassLoader()); try { JobConf jobConf = new JobConf(); try { jobConf.readFields(jobConfDataInput); } catch (IOException e) { throw new IgniteCheckedException(e); } // For map-reduce jobs prefer local writes. jobConf.setBooleanIfUnset(PARAM_IGFS_PREFER_LOCAL_WRITES, true); initializePartiallyRawComparator(jobConf); jobCtx = new JobContextImpl(jobConf, new JobID(jobId.globalId().toString(), jobId.localId())); useNewMapper = jobConf.getUseNewMapper(); useNewReducer = jobConf.getUseNewReducer(); useNewCombiner = jobConf.getCombinerClass() == null; } finally { HadoopCommonUtils.restoreContextClassLoader(oldLdr); } } /** {@inheritDoc} */ @Override public <T extends HadoopCounter> T counter(String grp, String name, Class<T> cls) { return cntrs.counter(grp, name, cls); } /** {@inheritDoc} */ @Override public HadoopCounters counters() { return cntrs; } /** * Creates appropriate task from current task info. * * @return Task. */ private HadoopTask createTask() { boolean isAbort = taskInfo().type() == HadoopTaskType.ABORT; switch (taskInfo().type()) { case SETUP: return useNewMapper ? new HadoopV2SetupTask(taskInfo()) : new HadoopV1SetupTask(taskInfo()); case MAP: return useNewMapper ? new HadoopV2MapTask(taskInfo()) : new HadoopV1MapTask(taskInfo()); case REDUCE: return useNewReducer ? new HadoopV2ReduceTask(taskInfo(), true) : new HadoopV1ReduceTask(taskInfo(), true); case COMBINE: return useNewCombiner ? new HadoopV2ReduceTask(taskInfo(), false) : new HadoopV1ReduceTask(taskInfo(), false); case COMMIT: case ABORT: return useNewReducer ? new HadoopV2CleanupTask(taskInfo(), isAbort) : new HadoopV1CleanupTask(taskInfo(), isAbort); default: return null; } } /** {@inheritDoc} */ @Override public void run() throws IgniteCheckedException { ClassLoader oldLdr = HadoopCommonUtils.setContextClassLoader(jobConf().getClassLoader()); try { try { task = createTask(); } catch (Throwable e) { if (e instanceof Error) throw e; throw transformException(e); } if (cancelled) throw new HadoopTaskCancelledException("Task cancelled."); try { task.run(this); } catch (Throwable e) { if (e instanceof Error) throw e; throw transformException(e); } } finally { task = null; HadoopCommonUtils.restoreContextClassLoader(oldLdr); } } /** {@inheritDoc} */ @Override public void cancel() { cancelled = true; HadoopTask t = task; if (t != null) t.cancel(); } /** {@inheritDoc} */ @Override public void prepareTaskEnvironment() throws IgniteCheckedException { File locDir; switch (taskInfo().type()) { case MAP: case REDUCE: job().prepareTaskEnvironment(taskInfo()); locDir = taskLocalDir(job.igniteWorkDirectory(), locNodeId, taskInfo()); break; default: locDir = jobLocalDir(job.igniteWorkDirectory(), locNodeId, taskInfo().jobId()); } ClassLoader oldLdr = HadoopCommonUtils.setContextClassLoader(jobConf().getClassLoader()); try { FileSystem.get(jobConf()); LocalFileSystem locFs = FileSystem.getLocal(jobConf()); locFs.setWorkingDirectory(new Path(locDir.getAbsolutePath())); } catch (Throwable e) { if (e instanceof Error) throw (Error) e; throw transformException(e); } finally { HadoopCommonUtils.restoreContextClassLoader(oldLdr); } } /** {@inheritDoc} */ @Override public void cleanupTaskEnvironment() throws IgniteCheckedException { job().cleanupTaskEnvironment(taskInfo()); } /** * Creates Hadoop attempt ID. * * @return Attempt ID. */ public TaskAttemptID attemptId() { TaskID tid = new TaskID(jobCtx.getJobID(), taskType(taskInfo().type()), taskInfo().taskNumber()); return new TaskAttemptID(tid, taskInfo().attempt()); } /** * @param type Task type. * @return Hadoop task type. */ private TaskType taskType(HadoopTaskType type) { switch (type) { case SETUP: return TaskType.JOB_SETUP; case MAP: case COMBINE: return TaskType.MAP; case REDUCE: return TaskType.REDUCE; case COMMIT: case ABORT: return TaskType.JOB_CLEANUP; default: return null; } } /** * Gets job configuration of the task. * * @return Job configuration. */ public JobConf jobConf() { return jobCtx.getJobConf(); } /** * Gets job context of the task. * * @return Job context. */ public JobContextImpl jobContext() { return jobCtx; } /** {@inheritDoc} */ @Override public HadoopPartitioner partitioner() throws IgniteCheckedException { Class<?> partClsOld = jobConf().getClass("mapred.partitioner.class", null); if (partClsOld != null) return new HadoopV1Partitioner(jobConf().getPartitionerClass(), jobConf()); try { return new HadoopV2Partitioner(jobCtx.getPartitionerClass(), jobConf()); } catch (ClassNotFoundException e) { throw new IgniteCheckedException(e); } } /** * Gets serializer for specified class. * * @param cls Class. * @param jobConf Job configuration. * @return Appropriate serializer. */ @SuppressWarnings("unchecked") private HadoopSerialization getSerialization(Class<?> cls, Configuration jobConf) throws IgniteCheckedException { A.notNull(cls, "cls"); SerializationFactory factory = new SerializationFactory(jobConf); Serialization<?> serialization = factory.getSerialization(cls); if (serialization == null) throw new IgniteCheckedException("Failed to find serialization for: " + cls.getName()); if (serialization.getClass() == WritableSerialization.class) return new HadoopWritableSerialization((Class<? extends Writable>) cls); return new HadoopSerializationWrapper(serialization, cls); } /** {@inheritDoc} */ @Override public HadoopSerialization keySerialization() throws IgniteCheckedException { return getSerialization(jobCtx.getMapOutputKeyClass(), jobConf()); } /** {@inheritDoc} */ @Override public HadoopSerialization valueSerialization() throws IgniteCheckedException { return getSerialization(jobCtx.getMapOutputValueClass(), jobConf()); } /** {@inheritDoc} */ @SuppressWarnings("unchecked") @Override public Comparator<Object> sortComparator() { return (Comparator<Object>) jobCtx.getSortComparator(); } /** {@inheritDoc} */ @SuppressWarnings("unchecked") @Override public PartiallyOffheapRawComparatorEx<Object> partialRawSortComparator() { Class cls = jobCtx.getJobConf().getClass(HadoopJobProperty.JOB_PARTIALLY_RAW_COMPARATOR.propertyName(), null); if (cls == null) return null; Object res = ReflectionUtils.newInstance(cls, jobConf()); if (res instanceof PartiallyOffheapRawComparatorEx) return (PartiallyOffheapRawComparatorEx) res; else return new HadoopV2DelegatingPartiallyOffheapRawComparator<>((PartiallyRawComparator) res); } /** {@inheritDoc} */ @SuppressWarnings("unchecked") @Override public Comparator<Object> groupComparator() { Comparator<?> res; switch (taskInfo().type()) { case COMBINE: res = COMBINE_KEY_GROUPING_SUPPORTED ? jobContext().getCombinerKeyGroupingComparator() : jobContext().getGroupingComparator(); break; case REDUCE: res = jobContext().getGroupingComparator(); break; default: return null; } if (res != null && res.getClass() != sortComparator().getClass()) return (Comparator<Object>) res; return null; } /** * @param split Split. * @return Native Hadoop split. * @throws IgniteCheckedException if failed. */ @SuppressWarnings("unchecked") public Object getNativeSplit(HadoopInputSplit split) throws IgniteCheckedException { if (split instanceof HadoopExternalSplit) return readExternalSplit((HadoopExternalSplit) split); if (split instanceof HadoopSplitWrapper) return unwrapSplit((HadoopSplitWrapper) split); throw new IllegalStateException("Unknown split: " + split); } /** * @param split External split. * @return Native input split. * @throws IgniteCheckedException If failed. */ @SuppressWarnings("unchecked") private Object readExternalSplit(HadoopExternalSplit split) throws IgniteCheckedException { Path jobDir = new Path(jobConf().get(MRJobConfig.MAPREDUCE_JOB_DIR)); FileSystem fs; try { fs = fileSystemForMrUserWithCaching(jobDir.toUri(), jobConf(), fsMap); } catch (IOException e) { throw new IgniteCheckedException(e); } try (FSDataInputStream in = fs.open(JobSubmissionFiles.getJobSplitFile(jobDir))) { in.seek(split.offset()); String clsName = Text.readString(in); Class<?> cls = jobConf().getClassByName(clsName); assert cls != null; Serialization serialization = new SerializationFactory(jobConf()).getSerialization(cls); Deserializer deserializer = serialization.getDeserializer(cls); deserializer.open(in); Object res = deserializer.deserialize(null); deserializer.close(); assert res != null; return res; } catch (IOException | ClassNotFoundException e) { throw new IgniteCheckedException(e); } } /** {@inheritDoc} */ @Override public <T> T runAsJobOwner(final Callable<T> c) throws IgniteCheckedException { String user = job.info().user(); user = IgfsUtils.fixUserName(user); assert user != null; String ugiUser; try { UserGroupInformation currUser = UserGroupInformation.getCurrentUser(); assert currUser != null; ugiUser = currUser.getShortUserName(); } catch (IOException ioe) { throw new IgniteCheckedException(ioe); } try { if (F.eq(user, ugiUser)) // if current UGI context user is the same, do direct call: return c.call(); else { UserGroupInformation ugi = UserGroupInformation.getBestUGI(null, user); return ugi.doAs(new PrivilegedExceptionAction<T>() { @Override public T run() throws Exception { return c.call(); } }); } } catch (Exception e) { throw new IgniteCheckedException(e); } } /** * Try initializing partially raw comparator for job. * * @param conf Configuration. */ private void initializePartiallyRawComparator(JobConf conf) { String clsName = conf.get(HadoopJobProperty.JOB_PARTIALLY_RAW_COMPARATOR.propertyName(), null); if (clsName == null) { Class keyCls = conf.getMapOutputKeyClass(); while (keyCls != null) { clsName = PARTIAL_COMPARATORS.get(keyCls.getName()); if (clsName != null) { conf.set(HadoopJobProperty.JOB_PARTIALLY_RAW_COMPARATOR.propertyName(), clsName); break; } keyCls = keyCls.getSuperclass(); } } } }