Java tutorial
/** * Copyright 2014-2015 SHAF-WORK * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.shaf.core.util; import java.io.DataInput; import java.io.DataOutput; import java.io.IOException; import java.lang.reflect.Array; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.io.BooleanWritable; import org.apache.hadoop.io.ByteWritable; import org.apache.hadoop.io.DoubleWritable; import org.apache.hadoop.io.FloatWritable; import org.apache.hadoop.io.IntWritable; import org.apache.hadoop.io.LongWritable; import org.apache.hadoop.io.ShortWritable; import org.apache.hadoop.io.Text; import org.apache.hadoop.io.Writable; /** * Utility function for Input/Output operations. * * @author Mykola Galushka */ public class IOUtils { /** * Returns the path to the Hadoop home directory. * * @return the path to the Hadoop home directory or {@code null} if Hadoop * environment is not available. * @throws IOException * if an I/O error occurs. */ public static final Path getHadoopHomePath() throws IOException { if (System.getenv().containsKey("HADOOP_HOME")) { Path path = new Path(System.getenv().get("HADOOP_HOME")); if (getLocalFileSystem().exists(path)) { return path; } } return null; } /** * Returns the path to the Hadoop configuration directory. * * @return the path to the Hadoop configuration directory or {@code null} if * Hadoop environment is not available. * @throws IOException * if an I/O error occurs. */ public static final Path getHadoopConfigPath() throws IOException { if (System.getenv().containsKey("HADOOP_CONF_DIR")) { Path path = new Path(System.getenv().get("HADOOP_CONF_DIR")); if (getLocalFileSystem().exists(path)) { return path; } } if (getHadoopHomePath() != null) { Path path = new Path(getHadoopHomePath(), "etc/hadoop"); if (getLocalFileSystem().exists(path)) { return path; } } return null; } /** * Returns the Hadoop configuration. * * @return the Hadoop configuration or {@code null} if Hadoop environment is * not available. * @throws IOException * if an I/O error occurs. */ public static final Configuration getHadoopConfig() throws IOException { Path path = getHadoopConfigPath(); if (path != null) { Configuration config = new Configuration(); config.addResource(new Path(path, "core-site.xml")); config.addResource(new Path(path, "hdfs-site.xml")); return config; } else { return null; } } /** * Tests if Hadoop environment is available. * * @return {@code true} if Hadoop environment is available and {@code false} * otherwise. * @throws IOException * if an I/O error occurs. */ public static final boolean isHadoopAvailable() throws IOException { return getHadoopConfig() != null; } /** * Returns the {@link FileSystem}. * * @return the file system. * @throws IOException * if an I/O error occurs. */ public static final FileSystem getFileSystem() throws IOException { if (isHadoopAvailable()) { return FileSystem.newInstance(getHadoopConfig()); } else { return FileSystem.newInstanceLocal(new Configuration()).getRaw(); } } /** * Returns the local {@link FileSystem}. * * @return the local file system. * @throws IOException * if an I/O error occurs. */ public static final FileSystem getLocalFileSystem() throws IOException { return FileSystem.newInstanceLocal(new Configuration()).getRaw(); } /** * Normalizes the {@link Path}. * * @param path * the path to normalize. * @return the normalized path. * @throws IOException * if an I/O error occurs. */ public static final Path normalizePath(final Path path) throws IOException { return Path.getPathWithoutSchemeAndAuthority(path); } /** * Normalizes the path. * * @param path * the path to normalize. * @return the normalized path. * @throws IOException * if an I/O error occurs. */ public static final Path normalizePath(final String path) throws IOException { return normalizePath(new Path(path)); } /** * Merges paths. * * @param paths * the paths to merge. * @return the merged path. * @throws IOException * if an I/O error occurs. */ public static final Path mergePath(final Path... paths) throws IOException { try { if (paths.length == 0) { throw new IllegalArgumentException("paths.length=0"); } if (paths.length == 1) { return paths[0]; } Path buf = null; for (int i = 0; i < paths.length; i++) { if (paths[i] == null) { throw new NullPointerException("paths[" + i + "]"); } Path norm = normalizePath(paths[i]); if (buf == null) { buf = norm; } else { if (norm.isAbsolute()) { norm = new Path(StringUtils.trim(norm.toString(), Path.SEPARATOR)); } buf = new Path(buf, norm); } } return buf; } catch (NullPointerException | IllegalArgumentException exc) { throw new IOException("Invalid input.", exc); } } /** * Merges paths. * * @param paths * the paths to merge. * @return the merged path. * @throws IOException * if an I/O error occurs. */ public static final Path mergePath(final String... paths) throws IOException { try { if (paths.length == 0) { throw new IllegalArgumentException("paths.length=0"); } Path buf = null; for (int i = 0; i < paths.length; i++) { if (paths[i] == null) { throw new NullPointerException("paths[" + i + "]"); } Path norm = normalizePath(paths[i]); if (buf == null) { buf = norm; } else { if (norm.isAbsolute()) { norm = new Path(StringUtils.trim(norm.toString(), Path.SEPARATOR)); } buf = new Path(buf, norm); } } return buf; } catch (NullPointerException | IllegalArgumentException exc) { throw new IOException("Invalid input.", exc); } } /** * Writes an {@link Object} to the {@link DataOutput}. * * @param obj * the object to write. * @param out * the data output stream. * @throws IOException * if I/O error occurs. */ public static final void writeObject(Object obj, DataOutput out) throws IOException { try { if (obj == null) { throw new IOException("Writing object is not defined: null."); } else if (ClassUtils.isBoolean(obj)) { (new BooleanWritable((boolean) obj)).write(out); } else if (ClassUtils.isByte(obj)) { (new ByteWritable((byte) obj)).write(out); } else if (ClassUtils.isShort(obj)) { (new ShortWritable((short) obj)).write(out); } else if (ClassUtils.isInteger(obj)) { (new IntWritable((int) obj)).write(out); } else if (ClassUtils.isLong(obj)) { (new LongWritable((long) obj)).write(out); } else if (ClassUtils.isFloat(obj)) { (new FloatWritable((float) obj)).write(out); } else if (ClassUtils.isDouble(obj)) { (new DoubleWritable((double) obj)).write(out); } else if (ClassUtils.isString(obj)) { Text.writeString(out, (String) obj); } else if (ClassUtils.isEnum(obj)) { (new IntWritable(((Enum<?>) obj).ordinal())).write(out); } else if (ClassUtils.isArray(obj)) { int length = Array.getLength(obj); writeObject(length, out); for (int j = 0; j < length; j++) { writeObject(Array.get(obj, j), out); } } else { ((Writable) obj).write(out); } } catch (IllegalArgumentException exc) { throw new IOException(exc); } } /** * Reads an {@link Object} of the specified type from the {@link DataInput}. * * @param cls * the type of the reading object. * @param in * the data input stream. * @return the read object. * @throws IOException * if I/O error occurs. */ public static final Object readObject(Class<?> cls, DataInput in) throws IOException { try { if (cls == null) { throw new IOException("Reading class is not defined: null."); } else if (ClassUtils.isBoolean(cls)) { BooleanWritable obj = new BooleanWritable(); obj.readFields(in); return obj.get(); } else if (ClassUtils.isByte(cls)) { ByteWritable obj = new ByteWritable(); obj.readFields(in); return obj.get(); } else if (ClassUtils.isShort(cls)) { ShortWritable obj = new ShortWritable(); obj.readFields(in); return obj.get(); } else if (ClassUtils.isInteger(cls)) { IntWritable obj = new IntWritable(); obj.readFields(in); return obj.get(); } else if (ClassUtils.isLong(cls)) { LongWritable obj = new LongWritable(); obj.readFields(in); return obj.get(); } else if (ClassUtils.isFloat(cls)) { FloatWritable obj = new FloatWritable(); obj.readFields(in); return obj.get(); } else if (ClassUtils.isDouble(cls)) { DoubleWritable obj = new DoubleWritable(); obj.readFields(in); return obj.get(); } else if (ClassUtils.isString(cls)) { return Text.readString(in); } else if (ClassUtils.isEnum(cls)) { IntWritable obj = new IntWritable(); obj.readFields(in); return cls.getEnumConstants()[obj.get()]; } else if (ClassUtils.isArray(cls)) { int length = (int) readObject(int.class, in); Object array = Array.newInstance(cls.getComponentType(), length); for (int j = 0; j < length; j++) { Object a = readObject(cls.getComponentType(), in); Array.set(array, j, a); } return array; } else { Object obj = cls.newInstance(); ((Writable) obj).readFields(in); return obj; } } catch (IllegalArgumentException | InstantiationException | IllegalAccessException exc) { throw new IOException(exc); } } }