Java tutorial
/* * Copyright 2014 University of Southern California * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package edu.usc.pgroup.floe.utils; import edu.usc.pgroup.floe.config.ConfigProperties; import edu.usc.pgroup.floe.config.FloeConfig; import org.apache.commons.lang.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.zeromq.ZMQ; import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; import java.io.File; import java.io.IOException; import java.io.ObjectInputStream; import java.io.ObjectOutputStream; import java.lang.reflect.Constructor; import java.net.Inet6Address; import java.net.InetAddress; import java.net.MalformedURLException; import java.net.NetworkInterface; import java.net.SocketException; import java.net.URL; import java.net.URLClassLoader; import java.nio.file.FileSystems; import java.nio.file.Path; import java.util.Enumeration; /** * Common Utility Class. * * @author Alok Kumbhare */ public final class Utils { /** * the global logger instance. */ private static final Logger LOGGER = LoggerFactory.getLogger(Utils.class); /** * Hiding public constructor. */ private Utils() { } /** * returns the canonical host name or reads it from the config file. * @return the hostname or ip read from the config file. */ public static String getHostNameOrIpAddress() { String hostnameOrIpAddr; if (FloeConfig.getConfig().containsKey(ConfigProperties.HOST_NAME)) { hostnameOrIpAddr = FloeConfig.getConfig().getString(ConfigProperties.HOST_NAME); } else { hostnameOrIpAddr = Utils.getIpAddress(); } //hostnameOrIpAddr = Utils.getIpAddress(); return hostnameOrIpAddr; } /** * returns the canonical host name. This assumes a unique host name for * each machine in the cluster does not apply. * FixMe: In local cloud environment (local eucalyptus in system mode) * where the DNS server is not running, this might be an issue. * * @return the canonical hostname. * public static String getHostName() { try { return InetAddress.getLocalHost().getCanonicalHostName(); } catch (UnknownHostException e) { LOGGER.error("Error occurred while retrieving hostname" + e.getMessage()); throw new RuntimeException("Error occurred while " + "retrieving hostname" + e.getMessage()); } }*/ /** * returns the canonical host name. This assumes a unique host name for * each machine in the cluster does not apply. * FixMe: In local cloud environment (local eucalyptus in system mode) * where the DNS server is not running, this might be an issue. * @return The first IPv4 address found for any interface that is up and * running. */ public static String getIpAddress() { String ip = null; try { Enumeration<NetworkInterface> interfaces = NetworkInterface.getNetworkInterfaces(); LOGGER.error("Getting ip"); while (interfaces.hasMoreElements()) { LOGGER.error("Next iface"); NetworkInterface current = interfaces.nextElement(); if (!current.isUp() || current.isLoopback() || current.isVirtual()) { continue; } Enumeration<InetAddress> addresses = current.getInetAddresses(); while (addresses.hasMoreElements()) { InetAddress currentAddr = addresses.nextElement(); if (currentAddr.isLoopbackAddress() || (currentAddr instanceof Inet6Address)) { continue; } ip = currentAddr.getHostAddress(); } } } catch (SocketException e) { LOGGER.error("Error occurred while retrieving hostname" + e.getMessage()); throw new RuntimeException("Error occurred while " + "retrieving hostname" + e.getMessage()); } return ip; } /** * Checks if the filename (to be uploaded or downloaded) is valid. * It should not contain ".." and should not be an absolute path. * * @param fileName filename to be uploaded or downloaded relative to the * coordinator's scratch folder. * @return returns true if the filename is valid and can be used in the * download or upload functions. */ public static boolean checkValidFileName(final String fileName) { if (fileName.contains("..")) { return false; } Path filePath = FileSystems.getDefault().getPath(fileName); if (filePath.isAbsolute()) { return true; } return true; } /** * Returns a complete command for launching a java child process. * with SAME classpath as the parent. * @param className Child's class name. * @param jvmParams Additional params to be passed to the JVM * @param appParams custom application params, appended into a single * string. * @return command string to launch the given class in a JVM */ public static String getJavaProcessLaunchCommand(final String className, final String appParams, final String... jvmParams) { String javaHome = FloeConfig.getConfig().getString(ConfigProperties.SYS_JAVA_HOME); String classPath = FloeConfig.getConfig().getString(ConfigProperties.SYS_JAVA_CLASS_PATH); //TODO: Add other required parameters String fileSeparator = FloeConfig.getConfig().getString(ConfigProperties.SYS_FILE_SEPARATOR); String javaCmd = javaHome + fileSeparator + "bin" + fileSeparator + "java"; String params = StringUtils.join(jvmParams, ' '); String command = StringUtils .join(new String[] { javaCmd, "-cp " + classPath, params, className, appParams }, ' '); return command; } /** * Use reflection to create an instance of the given class. * @param fqdnClassName the fully qualified class name in the default * class loader. * @return a new instance of the given class. NULL if there was an error * creating the instance. */ public static Object instantiateObject(final String fqdnClassName) { return instantiateObject(fqdnClassName, null); } /** * Use reflection to create an instance of the given class. * @param fqdnClassName the fully qualified class name. * @param loader class loader to be used. * @return a new instance of the given class. NULL if there was an error * creating the instance. */ public static Object instantiateObject(final String fqdnClassName, final ClassLoader loader) { Object instance = null; try { Class<?> klass; if (loader == null) { klass = Class.forName(fqdnClassName); } else { klass = Class.forName(fqdnClassName, true, loader); } instance = klass.newInstance(); } catch (ClassNotFoundException e) { LOGGER.error("Could not create an instance of {}, Exception:{}", fqdnClassName, e); } catch (InstantiationException e) { LOGGER.error("Could not create an instance of {}, Exception:{}", fqdnClassName, e); } catch (IllegalAccessException e) { LOGGER.error("Could not create an instance of {}, Exception:{}", fqdnClassName, e); } return instance; } /** * Use reflection to create an instance of the given class. * @param fqdnClassName the fully qualified class name. * @param params types of parameters to query for the constructor. * @return a new instance of the given class. NULL if there was an error * creating the instance. */ public static Constructor<?> getConstructor(final String fqdnClassName, final Class<?>... params) { try { Class<?> klass = Class.forName(fqdnClassName); return klass.getConstructor(params); } catch (ClassNotFoundException e) { LOGGER.error("Could not create an instance of {}, Exception:{}", fqdnClassName, e); } catch (NoSuchMethodException e) { LOGGER.error("No such constructor found, Exception:{}", fqdnClassName, e); } return null; } /** * Serializer used for storing data into zookeeper. We use the default * java serializer (since the amount of data to be serialized is usually * very small). * NOTE: THIS IS DIFFERENT FROM THE SERIALIZER USED DURING COMMUNICATION * BETWEEN DIFFERENT FLAKES. THAT ONE IS PLUGABBLE, THIS IS NOT. * * @param obj Object to be serialized * @return serialized byte array. */ public static byte[] serialize(final Object obj) { try { ByteArrayOutputStream bos = new ByteArrayOutputStream(); ObjectOutputStream oos = new ObjectOutputStream(bos); oos.writeObject(obj); oos.close(); return bos.toByteArray(); } catch (IOException ioe) { throw new RuntimeException(ioe); } } /** * Deserializer for Zookeeper data. See comments for serialize function. * * @param serialized serialized byte array (default java serialized) * obtained from the serialize function. * @return the constructed java object. Needs to be typecasted to * appropriate object before using. No checks are actionCompleted here. */ public static Object deserialize(final byte[] serialized) { try { ByteArrayInputStream bis = new ByteArrayInputStream(serialized); ObjectInputStream ois = new ObjectInputStream(bis); Object ret = ois.readObject(); ois.close(); return ret; } catch (IOException ioe) { throw new RuntimeException(ioe); } catch (ClassNotFoundException e) { throw new RuntimeException(e); } } /** * Deserializer for Zookeeper data. See comments for serialize function. * * @param serialized serialized byte array (default java serialized) * obtained from the serialize function. * @param classLoader Custom Class loader to used to deserialize the * object. * @return the constructed java object. Needs to be typecasted to * appropriate object before using. No checks are actionCompleted here. */ public static Object deserialize(final byte[] serialized, final ClassLoader classLoader) { try { ByteArrayInputStream bis = new ByteArrayInputStream(serialized); //ObjectInputStream ois = new ObjectInputStream(bis); ClassLoaderObjectInputStream ois = new ClassLoaderObjectInputStream(classLoader, bis); Object ret = ois.readObject(); ois.close(); return ret; } catch (IOException ioe) { throw new RuntimeException(ioe); } catch (ClassNotFoundException e) { throw new RuntimeException(e); } } /** * returns the pull local path of the application jar. * @param appName name of the application. * @param jarName jar name. * @return the path to the application jar. */ public static String getContainerJarDownloadPath(final String appName, final String jarName) { String downloadLocation = FloeConfig.getConfig().getString(ConfigProperties.FLOE_EXEC_SCRATCH_FOLDER) + Utils.Constants.FILE_PATH_SEPARATOR + FloeConfig.getConfig().getString(ConfigProperties.CONTAINER_LOCAL_FOLDER) + Utils.Constants.FILE_PATH_SEPARATOR + Utils.Constants.CONTAINER_APP_FOLDER + Utils.Constants.FILE_PATH_SEPARATOR + appName + Utils.Constants.FILE_PATH_SEPARATOR + jarName; return downloadLocation; } /** * Generates a unique flake id given cid and fid. * @param cid container's id on which this flake resides. * @param fid flake's id. * @return globally unique flake id. */ public static String generateFlakeId(final String cid, final String fid) { return cid + "-" + fid; } /** * Once the poller.poll returns, use this function as a component in the * proxy to forward messages from one socket to another. * @param from socket to read from. * @param to socket to send messages to */ public static void forwardCompleteMessage(final ZMQ.Socket from, final ZMQ.Socket to) { byte[] message; boolean more = false; while (true) { // receive message message = from.recv(0); more = from.hasReceiveMore(); // Broker it int flags = 0; if (more) { flags = ZMQ.SNDMORE; } to.send(message, flags); if (!more) { break; } } } /** * receives a complete message from the socket and ignores. * @param from the socket to read from. */ public static void recvAndignore(final ZMQ.Socket from) { byte[] message; boolean more = false; while (true) { // receive message message = from.recv(0); more = from.hasReceiveMore(); // Broker it int flags = 0; if (more) { flags = ZMQ.SNDMORE; } if (!more) { break; } } } /** * Creates and returns a class loader with the given jar file. * @param path path to the jar file. * @param parent parent class loader. * @return a new child class loader */ public static ClassLoader getClassLoader(final String path, final ClassLoader parent) { ClassLoader loader = null; try { File relativeJarLoc = new File(path); URL jarLoc = new URL("file://" + relativeJarLoc.getAbsolutePath()); LOGGER.info("Loading jar: {} into class loader.", jarLoc); loader = URLClassLoader.newInstance(new URL[] { jarLoc }, parent); } catch (MalformedURLException e) { e.printStackTrace(); LOGGER.error("Invalid Jar URL Exception: {}", e); } return loader; } /** * Various Constants used across the project. * Note: The string constants for configuration file, * and for the zookeeper paths are in config/ConfigProperties and * zookeeper/ZKConstants respectively. This file contains other generic * constants. */ public static final class Constants { /** * The system file separator. */ public static final String FILE_PATH_SEPARATOR = FloeConfig.getConfig() .getString(ConfigProperties.SYS_FILE_SEPARATOR); /** * The local execution mode for FLOE. */ public static final String LOCAL = "local"; /** * The distributed execution mode for FLOE. */ public static final String DISTRIBUTED = "distributed"; /** * Milli multiplier. */ public static final int MILLI = 1000; /** * Chunk size to read from file and send to the coordinator. */ public static final int BUFFER_SIZE = 1024 * 4; //public static final String FlakeReceiverFrontEndPrefix = "" /** * Flake receiver frontend prefix (this is suffixed by host and port). * Used for connecting to the predecessor flake. */ public static final String FLAKE_RECEIVER_FRONTEND_CONNECT_SOCK_PREFIX = "tcp://"; /** * Flake sender middle-end prefix (this is suffixed by a listening * port). * Receives data messages from the middle end. and uses PUSH to * send it to all flakes containing the pellet instances for the * succeeding pellet. */ public static final String FLAKE_SENDER_BACKEND_SOCK_PREFIX = "tcp://*:"; /** * Flake receiver backend prefix (this is suffixed by flake id). * Used for a PUSH socket to receive data from RECEIVER frontend and * send it to the pellets evenly. */ public static final String FLAKE_RECEIVER_BACKEND_SOCK_PREFIX = "inproc://receiver-backend-"; /** * Flake receiver backed for signals. This is suffixed by flake id. * Used to publish the signal to all pellet instances. Hence uses the * PUB/SUB model. */ public static final String FLAKE_RECEIVER_SIGNAL_BACKEND_SOCK_PREFIX = "inproc://receiver-signal-backend-"; /** * Flake receiver Control socket prefix (this is suffixed by flake id). * Used for receiving control signals from the container. */ public static final String FLAKE_CONTROL_SOCK_PREFIX = "ipc://flake-control-"; /** * Flake receiver Control socket prefix (this is suffixed by flake id). * Used for receiving control signals from the container. */ public static final String FLAKE_RECEIVER_MIDDLE_PREFIX = "inproc://flake-recv-middle-"; /** * Flake receiver Control socket prefix (this is suffixed by flake id). * Used for receiving control signals from the container. */ public static final String FLAKE_RECEIVER_CONTROL_FWD_PREFIX = "inproc://flake-control-fwd-"; /** * Flake sender front-end prefix (this is suffixed by flake id). * Used for receiving data messages from all pellet instances. */ public static final String FLAKE_SENDER_FRONTEND_SOCK_PREFIX = "inproc://sender-frontend-"; /** * Flake sender middle-end prefix (this is suffixed by flake id). * Receives data messages from the front end. and uses PUB to * send it all outgoing channels based on dispersion * strategy (currently only duplicate it to all outgoing channels). */ public static final String FLAKE_SENDER_MIDDLEEND_SOCK_PREFIX = "inproc://sender-middleend-"; /** * Flake backchannel sender prefix. */ public static final String FLAKE_BACKCHANNEL_SENDER_PREFIX = "inproc://flake-backchannel-sender-"; /** * Flake backchannel sender control prefix. Used to trigger the send. */ public static final String FLAKE_BACKCHANNEL_CONTROL_PREFIX = "inproc://flake-backchannel-control-"; /** * the endpoint to be used by flakes to send their heartbeat to the * container. */ public static final String FLAKE_HEARBEAT_SOCK_PREFIX = "ipc://flake-heartbeat-"; /** * Flake Component uses this to listen for start/stop notification * signals. */ public static final String FLAKE_COMPONENT_NOTIFY_PREFIX = "inproc://flake-comp-notify-"; /** * Flake Component uses this to send kill signal to the component * implementation. */ public static final String FLAKE_COMPONENT_KILL_PREFIX = "inproc://flake-comp-kill-"; /** * the endpoint to be used by flakes to send their state checkpoints. */ public static final String FLAKE_STATE_PUB_SOCK = "tcp://*:"; /** * the endpoint to be used by flakes to send their state checkpoints. */ public static final String FLAKE_MSG_BACKUP_PREFIX = "inproc://flake-msg-backup-"; /** * the endpoint to be used by backup component to start/stop recovery * process (like a control channel). */ public static final String FLAKE_MSG_BACKUP_CONTROL_PREFIX = "inproc://flake-msg-backup-control-"; /** * the endpoint to be used during recovery. Backup component binds to * it, flake receiver connects to it. */ public static final String FLAKE_MSG_RECOVERY_PREFIX = "inproc://flake-msg-backup-recovery-"; /** * the endpoint to be used by flakes to send their state checkpoints. */ public static final String FLAKE_STATE_BACKUP_PREFIX = "inproc://flake-state-backup-"; /** * the endpoint to be used by flakes to send their state checkpoints. */ public static final String FLAKE_STATE_SUB_SOCK_PREFIX = "tcp://"; /** * Number of i/o threads to be used by ZMQ for a single flake. */ public static final int FLAKE_NUM_IO_THREADS = 4; /** * Apps folder name relative to container local folder. */ public static final String CONTAINER_APP_FOLDER = "apps"; /** * Name of the default alternate. */ public static final String DEFAULT_ALTERNATE_NAME = "default_alternate"; /** * Name of the default publisher key to send messages to all * subscribers. */ public static final String PUB_ALL = "all"; /** * Name of the default output stream. */ public static final String DEFAULT_STREAM_NAME = "default_stream"; /** * The field name associated with the system timestamp of the tuple. */ public static final String SYSTEM_TS_FIELD_NAME = "SYSTEM_TS"; /** * The field name associated with the SRC pellet of the tuple. */ public static final String SYSTEM_SRC_PELLET_NAME = "SYSTEM_SRC_PELLET"; } }