Java tutorial
/* * Copyright 2015 Cask Data, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); you may not * use this file except in compliance with the License. You may obtain a copy of * the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the * License for the specific language governing permissions and limitations under * the License. */ package co.cask.cdap.internal.app.runtime.artifact; import co.cask.cdap.api.Config; import co.cask.cdap.api.annotation.Description; import co.cask.cdap.api.annotation.Name; import co.cask.cdap.api.annotation.Plugin; import co.cask.cdap.api.app.Application; import co.cask.cdap.api.artifact.ApplicationClass; import co.cask.cdap.api.artifact.ArtifactClasses; import co.cask.cdap.api.artifact.ArtifactId; import co.cask.cdap.api.data.schema.Schema; import co.cask.cdap.api.data.schema.UnsupportedTypeException; import co.cask.cdap.api.plugin.EndpointPluginContext; import co.cask.cdap.api.plugin.PluginClass; import co.cask.cdap.api.plugin.PluginConfig; import co.cask.cdap.api.plugin.PluginPropertyField; import co.cask.cdap.app.program.ManifestFields; import co.cask.cdap.common.InvalidArtifactException; import co.cask.cdap.common.conf.CConfiguration; import co.cask.cdap.common.conf.Constants; import co.cask.cdap.common.io.Locations; import co.cask.cdap.common.lang.jar.BundleJarUtil; import co.cask.cdap.common.utils.DirUtils; import co.cask.cdap.internal.app.runtime.plugin.PluginInstantiator; import co.cask.cdap.internal.io.ReflectionSchemaGenerator; import co.cask.cdap.proto.Id; import com.google.common.base.Function; import com.google.common.base.Predicate; import com.google.common.base.Throwables; import com.google.common.collect.AbstractIterator; import com.google.common.collect.ImmutableList; import com.google.common.collect.Iterators; import com.google.common.collect.Maps; import com.google.common.primitives.Primitives; import com.google.common.reflect.TypeToken; import org.apache.twill.filesystem.Location; import org.objectweb.asm.AnnotationVisitor; import org.objectweb.asm.ClassReader; import org.objectweb.asm.ClassVisitor; import org.objectweb.asm.Opcodes; import org.objectweb.asm.Type; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.io.File; import java.io.IOException; import java.io.InputStream; import java.lang.annotation.Annotation; import java.lang.reflect.Field; import java.lang.reflect.Method; import java.lang.reflect.Modifier; import java.net.URL; import java.util.ArrayList; import java.util.Enumeration; import java.util.HashSet; import java.util.Iterator; import java.util.List; import java.util.Map; import java.util.Set; import java.util.jar.Attributes; import java.util.jar.JarFile; import java.util.jar.Manifest; import java.util.zip.ZipException; import javax.annotation.Nullable; import javax.ws.rs.Path; /** * Inspects a jar file to determine metadata about the artifact. */ final class ArtifactInspector { private static final Logger LOG = LoggerFactory.getLogger(ArtifactInspector.class); private final CConfiguration cConf; private final ArtifactClassLoaderFactory artifactClassLoaderFactory; private final ReflectionSchemaGenerator schemaGenerator; ArtifactInspector(CConfiguration cConf, ArtifactClassLoaderFactory artifactClassLoaderFactory) { this.cConf = cConf; this.artifactClassLoaderFactory = artifactClassLoaderFactory; this.schemaGenerator = new ReflectionSchemaGenerator(false); } /** * Inspect the given artifact to determine the classes contained in the artifact. * * @param artifactId the id of the artifact to inspect * @param artifactFile the artifact file * @param parentClassLoader the parent classloader to use when inspecting plugins contained in the artifact. * For example, a ProgramClassLoader created from the artifact the input artifact extends * @return metadata about the classes contained in the artifact * @throws IOException if there was an exception opening the jar file * @throws InvalidArtifactException if the artifact is invalid. For example, if the application main class is not * actually an Application. */ ArtifactClasses inspectArtifact(Id.Artifact artifactId, File artifactFile, ClassLoader parentClassLoader) throws IOException, InvalidArtifactException { ArtifactClasses.Builder builder = inspectApplications(artifactId, ArtifactClasses.builder(), Locations.toLocation(artifactFile)); File tmpDir = new File(cConf.get(Constants.CFG_LOCAL_DATA_DIR), cConf.get(Constants.AppFabric.TEMP_DIR)) .getAbsoluteFile(); File stageDir = DirUtils.createTempDir(tmpDir); try (PluginInstantiator pluginInstantiator = new PluginInstantiator(cConf, parentClassLoader, stageDir)) { pluginInstantiator.addArtifact(Locations.toLocation(artifactFile), artifactId.toArtifactId()); inspectPlugins(builder, artifactFile, artifactId.toArtifactId(), pluginInstantiator); } finally { try { DirUtils.deleteDirectoryContents(stageDir); } catch (IOException e) { LOG.warn("Exception raised while deleting directory {}", stageDir, e); } } return builder.build(); } private ArtifactClasses.Builder inspectApplications(Id.Artifact artifactId, ArtifactClasses.Builder builder, Location artifactLocation) throws IOException, InvalidArtifactException { // right now we force users to include the application main class as an attribute in their manifest, // which forces them to have a single application class. // in the future, we may want to let users do this or maybe specify a list of classes or // a package that will be searched for applications, to allow multiple applications in a single artifact. String mainClassName; try { Manifest manifest = BundleJarUtil.getManifest(artifactLocation); if (manifest == null) { return builder; } Attributes manifestAttributes = manifest.getMainAttributes(); if (manifestAttributes == null) { return builder; } mainClassName = manifestAttributes.getValue(ManifestFields.MAIN_CLASS); } catch (ZipException e) { throw new InvalidArtifactException( String.format("Couldn't unzip artifact %s, please check it is a valid jar file.", artifactId), e); } if (mainClassName != null) { try (CloseableClassLoader artifactClassLoader = artifactClassLoaderFactory .createClassLoader(artifactLocation)) { Object appMain = artifactClassLoader.loadClass(mainClassName).newInstance(); if (!(appMain instanceof Application)) { // we don't want to error here, just don't record an application class. // possible for 3rd party plugin artifacts to have the main class set return builder; } Application app = (Application) appMain; java.lang.reflect.Type configType; // if the user parameterized their application, like 'xyz extends Application<T>', // we can deserialize the config into that object. Otherwise it'll just be a Config try { configType = Artifacts.getConfigType(app.getClass()); } catch (Exception e) { throw new InvalidArtifactException(String.format( "Could not resolve config type for Application class %s in artifact %s. " + "The type must extend Config and cannot be parameterized.", mainClassName, artifactId)); } Schema configSchema = configType == Config.class ? null : schemaGenerator.generate(configType); builder.addApp(new ApplicationClass(mainClassName, "", configSchema)); } catch (ClassNotFoundException e) { throw new InvalidArtifactException(String.format( "Could not find Application main class %s in artifact %s.", mainClassName, artifactId)); } catch (UnsupportedTypeException e) { throw new InvalidArtifactException(String.format( "Config for Application %s in artifact %s has an unsupported schema. " + "The type must extend Config and cannot be parameterized.", mainClassName, artifactId)); } catch (InstantiationException | IllegalAccessException e) { throw new InvalidArtifactException( String.format("Could not instantiate Application class %s in artifact %s.", mainClassName, artifactId), e); } } return builder; } /** * Inspects the plugin file and extracts plugin classes information. */ private ArtifactClasses.Builder inspectPlugins(ArtifactClasses.Builder builder, File artifactFile, ArtifactId artifactId, PluginInstantiator pluginInstantiator) throws IOException, InvalidArtifactException { // See if there are export packages. Plugins should be in those packages Set<String> exportPackages = getExportPackages(artifactFile); if (exportPackages.isEmpty()) { return builder; } try { ClassLoader pluginClassLoader = pluginInstantiator.getArtifactClassLoader(artifactId); for (Class<?> cls : getPluginClasses(exportPackages, pluginClassLoader)) { Plugin pluginAnnotation = cls.getAnnotation(Plugin.class); if (pluginAnnotation == null) { continue; } Map<String, PluginPropertyField> pluginProperties = Maps.newHashMap(); try { String configField = getProperties(TypeToken.of(cls), pluginProperties); Set<String> pluginEndpoints = getPluginEndpoints(cls); PluginClass pluginClass = new PluginClass(pluginAnnotation.type(), getPluginName(cls), getPluginDescription(cls), cls.getName(), configField, pluginProperties, pluginEndpoints); builder.addPlugin(pluginClass); } catch (UnsupportedTypeException e) { LOG.warn("Plugin configuration type not supported. Plugin ignored. {}", cls, e); } } } catch (Throwable t) { throw new InvalidArtifactException( String.format("Class could not be found while inspecting artifact for plugins. " + "Please check dependencies are available, and that the correct parent artifact was specified. " + "Error class: %s, message: %s.", t.getClass(), t.getMessage()), t); } return builder; } /** * Returns the set of package names that are declared in "Export-Package" in the jar file Manifest. */ private Set<String> getExportPackages(File file) throws IOException { try (JarFile jarFile = new JarFile(file)) { return ManifestFields.getExportPackages(jarFile.getManifest()); } } /** * Returns an {@link Iterable} of class name that are under the given list of package names that are loadable * through the plugin ClassLoader. */ private Iterable<Class<?>> getPluginClasses(final Iterable<String> packages, final ClassLoader pluginClassLoader) { return new Iterable<Class<?>>() { @Override public Iterator<Class<?>> iterator() { final Iterator<String> packageIterator = packages.iterator(); return new AbstractIterator<Class<?>>() { Iterator<String> classIterator = ImmutableList.<String>of().iterator(); String currentPackage; @Override protected Class<?> computeNext() { while (!classIterator.hasNext()) { if (!packageIterator.hasNext()) { return endOfData(); } currentPackage = packageIterator.next(); try { // Gets all package resource URL for the given package String resourceName = currentPackage.replace('.', File.separatorChar); Enumeration<URL> resources = pluginClassLoader.getResources(resourceName); List<Iterator<String>> iterators = new ArrayList<>(); // Go though all available resources and collect all class names that are plugin classes. while (resources.hasMoreElements()) { URL packageResource = resources.nextElement(); // Only inspect classes in the top level jar file for Plugins. // The jar manifest may have packages in Export-Package that are loadable from the bundled jar files, // which is for classloading purpose. Those classes won't be inspected for plugin classes. // There should be exactly one of resource that match, because it maps to a directory on the FS. if (packageResource.getProtocol().equals("file")) { Iterator<String> classFiles = DirUtils .list(new File(packageResource.toURI()), "class").iterator(); // Transform class file into class name and filter by @Plugin class only iterators.add(Iterators.filter( Iterators.transform(classFiles, new Function<String, String>() { @Override public String apply(String input) { return getClassName(currentPackage, input); } }), new Predicate<String>() { @Override public boolean apply(String className) { return isPlugin(className, pluginClassLoader); } })); } } if (!iterators.isEmpty()) { classIterator = Iterators.concat(iterators.iterator()); } } catch (Exception e) { // Cannot happen throw Throwables.propagate(e); } } try { return pluginClassLoader.loadClass(classIterator.next()); } catch (ClassNotFoundException | NoClassDefFoundError e) { // Cannot happen, since the class name is from the list of the class files under the classloader. throw Throwables.propagate(e); } } }; } }; } /** * Extracts and returns name of the plugin. */ private String getPluginName(Class<?> cls) { Name annotation = cls.getAnnotation(Name.class); return annotation == null || annotation.value().isEmpty() ? cls.getName() : annotation.value(); } /** * Extracts and returns set of endpoints in the plugin. * @throws IllegalArgumentException if there are duplicate endpoints found or * if the number of arguments is not 1 or 2, or if type of 2nd argument is not an EndpointPluginContext. */ private Set<String> getPluginEndpoints(Class<?> cls) throws IllegalArgumentException { Set<String> endpoints = new HashSet<>(); Method[] methods = cls.getMethods(); for (Method method : methods) { Path pathAnnotation = method.getAnnotation(Path.class); // method should have path annotation else continue if (pathAnnotation != null) { if (!endpoints.add(pathAnnotation.value())) { // if the endpoint already exists throw an exception saying, plugin has two methods with same endpoint name. throw new IllegalArgumentException( String.format("Two Endpoints with same name : %s found in Plugin : %s", pathAnnotation.value(), getPluginName(cls))); } // check that length of method parameters is 1 or 2. if length is 2, // check that 2nd param is of type EndpointPluginContext if (!(method.getParameterTypes().length == 1 || method.getParameterTypes().length == 2)) { throw new IllegalArgumentException(String.format( "Endpoint parameters can only be of length 1 or 2, " + "found endpoint %s with %s parameters", pathAnnotation.value(), method.getParameterTypes().length)); } if (method.getParameterTypes().length == 2 && !EndpointPluginContext.class.isAssignableFrom(method.getParameterTypes()[1])) { throw new IllegalArgumentException(String.format( "2nd parameter of endpoint should be EndpointPluginContext, " + "%s is not of type %s in endpoint %s", method.getParameterTypes()[1], EndpointPluginContext.class.getName(), pathAnnotation.value())); } } } return endpoints; } /** * Returns description for the plugin. */ private String getPluginDescription(Class<?> cls) { Description annotation = cls.getAnnotation(Description.class); return annotation == null ? "" : annotation.value(); } /** * Constructs the fully qualified class name based on the package name and the class file name. */ private String getClassName(String packageName, String classFileName) { return packageName + "." + classFileName.substring(0, classFileName.length() - ".class".length()); } /** * Gets all config properties for the given plugin. * * @return the name of the config field in the plugin class or {@code null} if the plugin doesn't have a config field */ @Nullable private String getProperties(TypeToken<?> pluginType, Map<String, PluginPropertyField> result) throws UnsupportedTypeException { // Get the config field for (TypeToken<?> type : pluginType.getTypes().classes()) { for (Field field : type.getRawType().getDeclaredFields()) { TypeToken<?> fieldType = TypeToken.of(field.getGenericType()); if (PluginConfig.class.isAssignableFrom(fieldType.getRawType())) { // Pick up all config properties inspectConfigField(fieldType, result); return field.getName(); } } } return null; } /** * Inspects the plugin config class and build up a map for {@link PluginPropertyField}. * * @param configType type of the config class * @param result map for storing the result * @throws UnsupportedTypeException if a field type in the config class is not supported */ private void inspectConfigField(TypeToken<?> configType, Map<String, PluginPropertyField> result) throws UnsupportedTypeException { for (TypeToken<?> type : configType.getTypes().classes()) { if (PluginConfig.class.equals(type.getRawType())) { break; } for (Field field : type.getRawType().getDeclaredFields()) { int modifiers = field.getModifiers(); if (Modifier.isTransient(modifiers) || Modifier.isStatic(modifiers) || field.isSynthetic()) { continue; } PluginPropertyField property = createPluginProperty(field, type); if (result.containsKey(property.getName())) { throw new IllegalArgumentException("Plugin config with name " + property.getName() + " already defined in " + configType.getRawType()); } result.put(property.getName(), property); } } } /** * Creates a {@link PluginPropertyField} based on the given field. */ private PluginPropertyField createPluginProperty(Field field, TypeToken<?> resolvingType) throws UnsupportedTypeException { TypeToken<?> fieldType = resolvingType.resolveType(field.getGenericType()); Class<?> rawType = fieldType.getRawType(); Name nameAnnotation = field.getAnnotation(Name.class); Description descAnnotation = field.getAnnotation(Description.class); String name = nameAnnotation == null ? field.getName() : nameAnnotation.value(); String description = descAnnotation == null ? "" : descAnnotation.value(); if (rawType.isPrimitive()) { return new PluginPropertyField(name, description, rawType.getName(), true); } rawType = Primitives.unwrap(rawType); if (!rawType.isPrimitive() && !String.class.equals(rawType)) { throw new UnsupportedTypeException("Only primitive and String types are supported"); } boolean required = true; for (Annotation annotation : field.getAnnotations()) { if (annotation.annotationType().getName().endsWith(".Nullable")) { required = false; break; } } return new PluginPropertyField(name, description, rawType.getSimpleName().toLowerCase(), required); } /** * Detects if a class is annotated with {@link Plugin} without loading the class. * * @param className name of the class * @param classLoader ClassLoader for loading the class file of the given class * @return true if the given class is annotated with {@link Plugin} */ private boolean isPlugin(String className, ClassLoader classLoader) { try (InputStream is = classLoader.getResourceAsStream(className.replace('.', '/') + ".class")) { if (is == null) { return false; } // Use ASM to inspect the class bytecode to see if it is annotated with @Plugin final boolean[] isPlugin = new boolean[1]; ClassReader cr = new ClassReader(is); cr.accept(new ClassVisitor(Opcodes.ASM5) { @Override public AnnotationVisitor visitAnnotation(String desc, boolean visible) { if (Plugin.class.getName().equals(Type.getType(desc).getClassName()) && visible) { isPlugin[0] = true; } return super.visitAnnotation(desc, visible); } }, ClassReader.SKIP_CODE | ClassReader.SKIP_DEBUG | ClassReader.SKIP_FRAMES); return isPlugin[0]; } catch (IOException e) { // If failed to open the class file, then it cannot be a plugin LOG.warn("Failed to open class file for {}", className, e); return false; } } }