Java tutorial
/** * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information * regarding copyright ownership. The ASF licenses this file * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.falcon.service; import org.apache.commons.lang3.StringUtils; import org.apache.falcon.FalconException; import org.apache.falcon.entity.ClusterHelper; import org.apache.falcon.entity.EntityUtil; import org.apache.falcon.entity.v0.Entity; import org.apache.falcon.entity.v0.EntityType; import org.apache.falcon.entity.v0.cluster.Cluster; import org.apache.falcon.entity.v0.cluster.ClusterLocationType; import org.apache.falcon.entity.v0.cluster.Interfacetype; import org.apache.falcon.hadoop.HadoopClientFactory; import org.apache.falcon.util.StartupProperties; import org.apache.falcon.extensions.store.ExtensionStore; import org.apache.falcon.extensions.ExtensionService; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.FileUtil; import org.apache.hadoop.fs.LocatedFileStatus; import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.RemoteIterator; import org.apache.hadoop.fs.permission.FsPermission; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.io.File; import java.io.IOException; import java.util.Arrays; import java.util.Properties; /** * Host shared libraries in oozie shared lib dir upon creation or modification of cluster. */ public class SharedLibraryHostingService implements ConfigurationChangeListener { private static final Logger LOG = LoggerFactory.getLogger(SharedLibraryHostingService.class); private static final String[] LIBS = StartupProperties.get().getProperty("shared.libs").split(","); private static class FalconLibPath implements FalconPathFilter { private String[] shareLibs; FalconLibPath(String[] libList) { this.shareLibs = Arrays.copyOf(libList, libList.length); } @Override public boolean accept(Path path) { for (String jarName : shareLibs) { if (path.getName().startsWith(jarName)) { return true; } } return false; } @Override public String getJarName(Path path) { for (String jarName : shareLibs) { if (path.getName().startsWith(jarName)) { return jarName; } } throw new IllegalArgumentException(path + " is not accepted!"); } } private void pushExtensionArtifactsToCluster(final Cluster cluster, final FileSystem clusterFs) throws FalconException { if (!Services.get().isRegistered(ExtensionService.SERVICE_NAME)) { LOG.info("ExtensionService not registered, return"); return; } ExtensionStore store = ExtensionStore.get(); if (!store.isExtensionStoreInitialized()) { LOG.info( "Extension store not initialized by Extension service. Make sure Extension service is added in " + "start up properties"); return; } final String filterPath = "/apps/falcon/extensions/mirroring/"; Path extensionStorePath = store.getExtensionStorePath(); LOG.info("extensionStorePath :{}", extensionStorePath); FileSystem falconFileSystem = HadoopClientFactory.get().createFalconFileSystem(extensionStorePath.toUri()); String nameNode = StringUtils .removeEnd(falconFileSystem.getConf().get(HadoopClientFactory.FS_DEFAULT_NAME_KEY), File.separator); String clusterStorageUrl = StringUtils.removeEnd(ClusterHelper.getStorageUrl(cluster), File.separator); // If default fs for Falcon server is same as cluster fs abort copy if (nameNode.equalsIgnoreCase(clusterStorageUrl)) { LOG.info("clusterStorageUrl :{} same return", clusterStorageUrl); return; } try { RemoteIterator<LocatedFileStatus> fileStatusListIterator = falconFileSystem .listFiles(extensionStorePath, true); while (fileStatusListIterator.hasNext()) { LocatedFileStatus srcfileStatus = fileStatusListIterator.next(); Path filePath = Path.getPathWithoutSchemeAndAuthority(srcfileStatus.getPath()); if (filePath != null && filePath.toString().startsWith(filterPath)) { /* HiveDR uses filter path as store path in DRStatusStore, so skip it. Copy only the extension artifacts */ continue; } if (srcfileStatus.isDirectory()) { if (!clusterFs.exists(filePath)) { HadoopClientFactory.mkdirs(clusterFs, filePath, srcfileStatus.getPermission()); } } else { if (clusterFs.exists(filePath)) { FileStatus targetfstat = clusterFs.getFileStatus(filePath); if (targetfstat.getLen() == srcfileStatus.getLen()) { continue; } } Path parentPath = filePath.getParent(); if (!clusterFs.exists(parentPath)) { FsPermission dirPerm = falconFileSystem.getFileStatus(parentPath).getPermission(); HadoopClientFactory.mkdirs(clusterFs, parentPath, dirPerm); } FileUtil.copy(falconFileSystem, srcfileStatus, clusterFs, filePath, false, true, falconFileSystem.getConf()); FileUtil.chmod(clusterFs.makeQualified(filePath).toString(), srcfileStatus.getPermission().toString()); } } } catch (IOException | InterruptedException e) { throw new FalconException("Failed to copy extension artifacts to cluster" + cluster.getName(), e); } } private void addLibsTo(Cluster cluster, FileSystem fs) throws FalconException { try { Path lib = new Path(ClusterHelper.getLocation(cluster, ClusterLocationType.WORKING).getPath(), "lib"); Path libext = new Path(ClusterHelper.getLocation(cluster, ClusterLocationType.WORKING).getPath(), "libext"); FalconPathFilter nonFalconJarFilter = new FalconLibPath(LIBS); Properties properties = StartupProperties.get(); pushLibsToHDFS(fs, properties.getProperty("system.lib.location"), lib, nonFalconJarFilter); pushLibsToHDFS(fs, properties.getProperty("libext.paths"), libext, null); pushLibsToHDFS(fs, properties.getProperty("libext.feed.paths"), new Path(libext, EntityType.FEED.name()), null); pushLibsToHDFS(fs, properties.getProperty("libext.feed.replication.paths"), new Path(libext, EntityType.FEED.name() + "/replication"), null); pushLibsToHDFS(fs, properties.getProperty("libext.feed.retention.paths"), new Path(libext, EntityType.FEED.name() + "/retention"), null); pushLibsToHDFS(fs, properties.getProperty("libext.process.paths"), new Path(libext, EntityType.PROCESS.name()), null); } catch (IOException e) { throw new FalconException("Failed to copy shared libs to cluster" + cluster.getName(), e); } } @SuppressWarnings("ConstantConditions") public static void pushLibsToHDFS(FileSystem fs, String src, Path target, FalconPathFilter pathFilter) throws IOException, FalconException { if (StringUtils.isEmpty(src)) { return; } LOG.debug("Copying libs from {}", src); createTargetPath(fs, target); for (String srcPaths : src.split(",")) { File srcFile = new File(srcPaths); File[] srcFiles = new File[] { srcFile }; if (srcFiles != null) { if (srcFile.isDirectory()) { srcFiles = srcFile.listFiles(); } } if (srcFiles != null) { for (File file : srcFiles) { Path path = new Path(file.getAbsolutePath()); String jarName = StringUtils.removeEnd(path.getName(), ".jar"); if (pathFilter != null) { if (!pathFilter.accept(path)) { continue; } jarName = pathFilter.getJarName(path); } Path targetFile = new Path(target, jarName + ".jar"); if (fs.exists(targetFile)) { FileStatus fstat = fs.getFileStatus(targetFile); if (fstat.getLen() == file.length()) { continue; } } fs.copyFromLocalFile(false, true, new Path(file.getAbsolutePath()), targetFile); fs.setPermission(targetFile, HadoopClientFactory.READ_EXECUTE_PERMISSION); LOG.info("Copied {} to {} in {}", file.getAbsolutePath(), targetFile.toString(), fs.getUri()); } } } } private static void createTargetPath(FileSystem fs, Path target) throws IOException, FalconException { // the dir and files MUST be readable by all users if (!fs.exists(target) && !FileSystem.mkdirs(fs, target, HadoopClientFactory.READ_EXECUTE_PERMISSION)) { throw new FalconException("mkdir " + target + " failed"); } } @Override public void onAdd(Entity entity) throws FalconException { if (entity.getEntityType() != EntityType.CLUSTER) { return; } Cluster cluster = (Cluster) entity; if (!EntityUtil.responsibleFor(cluster.getColo())) { return; } addLibsTo(cluster, getFilesystem(cluster)); pushExtensionArtifactsToCluster(cluster, getFilesystem(cluster)); } @Override public void onRemove(Entity entity) throws FalconException { // Do Nothing } @Override public void onChange(Entity oldEntity, Entity newEntity) throws FalconException { if (oldEntity.getEntityType() != EntityType.CLUSTER) { return; } Cluster oldCluster = (Cluster) oldEntity; Cluster newCluster = (Cluster) newEntity; if (!ClusterHelper.getInterface(oldCluster, Interfacetype.WRITE).getEndpoint() .equals(ClusterHelper.getInterface(newCluster, Interfacetype.WRITE).getEndpoint()) || !ClusterHelper.getInterface(oldCluster, Interfacetype.WORKFLOW).getEndpoint() .equals(ClusterHelper.getInterface(newCluster, Interfacetype.WORKFLOW).getEndpoint())) { addLibsTo(newCluster, getFilesystem(newCluster)); pushExtensionArtifactsToCluster(newCluster, getFilesystem(newCluster)); } } @Override public void onReload(Entity entity) throws FalconException { try { onAdd(entity); } catch (FalconException e) { LOG.error(e.getMessage(), e); } } private FileSystem getFilesystem(final Cluster cluster) throws FalconException { FileSystem fs; try { LOG.info("Initializing FS: {} for cluster: {}", ClusterHelper.getStorageUrl(cluster), cluster.getName()); fs = HadoopClientFactory.get().createFalconFileSystem(ClusterHelper.getConfiguration(cluster)); fs.getStatus(); return fs; } catch (Exception e) { throw new FalconException("Failed to initialize FS for cluster : " + cluster.getName(), e); } } }