Java tutorial
/* * Copyright 2013-2015 EMC Corporation. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"). * You may not use this file except in compliance with the License. * A copy of the License is located at * * http://www.apache.org/licenses/LICENSE-2.0.txt * * or in the "license" file accompanying this file. This file is distributed * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either * express or implied. See the License for the specific language governing * permissions and limitations under the License. */ package com.emc.ecs.sync; import com.emc.ecs.sync.cli.CliConfig; import com.emc.ecs.sync.cli.CliHelper; import com.emc.ecs.sync.config.ConfigUtil; import com.emc.ecs.sync.config.ConfigWrapper; import com.emc.ecs.sync.config.SyncConfig; import com.emc.ecs.sync.config.SyncOptions; import com.emc.ecs.sync.filter.SyncFilter; import com.emc.ecs.sync.model.*; import com.emc.ecs.sync.rest.RestServer; import com.emc.ecs.sync.service.*; import com.emc.ecs.sync.storage.SyncStorage; import com.emc.ecs.sync.util.*; import com.sun.management.OperatingSystemMXBean; import org.apache.commons.cli.ParseException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import javax.xml.bind.JAXBContext; import javax.xml.bind.JAXBException; import java.io.File; import java.lang.management.ManagementFactory; import java.util.ArrayList; import java.util.Date; import java.util.Iterator; import java.util.List; import java.util.concurrent.Executors; import java.util.concurrent.LinkedBlockingDeque; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.TimeUnit; public class EcsSync implements Runnable, RetryHandler { private static final Logger log = LoggerFactory.getLogger(EcsSync.class); public static final String VERSION = EcsSync.class.getPackage().getImplementationVersion(); public static void main(String[] args) { int exitCode = 0; System.out.println(versionLine()); RestServer restServer = null; try { // first, hush up the JDK logger (why does this default to INFO??) java.util.logging.LogManager.getLogManager().getLogger("").setLevel(java.util.logging.Level.WARNING); CliConfig cliConfig = CliHelper.parseCliConfig(args); if (cliConfig != null) { // configure logging for startup if (cliConfig.getLogLevel() != null) SyncJobService.getInstance().setLogLevel(cliConfig.getLogLevel()); // start REST service if (cliConfig.isRestEnabled()) { if (cliConfig.getRestEndpoint() != null) { String[] endpoint = cliConfig.getRestEndpoint().split(":"); restServer = new RestServer(endpoint[0], Integer.parseInt(endpoint[1])); } else { restServer = new RestServer(); restServer.setAutoPortEnabled(true); } // set DB connect string if provided if (cliConfig.getDbConnectString() != null) { SyncJobService.getInstance().setDbConnectString(cliConfig.getDbConnectString()); } restServer.start(); } // if REST-only, skip remaining logic (REST server thread will keep the VM running) if (cliConfig.isRestOnly()) return; try { // determine sync config SyncConfig syncConfig; if (cliConfig.getXmlConfig() != null) { syncConfig = loadXmlFile(new File(cliConfig.getXmlConfig())); } else { syncConfig = CliHelper.parseSyncConfig(cliConfig, args); } // create the sync instance final EcsSync sync = new EcsSync(); sync.setSyncConfig(syncConfig); // register for REST access SyncJobService.getInstance().registerJob(sync); // start sync job (this blocks until the sync is complete) sync.run(); // print completion stats System.out.print(sync.getStats().getStatsString()); if (sync.getStats().getObjectsFailed() > 0) exitCode = 3; } finally { if (restServer != null) try { restServer.stop(0); } catch (Throwable t) { log.warn("could not stop REST service", t); } } } } catch (ParseException e) { System.err.println(e.getMessage()); System.out.println(" use --help for a detailed (quite long) list of options"); exitCode = 1; } catch (Throwable t) { t.printStackTrace(); exitCode = 2; } System.exit(exitCode); } private static SyncConfig loadXmlFile(File xmlFile) throws JAXBException { List<Class> pluginClasses = new ArrayList<>(); pluginClasses.add(SyncConfig.class); for (ConfigWrapper<?> wrapper : ConfigUtil.allStorageConfigWrappers()) { pluginClasses.add(wrapper.getTargetClass()); } for (ConfigWrapper<?> wrapper : ConfigUtil.allFilterConfigWrappers()) { pluginClasses.add(wrapper.getTargetClass()); } return (SyncConfig) JAXBContext.newInstance(pluginClasses.toArray(new Class[pluginClasses.size()])) .createUnmarshaller().unmarshal(xmlFile); } private static String versionLine() { return EcsSync.class.getSimpleName() + (VERSION == null ? "" : " v" + VERSION); } private DbService dbService; private Throwable runError; private EnhancedThreadPoolExecutor listExecutor; private EnhancedThreadPoolExecutor syncExecutor; private EnhancedThreadPoolExecutor queryExecutor; private EnhancedThreadPoolExecutor estimateExecutor; private EnhancedThreadPoolExecutor retrySubmitter; private SyncFilter firstFilter; private SyncEstimate syncEstimate; private boolean paused, terminated; private SyncStats stats = new SyncStats(); private SyncConfig syncConfig; private SyncStorage<?> source; private SyncStorage<?> target; private List<SyncFilter> filters; private SyncVerifier verifier; private SyncControl syncControl = new SyncControl(); private int perfReportSeconds; private ScheduledExecutorService perfScheduler; public void run() { try { assert syncConfig != null : "syncConfig is null"; assert syncConfig.getOptions() != null : "syncConfig.options is null"; final SyncOptions options = syncConfig.getOptions(); // Some validation (must have source and target) assert source != null || syncConfig.getSource() != null : "source must be specified"; assert target != null || syncConfig.getTarget() != null : "target plugin must be specified"; if (source == null) source = PluginUtil.newStorageFromConfig(syncConfig.getSource(), options); else syncConfig.setSource(source.getConfig()); if (target == null) target = PluginUtil.newStorageFromConfig(syncConfig.getTarget(), options); else syncConfig.setTarget(target.getConfig()); if (filters == null) { if (syncConfig.getFilters() != null) filters = PluginUtil.newFiltersFromConfigList(syncConfig.getFilters(), options); else filters = new ArrayList<>(); } else { List<Object> filterConfigs = new ArrayList<>(); for (SyncFilter filter : filters) { filterConfigs.add(filter.getConfig()); } syncConfig.setFilters(filterConfigs); } // Summarize config for reference if (log.isInfoEnabled()) log.info(summarizeConfig()); // Ask each plugin to configure itself and validate the chain (resolves incompatible plugins) String currentPlugin = "source storage"; try { source.configure(source, filters.iterator(), target); currentPlugin = "target storage"; target.configure(source, filters.iterator(), target); for (SyncFilter filter : filters) { currentPlugin = filter.getClass().getSimpleName() + " filter"; filter.configure(source, filters.iterator(), target); } } catch (Exception e) { log.error("Error configuring " + currentPlugin); throw e; } // Build the plugin chain Iterator<SyncFilter> i = filters.iterator(); SyncFilter next, previous = null; while (i.hasNext()) { next = i.next(); if (previous != null) previous.setNext(next); previous = next; } // add target to chain SyncFilter targetFilter = new TargetFilter(target, options); if (previous != null) previous.setNext(targetFilter); firstFilter = filters.isEmpty() ? targetFilter : filters.get(0); // register for timings if (options.isTimingsEnabled()) TimingUtil.register(options); else TimingUtil.unregister(options); // in case of subsequent runs with same options instance log.info("Sync started at " + new Date()); // make sure any old stats are closed to terminate the counter threads try (SyncStats oldStats = stats) { stats = new SyncStats(); } stats.setStartTime(System.currentTimeMillis()); stats.setCpuStartTime( ((OperatingSystemMXBean) ManagementFactory.getOperatingSystemMXBean()).getProcessCpuTime() / 1000000); // initialize DB Service if necessary if (dbService == null) { if (options.getDbFile() != null) { dbService = new SqliteDbService(options.getDbFile()); } else if (options.getDbConnectString() != null) { dbService = new MySQLDbService(options.getDbConnectString(), null, null); } else { dbService = new NoDbService(); } if (options.getDbTable() != null) dbService.setObjectsTableName(options.getDbTable()); } // create thread pools listExecutor = new EnhancedThreadPoolExecutor(options.getThreadCount(), new LinkedBlockingDeque<Runnable>(options.getThreadCount() * 20), "list-pool"); estimateExecutor = new EnhancedThreadPoolExecutor(options.getThreadCount(), new LinkedBlockingDeque<Runnable>(options.getThreadCount() * 20), "estimate-pool"); queryExecutor = new EnhancedThreadPoolExecutor(options.getThreadCount() * 2, new LinkedBlockingDeque<Runnable>(), "query-pool"); syncExecutor = new EnhancedThreadPoolExecutor(options.getThreadCount(), new LinkedBlockingDeque<Runnable>(options.getThreadCount() * 20), "sync-pool"); retrySubmitter = new EnhancedThreadPoolExecutor(options.getThreadCount(), new LinkedBlockingDeque<Runnable>(), "retry-submitter"); // initialize verifier verifier = new Md5Verifier(options); // setup performance reporting startPerformanceReporting(); // set status to running syncControl.setRunning(true); stats.reset(); log.info("syncing from {} to {}", ConfigUtil.generateUri(syncConfig.getSource()), ConfigUtil.generateUri(syncConfig.getTarget())); // start estimating syncEstimate = new SyncEstimate(); estimateExecutor.submit(new Runnable() { @Override public void run() { // do we have a list-file? if (options.getSourceListFile() != null) { FileLineIterator lineIterator = new FileLineIterator(options.getSourceListFile()); while (lineIterator.hasNext()) { estimateExecutor .blockingSubmit(new EstimateTask(lineIterator.next(), source, syncEstimate)); } } else { for (ObjectSummary summary : source.allObjects()) { estimateExecutor.blockingSubmit(new EstimateTask(summary, source, syncEstimate)); } } } }); // iterate through root objects and submit tasks for syncing and crawling (querying). if (options.getSourceListFile() != null) { // do we have a list-file? FileLineIterator lineIterator = new FileLineIterator(options.getSourceListFile()); while (lineIterator.hasNext()) { if (!syncControl.isRunning()) break; final String listLine = lineIterator.next(); listExecutor.blockingSubmit(new Runnable() { @Override public void run() { ObjectSummary summary = source.parseListLine(listLine); submitForSync(source, summary); if (summary.isDirectory()) submitForQuery(source, summary); } }); } } else { for (ObjectSummary summary : source.allObjects()) { if (!syncControl.isRunning()) break; submitForSync(source, summary); if (summary.isDirectory()) submitForQuery(source, summary); } } // now we must wait until all submitted tasks are complete while (syncControl.isRunning()) { if (listExecutor.getUnfinishedTasks() <= 0 && queryExecutor.getUnfinishedTasks() <= 0 && syncExecutor.getUnfinishedTasks() <= 0) { // done log.info("all tasks complete"); break; } else { try { Thread.sleep(1000); } catch (InterruptedException e) { log.warn("interrupted while sleeping", e); } } } // run a final timing log TimingUtil.logTimings(options); } catch (Throwable t) { log.error("unexpected exception", t); runError = t; throw t; } finally { if (!syncControl.isRunning()) log.warn("terminated early!"); syncControl.setRunning(false); if (paused) { paused = false; // must interrupt the threads that are blocked if (listExecutor != null) listExecutor.shutdownNow(); if (estimateExecutor != null) estimateExecutor.shutdownNow(); if (queryExecutor != null) queryExecutor.shutdownNow(); if (retrySubmitter != null) retrySubmitter.shutdownNow(); if (syncExecutor != null) syncExecutor.shutdownNow(); } else { if (listExecutor != null) listExecutor.shutdown(); if (estimateExecutor != null) estimateExecutor.shutdown(); if (queryExecutor != null) queryExecutor.shutdown(); if (retrySubmitter != null) retrySubmitter.shutdown(); if (syncExecutor != null) syncExecutor.shutdown(); } if (stats != null) stats.setStopTime(System.currentTimeMillis()); // clean up any resources in the plugins cleanup(); } } private void startPerformanceReporting() { if (perfReportSeconds > 0) { perfScheduler = Executors.newSingleThreadScheduledExecutor(); perfScheduler.scheduleAtFixedRate(new Runnable() { @Override public void run() { if (isRunning()) { log.info("Source: read: {} b/s write: {} b/s", getSource().getReadRate(), getSource().getWriteRate()); log.info("Target: read: {} b/s write: {} b/s", getTarget().getReadRate(), getTarget().getWriteRate()); log.info("Objects: complete: {}/s failed: {}/s", getStats().getObjectCompleteRate(), getStats().getObjectErrorRate()); } } }, perfReportSeconds, perfReportSeconds, TimeUnit.SECONDS); } } /** * Stops the underlying executors from executing new tasks. Currently running tasks will complete and all threads * will then block until resumed * * @return true if the state was changed from running to pause; false if already paused * @throws IllegalStateException if the sync is complete or was terminated */ public boolean pause() { if (!syncControl.isRunning()) throw new IllegalStateException("sync is not running"); boolean changed = queryExecutor.pause() && syncExecutor.pause(); paused = true; stats.pause(); return changed; } /** * Resumes the underlying executors so they may continue to execute tasks * * @return true if the state was changed from paused to running; false if already running * @throws IllegalStateException if the sync is complete or was terminated * @see #pause() */ public boolean resume() { if (!syncControl.isRunning()) throw new IllegalStateException("sync is not running"); boolean changed = queryExecutor.resume() && syncExecutor.resume(); paused = false; stats.resume(); return changed; } public void terminate() { syncControl.setRunning(false); terminated = true; if (queryExecutor != null) queryExecutor.getQueue().clear(); if (retrySubmitter != null) retrySubmitter.getQueue().clear(); } public String summarizeConfig() { StringBuilder summary = new StringBuilder("Configuration Summary:\n"); summary.append(ConfigUtil.summarize(syncConfig.getOptions())); summary.append("Source: ").append(ConfigUtil.summarize(syncConfig.getSource())); summary.append("Target: ").append(ConfigUtil.summarize(syncConfig.getTarget())); if (syncConfig.getFilters() != null) { summary.append("Filters:\n"); for (Object filter : syncConfig.getFilters()) { summary.append(ConfigUtil.summarize(filter)); } } else { summary.append("Filters: none\n"); } return summary.toString(); } private void submitForQuery(SyncStorage source, ObjectSummary entry) { if (syncControl.isRunning()) queryExecutor.blockingSubmit(new QueryTask(source, entry)); else log.debug("not submitting task for query because terminate() was called: " + entry.getIdentifier()); } private void submitForSync(SyncStorage source, ObjectContext objectContext) { if (syncControl.isRunning()) { SyncTask syncTask = new SyncTask(objectContext, source, firstFilter, verifier, dbService, this, syncControl, stats); syncExecutor.blockingSubmit(syncTask); } else { log.debug("not submitting task for sync because terminate() was called: " + objectContext.getSourceSummary().getIdentifier()); } } private void submitForSync(SyncStorage source, ObjectSummary summary) { ObjectContext objectContext = new ObjectContext(); objectContext.setSourceSummary(summary); objectContext.setOptions(syncConfig.getOptions()); objectContext.setStatus(ObjectStatus.Queue); submitForSync(source, objectContext); } @Override public void submitForRetry(final SyncStorage source, final ObjectContext objectContext, Throwable t) throws Throwable { if (objectContext.getObject() == null || objectContext.getFailures() + 1 > syncConfig.getOptions().getRetryAttempts()) throw t; objectContext.incFailures(); // prepare for retry try { if (log.isInfoEnabled()) { log.info( "O--R object " + objectContext.getSourceSummary().getIdentifier() + " failed " + objectContext.getFailures() + " time" + (objectContext.getFailures() > 1 ? "s" : "") + " (queuing for retry)", SyncUtil.getCause(t)); } objectContext.setStatus(ObjectStatus.RetryQueue); dbService.setStatus(objectContext, SyncUtil.summarize(t), false); retrySubmitter.submit(new Runnable() { @Override public void run() { submitForSync(source, objectContext); } }); } catch (Throwable t2) { // could not retry, so bubble original error log.warn("retry for {} failed: {}", objectContext.getSourceSummary().getIdentifier(), SyncUtil.getCause(t2)); throw t; } } protected void cleanup() { safeClose(stats); safeClose(source); if (filters != null) for (SyncFilter filter : filters) { safeClose(filter); } safeClose(target); safeClose(verifier); if (perfScheduler != null) try { perfScheduler.shutdownNow(); } catch (Throwable t) { log.warn("could not shut down perf reporting", t); } } private void safeClose(AutoCloseable closeable) { try { if (closeable != null) closeable.close(); } catch (Throwable t) { log.warn("could not close " + closeable.getClass().getSimpleName(), t); } } public void setThreadCount(int threadCount) { syncConfig.getOptions().setThreadCount(threadCount); if (listExecutor != null) listExecutor.resizeThreadPool(threadCount); if (estimateExecutor != null) estimateExecutor.resizeThreadPool(threadCount); if (queryExecutor != null) queryExecutor.resizeThreadPool(threadCount); if (syncExecutor != null) syncExecutor.resizeThreadPool(threadCount); if (retrySubmitter != null) retrySubmitter.resizeThreadPool(threadCount); } public DbService getDbService() { return dbService; } public void setDbService(DbService dbService) { this.dbService = dbService; } public Throwable getRunError() { return runError; } public SyncStats getStats() { return stats; } public boolean isRunning() { return syncControl.isRunning(); } public boolean isPaused() { return paused; } public boolean isTerminated() { return terminated; } public boolean isEstimating() { return estimateExecutor != null && estimateExecutor.getUnfinishedTasks() > 0; } public long getEstimatedTotalObjects() { if (isEstimating() || syncEstimate == null) return -1; return syncEstimate.getTotalObjectCount(); } public long getEstimatedTotalBytes() { if (isEstimating() || syncEstimate == null) return -1; return syncEstimate.getTotalByteCount(); } public int getActiveQueryThreads() { if (queryExecutor != null) return queryExecutor.getActiveCount(); return 0; } public int getActiveSyncThreads() { int count = 0; if (syncExecutor != null) count += syncExecutor.getActiveCount(); return count; } /** * Counts the objects in the sync queue that have failed at least once (and are waiting to be retried) */ public int getObjectsAwaitingRetry() { if (syncExecutor == null) return 0; int retryCount = 0; for (Runnable runnable : syncExecutor.getQueue().toArray(new Runnable[0])) { if (runnable instanceof EnhancedFutureTask) { EnhancedFutureTask<?> task = (EnhancedFutureTask<?>) runnable; SyncTask syncTask = (SyncTask) task.getRunnable(); if (syncTask.getObjectContext().getStatus() == ObjectStatus.RetryQueue) retryCount++; } } return retryCount; } public SyncConfig getSyncConfig() { return syncConfig; } public void setSyncConfig(SyncConfig syncConfig) { this.syncConfig = syncConfig; } public int getPerfReportSeconds() { return perfReportSeconds; } public void setPerfReportSeconds(int perfReportSeconds) { this.perfReportSeconds = perfReportSeconds; } public SyncStorage<?> getSource() { return source; } public void setSource(SyncStorage<?> source) { this.source = source; } public SyncStorage<?> getTarget() { return target; } public void setTarget(SyncStorage<?> target) { this.target = target; } public List<SyncFilter> getFilters() { return filters; } public void setFilters(List<SyncFilter> filters) { this.filters = filters; } private class QueryTask implements Runnable { private SyncStorage<?> source; private ObjectSummary parent; QueryTask(SyncStorage source, ObjectSummary parent) { this.source = source; this.parent = parent; } @Override public void run() { if (!syncControl.isRunning()) { log.debug("aborting query task because terminate() was called: " + parent.getIdentifier()); return; } try { if (parent.isDirectory()) { log.debug(">>>> querying children of {}", parent.getIdentifier()); for (ObjectSummary child : source.children(parent)) { submitForSync(source, child); if (syncConfig.getOptions().isRecursive() && child.isDirectory()) { log.debug("{} is directory; submitting for query", child); submitForQuery(source, child); } } log.debug("<<<< finished querying children of {}", parent.getIdentifier()); } } catch (Throwable t) { log.warn(">>!! querying children of {} failed: {}", parent.getIdentifier(), SyncUtil.summarize(t)); } } } private class EstimateTask implements Runnable { private String listLine; private ObjectSummary summary; private SyncStorage<?> storage; private SyncEstimate syncEstimate; EstimateTask(ObjectSummary summary, SyncStorage storage, SyncEstimate syncEstimate) { this.summary = summary; this.storage = storage; this.syncEstimate = syncEstimate; } EstimateTask(String listLine, SyncStorage storage, SyncEstimate syncEstimate) { this.listLine = listLine; this.storage = storage; this.syncEstimate = syncEstimate; } @Override public void run() { if (!syncControl.isRunning()) { log.debug("aborting estimate task because terminate() was called: " + summary.getIdentifier()); return; } try { if (summary == null) summary = storage.parseListLine(listLine); syncEstimate.incTotalObjectCount(1); if (summary.isDirectory()) { queryExecutor.blockingSubmit(new Runnable() { @Override public void run() { log.debug("[est.]>>>> querying children of {}", summary.getIdentifier()); for (ObjectSummary child : storage.children(summary)) { estimateExecutor.blockingSubmit(new EstimateTask(child, storage, syncEstimate)); } log.debug("[est.]<<<< finished querying children of {}", summary.getIdentifier()); } }); } else { syncEstimate.incTotalByteCount(summary.getSize()); } } catch (Throwable t) { log.warn("unexpected exception", t); } } } }