Java tutorial
/* * z2env.org - (c) ZFabrik Software KG * * Licensed under Apache 2. * * www.z2-environment.net */ package org.z2env.impl.gitcr; import java.io.File; import java.io.IOException; import java.net.URISyntaxException; import java.util.Properties; import java.util.logging.Level; import java.util.logging.Logger; import org.eclipse.jgit.api.Git; import org.eclipse.jgit.lib.Repository; import org.eclipse.jgit.storage.file.FileRepository; import org.eclipse.jgit.transport.Transport; import org.eclipse.jgit.transport.TransportProtocol; import org.eclipse.jgit.transport.URIish; import org.eclipse.jgit.transport.UsernamePasswordCredentialsProvider; import org.z2env.components.provider.fs.AbstractFileSystemComponentRepository; import org.z2env.components.provider.fs.FileSystemImpl; import org.z2env.components.provider.util.FSComponentRepositoryDB; import org.z2env.impl.helper.GitTools; import org.z2env.util.Foundation; import org.z2env.util.fs.FileUtils; /** * Git based components repository based on {@link AbstractFileSystemComponentRepository} * <p> * The Git CR clones the source repo (i.e. the one defined by <code>gitcr.path</code>) into the z2-repo cache. * On sync the cloned repo pulls all changes from the source repo, so that git does all the ugly stuff :-) * * <p> * Known GitCR properties: * <table> * <tr><td><code>gitcr.uri</code></td><td>The URI to the source repo. Can be an absolute path, a local path relative to .../run/bin, or a remote URL</td></tr> * <tr><td><code>gitcr.priority</code> or <code>gitcr.prio</code></td><td>The priority of the repository</td></tr> * <tr><td><code>gitcr.branch</code></td><td>The branch to clone. Defaults to the active branch of the source repository</td></tr> * </table> * * @author Udo Offermann * */ public class GitComponentRepositoryImpl extends AbstractFileSystemComponentRepository { /** * URI of the git source repository. */ public final static String GITCR_URI = "gitcr.uri"; /** * Branch to clone from the source repository. */ public final static String GITCR_BRANCH = "gitcr.branch"; /** * User name for authentication at the repository */ public final static String GITCR_USER = "gitcr.user"; /** * Password for authentication at the repository */ public final static String GITCR_PWD = "gitcr.password"; /** * network timeout-value in seconds. Defaults to 10seconds. */ public final static String GITCR_TIMEOUT = "gitcr.timeout"; /** * The original Git-Repo */ private URIish originUri; /** * Credentials */ private UsernamePasswordCredentialsProvider credentials; /** * The branch to clone from the original repo */ private String branch; /** * true <=> this repository is optional */ private boolean isOptional; /** * network timeout */ private int timeout; public GitComponentRepositoryImpl(String name, Properties props) { super(name, getPrio(name, props, 500)); // get folder inside the repo-cache z2/work for the cloned git repository. File cacheRoot = getCacheRoot(); FileSystemImpl fs = new FileSystemImpl(new File(cacheRoot, GIT_CLONE_DIR)); setFileSystem(fs); // do we need a connection to the origin repository? this.isOptional = getIsOptional(name, props, true) || super.isRelaxedMode(); // get the original git repo from the props this.originUri = getOrigRepo(name, props, this.isOptional); // get the branch to clone this.branch = getBranchToClone(name, props); // get the credentials for (remote) repositories this.credentials = getCredentials(name, props); this.timeout = getTimeout(name, props, 10); if (!Foundation.isWorker()) { logger.info("Using " + this.toString()); } } /** * Scans for changes in this GitCR which implies: * <ul><li>if <code>current</code> is <code>null</code> the source repository will be cloned into the repository cache</li> * <li>if the repository */ public FSComponentRepositoryDB scan(FSComponentRepositoryDB current) { logger.fine("scanning " + getLabel()); if (current == null || !getFileSystem().getRoot().exists()) { // no DB exists yet, must clone the source repository doClone(); } else { // DB already exists, so try to pull the deltas from the origin. If this fails create a new clone doPull(); } // AbstractFileSystemComponentRepository.scan() performs the the real scan upon clone's working directory return super.scan(current); } /* * clone the repository and clean-up on failure */ private void doClone() { try { cloneRepository(); } catch (Exception e) { // delete any repository ruins FileUtils.delete(getFileSystem().getRoot()); logger.log(Level.WARNING, "WARNING: Failed cloning " + this.toString()); Throwable t = e; do { logger.log(Level.WARNING, "WARNING: caused by " + t.toString()); t = t.getCause(); } while (t != null); switch (GitTools.isValidRepository(this.originUri)) { case invalid: case cannotTell: if (!GitTools.isOriginReachable(this.originUri)) { logger.info("WARNING: '" + this.originUri.getHost() + "' IS NOT reachable!"); } else { logger.info("WARNING: '" + this.originUri.getHumanishName() + "' IS NOT a valid git repository on " + this.originUri.getHost() + "!"); } break; case valid: // what could it be? logger.info("WARNING: '" + this.originUri.getHumanishName() + "' is valid git repository on " + this.originUri.getHost() + "!"); break; } // something went wrong - maybe we are offline. if (this.isOptional) { logger.info("WARNING: '" + this.originUri.getHumanishName() + "' will be ignored because it's an optional repository or running in relaxed mode."); } else { throw new CloneFailedException(this, e); } } } /* * Clones the source-repository (defined by gitcr.uri) to the repository cache (defined by {@link AbstractFileSystemComponentRepository#getFileSystem()}). * This method does not perform any checks whether a clone already exists etc. */ private void cloneRepository() throws Exception { Repository clonedRepo = null; logger.info("Cloning " + getLabel() + ". This will take some seconds..."); // purging the folder allows to call clone in any situation FileUtils.delete(getFileSystem().getRoot()); try { // no clone available so far! Create it now... long tStart = System.currentTimeMillis(); // clone the origin repository into the z2 work folder clonedRepo = GitTools.cloneRepository(this.originUri, getFileSystem().getRoot(), this.credentials, this.timeout); // switch to target branch GitTools.switchBranch(clonedRepo, this.branch); logger.info("Created working-clone within " + (System.currentTimeMillis() - tStart) + " msec for " + getLabel()); } finally { try { clonedRepo.close(); } catch (Exception e) { /* ignore */ } } } /* * pull the deltas and try clone on failure */ private void doPull() { Repository clonedRepo = null; try { clonedRepo = new FileRepository(new File(getFileSystem().getRoot(), ".git")); pullRepository(clonedRepo); } catch (Exception e) { // pulling failed! // just log a short description logger.log(Level.WARNING, "WARNING: Updating " + this.toString() + " failed with exception: "); logger.log(Level.WARNING, "WARNING: " + e.toString()); switch (GitTools.isValidRepository(this.originUri)) { case invalid: case cannotTell: if (!GitTools.isOriginReachable(this.originUri)) { logger.info("WARNING: '" + this.originUri.getHost() + "' IS NOT reachable!"); } else { logger.info("WARNING: '" + this.originUri.getHumanishName() + "' IS NOT a valid git repository on " + this.originUri.getHost() + "!"); } if (this.isOptional) { // origin is not reachable, but repository is optional (maybe we are offline). Log this incident and try to continue logger.info("WARNING: '" + this.originUri.getHumanishName() + "' will be ignored because it's an optional repository or running in relaxed mode"); } else { // throw exception in production mode throw new PullFailedException(this, e); } break; case valid: // Origin is reachable which implies something went wrong locally. It could be for example that the fetch was successful but the merged failed. // In this case we will purge this clone and start from scratch. logger.info("WARNING: Will discard this clone and create a new one."); clonedRepo.close(); doClone(); } } finally { try { clonedRepo.close(); } catch (Exception e) { /* ignore */ } } } /* * Pulls the deltas from the source-repository (defined by gitcr.uri) into the repository cache (defined by {@link AbstractFileSystemComponentRepository#getFileSystem()}). * This method does not perform any checks whether a clone already exists etc. */ private void pullRepository(Repository clonedRepo) throws Exception { // clone already exists. Pull the deltas logger.fine("Pulling deltas from " + getLabel()); long tStart = System.currentTimeMillis(); if (hasUriChanged(clonedRepo)) { logger.info( "WARNING: URI has changed! Working-clone will be purged and a new clone will be created for " + toString()); cloneRepository(); return; } Git clonedGit = new Git(clonedRepo); // get the deltas from the origin repository clonedGit.pull().setCredentialsProvider(this.credentials). // set user/password if defined - can be null setTimeout(this.timeout).call(); logger.info("Pulled deltas within " + (System.currentTimeMillis() - tStart) + "msec from " + toString()); if (hasBranchChanged(clonedRepo)) { logger.info("Branch has changed! Will switch " + toString() + " to " + this.branch); GitTools.switchBranch(clonedRepo, this.branch); } } private boolean hasUriChanged(Repository clonedRepo) throws Exception { String currentRemote = clonedRepo.getConfig().getString("remote", "origin", "url"); boolean result = currentRemote == null || !this.originUri.equals(new URIish(currentRemote)); return result; } private boolean hasBranchChanged(Repository clonedRepo) throws IOException { String currentBranch = clonedRepo.getBranch(); boolean result = !currentBranch.equals(this.branch); return result; } @Override public String toString() { return getLabel(); } public String getLabel() { return "GIT-CR: " + super.toString() + ",origin:" + this.originUri + ",branch:" + this.branch + ",optional:" + this.isOptional; } // -- private --------------------------------------------------------------------------------- private static String normalize(String in) { if (in != null) { in = in.trim(); if (in.length() > 0) { return in; } } return null; } /* * Returns whether this gitcr is an optional repository (i.e. an invalid gitcr.uri property value will be ignored). */ private boolean getIsOptional(String name, Properties props, boolean defValue) { boolean result = false; if (Foundation.isDevelopmentMode()) { // http://redmine.z2-environment.net/issues/911: allow optional repositories only in dev-mode String isOptional = normalize(props.getProperty(COMPONENT_REPO_MODE)); result = isOptional == null ? defValue : COMPONENT_REPO_MODE_RELAXED.equals(isOptional); } logger.fine("read GitCR " + name + "::isOptional= " + result); return result; } /* * Returns the priority of the repository by reading 'gitcr.prio' from the z.properties. */ private static int getPrio(String name, Properties props, int defValue) { int prio; String prios = normalize(props.getProperty(COMPONENT_REPO_PRIO)); if (prios == null) { prio = defValue; } else { try { prio = Integer.parseInt(prios); } catch (NumberFormatException nfe) { throw new IllegalStateException("Git-CR '" + name + ": Invalid priority (" + COMPONENT_REPO_PRIO + ") specification (" + prios + "): " + name, nfe); } } logger.fine("read GitCR " + name + "::" + COMPONENT_REPO_PRIO + " = " + prio); return prio; } /* * Returns the URI to the source repository by reading 'gitcr.uri' from the z.properties */ private static URIish getOrigRepo(String name, Properties props, boolean isOptional) { String uri = normalize(props.getProperty(GITCR_URI)); logger.fine("read GitCR " + name + "::" + GITCR_URI + " = " + uri); if (uri == null) { throw new IllegalStateException("Missing property " + GITCR_URI + " in " + name); } URIish uriish; try { uriish = new URIish(uri); } catch (URISyntaxException e1) { throw new IllegalStateException("Git-CR '" + name + ": '" + uri + "' is not a valid Git-URI"); } // try normalize to work relative to Z2 home if (!uriish.isRemote()) { try { File f = new File(uri); if (!f.isAbsolute()) { // relative! String zHome = normalize(System.getProperty(Foundation.HOME)); if (zHome != null) { f = new File(new File(zHome), uri); } else { f = new File(uri); } } uri = f.getCanonicalPath(); uriish = new URIish(uri); } catch (Exception e1) { throw new IllegalStateException("Git-CR '" + name + ": '" + uri + "' is not a valid Git-URI"); } } if (isOptional) { // skip all checks for optional repositories return uriish; } boolean canHandle = false; for (TransportProtocol transp : Transport.getTransportProtocols()) { if (transp.canHandle(uriish)) { if (!canHandle && transp.getSchemes().contains("file")) { // do some checks in advance File gitDir = new File(uri); String absPath = null; try { absPath = gitDir.getCanonicalPath(); } catch (IOException e) { throw new IllegalStateException("Git-CR '" + name + ": The path " + gitDir + " defined in " + name + "::" + GITCR_URI + " cannot be canonicalized!", e); } if (!gitDir.exists()) { throw new IllegalStateException("Git-CR '" + name + ": The path " + gitDir + " (abs-path: " + absPath + ") defined in " + name + "::" + GITCR_URI + " does not exists!"); } if (!gitDir.isDirectory()) { throw new IllegalStateException("Git-CR '" + name + ": The path " + gitDir + " (abs-path: " + absPath + ") defined in " + name + "::" + GITCR_URI + " is not a directory!"); } if (!gitDir.canRead()) { throw new IllegalStateException("Git-CR '" + name + ": The path " + gitDir + " (abs-path: " + absPath + ") defined in " + name + "::" + GITCR_URI + " cannot be accessed! Please check permissions."); } } canHandle = true; } } if (!canHandle) { throw new IllegalStateException("Git-CR '" + name + ": The uri " + uri + " defined in " + GITCR_URI + " cannot be handled by this git implementation!"); } return uriish; } /* * Returns the branch to clone from the source repository by reading 'gitcr.branch' from the z.properties. */ private static String getBranchToClone(String name, Properties props) { String branch = normalize(props.getProperty(GITCR_BRANCH)); if (branch == null) { throw new IllegalStateException("Missing property " + GITCR_BRANCH + " in " + name); } logger.fine("read GitCR " + name + "::" + GITCR_BRANCH + " = " + branch); return branch; } /* * Returns a JGit-Credentials object containing the property-values for 'gitcr.user' and 'gitcr.password'. * Defaults to null, if no user and password are defined. */ private static UsernamePasswordCredentialsProvider getCredentials(String name, Properties props) { UsernamePasswordCredentialsProvider result = null; String user = normalize(props.getProperty(GITCR_USER)); if (user != null) { String passwd = normalize(props.getProperty(GITCR_PWD)); if (passwd == null) { throw new IllegalStateException( "GitCR '" + name + " has definded " + GITCR_USER + ", but " + GITCR_PWD + " is missing!"); } result = new UsernamePasswordCredentialsProvider(user, passwd); } logger.fine("read GitCR " + name + "::" + GITCR_USER + " = " + user); return result; } private int getTimeout(String name, Properties props, int defValue) { String timeout = normalize(props.getProperty(GITCR_TIMEOUT)); int result = timeout == null ? defValue : Integer.parseInt(timeout); logger.fine("read GitCR " + name + "::" + GITCR_TIMEOUT + " = " + result); return result; } /* the name of the folder containing the cloned repository */ private final static String GIT_CLONE_DIR = "git"; private final static Logger logger = Logger.getLogger(GitComponentRepositoryImpl.class.getName()); @SuppressWarnings("serial") private final static class CloneFailedException extends RuntimeException { public CloneFailedException(GitComponentRepositoryImpl gitcr, Exception e) { super("Failed to clone " + gitcr.toString() + " to " + gitcr.getFileSystem().getRoot(), e); } } @SuppressWarnings("serial") private final static class PullFailedException extends RuntimeException { public PullFailedException(GitComponentRepositoryImpl gitcr, Exception e) { super("Failed to pull from " + gitcr.toString(), e); } } }