Java tutorial
/* * Copyright 2014 gitblit.com. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package com.gitblit.tickets; import java.io.IOException; import java.text.MessageFormat; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Set; import java.util.TreeSet; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicLong; import org.eclipse.jgit.api.errors.ConcurrentRefUpdateException; import org.eclipse.jgit.dircache.DirCache; import org.eclipse.jgit.dircache.DirCacheBuilder; import org.eclipse.jgit.dircache.DirCacheEntry; import org.eclipse.jgit.events.RefsChangedEvent; import org.eclipse.jgit.events.RefsChangedListener; import org.eclipse.jgit.lib.FileMode; import org.eclipse.jgit.lib.ObjectId; import org.eclipse.jgit.lib.ObjectInserter; import org.eclipse.jgit.lib.PersonIdent; import org.eclipse.jgit.lib.Ref; import org.eclipse.jgit.lib.RefRename; import org.eclipse.jgit.lib.RefUpdate.Result; import org.eclipse.jgit.lib.Repository; import org.eclipse.jgit.revwalk.RevCommit; import org.eclipse.jgit.revwalk.RevTree; import org.eclipse.jgit.revwalk.RevWalk; import org.eclipse.jgit.transport.ReceiveCommand; import org.eclipse.jgit.treewalk.CanonicalTreeParser; import org.eclipse.jgit.treewalk.TreeWalk; import com.gitblit.Constants; import com.gitblit.git.ReceiveCommandEvent; import com.gitblit.manager.INotificationManager; import com.gitblit.manager.IPluginManager; import com.gitblit.manager.IRepositoryManager; import com.gitblit.manager.IRuntimeManager; import com.gitblit.manager.IUserManager; import com.gitblit.models.PathModel; import com.gitblit.models.PathModel.PathChangeModel; import com.gitblit.models.RefModel; import com.gitblit.models.RepositoryModel; import com.gitblit.models.TicketModel; import com.gitblit.models.TicketModel.Attachment; import com.gitblit.models.TicketModel.Change; import com.gitblit.utils.ArrayUtils; import com.gitblit.utils.JGitUtils; import com.gitblit.utils.StringUtils; import com.google.inject.Inject; import com.google.inject.Singleton; /** * Implementation of a ticket service based on an orphan branch. All tickets * are serialized as a list of JSON changes and persisted in a hashed directory * structure, similar to the standard git loose object structure. * * @author James Moger * */ @Singleton public class BranchTicketService extends ITicketService implements RefsChangedListener { public static final String BRANCH = "refs/meta/gitblit/tickets"; private static final String JOURNAL = "journal.json"; private static final String ID_PATH = "id/"; private final Map<String, AtomicLong> lastAssignedId; @Inject public BranchTicketService(IRuntimeManager runtimeManager, IPluginManager pluginManager, INotificationManager notificationManager, IUserManager userManager, IRepositoryManager repositoryManager) { super(runtimeManager, pluginManager, notificationManager, userManager, repositoryManager); lastAssignedId = new ConcurrentHashMap<String, AtomicLong>(); // register the branch ticket service for repository ref changes Repository.getGlobalListenerList().addRefsChangedListener(this); } @Override public void onStart() { log.info("{} started", getClass().getSimpleName()); } @Override protected void resetCachesImpl() { lastAssignedId.clear(); } @Override protected void resetCachesImpl(RepositoryModel repository) { if (lastAssignedId.containsKey(repository.name)) { lastAssignedId.get(repository.name).set(0); } } @Override protected void close() { } /** * Listen for tickets branch changes and (re)index tickets, as appropriate */ @Override public synchronized void onRefsChanged(RefsChangedEvent event) { if (!(event instanceof ReceiveCommandEvent)) { return; } ReceiveCommandEvent branchUpdate = (ReceiveCommandEvent) event; RepositoryModel repository = branchUpdate.model; ReceiveCommand cmd = branchUpdate.cmd; try { switch (cmd.getType()) { case CREATE: case UPDATE_NONFASTFORWARD: // reindex everything reindex(repository); break; case UPDATE: // incrementally index ticket updates resetCaches(repository); long start = System.nanoTime(); log.info("incrementally indexing {} ticket branch due to received ref update", repository.name); Repository db = repositoryManager.getRepository(repository.name); try { Set<Long> ids = new HashSet<Long>(); List<PathChangeModel> paths = JGitUtils.getFilesInRange(db, cmd.getOldId().getName(), cmd.getNewId().getName()); for (PathChangeModel path : paths) { String name = path.name.substring(path.name.lastIndexOf('/') + 1); if (!JOURNAL.equals(name)) { continue; } String tid = path.path.split("/")[2]; long ticketId = Long.parseLong(tid); if (!ids.contains(ticketId)) { ids.add(ticketId); TicketModel ticket = getTicket(repository, ticketId); log.info(MessageFormat.format("indexing ticket #{0,number,0}: {1}", ticketId, ticket.title)); indexer.index(ticket); } } long end = System.nanoTime(); log.info("incremental indexing of {0} ticket(s) completed in {1} msecs", ids.size(), TimeUnit.NANOSECONDS.toMillis(end - start)); } finally { db.close(); } break; default: log.warn("Unexpected receive type {} in BranchTicketService.onRefsChanged" + cmd.getType()); break; } } catch (Exception e) { log.error("failed to reindex " + repository.name, e); } } /** * Returns a RefModel for the refs/meta/gitblit/tickets branch in the repository. * If the branch can not be found, null is returned. * * @return a refmodel for the gitblit tickets branch or null */ private RefModel getTicketsBranch(Repository db) { List<RefModel> refs = JGitUtils.getRefs(db, "refs/"); Ref oldRef = null; for (RefModel ref : refs) { if (ref.reference.getName().equals(BRANCH)) { return ref; } else if (ref.reference.getName().equals("refs/gitblit/tickets")) { oldRef = ref.reference; } } if (oldRef != null) { // rename old ref to refs/meta/gitblit/tickets RefRename cmd; try { cmd = db.renameRef(oldRef.getName(), BRANCH); cmd.setRefLogIdent(new PersonIdent("Gitblit", "gitblit@localhost")); cmd.setRefLogMessage("renamed " + oldRef.getName() + " => " + BRANCH); Result res = cmd.rename(); switch (res) { case RENAMED: log.info(db.getDirectory() + " " + cmd.getRefLogMessage()); return getTicketsBranch(db); default: log.error("failed to rename " + oldRef.getName() + " => " + BRANCH + " (" + res.name() + ")"); } } catch (IOException e) { log.error("failed to rename tickets branch", e); } } return null; } /** * Creates the refs/meta/gitblit/tickets branch. * @param db */ private void createTicketsBranch(Repository db) { JGitUtils.createOrphanBranch(db, BRANCH, null); } /** * Returns the ticket path. This follows the same scheme as Git's object * store path where the first two characters of the hash id are the root * folder with the remaining characters as a subfolder within that folder. * * @param ticketId * @return the root path of the ticket content on the refs/meta/gitblit/tickets branch */ private String toTicketPath(long ticketId) { StringBuilder sb = new StringBuilder(); sb.append(ID_PATH); long m = ticketId % 100L; if (m < 10) { sb.append('0'); } sb.append(m); sb.append('/'); sb.append(ticketId); return sb.toString(); } /** * Returns the path to the attachment for the specified ticket. * * @param ticketId * @param filename * @return the path to the specified attachment */ private String toAttachmentPath(long ticketId, String filename) { return toTicketPath(ticketId) + "/attachments/" + filename; } /** * Reads a file from the tickets branch. * * @param db * @param file * @return the file content or null */ private String readTicketsFile(Repository db, String file) { RevWalk rw = null; try { ObjectId treeId = db.resolve(BRANCH + "^{tree}"); if (treeId == null) { return null; } rw = new RevWalk(db); RevTree tree = rw.lookupTree(treeId); if (tree != null) { return JGitUtils.getStringContent(db, tree, file, Constants.ENCODING); } } catch (IOException e) { log.error("failed to read " + file, e); } finally { if (rw != null) { rw.close(); } } return null; } /** * Writes a file to the tickets branch. * * @param db * @param file * @param content * @param createdBy * @param msg */ private void writeTicketsFile(Repository db, String file, String content, String createdBy, String msg) { if (getTicketsBranch(db) == null) { createTicketsBranch(db); } DirCache newIndex = DirCache.newInCore(); DirCacheBuilder builder = newIndex.builder(); ObjectInserter inserter = db.newObjectInserter(); try { // create an index entry for the revised index final DirCacheEntry idIndexEntry = new DirCacheEntry(file); idIndexEntry.setLength(content.length()); idIndexEntry.setLastModified(System.currentTimeMillis()); idIndexEntry.setFileMode(FileMode.REGULAR_FILE); // insert new ticket index idIndexEntry.setObjectId( inserter.insert(org.eclipse.jgit.lib.Constants.OBJ_BLOB, content.getBytes(Constants.ENCODING))); // add to temporary in-core index builder.add(idIndexEntry); Set<String> ignorePaths = new HashSet<String>(); ignorePaths.add(file); for (DirCacheEntry entry : JGitUtils.getTreeEntries(db, BRANCH, ignorePaths)) { builder.add(entry); } // finish temporary in-core index used for this commit builder.finish(); // commit the change commitIndex(db, newIndex, createdBy, msg); } catch (ConcurrentRefUpdateException e) { log.error("", e); } catch (IOException e) { log.error("", e); } finally { inserter.close(); } } /** * Ensures that we have a ticket for this ticket id. * * @param repository * @param ticketId * @return true if the ticket exists */ @Override public boolean hasTicket(RepositoryModel repository, long ticketId) { boolean hasTicket = false; Repository db = repositoryManager.getRepository(repository.name); try { RefModel ticketsBranch = getTicketsBranch(db); if (ticketsBranch == null) { return false; } String ticketPath = toTicketPath(ticketId); RevCommit tip = JGitUtils.getCommit(db, BRANCH); hasTicket = !JGitUtils.getFilesInPath(db, ticketPath, tip).isEmpty(); } finally { db.close(); } return hasTicket; } /** * Returns the assigned ticket ids. * * @return the assigned ticket ids */ @Override public synchronized Set<Long> getIds(RepositoryModel repository) { Repository db = repositoryManager.getRepository(repository.name); try { if (getTicketsBranch(db) == null) { return Collections.emptySet(); } Set<Long> ids = new TreeSet<Long>(); List<PathModel> paths = JGitUtils.getDocuments(db, Arrays.asList("json"), BRANCH); for (PathModel path : paths) { String name = path.name.substring(path.name.lastIndexOf('/') + 1); if (!JOURNAL.equals(name)) { continue; } String tid = path.path.split("/")[2]; long ticketId = Long.parseLong(tid); ids.add(ticketId); } return ids; } finally { if (db != null) { db.close(); } } } /** * Assigns a new ticket id. * * @param repository * @return a new long id */ @Override public synchronized long assignNewId(RepositoryModel repository) { long newId = 0L; Repository db = repositoryManager.getRepository(repository.name); try { if (getTicketsBranch(db) == null) { createTicketsBranch(db); } // identify current highest ticket id by scanning the paths in the tip tree if (!lastAssignedId.containsKey(repository.name)) { lastAssignedId.put(repository.name, new AtomicLong(0)); } AtomicLong lastId = lastAssignedId.get(repository.name); if (lastId.get() <= 0) { Set<Long> ids = getIds(repository); for (long id : ids) { if (id > lastId.get()) { lastId.set(id); } } } // assign the id and touch an empty journal to hold it's place newId = lastId.incrementAndGet(); String journalPath = toTicketPath(newId) + "/" + JOURNAL; writeTicketsFile(db, journalPath, "", "gitblit", "assigned id #" + newId); } finally { db.close(); } return newId; } /** * Returns all the tickets in the repository. Querying tickets from the * repository requires deserializing all tickets. This is an expensive * process and not recommended. Tickets are indexed by Lucene and queries * should be executed against that index. * * @param repository * @param filter * optional filter to only return matching results * @return a list of tickets */ @Override public List<TicketModel> getTickets(RepositoryModel repository, TicketFilter filter) { List<TicketModel> list = new ArrayList<TicketModel>(); Repository db = repositoryManager.getRepository(repository.name); try { RefModel ticketsBranch = getTicketsBranch(db); if (ticketsBranch == null) { return list; } // Collect the set of all json files List<PathModel> paths = JGitUtils.getDocuments(db, Arrays.asList("json"), BRANCH); // Deserialize each ticket and optionally filter out unwanted tickets for (PathModel path : paths) { String name = path.name.substring(path.name.lastIndexOf('/') + 1); if (!JOURNAL.equals(name)) { continue; } String json = readTicketsFile(db, path.path); if (StringUtils.isEmpty(json)) { // journal was touched but no changes were written continue; } try { // Reconstruct ticketId from the path // id/26/326/journal.json String tid = path.path.split("/")[2]; long ticketId = Long.parseLong(tid); List<Change> changes = TicketSerializer.deserializeJournal(json); if (ArrayUtils.isEmpty(changes)) { log.warn("Empty journal for {}:{}", repository, path.path); continue; } TicketModel ticket = TicketModel.buildTicket(changes); ticket.project = repository.projectPath; ticket.repository = repository.name; ticket.number = ticketId; // add the ticket, conditionally, to the list if (filter == null) { list.add(ticket); } else { if (filter.accept(ticket)) { list.add(ticket); } } } catch (Exception e) { log.error("failed to deserialize {}/{}\n{}", new Object[] { repository, path.path, e.getMessage() }); log.error(null, e); } } // sort the tickets by creation Collections.sort(list); return list; } finally { db.close(); } } /** * Retrieves the ticket from the repository by first looking-up the changeId * associated with the ticketId. * * @param repository * @param ticketId * @return a ticket, if it exists, otherwise null */ @Override protected TicketModel getTicketImpl(RepositoryModel repository, long ticketId) { Repository db = repositoryManager.getRepository(repository.name); try { List<Change> changes = getJournal(db, ticketId); if (ArrayUtils.isEmpty(changes)) { log.warn("Empty journal for {}:{}", repository, ticketId); return null; } TicketModel ticket = TicketModel.buildTicket(changes); if (ticket != null) { ticket.project = repository.projectPath; ticket.repository = repository.name; ticket.number = ticketId; } return ticket; } finally { db.close(); } } /** * Retrieves the journal for the ticket. * * @param repository * @param ticketId * @return a journal, if it exists, otherwise null */ @Override protected List<Change> getJournalImpl(RepositoryModel repository, long ticketId) { Repository db = repositoryManager.getRepository(repository.name); try { List<Change> changes = getJournal(db, ticketId); if (ArrayUtils.isEmpty(changes)) { log.warn("Empty journal for {}:{}", repository, ticketId); return null; } return changes; } finally { db.close(); } } /** * Returns the journal for the specified ticket. * * @param db * @param ticketId * @return a list of changes */ private List<Change> getJournal(Repository db, long ticketId) { RefModel ticketsBranch = getTicketsBranch(db); if (ticketsBranch == null) { return new ArrayList<Change>(); } if (ticketId <= 0L) { return new ArrayList<Change>(); } String journalPath = toTicketPath(ticketId) + "/" + JOURNAL; String json = readTicketsFile(db, journalPath); if (StringUtils.isEmpty(json)) { return new ArrayList<Change>(); } List<Change> list = TicketSerializer.deserializeJournal(json); return list; } @Override public boolean supportsAttachments() { return true; } /** * Retrieves the specified attachment from a ticket. * * @param repository * @param ticketId * @param filename * @return an attachment, if found, null otherwise */ @Override public Attachment getAttachment(RepositoryModel repository, long ticketId, String filename) { if (ticketId <= 0L) { return null; } // deserialize the ticket model so that we have the attachment metadata TicketModel ticket = getTicket(repository, ticketId); Attachment attachment = ticket.getAttachment(filename); // attachment not found if (attachment == null) { return null; } // retrieve the attachment content Repository db = repositoryManager.getRepository(repository.name); try { String attachmentPath = toAttachmentPath(ticketId, attachment.name); RevTree tree = JGitUtils.getCommit(db, BRANCH).getTree(); byte[] content = JGitUtils.getByteContent(db, tree, attachmentPath, false); attachment.content = content; attachment.size = content.length; return attachment; } finally { db.close(); } } /** * Deletes a ticket from the repository. * * @param ticket * @return true if successful */ @Override protected synchronized boolean deleteTicketImpl(RepositoryModel repository, TicketModel ticket, String deletedBy) { if (ticket == null) { throw new RuntimeException("must specify a ticket!"); } boolean success = false; Repository db = repositoryManager.getRepository(ticket.repository); try { RefModel ticketsBranch = getTicketsBranch(db); if (ticketsBranch == null) { throw new RuntimeException(BRANCH + " does not exist!"); } String ticketPath = toTicketPath(ticket.number); TreeWalk treeWalk = null; try { ObjectId treeId = db.resolve(BRANCH + "^{tree}"); // Create the in-memory index of the new/updated ticket DirCache index = DirCache.newInCore(); DirCacheBuilder builder = index.builder(); // Traverse HEAD to add all other paths treeWalk = new TreeWalk(db); int hIdx = -1; if (treeId != null) { hIdx = treeWalk.addTree(treeId); } treeWalk.setRecursive(true); while (treeWalk.next()) { String path = treeWalk.getPathString(); CanonicalTreeParser hTree = null; if (hIdx != -1) { hTree = treeWalk.getTree(hIdx, CanonicalTreeParser.class); } if (!path.startsWith(ticketPath)) { // add entries from HEAD for all other paths if (hTree != null) { final DirCacheEntry entry = new DirCacheEntry(path); entry.setObjectId(hTree.getEntryObjectId()); entry.setFileMode(hTree.getEntryFileMode()); // add to temporary in-core index builder.add(entry); } } } // finish temporary in-core index used for this commit builder.finish(); success = commitIndex(db, index, deletedBy, "- " + ticket.number); } catch (Throwable t) { log.error(MessageFormat.format("Failed to delete ticket {0,number,0} from {1}", ticket.number, db.getDirectory()), t); } finally { // release the treewalk if (treeWalk != null) { treeWalk.close(); } } } finally { db.close(); } return success; } /** * Commit a ticket change to the repository. * * @param repository * @param ticketId * @param change * @return true, if the change was committed */ @Override protected synchronized boolean commitChangeImpl(RepositoryModel repository, long ticketId, Change change) { boolean success = false; Repository db = repositoryManager.getRepository(repository.name); try { DirCache index = createIndex(db, ticketId, change); success = commitIndex(db, index, change.author, "#" + ticketId); } catch (Throwable t) { log.error(MessageFormat.format("Failed to commit ticket {0,number,0} to {1}", ticketId, db.getDirectory()), t); } finally { db.close(); } return success; } /** * Creates an in-memory index of the ticket change. * * @param changeId * @param change * @return an in-memory index * @throws IOException */ private DirCache createIndex(Repository db, long ticketId, Change change) throws IOException, ClassNotFoundException, NoSuchFieldException { String ticketPath = toTicketPath(ticketId); DirCache newIndex = DirCache.newInCore(); DirCacheBuilder builder = newIndex.builder(); ObjectInserter inserter = db.newObjectInserter(); Set<String> ignorePaths = new TreeSet<String>(); try { // create/update the journal // exclude the attachment content List<Change> changes = getJournal(db, ticketId); changes.add(change); String journal = TicketSerializer.serializeJournal(changes).trim(); byte[] journalBytes = journal.getBytes(Constants.ENCODING); String journalPath = ticketPath + "/" + JOURNAL; final DirCacheEntry journalEntry = new DirCacheEntry(journalPath); journalEntry.setLength(journalBytes.length); journalEntry.setLastModified(change.date.getTime()); journalEntry.setFileMode(FileMode.REGULAR_FILE); journalEntry.setObjectId(inserter.insert(org.eclipse.jgit.lib.Constants.OBJ_BLOB, journalBytes)); // add journal to index builder.add(journalEntry); ignorePaths.add(journalEntry.getPathString()); // Add any attachments to the index if (change.hasAttachments()) { for (Attachment attachment : change.attachments) { // build a path name for the attachment and mark as ignored String path = toAttachmentPath(ticketId, attachment.name); ignorePaths.add(path); // create an index entry for this attachment final DirCacheEntry entry = new DirCacheEntry(path); entry.setLength(attachment.content.length); entry.setLastModified(change.date.getTime()); entry.setFileMode(FileMode.REGULAR_FILE); // insert object entry.setObjectId(inserter.insert(org.eclipse.jgit.lib.Constants.OBJ_BLOB, attachment.content)); // add to temporary in-core index builder.add(entry); } } for (DirCacheEntry entry : JGitUtils.getTreeEntries(db, BRANCH, ignorePaths)) { builder.add(entry); } // finish the index builder.finish(); } finally { inserter.close(); } return newIndex; } private boolean commitIndex(Repository db, DirCache index, String author, String message) throws IOException, ConcurrentRefUpdateException { final boolean forceCommit = true; boolean success = false; ObjectId headId = db.resolve(BRANCH + "^{commit}"); if (headId == null) { // create the branch createTicketsBranch(db); } success = JGitUtils.commitIndex(db, BRANCH, index, headId, forceCommit, author, "gitblit@localhost", message); return success; } @Override protected boolean deleteAllImpl(RepositoryModel repository) { Repository db = repositoryManager.getRepository(repository.name); try { RefModel branch = getTicketsBranch(db); if (branch != null) { return JGitUtils.deleteBranchRef(db, BRANCH); } return true; } catch (Exception e) { log.error(null, e); } finally { if (db != null) { db.close(); } } return false; } @Override protected boolean renameImpl(RepositoryModel oldRepository, RepositoryModel newRepository) { return true; } @Override public String toString() { return getClass().getSimpleName(); } }