Java tutorial
// Copyright (C) 2017 The Android Open Source Project // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package com.google.gerrit.server.notedb; import static com.google.common.base.Preconditions.checkState; import static java.util.concurrent.TimeUnit.MILLISECONDS; import static java.util.concurrent.TimeUnit.NANOSECONDS; import static java.util.concurrent.TimeUnit.SECONDS; import com.github.rholder.retry.RetryException; import com.github.rholder.retry.Retryer; import com.github.rholder.retry.RetryerBuilder; import com.github.rholder.retry.StopStrategies; import com.github.rholder.retry.WaitStrategies; import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Stopwatch; import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; import com.google.gerrit.common.Nullable; import com.google.gerrit.common.TimeUtil; import com.google.gerrit.extensions.restapi.RestApiException; import com.google.gerrit.reviewdb.client.Account; import com.google.gerrit.reviewdb.client.Change; import com.google.gerrit.reviewdb.client.Project; import com.google.gerrit.reviewdb.client.RefNames; import com.google.gerrit.reviewdb.server.ReviewDb; import com.google.gerrit.reviewdb.server.ReviewDbUtil; import com.google.gerrit.server.InternalUser; import com.google.gerrit.server.config.AllUsersName; import com.google.gerrit.server.config.GerritServerConfig; import com.google.gerrit.server.git.GitRepositoryManager; import com.google.gerrit.server.git.RepoRefCache; import com.google.gerrit.server.index.change.ChangeField; import com.google.gerrit.server.notedb.NoteDbChangeState.PrimaryStorage; import com.google.gerrit.server.notedb.NoteDbChangeState.RefState; import com.google.gerrit.server.notedb.rebuild.ChangeRebuilder; import com.google.gerrit.server.project.ChangeControl; import com.google.gerrit.server.query.change.ChangeData; import com.google.gerrit.server.query.change.InternalChangeQuery; import com.google.gerrit.server.update.BatchUpdate; import com.google.gerrit.server.update.BatchUpdateOp; import com.google.gerrit.server.update.ChangeContext; import com.google.gerrit.server.update.UpdateException; import com.google.gwtorm.server.AtomicUpdate; import com.google.gwtorm.server.OrmException; import com.google.gwtorm.server.OrmRuntimeException; import com.google.inject.Inject; import com.google.inject.Provider; import com.google.inject.Singleton; import java.io.IOException; import java.sql.Timestamp; import java.util.List; import java.util.Objects; import java.util.Optional; import java.util.Set; import java.util.TreeSet; import java.util.concurrent.ExecutionException; import java.util.concurrent.atomic.AtomicBoolean; import org.eclipse.jgit.errors.RepositoryNotFoundException; import org.eclipse.jgit.lib.Config; import org.eclipse.jgit.lib.ObjectId; import org.eclipse.jgit.lib.Ref; import org.eclipse.jgit.lib.Repository; import org.slf4j.Logger; import org.slf4j.LoggerFactory; /** Helper to migrate the {@link PrimaryStorage} of individual changes. */ @Singleton public class PrimaryStorageMigrator { private static final Logger log = LoggerFactory.getLogger(PrimaryStorageMigrator.class); private final AllUsersName allUsers; private final BatchUpdate.Factory batchUpdateFactory; private final ChangeControl.GenericFactory changeControlFactory; private final ChangeRebuilder rebuilder; private final ChangeUpdate.Factory updateFactory; private final GitRepositoryManager repoManager; private final InternalUser.Factory internalUserFactory; private final Provider<InternalChangeQuery> queryProvider; private final Provider<ReviewDb> db; private final long skewMs; private final long timeoutMs; private final Retryer<NoteDbChangeState> testEnsureRebuiltRetryer; @Inject PrimaryStorageMigrator(@GerritServerConfig Config cfg, Provider<ReviewDb> db, GitRepositoryManager repoManager, AllUsersName allUsers, ChangeRebuilder rebuilder, ChangeControl.GenericFactory changeControlFactory, Provider<InternalChangeQuery> queryProvider, ChangeUpdate.Factory updateFactory, InternalUser.Factory internalUserFactory, BatchUpdate.Factory batchUpdateFactory) { this(cfg, db, repoManager, allUsers, rebuilder, null, changeControlFactory, queryProvider, updateFactory, internalUserFactory, batchUpdateFactory); } @VisibleForTesting public PrimaryStorageMigrator(Config cfg, Provider<ReviewDb> db, GitRepositoryManager repoManager, AllUsersName allUsers, ChangeRebuilder rebuilder, @Nullable Retryer<NoteDbChangeState> testEnsureRebuiltRetryer, ChangeControl.GenericFactory changeControlFactory, Provider<InternalChangeQuery> queryProvider, ChangeUpdate.Factory updateFactory, InternalUser.Factory internalUserFactory, BatchUpdate.Factory batchUpdateFactory) { this.db = db; this.repoManager = repoManager; this.allUsers = allUsers; this.rebuilder = rebuilder; this.testEnsureRebuiltRetryer = testEnsureRebuiltRetryer; this.changeControlFactory = changeControlFactory; this.queryProvider = queryProvider; this.updateFactory = updateFactory; this.internalUserFactory = internalUserFactory; this.batchUpdateFactory = batchUpdateFactory; skewMs = NoteDbChangeState.getReadOnlySkew(cfg); String s = "notedb"; timeoutMs = cfg.getTimeUnit(s, null, "primaryStorageMigrationTimeout", MILLISECONDS.convert(60, SECONDS), MILLISECONDS); } /** * Migrate a change's primary storage from ReviewDb to NoteDb. * * <p>This method will return only if the primary storage of the change is NoteDb afterwards. (It * may return early if the primary storage was already NoteDb.) * * <p>If this method throws an exception, then the primary storage of the change is probably not * NoteDb. (It is possible that the primary storage of the change is NoteDb in this case, but * there was an error reading the state.) Moreover, after an exception, the change may be * read-only until a lease expires. If the caller chooses to retry, they should wait until the * read-only lease expires; this method will fail relatively quickly if called on a read-only * change. * * <p>Note that if the change is read-only after this method throws an exception, that does not * necessarily guarantee that the read-only lease was acquired during that particular method * invocation; this call may have in fact failed because another thread acquired the lease first. * * @param id change ID. * @throws OrmException if a ReviewDb-level error occurs. * @throws IOException if a repo-level error occurs. */ public void migrateToNoteDbPrimary(Change.Id id) throws OrmException, IOException { // Since there are multiple non-atomic steps in this method, we need to // consider what happens when there is another writer concurrent with the // thread executing this method. // // Let: // * OR = other writer writes noteDbState & new data to ReviewDb (in one // transaction) // * ON = other writer writes to NoteDb // * MRO = migrator sets state to read-only // * MR = ensureRebuilt writes rebuilt noteDbState to ReviewDb (but does not // otherwise update ReviewDb in this transaction) // * MN = ensureRebuilt writes rebuilt state to NoteDb // // Consider all the interleavings of these operations. // // * OR,ON,MRO,... // Other writer completes before migrator begins; this is not a concurrent // write. // * MRO,...,OR,... // OR will fail, since it atomically checks that the noteDbState is not // read-only before proceeding. This results in an exception, but not a // concurrent write. // // Thus all the "interesting" interleavings start with OR,MRO, and differ on // where ON falls relative to MR/MN. // // * OR,MRO,ON,MR,MN // The other NoteDb write succeeds despite the noteDbState being // read-only. Because the read-only state from MRO includes the update // from OR, the change is up-to-date at this point. Thus MR,MN is a no-op. // The end result is an up-to-date, read-only change. // // * OR,MRO,MR,ON,MN // The change is out-of-date when ensureRebuilt begins, because OR // succeeded but the corresponding ON has not happened yet. ON will // succeed, because there have been no intervening NoteDb writes. MN will // fail, because ON updated the state in NoteDb to something other than // what MR claimed. This leaves the change in an out-of-date, read-only // state. // // If this method threw an exception in this case, the change would // eventually switch back to read-write when the read-only lease expires, // so this situation is recoverable. However, it would be inconvenient for // a change to be read-only for so long. // // Thus, as an optimization, we have a retry loop that attempts // ensureRebuilt while still holding the same read-only lease. This // effectively results in the interleaving OR,MR,ON,MR,MN; in contrast // with the previous case, here, MR/MN actually rebuilds the change. In // the case of a write failure, MR/MN might fail and get retried again. If // it exceeds the maximum number of retries, an exception is thrown. // // * OR,MRO,MR,MN,ON // The change is out-of-date when ensureRebuilt begins. The change is // rebuilt, leaving a new state in NoteDb. ON will fail, because the old // NoteDb state has changed since the ref state was read when the update // began (prior to OR). This results in an exception from ON, but the end // result is still an up-to-date, read-only change. The end user that // initiated the other write observes an error, but this is no different // from other errors that need retrying, e.g. due to a backend write // failure. Stopwatch sw = Stopwatch.createStarted(); Change readOnlyChange = setReadOnlyInReviewDb(id); // MRO if (readOnlyChange == null) { return; // Already migrated. } NoteDbChangeState rebuiltState; try { // MR,MN rebuiltState = ensureRebuiltRetryer(sw).call( () -> ensureRebuilt(readOnlyChange.getProject(), id, NoteDbChangeState.parse(readOnlyChange))); } catch (RetryException | ExecutionException e) { throw new OrmException(e); } // At this point, the noteDbState in ReviewDb is read-only, and it is // guaranteed to match the state actually in NoteDb. Now it is safe to set // the primary storage to NoteDb. setPrimaryStorageNoteDb(id, rebuiltState); log.info("Migrated change {} to NoteDb primary in {}ms", id, sw.elapsed(MILLISECONDS)); } private Change setReadOnlyInReviewDb(Change.Id id) throws OrmException { AtomicBoolean alreadyMigrated = new AtomicBoolean(false); Change result = db().changes().atomicUpdate(id, new AtomicUpdate<Change>() { @Override public Change update(Change change) { NoteDbChangeState state = NoteDbChangeState.parse(change); if (state == null) { // Could rebuild the change here, but that's more complexity, and this // really shouldn't happen. throw new OrmRuntimeException("change " + id + " has no note_db_state; rebuild it first"); } // If the change is already read-only, then the lease is held by another // (likely failed) migrator thread. Fail early, as we can't take over // the lease. NoteDbChangeState.checkNotReadOnly(change, skewMs); if (state.getPrimaryStorage() != PrimaryStorage.NOTE_DB) { Timestamp now = TimeUtil.nowTs(); Timestamp until = new Timestamp(now.getTime() + timeoutMs); change.setNoteDbState(state.withReadOnlyUntil(until).toString()); } else { alreadyMigrated.set(true); } return change; } }); return alreadyMigrated.get() ? null : result; } private Retryer<NoteDbChangeState> ensureRebuiltRetryer(Stopwatch sw) { if (testEnsureRebuiltRetryer != null) { return testEnsureRebuiltRetryer; } // Retry the ensureRebuilt step with backoff until half the timeout has // expired, leaving the remaining half for the rest of the steps. long remainingNanos = (MILLISECONDS.toNanos(timeoutMs) / 2) - sw.elapsed(NANOSECONDS); remainingNanos = Math.max(remainingNanos, 0); return RetryerBuilder.<NoteDbChangeState>newBuilder() .retryIfException(e -> (e instanceof IOException) || (e instanceof OrmException)) .withWaitStrategy(WaitStrategies.join(WaitStrategies.exponentialWait(250, MILLISECONDS), WaitStrategies.randomWait(50, MILLISECONDS))) .withStopStrategy(StopStrategies.stopAfterDelay(remainingNanos, NANOSECONDS)).build(); } private NoteDbChangeState ensureRebuilt(Project.NameKey project, Change.Id id, NoteDbChangeState readOnlyState) throws IOException, OrmException, RepositoryNotFoundException { try (Repository changeRepo = repoManager.openRepository(project); Repository allUsersRepo = repoManager.openRepository(allUsers)) { if (!readOnlyState.isUpToDate(new RepoRefCache(changeRepo), new RepoRefCache(allUsersRepo))) { NoteDbUpdateManager.Result r = rebuilder.rebuildEvenIfReadOnly(db(), id); checkState(r.newState().getReadOnlyUntil().equals(readOnlyState.getReadOnlyUntil()), "state after rebuilding has different read-only lease: %s != %s", r.newState(), readOnlyState); readOnlyState = r.newState(); } } return readOnlyState; } private void setPrimaryStorageNoteDb(Change.Id id, NoteDbChangeState expectedState) throws OrmException { db().changes().atomicUpdate(id, new AtomicUpdate<Change>() { @Override public Change update(Change change) { NoteDbChangeState state = NoteDbChangeState.parse(change); if (!Objects.equals(state, expectedState)) { throw new OrmRuntimeException(badState(state, expectedState)); } Timestamp until = state.getReadOnlyUntil().get(); if (TimeUtil.nowTs().after(until)) { throw new OrmRuntimeException("read-only lease on change " + id + " expired at " + until); } change.setNoteDbState(NoteDbChangeState.NOTE_DB_PRIMARY_STATE); return change; } }); } private ReviewDb db() { return ReviewDbUtil.unwrapDb(db.get()); } private String badState(NoteDbChangeState actual, NoteDbChangeState expected) { return "state changed unexpectedly: " + actual + " != " + expected; } public void migrateToReviewDbPrimary(Change.Id id, @Nullable Project.NameKey project) throws OrmException, IOException { // Migrating back to ReviewDb primary is much simpler than the original migration to NoteDb // primary, because when NoteDb is primary, each write only goes to one storage location rather // than both. We only need to consider whether a concurrent writer (OR) conflicts with the first // setReadOnlyInNoteDb step (MR) in this method. // // If OR wins, then either: // * MR will set read-only after OR is completed, which is not a concurrent write. // * MR will fail to set read-only with a lock failure. The caller will have to retry, but the // change is not in a read-only state, so behavior is not degraded in the meantime. // // If MR wins, then either: // * OR will fail with a read-only exception (via AbstractChangeNotes#apply). // * OR will fail with a lock failure. // // In all of these scenarios, the change is read-only if and only if MR succeeds. // // There will be no concurrent writes to ReviewDb for this change until // setPrimaryStorageReviewDb completes, because ReviewDb writes are not attempted when primary // storage is NoteDb. After the primary storage changes back, it is possible for subsequent // NoteDb writes to conflict with the releaseReadOnlyLeaseInNoteDb step, but at this point, // since ReviewDb is primary, we are back to ignoring them. Stopwatch sw = Stopwatch.createStarted(); if (project == null) { project = getProject(id); } ObjectId newMetaId = setReadOnlyInNoteDb(project, id); rebuilder.rebuildReviewDb(db(), project, id); setPrimaryStorageReviewDb(id, newMetaId); releaseReadOnlyLeaseInNoteDb(project, id); log.info("Migrated change {} to ReviewDb primary in {}ms", id, sw.elapsed(MILLISECONDS)); } private ObjectId setReadOnlyInNoteDb(Project.NameKey project, Change.Id id) throws OrmException, IOException { Timestamp now = TimeUtil.nowTs(); Timestamp until = new Timestamp(now.getTime() + timeoutMs); ChangeUpdate update = updateFactory .create(changeControlFactory.controlFor(db.get(), project, id, internalUserFactory.create())); update.setReadOnlyUntil(until); return update.commit(); } private void setPrimaryStorageReviewDb(Change.Id id, ObjectId newMetaId) throws OrmException, IOException { ImmutableMap.Builder<Account.Id, ObjectId> draftIds = ImmutableMap.builder(); try (Repository repo = repoManager.openRepository(allUsers)) { for (Ref draftRef : repo.getRefDatabase().getRefs(RefNames.refsDraftCommentsPrefix(id)).values()) { Account.Id accountId = Account.Id.fromRef(draftRef.getName()); if (accountId != null) { draftIds.put(accountId, draftRef.getObjectId().copy()); } } } NoteDbChangeState newState = new NoteDbChangeState(id, PrimaryStorage.REVIEW_DB, Optional.of(RefState.create(newMetaId, draftIds.build())), Optional.empty()); db().changes().atomicUpdate(id, new AtomicUpdate<Change>() { @Override public Change update(Change change) { if (PrimaryStorage.of(change) != PrimaryStorage.NOTE_DB) { throw new OrmRuntimeException( "change " + id + " is not NoteDb primary: " + change.getNoteDbState()); } change.setNoteDbState(newState.toString()); return change; } }); } private void releaseReadOnlyLeaseInNoteDb(Project.NameKey project, Change.Id id) throws OrmException { // Use a BatchUpdate since ReviewDb is primary at this point, so it needs to reflect the update. try (BatchUpdate bu = batchUpdateFactory.create(db.get(), project, internalUserFactory.create(), TimeUtil.nowTs())) { bu.addOp(id, new BatchUpdateOp() { @Override public boolean updateChange(ChangeContext ctx) { ctx.getUpdate(ctx.getChange().currentPatchSetId()).setReadOnlyUntil(new Timestamp(0)); return true; } }); bu.execute(); } catch (RestApiException | UpdateException e) { throw new OrmException(e); } } private Project.NameKey getProject(Change.Id id) throws OrmException { List<ChangeData> cds = queryProvider.get() .setRequestedFields(ImmutableSet.of(ChangeField.PROJECT.getName())).byLegacyChangeId(id); Set<Project.NameKey> projects = new TreeSet<>(); for (ChangeData cd : cds) { projects.add(cd.project()); } if (projects.size() != 1) { throw new OrmException("zero or multiple projects found for change " + id + ", must specify project explicitly: " + projects); } return projects.iterator().next(); } }