Java tutorial
/* * Copyright 2019 ThoughtWorks, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package com.thoughtworks.go.server.persistence; import com.thoughtworks.go.config.CaseInsensitiveString; import com.thoughtworks.go.config.materials.AbstractMaterial; import com.thoughtworks.go.config.materials.MaterialConfigs; import com.thoughtworks.go.config.materials.Materials; import com.thoughtworks.go.database.Database; import com.thoughtworks.go.database.QueryExtensions; import com.thoughtworks.go.domain.*; import com.thoughtworks.go.domain.materials.*; import com.thoughtworks.go.domain.materials.dependency.DependencyMaterialInstance; import com.thoughtworks.go.server.cache.CacheKeyGenerator; import com.thoughtworks.go.server.cache.GoCache; import com.thoughtworks.go.server.service.MaterialConfigConverter; import com.thoughtworks.go.server.service.MaterialExpansionService; import com.thoughtworks.go.server.transaction.TransactionSynchronizationManager; import com.thoughtworks.go.server.ui.ModificationForPipeline; import com.thoughtworks.go.server.ui.PipelineId; import com.thoughtworks.go.server.util.CollectionUtil; import com.thoughtworks.go.server.util.Pagination; import com.thoughtworks.go.util.SystemEnvironment; import org.apache.commons.collections4.CollectionUtils; import org.hibernate.*; import org.hibernate.criterion.*; import org.hibernate.type.LongType; import org.hibernate.type.StringType; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.orm.hibernate3.HibernateCallback; import org.springframework.orm.hibernate3.support.HibernateDaoSupport; import org.springframework.transaction.support.TransactionSynchronizationAdapter; import java.io.File; import java.math.BigInteger; import java.util.*; import java.util.stream.Collectors; import static com.thoughtworks.go.util.ExceptionUtils.bomb; import static org.hibernate.criterion.Restrictions.eq; import static org.hibernate.criterion.Restrictions.isNull; /** * @understands how to store and retrieve Materials from the database */ public class MaterialRepository extends HibernateDaoSupport { private static final Logger LOGGER = LoggerFactory.getLogger(MaterialRepository.class.getName()); private final GoCache goCache; private final TransactionSynchronizationManager transactionSynchronizationManager; private final MaterialConfigConverter materialConfigConverter; private final QueryExtensions queryExtensions; private final CacheKeyGenerator cacheKeyGenerator; private int latestModificationsCacheLimit; private MaterialExpansionService materialExpansionService; public MaterialRepository(SessionFactory sessionFactory, GoCache goCache, int latestModificationsCacheLimit, TransactionSynchronizationManager transactionSynchronizationManager, MaterialConfigConverter materialConfigConverter, MaterialExpansionService materialExpansionService, Database databaseStrategy) { this.goCache = goCache; this.latestModificationsCacheLimit = latestModificationsCacheLimit; this.transactionSynchronizationManager = transactionSynchronizationManager; this.materialConfigConverter = materialConfigConverter; this.materialExpansionService = materialExpansionService; this.queryExtensions = databaseStrategy.getQueryExtensions(); setSessionFactory(sessionFactory); this.cacheKeyGenerator = new CacheKeyGenerator(getClass()); } @SuppressWarnings({ "unchecked" }) public List<Modification> getModificationsForPipelineRange(final String pipelineName, final Integer fromCounter, final Integer toCounter) { return (List<Modification>) getHibernateTemplate().execute((HibernateCallback) session -> { final List<Long> fromInclusiveModificationList = fromInclusiveModificationsForPipelineRange(session, pipelineName, fromCounter, toCounter); final Set<Long> fromModifications = new TreeSet<>( fromInclusiveModificationsForPipelineRange(session, pipelineName, fromCounter, fromCounter)); final Set<Long> fromExclusiveModificationList = new HashSet<>(); for (Long modification : fromInclusiveModificationList) { if (fromModifications.contains(modification)) { fromModifications.remove(modification); } else { fromExclusiveModificationList.add(modification); } } SQLQuery query = session.createSQLQuery( "SELECT * FROM modifications WHERE id IN (:ids) ORDER BY materialId ASC, id DESC"); query.addEntity(Modification.class); query.setParameterList("ids", fromExclusiveModificationList.isEmpty() ? fromInclusiveModificationList : fromExclusiveModificationList); return query.list(); }); } private List<Long> fromInclusiveModificationsForPipelineRange(Session session, String pipelineName, Integer fromCounter, Integer toCounter) { String pipelineIdsSql = queryExtensions.queryFromInclusiveModificationsForPipelineRange(pipelineName, fromCounter, toCounter); SQLQuery pipelineIdsQuery = session.createSQLQuery(pipelineIdsSql); final List ids = pipelineIdsQuery.list(); if (ids.isEmpty()) { return new ArrayList<>(); } String minMaxQuery = " SELECT mods1.materialId as materialId, min(mods1.id) as min, max(mods1.id) as max" + " FROM modifications mods1 " + " INNER JOIN pipelineMaterialRevisions pmr ON (mods1.id >= pmr.actualFromRevisionId AND mods1.id <= pmr.toRevisionId) AND mods1.materialId = pmr.materialId " + " WHERE pmr.pipelineId IN (:ids) " + " GROUP BY mods1.materialId"; SQLQuery query = session .createSQLQuery("SELECT mods.id " + " FROM modifications mods" + " INNER JOIN (" + minMaxQuery + ") as edges on edges.materialId = mods.materialId and mods.id >= min and mods.id <= max" + " ORDER BY mods.materialId ASC, mods.id DESC"); query.addScalar("id", new LongType()); query.setParameterList("ids", ids); return query.list(); } public Map<Long, List<ModificationForPipeline>> findModificationsForPipelineIds(final List<Long> pipelineIds) { final int MODIFICATION = 0; final int RELEVANT_PIPELINE_ID = 1; final int RELEVANT_PIPELINE_NAME = 2; final int MATERIAL_TYPE = 3; final int MATERIAL_FINGERPRINT = 4; //noinspection unchecked return (Map<Long, List<ModificationForPipeline>>) getHibernateTemplate() .execute((HibernateCallback) session -> { if (pipelineIds.isEmpty()) { return new HashMap<Long, List<ModificationForPipeline>>(); } Map<PipelineId, Set<Long>> relevantToLookedUpMap = relevantToLookedUpDependencyMap(session, pipelineIds); SQLQuery query = session.createSQLQuery( "SELECT mods.*, pmr.pipelineId as pmrPipelineId, p.name as pmrPipelineName, m.type as materialType, m.fingerprint as fingerprint" + " FROM modifications mods " + " INNER JOIN pipelineMaterialRevisions pmr ON (mods.id >= pmr.fromRevisionId AND mods.id <= pmr.toRevisionId) AND mods.materialId = pmr.materialId " + " INNER JOIN pipelines p ON pmr.pipelineId = p.id" + " INNER JOIN materials m ON mods.materialId = m.id" + " WHERE pmr.pipelineId IN (:ids)"); @SuppressWarnings({ "unchecked" }) List<Object[]> allModifications = query.addEntity("mods", Modification.class) .addScalar("pmrPipelineId", new LongType()) .addScalar("pmrPipelineName", new StringType()) .addScalar("materialType", new StringType()).addScalar("fingerprint", new StringType()) .setParameterList("ids", relevantToLookedUpMap.keySet().stream() .map(PipelineId::getPipelineId).collect(Collectors.toList())) .list(); Map<Long, List<ModificationForPipeline>> modificationsForPipeline = new HashMap<>(); CollectionUtil.CollectionValueMap<Long, ModificationForPipeline> modsForPipeline = CollectionUtil .collectionValMap(modificationsForPipeline, new CollectionUtil.ArrayList<>()); for (Object[] modAndPmr : allModifications) { Modification mod = (Modification) modAndPmr[MODIFICATION]; Long relevantPipelineId = (Long) modAndPmr[RELEVANT_PIPELINE_ID]; String relevantPipelineName = (String) modAndPmr[RELEVANT_PIPELINE_NAME]; String materialType = (String) modAndPmr[MATERIAL_TYPE]; String materialFingerprint = (String) modAndPmr[MATERIAL_FINGERPRINT]; PipelineId relevantPipeline = new PipelineId(relevantPipelineName, relevantPipelineId); Set<Long> longs = relevantToLookedUpMap.get(relevantPipeline); for (Long lookedUpPipeline : longs) { modsForPipeline.put(lookedUpPipeline, new ModificationForPipeline(relevantPipeline, mod, materialType, materialFingerprint)); } } return modificationsForPipeline; }); } private Map<PipelineId, Set<Long>> relevantToLookedUpDependencyMap(Session session, List<Long> pipelineIds) { final int LOOKED_UP_PIPELINE_ID = 2; final int RELEVANT_PIPELINE_ID = 0; final int RELEVANT_PIPELINE_NAME = 1; String pipelineIdsSql = queryExtensions.queryRelevantToLookedUpDependencyMap(pipelineIds); SQLQuery pipelineIdsQuery = session.createSQLQuery(pipelineIdsSql); pipelineIdsQuery.addScalar("id", new LongType()); pipelineIdsQuery.addScalar("name", new StringType()); pipelineIdsQuery.addScalar("lookedUpId", new LongType()); final List<Object[]> ids = pipelineIdsQuery.list(); Map<Long, List<PipelineId>> lookedUpToParentMap = new HashMap<>(); CollectionUtil.CollectionValueMap<Long, PipelineId> lookedUpToRelevantMap = CollectionUtil .collectionValMap(lookedUpToParentMap, new CollectionUtil.ArrayList<>()); for (Object[] relevantAndLookedUpId : ids) { lookedUpToRelevantMap.put((Long) relevantAndLookedUpId[LOOKED_UP_PIPELINE_ID], new PipelineId((String) relevantAndLookedUpId[RELEVANT_PIPELINE_NAME], (Long) relevantAndLookedUpId[RELEVANT_PIPELINE_ID])); } return CollectionUtil.reverse(lookedUpToParentMap); } @SuppressWarnings("unchecked") public MaterialRevisions findMaterialRevisionsForPipeline(long pipelineId) { List<PipelineMaterialRevision> revisions = findPipelineMaterialRevisions(pipelineId); MaterialRevisions materialRevisions = new MaterialRevisions(); for (PipelineMaterialRevision revision : revisions) { List<Modification> modifications = findModificationsFor(revision); materialRevisions.addRevision(new MaterialRevision(revision.getMaterial(), revision.getChanged(), modifications.toArray(new Modification[modifications.size()]))); } return materialRevisions; } public void cacheMaterialRevisionsForPipelines(Set<Long> pipelineIds) { List<Long> ids = new ArrayList<>(pipelineIds); final int batchSize = 500; loadPMRsIntoCache(ids, batchSize); } private void loadPMRsIntoCache(List<Long> ids, int batchSize) { int total = ids.size(), remaining = total; while (!ids.isEmpty()) { LOGGER.info("Loading PMRs,Remaining {} Pipelines (Total: {})...", remaining, total); final List<Long> idsBatch = batchIds(ids, batchSize); loadPMRByPipelineIds(idsBatch); remaining -= batchSize; } } private <T> List<T> batchIds(List<T> items, int batchSize) { List<T> ids = new ArrayList<>(); for (int i = 0; i < batchSize; ++i) { if (items.isEmpty()) { break; } ids.add(items.remove(0)); } return ids; } public List findPipelineMaterialRevisions(long pipelineId) { String cacheKey = pipelinePmrsKey(pipelineId); synchronized (cacheKey) { List results = (List) goCache.get(cacheKey); if (results != null) { return results; } results = findPMRByPipelineId(pipelineId); goCache.put(cacheKey, results); return results; } } private List findPMRByPipelineId(long pipelineId) { return getHibernateTemplate().find("FROM PipelineMaterialRevision WHERE pipelineId = ? ORDER BY id", pipelineId); } private void loadPMRByPipelineIds(List<Long> pipelineIds) { List<PipelineMaterialRevision> pmrs = (List<PipelineMaterialRevision>) getHibernateTemplate() .findByCriteria(buildPMRDetachedQuery(pipelineIds)); sortPersistentObjectsById(pmrs, true); final Set<PipelineMaterialRevision> uniquePmrs = new HashSet<>(); for (PipelineMaterialRevision pmr : pmrs) { String cacheKey = pipelinePmrsKey(pmr.getPipelineId()); List<PipelineMaterialRevision> pmrsForId = (List<PipelineMaterialRevision>) goCache.get(cacheKey); if (pmrsForId == null) { pmrsForId = new ArrayList<>(); goCache.put(cacheKey, pmrsForId); } pmrsForId.add(pmr); putMaterialInstanceIntoCache(pmr.getToModification().getMaterialInstance()); uniquePmrs.add(pmr); } loadModificationsIntoCache(uniquePmrs); } private void sortPersistentObjectsById(List<? extends PersistentObject> persistentObjects, boolean asc) { Comparator<PersistentObject> ascendingSort = (po1, po2) -> (int) (po1.getId() - po2.getId()); Comparator<PersistentObject> descendingSort = (po1, po2) -> (int) (po2.getId() - po1.getId()); persistentObjects.sort(asc ? ascendingSort : descendingSort); } public void putMaterialInstanceIntoCache(MaterialInstance materialInstance) { String cacheKey = materialKey(materialInstance.getFingerprint()); goCache.put(cacheKey, materialInstance); } private DetachedCriteria buildPMRDetachedQuery(List<Long> pipelineIds) { DetachedCriteria criteria = DetachedCriteria.forClass(PipelineMaterialRevision.class); criteria.add(Restrictions.in("pipelineId", pipelineIds)); criteria.setResultTransformer(CriteriaSpecification.DISTINCT_ROOT_ENTITY); return criteria; } private void loadModificationsIntoCache(Set<PipelineMaterialRevision> pmrs) { List<PipelineMaterialRevision> pmrList = new ArrayList<>(pmrs); int batchSize = 100, total = pmrList.size(), remaining = total; while (!pmrList.isEmpty()) { LOGGER.info("Loading modifications, Remaining {} PMRs(Total: {})...", remaining, total); final List<PipelineMaterialRevision> pmrBatch = batchIds(pmrList, batchSize); loadModificationsForPMR(pmrBatch); remaining -= batchSize; } } private void loadModificationsForPMR(List<PipelineMaterialRevision> pmrs) { List<Criterion> criterions = new ArrayList<>(); for (PipelineMaterialRevision pmr : pmrs) { if (goCache.get(pmrModificationsKey(pmr)) != null) { continue; } final Criterion modificationClause = Restrictions.between("id", pmr.getFromModification().getId(), pmr.getToModification().getId()); final SimpleExpression idClause = Restrictions.eq("materialInstance", pmr.getMaterialInstance()); criterions.add(Restrictions.and(idClause, modificationClause)); } List<Modification> modifications = (List<Modification>) getHibernateTemplate() .findByCriteria(buildModificationDetachedQuery(criterions)); sortPersistentObjectsById(modifications, false); for (Modification modification : modifications) { List<String> cacheKeys = pmrModificationsKey(modification, pmrs); for (String cacheKey : cacheKeys) { List<Modification> modificationList = (List<Modification>) goCache.get(cacheKey); if (modificationList == null) { modificationList = new ArrayList<>(); goCache.put(cacheKey, modificationList); } modificationList.add(modification); } } } private DetachedCriteria buildModificationDetachedQuery(List<Criterion> criteria) { DetachedCriteria detachedCriteria = DetachedCriteria.forClass(Modification.class); Disjunction disjunction = Restrictions.disjunction(); detachedCriteria.add(disjunction); for (Criterion criterion : criteria) { disjunction.add(criterion); } detachedCriteria.setResultTransformer(CriteriaSpecification.DISTINCT_ROOT_ENTITY); return detachedCriteria; } private String pipelinePmrsKey(long pipelineId) { return (MaterialRepository.class.getName() + "_pipelinePMRs_" + pipelineId).intern(); } @SuppressWarnings("unchecked") List<Modification> findMaterialRevisionsForMaterial(long id) { return (List<Modification>) getHibernateTemplate().find("FROM Modification WHERE materialId = ?", new Object[] { id }); } @SuppressWarnings("unchecked") List<Modification> findModificationsFor(PipelineMaterialRevision pmr) { String cacheKey = pmrModificationsKey(pmr); List<Modification> modifications = (List<Modification>) goCache.get(cacheKey); if (modifications == null) { synchronized (cacheKey) { modifications = (List<Modification>) goCache.get(cacheKey); if (modifications == null) { modifications = (List<Modification>) getHibernateTemplate().find( "FROM Modification WHERE materialId = ? AND id BETWEEN ? AND ? ORDER BY id DESC", new Object[] { findMaterialInstance(pmr.getMaterial()).getId(), pmr.getFromModification().getId(), pmr.getToModification().getId() }); goCache.put(cacheKey, modifications); } } } return modifications; } private String pmrModificationsKey(PipelineMaterialRevision pmr) { // we intern() it because we might synchronize on the returned String return (MaterialRepository.class.getName() + "_pmrModifications_" + pmr.getId()).intern(); } private List<String> pmrModificationsKey(Modification modification, List<PipelineMaterialRevision> pmrs) { final long id = modification.getId(); final MaterialInstance materialInstance = modification.getMaterialInstance(); Collection<PipelineMaterialRevision> matchedPmrs = CollectionUtils.select(pmrs, pmr -> { long from = pmr.getFromModification().getId(); long to = pmr.getToModification().getId(); MaterialInstance pmi = findMaterialInstance(pmr.getMaterial()); return from <= id && id <= to && materialInstance.equals(pmi); }); List<String> keys = new ArrayList<>(matchedPmrs.size()); for (PipelineMaterialRevision matchedPmr : matchedPmrs) { keys.add(pmrModificationsKey(matchedPmr)); } return keys; } String latestMaterialModificationsKey(MaterialInstance materialInstance) { // we intern() it because we might synchronize on the returned String return (MaterialRepository.class.getName() + "_latestMaterialModifications_" + materialInstance.getId()) .intern(); } String materialModificationCountKey(MaterialInstance materialInstance) { // we intern() it because we might synchronize on the returned String return (MaterialRepository.class.getName() + "_materialModificationCount_" + materialInstance.getId()) .intern(); } String materialModificationsWithPaginationKey(MaterialInstance materialInstance) { // we intern() it because we might synchronize on the returned String return (MaterialRepository.class.getName() + "_materialModificationsWithPagination_" + materialInstance.getId()).intern(); } String materialModificationsWithPaginationSubKey(Pagination pagination) { return String.format("%s-%s", pagination.getOffset(), pagination.getPageSize()); } public void saveOrUpdate(MaterialInstance materialInstance) { String cacheKey = materialKey(materialInstance.getFingerprint()); synchronized (cacheKey) { getHibernateTemplate().saveOrUpdate(materialInstance); goCache.remove(cacheKey); goCache.put(cacheKey, materialInstance); } } public MaterialInstance find(long id) { return getHibernateTemplate().load(MaterialInstance.class, id); } public MaterialInstance saveMaterialRevision(MaterialRevision materialRevision) { LOGGER.info("Saving revision {}", materialRevision); MaterialInstance materialInstance = findOrCreateFrom(materialRevision.getMaterial()); saveModifications(materialInstance, materialRevision.getModifications()); return materialInstance; } // Used in tests public void saveModification(MaterialInstance materialInstance, Modification modification) { modification.setMaterialInstance(materialInstance); try { getHibernateTemplate().saveOrUpdate(modification); removeLatestCachedModification(materialInstance, modification); removeCachedModificationCountFor(materialInstance); removeCachedModificationsFor(materialInstance); } catch (Exception e) { String message = "Cannot save modification " + modification; LOGGER.error(message, e); throw new RuntimeException(message, e); } } public MaterialInstance findOrCreateFrom(Material material) { String cacheKey = materialKey(material); synchronized (cacheKey) { MaterialInstance materialInstance = findMaterialInstance(material); if (materialInstance == null) { LOGGER.debug( "Material instance for material '{}' not found in the database, creating a new instance now.", material); materialInstance = material.createMaterialInstance(); saveOrUpdate(materialInstance); } return materialInstance; } } final String materialKey(Material material) { // we intern() it because we synchronize on the returned String return materialKey(material.getFingerprint()); } private String materialKey(String fingerprint) { return (MaterialRepository.class.getName() + "_materialInstance_" + fingerprint).intern(); } public MaterialInstance findMaterialInstance(Material material) { String cacheKey = materialKey(material); MaterialInstance materialInstance = (MaterialInstance) goCache.get(cacheKey); if (materialInstance == null) { synchronized (cacheKey) { materialInstance = (MaterialInstance) goCache.get(cacheKey); if (materialInstance == null) { DetachedCriteria hibernateCriteria = DetachedCriteria.forClass(material.getInstanceType()); for (Map.Entry<String, Object> property : material.getSqlCriteria().entrySet()) { if (!property.getKey().equals(AbstractMaterial.SQL_CRITERIA_TYPE)) {//type is polymorphic mapping discriminator if (property.getValue() == null) { hibernateCriteria.add(isNull(property.getKey())); } else { hibernateCriteria.add(eq(property.getKey(), property.getValue())); } } } materialInstance = (MaterialInstance) firstResult(hibernateCriteria); if (materialInstance != null) { goCache.put(cacheKey, materialInstance); } } } } return materialInstance;//TODO: clone me, caller may mutate } private String buildMaterialInstanceQuery(List<Long> materialIds) { StringBuilder queryBuilder = new StringBuilder("FROM MaterialInstance WHERE id IN ("); for (Long materialId : materialIds) { queryBuilder.append(materialId + ","); } queryBuilder.append(")"); return queryBuilder.toString().replace(",)", ")"); //hack to remove the last comma } public MaterialInstance findMaterialInstance(MaterialConfig materialConfig) { String cacheKey = materialKey(materialConfig.getFingerprint()); MaterialInstance materialInstance = (MaterialInstance) goCache.get(cacheKey); if (materialInstance == null) { synchronized (cacheKey) { materialInstance = (MaterialInstance) goCache.get(cacheKey); if (materialInstance == null) { DetachedCriteria hibernateCriteria = DetachedCriteria .forClass(materialConfigConverter.getInstanceType(materialConfig)); for (Map.Entry<String, Object> property : materialConfig.getSqlCriteria().entrySet()) { if (!property.getKey().equals(AbstractMaterial.SQL_CRITERIA_TYPE)) { //type is polymorphic mapping discriminator if (property.getValue() == null) { hibernateCriteria.add(isNull(property.getKey())); } else { hibernateCriteria.add(eq(property.getKey(), property.getValue())); } } } materialInstance = (MaterialInstance) firstResult(hibernateCriteria); if (materialInstance != null) { goCache.put(cacheKey, materialInstance); } } } } return materialInstance;//TODO: clone me, caller may mutate } private Object firstResult(DetachedCriteria criteria) { List results = getHibernateTemplate().findByCriteria(criteria); if (results.isEmpty()) { return null; } return results.get(0); } private Object uniqueResult(DetachedCriteria criteria) { List results = getHibernateTemplate().findByCriteria(criteria); if (results.isEmpty()) { return null; } if (results.size() > 1) { throw bomb("expected unique results, got " + results.size() + ": " + results); } return results.get(0); } public void savePipelineMaterialRevision(Pipeline pipeline, long pipelineId, MaterialRevision materialRevision) { Modification from = materialRevision.getOldestModification(); Modification to = materialRevision.getLatestModification(); Long actualFromModificationId = getLastBuiltModificationId(pipeline, to.getMaterialInstance(), from); if (!from.hasId() || !to.hasId()) { throw bomb( "You cannot save a PipelineMaterialRevision unless the modifications have already been saved."); } PipelineMaterialRevision revision = new PipelineMaterialRevision(pipelineId, materialRevision, actualFromModificationId); save(revision, pipeline.getName()); } private Long getLastBuiltModificationId(final Pipeline pipeline, final MaterialInstance materialInstance, Modification from) { if (materialInstance instanceof DependencyMaterialInstance) { Long id = findLastBuiltModificationId(pipeline, materialInstance); if (id == null) { return from.getId(); } else { return modificationAfter(id, materialInstance); } } return from.getId(); } private long modificationAfter(final long id, final MaterialInstance materialInstance) { BigInteger result = (BigInteger) getHibernateTemplate().execute((HibernateCallback) session -> { String sql = "SELECT id " + " FROM modifications " + " WHERE materialId = ? " + " AND id > ?" + " ORDER BY id" + " LIMIT 1"; SQLQuery query = session.createSQLQuery(sql); query.setLong(0, materialInstance.getId()); query.setLong(1, id); return query.uniqueResult(); }); return result == null ? id : result.longValue(); } private Long findLastBuiltModificationId(final Pipeline pipeline, final MaterialInstance materialInstance) { BigInteger result = (BigInteger) getHibernateTemplate().execute((HibernateCallback) session -> { String sql = "SELECT fromRevisionId " + " FROM pipelineMaterialRevisions pmr " + " INNER JOIN pipelines p on p.id = pmr.pipelineId " + " WHERE materialId = ? " + " AND p.name = ? " + " AND pipelineId < ? " + " ORDER BY pmr.id DESC" + " LIMIT 1"; SQLQuery query = session.createSQLQuery(sql); query.setLong(0, materialInstance.getId()); query.setString(1, pipeline.getName()); query.setLong(2, pipeline.getId()); return query.uniqueResult(); }); return result == null ? null : result.longValue(); } private boolean hasSameMaterialName(Material material, PipelineMaterialRevision pmr) { if (material.getName() == null && pmr.getMaterialName() == null) { return true; } if (material.getName() == null && pmr.getMaterialName() != null) { return false; } return material.getName().equals(new CaseInsensitiveString(pmr.getMaterialName())); } private boolean hasSameFolder(Material material, PipelineMaterialRevision pmr) { if (material.getFolder() == null && pmr.getFolder() == null) { return true; } if (material.getFolder() == null && pmr.getFolder() != null) { return false; } return material.getFolder().equals(pmr.getFolder()); } private void save(final PipelineMaterialRevision pipelineMaterialRevision, final String pipelineName) { getHibernateTemplate().save(pipelineMaterialRevision); transactionSynchronizationManager.registerSynchronization(new TransactionSynchronizationAdapter() { @Override public void afterCommit() { String key = cacheKeyForLatestPmrForPipelineKey(pipelineMaterialRevision.getMaterialId(), pipelineName.toLowerCase()); synchronized (key) { goCache.remove(key); } } }); } public void createPipelineMaterialRevisions(Pipeline pipeline, Long pipelineId, MaterialRevisions materialRevisions) { for (MaterialRevision materialRevision : materialRevisions) { savePipelineMaterialRevision(pipeline, pipelineId, materialRevision); } } /** * @deprecated Not used in production */ public void save(MaterialRevisions materialRevisions) { for (MaterialRevision materialRevision : materialRevisions) { saveMaterialRevision(materialRevision); } } public List<Modification> findModificationsSinceAndUptil(Material material, MaterialRevision materialRevision, PipelineTimelineEntry.Revision scmRevision) { List<Modification> modificationsSince = findModificationsSince(material, materialRevision); if (scmRevision == null) { return modificationsSince; } ArrayList<Modification> modificationsUptil = new ArrayList<>(); for (Modification modification : modificationsSince) { if (modification.getId() <= scmRevision.id) { modificationsUptil.add(modification); } } return modificationsUptil; } @SuppressWarnings("unchecked") public List<Modification> findModificationsSince(Material material, MaterialRevision revision) { MaterialInstance materialInstance = findOrCreateFrom(material); String cacheKey = latestMaterialModificationsKey(materialInstance); synchronized (cacheKey) { long sinceModificationId = revision.getLatestModification().getId(); Modifications modifications = cachedModifications(materialInstance); if (!modificationExists(sinceModificationId, modifications)) { LOGGER.debug("CACHE-MISS for findModificationsSince - {}: {}", materialInstance, revision.getLatestModification()); modifications = _findModificationsSince(materialInstance, sinceModificationId); if (shouldCache(modifications)) { goCache.put(cacheKey, modifications); } else { goCache.remove(cacheKey); } } return modifications.since(sinceModificationId); } } private boolean modificationExists(long sinceModificationId, Modifications modifications) { return modifications != null && modifications.hasModfication(sinceModificationId); } private boolean shouldCache(Modifications modifications) { return modifications.size() <= latestModificationsCacheLimit; } private Modifications _findModificationsSince(MaterialInstance materialInstance, long sinceModificationId) { return new Modifications((List<Modification>) getHibernateTemplate().find( "FROM Modification WHERE materialId = ? AND id >= ? ORDER BY id DESC", new Object[] { materialInstance.getId(), sinceModificationId })); } private void removeLatestCachedModification(final MaterialInstance materialInstance, Modification latest) { transactionSynchronizationManager.registerSynchronization(new TransactionSynchronizationAdapter() { @Override public void afterCommit() { String cacheKey = latestMaterialModificationsKey(materialInstance); synchronized (cacheKey) { goCache.remove(cacheKey); } } }); } private void removeCachedModificationCountFor(final MaterialInstance materialInstance) { transactionSynchronizationManager.registerSynchronization(new TransactionSynchronizationAdapter() { @Override public void afterCommit() { String key = materialModificationCountKey(materialInstance); synchronized (key) { goCache.remove(key); } } }); } private void removeCachedModificationsFor(final MaterialInstance materialInstance) { transactionSynchronizationManager.registerSynchronization(new TransactionSynchronizationAdapter() { @Override public void afterCommit() { String key = materialModificationsWithPaginationKey(materialInstance); synchronized (key) { goCache.remove(key); } } }); } Modifications cachedModifications(MaterialInstance materialInstance) { return (Modifications) goCache.get(latestMaterialModificationsKey(materialInstance)); } public MaterialRevisions findLatestModification(Material material) { MaterialInstance materialInstance = findMaterialInstance(material); if (materialInstance == null) { return new MaterialRevisions(); } Materials materials = new Materials(); materialExpansionService.expandForHistory(material, materials); MaterialRevisions allModifications = new MaterialRevisions(); for (Material expanded : materials) { final MaterialInstance expandedInstance = findOrCreateFrom(expanded); Modification modification = findLatestModification(expandedInstance); if (modification != null) { allModifications.addRevision(expanded, modification); } } return allModifications; } Modification findLatestModification(final MaterialInstance expandedInstance) { Modifications modifications = cachedModifications(expandedInstance); if (modifications != null && !modifications.isEmpty()) { return modifications.get(0); } String cacheKey = latestMaterialModificationsKey(expandedInstance); synchronized (cacheKey) { Modification modification = (Modification) getHibernateTemplate() .execute((HibernateCallback) session -> { Query query = session .createQuery("FROM Modification WHERE materialId = ? ORDER BY id DESC"); query.setMaxResults(1); query.setLong(0, expandedInstance.getId()); return query.uniqueResult(); }); goCache.put(cacheKey, new Modifications(modification)); return modification; } } public void saveModifications(MaterialInstance materialInstance, List<Modification> newChanges) { if (newChanges.isEmpty()) { return; } ArrayList<Modification> list = new ArrayList<>(newChanges); Collections.reverse(list); for (Modification modification : list) { modification.setMaterialInstance(materialInstance); } try { checkAndRemoveDuplicates(materialInstance, newChanges, list); for (Modification modification : list) { getHibernateTemplate().saveOrUpdate(modification); } } catch (Exception e) { String message = "Cannot save modification: "; LOGGER.error(message, e); throw new RuntimeException(message + e.getMessage(), e); } for (Modification modification : list) { removeLatestCachedModification(materialInstance, modification); } removeCachedModificationCountFor(materialInstance); removeCachedModificationsFor(materialInstance); } private void checkAndRemoveDuplicates(MaterialInstance materialInstance, List<Modification> newChanges, ArrayList<Modification> list) { if (!new SystemEnvironment().get(SystemEnvironment.CHECK_AND_REMOVE_DUPLICATE_MODIFICATIONS)) { return; } DetachedCriteria criteria = DetachedCriteria.forClass(Modification.class); criteria.setProjection(Projections.projectionList().add(Projections.property("revision"))); criteria.add(Restrictions.eq("materialInstance.id", materialInstance.getId())); ArrayList<String> revisions = new ArrayList<>(); for (Modification modification : newChanges) { revisions.add(modification.getRevision()); } criteria.add(Restrictions.in("revision", revisions)); List<String> matchingRevisionsFromDb = (List<String>) getHibernateTemplate().findByCriteria(criteria); if (!matchingRevisionsFromDb.isEmpty()) { for (final String revision : matchingRevisionsFromDb) { Modification modification = list.stream().filter(item -> item.getRevision().equals(revision)) .findFirst().orElse(null); list.remove(modification); } } if (!newChanges.isEmpty() && list.isEmpty()) { LOGGER.debug("All modifications already exist in db [{}]", revisions); } if (!matchingRevisionsFromDb.isEmpty()) { LOGGER.info("Saving revisions for material [{}] after removing the following duplicates {}", materialInstance.toOldMaterial(null, null, null).getLongDescription(), matchingRevisionsFromDb); } } public Modification findModificationWithRevision(final Material material, final String revision) { return (Modification) getHibernateTemplate().execute((HibernateCallback) session -> { try { final long materialId = findOrCreateFrom(material).getId(); return MaterialRepository.this.findModificationWithRevision(session, materialId, revision); } catch (Exception e) { LOGGER.error("Error while retrieving modification with material [{}] containing revision [{}]", material, revision, e); throw e instanceof HibernateException ? (HibernateException) e : new RuntimeException(e); } }); } Modification findModificationWithRevision(Session session, long materialId, String revision) { Modification modification; String key = cacheKeyForModificationWithRevision(materialId, revision); modification = (Modification) goCache.get(key); if (modification == null) { synchronized (key) { modification = (Modification) goCache.get(key); if (modification == null) { Query query = session.createQuery( "FROM Modification WHERE materialId = ? and revision = ? ORDER BY id DESC"); query.setLong(0, materialId); query.setString(1, revision); modification = (Modification) query.uniqueResult(); goCache.put(key, modification); } } } return modification; } public MaterialRevisions findLatestRevisions(MaterialConfigs materialConfigs) { MaterialRevisions materialRevisions = new MaterialRevisions(); for (MaterialConfig materialConfig : materialConfigs) { MaterialInstance materialInstance = findMaterialInstance(materialConfig); if (materialInstance != null) { Modification modification = findLatestModification(materialInstance); Material material = materialConfigConverter.toMaterial(materialConfig); materialRevisions.addRevision(modification == null ? new MaterialRevision(material) : new MaterialRevision(material, modification)); } } return materialRevisions; } public boolean hasPipelineEverRunWith(final String pipelineName, final MaterialRevisions revisions) { return (Boolean) getHibernateTemplate().execute((HibernateCallback) session -> { int numberOfMaterials = revisions.getRevisions().size(); int match = 0; for (MaterialRevision revision : revisions) { long materialId = findOrCreateFrom(revision.getMaterial()).getId(); long modificationId = revision.getLatestModification().getId(); String key = cacheKeyForHasPipelineEverRunWithModification(pipelineName, materialId, modificationId); if (goCache.get(key) != null) { match++; continue; } String sql = "SELECT materials.id" + " FROM pipelineMaterialRevisions" + " INNER JOIN pipelines ON pipelineMaterialRevisions.pipelineId = pipelines.id" + " INNER JOIN modifications on modifications.id = pipelineMaterialRevisions.torevisionId" + " INNER JOIN materials on modifications.materialId = materials.id" + " WHERE materials.id = ? AND pipelineMaterialRevisions.toRevisionId >= ? AND pipelineMaterialRevisions.fromRevisionId <= ? AND pipelines.name = ?" + " GROUP BY materials.id;"; SQLQuery query = session.createSQLQuery(sql); query.setLong(0, materialId); query.setLong(1, modificationId); query.setLong(2, modificationId); query.setString(3, pipelineName); if (!query.list().isEmpty()) { match++; goCache.put(key, Boolean.TRUE); } } return match == numberOfMaterials; }); } private String cacheKeyForHasPipelineEverRunWithModification(Object pipelineName, long materialId, long modificationId) { return cacheKeyGenerator.generate("hasPipelineEverRunWithModification", pipelineName, materialId, modificationId); } @SuppressWarnings("unchecked") public List<MatchedRevision> findRevisionsMatching(final MaterialConfig materialConfig, final String searchString) { return (List<MatchedRevision>) getHibernateTemplate().execute((HibernateCallback) session -> { String sql = "SELECT m.*" + " FROM modifications AS m" + " INNER JOIN materials mat ON mat.id = m.materialId" + " WHERE mat.fingerprint = :finger_print" + " AND (m.revision || ' ' || COALESCE(m.username, '') || ' ' || COALESCE(m.comment, '') LIKE :search_string OR m.pipelineLabel LIKE :search_string)" + " ORDER BY m.id DESC" + " LIMIT 5"; SQLQuery query = session.createSQLQuery(sql); query.addEntity("m", Modification.class); Material material = materialConfigConverter.toMaterial(materialConfig); query.setString("finger_print", material.getFingerprint()); query.setString("search_string", "%" + searchString + "%"); final List<MatchedRevision> list = new ArrayList<>(); for (Modification mod : (List<Modification>) query.list()) { list.add(material.createMatchedRevision(mod, searchString)); } return list; }); } public List<Modification> modificationFor(final StageIdentifier stageIdentifier) { if (stageIdentifier == null) { return null; } String key = cacheKeyForModificationsForStageLocator(stageIdentifier); List<Modification> modifications = (List<Modification>) goCache.get(key); if (modifications == null) { synchronized (key) { modifications = (List<Modification>) goCache.get(key); if (modifications == null) { modifications = (List<Modification>) getHibernateTemplate() .execute((HibernateCallback) session -> { Query q = session.createQuery( "FROM Modification WHERE revision = :revision ORDER BY id DESC"); q.setParameter("revision", stageIdentifier.getStageLocator()); return q.list(); }); if (!modifications.isEmpty()) { goCache.put(key, modifications); } } } } return modifications; } public Long getTotalModificationsFor(final MaterialInstance materialInstance) { String key = materialModificationCountKey(materialInstance); Long totalCount = (Long) goCache.get(key); if (totalCount == null || totalCount == 0) { synchronized (key) { totalCount = (Long) goCache.get(key); if (totalCount == null || totalCount == 0) { totalCount = (Long) getHibernateTemplate().execute((HibernateCallback) session -> { Query q = session.createQuery("select count(*) FROM Modification WHERE materialId = ?"); q.setLong(0, materialInstance.getId()); return q.uniqueResult(); }); goCache.put(key, totalCount); } } } return totalCount; } public Modifications getModificationsFor(final MaterialInstance materialInstance, final Pagination pagination) { String key = materialModificationsWithPaginationKey(materialInstance); String subKey = materialModificationsWithPaginationSubKey(pagination); Modifications modifications = (Modifications) goCache.get(key, subKey); if (modifications == null) { synchronized (key) { modifications = (Modifications) goCache.get(key, subKey); if (modifications == null) { List<Modification> modificationsList = (List<Modification>) getHibernateTemplate() .execute((HibernateCallback) session -> { Query q = session .createQuery("FROM Modification WHERE materialId = ? ORDER BY id DESC"); q.setFirstResult(pagination.getOffset()); q.setMaxResults(pagination.getPageSize()); q.setLong(0, materialInstance.getId()); return q.list(); }); if (!modificationsList.isEmpty()) { modifications = new Modifications(modificationsList); goCache.put(key, subKey, modifications); } } } } return modifications; } public Long latestModificationRunByPipeline(final CaseInsensitiveString pipelineName, final Material material) { final long materialId = findMaterialInstance(material).getId(); String key = cacheKeyForLatestPmrForPipelineKey(materialId, pipelineName.toLower()); Long modificationId = (Long) goCache.get(key); if (modificationId == null) { synchronized (key) { modificationId = (Long) goCache.get(key); if (modificationId == null) { modificationId = (Long) getHibernateTemplate().execute((HibernateCallback) session -> { SQLQuery sqlQuery = session.createSQLQuery("SELECT MAX(pmr.toRevisionId) toRevisionId " + "FROM (SELECT torevisionid, pipelineid FROM pipelineMaterialRevisions WHERE materialid = :material_id) AS pmr\n" + "INNER JOIN pipelines p ON ( p.name = :pipeline_name AND p.id = pmr.pipelineId)"); sqlQuery.setParameter("material_id", materialId); sqlQuery.setParameter("pipeline_name", pipelineName.toString()); sqlQuery.addScalar("toRevisionId", new LongType()); return sqlQuery.uniqueResult(); }); if (modificationId == null) { modificationId = -1L; } goCache.put(key, modificationId); } } } return modificationId; } private String cacheKeyForLatestPmrForPipelineKey(long materialId, final String lowerCasePipelineName) { return cacheKeyGenerator.generate("latestPmrForPipeline", lowerCasePipelineName, "andMaterial", materialId); } String cacheKeyForModificationWithRevision(long materialId, String revision) { return cacheKeyGenerator.generate("findModificationWithRevision", "ForMaterialId", materialId, "andRevision", revision); } String cacheKeyForModificationsForStageLocator(StageIdentifier stageIdentifier) { return cacheKeyGenerator.generate("modificationsFor", stageIdentifier.getStageLocator()); } public File folderFor(Material material) { MaterialInstance materialInstance = this.findOrCreateFrom(material); return new File(new File("pipelines", "flyweight"), materialInstance.getFlyweightName()); } }