Java tutorial
/** * The contents of this file are subject to the license and copyright * detailed in the LICENSE file at the root of the source * tree and available online at * * https://github.com/keeps/roda */ package org.roda.core.plugins; import java.io.ByteArrayInputStream; import java.io.IOException; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.attribute.PosixFilePermission; import java.nio.file.attribute.PosixFilePermissions; import java.util.ArrayList; import java.util.Arrays; import java.util.HashMap; import java.util.HashSet; import java.util.Iterator; import java.util.List; import java.util.stream.Collectors; import org.apache.commons.lang3.RandomStringUtils; import org.apache.solr.client.solrj.SolrServerException; import org.roda.core.RodaCoreFactory; import org.roda.core.TestsHelper; import org.roda.core.common.iterables.CloseableIterable; import org.roda.core.common.monitor.TransferredResourcesScanner; import org.roda.core.data.common.RodaConstants; import org.roda.core.data.common.RodaConstants.SolrType; import org.roda.core.data.exceptions.AlreadyExistsException; import org.roda.core.data.exceptions.AuthorizationDeniedException; import org.roda.core.data.exceptions.GenericException; import org.roda.core.data.exceptions.InvalidParameterException; import org.roda.core.data.exceptions.JobAlreadyStartedException; import org.roda.core.data.exceptions.NotFoundException; import org.roda.core.data.exceptions.RequestNotValidException; import org.roda.core.data.v2.common.OptionalWithCause; import org.roda.core.data.v2.index.select.SelectedItemsAll; import org.roda.core.data.v2.index.select.SelectedItemsList; import org.roda.core.data.v2.ip.AIP; import org.roda.core.data.v2.ip.File; import org.roda.core.data.v2.ip.Representation; import org.roda.core.data.v2.ip.TransferredResource; import org.roda.core.data.v2.jobs.Job; import org.roda.core.data.v2.jobs.PluginType; import org.roda.core.index.IndexService; import org.roda.core.model.ModelService; import org.roda.core.model.utils.ModelUtils; import org.roda.core.plugins.orchestrate.JobsHelper; import org.roda.core.plugins.plugins.base.ReplicationPlugin; import org.roda.core.plugins.plugins.ingest.TransferredResourceToAIPPlugin; import org.roda.core.storage.Binary; import org.roda.core.storage.StorageService; import org.roda.core.storage.fs.FSUtils; import org.roda.core.util.CommandException; import org.roda.core.util.CommandUtility; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.testng.Assert; import org.testng.AssertJUnit; import org.testng.annotations.AfterClass; import org.testng.annotations.BeforeClass; import org.testng.annotations.Test; @Test(groups = { RodaConstants.TEST_GROUP_ALL, RodaConstants.TEST_GROUP_TRAVIS }) public class ReplicationPluginTest { private static final Logger LOGGER = LoggerFactory.getLogger(ReplicationPluginTest.class); private static final int GENERATED_FILE_SIZE = 100; private static final int NUMBER_OF_AIPS = 1000; private static Path basePath; private static Path testPath; private static StorageService storage; private static ModelService model; private static IndexService index; private static String parentResourceUUID = null; private static int resourceCounter = 0; @BeforeClass public void setUp() throws Exception { basePath = TestsHelper.createBaseTempDir(getClass(), true, PosixFilePermissions.asFileAttribute(new HashSet<>(Arrays.asList(PosixFilePermission.OWNER_READ, PosixFilePermission.OWNER_WRITE, PosixFilePermission.OWNER_EXECUTE, PosixFilePermission.OTHERS_READ, PosixFilePermission.OTHERS_EXECUTE)))); boolean deploySolr = true; boolean deployLdap = true; boolean deployFolderMonitor = true; boolean deployOrchestrator = true; boolean deployPluginManager = true; boolean deployDefaultResources = false; // embedded Apache Solr RodaCoreFactory.instantiateTest(deploySolr, deployLdap, deployFolderMonitor, deployOrchestrator, deployPluginManager, deployDefaultResources, SolrType.EMBEDDED); // // HTTP Apache Solr // RodaCoreFactory.instantiateTest(deploySolr, deployLdap, // deployFolderMonitor, deployOrchestrator, // deployPluginManager, deployDefaultResources, SolrType.HTTP); // // Cloud Apache Solr // RodaCoreFactory.instantiateTest(deploySolr, deployLdap, // deployFolderMonitor, deployOrchestrator, // deployPluginManager, deployDefaultResources, SolrType.HTTP_CLOUD); model = RodaCoreFactory.getModelService(); index = RodaCoreFactory.getIndexService(); storage = RodaCoreFactory.getStorageService(); testPath = TestsHelper.createBaseTempDir(getClass(), true, PosixFilePermissions.asFileAttribute(new HashSet<>(Arrays.asList(PosixFilePermission.OWNER_READ, PosixFilePermission.OWNER_WRITE, PosixFilePermission.OWNER_EXECUTE, PosixFilePermission.OTHERS_READ, PosixFilePermission.OTHERS_EXECUTE)))); Files.createDirectories( testPath.resolve(RodaConstants.CORE_STORAGE_FOLDER).resolve(RodaConstants.STORAGE_CONTAINER_AIP)); Files.createDirectories(testPath.resolve(RodaConstants.CORE_STORAGE_FOLDER) .resolve(RodaConstants.STORAGE_CONTAINER_PRESERVATION) .resolve(RodaConstants.STORAGE_CONTAINER_PRESERVATION_AGENTS)); LOGGER.info("Running {} tests under storage {}", getClass().getSimpleName(), basePath); } @AfterClass public void tearDown() throws Exception { RodaCoreFactory.shutdown(); FSUtils.deletePath(basePath); FSUtils.deletePath(testPath); } private ByteArrayInputStream generateContentData() { return new ByteArrayInputStream(RandomStringUtils.randomAscii(GENERATED_FILE_SIZE).getBytes()); } private String createCorpora() throws InterruptedException, IOException, NotFoundException, GenericException, RequestNotValidException, AlreadyExistsException { TransferredResourcesScanner f = RodaCoreFactory.getTransferredResourcesScanner(); String parentUUID = f.createFolder(null, "test").getUUID(); index.commit(TransferredResource.class); index.retrieve(TransferredResource.class, parentUUID, new ArrayList<>()); return parentUUID; } private TransferredResource createCorpora(String parentUUID, int i) throws InterruptedException, IOException, NotFoundException, GenericException, RequestNotValidException, AlreadyExistsException { TransferredResourcesScanner f = RodaCoreFactory.getTransferredResourcesScanner(); String test1UUID = f.createFolder(parentUUID, "test" + i).getUUID(); f.createFile(test1UUID, "test" + i + ".txt", generateContentData()); index.commit(TransferredResource.class); return index.retrieve(TransferredResource.class, test1UUID, new ArrayList<>()); } private List<String> ingestCorpora() throws RequestNotValidException, NotFoundException, GenericException, AlreadyExistsException, AuthorizationDeniedException, InvalidParameterException, InterruptedException, IOException, SolrServerException { if (parentResourceUUID == null) { parentResourceUUID = createCorpora(); } List<String> resourceUUIDs = new ArrayList<>(); for (int i = 0; i < NUMBER_OF_AIPS / 2; i++) { TransferredResource transferredResource = createCorpora(parentResourceUUID, resourceCounter); AssertJUnit.assertNotNull(transferredResource); resourceUUIDs.add(transferredResource.getUUID()); resourceCounter++; } Job job = TestsHelper.executeJob(TransferredResourceToAIPPlugin.class, new HashMap<>(), PluginType.SIP_TO_AIP, SelectedItemsList.create(TransferredResource.class, resourceUUIDs)); index.commitAIPs(); return TestsHelper.getJobReports(index, job).stream().map(r -> r.getOutcomeObjectId()) .collect(Collectors.toList()); } @Test private void testRsyncViaListOfIds() throws RequestNotValidException, NotFoundException, GenericException, AlreadyExistsException, AuthorizationDeniedException, InvalidParameterException, InterruptedException, IOException, SolrServerException, JobAlreadyStartedException, CommandException { // increase number of job workers int originalNumberOfJobWorkers = JobsHelper.getNumberOfJobsWorkers(); JobsHelper.setNumberOfJobsWorkers(originalNumberOfJobWorkers * 2); // increase sync job execution timeout in seconds JobsHelper.setSyncTimeout(1200); LOGGER.info("Ingesting first pack of corpora..."); List<String> aips = ingestCorpora(); LOGGER.info("Rsyncing first pack of corpora..."); executeJobWithReplicationPlugin(aips); LOGGER.info("Ingesting second pack of corpora..."); List<String> newAips = ingestCorpora(); LOGGER.info("Rsyncing second pack of corpora..."); executeJobWithReplicationPlugin(newAips); LOGGER.info("Testing if destination has the number of AIPs that was supposed"); Assert.assertEquals(FSUtils.countPath( testPath.resolve(RodaConstants.CORE_STORAGE_FOLDER).resolve(RodaConstants.STORAGE_CONTAINER_AIP)) .intValue(), NUMBER_OF_AIPS); AIP aip1 = model.retrieveAIP(aips.get(0)); AIP aip2 = model.retrieveAIP(aips.get(1)); Binary binary = null; Representation rep1 = aip1.getRepresentations().get(0); CloseableIterable<OptionalWithCause<File>> listFilesUnder1 = model.listFilesUnder(aip1.getId(), rep1.getId(), false); Iterator<OptionalWithCause<File>> iterator1 = listFilesUnder1.iterator(); while (iterator1.hasNext()) { OptionalWithCause<File> next = iterator1.next(); File file = next.get(); binary = storage.getBinary(ModelUtils.getFileStoragePath(file.getAipId(), file.getRepresentationId(), file.getPath(), file.getId())); } Representation rep2 = aip2.getRepresentations().get(0); CloseableIterable<OptionalWithCause<File>> listFilesUnder2 = model.listFilesUnder(aip2.getId(), rep2.getId(), false); Iterator<OptionalWithCause<File>> iterator2 = listFilesUnder2.iterator(); while (iterator2.hasNext()) { OptionalWithCause<File> next = iterator2.next(); File file = next.get(); if (binary != null) { model.updateFile(file.getAipId(), file.getRepresentationId(), file.getPath(), file.getId(), binary.getContent(), true, true); } } executeJobWithReplicationPlugin(Arrays.asList(aip2.getId())); List<String> diffCommand = new ArrayList<>(); diffCommand.add("diff"); diffCommand.add("-rq"); diffCommand.add(RodaCoreFactory.getDataPath().resolve(RodaConstants.CORE_STORAGE_FOLDER) .resolve(RodaConstants.STORAGE_CONTAINER_AIP).toString()); diffCommand.add(testPath.resolve(RodaConstants.CORE_STORAGE_FOLDER) .resolve(RodaConstants.STORAGE_CONTAINER_AIP).toString()); LOGGER.info("Executing diff to compare two folders: {}", diffCommand); CommandUtility.execute(diffCommand); } @Test(enabled = false) private void testRsyncViaFilter() throws RequestNotValidException, NotFoundException, GenericException, AlreadyExistsException, AuthorizationDeniedException, InvalidParameterException, InterruptedException, IOException, SolrServerException, JobAlreadyStartedException, CommandException { // increase sync job execution timeout in seconds JobsHelper.setSyncTimeout(1200); JobsHelper.setBlockSize(100); LOGGER.info("Ingesting first pack of corpora..."); ingestCorpora(); LOGGER.info("Ingesting second pack of corpora..."); ingestCorpora(); LOGGER.info("Rsyncing corpora via filter..."); executeJobWithReplicationPlugin(); LOGGER.info("Testing if destination has the number of AIPs that was supposed"); Assert.assertEquals(FSUtils.countPath( testPath.resolve(RodaConstants.CORE_STORAGE_FOLDER).resolve(RodaConstants.STORAGE_CONTAINER_AIP)) .intValue(), NUMBER_OF_AIPS); List<String> diffCommand = new ArrayList<>(); diffCommand.add("diff"); diffCommand.add("-rq"); diffCommand.add(RodaCoreFactory.getDataPath().resolve(RodaConstants.CORE_STORAGE_FOLDER) .resolve(RodaConstants.STORAGE_CONTAINER_AIP).toString()); diffCommand.add(testPath.resolve(RodaConstants.CORE_STORAGE_FOLDER) .resolve(RodaConstants.STORAGE_CONTAINER_AIP).toString()); LOGGER.info("Executing diff to compare two folders: {}", diffCommand); CommandUtility.execute(diffCommand); } private void executeJobWithReplicationPlugin(List<String> aipIds) throws CommandException, IOException, JobAlreadyStartedException, GenericException, RequestNotValidException, NotFoundException, AuthorizationDeniedException { String targetUser = RodaCoreFactory.getRodaConfigurationAsString("core", "aip_rsync", "target"); RodaCoreFactory.getRodaConfiguration().setProperty("core.aip_rsync.target", testPath.toString() + "/"); Job job = TestsHelper.executeJob(ReplicationPlugin.class, new HashMap<>(), PluginType.MISC, SelectedItemsList.create(AIP.class, aipIds)); TestsHelper.getJobReports(index, job, true); RodaCoreFactory.getRodaConfiguration().setProperty("core.aip_rsync.target", targetUser); } private void executeJobWithReplicationPlugin() throws CommandException, IOException, JobAlreadyStartedException, GenericException, RequestNotValidException, NotFoundException, AuthorizationDeniedException { String targetUser = RodaCoreFactory.getRodaConfigurationAsString("core", "aip_rsync", "target"); RodaCoreFactory.getRodaConfiguration().setProperty("core.aip_rsync.target", testPath.toString()); TestsHelper.executeJob(ReplicationPlugin.class, PluginType.MISC, SelectedItemsAll.create(AIP.class)); RodaCoreFactory.getRodaConfiguration().setProperty("core.aip_rsync.target", targetUser); } }