Java tutorial
/** * The contents of this file are subject to the license and copyright * detailed in the LICENSE file at the root of the source * tree and available online at * * https://github.com/keeps/roda */ package org.roda.core.plugins; import java.io.ByteArrayInputStream; import java.io.IOException; import java.nio.file.Path; import java.nio.file.attribute.PosixFilePermission; import java.nio.file.attribute.PosixFilePermissions; import java.util.ArrayList; import java.util.Arrays; import java.util.HashMap; import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.stream.Collectors; import org.apache.commons.io.IOUtils; import org.apache.commons.lang3.RandomStringUtils; import org.apache.solr.client.solrj.SolrServerException; import org.apache.xmlbeans.XmlException; import org.hamcrest.MatcherAssert; import org.hamcrest.Matchers; import org.hamcrest.core.Is; import org.roda.core.RodaCoreFactory; import org.roda.core.TestsHelper; import org.roda.core.common.PremisV3Utils; import org.roda.core.common.iterables.CloseableIterable; import org.roda.core.common.monitor.TransferredResourcesScanner; import org.roda.core.data.common.RodaConstants; import org.roda.core.data.common.RodaConstants.PreservationEventType; import org.roda.core.data.exceptions.AlreadyExistsException; import org.roda.core.data.exceptions.AuthorizationDeniedException; import org.roda.core.data.exceptions.GenericException; import org.roda.core.data.exceptions.InvalidParameterException; import org.roda.core.data.exceptions.NotFoundException; import org.roda.core.data.exceptions.RODAException; import org.roda.core.data.exceptions.RequestNotValidException; import org.roda.core.data.v2.IsRODAObject; import org.roda.core.data.v2.common.OptionalWithCause; import org.roda.core.data.v2.index.IndexResult; import org.roda.core.data.v2.index.IndexRunnable; import org.roda.core.data.v2.index.filter.Filter; import org.roda.core.data.v2.index.filter.SimpleFilterParameter; import org.roda.core.data.v2.index.select.SelectedItemsList; import org.roda.core.data.v2.index.sublist.Sublist; import org.roda.core.data.v2.ip.AIP; import org.roda.core.data.v2.ip.AIPState; import org.roda.core.data.v2.ip.File; import org.roda.core.data.v2.ip.IndexedAIP; import org.roda.core.data.v2.ip.IndexedFile; import org.roda.core.data.v2.ip.Permissions; import org.roda.core.data.v2.ip.TransferredResource; import org.roda.core.data.v2.ip.metadata.IndexedPreservationEvent; import org.roda.core.data.v2.ip.metadata.PreservationMetadata; import org.roda.core.data.v2.ip.metadata.PreservationMetadata.PreservationMetadataType; import org.roda.core.data.v2.jobs.Job; import org.roda.core.data.v2.jobs.PluginType; import org.roda.core.index.IndexService; import org.roda.core.model.ModelService; import org.roda.core.plugins.plugins.PluginHelper; import org.roda.core.plugins.plugins.antivirus.AntivirusPlugin; import org.roda.core.plugins.plugins.characterization.PremisSkeletonPlugin; import org.roda.core.plugins.plugins.characterization.SiegfriedPlugin; import org.roda.core.plugins.plugins.ingest.AutoAcceptSIPPlugin; import org.roda.core.plugins.plugins.ingest.TransferredResourceToAIPPlugin; import org.roda.core.storage.Binary; import org.roda.core.storage.fs.FSUtils; import org.roda.core.util.IdUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.testng.AssertJUnit; import org.testng.annotations.AfterClass; import org.testng.annotations.AfterMethod; import org.testng.annotations.BeforeClass; import org.testng.annotations.Test; import org.w3c.util.InvalidDateException; import com.google.common.collect.Iterables; import gov.loc.premis.v3.EventComplexType; import gov.loc.premis.v3.FormatComplexType; import gov.loc.premis.v3.FormatRegistryComplexType; import gov.loc.premis.v3.LinkingAgentIdentifierComplexType; import gov.loc.premis.v3.ObjectCharacteristicsComplexType; import gov.loc.premis.v3.Representation; import jersey.repackaged.com.google.common.collect.Lists; @Test(groups = { RodaConstants.TEST_GROUP_ALL, RodaConstants.TEST_GROUP_TRAVIS }) public class InternalPluginsTest { private static final String FAKE_REPORTING_CLASS = "NONE"; private static final Logger LOGGER = LoggerFactory.getLogger(InternalPluginsTest.class); private static final int CORPORA_FILES_COUNT = 13; private static final int CORPORA_FOLDERS_COUNT = 3; private static final String CORPORA_PDF = "test.docx"; private static final String CORPORA_TEST1 = "test1"; private static final String CORPORA_TEST1_TXT = "test1.txt"; private static final int GENERATED_FILE_SIZE = 100; private static Path basePath; private static ModelService model; private static IndexService index; @BeforeClass public void setUp() throws Exception { basePath = TestsHelper.createBaseTempDir(getClass(), true, PosixFilePermissions.asFileAttribute(new HashSet<>(Arrays.asList(PosixFilePermission.OWNER_READ, PosixFilePermission.OWNER_WRITE, PosixFilePermission.OWNER_EXECUTE, PosixFilePermission.OTHERS_READ, PosixFilePermission.OTHERS_EXECUTE)))); boolean deploySolr = true; boolean deployLdap = true; boolean deployFolderMonitor = true; boolean deployOrchestrator = true; boolean deployPluginManager = true; boolean deployDefaultResources = false; RodaCoreFactory.instantiateTest(deploySolr, deployLdap, deployFolderMonitor, deployOrchestrator, deployPluginManager, deployDefaultResources); model = RodaCoreFactory.getModelService(); index = RodaCoreFactory.getIndexService(); LOGGER.info("Running internal plugins tests under storage {}", basePath); } @AfterClass public void tearDown() throws Exception { RodaCoreFactory.shutdown(); FSUtils.deletePath(basePath); } @AfterMethod public void cleanUp() throws RODAException { // delete all AIPs index.execute(IndexedAIP.class, Filter.ALL, new ArrayList<>(), new IndexRunnable<IndexedAIP>() { @Override public void run(IndexedAIP item) throws GenericException, RequestNotValidException, AuthorizationDeniedException { try { model.deleteAIP(item.getId()); } catch (NotFoundException e) { // do nothing } } }); // delete all Transferred Resources index.execute(TransferredResource.class, Filter.ALL, new ArrayList<>(), new IndexRunnable<TransferredResource>() { @Override public void run(TransferredResource item) throws GenericException, RequestNotValidException, AuthorizationDeniedException { model.deleteTransferredResource(item); } }); } private ByteArrayInputStream generateContentData() { return new ByteArrayInputStream(RandomStringUtils.randomAscii(GENERATED_FILE_SIZE).getBytes()); } private TransferredResource createCorpora() throws InterruptedException, IOException, NotFoundException, GenericException, RequestNotValidException, AlreadyExistsException { TransferredResourcesScanner f = RodaCoreFactory.getTransferredResourcesScanner(); String parentUUID = f.createFolder(null, "test").getUUID(); String test1UUID = f.createFolder(parentUUID, CORPORA_TEST1).getUUID(); String test2UUID = f.createFolder(parentUUID, "test2").getUUID(); String test3UUID = f.createFolder(parentUUID, "test3").getUUID(); f.createFile(parentUUID, CORPORA_TEST1_TXT, generateContentData()); f.createFile(parentUUID, "test2.txt", generateContentData()); f.createFile(parentUUID, "test3.txt", generateContentData()); f.createFile(test1UUID, CORPORA_TEST1_TXT, generateContentData()); f.createFile(test1UUID, "test2.txt", generateContentData()); f.createFile(test1UUID, "test3.txt", generateContentData()); f.createFile(test2UUID, CORPORA_TEST1_TXT, generateContentData()); f.createFile(test2UUID, "test2.txt", generateContentData()); f.createFile(test2UUID, "test3.txt", generateContentData()); f.createFile(test3UUID, CORPORA_TEST1_TXT, generateContentData()); f.createFile(test3UUID, "test2.txt", generateContentData()); f.createFile(test3UUID, "test3.txt", generateContentData()); f.createFile(parentUUID, CORPORA_PDF, getClass().getResourceAsStream("/corpora/Media/" + CORPORA_PDF)); index.commit(TransferredResource.class); TransferredResource transferredResource = index.retrieve(TransferredResource.class, IdUtils.createUUID("test"), new ArrayList<>()); return transferredResource; } private AIP ingestCorpora() throws RequestNotValidException, NotFoundException, GenericException, AlreadyExistsException, AuthorizationDeniedException, InvalidParameterException, InterruptedException, IOException, SolrServerException { String parentId = null; String aipType = RodaConstants.AIP_TYPE_MIXED; AIP root = model.createAIP(parentId, aipType, new Permissions(), RodaConstants.ADMIN); Map<String, String> parameters = new HashMap<>(); parameters.put(RodaConstants.PLUGIN_PARAMS_PARENT_ID, root.getId()); TransferredResource transferredResource = createCorpora(); AssertJUnit.assertNotNull(transferredResource); Job job = TestsHelper.executeJob(TransferredResourceToAIPPlugin.class, parameters, PluginType.SIP_TO_AIP, SelectedItemsList.create(TransferredResource.class, transferredResource.getUUID())); TestsHelper.getJobReports(index, job, true); index.commitAIPs(); IndexResult<IndexedAIP> find = index.find(IndexedAIP.class, new Filter(new SimpleFilterParameter(RodaConstants.AIP_PARENT_ID, root.getId())), null, new Sublist(0, 10), new ArrayList<>()); AssertJUnit.assertEquals(1L, find.getTotalCount()); IndexedAIP indexedAIP = find.getResults().get(0); AIP aip = model.retrieveAIP(indexedAIP.getId()); return aip; } @Test public void testIngestTransferredResource() throws IOException, InterruptedException, RODAException, SolrServerException { AIP aip = ingestCorpora(); AssertJUnit.assertEquals(1, aip.getRepresentations().size()); CloseableIterable<OptionalWithCause<File>> allFiles = model.listFilesUnder(aip.getId(), aip.getRepresentations().get(0).getId(), true); List<File> reusableAllFiles = new ArrayList<>(); Iterables.addAll(reusableAllFiles, Lists.newArrayList(allFiles).stream().filter(f -> f.isPresent()) .map(f -> f.get()).collect(Collectors.toList())); // All folders and files AssertJUnit.assertEquals(CORPORA_FOLDERS_COUNT + CORPORA_FILES_COUNT, reusableAllFiles.size()); } @Test public void testVirusCheck() throws RODAException, InterruptedException, IOException, SolrServerException, XmlException { AIP aip = ingestCorpora(); Job job = TestsHelper.executeJob(AntivirusPlugin.class, PluginType.AIP_TO_AIP, SelectedItemsList.create(AIP.class, aip.getId())); TestsHelper.getJobReports(index, job, true); aip = model.retrieveAIP(aip.getId()); Plugin<? extends IsRODAObject> plugin = RodaCoreFactory.getPluginManager() .getPlugin(AntivirusPlugin.class.getName()); String agentID = PluginHelper.getPluginAgentId(plugin); boolean found = false; CloseableIterable<OptionalWithCause<PreservationMetadata>> preservationMetadataList = model .listPreservationMetadata(aip.getId(), true); for (OptionalWithCause<PreservationMetadata> opm : preservationMetadataList) { if (opm.isPresent()) { PreservationMetadata pm = opm.get(); if (pm.getType().equals(PreservationMetadataType.EVENT)) { EventComplexType event = PremisV3Utils.binaryToEvent(model .retrievePreservationEvent(pm.getAipId(), pm.getRepresentationId(), pm.getFileDirectoryPath(), pm.getFileId(), pm.getId()) .getContent().createInputStream()); if (event.getLinkingAgentIdentifierArray() != null && event.getLinkingAgentIdentifierArray().length > 0) { for (LinkingAgentIdentifierComplexType laict : event.getLinkingAgentIdentifierArray()) { if (laict.getLinkingAgentIdentifierValue() != null && laict.getLinkingAgentIdentifierValue().equalsIgnoreCase(agentID)) { found = true; break; } } if (found) { break; } } } } } IOUtils.closeQuietly(preservationMetadataList); AssertJUnit.assertTrue(found); index.commitAIPs(); Filter filter = new Filter(); filter.add(new SimpleFilterParameter(RodaConstants.PRESERVATION_EVENT_TYPE, PreservationEventType.VIRUS_CHECK.toString())); filter.add(new SimpleFilterParameter(RodaConstants.PRESERVATION_EVENT_AIP_ID, aip.getId())); IndexResult<IndexedPreservationEvent> events = index.find(IndexedPreservationEvent.class, filter, null, new Sublist(0, 10), new ArrayList<>()); AssertJUnit.assertEquals(1, events.getTotalCount()); } @SuppressWarnings("unchecked") @Test public void testPremisSkeleton() throws RODAException, InterruptedException, IOException, SolrServerException { AIP aip = ingestCorpora(); TestsHelper.executeJob(PremisSkeletonPlugin.class, PluginType.AIP_TO_AIP, SelectedItemsList.create(AIP.class, aip.getId())); aip = model.retrieveAIP(aip.getId()); CloseableIterable<OptionalWithCause<PreservationMetadata>> preservationMetadata = model .listPreservationMetadata(aip.getId(), true); // Files plus one representation + 2 SIP To AIP Event + 1 Premis Skeleton // Event AssertJUnit.assertEquals(CORPORA_FILES_COUNT + 4, Iterables.size(preservationMetadata)); preservationMetadata.close(); Binary rpo_bin = model.retrievePreservationRepresentation(aip.getId(), aip.getRepresentations().get(0).getId()); Representation rpo = PremisV3Utils.binaryToRepresentation(rpo_bin.getContent(), true); // Relates to files AssertJUnit.assertEquals(CORPORA_FILES_COUNT, rpo.getRelationshipArray().length); Binary fpo_bin = model.retrievePreservationFile(aip.getId(), aip.getRepresentations().get(0).getId(), Arrays.asList(CORPORA_TEST1), CORPORA_TEST1_TXT); gov.loc.premis.v3.File fpo = PremisV3Utils.binaryToFile(fpo_bin.getContent(), true); ObjectCharacteristicsComplexType fileCharacteristics = fpo.getObjectCharacteristicsArray(0); // check a fixity was generated AssertJUnit.assertTrue("No fixity checks", fileCharacteristics.getFixityArray().length > 0); // check file size long size = fileCharacteristics.getSize(); AssertJUnit.assertTrue("File size is zero", size > 0); // check file original name String originalName = fpo.getOriginalName().getStringValue(); AssertJUnit.assertEquals(CORPORA_TEST1_TXT, originalName); } @SuppressWarnings("unchecked") @Test public void testSiegfried() throws RODAException, InterruptedException, IOException, SolrServerException, XmlException { AIP aip = ingestCorpora(); // ensure PREMIS objects are created TestsHelper.executeJob(PremisSkeletonPlugin.class, PluginType.AIP_TO_AIP, SelectedItemsList.create(AIP.class, aip.getId())); // run siegfried Map<String, String> parameters = new HashMap<>(); parameters.put(RodaConstants.PLUGIN_PARAMS_REPORTING_CLASS, FAKE_REPORTING_CLASS); Job job = TestsHelper.executeJob(SiegfriedPlugin.class, parameters, PluginType.AIP_TO_AIP, SelectedItemsList.create(AIP.class, aip.getId())); TestsHelper.getJobReports(index, job, true); aip = model.retrieveAIP(aip.getId()); // Files with Siegfried output AssertJUnit.assertEquals(CORPORA_FILES_COUNT, Iterables .size(model.listOtherMetadata(aip.getId(), RodaConstants.OTHER_METADATA_TYPE_SIEGFRIED, true))); Binary om = model.retrieveOtherMetadataBinary(aip.getId(), aip.getRepresentations().get(0).getId(), Arrays.asList(CORPORA_TEST1), CORPORA_TEST1_TXT, SiegfriedPlugin.FILE_SUFFIX, RodaConstants.OTHER_METADATA_TYPE_SIEGFRIED); AssertJUnit.assertNotNull(om); Binary fpo_bin = model.retrievePreservationFile(aip.getId(), aip.getRepresentations().get(0).getId(), Arrays.asList(CORPORA_TEST1), CORPORA_TEST1_TXT); gov.loc.premis.v3.File fpo = PremisV3Utils.binaryToFile(fpo_bin.getContent(), true); FormatComplexType format = fpo.getObjectCharacteristicsArray(0).getFormatArray(0); AssertJUnit.assertEquals("Plain Text File", format.getFormatDesignation().getFormatName().getStringValue()); FormatRegistryComplexType pronomRegistry = PremisV3Utils.getFormatRegistry(fpo, RodaConstants.PRESERVATION_REGISTRY_PRONOM); AssertJUnit.assertEquals(RodaConstants.PRESERVATION_REGISTRY_PRONOM, pronomRegistry.getFormatRegistryName().getStringValue()); AssertJUnit.assertEquals("x-fmt/111", pronomRegistry.getFormatRegistryKey().getStringValue()); FormatRegistryComplexType mimeRegistry = PremisV3Utils.getFormatRegistry(fpo, RodaConstants.PRESERVATION_REGISTRY_MIME); String mimetype = "text/plain"; AssertJUnit.assertEquals(mimetype, mimeRegistry.getFormatRegistryKey().getStringValue()); index.commitAIPs(); IndexedFile indFile = index.retrieve(IndexedFile.class, IdUtils.getFileId(aip.getId(), aip.getRepresentations().get(0).getId(), Arrays.asList(CORPORA_TEST1), CORPORA_TEST1_TXT), new ArrayList<>()); AssertJUnit.assertEquals(mimetype, indFile.getFileFormat().getMimeType()); AssertJUnit.assertEquals("x-fmt/111", indFile.getFileFormat().getPronom()); AssertJUnit.assertEquals("Plain Text File", indFile.getFileFormat().getFormatDesignationName()); List<String> suggest = index.suggest(IndexedFile.class, RodaConstants.FILE_FORMAT_MIMETYPE, mimetype.substring(0, 1), null, false, false); MatcherAssert.assertThat(suggest, Matchers.contains(mimetype)); Plugin<? extends IsRODAObject> plugin = RodaCoreFactory.getPluginManager() .getPlugin(SiegfriedPlugin.class.getName()); String agentID = PluginHelper.getPluginAgentId(plugin); boolean found = false; CloseableIterable<OptionalWithCause<PreservationMetadata>> preservationMetadataList = model .listPreservationMetadata(aip.getId(), true); for (OptionalWithCause<PreservationMetadata> opm : preservationMetadataList) { if (opm.isPresent()) { PreservationMetadata pm = opm.get(); if (pm.getType().equals(PreservationMetadataType.EVENT)) { EventComplexType event = PremisV3Utils.binaryToEvent(model .retrievePreservationEvent(pm.getAipId(), pm.getRepresentationId(), pm.getFileDirectoryPath(), pm.getFileId(), pm.getId()) .getContent().createInputStream()); if (event.getLinkingAgentIdentifierArray() != null && event.getLinkingAgentIdentifierArray().length > 0) { for (LinkingAgentIdentifierComplexType laict : event.getLinkingAgentIdentifierArray()) { if (laict.getLinkingAgentIdentifierValue() != null && laict.getLinkingAgentIdentifierValue().equalsIgnoreCase(agentID)) { found = true; break; } } if (found) { break; } } } } } IOUtils.closeQuietly(preservationMetadataList); AssertJUnit.assertTrue(found); Filter filter = new Filter(); filter.add(new SimpleFilterParameter(RodaConstants.PRESERVATION_EVENT_TYPE, PreservationEventType.FORMAT_IDENTIFICATION.toString())); filter.add(new SimpleFilterParameter(RodaConstants.PRESERVATION_EVENT_AIP_ID, aip.getId())); IndexResult<IndexedPreservationEvent> events = index.find(IndexedPreservationEvent.class, filter, null, new Sublist(0, 10), new ArrayList<>()); AssertJUnit.assertEquals(1, events.getTotalCount()); } @Test public void testAutoAccept() throws RODAException, InterruptedException, IOException, InvalidDateException, SolrServerException { AIP aip = ingestCorpora(); TestsHelper.executeJob(AutoAcceptSIPPlugin.class, PluginType.AIP_TO_AIP, SelectedItemsList.create(AIP.class, aip.getId())); aip = model.retrieveAIP(aip.getId()); MatcherAssert.assertThat(aip.getState(), Is.is(AIPState.ACTIVE)); } }