Java tutorial
package com.thinkbiganalytics.nifi.v2.hdfs; /*- * #%L * thinkbig-nifi-hadoop-processors * %% * Copyright (C) 2017 ThinkBig Analytics * %% * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. * #L% */ import com.google.gson.Gson; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.MD5MD5CRC32FileChecksum; import org.apache.hadoop.fs.Path; import org.apache.hadoop.io.MD5Hash; import org.apache.nifi.components.ValidationResult; import org.apache.nifi.processor.ProcessContext; import org.apache.nifi.util.MockProcessContext; import org.apache.nifi.util.TestRunner; import org.apache.nifi.util.TestRunners; import org.junit.Assert; import org.junit.Before; import org.junit.Test; import org.mockito.InOrder; import org.mockito.Mockito; import java.io.FileNotFoundException; import java.io.IOException; import java.util.Collection; import javax.annotation.Nonnull; import javax.annotation.Nullable; import static org.mockito.Matchers.any; public class ComputeHDFSChecksumsTest { /** * Mock file system */ private final FileSystem fileSystem = Mockito.mock(FileSystem.class); /** * Test runner */ private final TestRunner runner = TestRunners.newTestRunner(new TestableComputeHDFSChecksums()); private final String fileEntry = "{\n" + "\"name\": \"%s\",\n" + "\"size\": 131665,\n" + "\"checksum\": {\n" + "\"length\": 28,\n" + "\"value\": \"%s\",\n" + "\"algorithm\": \"MD5-of-0MD5-of-512CRC32C\"\n" + "}\n" + "}"; /** * Initialize instance variables */ @Before public void setUp() throws Exception { // Setup mock file system Mockito.when(fileSystem.delete(any(Path.class), Mockito.eq(true))).thenReturn(true); // Setup test runner runner.setValidateExpressionUsage(false); } /** * Verify required properties. */ @Test public void testValidators() { String files = ComputeHDFSChecksums.FILES.getName(); // Test with no properties Collection<ValidationResult> results = validate(runner); Assert.assertEquals(1, results.size()); results.forEach((ValidationResult result) -> Assert.assertEquals( String.format("'%s' is invalid because %s is required", files, files), result.toString())); // Test with required properties present runner.setProperty(ComputeHDFSChecksums.FILES, "[]"); results = validate(runner); Assert.assertEquals(0, results.size()); // Test with additional property DIRECTORY set runner.setProperty(ComputeHDFSChecksums.DIRECTORY, "/dropzone"); results = validate(runner); Assert.assertEquals(0, results.size()); } @Test public void testFilesListAttributeNotJSON() { runner.setProperty(ComputeHDFSChecksums.FILES, "a"); runner.enqueue(new byte[0]); runner.run(); // Check relationships Assert.assertEquals(1, runner.getFlowFilesForRelationship(ComputeHDFSChecksums.REL_FAILURE).size()); Assert.assertEquals(0, runner.getFlowFilesForRelationship(ComputeHDFSChecksums.REL_SUCCESS).size()); // Check file system calls InOrder inOrder = Mockito.inOrder(fileSystem); inOrder.verifyNoMoreInteractions(); } @Test public void testFileListAttributeNotArray() { runner.setProperty(ComputeHDFSChecksums.FILES, "{}"); runner.enqueue(new byte[0]); runner.run(); // Check relationships Assert.assertEquals(1, runner.getFlowFilesForRelationship(ComputeHDFSChecksums.REL_FAILURE).size()); Assert.assertEquals(0, runner.getFlowFilesForRelationship(ComputeHDFSChecksums.REL_SUCCESS).size()); // Check file system calls InOrder inOrder = Mockito.inOrder(fileSystem); inOrder.verifyNoMoreInteractions(); } @Test public void testFileListAttributeEmtpy() { runner.setProperty(ComputeHDFSChecksums.FILES, ""); runner.enqueue(new byte[0]); runner.run(); // Check relationships Assert.assertEquals(0, runner.getFlowFilesForRelationship(ComputeHDFSChecksums.REL_FAILURE).size()); Assert.assertEquals(1, runner.getFlowFilesForRelationship(ComputeHDFSChecksums.REL_SUCCESS).size()); // Check file system calls InOrder inOrder = Mockito.inOrder(fileSystem); inOrder.verifyNoMoreInteractions(); } @Test public void testFileListAttributeEmptyArray() { runner.setProperty(ComputeHDFSChecksums.FILES, "[]"); runner.enqueue(new byte[0]); runner.run(); // Check relationships Assert.assertEquals(0, runner.getFlowFilesForRelationship(ComputeHDFSChecksums.REL_FAILURE).size()); Assert.assertEquals(1, runner.getFlowFilesForRelationship(ComputeHDFSChecksums.REL_SUCCESS).size()); // Check file system calls InOrder inOrder = Mockito.inOrder(fileSystem); inOrder.verifyNoMoreInteractions(); } @Test public void testSingleFileInListDontFailOnWrongChecksum() throws Exception { String fileName = "000000_0"; Mockito.doReturn(new MD5MD5CRC32FileChecksum(0, 512, new MD5Hash("112233445566778899aabbccddeeff00"))) .when(fileSystem).getFileChecksum(any(Path.class)); runner.setProperty(ComputeHDFSChecksums.FAIL_IF_INCORRECT_CHECKSUM, "False"); runner.setProperty(ComputeHDFSChecksums.FILES, String.format("[" + fileEntry + "]", fileName, "AAACAAAAAAAAAAAArRnBpxcZ9ze14XqfLMB4yA==")); runner.enqueue(new byte[0]); runner.run(); // Check relationships Assert.assertEquals(0, runner.getFlowFilesForRelationship(ComputeHDFSChecksums.REL_FAILURE).size()); Assert.assertEquals(1, runner.getFlowFilesForRelationship(ComputeHDFSChecksums.REL_SUCCESS).size()); // Check whether checksum was passed correctly to attributes String filesJSON = runner.getFlowFilesForRelationship(ComputeHDFSChecksums.REL_SUCCESS).get(0) .getAttribute("files"); Gson jsonParser = new Gson(); ComputeHDFSChecksums.File[] files = jsonParser.fromJson(filesJSON, ComputeHDFSChecksums.File[].class); Assert.assertEquals(files[0].getComputedChecksum().getValue(), "AAAAAAAAAAAAAAIAESIzRFVmd4iZqrvM3e7/AAAAAAA="); // Check file system calls verifyGetFileChecksumCall(fileName); } @Test public void testSingleFileInListFailOnWrongChecksum() throws Exception { String fileName = "000000_0"; Mockito.doReturn(new MD5MD5CRC32FileChecksum(0, 512, new MD5Hash("112233445566778899aabbccddeeff00"))) .when(fileSystem).getFileChecksum(any(Path.class)); runner.setProperty(ComputeHDFSChecksums.FAIL_IF_INCORRECT_CHECKSUM, "True"); runner.setProperty(ComputeHDFSChecksums.FILES, String.format("[" + fileEntry + "]", fileName, "AAACAAAAAAAAAAAArRnBpxcZ9ze14XqfLMB4yA==")); runner.enqueue(new byte[0]); runner.run(); // Check relationships Assert.assertEquals(1, runner.getFlowFilesForRelationship(ComputeHDFSChecksums.REL_FAILURE).size()); Assert.assertEquals(0, runner.getFlowFilesForRelationship(ComputeHDFSChecksums.REL_SUCCESS).size()); // Check file system calls verifyGetFileChecksumCall(fileName); } @Test public void testSingleFileProperChecksum() throws Exception { String fileName = "000000_0"; Mockito.doReturn(new MD5MD5CRC32FileChecksum(0, 512, new MD5Hash("112233445566778899aabbccddeeff00"))) .when(fileSystem).getFileChecksum(any(Path.class)); runner.setProperty(ComputeHDFSChecksums.FAIL_IF_INCORRECT_CHECKSUM, "True"); runner.setProperty(ComputeHDFSChecksums.FILES, String.format("[" + fileEntry + "]", fileName, "AAAAAAAAAAAAAAIAESIzRFVmd4iZqrvM3e7/AAAAAAA=")); runner.enqueue(new byte[0]); runner.run(); // Check relationships Assert.assertEquals(0, runner.getFlowFilesForRelationship(ComputeHDFSChecksums.REL_FAILURE).size()); Assert.assertEquals(1, runner.getFlowFilesForRelationship(ComputeHDFSChecksums.REL_SUCCESS).size()); // Check file system calls verifyGetFileChecksumCall(fileName); } @Test public void testMultipleFilesFailOnSingleWrongChecksum() throws Exception { String fileName = "000000_0"; String fileName2 = "000000_1"; String fileName3 = "000000_2"; Mockito.when(fileSystem.getFileChecksum(any(Path.class))) .thenReturn(new MD5MD5CRC32FileChecksum(0, 512, new MD5Hash("112233445566778899aabbccddeeff00"))) .thenReturn(new MD5MD5CRC32FileChecksum(0, 512, new MD5Hash("112233445566778899aabbccddeeff01"))) .thenReturn(new MD5MD5CRC32FileChecksum(0, 512, new MD5Hash("112233445566778899aabbccddeeff02"))); runner.setProperty(ComputeHDFSChecksums.FAIL_IF_INCORRECT_CHECKSUM, "True"); runner.setProperty(ComputeHDFSChecksums.FILES, String.format("[" + fileEntry + "," + fileEntry + "," + fileEntry + "]", fileName, "AAAAAAAAAAAAAAIAESIzRFVmd4iZqrvM3e7/AAAAAAA=", fileName2, "AAAAAAAAAAAAAAIAESIzRFVmd4iZqrvM3e7/AAAAAAA=", fileName3, "AAAAAAAAAAAAAAIAESIzRFVmd4iZqrvM3e7/AgAAAAA=")); runner.enqueue(new byte[0]); runner.run(); // Check relationships Assert.assertEquals(1, runner.getFlowFilesForRelationship(ComputeHDFSChecksums.REL_FAILURE).size()); Assert.assertEquals(0, runner.getFlowFilesForRelationship(ComputeHDFSChecksums.REL_SUCCESS).size()); // Check file system calls InOrder inOrder = Mockito.inOrder(fileSystem); inOrder.verify(fileSystem).getFileChecksum(new Path(fileName)); inOrder.verify(fileSystem).getFileChecksum(new Path(fileName2)); inOrder.verifyNoMoreInteractions(); } @Test public void testMultipleFilesWithDirectoryDefined() throws Exception { String fileName = "000000_0"; String fileName2 = "000000_1"; String fileName3 = "000000_2"; String directory = "/dropzone"; Mockito.when(fileSystem.getFileChecksum(any(Path.class))) .thenReturn(new MD5MD5CRC32FileChecksum(0, 512, new MD5Hash("112233445566778899aabbccddeeff00"))) .thenReturn(new MD5MD5CRC32FileChecksum(0, 512, new MD5Hash("112233445566778899aabbccddeeff01"))) .thenReturn(new MD5MD5CRC32FileChecksum(0, 512, new MD5Hash("112233445566778899aabbccddeeff02"))); runner.setProperty(ComputeHDFSChecksums.DIRECTORY, directory); runner.setProperty(ComputeHDFSChecksums.FAIL_IF_INCORRECT_CHECKSUM, "True"); runner.setProperty(ComputeHDFSChecksums.FILES, String.format("[" + fileEntry + "," + fileEntry + "," + fileEntry + "]", fileName, "AAAAAAAAAAAAAAIAESIzRFVmd4iZqrvM3e7/AAAAAAA=", fileName2, "AAAAAAAAAAAAAAIAESIzRFVmd4iZqrvM3e7/AQAAAAA=", fileName3, "AAAAAAAAAAAAAAIAESIzRFVmd4iZqrvM3e7/AgAAAAA=")); runner.enqueue(new byte[0]); runner.run(); // Check relationships Assert.assertEquals(0, runner.getFlowFilesForRelationship(ComputeHDFSChecksums.REL_FAILURE).size()); Assert.assertEquals(1, runner.getFlowFilesForRelationship(ComputeHDFSChecksums.REL_SUCCESS).size()); // Check file system calls InOrder inOrder = Mockito.inOrder(fileSystem); inOrder.verify(fileSystem).getFileChecksum(new Path(directory, fileName)); inOrder.verify(fileSystem).getFileChecksum(new Path(directory, fileName2)); inOrder.verify(fileSystem).getFileChecksum(new Path(directory, fileName3)); inOrder.verifyNoMoreInteractions(); } @Test public void testFileNotFoundException() throws Exception { String fileName = "000000_0"; Mockito.doThrow(new FileNotFoundException()).when(fileSystem).getFileChecksum(any(Path.class)); runner.setProperty(ComputeHDFSChecksums.FAIL_IF_INCORRECT_CHECKSUM, "True"); runner.setProperty(ComputeHDFSChecksums.FILES, String.format("[" + fileEntry + "]", fileName, "AAAAAAAAAAAAAAIAESIzRFVmd4iZqrvM3e7/AAAAAAA=")); runner.enqueue(new byte[0]); runner.run(); // Check relationships Assert.assertEquals(1, runner.getFlowFilesForRelationship(ComputeHDFSChecksums.REL_FAILURE).size()); Assert.assertEquals(0, runner.getFlowFilesForRelationship(ComputeHDFSChecksums.REL_SUCCESS).size()); // Check file system calls verifyGetFileChecksumCall(fileName); } /** * Enqueues a {@code FlowFile} and validates its properties. * * @param runner the test runner * @return the validation results */ @Nonnull private Collection<ValidationResult> validate(@Nonnull final TestRunner runner) { runner.enqueue(new byte[0]); return ((MockProcessContext) runner.getProcessContext()).validate(); } private void verifyGetFileChecksumCall(String fileName) throws Exception { InOrder inOrder = Mockito.inOrder(fileSystem); inOrder.verify(fileSystem).getFileChecksum(new Path(fileName)); inOrder.verifyNoMoreInteractions(); } /** * A mock {@code ComputeHDFSChecksums} for testing. */ private class TestableComputeHDFSChecksums extends ComputeHDFSChecksums { @Nullable @Override protected FileSystem getFileSystem(@Nonnull ProcessContext context) { return fileSystem; } @Override HdfsResources resetHDFSResources(String configResources, String dir, ProcessContext context) throws IOException { return null; } } }