Java tutorial
/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. * The ASF licenses this file to You under the Apache License, Version 2.0 * (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.nifi.processors.hadoop; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.security.UserGroupInformation; import org.apache.nifi.annotation.behavior.TriggerWhenEmpty; import org.apache.nifi.annotation.documentation.CapabilityDescription; import org.apache.nifi.annotation.documentation.SeeAlso; import org.apache.nifi.annotation.documentation.Tags; import org.apache.nifi.components.PropertyDescriptor; import org.apache.nifi.flowfile.FlowFile; import org.apache.nifi.logging.ComponentLog; import org.apache.nifi.processor.DataUnit; import org.apache.nifi.processor.ProcessContext; import org.apache.nifi.processor.ProcessSession; import org.apache.nifi.processors.hadoop.util.SequenceFileReader; import org.apache.nifi.util.StopWatch; import java.security.PrivilegedExceptionAction; import java.util.ArrayList; import java.util.Collections; import java.util.List; import java.util.Set; import java.util.concurrent.TimeUnit; /** * This processor is used to pull files from HDFS. The files being pulled in MUST be SequenceFile formatted files. The processor creates a flow file for each key/value entry in the ingested * SequenceFile. The created flow file's content depends on the value of the optional configuration property FlowFile Content. Currently, there are two choices: VALUE ONLY and KEY VALUE PAIR. With the * prior, only the SequenceFile value element is written to the flow file contents. With the latter, the SequenceFile key and value are written to the flow file contents as serialized objects; the * format is key length (int), key(String), value length(int), value(bytes). The default is VALUE ONLY. * <p> * NOTE: This processor loads the entire value entry into memory. While the size limit for a value entry is 2GB, this will cause memory problems if there are too many concurrent tasks and the data * being ingested is large. * */ @TriggerWhenEmpty @Tags({ "hadoop", "HDFS", "get", "fetch", "ingest", "source", "sequence file" }) @CapabilityDescription("Fetch sequence files from Hadoop Distributed File System (HDFS) into FlowFiles") @SeeAlso(PutHDFS.class) public class GetHDFSSequenceFile extends GetHDFS { static final String VALUE_ONLY = "VALUE ONLY"; static final PropertyDescriptor FLOWFILE_CONTENT = new PropertyDescriptor.Builder().name("FlowFile Content") .description( "Indicate if the content is to be both the key and value of the Sequence File, or just the value.") .allowableValues(VALUE_ONLY, "KEY VALUE PAIR").defaultValue(VALUE_ONLY).required(true).build(); @Override protected List<PropertyDescriptor> getSupportedPropertyDescriptors() { List<PropertyDescriptor> someProps = new ArrayList<>(super.getSupportedPropertyDescriptors()); someProps.add(FLOWFILE_CONTENT); return Collections.unmodifiableList(someProps); } @Override protected void processBatchOfFiles(final List<Path> files, final ProcessContext context, final ProcessSession session) { final Configuration conf = getConfiguration(); final FileSystem hdfs = getFileSystem(); final String flowFileContentValue = context.getProperty(FLOWFILE_CONTENT).getValue(); final boolean keepSourceFiles = context.getProperty(KEEP_SOURCE_FILE).asBoolean(); final Double bufferSizeProp = context.getProperty(BUFFER_SIZE).asDataSize(DataUnit.B); if (bufferSizeProp != null) { int bufferSize = bufferSizeProp.intValue(); conf.setInt(BUFFER_SIZE_KEY, bufferSize); } ComponentLog logger = getLogger(); final SequenceFileReader<Set<FlowFile>> reader; if (flowFileContentValue.equalsIgnoreCase(VALUE_ONLY)) { reader = new ValueReader(session); } else { reader = new KeyValueReader(session); } Set<FlowFile> flowFiles = Collections.emptySet(); for (final Path file : files) { if (!this.isScheduled()) { break; // This processor should stop running immediately. } final StopWatch stopWatch = new StopWatch(false); try { stopWatch.start(); if (!hdfs.exists(file)) { continue; // If file is no longer here move on. } logger.debug("Reading file"); flowFiles = getFlowFiles(conf, hdfs, reader, file); if (!keepSourceFiles && !hdfs.delete(file, false)) { logger.warn("Unable to delete path " + file.toString() + " from HDFS. Will likely be picked up over and over..."); } } catch (Throwable t) { logger.error("Error retrieving file {} from HDFS due to {}", new Object[] { file, t }); session.rollback(); context.yield(); } finally { stopWatch.stop(); long totalSize = 0; for (FlowFile flowFile : flowFiles) { totalSize += flowFile.getSize(); session.getProvenanceReporter().receive(flowFile, file.toString()); } if (totalSize > 0) { final String dataRate = stopWatch.calculateDataRate(totalSize); final long millis = stopWatch.getDuration(TimeUnit.MILLISECONDS); logger.info( "Created {} flowFiles from SequenceFile {}. Ingested in {} milliseconds at a rate of {}", new Object[] { flowFiles.size(), file.toUri().toASCIIString(), millis, dataRate }); logger.info("Transferred flowFiles {} to success", new Object[] { flowFiles }); session.transfer(flowFiles, REL_SUCCESS); } } } } protected Set<FlowFile> getFlowFiles(final Configuration conf, final FileSystem hdfs, final SequenceFileReader<Set<FlowFile>> reader, final Path file) throws Exception { PrivilegedExceptionAction<Set<FlowFile>> privilegedExceptionAction = new PrivilegedExceptionAction<Set<FlowFile>>() { @Override public Set<FlowFile> run() throws Exception { return reader.readSequenceFile(file, conf, hdfs); } }; UserGroupInformation userGroupInformation = getUserGroupInformation(); if (userGroupInformation == null) { return privilegedExceptionAction.run(); } else { return userGroupInformation.doAs(privilegedExceptionAction); } } }