com.sunteng.flume.source.tail.TailFileEventReader.java Source code

Java tutorial

Introduction

Here is the source code for com.sunteng.flume.source.tail.TailFileEventReader.java

Source

/*
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing,
 * software distributed under the License is distributed on an
 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
 * KIND, either express or implied.  See the License for the
 * specific language governing permissions and limitations
 * under the License.
 */
package com.sunteng.flume.source.tail;

import com.google.common.base.Preconditions;
import org.apache.commons.codec.binary.Base64;
import org.apache.flume.Context;
import org.apache.flume.Event;
import org.apache.flume.client.avro.ReliableEventReader;
import org.apache.flume.serialization.DecodeErrorPolicy;
import org.apache.flume.serialization.EventDeserializer;
import org.apache.flume.serialization.EventDeserializerFactory;
import org.apache.flume.serialization.ResettableFileInputStream;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.io.File;
import java.io.FileFilter;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.nio.charset.Charset;
import java.nio.file.Files;
import java.nio.file.attribute.BasicFileAttributes;
import java.util.Arrays;
import java.util.Collections;
import java.util.List;

public class TailFileEventReader implements ReliableEventReader {
    private static final Logger logger = LoggerFactory.getLogger(TailFileEventReader.class);

    private FinishMarker finishMarker;

    private String deserializerType;
    private Context deserializerContext;

    private final Charset inputCharset;
    private final DecodeErrorPolicy decodeErrorPolicy;

    private String pattern;
    private String fileMatchPattern;
    private FileReader curFileReader;
    private File metaDirectory;
    private File metaFile;
    private File cleanLockFile; // to determine whether if TailSource first run
    private boolean scanAllOnBegin;
    private boolean isSingleRotate;
    private boolean committed = true;

    public TailFileEventReader(String pattern, File metaDirectory, String fileMatchPattern, String deserializerType,
            Context deserializerContext, DecodeErrorPolicy decodeErrorPolicy, String inputCharset,
            boolean scanAllOnBegin, FinishMarker finishMarker) {
        this.deserializerContext = deserializerContext;
        this.metaDirectory = metaDirectory;
        this.deserializerType = deserializerType;
        this.inputCharset = Charset.forName(inputCharset);
        this.decodeErrorPolicy = Preconditions.checkNotNull(decodeErrorPolicy);
        this.pattern = pattern;
        this.fileMatchPattern = fileMatchPattern;
        this.scanAllOnBegin = scanAllOnBegin;
        this.finishMarker = finishMarker;
        isSingleRotate = !fileMatchPattern.contains("*");
        metaFile = metaNameEncoding(pattern);
        cleanLockFile = cleanLockFileEncoding(pattern);
    }

    private File cleanLockFileEncoding(String pattern) {
        String filePath = new String(Base64.encodeBase64(pattern.getBytes())) + ".cleanlock";
        return new File(metaDirectory, filePath);
    }

    private File metaNameEncoding(String pattern) {
        String filePath = new String(Base64.encodeBase64(pattern.getBytes())) + ".meta";
        return new File(metaDirectory, filePath);
    }

    @Override
    public void commit() throws IOException {
        if (!committed && curFileReader != null) {
            committed = true;
            curFileReader.commit();
        }
    }

    @Override
    public Event readEvent() throws IOException {
        throw new IllegalStateException("readEvent not supported yet");
    }

    public boolean rotated() {
        if (curFileReader != null) {
            return curFileReader.rotated;
        }
        return false;
    }

    int lastEmpty = 0;

    @Override
    public List<Event> readEvents(int numEvents) throws IOException {

        while (true) {
            if (!committed) {
                if (curFileReader == null) {
                    throw new IllegalStateException("File should not roll when commit is outstanding.");
                }
                logger.debug("Last read was never committed - resetting mark position.");
                curFileReader.reset();
            } else {
                // Check if new files have arrived since last call
                if (curFileReader == null) {
                    curFileReader = selectOpenFiles();
                }
                // couldn't find any available file
                if (curFileReader == null) {
                    return Collections.emptyList();
                }
            }

            List<Event> events = curFileReader.readEvents(numEvents);

            if (events.isEmpty()) { // eof
                File next = getEarliestFile(curFileReader.file);
                logger.info("next file: " + next + ", current file: " + curFileReader.file);
                if (curFileReader.isNeedClean(next)) { // I think I will have a new file
                    logger.info("clean meta file");
                    try {
                        cleanCurrentFile();
                    } catch (IOException ignored) {
                        // catch and try next time.
                    }
                    continue;
                }

                logger.debug("no need to clean, return empty: " + lastEmpty);
                lastEmpty++;
                return Collections.emptyList();
            } else {
                if (lastEmpty > 0) {
                    logger.debug("lastEmpty: " + lastEmpty, "now finally got one");
                    lastEmpty = 0;
                }
            }

            committed = false;
            return events;
        }
    }

    private void cleanCurrentFile() throws IOException {
        if (curFileReader == null)
            return;
        if (!isSingleRotate) {
            curFileReader.markFinish();
        }

        deleteFile(metaFile);
        try {
            curFileReader.close();
        } catch (IOException ignored) {
        }
        curFileReader = null;
    }

    @Override
    public void close() throws IOException {
        if (curFileReader != null) {
            curFileReader.close();
        }
    }

    private static class FileReader {
        private final File file;
        private long prevOffset;
        private long currentOffset;
        private String currentFileKey;
        private final EventDeserializer deserializer;
        private boolean rotated;
        private Event lastEvent;
        private String lastEventIno;
        private boolean isSingleRotate;
        private DurablePositionTrack tracker;
        private ResettableFileInputStream in;
        private FinishMarker finishMarker;

        public FileReader(File file, EventDeserializer deserializer, ResettableFileInputStream in,
                DurablePositionTrack tracker, boolean isSingleRotate, FinishMarker finishMarker) {
            this.file = file;
            this.in = in;
            this.isSingleRotate = isSingleRotate;
            this.tracker = tracker;
            this.deserializer = deserializer;
            this.currentFileKey = getFileKey(file);
            this.finishMarker = finishMarker;
        }

        public List<Event> readEvents(int numEvents) throws IOException {
            List<Event> ret = deserializer.readEvents(numEvents);
            if (!ret.isEmpty()) {
                if (lastEvent == null) {
                    logger.debug("first event:" + new String(ret.get(0).getBody()) + "; ino:" + tracker.getIno());
                }

                lastEvent = ret.get(ret.size() - 1);
                lastEventIno = tracker.getIno();
            }
            if (isSingleRotate) {
                prevOffset = currentOffset;
                currentOffset = file.length();
                if (currentOffset < prevOffset) {
                    rotated = true;
                }
            }
            // logger.info("readEvent: prev:" + prevOffset + ", current:" + currentOffset);
            return ret;
        }

        // decide whether clean the meta file when we reach eof(got empty events)
        public boolean isNeedClean(File nextFile) {
            // size increase
            if (isSingleRotate && !rotated) {
                logger.debug("not rotated!, not clean: " + currentOffset + "," + prevOffset);
                return false;
            }

            // watch file not exists
            if (nextFile == null || !nextFile.exists()) {
                logger.debug("file.exists(), not clean:" + currentOffset + "," + prevOffset);
                return false;
            }

            if (getFileKey(nextFile).equals(currentFileKey)) {
                logger.debug("currentOffset == prevOffset: " + "newFileKey.equals(currentFileKey):");
                return false;
            }

            return true;
        }

        public void reset() throws IOException {
            deserializer.reset();
        }

        public void commit() throws IOException {
            deserializer.mark();
        }

        public void markFinish() throws IOException {
            finishMarker.mark(file.getPath());
        }

        public void close() throws IOException {
            if (lastEvent != null) {
                logger.debug("last event:" + new String(lastEvent.getBody()) + "; ino:" + lastEventIno);
            }
            logger.debug("in last pos:" + in.getMarkPosition());
            deserializer.close();
        }
    }

    public static String getFileKey(File file) {
        BasicFileAttributes basicAttr;
        try {
            basicAttr = Files.readAttributes(file.toPath(), BasicFileAttributes.class);
        } catch (IOException e) {
            return new String();
        }
        Object obj = basicAttr.fileKey();
        if (obj == null) {
            return new String();
        }
        return obj.toString();
    }

    private FileReader selectOpenFiles() {
        File earliestFile = getEarliestFile(null);
        if (earliestFile == null) {
            return null;
        }
        return openfile(earliestFile, metaFile, isSingleRotate);
    }

    private File getEarliestFile(File now) {
        if (isSingleRotate && now != null) {
            logger.info("getearliestfile:" + now);
            return now;
        }
        File[] files = new File[0];
        try {
            files = FileMatcher.listFiles(pattern, finishMarker);
        } catch (IOException e) {
            e.printStackTrace();
        }
        if (files.length == 0) {
            logger.warn("Could not find file");
            return null;
        }
        Arrays.sort(files);
        if (now == null) {
            return files[0];
        }
        logger.debug("selecting file:" + now);
        for (int i = 0; i < files.length; i++) {
            if (!files[i].equals(now)) {
                continue;
            }
            if (i + 1 < files.length) {
                return files[i + 1];
            }
            return null;
        }
        return null;
    }

    private FileReader openfile(File file, File metaFile, boolean isSingleRotate) {
        try {
            String currentPath = file.getPath();
            logger.info("try to openfile...");
            boolean exist = metaFile.exists();

            DurablePositionTrack tracker = DurablePositionTrack.getInstance(metaFile, currentPath);
            boolean change = !tracker.getTarget().equals(currentPath);
            if (!change) {
                String ino1 = tracker.getIno();
                String ino2 = getFileKey(new File(currentPath));
                logger.debug("event: not change, " + ino1 + "," + ino2 + ":" + exist);
                change = !ino1.equals(ino2);
            }
            if (change) {
                logger.debug("tracker rotate...(target not same)");
                tracker.close();
                deleteFile(metaFile);
                tracker = DurablePositionTrack.getInstance(metaFile, currentPath);
            }

            if (!tracker.isExists && !scanAllOnBegin && !cleanLockFile.exists()) {
                tracker.storePosition(file.length());
                // lock it
                cleanLockFile.createNewFile();
            }

            // sanity check
            Preconditions.checkState(tracker.getTarget().equals(currentPath),
                    "Tracker target %s does not equal expected filename %s", tracker.getTarget(), currentPath);

            ResettableFileInputStream in = new ResettableFileInputStream(file, tracker,
                    ResettableFileInputStream.DEFAULT_BUF_SIZE, inputCharset, decodeErrorPolicy);
            logger.info("get last use of offset:" + in.getMarkPosition());
            EventDeserializer deserializer = EventDeserializerFactory.getInstance(deserializerType,
                    deserializerContext, in);

            return new FileReader(file, deserializer, in, tracker, isSingleRotate, finishMarker);
        } catch (FileNotFoundException e) {
            // File could have been deleted in the interim
            logger.warn("Could not find file: ", e);
            return null;
        } catch (IOException e) {
            logger.error("Exception opening file: " + file, e);
            return null;
        }
    }

    private void deleteFile(File metaFile) throws IOException {
        if (metaFile.exists() && !metaFile.delete()) {
            throw new IOException("Unable to delete old meta file " + metaFile);
        }
    }

}