Java tutorial
/** * Copyright (c) Acroquest Technology Co, Ltd. All Rights Reserved. * Please read the associated COPYRIGHTS file for more details. * * THE SOFTWARE IS PROVIDED BY Acroquest Technolog Co., Ltd., * WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING * BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. * IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDER BE LIABLE FOR ANY * CLAIM, DAMAGES SUFFERED BY LICENSEE AS A RESULT OF USING, MODIFYING * OR DISTRIBUTING THIS SOFTWARE OR ITS DERIVATIVES. */ package acromusashi.stream.bolt.hdfs; import java.io.IOException; import java.text.DateFormat; import java.text.MessageFormat; import java.text.ParseException; import java.text.SimpleDateFormat; import java.util.Date; import java.util.concurrent.TimeUnit; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import acromusashi.stream.util.TimeIntervalFormatUtil; import acromusashi.stream.util.TimeUnitUtil; /** * HDFS?Writer?????????<br> * ??????????????????????????????????? * * @author kimura */ public class HdfsOutputSwitcher { /** logger */ private static final Logger logger = LoggerFactory.getLogger(HdfsOutputSwitcher.class); /** ? */ private static final int TMP_MAX = 50; /** HDFS */ private FileSystem fileSystem = null; /** HDFS */ private HdfsStoreConfig config = null; /** HDFSUri */ public String outputDirUri = ""; /** ????Writer */ private HdfsStreamWriter currentWriter = null; /** ?????URI */ private String currentOutputUri = null; /** ????? */ private String currentSuffix = null; /** ?? */ private long nextSwitchTime = 0; /** ???? */ private SimpleDateFormat dateFormat = null; /** ? */ private int switchTimeInterval = HdfsStoreConfig.DEFAULT_INTERVAL; /** ??? */ private TimeUnit switchTimeUnit = TimeUnit.MINUTES; /** * ????? */ public HdfsOutputSwitcher() { } /** * HDFS??? * * @param fileSystem HDFS * @param config HDFS * @param initializeTime ? * @throws IOException * @throws ParseException */ public void initialize(FileSystem fileSystem, HdfsStoreConfig config, long initializeTime) throws IOException, ParseException { this.fileSystem = fileSystem; this.config = config; if (this.config.getOutputUri().endsWith("/") == false) { this.outputDirUri = this.config.getOutputUri() + "/"; } else { this.outputDirUri = this.config.getOutputUri(); } // ???????? // ??????????? initializeIntervalConf(this.config.getFileSwitchIntarval(), this.config.getFileSwitchIntervalUnit()); // ?????? this.dateFormat = new SimpleDateFormat(TimeUnitUtil.getDatePattern(this.switchTimeUnit)); // ?????? long initialBaseTime = TimeIntervalFormatUtil.generateInitialBaseTime(initializeTime, this.switchTimeInterval, this.switchTimeUnit); this.currentOutputUri = generateOutputFileBase(this.outputDirUri, this.config.getFileNameHeader(), this.config.getFileNameBody(), this.dateFormat, initialBaseTime); this.nextSwitchTime = initialBaseTime + this.switchTimeUnit.toMillis(this.switchTimeInterval); updateWriter(); logger.info("HDFSOutputSwitcher initialized."); } /** * HDFS?? * * @param target * @param nowTime * @throws IOException */ public void append(String target, long nowTime) throws IOException { if (this.nextSwitchTime <= nowTime) { switchWriter(nowTime); } this.currentWriter.append(target); } /** * HDFS???? * * @param target * @param nowTime * @throws IOException */ public void appendLine(String target, long nowTime) throws IOException { if (this.nextSwitchTime <= nowTime) { switchWriter(nowTime); } this.currentWriter.appendLine(target); } /** * ? * * @param nowTime */ private void switchWriter(long nowTime) { closeRenameTmp2BaseFile(); // ?????? // ??????????????????? long nextBaseTime = TimeIntervalFormatUtil.generateNextFileBaseTime(nowTime, this.nextSwitchTime, this.switchTimeInterval, this.switchTimeUnit); this.currentOutputUri = generateOutputFileBase(this.outputDirUri, this.config.getFileNameHeader(), this.config.getFileNameBody(), this.dateFormat, nextBaseTime); this.nextSwitchTime = nextBaseTime + this.config.getFileSwitchIntervalUnit().toMillis(this.switchTimeInterval); updateWriter(); } /** * ????????????? */ private void closeRenameTmp2BaseFile() { try { this.currentWriter.close(); } catch (IOException ex) { String logFormat = "Failed to HDFS file close. Continue file switch. : TargetUri={0}"; String logMessage = MessageFormat.format(logFormat, this.currentOutputUri + this.currentSuffix); logger.warn(logMessage, ex); } boolean isFileExists = true; try { isFileExists = this.fileSystem.exists(new Path(this.currentOutputUri)); } catch (IOException ioex) { String logFormat = "Failed to search target file exists. Skip file rename. : TargetUri={0}"; String logMessage = MessageFormat.format(logFormat, this.currentOutputUri); logger.warn(logMessage, ioex); return; } if (isFileExists) { String logFormat = "File exists renamed target. Skip file rename. : BeforeUri={0} , AfterUri={1}"; String logMessage = MessageFormat.format(logFormat, this.currentOutputUri + this.currentSuffix, this.currentOutputUri); logger.warn(logMessage); } else { try { this.fileSystem.rename(new Path(this.currentOutputUri + this.currentSuffix), new Path(this.currentOutputUri)); } catch (IOException ex) { String logFormat = "Failed to HDFS file rename. Skip rename file. : BeforeUri={0} , AfterUri={1}"; String logMessage = MessageFormat.format(logFormat, this.currentOutputUri + this.currentSuffix, this.currentOutputUri); logger.warn(logMessage, ex); } } } /** * Writer?????50??? */ public void updateWriter() { HdfsStreamWriter result = new HdfsStreamWriter(); String suffix = this.config.getTmpFileSuffix(); int suffixIndex = 0; boolean isFileSyncEachTime = this.config.isFileSyncEachTime(); boolean isSucceed = false; while (suffixIndex < TMP_MAX) { try { result.open(this.currentOutputUri + suffix, this.fileSystem, isFileSyncEachTime); isSucceed = true; break; } catch (IOException ex) { String logFormat = "Failed to HDFS file open. Skip and retry next file. : TargetUri={0}"; String logMessage = MessageFormat.format(logFormat, this.currentOutputUri + suffix); logger.warn(logMessage, ex); suffixIndex++; suffix = this.config.getTmpFileSuffix() + suffixIndex; } } if (isSucceed) { this.currentWriter = result; this.currentSuffix = suffix; } else { String logFormat = "HDFS file open failure is retry overed. Skip HDFS file open. : TargetUri={0}"; String logMessage = MessageFormat.format(logFormat, this.currentOutputUri + suffix); logger.warn(logMessage); } } /** * ???????????? * * @throws IOException */ public void close() throws IOException { closeRenameTmp2BaseFile(); logger.info("HDFSOutputSwitcher closed."); } /** * ?????????????<br> * ??????????????????????(10)??<br> * * @param interval ? * @param unit ?? */ public void initializeIntervalConf(int interval, TimeUnit unit) { boolean isIntervalValid = TimeIntervalFormatUtil.checkValidInterval(interval, unit); if (isIntervalValid == false) { String logFormat = "File switch interval is invalid. Apply default interval 10 minutes. : Interval={0} , TimeUnit={1}"; String logMessage = MessageFormat.format(logFormat, interval, unit.toString()); logger.warn(logMessage); } else { this.switchTimeInterval = interval; this.switchTimeUnit = unit; } } /** * ??????????? */ /** * ????? * * @param baseDir * @param fileNameHeader ?? * @param fileNameBody ?? * @param dateFormat * @param targetDate * @return ?? */ private String generateOutputFileBase(String baseDir, String fileNameHeader, String fileNameBody, DateFormat dateFormat, long targetDate) { StringBuilder baseFileNameBuilder = new StringBuilder(); baseFileNameBuilder.append(baseDir).append(fileNameHeader).append(fileNameBody); baseFileNameBuilder.append(dateFormat.format(new Date(targetDate))); String result = baseFileNameBuilder.toString(); return result; } }