Java tutorial
/* * Copyright (C) 2010-2101 Alibaba Group Holding Limited. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package com.alibaba.otter.node.etl.common.pipe.impl.http; import java.io.File; import java.io.InputStream; import java.text.MessageFormat; import java.text.SimpleDateFormat; import java.util.Date; import java.util.List; import org.apache.commons.io.FilenameUtils; import org.apache.commons.lang.ClassUtils; import org.apache.commons.lang.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.slf4j.MDC; import org.springframework.beans.BeansException; import org.springframework.beans.factory.BeanFactory; import org.springframework.beans.factory.BeanFactoryAware; import com.alibaba.otter.node.etl.OtterConstants; import com.alibaba.otter.node.etl.common.io.EncryptedData; import com.alibaba.otter.node.etl.common.io.download.DataRetriever; import com.alibaba.otter.node.etl.common.pipe.PipeDataType; import com.alibaba.otter.node.etl.common.pipe.exception.PipeException; import com.alibaba.otter.node.etl.common.pipe.impl.http.archive.ArchiveBean; import com.alibaba.otter.node.etl.common.pipe.impl.http.archive.ArchiveRetriverCallback; import com.alibaba.otter.node.etl.common.pipe.impl.http.archive.LazyFileInputStream; import com.alibaba.otter.node.etl.load.loader.db.FileloadDumper; import com.alibaba.otter.shared.common.model.config.pipeline.Pipeline; import com.alibaba.otter.shared.etl.model.FileBatch; import com.alibaba.otter.shared.etl.model.FileData; import com.alibaba.otter.shared.etl.model.Identity; /** * http??? * * @author jianghang 2011-10-17 ?03:11:44 * @version 4.0.0 */ public class AttachmentHttpPipe extends AbstractHttpPipe<Object, HttpPipeKey> implements BeanFactoryAware { private static final Logger logger = LoggerFactory.getLogger(AttachmentHttpPipe.class); private BeanFactory beanFactory; private boolean encrypt = false; @Override public HttpPipeKey put(Object data) throws PipeException { if (data instanceof FileBatch) { return archiveFile((FileBatch) data); } else { throw new IllegalArgumentException("error argument"); } } public File get(HttpPipeKey key) throws PipeException { return unpackFile(key); } // ? private HttpPipeKey archiveFile(final FileBatch fileBatch) { // ?url String filename = buildFileName(fileBatch.getIdentity(), ClassUtils.getShortClassName(fileBatch.getClass())); File file = new File(htdocsDir, filename); // ? List<FileData> fileDatas = fileBatch.getFiles(); Pipeline pipeline = configClientService.findPipeline(fileBatch.getIdentity().getPipelineId()); int poolSize = pipeline.getParameters().getFileLoadPoolSize(); boolean useLocalFileMutliThread = pipeline.getParameters().getUseLocalFileMutliThread(); ArchiveBean archiveBean = getArchiveBean(); archiveBean.adjustPoolSize(poolSize);// ? archiveBean.setUseLocalFileMutliThread(useLocalFileMutliThread);// ??local? boolean done = archiveBean.pack(file, fileDatas, new ArchiveRetriverCallback<FileData>() { public InputStream retrive(FileData fileData) { boolean miss = false; try { if (StringUtils.isNotEmpty(fileData.getNameSpace())) { throw new RuntimeException(fileData + " is not support!"); } else { File source = new File(fileData.getPath()); if (source.exists() && source.isFile()) { return new LazyFileInputStream(source); } else { miss = true; return null; } } } finally { if (miss && logger.isInfoEnabled()) { MDC.put(OtterConstants.splitPipelineLoadLogFileKey, String.valueOf(fileBatch.getIdentity().getPipelineId())); logger.info(FileloadDumper.dumpMissFileDatas(fileBatch.getIdentity(), fileData)); } } } }); if (done == false) { return null; // } HttpPipeKey key = new HttpPipeKey(); key.setUrl(remoteUrlBuilder.getUrl(fileBatch.getIdentity().getPipelineId(), filename)); key.setDataType(PipeDataType.FILE_BATCH); key.setIdentity(fileBatch.getIdentity()); if (encrypt || pipeline.getParameters().getUseFileEncrypt()) { // ? EncryptedData encryptedData = encryptFile(file); key.setKey(encryptedData.getKey()); key.setCrc(encryptedData.getCrc()); } return key; } // ? private File unpackFile(HttpPipeKey key) { Pipeline pipeline = configClientService.findPipeline(key.getIdentity().getPipelineId()); DataRetriever dataRetriever = dataRetrieverFactory.createRetriever(pipeline.getParameters().getRetriever(), key.getUrl(), downloadDir); File archiveFile = null; try { dataRetriever.connect(); dataRetriever.doRetrieve(); archiveFile = dataRetriever.getDataAsFile(); } catch (Exception e) { dataRetriever.abort(); throw new PipeException("download_error", e); } finally { dataRetriever.disconnect(); } // ?? if (StringUtils.isNotEmpty(key.getKey()) && StringUtils.isNotEmpty(key.getCrc())) { decodeFile(archiveFile, key.getKey(), key.getCrc()); } // .gzip?? String dir = StringUtils.removeEnd(archiveFile.getPath(), FilenameUtils.EXTENSION_SEPARATOR_STR + FilenameUtils.getExtension(archiveFile.getPath())); File unpackDir = new File(dir); // getArchiveBean().unpack(archiveFile, unpackDir); return unpackDir; } // ?? private String buildFileName(Identity identity, String prefix) { Date now = new Date(); String time = new SimpleDateFormat(DATE_FORMAT).format(now); return MessageFormat.format("{0}-{1}-{2}-{3}-{4}.gzip", prefix, time, String.valueOf(identity.getChannelId()), String.valueOf(identity.getPipelineId()), String.valueOf(identity.getProcessId())); } private ArchiveBean getArchiveBean() { // archiveBean????? return (ArchiveBean) beanFactory.getBean("archiveBean", ArchiveBean.class); } // ================== setter / getter =================== public void setBeanFactory(BeanFactory beanFactory) throws BeansException { this.beanFactory = beanFactory; } public void setEncrypt(boolean encrypt) { this.encrypt = encrypt; } }