com.alibaba.otter.node.etl.common.pipe.impl.http.AbstractHttpPipe.java Source code

Java tutorial

Introduction

Here is the source code for com.alibaba.otter.node.etl.common.pipe.impl.http.AbstractHttpPipe.java

Source

/*
 * Copyright (C) 2010-2101 Alibaba Group Holding Limited.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package com.alibaba.otter.node.etl.common.pipe.impl.http;

import java.io.File;
import java.io.RandomAccessFile;
import java.io.UnsupportedEncodingException;
import java.util.Set;
import java.util.concurrent.Executors;
import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.TimeUnit;

import org.apache.commons.io.FileUtils;
import org.apache.commons.io.IOUtils;
import org.apache.commons.lang.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.InitializingBean;
import org.springframework.util.Assert;

import com.alibaba.otter.node.common.config.ConfigClientService;
import com.alibaba.otter.node.etl.common.io.EncryptUtils;
import com.alibaba.otter.node.etl.common.io.EncryptedData;
import com.alibaba.otter.node.etl.common.io.download.DataRetrieverFactory;
import com.alibaba.otter.node.etl.common.io.signature.ChecksumException;
import com.alibaba.otter.node.etl.common.jetty.JettyEmbedServer;
import com.alibaba.otter.node.etl.common.pipe.Pipe;
import com.alibaba.otter.node.etl.common.pipe.exception.PipeException;
import com.alibaba.otter.shared.common.utils.ByteUtils;
import com.alibaba.otter.shared.common.utils.NioUtils;
import com.alibaba.otter.shared.common.utils.thread.NamedThreadFactory;
import com.google.common.collect.Sets;

/**
 * httppipe
 * 
 * @author jianghang 2011-10-13 ?06:31:13
 * @version 4.0.0
 */
public abstract class AbstractHttpPipe<T, KEY extends HttpPipeKey> implements Pipe<T, KEY>, InitializingBean {

    protected static final Long DEFAULT_PERIOD = 60 * 1000L;
    protected static final String UTF_8 = "UTF-8";
    protected static final String DATE_FORMAT = "yyyy-MM-dd-HH-mm-ss";
    protected static ScheduledExecutorService schedulor = Executors.newScheduledThreadPool(1,
            new NamedThreadFactory("HttpPipe-Cleaner"));;
    protected Logger logger = LoggerFactory.getLogger(this.getClass());
    protected JettyEmbedServer jettyEmbedServer; // ?server??
    protected Long period = DEFAULT_PERIOD;
    protected ConfigClientService configClientService;
    protected Long timeout = 24 * 60 * 60 * 1000L; // ,24?
    protected String htdocsDir; // http?
    protected String downloadDir; // ??
    protected RemoteUrlBuilder remoteUrlBuilder;
    protected DataRetrieverFactory dataRetrieverFactory;

    public void afterPropertiesSet() throws Exception {
        Assert.notNull(remoteUrlBuilder);
        Assert.notNull(htdocsDir);
        NioUtils.create(new File(htdocsDir), false, 3);
        if (StringUtils.isEmpty(downloadDir)) {
            downloadDir = htdocsDir;
        } else {
            NioUtils.create(new File(downloadDir), false, 3);
        }
        // ??
        schedulor.scheduleAtFixedRate(new Runnable() {

            public void run() {
                try {
                    long threshold = System.currentTimeMillis() - timeout;
                    File htdocsDirFile = new File(htdocsDir);
                    File[] htdocsFiles = htdocsDirFile.listFiles();
                    Set<File> files = Sets.newHashSet();
                    for (File htdocsFile : htdocsFiles) {
                        files.add(htdocsFile);
                    }
                    if (downloadDir.equals(htdocsDir) == false) {
                        File downloadDirFile = new File(downloadDir);
                        File[] downloadFiles = downloadDirFile.listFiles();
                        for (File downloadFile : downloadFiles) {
                            files.add(downloadFile);
                        }
                    }

                    for (File file : files) {
                        boolean isOld = FileUtils.isFileOlder(file, threshold);
                        if (isOld) {
                            NioUtils.delete(file, 3);
                        }
                    }
                } catch (Exception e) {
                    logger.error("old_file_clean_error", e);
                }
            }
        }, DEFAULT_PERIOD, DEFAULT_PERIOD, TimeUnit.MILLISECONDS);
    }

    protected EncryptedData encryptFile(File file) {
        // ?file path??
        EncryptedData encryptedData = null;
        try {
            encryptedData = EncryptUtils.encrypt(file.getPath().getBytes("UTF-8"));
        } catch (UnsupportedEncodingException e) {
            // ignore
        }

        // ?
        RandomAccessFile raf = null;
        try {
            raf = new RandomAccessFile(file, "rw");
            long origLength = file.length();
            int keyLength = ByteUtils.stringToBytes(encryptedData.getKey()).length;
            int crcLength = ByteUtils.stringToBytes(encryptedData.getCrc()).length;
            long totalLength = origLength + crcLength + keyLength;
            raf.setLength(totalLength);
            raf.seek(origLength);
            raf.write(ByteUtils.stringToBytes(encryptedData.getKey()), 0, keyLength);
            raf.seek(origLength + keyLength);
            raf.write(ByteUtils.stringToBytes(encryptedData.getCrc()), 0, crcLength);
        } catch (Exception e) {
            throw new PipeException("write_encrypted_error", e);
        } finally {
            IOUtils.closeQuietly(raf);
        }

        return encryptedData;
    }

    protected void decodeFile(File file, String key, String crc) {
        // ??
        RandomAccessFile raf = null;
        try {
            raf = new RandomAccessFile(file, "rw");

            long totallength = file.length();
            int keyLength = ByteUtils.stringToBytes(key).length;
            int crcLength = ByteUtils.stringToBytes(crc).length;
            // ?
            long pos = totallength - keyLength - crcLength;
            // 
            raf.seek(pos);
            // ?key
            byte[] keyBytes = new byte[keyLength];
            raf.read(keyBytes, 0, keyLength);
            String keystr = ByteUtils.bytesToString(keyBytes);
            if (!key.equals(keystr)) {
                throw new ChecksumException("unmatch garble key with[" + key + "],[" + keystr + "]");
            }

            // ??
            raf.seek(pos + keyLength);
            byte[] crcBytes = new byte[crcLength];
            raf.read(crcBytes, 0, crcLength);
            String crcStr = ByteUtils.bytesToString(crcBytes);
            if (!crc.equals(crcStr)) {
                throw new ChecksumException("unmatch crc with[" + crc + "],[" + crcStr + "]");
            }

            // 
            raf.setLength(pos);
        } catch (Exception e) {
            throw new PipeException("read_encrypted_error", e);
        } finally {
            IOUtils.closeQuietly(raf);
        }
    }

    // ==================== setter / getter ==================

    public void setConfigClientService(ConfigClientService configClientService) {
        this.configClientService = configClientService;
    }

    public void setJettyEmbedServer(JettyEmbedServer jettyEmbedServer) {
        this.jettyEmbedServer = jettyEmbedServer;
    }

    public void setRemoteUrlBuilder(RemoteUrlBuilder remoteUrlBuilder) {
        this.remoteUrlBuilder = remoteUrlBuilder;
    }

    public void setHtdocsDir(String htdocsDir) {
        this.htdocsDir = htdocsDir;
    }

    public void setDownloadDir(String downloadDir) {
        this.downloadDir = downloadDir;
    }

    public void setTimeout(Long timeout) {
        this.timeout = timeout;
    }

    public void setPeriod(Long period) {
        this.period = period;
    }

    public void setDataRetrieverFactory(DataRetrieverFactory dataRetrieverFactory) {
        this.dataRetrieverFactory = dataRetrieverFactory;
    }

}