com.gagein.crawler.FileDownLoader.java Source code

Introduction

Here is the source code for com.gagein.crawler.FileDownLoader.java

Source

//
//                               GageIn, Inc.
//                     Confidential and Proprietary
//                         ALL RIGHTS RESERVED.
//
//      This software is provided under license and may be used
//      or distributed only in accordance with the terms of
//      such license.
//
//
// History: Date        By whom      what
//          Jul 4, 2014     Administrator      Created
//
// CVS version control block - do not edit manually
// $Id: $

package com.gagein.crawler;

import java.io.DataOutputStream;
import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;

import org.apache.commons.httpclient.DefaultHttpMethodRetryHandler;
import org.apache.commons.httpclient.HttpClient;
import org.apache.commons.httpclient.HttpException;
import org.apache.commons.httpclient.HttpStatus;
import org.apache.commons.httpclient.methods.GetMethod;
import org.apache.commons.httpclient.params.HttpMethodParams;
import org.apache.log4j.Logger;

import com.gagein.bean.HtmlFileBean;
import com.gagein.dp.platform.common.api.util.HttpUtil;
import com.gagein.dp.platform.common.api.util.HttpUtil.HttpUtilResponse;
import com.gagein.util.FileUtils;

public class FileDownLoader {
    /** logger */
    private static final Logger logger = Logger.getLogger(FileDownLoader.class);

    /**
     * ? filePath ???
     */
    private void saveToLocal(byte[] data, String filePath) {
        try {
            System.out.println("========SaveToLocal========filePath===" + filePath);
            DataOutputStream out = new DataOutputStream(new FileOutputStream(new File(filePath)));
            for (int i = 0; i < data.length; i++)
                out.write(data[i]);
            out.flush();
            out.close();
        } catch (IOException e) {
            e.printStackTrace();
        }
    }

    /*  ? ,?? */
    public void downloadMainPage(HtmlFileBean hfb) {
        String charset = null;
        try {
            HttpUtilResponse res = HttpUtil.retrievePageInfo(hfb.getUrl());
            if (res != null) {
                FileUtils.writeByteArrayToFile(new File(hfb.getMainFilePath()), res.getContent());
                charset = res.getCharset();
                hfb.setCharset(charset);
            } else {
                System.out.println("on");
            }
        } catch (IOException e) {
            e.printStackTrace();
        }

    }

    /*  url ? */
    public String downloadLink(String url, HtmlFileBean hfb) {
        /* 1.? HttpClinet ? */
        HttpClient httpClient = new HttpClient();
        //  Http  5s
        httpClient.getHttpConnectionManager().getParams().setConnectionTimeout(5000);
        /* 2.? GetMethod ? */
        GetMethod getMethod = new GetMethod(url);
        //  get  5s
        getMethod.getParams().setParameter(HttpMethodParams.SO_TIMEOUT, 5000);
        // ??
        getMethod.getParams().setParameter(HttpMethodParams.RETRY_HANDLER, new DefaultHttpMethodRetryHandler());

        /* 3. HTTP GET  */
        try {
            int statusCode = httpClient.executeMethod(getMethod);
            if (statusCode != HttpStatus.SC_OK) {
                System.err.println("Method failed: " + getMethod.getStatusLine());
            }
            /* 4.? HTTP ? */
            byte[] responseBody = getMethod.getResponseBody();// ?

            // ? url ????
            url = FileUtils.getFileNameByUrl(url);

            saveToLocal(responseBody, hfb.getDirPath() + File.separator + url);
        } catch (HttpException e) {
            // ?????
            logger.debug("Please check your provided http " + "address!");
            e.printStackTrace();
        } catch (IOException e) {
            // ?
            logger.debug("?");
            e.printStackTrace();
        } finally {
            // 
            logger.debug("=======================" + url + "=============?");
            getMethod.releaseConnection();
        }
        return hfb.getDirPath() + url;
    }

    //  main 
    public static void main(String[] args) {

    }
}