com.unisa.test.TestCSVSerializer.java Source code

Introduction

Here is the source code for com.unisa.test.TestCSVSerializer.java

Source

package com.unisa.test;
/**
 * This file is part of UniSAHadoop.
 * 
 *     UniSAHadoop is free software: you can redistribute it and/or modify
 *     it under the terms of the GNU General Public License as published by  
 *     the Free Software Foundation, either version 3 of the License, or
 *     (at your option) any later version.
 * 
 *     UniSAHadoop is distributed in the hope that it will be useful,
 *     but WITHOUT ANY WARRANTY; without even the implied warranty of
 *     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 *     GNU General Public License for more details.
 * 
 *     You should have received a copy of the GNU General Public License
 *     along with UniSAHadoop.  If not, see <http://www.gnu.org/licenses/>.
 */

import java.io.BufferedReader;
import java.io.File;
import java.io.FileOutputStream;
import java.io.FileReader;
import java.io.OutputStream;
import java.util.HashMap;
import java.util.Map;

import org.apache.commons.io.FileUtils;
import org.apache.flume.Context;
import org.apache.flume.event.EventBuilder;
import org.apache.flume.interceptor.TimestampInterceptor;
import org.apache.flume.serialization.EventSerializer;
import org.apache.flume.serialization.EventSerializerFactory;
import junit.framework.Assert;
import junit.framework.Test;

import com.unisa.flume.serializer.CSVSerializer;

import com.google.common.base.Charsets;

public class TestCSVSerializer {

    private File testFile = new File("src/events.txt");
    private String msgText1 = "Dec 11 00:03:10 ltm-cw-man.cw.unisa.edu.au local/tmm1 info tmm1[5119]: Rule IR_mail.unisa.edu.au_owa_append <HTTP_RESPONSE>: #EXCHANGE#: Client: 1.125.48.172:1162 -> VIP:130.220.1.193:443 mail.unisa.edu.au/Microsoft-Server-ActiveSync?Cmd=Sync&User=uninet%5Clawtonlk&DeviceId=SEC15C5A37F9906C&DeviceType=SAMSUNGSMN915G -> Node: 10.78.2.10:443 with response 200";
    private String msgText2 = "Dec 11 14:46:22 ltm-cw-man.cw.unisa.edu.au local/tmm info tmm[5118]: Rule IR_mail.unisa.edu.au_owa_append <HTTP_RESPONSE>: #EXCHANGE#: Client: 118.211.99.72:54369 -> VIP:130.220.1.193:443 mail.unisa.edu.au/Microsoft-Server-ActiveSync?User=johamch&DeviceId=ApplDYTHTR6XDJ8T&DeviceType=iPad&Cmd=Sync -> Node: 10.78.2.10:443 with response 200";
    private String line;

    public void testCSVAndColumns1() throws Exception {
        Map<String, String> headers = new HashMap<String, String>();
        headers.put(TimestampInterceptor.Constants.TIMESTAMP, "1390883411761");
        OutputStream out = new FileOutputStream(testFile);
        Context context = new Context();
        context.put(CSVSerializer.FORMAT, "CSV");
        context.put(CSVSerializer.REGEX,
                "^([^ ]+ [^ ]+) ([^ ]+).* Client: ([^:]+).* VIP:([^:]+).*/([^?]+).*User=([^&]+).*DeviceType=([^& ]+).* Node: ([^:]+).* response ([^ ]+)");
        context.put(CSVSerializer.REGEX_ORDER, "1 2 3 4 5 6 7 8 9");
        EventSerializer serializer = EventSerializerFactory
                .getInstance("com.unisa.flume.serializer.CSVSerializer$Builder", context, out);
        serializer.afterCreate();
        serializer.write(EventBuilder.withBody(msgText1, Charsets.UTF_8, headers));
        serializer.flush();
        serializer.beforeClose();
        out.flush();
        out.close();

        BufferedReader reader = new BufferedReader(new FileReader(testFile));
        line = reader.readLine();
        System.out.println(line);
        Assert.assertEquals(
                "1390883411761,Dec 11,00:03:10,1.125.48.172,130.220.1.193,Microsoft-Server-ActiveSync,uninet%5Clawtonlk,SAMSUNGSMN915G,10.78.2.10,200",
                line);
        reader.close();

        FileUtils.forceDelete(testFile);
    }

    public void testCSVAndColumns2() throws Exception {
        Map<String, String> headers = new HashMap<String, String>();
        headers.put(TimestampInterceptor.Constants.TIMESTAMP, "1390883411761");
        OutputStream out = new FileOutputStream(testFile);
        Context context = new Context();
        context.put(CSVSerializer.FORMAT, "CSV");
        context.put(CSVSerializer.REGEX,
                "^([^ ]+ [^ ]+) ([^ ]+).* Client: ([^:]+).* VIP:([^:]+).*/([^?]+).*User=([^&]+).*DeviceType=([^& ]+).* Node: ([^:]+).* response ([^ ]+)");
        context.put(CSVSerializer.REGEX_ORDER, "1 2 3 4 5 6 7 8 9");

        EventSerializer serializer = EventSerializerFactory
                .getInstance("com.unisa.flume.serializer.CSVSerializer$Builder", context, out);
        serializer.afterCreate();
        serializer.write(EventBuilder.withBody(msgText2, Charsets.UTF_8, headers));
        serializer.flush();
        serializer.beforeClose();
        out.flush();
        out.close();

        BufferedReader reader = new BufferedReader(new FileReader(testFile));
        line = reader.readLine();
        System.out.println(line);
        Assert.assertEquals(
                "1390883411761,Dec 11,14:46:22,118.211.99.72,130.220.1.193,Microsoft-Server-ActiveSync,johamch,iPad,10.78.2.10,200",
                line);
        reader.close();

        FileUtils.forceDelete(testFile);
    }

}