infinidb.hadoop.db.IDBFileInputFormat.java Source code

Introduction

Here is the source code for infinidb.hadoop.db.IDBFileInputFormat.java

Source

/*
 * Copyright (c) 2014 InfiniDB, Inc.
 *
 * InfiniDB, Inc. licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License. You may obtain a copy of the License at
 *
 *   http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing,
 * software distributed under the License is distributed on an
 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
 * KIND, either express or implied. See the License for the
 * specific language governing permissions and limitations
 * under the License.
 */

package infinidb.hadoop.db;

import infinidb.hadoop.db.InfiniDBConfiguration;

import java.io.*;
import java.sql.*;
import java.util.Date;
import java.util.Formatter;
import java.io.IOException;

import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.mapred.LineRecordReader;
import org.apache.hadoop.mapred.FileSplit;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.NullWritable;
import org.apache.hadoop.io.Writable;
import org.apache.hadoop.mapred.lib.db.*;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.lib.input.*;
import org.apache.hadoop.mapreduce.lib.output.*;
import org.apache.hadoop.mapred.JobClient;
import org.apache.hadoop.mapred.JobConf;
import org.apache.hadoop.mapred.Mapper;
import org.apache.hadoop.mapred.Reducer;
import org.apache.hadoop.conf.*;
import org.apache.hadoop.util.*;
import org.apache.hadoop.*;
import org.apache.hadoop.mapred.FileInputFormat;
import org.apache.hadoop.mapred.*;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

public class IDBFileInputFormat extends org.apache.hadoop.mapred.FileInputFormat<NullWritable, NullWritable> {

    private static final Logger LOG = LoggerFactory.getLogger(DBInputFormat.class);

    @Override
    public RecordReader<NullWritable, NullWritable> getRecordReader(InputSplit arg0, JobConf arg1, Reporter arg2)
            throws IOException {
        final String filename = ((FileSplit) arg0).getPath().toString();
        final JobConf job = arg1;

        return new RecordReader<NullWritable, NullWritable>() {
            private boolean unread = true;

            @Override
            public void close() throws IOException {
            }

            @Override
            public NullWritable createKey() {
                return NullWritable.get();
            }

            @Override
            public NullWritable createValue() {
                return NullWritable.get();
            }

            @Override
            public long getPos() throws IOException {
                return 0;
            }

            @Override
            public float getProgress() throws IOException {
                return unread ? 0 : 1;
            }

            @Override
            /* spawn a cpimport process for each input file */
            public boolean next(NullWritable arg0, NullWritable arg1) throws IOException {
                InfiniDBConfiguration dbConf = new InfiniDBConfiguration(job);
                String schemaName = dbConf.getOutputSchemaName();
                String tableName = (filename.substring(filename.lastIndexOf('/') + 1, filename.length()));
                tableName = tableName.substring(0, tableName.lastIndexOf('.'));
                String output = job.get("mapred.output.dir");
                if (unread) {
                    try {
                        StringBuilder loadCmdStr = new StringBuilder();
                        loadCmdStr.append(dbConf.getInfiniDBHome());
                        loadCmdStr.append("/bin/");
                        loadCmdStr.append("infinidoop_load.sh ");
                        loadCmdStr.append(filename);
                        loadCmdStr.append(" ");
                        loadCmdStr.append(schemaName);
                        loadCmdStr.append(" ");
                        loadCmdStr.append(tableName);

                        Process lChldProc = Runtime.getRuntime().exec(loadCmdStr.toString());

                        // Wait for the child to exit
                        lChldProc.waitFor();
                        BufferedReader lChldProcOutStream = new BufferedReader(
                                new InputStreamReader(lChldProc.getInputStream()));
                        BufferedReader stdError = new BufferedReader(
                                new InputStreamReader(lChldProc.getErrorStream()));

                        String lChldProcOutPutStr = null;
                        StringBuffer outpath = new StringBuffer();
                        outpath.append(job.getWorkingDirectory());
                        outpath.append("/");
                        outpath.append(output);
                        outpath.append("/");
                        outpath.append(tableName);
                        outpath.append(".log");

                        Path pt = new Path(outpath.toString());
                        FileSystem fs = FileSystem.get(new Configuration());
                        BufferedWriter br = new BufferedWriter(new OutputStreamWriter(fs.create(pt, false)));

                        // catch output
                        while ((lChldProcOutPutStr = lChldProcOutStream.readLine()) != null) {
                            br.write(lChldProcOutPutStr);
                            br.newLine();
                        }

                        // catch error
                        while ((lChldProcOutPutStr = stdError.readLine()) != null) {
                            br.write(lChldProcOutPutStr);
                            br.newLine();
                        }

                        //br.write(outpath.toString());
                        //br.newLine();
                        //br.write(loadCmdStr.toString());
                        //br.newLine();
                        //br.write(filename);
                        br.close();

                        lChldProcOutStream.close();
                    } catch (Exception e) {
                        e.printStackTrace();
                    }
                    unread = false;
                    return true;
                } else {
                    return false;
                }
            }
        };
    }

    @Override
    protected boolean isSplitable(FileSystem fs, Path filename) {
        return false;
    }

}