Java tutorial
/** * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with this * work for additional information regarding copyright ownership. The ASF * licenses this file to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the * License for the specific language governing permissions and limitations under * the License. */ package org.apache.pig.piggybank.storage; import java.io.IOException; import java.util.ArrayList; import java.util.Arrays; import java.util.LinkedHashSet; import java.util.List; import java.util.Map; import java.util.Properties; import java.util.Set; import java.util.regex.Pattern; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.ql.session.SessionState; import org.apache.hadoop.hive.serde.Constants; import org.apache.hadoop.hive.serde2.SerDeException; import org.apache.hadoop.hive.serde2.columnar.BytesRefArrayWritable; import org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe; import org.apache.hadoop.hive.serde2.columnar.ColumnarStruct; import org.apache.hadoop.io.LongWritable; import org.apache.hadoop.mapreduce.InputFormat; import org.apache.hadoop.mapreduce.Job; import org.apache.hadoop.mapreduce.RecordReader; import org.apache.hadoop.mapreduce.lib.input.FileInputFormat; import org.apache.pig.Expression; import org.apache.pig.FileInputLoadFunc; import org.apache.pig.LoadMetadata; import org.apache.pig.LoadPushDown; import org.apache.pig.ResourceSchema; import org.apache.pig.ResourceSchema.ResourceFieldSchema; import org.apache.pig.ResourceStatistics; import org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.PigSplit; import org.apache.pig.data.DataType; import org.apache.pig.data.Tuple; import org.apache.pig.data.TupleFactory; import org.apache.pig.impl.logicalLayer.FrontendException; import org.apache.pig.impl.logicalLayer.schema.Schema; import org.apache.pig.impl.logicalLayer.schema.Schema.FieldSchema; import org.apache.pig.impl.util.UDFContext; import org.apache.pig.piggybank.storage.hiverc.HiveRCInputFormat; import org.apache.pig.piggybank.storage.hiverc.HiveRCRecordReader; import org.apache.pig.piggybank.storage.hiverc.HiveRCSchemaUtil; import org.apache.pig.piggybank.storage.partition.PathPartitionHelper; /** * Loader for Hive RC Columnar files.<br/> * Supports the following types:<br/> * * * <table> * <tr> * <th>Hive Type</th> * <th>Pig Type from DataType</th> * </tr> * <tr> * <td>string</td> * <td>CHARARRAY</td> * </tr> * <tr> * <td>int</td> * <td>INTEGER</td> * </tr> * <tr> * <td>bigint or long</td> * <td>LONG</td> * </tr> * <tr> * <td>float</td> * <td>float</td> * </tr> * <tr> * <td>double</td> * <td>DOUBLE</td> * </tr> * <tr> * <td>boolean</td> * <td>BOOLEAN</td> * </tr> * <tr> * <td>byte</td> * <td>BYTE</td> * </tr> * <tr> * <td>array</td> * <td>TUPLE</td> * </tr> * <tr> * <td>map</td> * <td>MAP</td> * </tr> * </table> * * <p/> * <b>Partitions</b><br/> * The input paths are scanned by the loader for [partition name]=[value] * patterns in the subdirectories.<br/> * If detected these partitions are appended to the table schema.<br/> * For example if you have the directory structure:<br/> * * <pre> * /user/hive/warehouse/mytable * /year=2010/month=02/day=01 * </pre> * * The mytable schema is (id int,name string).<br/> * The final schema returned in pig will be (id:int, name:chararray, * year:chararray, month:chararray, day:chararray).<br/> * <p/> * Usage 1: * <p/> * To load a hive table: uid bigint, ts long, arr ARRAY<string,string>, m * MAP<String, String> <br/> * <code> * <pre> * a = LOAD 'file' USING HiveColumnarLoader("uid bigint, ts long, arr array<string,string>, m map<string,string>"); * -- to reference the fields * b = FOREACH GENERATE a.uid, a.ts, a.arr, a.m; * </pre> * </code> * <p/> * Usage 2: * <p/> * To load a hive table: uid bigint, ts long, arr ARRAY<string,string>, m * MAP<String, String> only processing dates 2009-10-01 to 2009-10-02 in a <br/> * date partitioned hive table.<br/> * <b>Old Usage</b><br/> * <b>Note:</b> The partitions can be filtered by using pig's FILTER operator.<br/> * <code> * <pre> * a = LOAD 'file' USING HiveColumnarLoader("uid bigint, ts long, arr array<string,string>, m map<string,string>", "2009-10-01:2009-10-02"); * -- to reference the fields * b = FOREACH GENERATE a.uid, a.ts, a.arr, a.m; * </pre> * </code> <br/> * <b>New Usage</b/><br/> * <code> * <pre> * a = LOAD 'file' USING HiveColumnarLoader("uid bigint, ts long, arr array<string,string>, m map<string,string>"); * f = FILTER a BY daydate>='2009-10-01' AND daydate >='2009-10-02'; * </pre> * </code> * <p/> * Usage 3: * <p/> * To load a hive table: uid bigint, ts long, arr ARRAY<string,string>, m * MAP<String, String> only reading column uid and ts for dates 2009-10-01 to * 2009-10-02.<br/ <br/> * <b>Old Usage</b><br/> * <b>Note:<b/> This behaviour is now supported in pig by LoadPushDown adding * the columns needed to be loaded like below is ignored and pig will * automatically send the columns used by the script to the loader.<br/> * <code> * <pre> * a = LOAD 'file' USING HiveColumnarLoader("uid bigint, ts long, arr array<string,string>, m map<string,string>"); * f = FILTER a BY daydate>='2009-10-01' AND daydate >='2009-10-02'; * -- to reference the fields * b = FOREACH a GENERATE uid, ts, arr, m; * </pre> * </code> * <p/> * <b>Issues</b> * <p/> * <u>Table schema definition</u><br/> * The schema definition must be column name followed by a space then a comma * then no space and the next column name and so on.<br/> * This so column1 string, column2 string will not work, it must be column1 * string,column2 string * <p/> * <u>Partitioning</u><br/> * Partitions must be in the format [partition name]=[partition value]<br/> * Only strings are supported in the partitioning.<br/> * Partitions must follow the same naming convention for all sub directories in * a table<br/> * For example:<br/> * The following is not valid:<br/> * * <pre> * mytable/hour=00 * mytable/day=01/hour=00 * </pre> * **/ public class HiveColumnarLoader extends FileInputLoadFunc implements LoadMetadata, LoadPushDown { public static final String PROJECTION_ID = HiveColumnarLoader.class.getName() + ".projection"; public static final String DATE_RANGE = HiveColumnarLoader.class.getName() + ".date-range"; /** * Regex to filter out column names */ protected static final Pattern pcols = Pattern.compile("[a-zA-Z_0-9]*[ ]"); protected static final Log LOG = LogFactory.getLog(HiveColumnarLoader.class); protected TupleFactory tupleFactory = TupleFactory.getInstance(); String signature = ""; // we need to save the dateRange from the constructor if provided to add to // the UDFContext only when the signature is available. String dateRange = null; HiveRCRecordReader reader; ColumnarSerDe serde = null; Configuration conf = null; ResourceSchema pigSchema; boolean partitionKeysSet = false; BytesRefArrayWritable buff = null; private Properties props; private HiveConf hiveConf; transient int[] requiredColumns; transient Set<String> partitionColumns; /** * Implements the logic for searching partition keys and applying parition * filtering */ transient PathPartitionHelper pathPartitionerHelper = new PathPartitionHelper(); transient Path currentPath = null; transient Map<String, String> currentPathPartitionKeyMap; /** * Table schema should be a space and comma separated string describing the * Hive schema.<br/> * For example uid BIGINT, pid long, means 1 column of uid type BIGINT and * one column of pid type LONG.<br/> * The types are not case sensitive. * * @param table_schema * This property cannot be null */ public HiveColumnarLoader(String table_schema) { setup(table_schema); } /** * This constructor is for backward compatibility. * * Table schema should be a space and comma separated string describing the * Hive schema.<br/> * For example uid BIGINT, pid long, means 1 column of uid type BIGINT and * one column of pid type LONG.<br/> * The types are not case sensitive. * * @param table_schema * This property cannot be null * @param dateRange * String * @param columns * String not used any more */ public HiveColumnarLoader(String table_schema, String dateRange, String columns) { setup(table_schema); this.dateRange = dateRange; } /** * This constructor is for backward compatibility. * * Table schema should be a space and comma separated string describing the * Hive schema.<br/> * For example uid BIGINT, pid long, means 1 column of uid type BIGINT and * one column of pid type LONG.<br/> * The types are not case sensitive. * * @param table_schema * This property cannot be null * @param dateRange * String */ public HiveColumnarLoader(String table_schema, String dateRange) { setup(table_schema); this.dateRange = dateRange; } private Properties getUDFContext() { return UDFContext.getUDFContext().getUDFProperties(this.getClass(), new String[] { signature }); } @Override public InputFormat<LongWritable, BytesRefArrayWritable> getInputFormat() throws IOException { LOG.info("Signature: " + signature); return new HiveRCInputFormat(signature); } @Override public Tuple getNext() throws IOException { Tuple tuple = null; try { if (reader.nextKeyValue()) { BytesRefArrayWritable buff = reader.getCurrentValue(); ColumnarStruct struct = readColumnarStruct(buff); tuple = readColumnarTuple(struct, reader.getSplitPath()); } } catch (InterruptedException e) { throw new IOException(e.toString(), e); } return tuple; } @Override public void prepareToRead(@SuppressWarnings("rawtypes") RecordReader reader, PigSplit split) throws IOException { this.reader = (HiveRCRecordReader) reader; // check that the required indexes actually exist i.e. the columns that // should be read. // assuming this is always defined simplifies the readColumnarTuple // logic. int requiredIndexes[] = getRequiredColumns(); if (requiredIndexes == null) { int fieldLen = pigSchema.getFields().length; // if any the partition keys should already exist String[] partitionKeys = getPartitionKeys(null, null); if (partitionKeys != null) { fieldLen += partitionKeys.length; } requiredIndexes = new int[fieldLen]; for (int i = 0; i < fieldLen; i++) { requiredIndexes[i] = i; } this.requiredColumns = requiredIndexes; } try { serde = new ColumnarSerDe(); serde.initialize(hiveConf, props); } catch (SerDeException e) { LOG.error(e.toString(), e); throw new IOException(e); } } @Override public void setLocation(String location, Job job) throws IOException { FileInputFormat.setInputPaths(job, location); } /** * Does the configuration setup and schema parsing and setup. * * @param table_schema * String * @param columnsToRead * String */ private void setup(String table_schema) { if (table_schema == null) throw new RuntimeException( "The table schema must be defined as colname type, colname type. All types are hive types"); // create basic configuration for hdfs and hive conf = new Configuration(); hiveConf = new HiveConf(conf, SessionState.class); // parse the table_schema string List<String> types = HiveRCSchemaUtil.parseSchemaTypes(table_schema); List<String> cols = HiveRCSchemaUtil.parseSchema(pcols, table_schema); List<FieldSchema> fieldSchemaList = new ArrayList<FieldSchema>(cols.size()); for (int i = 0; i < cols.size(); i++) { fieldSchemaList.add(new FieldSchema(cols.get(i), HiveRCSchemaUtil.findPigDataType(types.get(i)))); } pigSchema = new ResourceSchema(new Schema(fieldSchemaList)); props = new Properties(); // setting table schema properties for ColumnarSerDe // these properties are never changed by the columns to read filter, // because the columnar serde needs to now the // complete format of each record. props.setProperty(Constants.LIST_COLUMNS, HiveRCSchemaUtil.listToString(cols)); props.setProperty(Constants.LIST_COLUMN_TYPES, HiveRCSchemaUtil.listToString(types)); } /** * Uses the ColumnarSerde to deserialize the buff:BytesRefArrayWritable into * a ColumnarStruct instance. * * @param buff * BytesRefArrayWritable * @return ColumnarStruct */ private ColumnarStruct readColumnarStruct(BytesRefArrayWritable buff) { // use ColumnarSerDe to deserialize row ColumnarStruct struct = null; try { struct = (ColumnarStruct) serde.deserialize(buff); } catch (SerDeException e) { LOG.error(e.toString(), e); throw new RuntimeException(e.toString(), e); } return struct; } /** * Only read the columns that were requested in the constructor.<br/> * * @param struct * ColumnarStruct * @param path * Path * @return Tuple * @throws IOException */ private Tuple readColumnarTuple(ColumnarStruct struct, Path path) throws IOException { int[] columnIndexes = getRequiredColumns(); // the partition keys if any will already be in the UDFContext here. String[] partitionKeys = getPartitionKeys(null, null); // only if the path has changed should be run the if (currentPath == null || !currentPath.equals(path)) { currentPathPartitionKeyMap = (partitionKeys == null) ? null : pathPartitionerHelper.getPathPartitionKeyValues(path.toString()); currentPath = path; } // if the partitionColumns is null this value will stop the for loop // below from trynig to add any partition columns // that do not exist int partitionColumnStartIndex = Integer.MAX_VALUE; if (!(partitionColumns == null || partitionColumns.size() == 0)) { // partition columns are always appended to the schema fields. partitionColumnStartIndex = pigSchema.getFields().length; } // create tuple with determined previous size Tuple t = tupleFactory.newTuple(columnIndexes.length); // read in all columns for (int i = 0; i < columnIndexes.length; i++) { int columnIndex = columnIndexes[i]; if (columnIndex < partitionColumnStartIndex) { Object obj = struct.getField(columnIndex); Object pigType = HiveRCSchemaUtil.extractPigTypeFromHiveType(obj); t.set(i, pigType); } else { // read the partition columns // will only be executed if partitionColumns is not null String key = partitionKeys[columnIndex - partitionColumnStartIndex]; Object value = currentPathPartitionKeyMap.get(key); t.set(i, value); } } return t; } /** * Will parse the required columns from the UDFContext properties if the * requiredColumns[] variable is null, or else just return the * requiredColumns. * * @return int[] */ private int[] getRequiredColumns() { if (requiredColumns == null) { Properties properties = getUDFContext(); String projectionStr = properties.getProperty(PROJECTION_ID); if (projectionStr != null) { String[] split = projectionStr.split(","); int columnIndexes[] = new int[split.length]; int index = 0; for (String splitItem : split) { columnIndexes[index++] = Integer.parseInt(splitItem); } requiredColumns = columnIndexes; } } return requiredColumns; } /** * Reads the partition columns * * @param location * @param job * @return */ private Set<String> getPartitionColumns(String location, Job job) { if (partitionColumns == null) { // read the partition columns from the UDF Context first. // if not in the UDF context then read it using the PathPartitioner. Properties properties = getUDFContext(); if (properties == null) properties = new Properties(); String partitionColumnStr = properties.getProperty(PathPartitionHelper.PARTITION_COLUMNS); if (partitionColumnStr == null && !(location == null || job == null)) { // if it hasn't been written yet. Set<String> partitionColumnSet; try { partitionColumnSet = pathPartitionerHelper.getPartitionKeys(location, job.getConfiguration()); } catch (IOException e) { RuntimeException rte = new RuntimeException(e); rte.setStackTrace(e.getStackTrace()); throw rte; } if (partitionColumnSet != null) { StringBuilder buff = new StringBuilder(); int i = 0; for (String column : partitionColumnSet) { if (i++ != 0) { buff.append(','); } buff.append(column); } String buffStr = buff.toString().trim(); if (buffStr.length() > 0) { properties.setProperty(PathPartitionHelper.PARTITION_COLUMNS, buff.toString()); } partitionColumns = partitionColumnSet; } } else { // the partition columns has been set already in the UDF Context if (partitionColumnStr != null) { String split[] = partitionColumnStr.split(","); partitionColumns = new LinkedHashSet<String>(); if (split.length > 0) { for (String splitItem : split) { partitionColumns.add(splitItem); } } } } } return partitionColumns; } @Override public String[] getPartitionKeys(String location, Job job) throws IOException { Set<String> partitionKeys = getPartitionColumns(location, job); return partitionKeys == null ? null : partitionKeys.toArray(new String[] {}); } @Override public ResourceSchema getSchema(String location, Job job) throws IOException { if (!partitionKeysSet) { Set<String> keys = getPartitionColumns(location, job); if (!(keys == null || keys.size() == 0)) { // re-edit the pigSchema to contain the new partition keys. ResourceFieldSchema[] fields = pigSchema.getFields(); LOG.debug("Schema: " + Arrays.toString(fields)); ResourceFieldSchema[] newFields = Arrays.copyOf(fields, fields.length + keys.size()); int index = fields.length; for (String key : keys) { newFields[index++] = new ResourceFieldSchema(new FieldSchema(key, DataType.CHARARRAY)); } pigSchema.setFields(newFields); LOG.debug("Added partition fields: " + keys + " to loader schema"); LOG.debug("Schema is: " + Arrays.toString(newFields)); } partitionKeysSet = true; } return pigSchema; } @Override public ResourceStatistics getStatistics(String location, Job job) throws IOException { return null; } @Override public void setPartitionFilter(Expression partitionFilter) throws IOException { getUDFContext().setProperty(PathPartitionHelper.PARITITION_FILTER_EXPRESSION, partitionFilter.toString()); } @Override public List<OperatorSet> getFeatures() { return Arrays.asList(LoadPushDown.OperatorSet.PROJECTION); } @Override public RequiredFieldResponse pushProjection(RequiredFieldList requiredFieldList) throws FrontendException { // save the required field list to the UDFContext properties. StringBuilder buff = new StringBuilder(); int i = 0; for (RequiredField f : requiredFieldList.getFields()) { if (i++ != 0) buff.append(','); buff.append(f.getIndex()); } Properties properties = getUDFContext(); properties.setProperty(PROJECTION_ID, buff.toString()); return new RequiredFieldResponse(true); } @Override public void setUDFContextSignature(String signature) { super.setUDFContextSignature(signature); LOG.debug("Signature: " + signature); this.signature = signature; // this provides backwards compatibility // the HiveRCInputFormat will read this and if set will perform the // needed partitionFiltering if (dateRange != null) { getUDFContext().setProperty(DATE_RANGE, dateRange); } } }