Java tutorial
// Copyright (c) 2013, Facebook, Inc. All rights reserved. /** * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information * regarding copyright ownership. The ASF licenses this file * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package com.facebook.hive.orc; import java.io.DataInput; import java.io.DataOutput; import java.io.IOException; import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; import org.apache.hadoop.hive.serde2.objectinspector.ListObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.MapObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.SettableListObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.SettableMapObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.SettableStructObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.StructField; import org.apache.hadoop.hive.serde2.typeinfo.ListTypeInfo; import org.apache.hadoop.hive.serde2.typeinfo.MapTypeInfo; import org.apache.hadoop.hive.serde2.typeinfo.StructTypeInfo; import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo; import org.apache.hadoop.io.Writable; import com.facebook.hive.orc.lazy.OrcLazyObjectInspectorUtils; public final class OrcStruct implements Writable { private Object[] fields; private List<String> fieldNames; public OrcStruct() { this.fields = new Object[0]; this.fieldNames = new ArrayList<String>(); } public OrcStruct(List<String> fieldNames) { this.fields = new Object[fieldNames.size()]; this.fieldNames = fieldNames; } public Object getFieldValue(int fieldIndex) { return fields[fieldIndex]; } public void setFieldValue(int fieldIndex, Object value) { fields[fieldIndex] = value; } public int getNumFields() { return fields.length; } public List<String> getFieldNames() { return fieldNames; } /** * Change the names and number of fields in the struct. No effect if the number of * fields is the same. The old field values are copied to the new array. * @param numFields the new number of fields */ public void setFieldNames(List<String> fieldNames) { this.fieldNames = fieldNames; if (fields.length != fieldNames.size()) { Object[] oldFields = fields; fields = new Object[fieldNames.size()]; System.arraycopy(oldFields, 0, fields, 0, Math.min(oldFields.length, fieldNames.size())); } } public void appendField(String fieldName) { fieldNames.add(fieldName); setFieldNames(fieldNames); } @Override public void write(DataOutput dataOutput) throws IOException { throw new UnsupportedOperationException("write unsupported"); } @Override public void readFields(DataInput dataInput) throws IOException { throw new UnsupportedOperationException("readFields unsupported"); } @Override public boolean equals(Object other) { if (other == null || other.getClass() != OrcStruct.class) { return false; } else { OrcStruct oth = (OrcStruct) other; if (fields.length != oth.fields.length) { return false; } for (int i = 0; i < fields.length; ++i) { if (fields[i] == null) { if (oth.fields[i] != null) { return false; } } else { if (!fields[i].equals(oth.fields[i])) { return false; } } } return true; } } @Override public int hashCode() { int result = fields.length; for (Object field : fields) { if (field != null) { result ^= field.hashCode(); } } return result; } @Override public String toString() { StringBuilder buffer = new StringBuilder(); buffer.append("{"); for (int i = 0; i < fields.length; ++i) { if (i != 0) { buffer.append(", "); } buffer.append(fields[i]); } buffer.append("}"); return buffer.toString(); } public static class Field implements StructField { private final String name; private final ObjectInspector inspector; private final int offset; public Field(String name, ObjectInspector inspector, int offset) { this.name = name; this.inspector = inspector; this.offset = offset; } @Override public String getFieldName() { return name; } @Override public ObjectInspector getFieldObjectInspector() { return inspector; } @Override public String getFieldComment() { return null; } public int getOffset() { return offset; } } public static class OrcStructInspector extends SettableStructObjectInspector { private final List<StructField> fields; public OrcStructInspector(StructTypeInfo info) { ArrayList<String> fieldNames = info.getAllStructFieldNames(); ArrayList<TypeInfo> fieldTypes = info.getAllStructFieldTypeInfos(); fields = new ArrayList<StructField>(fieldNames.size()); for (int i = 0; i < fieldNames.size(); ++i) { fields.add(new Field(fieldNames.get(i), OrcLazyObjectInspectorUtils.createWritableObjectInspector(fieldTypes.get(i)), i)); } } public OrcStructInspector(int columnId, List<OrcProto.Type> types) { OrcProto.Type type = types.get(columnId); int fieldCount = type.getSubtypesCount(); fields = new ArrayList<StructField>(fieldCount); for (int i = 0; i < fieldCount; ++i) { int fieldType = type.getSubtypes(i); fields.add(new Field(type.getFieldNames(i), OrcLazyObjectInspectorUtils.createWritableObjectInspector(fieldType, types), i)); } } @Override public List<StructField> getAllStructFieldRefs() { return fields; } @Override public StructField getStructFieldRef(String s) { s = s.toLowerCase(); for (StructField field : fields) { if (field.getFieldName().equals(s)) { return field; } } return null; } @Override public Object getStructFieldData(Object object, StructField field) { if (object == null) { return null; } int offset = ((Field) field).offset; OrcStruct struct = (OrcStruct) object; if (offset >= struct.fields.length) { return null; } return struct.fields[offset]; } @Override public List<Object> getStructFieldsDataAsList(Object object) { if (object == null) { return null; } OrcStruct struct = (OrcStruct) object; List<Object> result = new ArrayList<Object>(struct.fields.length); for (Object child : struct.fields) { result.add(child); } return result; } @Override public String getTypeName() { StringBuilder buffer = new StringBuilder(); buffer.append("struct<"); for (int i = 0; i < fields.size(); ++i) { StructField field = fields.get(i); if (i != 0) { buffer.append(","); } buffer.append(field.getFieldName()); buffer.append(":"); buffer.append(field.getFieldObjectInspector().getTypeName()); } buffer.append(">"); return buffer.toString(); } @Override public Category getCategory() { return Category.STRUCT; } @Override public Object create() { return new OrcStruct(); } @Override public Object setStructFieldData(Object struct, StructField field, Object fieldValue) { OrcStruct orcStruct = (OrcStruct) struct; int offset = ((Field) field).offset; // if the offset is bigger than our current number of fields, grow it if (orcStruct.getNumFields() <= offset) { orcStruct.appendField(((Field) field).getFieldName()); } orcStruct.setFieldValue(offset, fieldValue); return struct; } @Override public boolean equals(Object o) { if (o == null || o.getClass() != getClass()) { return false; } else if (o == this) { return true; } else { List<StructField> other = ((OrcStructInspector) o).fields; if (other.size() != fields.size()) { return false; } for (int i = 0; i < fields.size(); ++i) { StructField left = other.get(i); StructField right = fields.get(i); if (!(left.getFieldName().equals(right.getFieldName()) && left.getFieldObjectInspector().equals(right.getFieldObjectInspector()))) { return false; } } return true; } } } public static class OrcMapObjectInspector implements MapObjectInspector, SettableMapObjectInspector { private final ObjectInspector key; private final ObjectInspector value; public OrcMapObjectInspector(MapTypeInfo info) { key = OrcLazyObjectInspectorUtils.createWritableObjectInspector(info.getMapKeyTypeInfo()); value = OrcLazyObjectInspectorUtils.createWritableObjectInspector(info.getMapValueTypeInfo()); } public OrcMapObjectInspector(int columnId, List<OrcProto.Type> types) { OrcProto.Type type = types.get(columnId); key = OrcLazyObjectInspectorUtils.createWritableObjectInspector(type.getSubtypes(0), types); value = OrcLazyObjectInspectorUtils.createWritableObjectInspector(type.getSubtypes(1), types); } @Override public ObjectInspector getMapKeyObjectInspector() { return key; } @Override public ObjectInspector getMapValueObjectInspector() { return value; } @Override public Object getMapValueElement(Object map, Object key) { if (map == null) { return null; } return ((Map<?, ?>) map).get(key); } @Override @SuppressWarnings("unchecked") public Map<Object, Object> getMap(Object map) { return (Map<Object, Object>) map; } @Override public int getMapSize(Object map) { if (map == null) { return -1; } return ((Map<?, ?>) map).size(); } @Override public String getTypeName() { return "map<" + key.getTypeName() + "," + value.getTypeName() + ">"; } @Override public Category getCategory() { return Category.MAP; } @Override public Object create() { return new HashMap<Object, Object>(); } @Override public Object put(Object map, Object key, Object value) { ((Map) map).put(key, value); return map; } @Override public Object remove(Object map, Object key) { ((Map<?, ?>) map).remove(key); return map; } @Override public Object clear(Object map) { ((Map<?, ?>) map).clear(); return map; } @Override public boolean equals(Object o) { if (o == null || o.getClass() != getClass()) { return false; } else if (o == this) { return true; } else { OrcMapObjectInspector other = (OrcMapObjectInspector) o; return other.key.equals(key) && other.value.equals(value); } } } public static class OrcListObjectInspector implements ListObjectInspector, SettableListObjectInspector { private final ObjectInspector child; public OrcListObjectInspector(ListTypeInfo info) { child = OrcLazyObjectInspectorUtils.createWritableObjectInspector(info.getListElementTypeInfo()); } public OrcListObjectInspector(int columnId, List<OrcProto.Type> types) { OrcProto.Type type = types.get(columnId); child = OrcLazyObjectInspectorUtils.createWritableObjectInspector(type.getSubtypes(0), types); } @Override public ObjectInspector getListElementObjectInspector() { return child; } @Override public Object getListElement(Object list, int i) { if (list == null || i < 0 || i >= ((List<?>) list).size()) { return null; } return ((List<?>) list).get(i); } @Override public int getListLength(Object list) { if (list == null) { return -1; } return ((List<?>) list).size(); } @Override @SuppressWarnings("unchecked") public List<?> getList(Object list) { return (List<?>) list; } @Override public String getTypeName() { return "array<" + child.getTypeName() + ">"; } @Override public Category getCategory() { return Category.LIST; } @Override public Object create(int size) { ArrayList<Object> result = new ArrayList<Object>(size); for (int i = 0; i < size; ++i) { result.add(null); } return result; } @Override public Object set(Object list, int index, Object element) { List l = (List) list; for (int i = l.size(); i < index + 1; ++i) { l.add(null); } l.set(index, element); return list; } @Override public Object resize(Object list, int newSize) { ((ArrayList<?>) list).ensureCapacity(newSize); return list; } @Override public boolean equals(Object o) { if (o == null || o.getClass() != getClass()) { return false; } else if (o == this) { return true; } else { ObjectInspector other = ((OrcListObjectInspector) o).child; return other.equals(child); } } } }