Java tutorial
/* * * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information * regarding copyright ownership. The ASF licenses this file * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.hadoop.hive.metastore; import com.google.common.base.Joiner; import org.apache.commons.lang.BooleanUtils; import org.apache.hadoop.hive.metastore.api.FieldSchema; import org.apache.hadoop.hive.metastore.api.MetaException; import org.apache.hadoop.hive.metastore.api.Order; import org.apache.hadoop.hive.metastore.api.Partition; import org.apache.hadoop.hive.metastore.api.SerDeInfo; import org.apache.hadoop.hive.metastore.api.SkewedInfo; import org.apache.hadoop.hive.metastore.api.StorageDescriptor; import org.apache.hadoop.hive.metastore.utils.MetaStoreServerUtils; import org.apache.hadoop.hive.metastore.utils.MetaStoreUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import javax.jdo.PersistenceManager; import javax.jdo.Query; import java.sql.Blob; import java.sql.Clob; import java.sql.SQLException; import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; import java.util.Iterator; import java.util.List; import java.util.Map; import java.util.TreeMap; import java.util.stream.Collectors; /** * Helper utilities used by DirectSQL code in HiveMetastore. */ class MetastoreDirectSqlUtils { private static final Logger LOG = LoggerFactory.getLogger(MetastoreDirectSqlUtils.class); private MetastoreDirectSqlUtils() { } @SuppressWarnings("unchecked") static <T> T executeWithArray(Query query, Object[] params, String sql) throws MetaException { try { return (T) ((params == null) ? query.execute() : query.executeWithArray(params)); } catch (Exception ex) { StringBuilder errorBuilder = new StringBuilder("Failed to execute [" + sql + "] with parameters ["); if (params != null) { boolean isFirst = true; for (Object param : params) { errorBuilder.append((isFirst ? "" : ", ") + param); isFirst = false; } } LOG.warn(errorBuilder.toString() + "]", ex); // We just logged an exception with (in case of JDO) a humongous callstack. Make a new one. throw new MetaException("See previous errors; " + ex.getMessage()); } } @SuppressWarnings("unchecked") static List<Object[]> ensureList(Object result) throws MetaException { if (!(result instanceof List<?>)) { throw new MetaException("Wrong result type " + result.getClass()); } return (List<Object[]>) result; } static Long extractSqlLong(Object obj) throws MetaException { if (obj == null) return null; if (!(obj instanceof Number)) { throw new MetaException("Expected numeric type but got " + obj.getClass().getName()); } return ((Number) obj).longValue(); } static void timingTrace(boolean doTrace, String queryText, long start, long queryTime) { if (!doTrace) return; LOG.debug("Direct SQL query in " + (queryTime - start) / 1000000.0 + "ms + " + (System.nanoTime() - queryTime) / 1000000.0 + "ms, the query is [" + queryText + "]"); } static <T> int loopJoinOrderedResult(PersistenceManager pm, TreeMap<Long, T> tree, String queryText, int keyIndex, ApplyFunc<T> func) throws MetaException { return loopJoinOrderedResult(pm, tree, queryText, null, keyIndex, func); } /** * Merges applies the result of a PM SQL query into a tree of object. * Essentially it's an object join. DN could do this for us, but it issues queries * separately for every object, which is suboptimal. * @param pm * @param tree The object tree, by ID. * @param queryText The query text. * @param keyIndex Index of the Long column corresponding to the map ID in query result rows. * @param func The function that is called on each (object,row) pair with the same id. * @return the count of results returned from the query. */ static <T> int loopJoinOrderedResult(PersistenceManager pm, TreeMap<Long, T> tree, String queryText, Object[] parameters, int keyIndex, ApplyFunc<T> func) throws MetaException { boolean doTrace = LOG.isDebugEnabled(); long start = doTrace ? System.nanoTime() : 0; Query query = pm.newQuery("javax.jdo.query.SQL", queryText); Object result = null; if (parameters == null || parameters.length == 0) { result = query.execute(); } else { result = query.executeWithArray(parameters); } long queryTime = doTrace ? System.nanoTime() : 0; if (result == null) { query.closeAll(); return 0; } List<Object[]> list = ensureList(result); Iterator<Object[]> iter = list.iterator(); Object[] fields = null; for (Map.Entry<Long, T> entry : tree.entrySet()) { if (fields == null && !iter.hasNext()) break; long id = entry.getKey(); while (fields != null || iter.hasNext()) { if (fields == null) { fields = iter.next(); } long nestedId = extractSqlLong(fields[keyIndex]); if (nestedId < id) throw new MetaException("Found entries for unknown ID " + nestedId); if (nestedId > id) break; // fields belong to one of the next entries func.apply(entry.getValue(), fields); fields = null; } Deadline.checkTimeout(); } int rv = list.size(); query.closeAll(); timingTrace(doTrace, queryText, start, queryTime); return rv; } static void setPartitionParameters(String PARTITION_PARAMS, boolean convertMapNullsToEmptyStrings, PersistenceManager pm, String partIds, TreeMap<Long, Partition> partitions) throws MetaException { String queryText; queryText = "select \"PART_ID\", \"PARAM_KEY\", \"PARAM_VALUE\" from " + PARTITION_PARAMS + "" + " where \"PART_ID\" in (" + partIds + ") and \"PARAM_KEY\" is not null" + " order by \"PART_ID\" asc"; loopJoinOrderedResult(pm, partitions, queryText, 0, new ApplyFunc<Partition>() { @Override public void apply(Partition t, Object[] fields) { t.putToParameters((String) fields[1], extractSqlClob(fields[2])); } }); // Perform conversion of null map values for (Partition t : partitions.values()) { t.setParameters(MetaStoreServerUtils.trimMapNulls(t.getParameters(), convertMapNullsToEmptyStrings)); } } static void setPartitionParametersWithFilter(String PARTITION_PARAMS, boolean convertMapNullsToEmptyStrings, PersistenceManager pm, String partIds, TreeMap<Long, Partition> partitions, String includeParamKeyPattern, String excludeParamKeyPattern) throws MetaException { StringBuilder queryTextBuilder = new StringBuilder( "select \"PART_ID\", \"PARAM_KEY\", \"PARAM_VALUE\" from ").append(PARTITION_PARAMS) .append(" where \"PART_ID\" in (").append(partIds) .append(") and \"PARAM_KEY\" is not null"); List<Object> queryParams = new ArrayList<>(2); ; if (includeParamKeyPattern != null && !includeParamKeyPattern.isEmpty()) { queryTextBuilder.append(" and \"PARAM_KEY\" LIKE (?)"); queryParams.add(includeParamKeyPattern); } if (excludeParamKeyPattern != null && !excludeParamKeyPattern.isEmpty()) { queryTextBuilder.append(" and \"PARAM_KEY\" NOT LIKE (?)"); queryParams.add(excludeParamKeyPattern); } queryTextBuilder.append(" order by \"PART_ID\" asc"); String queryText = queryTextBuilder.toString(); loopJoinOrderedResult(pm, partitions, queryText, queryParams.toArray(), 0, new ApplyFunc<Partition>() { @Override public void apply(Partition t, Object[] fields) { t.putToParameters((String) fields[1], extractSqlClob(fields[2])); } }); // Perform conversion of null map values for (Partition t : partitions.values()) { t.setParameters(MetaStoreServerUtils.trimMapNulls(t.getParameters(), convertMapNullsToEmptyStrings)); } } static void setPartitionValues(String PARTITION_KEY_VALS, PersistenceManager pm, String partIds, TreeMap<Long, Partition> partitions) throws MetaException { String queryText; queryText = "select \"PART_ID\", \"PART_KEY_VAL\" from " + PARTITION_KEY_VALS + "" + " where \"PART_ID\" in (" + partIds + ")" + " order by \"PART_ID\" asc, \"INTEGER_IDX\" asc"; loopJoinOrderedResult(pm, partitions, queryText, 0, new ApplyFunc<Partition>() { @Override public void apply(Partition t, Object[] fields) { t.addToValues((String) fields[1]); } }); } static String extractSqlClob(Object value) { if (value == null) return null; try { if (value instanceof Clob) { // we trim the Clob value to a max length an int can hold int maxLength = (((Clob) value).length() < Integer.MAX_VALUE - 2) ? (int) ((Clob) value).length() : Integer.MAX_VALUE - 2; return ((Clob) value).getSubString(1L, maxLength); } else { return value.toString(); } } catch (SQLException sqle) { return null; } } static void setSDParameters(String SD_PARAMS, boolean convertMapNullsToEmptyStrings, PersistenceManager pm, TreeMap<Long, StorageDescriptor> sds, String sdIds) throws MetaException { String queryText; queryText = "select \"SD_ID\", \"PARAM_KEY\", \"PARAM_VALUE\" from " + SD_PARAMS + "" + " where \"SD_ID\" in (" + sdIds + ") and \"PARAM_KEY\" is not null" + " order by \"SD_ID\" asc"; loopJoinOrderedResult(pm, sds, queryText, 0, new ApplyFunc<StorageDescriptor>() { @Override public void apply(StorageDescriptor t, Object[] fields) { t.putToParameters((String) fields[1], extractSqlClob(fields[2])); } }); // Perform conversion of null map values for (StorageDescriptor t : sds.values()) { t.setParameters(MetaStoreServerUtils.trimMapNulls(t.getParameters(), convertMapNullsToEmptyStrings)); } } static int extractSqlInt(Object field) { return ((Number) field).intValue(); } static void setSDSortCols(String SORT_COLS, List<String> columnNames, PersistenceManager pm, TreeMap<Long, StorageDescriptor> sds, String sdIds) throws MetaException { StringBuilder queryTextBuilder = new StringBuilder("select \"SD_ID\""); int counter = 0; if (columnNames.contains("col")) { counter++; queryTextBuilder.append(", \"COLUMN_NAME\""); } if (columnNames.contains("order")) { counter++; queryTextBuilder.append(", \"ORDER\""); } queryTextBuilder.append(" from ").append(SORT_COLS).append(" where \"SD_ID\" in (").append(sdIds) .append(") order by \"SD_ID\" asc, \"INTEGER_IDX\" asc"); String queryText = queryTextBuilder.toString(); final int finalCounter = counter; loopJoinOrderedResult(pm, sds, queryText, 0, new ApplyFunc<StorageDescriptor>() { @Override public void apply(StorageDescriptor t, Object[] fields) { if (finalCounter > 1 && fields[2] == null) { return; } Order order = new Order(); if (finalCounter > 0) { order.setCol((String) fields[1]); } if (finalCounter > 1) { order.setOrder(extractSqlInt(fields[2])); } t.addToSortCols(order); } }); } static void setSDSortCols(String SORT_COLS, PersistenceManager pm, TreeMap<Long, StorageDescriptor> sds, String sdIds) throws MetaException { String queryText; queryText = "select \"SD_ID\", \"COLUMN_NAME\", " + SORT_COLS + ".\"ORDER\"" + " from " + SORT_COLS + "" + " where \"SD_ID\" in (" + sdIds + ")" + " order by \"SD_ID\" asc, \"INTEGER_IDX\" asc"; loopJoinOrderedResult(pm, sds, queryText, 0, new ApplyFunc<StorageDescriptor>() { @Override public void apply(StorageDescriptor t, Object[] fields) { if (fields[2] == null) return; t.addToSortCols(new Order((String) fields[1], extractSqlInt(fields[2]))); } }); } static void setSDBucketCols(String BUCKETING_COLS, PersistenceManager pm, TreeMap<Long, StorageDescriptor> sds, String sdIds) throws MetaException { String queryText; queryText = "select \"SD_ID\", \"BUCKET_COL_NAME\" from " + BUCKETING_COLS + "" + " where \"SD_ID\" in (" + sdIds + ")" + " order by \"SD_ID\" asc, \"INTEGER_IDX\" asc"; loopJoinOrderedResult(pm, sds, queryText, 0, new ApplyFunc<StorageDescriptor>() { @Override public void apply(StorageDescriptor t, Object[] fields) { t.addToBucketCols((String) fields[1]); } }); } static boolean setSkewedColNames(String SKEWED_COL_NAMES, PersistenceManager pm, TreeMap<Long, StorageDescriptor> sds, String sdIds) throws MetaException { String queryText; queryText = "select \"SD_ID\", \"SKEWED_COL_NAME\" from " + SKEWED_COL_NAMES + "" + " where \"SD_ID\" in (" + sdIds + ")" + " order by \"SD_ID\" asc, \"INTEGER_IDX\" asc"; return loopJoinOrderedResult(pm, sds, queryText, 0, new ApplyFunc<StorageDescriptor>() { @Override public void apply(StorageDescriptor t, Object[] fields) { if (!t.isSetSkewedInfo()) t.setSkewedInfo(new SkewedInfo()); t.getSkewedInfo().addToSkewedColNames((String) fields[1]); } }) > 0; } static void setSkewedColValues(String SKEWED_STRING_LIST_VALUES, String SKEWED_VALUES, PersistenceManager pm, TreeMap<Long, StorageDescriptor> sds, String sdIds) throws MetaException { String queryText; queryText = "select " + SKEWED_VALUES + ".\"SD_ID_OID\"," + " " + SKEWED_STRING_LIST_VALUES + ".\"STRING_LIST_ID\"," + " " + SKEWED_STRING_LIST_VALUES + ".\"STRING_LIST_VALUE\" " + "from " + SKEWED_VALUES + " " + " left outer join " + SKEWED_STRING_LIST_VALUES + " on " + SKEWED_VALUES + "." + "\"STRING_LIST_ID_EID\" = " + SKEWED_STRING_LIST_VALUES + ".\"STRING_LIST_ID\" " + "where " + SKEWED_VALUES + ".\"SD_ID_OID\" in (" + sdIds + ") " + " and " + SKEWED_VALUES + ".\"STRING_LIST_ID_EID\" is not null " + " and " + SKEWED_VALUES + ".\"INTEGER_IDX\" >= 0 " + "order by " + SKEWED_VALUES + ".\"SD_ID_OID\" asc, " + SKEWED_VALUES + ".\"INTEGER_IDX\" asc," + " " + SKEWED_STRING_LIST_VALUES + ".\"INTEGER_IDX\" asc"; loopJoinOrderedResult(pm, sds, queryText, 0, new ApplyFunc<StorageDescriptor>() { private Long currentListId; private List<String> currentList; @Override public void apply(StorageDescriptor t, Object[] fields) throws MetaException { if (!t.isSetSkewedInfo()) t.setSkewedInfo(new SkewedInfo()); // Note that this is not a typical list accumulator - there's no call to finalize // the last list. Instead we add list to SD first, as well as locally to add elements. if (fields[1] == null) { currentList = null; // left outer join produced a list with no values currentListId = null; t.getSkewedInfo().addToSkewedColValues(Collections.<String>emptyList()); } else { long fieldsListId = extractSqlLong(fields[1]); if (currentListId == null || fieldsListId != currentListId) { currentList = new ArrayList<String>(); currentListId = fieldsListId; t.getSkewedInfo().addToSkewedColValues(currentList); } currentList.add((String) fields[2]); } } }); } static void setSkewedColLocationMaps(String SKEWED_COL_VALUE_LOC_MAP, String SKEWED_STRING_LIST_VALUES, PersistenceManager pm, TreeMap<Long, StorageDescriptor> sds, String sdIds) throws MetaException { String queryText; queryText = "select " + SKEWED_COL_VALUE_LOC_MAP + ".\"SD_ID\"," + " " + SKEWED_STRING_LIST_VALUES + ".STRING_LIST_ID," + " " + SKEWED_COL_VALUE_LOC_MAP + ".\"LOCATION\"," + " " + SKEWED_STRING_LIST_VALUES + ".\"STRING_LIST_VALUE\" " + "from " + SKEWED_COL_VALUE_LOC_MAP + "" + " left outer join " + SKEWED_STRING_LIST_VALUES + " on " + SKEWED_COL_VALUE_LOC_MAP + "." + "\"STRING_LIST_ID_KID\" = " + SKEWED_STRING_LIST_VALUES + ".\"STRING_LIST_ID\" " + "where " + SKEWED_COL_VALUE_LOC_MAP + ".\"SD_ID\" in (" + sdIds + ")" + " and " + SKEWED_COL_VALUE_LOC_MAP + ".\"STRING_LIST_ID_KID\" is not null " + "order by " + SKEWED_COL_VALUE_LOC_MAP + ".\"SD_ID\" asc," + " " + SKEWED_STRING_LIST_VALUES + ".\"STRING_LIST_ID\" asc," + " " + SKEWED_STRING_LIST_VALUES + ".\"INTEGER_IDX\" asc"; loopJoinOrderedResult(pm, sds, queryText, 0, new ApplyFunc<StorageDescriptor>() { private Long currentListId; private List<String> currentList; @Override public void apply(StorageDescriptor t, Object[] fields) throws MetaException { if (!t.isSetSkewedInfo()) { SkewedInfo skewedInfo = new SkewedInfo(); skewedInfo.setSkewedColValueLocationMaps(new HashMap<List<String>, String>()); t.setSkewedInfo(skewedInfo); } Map<List<String>, String> skewMap = t.getSkewedInfo().getSkewedColValueLocationMaps(); // Note that this is not a typical list accumulator - there's no call to finalize // the last list. Instead we add list to SD first, as well as locally to add elements. if (fields[1] == null) { currentList = new ArrayList<String>(); // left outer join produced a list with no values currentListId = null; } else { long fieldsListId = extractSqlLong(fields[1]); if (currentListId == null || fieldsListId != currentListId) { currentList = new ArrayList<String>(); currentListId = fieldsListId; } else { skewMap.remove(currentList); // value based compare.. remove first } currentList.add((String) fields[3]); } skewMap.put(currentList, (String) fields[2]); } }); } static void setSDCols(String COLUMNS_V2, List<String> columnNames, PersistenceManager pm, TreeMap<Long, List<FieldSchema>> colss, String colIds) throws MetaException { StringBuilder queryTextBuilder = new StringBuilder("select \"CD_ID\""); int counter = 0; if (columnNames.contains("name")) { counter++; queryTextBuilder.append(", \"COLUMN_NAME\""); } if (columnNames.contains("type")) { counter++; queryTextBuilder.append(", \"TYPE_NAME\""); } if (columnNames.contains("comment")) { counter++; queryTextBuilder.append(", \"COMMENT\""); } queryTextBuilder.append(" from ").append(COLUMNS_V2).append(" where \"CD_ID\" in (").append(colIds) .append(") order by \"CD_ID\" asc, \"INTEGER_IDX\" asc"); String queryText = queryTextBuilder.toString(); int finalCounter = counter; loopJoinOrderedResult(pm, colss, queryText, 0, new ApplyFunc<List<FieldSchema>>() { @Override public void apply(List<FieldSchema> t, Object[] fields) { FieldSchema fieldSchema = new FieldSchema(); if (finalCounter > 0) { fieldSchema.setName((String) fields[1]); } if (finalCounter > 1) { fieldSchema.setType(extractSqlClob(fields[2])); } if (finalCounter > 2) { fieldSchema.setComment((String) fields[3]); } t.add(fieldSchema); } }); } static void setSDCols(String COLUMNS_V2, PersistenceManager pm, TreeMap<Long, List<FieldSchema>> colss, String colIds) throws MetaException { String queryText; queryText = "select \"CD_ID\", \"COMMENT\", \"COLUMN_NAME\", \"TYPE_NAME\"" + " from " + COLUMNS_V2 + " where \"CD_ID\" in (" + colIds + ")" + " order by \"CD_ID\" asc, \"INTEGER_IDX\" asc"; loopJoinOrderedResult(pm, colss, queryText, 0, new ApplyFunc<List<FieldSchema>>() { @Override public void apply(List<FieldSchema> t, Object[] fields) { t.add(new FieldSchema((String) fields[2], extractSqlClob(fields[3]), (String) fields[1])); } }); } static void setSerdeParams(String SERDE_PARAMS, boolean convertMapNullsToEmptyStrings, PersistenceManager pm, TreeMap<Long, SerDeInfo> serdes, String serdeIds) throws MetaException { String queryText; queryText = "select \"SERDE_ID\", \"PARAM_KEY\", \"PARAM_VALUE\" from " + SERDE_PARAMS + "" + " where \"SERDE_ID\" in (" + serdeIds + ") and \"PARAM_KEY\" is not null" + " order by \"SERDE_ID\" asc"; loopJoinOrderedResult(pm, serdes, queryText, 0, new ApplyFunc<SerDeInfo>() { @Override public void apply(SerDeInfo t, Object[] fields) { t.putToParameters((String) fields[1], extractSqlClob(fields[2])); } }); // Perform conversion of null map values for (SerDeInfo t : serdes.values()) { t.setParameters(MetaStoreServerUtils.trimMapNulls(t.getParameters(), convertMapNullsToEmptyStrings)); } } /** * Convert a boolean value returned from the RDBMS to a Java Boolean object. * MySQL has booleans, but e.g. Derby uses 'Y'/'N' mapping. * * @param value * column value from the database * @return The Boolean value of the database column value, null if the column * value is null * @throws MetaException * if the column value cannot be converted into a Boolean object */ static Boolean extractSqlBoolean(Object value) throws MetaException { if (value == null) { return null; } if (value instanceof Boolean) { return (Boolean) value; } if (value instanceof String) { try { return BooleanUtils.toBooleanObject((String) value, "Y", "N", null); } catch (IllegalArgumentException iae) { // NOOP } } throw new MetaException("Cannot extract boolean from column value " + value); } static String extractSqlString(Object value) { if (value == null) return null; return value.toString(); } static Double extractSqlDouble(Object obj) throws MetaException { if (obj == null) return null; if (!(obj instanceof Number)) { throw new MetaException("Expected numeric type but got " + obj.getClass().getName()); } return ((Number) obj).doubleValue(); } static byte[] extractSqlBlob(Object value) throws MetaException { if (value == null) return null; if (value instanceof Blob) { //derby, oracle try { // getBytes function says: pos the ordinal position of the first byte in // the BLOB value to be extracted; the first byte is at position 1 return ((Blob) value).getBytes(1, (int) ((Blob) value).length()); } catch (SQLException e) { throw new MetaException("Encounter error while processing blob."); } } else if (value instanceof byte[]) { // mysql, postgres, sql server return (byte[]) value; } else { // this may happen when enablebitvector is false LOG.debug("Expected blob type but got " + value.getClass().getName()); return null; } } @FunctionalInterface static interface ApplyFunc<Target> { void apply(Target t, Object[] fields) throws MetaException; } }