Java tutorial
/** * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. * The ASF licenses this file to You under the Apache License, Version 2.0 * (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.blur.hive; import static org.junit.Assert.assertEquals; import java.io.File; import java.io.FileNotFoundException; import java.io.FileOutputStream; import java.io.IOException; import java.io.PrintWriter; import java.lang.reflect.Field; import java.net.ServerSocket; import java.sql.Connection; import java.sql.Date; import java.sql.DriverManager; import java.sql.ResultSet; import java.sql.ResultSetMetaData; import java.sql.SQLException; import java.sql.Statement; import java.text.SimpleDateFormat; import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Properties; import org.apache.blur.MiniCluster; import org.apache.blur.mapreduce.lib.BlurColumn; import org.apache.blur.mapreduce.lib.BlurRecord; import org.apache.blur.thirdparty.thrift_0_9_0.TException; import org.apache.blur.thrift.BlurClient; import org.apache.blur.thrift.generated.Blur.Iface; import org.apache.blur.thrift.generated.BlurException; import org.apache.blur.thrift.generated.BlurQuery; import org.apache.blur.thrift.generated.BlurResults; import org.apache.blur.thrift.generated.ColumnDefinition; import org.apache.blur.thrift.generated.Query; import org.apache.blur.thrift.generated.TableDescriptor; import org.apache.blur.utils.BlurConstants; import org.apache.blur.utils.GCWatcher; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.LocalFileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.permission.FsAction; import org.apache.hadoop.fs.permission.FsPermission; import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.serde2.SerDeException; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; import org.apache.hadoop.security.UserGroupInformation; import org.apache.hive.jdbc.HiveDriver; import org.apache.hive.service.server.HiveServer2; import org.junit.After; import org.junit.AfterClass; import org.junit.Before; import org.junit.BeforeClass; import org.junit.Test; import com.google.common.base.Splitter; public class BlurSerDeTest { public static final File WAREHOUSE = new File("./target/tmp/warehouse"); public static final String COLUMN_SEP = new String(new char[] { 1 }); public static final String ITEM_SEP = new String(new char[] { 2 }); public static final File DERBY_FILE = new File("derby.log"); public static final File METASTORE_DB_FILE = new File("metastore_db"); private static final String FAM = "fam0"; private static final String YYYYMMDD = "yyyyMMdd"; private static final String YYYY_MM_DD = "yyyy-MM-dd"; private static final String TEST = "test"; private static final File TMPDIR = new File(System.getProperty("blur.tmp.dir", "./target/tmp_BlurSerDeTest")); private static MiniCluster miniCluster; private static boolean externalProcesses = false; @BeforeClass public static void startCluster() throws IOException { System.setProperty("hadoop.log.dir", "./target/tmp_BlurSerDeTest_hadoop_log"); GCWatcher.init(0.60); LocalFileSystem localFS = FileSystem.getLocal(new Configuration()); File testDirectory = new File(TMPDIR, "blur-SerDe-test").getAbsoluteFile(); testDirectory.mkdirs(); Path directory = new Path(testDirectory.getPath()); FsPermission dirPermissions = localFS.getFileStatus(directory).getPermission(); FsAction userAction = dirPermissions.getUserAction(); FsAction groupAction = dirPermissions.getGroupAction(); FsAction otherAction = dirPermissions.getOtherAction(); StringBuilder builder = new StringBuilder(); builder.append(userAction.ordinal()); builder.append(groupAction.ordinal()); builder.append(otherAction.ordinal()); String dirPermissionNum = builder.toString(); System.setProperty("dfs.datanode.data.dir.perm", dirPermissionNum); testDirectory.delete(); miniCluster = new MiniCluster(); miniCluster.startBlurCluster(new File(testDirectory, "cluster").getAbsolutePath(), 2, 3, true, externalProcesses); miniCluster.startMrMiniCluster(); } @AfterClass public static void shutdownCluster() throws IOException { miniCluster.stopMrMiniCluster(); miniCluster.shutdownBlurCluster(); } private String _mrWorkingPath; @Before public void setup() throws BlurException, TException, IOException { _mrWorkingPath = miniCluster.getFileSystemUri().toString() + "/mrworkingpath"; String controllerConnectionStr = miniCluster.getControllerConnectionStr(); Iface client = BlurClient.getClient(controllerConnectionStr); List<String> tableList = client.tableList(); if (!tableList.contains(TEST)) { TableDescriptor tableDescriptor = new TableDescriptor(); tableDescriptor.setName(TEST); tableDescriptor.setShardCount(1); tableDescriptor.setTableUri(miniCluster.getFileSystemUri().toString() + "/blur/tables/test"); tableDescriptor.putToTableProperties(BlurConstants.BLUR_BULK_UPDATE_WORKING_PATH, _mrWorkingPath); client.createTable(tableDescriptor); Map<String, String> props = new HashMap<String, String>(); props.put("dateFormat", YYYYMMDD); client.addColumnDefinition(TEST, cd(false, FAM, "string-col-single", "string")); client.addColumnDefinition(TEST, cd(false, FAM, "text-col-single", "text")); client.addColumnDefinition(TEST, cd(false, FAM, "stored-col-single", "stored")); client.addColumnDefinition(TEST, cd(false, FAM, "double-col-single", "double")); client.addColumnDefinition(TEST, cd(false, FAM, "float-col-single", "float")); client.addColumnDefinition(TEST, cd(false, FAM, "long-col-single", "long")); client.addColumnDefinition(TEST, cd(false, FAM, "int-col-single", "int")); client.addColumnDefinition(TEST, cd(false, FAM, "date-col-single", "date", props)); client.addColumnDefinition(TEST, cd(false, FAM, "geo-col-single", "geo-pointvector")); client.addColumnDefinition(TEST, cd(true, FAM, "string-col-multi", "string")); client.addColumnDefinition(TEST, cd(true, FAM, "text-col-multi", "text")); client.addColumnDefinition(TEST, cd(true, FAM, "stored-col-multi", "stored")); client.addColumnDefinition(TEST, cd(true, FAM, "double-col-multi", "double")); client.addColumnDefinition(TEST, cd(true, FAM, "float-col-multi", "float")); client.addColumnDefinition(TEST, cd(true, FAM, "long-col-multi", "long")); client.addColumnDefinition(TEST, cd(true, FAM, "int-col-multi", "int")); client.addColumnDefinition(TEST, cd(true, FAM, "date-col-multi", "date", props)); } rmr(WAREHOUSE); rmr(METASTORE_DB_FILE); rmr(DERBY_FILE); } @After public void teardown() { rmr(WAREHOUSE); rmr(METASTORE_DB_FILE); rmr(DERBY_FILE); } public static void rmr(File file) { if (!file.exists()) { return; } if (file.isDirectory()) { for (File f : file.listFiles()) { rmr(f); } } file.delete(); } private ColumnDefinition cd(boolean multiValue, String family, String columnName, String type) { return cd(multiValue, family, columnName, type, null); } private ColumnDefinition cd(boolean multiValue, String family, String columnName, String type, Map<String, String> props) { ColumnDefinition columnDefinition = new ColumnDefinition(family, columnName, null, false, type, props, false); columnDefinition.setMultiValueField(multiValue); return columnDefinition; } @Test public void test1() throws SerDeException { long now = System.currentTimeMillis(); Date date = new Date(now); SimpleDateFormat simpleDateFormat = new SimpleDateFormat(YYYYMMDD); BlurSerDe blurSerDe = new BlurSerDe(); Configuration conf = new Configuration(); Properties tbl = new Properties(); tbl.put(BlurSerDe.TABLE, TEST); tbl.put(BlurSerDe.FAMILY, FAM); tbl.put(BlurSerDe.ZK, miniCluster.getZkConnectionString()); blurSerDe.initialize(conf, tbl); ObjectInspector objectInspector = blurSerDe.getObjectInspector(); Object[] row = new Object[19]; int c = 0; row[c++] = "rowid"; row[c++] = "recordid"; row[c++] = new Object[] { date, date }; row[c++] = date; row[c++] = new Object[] { 1234.5678, 4321.5678 }; row[c++] = 1234.5678; row[c++] = new Object[] { 1234.567f, 4321.567f }; row[c++] = 1234.567f; row[c++] = new Object[] { 1.0f, 2.0f }; row[c++] = new Object[] { 12345678, 87654321 }; row[c++] = 12345678; row[c++] = new Object[] { 12345678l, 87654321l }; row[c++] = 12345678l; row[c++] = new Object[] { "stored input1", "stored input2" }; row[c++] = "stored input"; row[c++] = new Object[] { "string input1", "string input2" }; row[c++] = "string input"; row[c++] = new Object[] { "text input1", "text input2" }; row[c++] = "text input"; BlurRecord blurRecord = (BlurRecord) blurSerDe.serialize(row, objectInspector); assertEquals("rowid", blurRecord.getRowId()); assertEquals("recordid", blurRecord.getRecordId()); Map<String, List<String>> columns = toMap(blurRecord.getColumns()); assertEquals(list("string input"), columns.get("string-col-single")); assertEquals(list("string input1", "string input2"), columns.get("string-col-multi")); assertEquals(list("text input"), columns.get("text-col-single")); assertEquals(list("text input1", "text input2"), columns.get("text-col-multi")); assertEquals(list("stored input"), columns.get("stored-col-single")); assertEquals(list("stored input1", "stored input2"), columns.get("stored-col-multi")); assertEquals(list("1234.5678"), columns.get("double-col-single")); assertEquals(list("1234.5678", "4321.5678"), columns.get("double-col-multi")); assertEquals(list("1234.567"), columns.get("float-col-single")); assertEquals(list("1234.567", "4321.567"), columns.get("float-col-multi")); assertEquals(list("12345678"), columns.get("long-col-single")); assertEquals(list("12345678", "87654321"), columns.get("long-col-multi")); assertEquals(list("12345678"), columns.get("int-col-single")); assertEquals(list("12345678", "87654321"), columns.get("int-col-multi")); assertEquals(list(simpleDateFormat.format(date)), columns.get("date-col-single")); assertEquals(list(simpleDateFormat.format(date), simpleDateFormat.format(date)), columns.get("date-col-multi")); assertEquals(list("1.0,2.0"), columns.get("geo-col-single")); } @Test public void test2() throws SQLException, ClassNotFoundException, IOException, BlurException, TException, InterruptedException { int totalRecords = runLoad(true); Iface client = BlurClient.getClientFromZooKeeperConnectionStr(miniCluster.getZkConnectionString()); BlurQuery blurQuery = new BlurQuery(); Query query = new Query(); query.setQuery("*"); blurQuery.setQuery(query); BlurResults results = client.query(TEST, blurQuery); assertEquals(totalRecords, results.getTotalResults()); } @Test public void test3() throws Exception { int totalRecords = runLoad(false); Iface client = BlurClient.getClientFromZooKeeperConnectionStr(miniCluster.getZkConnectionString()); BlurQuery blurQuery = new BlurQuery(); Query query = new Query(); query.setQuery("*"); blurQuery.setQuery(query); BlurResults results = client.query(TEST, blurQuery); assertEquals(totalRecords, results.getTotalResults()); } private int runLoad(boolean disableMrUpdate) throws IOException, InterruptedException, ClassNotFoundException, SQLException { Configuration configuration = miniCluster.getMRConfiguration(); writeSiteFiles(configuration); HiveConf hiveConf = new HiveConf(configuration, getClass()); hiveConf.set("hive.server2.thrift.port", "0"); HiveServer2 hiveServer2 = new HiveServer2(); hiveServer2.init(hiveConf); hiveServer2.start(); int port = waitForStartupAndGetPort(hiveServer2); Class.forName(HiveDriver.class.getName()); String userName = UserGroupInformation.getCurrentUser().getShortUserName(); Connection connection = DriverManager.getConnection("jdbc:hive2://localhost:" + port, userName, ""); UserGroupInformation currentUser = UserGroupInformation.getCurrentUser(); run(connection, "set blur.user.name=" + currentUser.getUserName()); run(connection, "set blur.mr.update.disabled=" + disableMrUpdate); run(connection, "set hive.metastore.warehouse.dir=" + WAREHOUSE.toURI().toString()); run(connection, "create database if not exists testdb"); run(connection, "use testdb"); run(connection, "CREATE TABLE if not exists testtable ROW FORMAT SERDE 'org.apache.blur.hive.BlurSerDe' " + "WITH SERDEPROPERTIES ( 'blur.zookeeper.connection'='" + miniCluster.getZkConnectionString() + "', " + "'blur.table'='" + TEST + "', 'blur.family'='" + FAM + "' ) " + "STORED BY 'org.apache.blur.hive.BlurHiveStorageHandler'"); run(connection, "desc testtable"); String createLoadTable = buildCreateLoadTable(connection); run(connection, createLoadTable); File dbDir = new File(WAREHOUSE, "testdb.db"); File tableDir = new File(dbDir, "loadtable"); int totalRecords = 100; generateData(tableDir, totalRecords); run(connection, "select * from loadtable"); run(connection, "set " + BlurSerDe.BLUR_BLOCKING_APPLY + "=true"); run(connection, "insert into table testtable select * from loadtable"); connection.close(); hiveServer2.stop(); return totalRecords; } private void writeSiteFiles(Configuration configuration) throws FileNotFoundException, IOException { String name = BlurHiveMRLoaderOutputCommitter.MAPRED_SITE_XML; if (miniCluster.useYarn()) { name = BlurHiveMRLoaderOutputCommitter.YARN_SITE_XML; } String classPath = System.getProperty("java.class.path"); for (String path : Splitter.on(":").split(classPath)) { File file = new File(path); if (file.getName().equals("test-classes")) { writeFile(new File(file, name), configuration); return; } } } private void writeFile(File file, Configuration configuration) throws FileNotFoundException, IOException { FileOutputStream outputStream = new FileOutputStream(file); configuration.writeXml(outputStream); outputStream.close(); } private void generateData(File file, int totalRecords) throws IOException { SimpleDateFormat simpleDateFormat = new SimpleDateFormat(YYYY_MM_DD); file.mkdirs(); PrintWriter print = new PrintWriter(new File(file, "data")); Date date = new Date(System.currentTimeMillis()); for (int i = 0; i < totalRecords; i++) { // rowid print.print("rowid" + i); print.print(COLUMN_SEP); // recordid print.print("recordid" + i); print.print(COLUMN_SEP); { // date_col_multi print.print(simpleDateFormat.format(date)); print.print(ITEM_SEP); print.print(simpleDateFormat.format(date)); } print.print(COLUMN_SEP); // date_col_single print.print(simpleDateFormat.format(date)); print.print(COLUMN_SEP); { // double_col_multi print.print("1.0"); print.print(ITEM_SEP); print.print("2.0"); } print.print(COLUMN_SEP); // double_col_single print.print("3.0"); print.print(COLUMN_SEP); { // float_col_multi print.print("4.0"); print.print(ITEM_SEP); print.print("5.0"); } print.print(COLUMN_SEP); // float_col_single print.print("6.0"); print.print(COLUMN_SEP); // geo_col_single print.print("10.0"); print.print(ITEM_SEP); print.print("10.0"); print.print(COLUMN_SEP); { // int_col_multi print.print("1"); print.print(ITEM_SEP); print.print("2"); } print.print(COLUMN_SEP); // int_col_single print.print("3"); print.print(COLUMN_SEP); { // long_col_multi print.print("4"); print.print(ITEM_SEP); print.print("5"); } print.print(COLUMN_SEP); // long_col_single print.print("6"); print.print(COLUMN_SEP); { // stored_col_multi print.print("stored_1"); print.print(ITEM_SEP); print.print("stored_2"); } print.print(COLUMN_SEP); // stored_col_single print.print("stored_3"); print.print(COLUMN_SEP); { // string_col_multi print.print("string_1"); print.print(ITEM_SEP); print.print("string_2"); } print.print(COLUMN_SEP); // string_col_single print.print("string_3"); print.print(COLUMN_SEP); { // text_col_multi print.print("text_1"); print.print(ITEM_SEP); print.print("text_2"); } print.print(COLUMN_SEP); // text_col_single print.print("text_3"); print.println(); } print.close(); } private String buildCreateLoadTable(Connection connection) throws SQLException { StringBuilder builder = new StringBuilder("create TABLE if not exists loadtable ("); Statement statement = connection.createStatement(); if (statement.execute("desc testtable")) { ResultSet resultSet = statement.getResultSet(); boolean first = true; while (resultSet.next()) { if (!first) { builder.append(", "); } Object name = resultSet.getObject(1); Object type = resultSet.getObject(2); builder.append(name.toString()); builder.append(' '); builder.append(type.toString()); first = false; } builder.append(")"); return builder.toString(); } throw new RuntimeException("Can't build create table script."); } public static void run(Connection connection, String sql) throws SQLException { System.out.println("Running:" + sql); Statement statement = connection.createStatement(); if (statement.execute(sql)) { ResultSet resultSet = statement.getResultSet(); while (resultSet.next()) { ResultSetMetaData metaData = resultSet.getMetaData(); int columnCount = metaData.getColumnCount(); for (int i = 1; i <= columnCount; i++) { System.out.print(resultSet.getObject(i) + "\t"); } System.out.println(); } } statement.close(); } private List<String> list(String... sarray) { List<String> list = new ArrayList<String>(); for (String s : sarray) { list.add(s); } return list; } private Map<String, List<String>> toMap(List<BlurColumn> columns) { Map<String, List<String>> map = new HashMap<String, List<String>>(); for (BlurColumn blurColumn : columns) { String name = blurColumn.getName(); List<String> list = map.get(name); if (list == null) { map.put(name, list = new ArrayList<String>()); } list.add(blurColumn.getValue()); } return map; } @SuppressWarnings("resource") private int waitForStartupAndGetPort(HiveServer2 hiveServer2) throws InterruptedException { while (true) { // thriftCLIService->server->serverTransport_->serverSocket_ Thread.sleep(100); Object o1 = getObject(hiveServer2, "thriftCLIService"); if (o1 == null) { continue; } Object o2 = getObject(o1, "server"); if (o2 == null) { continue; } Object o3 = getObject(o2, "serverTransport_"); if (o3 == null) { continue; } Object o4 = getObject(o3, "serverSocket_"); if (o4 == null) { continue; } ServerSocket socket = (ServerSocket) o4; return socket.getLocalPort(); } } private Object getObject(Object o, String field) { return getObject(o, field, o.getClass()); } private Object getObject(Object o, String field, Class<? extends Object> clazz) { try { Field declaredField = clazz.getDeclaredField(field); return getObject(o, declaredField); } catch (NoSuchFieldException e) { return getObject(o, field, clazz.getSuperclass()); } catch (SecurityException e) { throw new RuntimeException(e); } } private Object getObject(Object o, Field field) { field.setAccessible(true); try { return field.get(o); } catch (Exception e) { throw new RuntimeException(e); } } }