devbox
weixin_40725706
这个屌丝很懒，什么也没留下！
关注作者
热门标签
热门文章
当前位置:   article > 正文

HiveMetaStoreClient用法示例

作者：weixin_40725706 | 2024-04-25 07:51:17

赞

踩

hivemetastoreclient
  1. /**
  2.  * Licensed to the Apache Software Foundation (ASF) under one
  3.  * or more contributor license agreements.  See the NOTICE file
  4.  * distributed with this work for additional information
  5.  * regarding copyright ownership.  The ASF licenses this file
  6.  * to you under the Apache License, Version 2.0 (the
  7.  * "License"); you may not use this file except in compliance
  8.  * with the License.  You may obtain a copy of the License at
  9.  *
  10.  *     http://www.apache.org/licenses/LICENSE-2.0
  11.  *
  12.  * Unless required by applicable law or agreed to in writing, software
  13.  * distributed under the License is distributed on an "AS IS" BASIS,
  14.  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  15.  * See the License for the specific language governing permissions and
  16.  * limitations under the License.
  17.  */
  18.  
  19. package org.apache.hadoop.hive.metastore;
  20.  
  21. import java.util.ArrayList;
  22. import java.util.HashMap;
  23. import java.util.HashSet;
  24. import java.util.LinkedHashMap;
  25. import java.util.List;
  26. import java.util.Map;
  27. import java.util.Set;
  28.  
  29. import junit.framework.TestCase;
  30.  
  31. import org.apache.hadoop.fs.FileSystem;
  32. import org.apache.hadoop.fs.Path;
  33. import org.apache.hadoop.hive.conf.HiveConf;
  34. import org.apache.hadoop.hive.metastore.api.AlreadyExistsException;
  35. import org.apache.hadoop.hive.metastore.api.ConfigValSecurityException;
  36. import org.apache.hadoop.hive.metastore.api.Database;
  37. import org.apache.hadoop.hive.metastore.api.FieldSchema;
  38. import org.apache.hadoop.hive.metastore.api.InvalidObjectException;
  39. import org.apache.hadoop.hive.metastore.api.InvalidOperationException;
  40. import org.apache.hadoop.hive.metastore.api.MetaException;
  41. import org.apache.hadoop.hive.metastore.api.NoSuchObjectException;
  42. import org.apache.hadoop.hive.metastore.api.Order;
  43. import org.apache.hadoop.hive.metastore.api.Partition;
  44. import org.apache.hadoop.hive.metastore.api.SerDeInfo;
  45. import org.apache.hadoop.hive.metastore.api.StorageDescriptor;
  46. import org.apache.hadoop.hive.metastore.api.Table;
  47. import org.apache.hadoop.hive.metastore.api.Type;
  48. import org.apache.hadoop.hive.serde.Constants;
  49. import org.apache.hadoop.util.StringUtils;
  50. import org.apache.thrift.TException;
  51.  
  52. public abstract class TestHiveMetaStore extends TestCase {
  53.   protected static HiveMetaStoreClient client;
  54.   protected static HiveConf hiveConf;
  55.   protected static Warehouse warehouse;
  56.   protected static boolean isThriftClient = false;
  57.  
  58.   private static final String TEST_DB1_NAME = "testdb1";
  59.   private static final String TEST_DB2_NAME = "testdb2";
  60.  
  61.   @Override
  62.   protected void setUp() throws Exception {
  63.     hiveConf = new HiveConf(this.getClass());
  64.     warehouse = new Warehouse(hiveConf);
  65.  
  66.     // set some values to use for getting conf. vars
  67.     hiveConf.set("hive.metastore.metrics.enabled","true");
  68.     hiveConf.set("hive.key1", "value1");
  69.     hiveConf.set("hive.key2", "http://www.example.com");
  70.     hiveConf.set("hive.key3", "");
  71.     hiveConf.set("hive.key4", "0");
  72.   }
  73.  
  74.   public void testNameMethods() {
  75.     Map<String, String> spec = new LinkedHashMap<String, String>();
  76.     spec.put("ds", "2008-07-01 14:13:12");
  77.     spec.put("hr", "14");
  78.     List<String> vals = new ArrayList<String>();
  79.     for(String v : spec.values()) {
  80.       vals.add(v);
  81.     }
  82.     String partName = "ds=2008-07-01 14%3A13%3A12/hr=14";
  83.  
  84.     try {
  85.       List<String> testVals = client.partitionNameToVals(partName);
  86.       assertTrue("Values from name are incorrect", vals.equals(testVals));
  87.  
  88.       Map<String, String> testSpec = client.partitionNameToSpec(partName);
  89.       assertTrue("Spec from name is incorrect", spec.equals(testSpec));
  90.  
  91.       List<String> emptyVals = client.partitionNameToVals("");
  92.       assertTrue("Values should be empty", emptyVals.size() == 0);
  93.  
  94.       Map<String, String> emptySpec =  client.partitionNameToSpec("");
  95.       assertTrue("Spec should be empty", emptySpec.size() == 0);
  96.     } catch (Exception e) {
  97.       assert(false);
  98.     }
  99.   }
  100.  
  101.   /**
  102.    * tests create table and partition and tries to drop the table without
  103.    * droppping the partition
  104.    *
  105.    * @throws Exception
  106.    */
  107.   public void testPartition() throws Exception {
  108.     partitionTester(client, hiveConf);
  109.   }
  110.  
  111.   public static void partitionTester(HiveMetaStoreClient client, HiveConf hiveConf)
  112.     throws Exception {
  113.     try {
  114.       String dbName = "compdb";
  115.       String tblName = "comptbl";
  116.       String typeName = "Person";
  117.       List<String> vals = new ArrayList<String>(2);
  118.       vals.add("2008-07-01 14:13:12");
  119.       vals.add("14");
  120.       List <String> vals2 = new ArrayList<String>(2);
  121.       vals2.add("2008-07-01 14:13:12");
  122.       vals2.add("15");
  123.       List <String> vals3 = new ArrayList<String>(2);
  124.       vals3 = new ArrayList<String>(2);
  125.       vals3.add("2008-07-02 14:13:12");
  126.       vals3.add("15");
  127.       List <String> vals4 = new ArrayList<String>(2);
  128.       vals4 = new ArrayList<String>(2);
  129.       vals4.add("2008-07-03 14:13:12");
  130.       vals4.add("151");
  131.  
  132.       client.dropTable(dbName, tblName);
  133.       silentDropDatabase(dbName);
  134.       Database db = new Database();
  135.       db.setName(dbName);
  136.       client.createDatabase(db);
  137.  
  138.       client.dropType(typeName);
  139.       Type typ1 = new Type();
  140.       typ1.setName(typeName);
  141.       typ1.setFields(new ArrayList<FieldSchema>(2));
  142.       typ1.getFields().add(
  143.           new FieldSchema("name", Constants.STRING_TYPE_NAME, ""));
  144.       typ1.getFields().add(
  145.           new FieldSchema("income", Constants.INT_TYPE_NAME, ""));
  146.       client.createType(typ1);
  147.  
  148.       Table tbl = new Table();
  149.       tbl.setDbName(dbName);
  150.       tbl.setTableName(tblName);
  151.       StorageDescriptor sd = new StorageDescriptor();
  152.       tbl.setSd(sd);
  153.       sd.setCols(typ1.getFields());
  154.       sd.setCompressed(false);
  155.       sd.setNumBuckets(1);
  156.       sd.setParameters(new HashMap<String, String>());
  157.       sd.getParameters().put("test_param_1", "Use this for comments etc");
  158.       sd.setBucketCols(new ArrayList<String>(2));
  159.       sd.getBucketCols().add("name");
  160.       sd.setSerdeInfo(new SerDeInfo());
  161.       sd.getSerdeInfo().setName(tbl.getTableName());
  162.       sd.getSerdeInfo().setParameters(new HashMap<String, String>());
  163.       sd.getSerdeInfo().getParameters()
  164.           .put(Constants.SERIALIZATION_FORMAT, "1");
  165.       sd.setSortCols(new ArrayList<Order>());
  166.  
  167.       tbl.setPartitionKeys(new ArrayList<FieldSchema>(2));
  168.       tbl.getPartitionKeys().add(
  169.           new FieldSchema("ds", Constants.STRING_TYPE_NAME, ""));
  170.       tbl.getPartitionKeys().add(
  171.           new FieldSchema("hr", Constants.STRING_TYPE_NAME, ""));
  172.  
  173.       client.createTable(tbl);
  174.  
  175.       if (isThriftClient) {
  176.         // the createTable() above does not update the location in the 'tbl'
  177.         // object when the client is a thrift client and the code below relies
  178.         // on the location being present in the 'tbl' object - so get the table
  179.         // from the metastore
  180.         tbl = client.getTable(dbName, tblName);
  181.       }
  182.  
  183.       Partition part = new Partition();
  184.       part.setDbName(dbName);
  185.       part.setTableName(tblName);
  186.       part.setValues(vals);
  187.       part.setParameters(new HashMap<String, String>());
  188.       part.setSd(tbl.getSd());
  189.       part.getSd().setSerdeInfo(tbl.getSd().getSerdeInfo());
  190.       part.getSd().setLocation(tbl.getSd().getLocation() + "/part1");
  191.  
  192.       Partition part2 = new Partition();
  193.       part2.setDbName(dbName);
  194.       part2.setTableName(tblName);
  195.       part2.setValues(vals2);
  196.       part2.setParameters(new HashMap<String, String>());
  197.       part2.setSd(tbl.getSd());
  198.       part2.getSd().setSerdeInfo(tbl.getSd().getSerdeInfo());
  199.       part2.getSd().setLocation(tbl.getSd().getLocation() + "/part2");
  200.  
  201.       Partition part3 = new Partition();
  202.       part3.setDbName(dbName);
  203.       part3.setTableName(tblName);
  204.       part3.setValues(vals3);
  205.       part3.setParameters(new HashMap<String, String>());
  206.       part3.setSd(tbl.getSd());
  207.       part3.getSd().setSerdeInfo(tbl.getSd().getSerdeInfo());
  208.       part3.getSd().setLocation(tbl.getSd().getLocation() + "/part3");
  209.  
  210.       Partition part4 = new Partition();
  211.       part4.setDbName(dbName);
  212.       part4.setTableName(tblName);
  213.       part4.setValues(vals4);
  214.       part4.setParameters(new HashMap<String, String>());
  215.       part4.setSd(tbl.getSd());
  216.       part4.getSd().setSerdeInfo(tbl.getSd().getSerdeInfo());
  217.       part4.getSd().setLocation(tbl.getSd().getLocation() + "/part4");
  218.  
  219.       // check if the partition exists (it shouldn;t)
  220.       boolean exceptionThrown = false;
  221.       try {
  222.         Partition p = client.getPartition(dbName, tblName, vals);
  223.       } catch(Exception e) {
  224.         assertEquals("partition should not have existed",
  225.             NoSuchObjectException.class, e.getClass());
  226.         exceptionThrown = true;
  227.       }
  228.       assertTrue("getPartition() should have thrown NoSuchObjectException", exceptionThrown);
  229.       Partition retp = client.add_partition(part);
  230.       assertNotNull("Unable to create partition " + part, retp);
  231.       Partition retp2 = client.add_partition(part2);
  232.       assertNotNull("Unable to create partition " + part2, retp2);
  233.       Partition retp3 = client.add_partition(part3);
  234.       assertNotNull("Unable to create partition " + part3, retp3);
  235.       Partition retp4 = client.add_partition(part4);
  236.       assertNotNull("Unable to create partition " + part4, retp4);
  237.  
  238.       Partition part_get = client.getPartition(dbName, tblName, part.getValues());
  239.       if(isThriftClient) {
  240.         // since we are using thrift, 'part' will not have the create time and
  241.         // last DDL time set since it does not get updated in the add_partition()
  242.         // call - likewise part2 and part3 - set it correctly so that equals check
  243.         // doesn't fail
  244.         adjust(client, part, dbName, tblName);
  245.         adjust(client, part2, dbName, tblName);
  246.         adjust(client, part3, dbName, tblName);
  247.       }
  248.       assertTrue("Partitions are not same", part.equals(part_get));
  249.  
  250.       String partName = "ds=2008-07-01 14%3A13%3A12/hr=14";
  251.       String part2Name = "ds=2008-07-01 14%3A13%3A12/hr=15";
  252.       String part3Name ="ds=2008-07-02 14%3A13%3A12/hr=15";
  253.  
  254.       part_get = client.getPartition(dbName, tblName, partName);
  255.       assertTrue("Partitions are not the same", part.equals(part_get));
  256.  
  257.       // Test partition listing with a partial spec - ds is specified but hr is not
  258.       List<String> partialVals = new ArrayList<String>();
  259.       partialVals.add(vals.get(0));
  260.       Set<Partition> parts = new HashSet<Partition>();
  261.       parts.add(part);
  262.       parts.add(part2);
  263.  
  264.       List<Partition> partial = client.listPartitions(dbName, tblName, partialVals,
  265.           (short) -1);
  266.       assertTrue("Should have returned 2 partitions", partial.size() == 2);
  267.       assertTrue("Not all parts returned", partial.containsAll(parts));
  268.  
  269.       Set<String> partNames = new HashSet<String>();
  270.       partNames.add(partName);
  271.       partNames.add(part2Name);
  272.       List<String> partialNames = client.listPartitionNames(dbName, tblName, partialVals,
  273.           (short) -1);
  274.       assertTrue("Should have returned 2 partition names", partialNames.size() == 2);
  275.       assertTrue("Not all part names returned", partialNames.containsAll(partNames));
  276.  
  277.       // Test partition listing with a partial spec - hr is specified but ds is not
  278.       parts.clear();
  279.       parts.add(part2);
  280.       parts.add(part3);
  281.  
  282.       partialVals.clear();
  283.       partialVals.add("");
  284.       partialVals.add(vals2.get(1));
  285.  
  286.       partial = client.listPartitions(dbName, tblName, partialVals, (short) -1);
  287.       assertEquals("Should have returned 2 partitions", 2, partial.size());
  288.       assertTrue("Not all parts returned", partial.containsAll(parts));
  289.  
  290.       partNames.clear();
  291.       partNames.add(part2Name);
  292.       partNames.add(part3Name);
  293.       partialNames = client.listPartitionNames(dbName, tblName, partialVals,
  294.           (short) -1);
  295.       assertEquals("Should have returned 2 partition names", 2, partialNames.size());
  296.       assertTrue("Not all part names returned", partialNames.containsAll(partNames));
  297.  
  298.       // Verify escaped partition names don't return partitions
  299.       exceptionThrown = false;
  300.       try {
  301.         String badPartName = "ds=2008-07-01 14%3A13%3A12/hrs=14";
  302.         client.getPartition(dbName, tblName, badPartName);
  303.       } catch(NoSuchObjectException e) {
  304.         exceptionThrown = true;
  305.       }
  306.       assertTrue("Bad partition spec should have thrown an exception", exceptionThrown);
  307.  
  308.       Path partPath = new Path(part2.getSd().getLocation());
  309.       FileSystem fs = FileSystem.get(partPath.toUri(), hiveConf);
  310.  
  311.  
  312.       assertTrue(fs.exists(partPath));
  313.       client.dropPartition(dbName, tblName, part.getValues(), true);
  314.       assertFalse(fs.exists(partPath));
  315.  
  316.       // Test append_partition_by_name
  317.       client.appendPartition(dbName, tblName, partName);
  318.       Partition part5 = client.getPartition(dbName, tblName, part.getValues());
  319.       assertTrue("Append partition by name failed", part5.getValues().equals(vals));;
  320.       Path part5Path = new Path(part5.getSd().getLocation());
  321.       assertTrue(fs.exists(part5Path));
  322.  
  323.       // Test drop_partition_by_name
  324.       assertTrue("Drop partition by name failed",
  325.           client.dropPartition(dbName, tblName, partName, true));
  326.       assertFalse(fs.exists(part5Path));
  327.  
  328.       // add the partition again so that drop table with a partition can be
  329.       // tested
  330.       retp = client.add_partition(part);
  331.       assertNotNull("Unable to create partition " + part, retp);
  332.  
  333.       client.dropTable(dbName, tblName);
  334.  
  335.       client.dropType(typeName);
  336.  
  337.       // recreate table as external, drop partition and it should
  338.       // still exist
  339.       tbl.setParameters(new HashMap<String, String>());
  340.       tbl.getParameters().put("EXTERNAL", "TRUE");
  341.       client.createTable(tbl);
  342.       retp = client.add_partition(part);
  343.       assertTrue(fs.exists(partPath));
  344.       client.dropPartition(dbName, tblName, part.getValues(), true);
  345.       assertTrue(fs.exists(partPath));
  346.  
  347.       for (String tableName : client.getTables(dbName, "*")) {
  348.         client.dropTable(dbName, tableName);
  349.       }
  350.  
  351.       client.dropDatabase(dbName);
  352.  
  353.     } catch (Exception e) {
  354.       System.err.println(StringUtils.stringifyException(e));
  355.       System.err.println("testPartition() failed.");
  356.       throw e;
  357.     }
  358.   }
  359.  
  360.   public void testAlterPartition() throws Throwable {
  361.  
  362.     try {
  363.       String dbName = "compdb";
  364.       String tblName = "comptbl";
  365.       List<String> vals = new ArrayList<String>(2);
  366.       vals.add("2008-07-01");
  367.       vals.add("14");
  368.  
  369.       client.dropTable(dbName, tblName);
  370.       silentDropDatabase(dbName);
  371.       Database db = new Database();
  372.       db.setName(dbName);
  373.       db.setDescription("Alter Partition Test database");
  374.       client.createDatabase(db);
  375.  
  376.       ArrayList<FieldSchema> cols = new ArrayList<FieldSchema>(2);
  377.       cols.add(new FieldSchema("name", Constants.STRING_TYPE_NAME, ""));
  378.       cols.add(new FieldSchema("income", Constants.INT_TYPE_NAME, ""));
  379.  
  380.       Table tbl = new Table();
  381.       tbl.setDbName(dbName);
  382.       tbl.setTableName(tblName);
  383.       StorageDescriptor sd = new StorageDescriptor();
  384.       tbl.setSd(sd);
  385.       sd.setCols(cols);
  386.       sd.setCompressed(false);
  387.       sd.setNumBuckets(1);
  388.       sd.setParameters(new HashMap<String, String>());
  389.       sd.getParameters().put("test_param_1", "Use this for comments etc");
  390.       sd.setBucketCols(new ArrayList<String>(2));
  391.       sd.getBucketCols().add("name");
  392.       sd.setSerdeInfo(new SerDeInfo());
  393.       sd.getSerdeInfo().setName(tbl.getTableName());
  394.       sd.getSerdeInfo().setParameters(new HashMap<String, String>());
  395.       sd.getSerdeInfo().getParameters()
  396.           .put(Constants.SERIALIZATION_FORMAT, "1");
  397.       sd.setSortCols(new ArrayList<Order>());
  398.  
  399.       tbl.setPartitionKeys(new ArrayList<FieldSchema>(2));
  400.       tbl.getPartitionKeys().add(
  401.           new FieldSchema("ds", Constants.STRING_TYPE_NAME, ""));
  402.       tbl.getPartitionKeys().add(
  403.           new FieldSchema("hr", Constants.INT_TYPE_NAME, ""));
  404.  
  405.       client.createTable(tbl);
  406.  
  407.       if (isThriftClient) {
  408.         // the createTable() above does not update the location in the 'tbl'
  409.         // object when the client is a thrift client and the code below relies
  410.         // on the location being present in the 'tbl' object - so get the table
  411.         // from the metastore
  412.         tbl = client.getTable(dbName, tblName);
  413.       }
  414.  
  415.       Partition part = new Partition();
  416.       part.setDbName(dbName);
  417.       part.setTableName(tblName);
  418.       part.setValues(vals);
  419.       part.setParameters(new HashMap<String, String>());
  420.       part.setSd(tbl.getSd());
  421.       part.getSd().setSerdeInfo(tbl.getSd().getSerdeInfo());
  422.       part.getSd().setLocation(tbl.getSd().getLocation() + "/part1");
  423.  
  424.       client.add_partition(part);
  425.  
  426.       Partition part2 = client.getPartition(dbName, tblName, part.getValues());
  427.  
  428.       part2.getParameters().put("retention", "10");
  429.       part2.getSd().setNumBuckets(12);
  430.       part2.getSd().getSerdeInfo().getParameters().put("abc", "1");
  431.       client.alter_partition(dbName, tblName, part2);
  432.  
  433.       Partition part3 = client.getPartition(dbName, tblName, part.getValues());
  434.       assertEquals("couldn't alter partition", part3.getParameters().get(
  435.           "retention"), "10");
  436.       assertEquals("couldn't alter partition", part3.getSd().getSerdeInfo()
  437.           .getParameters().get("abc"), "1");
  438.       assertEquals("couldn't alter partition", part3.getSd().getNumBuckets(),
  439.           12);
  440.  
  441.       client.dropTable(dbName, tblName);
  442.  
  443.       client.dropDatabase(dbName);
  444.     } catch (Exception e) {
  445.       System.err.println(StringUtils.stringifyException(e));
  446.       System.err.println("testPartition() failed.");
  447.       throw e;
  448.     }
  449.   }
  450.  
  451.   public void testDatabase() throws Throwable {
  452.     try {
  453.       // clear up any existing databases
  454.       silentDropDatabase(TEST_DB1_NAME);
  455.       silentDropDatabase(TEST_DB2_NAME);
  456.  
  457.       Database db = new Database();
  458.       db.setName(TEST_DB1_NAME);
  459.       client.createDatabase(db);
  460.  
  461.       db = client.getDatabase(TEST_DB1_NAME);
  462.  
  463.       assertEquals("name of returned db is different from that of inserted db",
  464.           TEST_DB1_NAME, db.getName());
  465.       assertEquals("location of the returned db is different from that of inserted db",
  466.           warehouse.getDefaultDatabasePath(TEST_DB1_NAME).toString(), db.getLocationUri());
  467.  
  468.       Database db2 = new Database();
  469.       db2.setName(TEST_DB2_NAME);
  470.       client.createDatabase(db2);
  471.  
  472.       db2 = client.getDatabase(TEST_DB2_NAME);
  473.  
  474.       assertEquals("name of returned db is different from that of inserted db",
  475.           TEST_DB2_NAME, db2.getName());
  476.       assertEquals("location of the returned db is different from that of inserted db",
  477.           warehouse.getDefaultDatabasePath(TEST_DB2_NAME).toString(), db2.getLocationUri());
  478.  
  479.       List<String> dbs = client.getDatabases(".*");
  480.  
  481.       assertTrue("first database is not " + TEST_DB1_NAME, dbs.contains(TEST_DB1_NAME));
  482.       assertTrue("second database is not " + TEST_DB2_NAME, dbs.contains(TEST_DB2_NAME));
  483.  
  484.       client.dropDatabase(TEST_DB1_NAME);
  485.       client.dropDatabase(TEST_DB2_NAME);
  486.       silentDropDatabase(TEST_DB1_NAME);
  487.       silentDropDatabase(TEST_DB2_NAME);
  488.     } catch (Throwable e) {
  489.       System.err.println(StringUtils.stringifyException(e));
  490.       System.err.println("testDatabase() failed.");
  491.       throw e;
  492.     }
  493.   }
  494.  
  495.   public void testSimpleTypeApi() throws Exception {
  496.     try {
  497.       client.dropType(Constants.INT_TYPE_NAME);
  498.  
  499.       Type typ1 = new Type();
  500.       typ1.setName(Constants.INT_TYPE_NAME);
  501.       boolean ret = client.createType(typ1);
  502.       assertTrue("Unable to create type", ret);
  503.  
  504.       Type typ1_2 = client.getType(Constants.INT_TYPE_NAME);
  505.       assertNotNull(typ1_2);
  506.       assertEquals(typ1.getName(), typ1_2.getName());
  507.  
  508.       ret = client.dropType(Constants.INT_TYPE_NAME);
  509.       assertTrue("unable to drop type integer", ret);
  510.  
  511.       boolean exceptionThrown = false;
  512.       try {
  513.         client.getType(Constants.INT_TYPE_NAME);
  514.       } catch (NoSuchObjectException e) {
  515.         exceptionThrown = true;
  516.       }
  517.       assertTrue("Expected NoSuchObjectException", exceptionThrown);
  518.     } catch (Exception e) {
  519.       System.err.println(StringUtils.stringifyException(e));
  520.       System.err.println("testSimpleTypeApi() failed.");
  521.       throw e;
  522.     }
  523.   }
  524.  
  525.   // TODO:pc need to enhance this with complex fields and getType_all function
  526.   public void testComplexTypeApi() throws Exception {
  527.     try {
  528.       client.dropType("Person");
  529.  
  530.       Type typ1 = new Type();
  531.       typ1.setName("Person");
  532.       typ1.setFields(new ArrayList<FieldSchema>(2));
  533.       typ1.getFields().add(
  534.           new FieldSchema("name", Constants.STRING_TYPE_NAME, ""));
  535.       typ1.getFields().add(
  536.           new FieldSchema("income", Constants.INT_TYPE_NAME, ""));
  537.       boolean ret = client.createType(typ1);
  538.       assertTrue("Unable to create type", ret);
  539.  
  540.       Type typ1_2 = client.getType("Person");
  541.       assertNotNull("type Person not found", typ1_2);
  542.       assertEquals(typ1.getName(), typ1_2.getName());
  543.       assertEquals(typ1.getFields().size(), typ1_2.getFields().size());
  544.       assertEquals(typ1.getFields().get(0), typ1_2.getFields().get(0));
  545.       assertEquals(typ1.getFields().get(1), typ1_2.getFields().get(1));
  546.  
  547.       client.dropType("Family");
  548.  
  549.       Type fam = new Type();
  550.       fam.setName("Family");
  551.       fam.setFields(new ArrayList<FieldSchema>(2));
  552.       fam.getFields().add(
  553.           new FieldSchema("name", Constants.STRING_TYPE_NAME, ""));
  554.       fam.getFields().add(
  555.           new FieldSchema("members",
  556.               MetaStoreUtils.getListType(typ1.getName()), ""));
  557.  
  558.       ret = client.createType(fam);
  559.       assertTrue("Unable to create type " + fam.getName(), ret);
  560.  
  561.       Type fam2 = client.getType("Family");
  562.       assertNotNull("type Person not found", fam2);
  563.       assertEquals(fam.getName(), fam2.getName());
  564.       assertEquals(fam.getFields().size(), fam2.getFields().size());
  565.       assertEquals(fam.getFields().get(0), fam2.getFields().get(0));
  566.       assertEquals(fam.getFields().get(1), fam2.getFields().get(1));
  567.  
  568.       ret = client.dropType("Family");
  569.       assertTrue("unable to drop type Family", ret);
  570.  
  571.       ret = client.dropType("Person");
  572.       assertTrue("unable to drop type Person", ret);
  573.  
  574.       boolean exceptionThrown = false;
  575.       try {
  576.         client.getType("Person");
  577.       } catch (NoSuchObjectException e) {
  578.         exceptionThrown = true;
  579.       }
  580.       assertTrue("Expected NoSuchObjectException", exceptionThrown);
  581.     } catch (Exception e) {
  582.       System.err.println(StringUtils.stringifyException(e));
  583.       System.err.println("testComplexTypeApi() failed.");
  584.       throw e;
  585.     }
  586.   }
  587.  
  588.   public void testSimpleTable() throws Exception {
  589.     try {
  590.       String dbName = "simpdb";
  591.       String tblName = "simptbl";
  592.       String tblName2 = "simptbl2";
  593.       String typeName = "Person";
  594.  
  595.       client.dropTable(dbName, tblName);
  596.       silentDropDatabase(dbName);
  597.  
  598.       Database db = new Database();
  599.       db.setName(dbName);
  600.       client.createDatabase(db);
  601.  
  602.       client.dropType(typeName);
  603.       Type typ1 = new Type();
  604.       typ1.setName(typeName);
  605.       typ1.setFields(new ArrayList<FieldSchema>(2));
  606.       typ1.getFields().add(
  607.           new FieldSchema("name", Constants.STRING_TYPE_NAME, ""));
  608.       typ1.getFields().add(
  609.           new FieldSchema("income", Constants.INT_TYPE_NAME, ""));
  610.       client.createType(typ1);
  611.  
  612.       Table tbl = new Table();
  613.       tbl.setDbName(dbName);
  614.       tbl.setTableName(tblName);
  615.       StorageDescriptor sd = new StorageDescriptor();
  616.       tbl.setSd(sd);
  617.       sd.setCols(typ1.getFields());
  618.       sd.setCompressed(false);
  619.       sd.setNumBuckets(1);
  620.       sd.setParameters(new HashMap<String, String>());
  621.       sd.getParameters().put("test_param_1", "Use this for comments etc");
  622.       sd.setBucketCols(new ArrayList<String>(2));
  623.       sd.getBucketCols().add("name");
  624.       sd.setSerdeInfo(new SerDeInfo());
  625.       sd.getSerdeInfo().setName(tbl.getTableName());
  626.       sd.getSerdeInfo().setParameters(new HashMap<String, String>());
  627.       sd.getSerdeInfo().getParameters().put(
  628.           org.apache.hadoop.hive.serde.Constants.SERIALIZATION_FORMAT, "1");
  629.       sd.getSerdeInfo().setSerializationLib(
  630.           org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe.class.getName());
  631.       tbl.setPartitionKeys(new ArrayList<FieldSchema>());
  632.  
  633.       client.createTable(tbl);
  634.  
  635.       if (isThriftClient) {
  636.         // the createTable() above does not update the location in the 'tbl'
  637.         // object when the client is a thrift client and the code below relies
  638.         // on the location being present in the 'tbl' object - so get the table
  639.         // from the metastore
  640.         tbl = client.getTable(dbName, tblName);
  641.       }
  642.  
  643.       Table tbl2 = client.getTable(dbName, tblName);
  644.       assertNotNull(tbl2);
  645.       assertEquals(tbl2.getDbName(), dbName);
  646.       assertEquals(tbl2.getTableName(), tblName);
  647.       assertEquals(tbl2.getSd().getCols().size(), typ1.getFields().size());
  648.       assertEquals(tbl2.getSd().isCompressed(), false);
  649.       assertEquals(tbl2.getSd().getNumBuckets(), 1);
  650.       assertEquals(tbl2.getSd().getLocation(), tbl.getSd().getLocation());
  651.       assertNotNull(tbl2.getSd().getSerdeInfo());
  652.       sd.getSerdeInfo().setParameters(new HashMap<String, String>());
  653.       sd.getSerdeInfo().getParameters().put(
  654.           org.apache.hadoop.hive.serde.Constants.SERIALIZATION_FORMAT, "1");
  655.  
  656.       tbl2.setTableName(tblName2);
  657.       tbl2.setParameters(new HashMap<String, String>());
  658.       tbl2.getParameters().put("EXTERNAL", "TRUE");
  659.       tbl2.getSd().setLocation(tbl.getSd().getLocation() + "-2");
  660.  
  661.       List<FieldSchema> fieldSchemas = client.getFields(dbName, tblName);
  662.       assertNotNull(fieldSchemas);
  663.       assertEquals(fieldSchemas.size(), tbl.getSd().getCols().size());
  664.       for (FieldSchema fs : tbl.getSd().getCols()) {
  665.         assertTrue(fieldSchemas.contains(fs));
  666.       }
  667.  
  668.       List<FieldSchema> fieldSchemasFull = client.getSchema(dbName, tblName);
  669.       assertNotNull(fieldSchemasFull);
  670.       assertEquals(fieldSchemasFull.size(), tbl.getSd().getCols().size()
  671.           + tbl.getPartitionKeys().size());
  672.       for (FieldSchema fs : tbl.getSd().getCols()) {
  673.         assertTrue(fieldSchemasFull.contains(fs));
  674.       }
  675.       for (FieldSchema fs : tbl.getPartitionKeys()) {
  676.         assertTrue(fieldSchemasFull.contains(fs));
  677.       }
  678.  
  679.       client.createTable(tbl2);
  680.       if (isThriftClient) {
  681.         tbl2 = client.getTable(tbl2.getDbName(), tbl2.getTableName());
  682.       }
  683.  
  684.       Table tbl3 = client.getTable(dbName, tblName2);
  685.       assertNotNull(tbl3);
  686.       assertEquals(tbl3.getDbName(), dbName);
  687.       assertEquals(tbl3.getTableName(), tblName2);
  688.       assertEquals(tbl3.getSd().getCols().size(), typ1.getFields().size());
  689.       assertEquals(tbl3.getSd().isCompressed(), false);
  690.       assertEquals(tbl3.getSd().getNumBuckets(), 1);
  691.       assertEquals(tbl3.getSd().getLocation(), tbl2.getSd().getLocation());
  692.       assertEquals(tbl3.getParameters(), tbl2.getParameters());
  693.  
  694.       fieldSchemas = client.getFields(dbName, tblName2);
  695.       assertNotNull(fieldSchemas);
  696.       assertEquals(fieldSchemas.size(), tbl2.getSd().getCols().size());
  697.       for (FieldSchema fs : tbl2.getSd().getCols()) {
  698.         assertTrue(fieldSchemas.contains(fs));
  699.       }
  700.  
  701.       fieldSchemasFull = client.getSchema(dbName, tblName2);
  702.       assertNotNull(fieldSchemasFull);
  703.       assertEquals(fieldSchemasFull.size(), tbl2.getSd().getCols().size()
  704.           + tbl2.getPartitionKeys().size());
  705.       for (FieldSchema fs : tbl2.getSd().getCols()) {
  706.         assertTrue(fieldSchemasFull.contains(fs));
  707.       }
  708.       for (FieldSchema fs : tbl2.getPartitionKeys()) {
  709.         assertTrue(fieldSchemasFull.contains(fs));
  710.       }
  711.  
  712.       assertEquals("Use this for comments etc", tbl2.getSd().getParameters()
  713.           .get("test_param_1"));
  714.       assertEquals("name", tbl2.getSd().getBucketCols().get(0));
  715.       assertTrue("Partition key list is not empty",
  716.           (tbl2.getPartitionKeys() == null)
  717.               || (tbl2.getPartitionKeys().size() == 0));
  718.  
  719.       FileSystem fs = FileSystem.get((new Path(tbl.getSd().getLocation())).toUri(), hiveConf);
  720.       client.dropTable(dbName, tblName);
  721.       assertFalse(fs.exists(new Path(tbl.getSd().getLocation())));
  722.  
  723.       client.dropTable(dbName, tblName2);
  724.       assertTrue(fs.exists(new Path(tbl2.getSd().getLocation())));
  725.  
  726.       client.dropType(typeName);
  727.       client.dropDatabase(dbName);
  728.     } catch (Exception e) {
  729.       System.err.println(StringUtils.stringifyException(e));
  730.       System.err.println("testSimpleTable() failed.");
  731.       throw e;
  732.     }
  733.   }
  734.  
  735.   public void testAlterTable() throws Exception {
  736.     String dbName = "alterdb";
  737.     String invTblName = "alter-tbl";
  738.     String tblName = "altertbl";
  739.  
  740.     try {
  741.       client.dropTable(dbName, tblName);
  742.       silentDropDatabase(dbName);
  743.  
  744.       Database db = new Database();
  745.       db.setName(dbName);
  746.       client.createDatabase(db);
  747.  
  748.       ArrayList<FieldSchema> invCols = new ArrayList<FieldSchema>(2);
  749.       invCols.add(new FieldSchema("n-ame", Constants.STRING_TYPE_NAME, ""));
  750.       invCols.add(new FieldSchema("in.come", Constants.INT_TYPE_NAME, ""));
  751.  
  752.       Table tbl = new Table();
  753.       tbl.setDbName(dbName);
  754.       tbl.setTableName(invTblName);
  755.       StorageDescriptor sd = new StorageDescriptor();
  756.       tbl.setSd(sd);
  757.       sd.setCols(invCols);
  758.       sd.setCompressed(false);
  759.       sd.setNumBuckets(1);
  760.       sd.setParameters(new HashMap<String, String>());
  761.       sd.getParameters().put("test_param_1", "Use this for comments etc");
  762.       sd.setBucketCols(new ArrayList<String>(2));
  763.       sd.getBucketCols().add("name");
  764.       sd.setSerdeInfo(new SerDeInfo());
  765.       sd.getSerdeInfo().setName(tbl.getTableName());
  766.       sd.getSerdeInfo().setParameters(new HashMap<String, String>());
  767.       sd.getSerdeInfo().getParameters().put(
  768.           org.apache.hadoop.hive.serde.Constants.SERIALIZATION_FORMAT, "1");
  769.       boolean failed = false;
  770.       try {
  771.         client.createTable(tbl);
  772.       } catch (InvalidObjectException ex) {
  773.         failed = true;
  774.       }
  775.       if (!failed) {
  776.         assertTrue("Able to create table with invalid name: " + invTblName,
  777.             false);
  778.       }
  779.       ArrayList<FieldSchema> cols = new ArrayList<FieldSchema>(2);
  780.       cols.add(new FieldSchema("name", Constants.STRING_TYPE_NAME, ""));
  781.       cols.add(new FieldSchema("income", Constants.INT_TYPE_NAME, ""));
  782.  
  783.       // create a valid table
  784.       tbl.setTableName(tblName);
  785.       tbl.getSd().setCols(cols);
  786.       client.createTable(tbl);
  787.  
  788.       if (isThriftClient) {
  789.         tbl = client.getTable(tbl.getDbName(), tbl.getTableName());
  790.       }
  791.  
  792.       // now try to invalid alter table
  793.       Table tbl2 = client.getTable(dbName, tblName);
  794.       failed = false;
  795.       try {
  796.         tbl2.setTableName(invTblName);
  797.         tbl2.getSd().setCols(invCols);
  798.         client.alter_table(dbName, tblName, tbl2);
  799.       } catch (InvalidOperationException ex) {
  800.         failed = true;
  801.       }
  802.       if (!failed) {
  803.         assertTrue("Able to rename table with invalid name: " + invTblName,
  804.             false);
  805.       }
  806.       // try a valid alter table
  807.       tbl2.setTableName(tblName + "_renamed");
  808.       tbl2.getSd().setCols(cols);
  809.       tbl2.getSd().setNumBuckets(32);
  810.       client.alter_table(dbName, tblName, tbl2);
  811.       Table tbl3 = client.getTable(dbName, tbl2.getTableName());
  812.       assertEquals("Alter table didn't succeed. Num buckets is different ",
  813.           tbl2.getSd().getNumBuckets(), tbl3.getSd().getNumBuckets());
  814.       // check that data has moved
  815.       FileSystem fs = FileSystem.get((new Path(tbl.getSd().getLocation())).toUri(), hiveConf);
  816.       assertFalse("old table location still exists", fs.exists(new Path(tbl
  817.           .getSd().getLocation())));
  818.       assertTrue("data did not move to new location", fs.exists(new Path(tbl3
  819.           .getSd().getLocation())));
  820.  
  821.       if (!isThriftClient) {
  822.         assertEquals("alter table didn't move data correct location", tbl3
  823.             .getSd().getLocation(), tbl2.getSd().getLocation());
  824.       }
  825.     } catch (Exception e) {
  826.       System.err.println(StringUtils.stringifyException(e));
  827.       System.err.println("testSimpleTable() failed.");
  828.       throw e;
  829.     } finally {
  830.       silentDropDatabase(dbName);
  831.     }
  832.   }
  833.  
  834.   public void testComplexTable() throws Exception {
  835.  
  836.     String dbName = "compdb";
  837.     String tblName = "comptbl";
  838.     String typeName = "Person";
  839.  
  840.     try {
  841.       client.dropTable(dbName, tblName);
  842.       silentDropDatabase(dbName);
  843.       Database db = new Database();
  844.       db.setName(dbName);
  845.       client.createDatabase(db);
  846.  
  847.       client.dropType(typeName);
  848.       Type typ1 = new Type();
  849.       typ1.setName(typeName);
  850.       typ1.setFields(new ArrayList<FieldSchema>(2));
  851.       typ1.getFields().add(
  852.           new FieldSchema("name", Constants.STRING_TYPE_NAME, ""));
  853.       typ1.getFields().add(
  854.           new FieldSchema("income", Constants.INT_TYPE_NAME, ""));
  855.       client.createType(typ1);
  856.  
  857.       Table tbl = new Table();
  858.       tbl.setDbName(dbName);
  859.       tbl.setTableName(tblName);
  860.       StorageDescriptor sd = new StorageDescriptor();
  861.       tbl.setSd(sd);
  862.       sd.setCols(typ1.getFields());
  863.       sd.setCompressed(false);
  864.       sd.setNumBuckets(1);
  865.       sd.setParameters(new HashMap<String, String>());
  866.       sd.getParameters().put("test_param_1", "Use this for comments etc");
  867.       sd.setBucketCols(new ArrayList<String>(2));
  868.       sd.getBucketCols().add("name");
  869.       sd.setSerdeInfo(new SerDeInfo());
  870.       sd.getSerdeInfo().setName(tbl.getTableName());
  871.       sd.getSerdeInfo().setParameters(new HashMap<String, String>());
  872.       sd.getSerdeInfo().getParameters().put(
  873.           org.apache.hadoop.hive.serde.Constants.SERIALIZATION_FORMAT, "9");
  874.       sd.getSerdeInfo().setSerializationLib(
  875.           org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe.class.getName());
  876.  
  877.       tbl.setPartitionKeys(new ArrayList<FieldSchema>(2));
  878.       tbl.getPartitionKeys().add(
  879.           new FieldSchema("ds",
  880.               org.apache.hadoop.hive.serde.Constants.DATE_TYPE_NAME, ""));
  881.       tbl.getPartitionKeys().add(
  882.           new FieldSchema("hr",
  883.               org.apache.hadoop.hive.serde.Constants.INT_TYPE_NAME, ""));
  884.  
  885.       client.createTable(tbl);
  886.  
  887.       Table tbl2 = client.getTable(dbName, tblName);
  888.       assertEquals(tbl2.getDbName(), dbName);
  889.       assertEquals(tbl2.getTableName(), tblName);
  890.       assertEquals(tbl2.getSd().getCols().size(), typ1.getFields().size());
  891.       assertFalse(tbl2.getSd().isCompressed());
  892.       assertEquals(tbl2.getSd().getNumBuckets(), 1);
  893.  
  894.       assertEquals("Use this for comments etc", tbl2.getSd().getParameters()
  895.           .get("test_param_1"));
  896.       assertEquals("name", tbl2.getSd().getBucketCols().get(0));
  897.  
  898.       assertNotNull(tbl2.getPartitionKeys());
  899.       assertEquals(2, tbl2.getPartitionKeys().size());
  900.       assertEquals(Constants.DATE_TYPE_NAME, tbl2.getPartitionKeys().get(0)
  901.           .getType());
  902.       assertEquals(Constants.INT_TYPE_NAME, tbl2.getPartitionKeys().get(1)
  903.           .getType());
  904.       assertEquals("ds", tbl2.getPartitionKeys().get(0).getName());
  905.       assertEquals("hr", tbl2.getPartitionKeys().get(1).getName());
  906.  
  907.       List<FieldSchema> fieldSchemas = client.getFields(dbName, tblName);
  908.       assertNotNull(fieldSchemas);
  909.       assertEquals(fieldSchemas.size(), tbl.getSd().getCols().size());
  910.       for (FieldSchema fs : tbl.getSd().getCols()) {
  911.         assertTrue(fieldSchemas.contains(fs));
  912.       }
  913.  
  914.       List<FieldSchema> fieldSchemasFull = client.getSchema(dbName, tblName);
  915.       assertNotNull(fieldSchemasFull);
  916.       assertEquals(fieldSchemasFull.size(), tbl.getSd().getCols().size()
  917.           + tbl.getPartitionKeys().size());
  918.       for (FieldSchema fs : tbl.getSd().getCols()) {
  919.         assertTrue(fieldSchemasFull.contains(fs));
  920.       }
  921.       for (FieldSchema fs : tbl.getPartitionKeys()) {
  922.         assertTrue(fieldSchemasFull.contains(fs));
  923.       }
  924.     } catch (Exception e) {
  925.       System.err.println(StringUtils.stringifyException(e));
  926.       System.err.println("testComplexTable() failed.");
  927.       throw e;
  928.     } finally {
  929.       client.dropTable(dbName, tblName);
  930.       boolean ret = client.dropType(typeName);
  931.       assertTrue("Unable to drop type " + typeName, ret);
  932.       client.dropDatabase(dbName);
  933.     }
  934.   }
  935.  
  936.   public void testGetConfigValue() {
  937.  
  938.     String val = "value";
  939.  
  940.     if (!isThriftClient) {
  941.       try {
  942.         assertEquals(client.getConfigValue("hive.key1", val), "value1");
  943.         assertEquals(client.getConfigValue("hive.key2", val), "http://www.example.com");
  944.         assertEquals(client.getConfigValue("hive.key3", val), "");
  945.         assertEquals(client.getConfigValue("hive.key4", val), "0");
  946.         assertEquals(client.getConfigValue("hive.key5", val), val);
  947.         assertEquals(client.getConfigValue(null, val), val);
  948.       } catch (TException e) {
  949.         e.printStackTrace();
  950.           assert (false);
  951.       } catch (ConfigValSecurityException e) {
  952.         e.printStackTrace();
  953.         assert (false);
  954.       }
  955.     }
  956.  
  957.     boolean threwException = false;
  958.     try {
  959.       // Attempting to get the password should throw an exception
  960.       client.getConfigValue("javax.jdo.option.ConnectionPassword", "password");
  961.     } catch (TException e) {
  962.       e.printStackTrace();
  963.       assert (false);
  964.     } catch (ConfigValSecurityException e) {
  965.       threwException = true;
  966.     }
  967.     assert (threwException);
  968.   }
  969.  
  970.   private static void adjust(HiveMetaStoreClient client, Partition part,
  971.       String dbName, String tblName)
  972.   throws NoSuchObjectException, MetaException, TException {
  973.     Partition part_get = client.getPartition(dbName, tblName, part.getValues());
  974.     part.setCreateTime(part_get.getCreateTime());
  975.     part.putToParameters(org.apache.hadoop.hive.metastore.api.Constants.DDL_TIME, Long.toString(part_get.getCreateTime()));
  976.   }
  977.  
  978.   private static void silentDropDatabase(String dbName) throws MetaException, TException {
  979.     try {
  980.       for (String tableName : client.getTables(dbName, "*")) {
  981.         client.dropTable(dbName, tableName);
  982.       }
  983.       client.dropDatabase(dbName);
  984.     } catch (NoSuchObjectException e) {
  985.     } catch (InvalidOperationException e) {
  986.     }
  987.   }
  988.  
  989.   /**
  990.    * Tests for list partition by filter functionality.
  991.    * @throws Exception
  992.    */
  993.   public void testPartitionFilter() throws Exception {
  994.       String dbName = "filterdb";
  995.       String tblName = "filtertbl";
  996.  
  997.       List<String> vals = new ArrayList<String>(3);
  998.       vals.add("p11");
  999.       vals.add("p21");
  1000.       vals.add("p31");
  1001.       List <String> vals2 = new ArrayList<String>(3);
  1002.       vals2.add("p11");
  1003.       vals2.add("p22");
  1004.       vals2.add("p31");
  1005.       List <String> vals3 = new ArrayList<String>(3);
  1006.       vals3.add("p12");
  1007.       vals3.add("p21");
  1008.       vals3.add("p31");
  1009.       List <String> vals4 = new ArrayList<String>(3);
  1010.       vals4.add("p12");
  1011.       vals4.add("p23");
  1012.       vals4.add("p31");
  1013.       List <String> vals5 = new ArrayList<String>(3);
  1014.       vals5.add("p13");
  1015.       vals5.add("p24");
  1016.       vals5.add("p31");
  1017.       List <String> vals6 = new ArrayList<String>(3);
  1018.       vals6.add("p13");
  1019.       vals6.add("p25");
  1020.       vals6.add("p31");
  1021.  
  1022.       silentDropDatabase(dbName);
  1023.  
  1024.       Database db = new Database();
  1025.       db.setName(dbName);
  1026.       client.createDatabase(db);
  1027.  
  1028.       ArrayList<FieldSchema> cols = new ArrayList<FieldSchema>(2);
  1029.       cols.add(new FieldSchema("c1", Constants.STRING_TYPE_NAME, ""));
  1030.       cols.add(new FieldSchema("c2", Constants.INT_TYPE_NAME, ""));
  1031.  
  1032.       ArrayList<FieldSchema> partCols = new ArrayList<FieldSchema>(3);
  1033.       partCols.add(new FieldSchema("p1", Constants.STRING_TYPE_NAME, ""));
  1034.       partCols.add(new FieldSchema("p2", Constants.STRING_TYPE_NAME, ""));
  1035.       partCols.add(new FieldSchema("p3", Constants.INT_TYPE_NAME, ""));
  1036.  
  1037.       Table tbl = new Table();
  1038.       tbl.setDbName(dbName);
  1039.       tbl.setTableName(tblName);
  1040.       StorageDescriptor sd = new StorageDescriptor();
  1041.       tbl.setSd(sd);
  1042.       sd.setCols(cols);
  1043.       sd.setCompressed(false);
  1044.       sd.setNumBuckets(1);
  1045.       sd.setParameters(new HashMap<String, String>());
  1046.       sd.setBucketCols(new ArrayList<String>());
  1047.       sd.setSerdeInfo(new SerDeInfo());
  1048.       sd.getSerdeInfo().setName(tbl.getTableName());
  1049.       sd.getSerdeInfo().setParameters(new HashMap<String, String>());
  1050.       sd.getSerdeInfo().getParameters()
  1051.           .put(Constants.SERIALIZATION_FORMAT, "1");
  1052.       sd.setSortCols(new ArrayList<Order>());
  1053.  
  1054.       tbl.setPartitionKeys(partCols);
  1055.       client.createTable(tbl);
  1056.  
  1057.       tbl = client.getTable(dbName, tblName);
  1058.  
  1059.       add_partition(client, tbl, vals, "part1");
  1060.       add_partition(client, tbl, vals2, "part2");
  1061.       add_partition(client, tbl, vals3, "part3");
  1062.       add_partition(client, tbl, vals4, "part4");
  1063.       add_partition(client, tbl, vals5, "part5");
  1064.       add_partition(client, tbl, vals6, "part6");
  1065.  
  1066.       checkFilter(client, dbName, tblName, "p1 = \"p11\"", 2);
  1067.       checkFilter(client, dbName, tblName, "p1 = \"p12\"", 2);
  1068.       checkFilter(client, dbName, tblName, "p2 = \"p21\"", 2);
  1069.       checkFilter(client, dbName, tblName, "p2 = \"p23\"", 1);
  1070.       checkFilter(client, dbName, tblName, "p1 = \"p11\" and p2=\"p22\"", 1);
  1071.       checkFilter(client, dbName, tblName, "p1 = \"p11\" or p2=\"p23\"", 3);
  1072.       checkFilter(client, dbName, tblName, "p1 = \"p11\" or p1=\"p12\"", 4);
  1073.  
  1074.       checkFilter(client, dbName, tblName,
  1075.           "p1 = \"p11\" or (p1=\"p12\" and p2=\"p21\")", 3);
  1076.       checkFilter(client, dbName, tblName,
  1077.          "p1 = \"p11\" or (p1=\"p12\" and p2=\"p21\") Or " +
  1078.          "(p1=\"p13\" aNd p2=\"p24\")", 4);
  1079.       //test for and or precedence
  1080.       checkFilter(client, dbName, tblName,
  1081.          "p1=\"p12\" and (p2=\"p27\" Or p2=\"p21\")", 1);
  1082.       checkFilter(client, dbName, tblName,
  1083.          "p1=\"p12\" and p2=\"p27\" Or p2=\"p21\"", 2);
  1084.  
  1085.       checkFilter(client, dbName, tblName, "p1 > \"p12\"", 2);
  1086.       checkFilter(client, dbName, tblName, "p1 >= \"p12\"", 4);
  1087.       checkFilter(client, dbName, tblName, "p1 < \"p12\"", 2);
  1088.       checkFilter(client, dbName, tblName, "p1 <= \"p12\"", 4);
  1089.       checkFilter(client, dbName, tblName, "p1 <> \"p12\"", 4);
  1090.       checkFilter(client, dbName, tblName, "p1 like \"p1.*\"", 6);
  1091.       checkFilter(client, dbName, tblName, "p2 like \"p.*3\"", 1);
  1092.  
  1093.       //Test for setting the maximum partition count
  1094.       List<Partition> partitions = client.listPartitionsByFilter(dbName,
  1095.           tblName, "p1 >= \"p12\"", (short) 2);
  1096.       assertEquals("User specified row limit for partitions",
  1097.           2, partitions.size());
  1098.  
  1099.       //Negative tests
  1100.       Exception me = null;
  1101.       try {
  1102.         client.listPartitionsByFilter(dbName,
  1103.             tblName, "p3 >= \"p12\"", (short) -1);
  1104.       } catch(MetaException e) {
  1105.         me = e;
  1106.       }
  1107.       assertNotNull(me);
  1108.       assertTrue("Filter on int partition key", me.getMessage().contains(
  1109.             "Filtering is supported only on partition keys of type string"));
  1110.  
  1111.       me = null;
  1112.       try {
  1113.         client.listPartitionsByFilter(dbName,
  1114.             tblName, "c1 >= \"p12\"", (short) -1);
  1115.       } catch(MetaException e) {
  1116.         me = e;
  1117.       }
  1118.       assertNotNull(me);
  1119.       assertTrue("Filter on invalid key", me.getMessage().contains(
  1120.             "<c1> is not a partitioning key for the table"));
  1121.  
  1122.       me = null;
  1123.       try {
  1124.         client.listPartitionsByFilter(dbName,
  1125.             tblName, "c1 >= ", (short) -1);
  1126.       } catch(MetaException e) {
  1127.         me = e;
  1128.       }
  1129.       assertNotNull(me);
  1130.       assertTrue("Invalid filter string", me.getMessage().contains(
  1131.             "Error parsing partition filter"));
  1132.  
  1133.       me = null;
  1134.       try {
  1135.         client.listPartitionsByFilter("invDBName",
  1136.             "invTableName", "p1 = \"p11\"", (short) -1);
  1137.       } catch(NoSuchObjectException e) {
  1138.         me = e;
  1139.       }
  1140.       assertNotNull(me);
  1141.       assertTrue("NoSuchObject exception", me.getMessage().contains(
  1142.             "database/table does not exist"));
  1143.  
  1144.       client.dropTable(dbName, tblName);
  1145.       client.dropDatabase(dbName);
  1146.   }
  1147.  
  1148.   /**
  1149.    * Test filtering on table with single partition
  1150.    * @throws Exception
  1151.    */
  1152.   public void testFilterSinglePartition() throws Exception {
  1153.       String dbName = "filterdb";
  1154.       String tblName = "filtertbl";
  1155.  
  1156.       List<String> vals = new ArrayList<String>(1);
  1157.       vals.add("p11");
  1158.       List <String> vals2 = new ArrayList<String>(1);
  1159.       vals2.add("p12");
  1160.       List <String> vals3 = new ArrayList<String>(1);
  1161.       vals3.add("p13");
  1162.  
  1163.       silentDropDatabase(dbName);
  1164.  
  1165.       Database db = new Database();
  1166.       db.setName(dbName);
  1167.       client.createDatabase(db);
  1168.  
  1169.       ArrayList<FieldSchema> cols = new ArrayList<FieldSchema>(2);
  1170.       cols.add(new FieldSchema("c1", Constants.STRING_TYPE_NAME, ""));
  1171.       cols.add(new FieldSchema("c2", Constants.INT_TYPE_NAME, ""));
  1172.  
  1173.       ArrayList<FieldSchema> partCols = new ArrayList<FieldSchema>(1);
  1174.       partCols.add(new FieldSchema("p1", Constants.STRING_TYPE_NAME, ""));
  1175.  
  1176.       Table tbl = new Table();
  1177.       tbl.setDbName(dbName);
  1178.       tbl.setTableName(tblName);
  1179.       StorageDescriptor sd = new StorageDescriptor();
  1180.       tbl.setSd(sd);
  1181.       sd.setCols(cols);
  1182.       sd.setCompressed(false);
  1183.       sd.setNumBuckets(1);
  1184.       sd.setParameters(new HashMap<String, String>());
  1185.       sd.setBucketCols(new ArrayList<String>());
  1186.       sd.setSerdeInfo(new SerDeInfo());
  1187.       sd.getSerdeInfo().setName(tbl.getTableName());
  1188.       sd.getSerdeInfo().setParameters(new HashMap<String, String>());
  1189.       sd.getSerdeInfo().getParameters()
  1190.           .put(Constants.SERIALIZATION_FORMAT, "1");
  1191.       sd.setSortCols(new ArrayList<Order>());
  1192.  
  1193.       tbl.setPartitionKeys(partCols);
  1194.       client.createTable(tbl);
  1195.  
  1196.       tbl = client.getTable(dbName, tblName);
  1197.  
  1198.       add_partition(client, tbl, vals, "part1");
  1199.       add_partition(client, tbl, vals2, "part2");
  1200.       add_partition(client, tbl, vals3, "part3");
  1201.  
  1202.       checkFilter(client, dbName, tblName, "p1 = \"p12\"", 1);
  1203.       checkFilter(client, dbName, tblName, "p1 < \"p12\"", 1);
  1204.       checkFilter(client, dbName, tblName, "p1 > \"p12\"", 1);
  1205.       checkFilter(client, dbName, tblName, "p1 >= \"p12\"", 2);
  1206.       checkFilter(client, dbName, tblName, "p1 <= \"p12\"", 2);
  1207.       checkFilter(client, dbName, tblName, "p1 <> \"p12\"", 2);
  1208.       checkFilter(client, dbName, tblName, "p1 like \"p1.*\"", 3);
  1209.       checkFilter(client, dbName, tblName, "p1 like \"p.*2\"", 1);
  1210.  
  1211.       client.dropTable(dbName, tblName);
  1212.       client.dropDatabase(dbName);
  1213.   }
  1214.  
  1215.   /**
  1216.    * Test filtering based on the value of the last partition
  1217.    * @throws Exception
  1218.    */
  1219.   public void testFilterLastPartition() throws Exception {
  1220.       String dbName = "filterdb";
  1221.       String tblName = "filtertbl";
  1222.  
  1223.       List<String> vals = new ArrayList<String>(2);
  1224.       vals.add("p11");
  1225.       vals.add("p21");
  1226.       List <String> vals2 = new ArrayList<String>(2);
  1227.       vals2.add("p11");
  1228.       vals2.add("p22");
  1229.       List <String> vals3 = new ArrayList<String>(2);
  1230.       vals3.add("p12");
  1231.       vals3.add("p21");
  1232.  
  1233.       silentDropDatabase(dbName);
  1234.  
  1235.       Database db = new Database();
  1236.       db.setName(dbName);
  1237.       client.createDatabase(db);
  1238.  
  1239.       ArrayList<FieldSchema> cols = new ArrayList<FieldSchema>(2);
  1240.       cols.add(new FieldSchema("c1", Constants.STRING_TYPE_NAME, ""));
  1241.       cols.add(new FieldSchema("c2", Constants.INT_TYPE_NAME, ""));
  1242.  
  1243.       ArrayList<FieldSchema> partCols = new ArrayList<FieldSchema>(2);
  1244.       partCols.add(new FieldSchema("p1", Constants.STRING_TYPE_NAME, ""));
  1245.       partCols.add(new FieldSchema("p2", Constants.STRING_TYPE_NAME, ""));
  1246.  
  1247.       Table tbl = new Table();
  1248.       tbl.setDbName(dbName);
  1249.       tbl.setTableName(tblName);
  1250.       StorageDescriptor sd = new StorageDescriptor();
  1251.       tbl.setSd(sd);
  1252.       sd.setCols(cols);
  1253.       sd.setCompressed(false);
  1254.       sd.setNumBuckets(1);
  1255.       sd.setParameters(new HashMap<String, String>());
  1256.       sd.setBucketCols(new ArrayList<String>());
  1257.       sd.setSerdeInfo(new SerDeInfo());
  1258.       sd.getSerdeInfo().setName(tbl.getTableName());
  1259.       sd.getSerdeInfo().setParameters(new HashMap<String, String>());
  1260.       sd.getSerdeInfo().getParameters()
  1261.           .put(Constants.SERIALIZATION_FORMAT, "1");
  1262.       sd.setSortCols(new ArrayList<Order>());
  1263.  
  1264.       tbl.setPartitionKeys(partCols);
  1265.       client.createTable(tbl);
  1266.  
  1267.       tbl = client.getTable(dbName, tblName);
  1268.  
  1269.       add_partition(client, tbl, vals, "part1");
  1270.       add_partition(client, tbl, vals2, "part2");
  1271.       add_partition(client, tbl, vals3, "part3");
  1272.  
  1273.       checkFilter(client, dbName, tblName, "p2 = \"p21\"", 2);
  1274.       checkFilter(client, dbName, tblName, "p2 < \"p23\"", 3);
  1275.       checkFilter(client, dbName, tblName, "p2 > \"p21\"", 1);
  1276.       checkFilter(client, dbName, tblName, "p2 >= \"p21\"", 3);
  1277.       checkFilter(client, dbName, tblName, "p2 <= \"p21\"", 2);
  1278.       checkFilter(client, dbName, tblName, "p2 <> \"p12\"", 3);
  1279.       checkFilter(client, dbName, tblName, "p2 like \"p2.*\"", 3);
  1280.       checkFilter(client, dbName, tblName, "p2 like \"p.*2\"", 1);
  1281.  
  1282.       client.dropTable(dbName, tblName);
  1283.       client.dropDatabase(dbName);
  1284.   }
  1285.  
  1286.   private void checkFilter(HiveMetaStoreClient client, String dbName,
  1287.         String tblName, String filter, int expectedCount)
  1288.         throws MetaException, NoSuchObjectException, TException {
  1289.     List<Partition> partitions = client.listPartitionsByFilter(dbName,
  1290.             tblName, filter, (short) -1);
  1291.  
  1292.     assertEquals("Partition count expected for filter " + filter,
  1293.             expectedCount, partitions.size());
  1294.   }
  1295.  
  1296.   private void add_partition(HiveMetaStoreClient client, Table table,
  1297.       List<String> vals, String location) throws InvalidObjectException,
  1298.         AlreadyExistsException, MetaException, TException {
  1299.  
  1300.     Partition part = new Partition();
  1301.     part.setDbName(table.getDbName());
  1302.     part.setTableName(table.getTableName());
  1303.     part.setValues(vals);
  1304.     part.setParameters(new HashMap<String, String>());
  1305.     part.setSd(table.getSd());
  1306.     part.getSd().setSerdeInfo(table.getSd().getSerdeInfo());
  1307.     part.getSd().setLocation(table.getSd().getLocation() + location);
  1308.  
  1309.     client.add_partition(part);
  1310.   }
  1311.  
  1312.   /**
  1313.    * Tests {@link HiveMetaStoreClient#newSynchronizedClient}.  Does not
  1314.    * actually test multithreading, but does verify that the proxy
  1315.    * at least works correctly.
  1316.    */
  1317.   public void testSynchronized() throws Exception {
  1318.     IMetaStoreClient synchronizedClient =
  1319.       HiveMetaStoreClient.newSynchronizedClient(client);
  1320.     List<String> databases = synchronizedClient.getAllDatabases();
  1321.     assertEquals(1, databases.size());
  1322.   }
  1323. }

声明：本文内容由网友自发贡献，不代表【wpsshop博客】立场，版权归原作者所有，本站不承担相应法律责任。如您发现有侵权的内容，请联系我们。转载请注明出处：https://www.wpsshop.cn/w/weixin_40725706/article/detail/484111
推荐阅读
相关标签

Copyright © 2003-2013 www.wpsshop.cn 版权所有，并保留所有权利。

  

闽ICP备14008679号