使用 HiveMetaStoreClient(即 HiveConf) 检索 Hive 元数据

【问题标题】：Use of HiveMetaStoreClient(thereby, HiveConf) to retrieve Hive metadata使用 HiveMetaStoreClient(即 HiveConf) 检索 Hive 元数据
【发布时间】：2017-11-23 15:03:31
【问题描述】：

Kerberized HDP-2.6.3.0。

我可以使用 Hive JDBC 驱动程序从我的 Windows 机器连接到 Hive，但是，我需要使用 HiveMetaStoreClient 的一些方法。我翻阅了 api 并编写了一个我正在从 IDE 执行的测试代码。

private static void connectHiveMetastore() throws MetaException, MalformedURLException {
        //System.setProperty("javax.security.auth.useSubjectCredsOnly","false");
        //System.setProperty("java.security.krb5.conf","C:\\kerb5.conf");
        Configuration configuration = new Configuration();
        //configuration.addResource("E:\\hdp\\client_config\\HDFS_CLIENT\\core-site.xml");
        //configuration.addResource("E:\\hdp\\client_config\\HDFS_CLIENT\\hdfs-site.xml");
        HiveConf hiveConf = new HiveConf(configuration,Configuration.class);
        //URL url = new File("E:\\hdp\\client_config\\HDFS_CLIENT\\hive-site.xml").toURI().toURL();
        //hiveConf.setHiveSiteLocation(url);
        //hiveConf.setVar(HiveConf.ConfVars.METASTOREURIS,"thrift://l4283t.sss.com:9083,thrift://l4284t.sss.com:9083");
        HiveMetaStoreClient hiveMetaStoreClient = new HiveMetaStoreClient(hiveConf);
    }

pom文件中的依赖：

</dependencies>
<dependency>
        <groupId>junit</groupId>
        <artifactId>junit</artifactId>
        <version>3.8.1</version>
        <scope>test</scope>
    </dependency>
    <!-- https://mvnrepository.com/artifact/org.apache.hive/hive-metastore -->
    <dependency>
        <groupId>org.apache.hive</groupId>
        <artifactId>hive-metastore</artifactId>
        <version>2.3.2</version>
    </dependency>
    <!-- https://mvnrepository.com/artifact/org.apache.hive/hive-exec -->
    <dependency>
        <groupId>org.apache.hive</groupId>
        <artifactId>hive-exec</artifactId>
        <version>2.3.2</version>
    </dependency>
    <!-- https://mvnrepository.com/artifact/org.apache.hadoop/hadoop-common -->
    <dependency>
        <groupId>org.apache.hadoop</groupId>
        <artifactId>hadoop-common</artifactId>
        <version>2.9.0</version>
    </dependency>
</dependencies>

无论我是评论还是取消评论与配置和 Kerberos 相关的行，我都会收到以下异常，该异常在 Hive wiki 中进行了解释：

15:35:27.139 [main] ERROR org.apache.hadoop.hive.metastore.RetryingHMSHandler - MetaException(message:Version information not found in metastore. )
at org.apache.hadoop.hive.metastore.ObjectStore.checkSchema(ObjectStore.java:7564)
at org.apache.hadoop.hive.metastore.ObjectStore.verifySchema(ObjectStore.java:7542)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at org.apache.hadoop.hive.metastore.RawStoreProxy.invoke(RawStoreProxy.java:101)
at com.sun.proxy.$Proxy8.verifySchema(Unknown Source)
at org.apache.hadoop.hive.metastore.HiveMetaStore$HMSHandler.getMSForConf(HiveMetaStore.java:591)
at org.apache.hadoop.hive.metastore.HiveMetaStore$HMSHandler.getMS(HiveMetaStore.java:584)
at org.apache.hadoop.hive.metastore.HiveMetaStore$HMSHandler.createDefaultDB(HiveMetaStore.java:651)
at org.apache.hadoop.hive.metastore.HiveMetaStore$HMSHandler.init(HiveMetaStore.java:427)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at org.apache.hadoop.hive.metastore.RetryingHMSHandler.invokeInternal(RetryingHMSHandler.java:148)
at org.apache.hadoop.hive.metastore.RetryingHMSHandler.invoke(RetryingHMSHandler.java:107)
at org.apache.hadoop.hive.metastore.RetryingHMSHandler.<init>(RetryingHMSHandler.java:79)
at org.apache.hadoop.hive.metastore.RetryingHMSHandler.getProxy(RetryingHMSHandler.java:92)
at org.apache.hadoop.hive.metastore.HiveMetaStore.newRetryingHMSHandler(HiveMetaStore.java:6893)
at org.apache.hadoop.hive.metastore.HiveMetaStoreClient.<init>(HiveMetaStoreClient.java:164)
at org.apache.hadoop.hive.metastore.HiveMetaStoreClient.<init>(HiveMetaStoreClient.java:129)
at com.my.App.connectHiveMetastore(App.java:58)
at com.my.App.main(App.java:37)
15:35:27.141 [main] ERROR org.apache.hadoop.hive.metastore.RetryingHMSHandler - HMSHandler Fatal error: MetaException(message:Version information not found in metastore. )
at org.apache.hadoop.hive.metastore.ObjectStore.checkSchema(ObjectStore.java:7564)
at org.apache.hadoop.hive.metastore.ObjectStore.verifySchema(ObjectStore.java:7542)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at org.apache.hadoop.hive.metastore.RawStoreProxy.invoke(RawStoreProxy.java:101)
at com.sun.proxy.$Proxy8.verifySchema(Unknown Source)
at org.apache.hadoop.hive.metastore.HiveMetaStore$HMSHandler.getMSForConf(HiveMetaStore.java:591)
at org.apache.hadoop.hive.metastore.HiveMetaStore$HMSHandler.getMS(HiveMetaStore.java:584)
at org.apache.hadoop.hive.metastore.HiveMetaStore$HMSHandler.createDefaultDB(HiveMetaStore.java:651)
at org.apache.hadoop.hive.metastore.HiveMetaStore$HMSHandler.init(HiveMetaStore.java:427)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at org.apache.hadoop.hive.metastore.RetryingHMSHandler.invokeInternal(RetryingHMSHandler.java:148)
at org.apache.hadoop.hive.metastore.RetryingHMSHandler.invoke(RetryingHMSHandler.java:107)
at org.apache.hadoop.hive.metastore.RetryingHMSHandler.<init>(RetryingHMSHandler.java:79)
at org.apache.hadoop.hive.metastore.RetryingHMSHandler.getProxy(RetryingHMSHandler.java:92)
at org.apache.hadoop.hive.metastore.HiveMetaStore.newRetryingHMSHandler(HiveMetaStore.java:6893)
at org.apache.hadoop.hive.metastore.HiveMetaStoreClient.<init>(HiveMetaStoreClient.java:164)
at org.apache.hadoop.hive.metastore.HiveMetaStoreClient.<init>(HiveMetaStoreClient.java:129)
at com.my.App.connectHiveMetastore(App.java:58)
at com.my.App.main(App.java:37)
Exception in thread "main" MetaException(message:Version information not found in metastore. )
at org.apache.hadoop.hive.metastore.RetryingHMSHandler.<init>(RetryingHMSHandler.java:83)
at org.apache.hadoop.hive.metastore.RetryingHMSHandler.getProxy(RetryingHMSHandler.java:92)
at org.apache.hadoop.hive.metastore.HiveMetaStore.newRetryingHMSHandler(HiveMetaStore.java:6893)
at org.apache.hadoop.hive.metastore.HiveMetaStoreClient.<init>(HiveMetaStoreClient.java:164)
at org.apache.hadoop.hive.metastore.HiveMetaStoreClient.<init>(HiveMetaStoreClient.java:129)
at com.my.App.connectHiveMetastore(App.java:58)
at com.my.App.main(App.java:37)
Caused by: MetaException(message:Version information not found in metastore. )
at org.apache.hadoop.hive.metastore.ObjectStore.checkSchema(ObjectStore.java:7564)
at org.apache.hadoop.hive.metastore.ObjectStore.verifySchema(ObjectStore.java:7542)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at org.apache.hadoop.hive.metastore.RawStoreProxy.invoke(RawStoreProxy.java:101)
at com.sun.proxy.$Proxy8.verifySchema(Unknown Source)
at org.apache.hadoop.hive.metastore.HiveMetaStore$HMSHandler.getMSForConf(HiveMetaStore.java:591)
at org.apache.hadoop.hive.metastore.HiveMetaStore$HMSHandler.getMS(HiveMetaStore.java:584)
at org.apache.hadoop.hive.metastore.HiveMetaStore$HMSHandler.createDefaultDB(HiveMetaStore.java:651)
at org.apache.hadoop.hive.metastore.HiveMetaStore$HMSHandler.init(HiveMetaStore.java:427)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at org.apache.hadoop.hive.metastore.RetryingHMSHandler.invokeInternal(RetryingHMSHandler.java:148)
at org.apache.hadoop.hive.metastore.RetryingHMSHandler.invoke(RetryingHMSHandler.java:107)
at org.apache.hadoop.hive.metastore.RetryingHMSHandler.<init>(RetryingHMSHandler.java:79)
... 6 more
Process finished with exit code 1

我有以下问题/疑虑：

  1. 我考虑和连接 HiveMetaStoreClient 的方式是否正确？如果没有，我如何检索HiveMetaStoreClient 方法提供的元数据信息？
  2. 代码肯定没有“到达”集群。上述异常是否与依赖版本有关？如果不是，那么根本原因是什么？

【问题讨论】：

  • 只需将一个有效的hive-site.xml 文件放到本地目录中，将该目录 添加到CLASSPATH，然后构建一个new HiveConf(...)。然后检查hive.metastore.uris 是否已正确初始化——如果 Hadoop 客户端在 CLASSPATH 中找不到它们的配置文件，它们会恢复为硬设置的默认值——这意味着由易失 Derby 数据库支持的嵌入式元存储。跨度>
  • 另外，不要再玩弄 POM 文件了。只需在集群上获取 Hive JAR - 加上几个 Hadoop JAR，因为 Kerberos 身份验证（例如 hadoop-auth 和 commons-configuration）。
  • 说到 Kerberos，要么你有一个可用的 Java 风格的 Kerberos 票证，要么你需要用 principal & keytab 显式初始化静态UserGroupInformation... （而 JDBC 可以得到principal / keytab 直接来自 JAAS 配置）
  • 我将继续您提到的方法。即使我想连接到远程集群，是否需要有一个 winutil？我已将 core-site、hdfs-site 和 hive-site 添加到类路径中，但我得到 Did not find winutils.exe: java.io.FileNotFoundException: java.io.FileNotFoundException: HADOOP_HOME and hadoop.home.dir are unset . -见wiki.apache.org/hadoop/WindowsProblems
  • 啊，由于糟糕的 Kerberos 实现（仅由单元测试使用，但始终需要），又是一个糟糕的依赖项。

标签： hadoop hive hive-metastore


【解决方案1】：

按照@Samson（在 cmets 中）提供的线索，我给出了 Maven 方法，并简单地继续复制集群所需的 jar。是的，这花了很长时间才把事情整理好，但我确实取得了一些进展。 下面是我正在使用的类，我仍然得到与 SASL 相关的异常，但至少请求到达了服务器。

private static void connectHiveMetastore() throws MetaException, MalformedURLException {

        System.setProperty("hadoop.home.dir", "E:\\Software\\Virtualization");

        /*Start : Commented or un-commented, immaterial ...*/
        System.setProperty("javax.security.auth.useSubjectCredsOnly","false");
        System.setProperty("java.security.auth.login.config","E:\\lib\\hdp\\loginconf.ini");
        System.setProperty("java.security.krb5.conf","E:\\lib\\hdp\\krb5.conf");
        /*End : Commented or un-commented, immaterial ...*/

        Configuration configuration = new Configuration();
        /*Start : Commented or un-commented, immaterial ...*/
        configuration.addResource("E:\\lib\\hdp\\client_config\\HDFS_CLIENT\\core-site.xml");
        configuration.addResource("E:\\lib\\hdp\\client_config\\HDFS_CLIENT\\hdfs-site.xml");
        configuration.addResource("E:\\lib\\hdp\\client_config\\HIVE_CLIENT\\hive-site.xml");
        configuration.set("hive.server2.authentication","KERBEROS");
        configuration.set("hadoop.security.authentication", "Kerberos");
        /*End : Commented or un-commented, immaterial ...*/

        HiveConf hiveConf = new HiveConf(configuration,Configuration.class);
        URL url = new File("E:\\lib\\hdp\\client_config\\HIVE_CLIENT\\hive-site.xml").toURI().toURL();
        HiveConf.setHiveSiteLocation(url);
        hiveConf.setVar(HiveConf.ConfVars.METASTOREURIS,"thrift://l4283t.sss.se.com:9083,thrift://l4284t.sss.se.com:9083");
        /*Start : Commented or un-commented, immaterial ...*/
        hiveConf.setVar(HiveConf.ConfVars.HIVE_SERVER2_AUTHENTICATION,"KERBEROS");
        /*End : Commented or un-commented, immaterial ...*/

        HiveMetaStoreClient hiveMetaStoreClient = new HiveMetaStoreClient(hiveConf);

        System.out.println("Metastore client : "+hiveMetaStoreClient);
        System.out.println("Is local metastore ? "+hiveMetaStoreClient.isLocalMetaStore());
        System.out.println(hiveMetaStoreClient.getAllDatabases());

        hiveMetaStoreClient.close();
    }

【讨论】：

    猜你喜欢
    • 2012-10-31
    • 1970-01-01
    • 1970-01-01
    • 1970-01-01
    • 2013-04-24
    • 1970-01-01
    • 1970-01-01
    • 1970-01-01
    • 2016-02-14
    相关资源
    最近更新 更多
    热门标签
    Java Python linux javascript Mysql C# Docker 算法 前端 SpringBoot Redis Vue spring 设计模式 .net core .net kubernetes c++ 数据库 数据结构 大数据 js 机器学习 微服务 Android Go 程序员 面试 JVM ASP.net core 云原生 人工智能 后端 PHP git CSS golang k8s Nginx Django mybatis 深度学习 多线程 React 架构 devops 爬虫 云计算 Spring Boot LeetCode