Archive for October, 2021

Hive/Impala Tester – org.apache.hive.jdbc.HiveDriver

Posted: October 29, 2021 in Hadoop
Tags:
0

Prerequisites

Hive/Impala Tester

HiveServer2 Beeline tests

beeline -u 'jdbc:hive2://<hs2_node>:10000/default;ssl=true;principal=hive/_HOST@<kerberos_realm>'

beeline -u 'jdbc:hive2://<impala-load-balancer>:21050/default;ssl=true;principal=impala/_HOST@<kerberos_realm>'

note: Beeline Command Options

Obtain TGT in ticket cache

# renewal of validity for Kerberos ticket: scheduled e.g. Cron job
su - u hiveimpalatester
# kinit -kt <path-to-keytab> <client-principal>
kinit -kt /etc/security/keytabs/hiveimpalatester.keytab hiveimpalatester
#klist

hiveimpala_jdbc_tester.java

//import org.slf4j.Logger;
//import org.slf4j.LoggerFactory;
import java.io.IOException;
import java.util.Properties;
import java.io.StringReader;
import java.sql.*;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.security.UserGroupInformation;
 
public class hiveimpala_jdbc_tester {
  //private static final Logger LOGGER = 
  LoggerFactory.getLogger(hiveimpala_jdbc_tester.class);
  private static String HIVESERVE2DRIVER = "org.apache.hive.jdbc.HiveDriver";
 
  public static void main(String[] args) throws SQLException, IOException {
    String principal = args[0];
    String keytab    = args[1];
    String url       = args[2];
    String fetchSize = args[3];
    String sqlString = args[4];
    int fSize = Integer.parseInt(fetchSize.trim());
 
    // attaching Kerberos conf information: realm and kdc
    System.setProperty("java.security.krb5.conf", "/<path>/krb5.conf");
    //System.setProperty("sun.security.krb5.debug", "true");
 
    // extract serverPrincipal
    Properties p = new Properties();
    p.load(new StringReader(url.replaceAll(";", "\n")));
    String serverPrincipal=p.getProperty("principal");
    Configuration conf = new Configuration();
    conf.set("hadoop.security.authentication", "kerberos");
    UserGroupInformation.setConfiguration(conf);
    UserGroupInformation.loginUserFromKeytab(principal, keytab);
 
    // load the driver
    try {
      Class.forName(HIVESERVE2DRIVER);
    } catch (Exception e) {
      //LOGGER.error("Driver not found");
      System.out.println("Driver not found");
    }
 
    // get connected
    //LOGGER.info("");
    //LOGGER.info("Connecting...");
    System.out.println("");
    System.out.println("Connecting...");
    Connection conn = DriverManager.getConnection(url);
    Statement stmt = conn.createStatement();
    // HIVE only:
    //stmt.execute("SET hive.query.results.cache.enabled=false");
    // set fetch task if needed
    //stmt.execute("SET hive.fetch.task.conversion=more");
    //stmt.execute("SET hive.fetch.task.conversion.threshold=-1");
    stmt.setFetchSize(fSize);
    //LOGGER.info("command: " + sqlString);
    System.out.println("command: " + sqlString);
    //LOGGER.info("Executing...");
    System.out.println("Executing...");
    ResultSet res = stmt.executeQuery(sqlString);
    //LOGGER.info("Fetching...");
    System.out.println("Fetching...");
    ResultSetMetaData resMD = res.getMetaData();
    int cols = resMD.getColumnCount();
     while ( res.next() ) {
      for (int i = 1; i <= cols; i++) {
          if (i > 1) System.out.print(",  ");
              String value = res.getString(i);
              System.out.print(value);
          }
          System.out.println("");
    }
    //LOGGER.info("Exiting...");
    System.out.println("Exiting...");
    conn.close();
  }
}

build_java.sh

javac -d . hiveimpala_jdbc_tester.java -cp "/<path>/jars/*:."

set permissions & execute build_java.sh

chmod +x ./build_java.sh
./build_java.sh

run_tester.sh

# DEBUG: Prints all Hadoop and Hive configuration variables
#set -v
 
# Number of rows that should be fetched from the database
FETCHSIZE="10000"
 
# DEVE/TEST: For secure cluster with Kerberos authentication and TLS enabled; Here we allow the SSL certificate used by the server to be self-signed, by sending the AllowSelfSignedCerts property=1
#HS2="jdbc:hive2://<hs2_node>:10000/;ssl=true;AllowSelfSignedCerts=1;principal=hive/_HOST@<kerberos_realm>;"
#HS2="jdbc:hive2://<impala-load-balancer>:21050/;ssl=true;AllowSelfSignedCerts=1;principal=impala/_HOST@<kerberos_realm>;"
 
# PROD: For secure cluster with Kerberos authentication and TLS enabled; here we allow the SSL certificate used by the server to be signed by a Certificate Authority* (CA)
# To import the certificate to a truststore on a client in two steps as follows:
# Copy /opt/cloudera/CMCA/trust-store/cm-auto-global_cacerts.pem from CM server to localhost
# Run the following command to import the certificate to the truststore:
# keytool -import -keystore /tmp/certs.jks -alias autotls -file /<path>/certs/cm-auto-global_cacerts.pem -keypass changeit -storepass changeit -noprompt
# Certificate was added to keystore
#HS2="jdbc:hive2://<hs2_node>:10000/;ssl=true;sslTrustStore=/<path>/certs.jks;trustStorePassword=changeit;principal=impala/_HOST@<kerberos_realm>;"
HS2="jdbc:hive2://<impala-load-balancer>:21050/;ssl=true;sslTrustStore=/<path>/certs.jks;trustStorePassword=changeit;principal=impala/_HOST@<kerberos_realm>;"
 
KRB_PRINC="hiveimpalatester@<kerberos_realm>"
 
KRB_KEYTB="/etc/security/keytabs/hiveimpalatester.keytab"
 
time java -cp "/<path>/jars/*:." \
hiveimpala_jdbc_tester "${KRB_PRINC}" "${KRB_KEYTB}" "${HS2};fetchSize=${FETCHSIZE}" "${FETCHSIZE}" \
"select * from <db>.<table> limit 1" 2>&1

set permissions & execute run_tester.sh

chmod +x ./run_tester.sh
./run_tester.sh

Advertisement