Java Code Examples for org.apache.hadoop.hive.metastore.api.Function

The following examples show how to use org.apache.hadoop.hive.metastore.api.Function. These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
public static Function convertFunction(final String dbName,
                                       final com.amazonaws.services.glue.model.UserDefinedFunction catalogFunction) {
  if (catalogFunction ==  null) {
    return null;
  }
  Function hiveFunction = new Function();
  hiveFunction.setClassName(catalogFunction.getClassName());
  hiveFunction.setCreateTime((int)(catalogFunction.getCreateTime().getTime() / 1000));
  hiveFunction.setDbName(dbName);
  hiveFunction.setFunctionName(catalogFunction.getFunctionName());
  hiveFunction.setFunctionType(FunctionType.JAVA);
  hiveFunction.setOwnerName(catalogFunction.getOwnerName());
  hiveFunction.setOwnerType(convertPrincipalType(com.amazonaws.services.glue.model.PrincipalType.fromValue(catalogFunction.getOwnerType())));
  hiveFunction.setResourceUris(convertResourceUriList(catalogFunction.getResourceUris()));
  return hiveFunction;
}
 
Example 2
Source Project: flink   Source File: HiveShimV1.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public Function getFunction(IMetaStoreClient client, String dbName, String functionName) throws NoSuchObjectException, TException {
	try {
		// hive-1.x doesn't throw NoSuchObjectException if function doesn't exist, instead it throws a MetaException
		return client.getFunction(dbName, functionName);
	} catch (MetaException e) {
		// need to check the cause and message of this MetaException to decide whether it should actually be a NoSuchObjectException
		if (e.getCause() instanceof NoSuchObjectException) {
			throw (NoSuchObjectException) e.getCause();
		}
		if (e.getMessage().startsWith(NoSuchObjectException.class.getSimpleName())) {
			throw new NoSuchObjectException(e.getMessage());
		}
		throw e;
	}
}
 
Example 3
Source Project: flink   Source File: HiveCatalog.java    License: Apache License 2.0 6 votes vote down vote up
private static Function instantiateHiveFunction(ObjectPath functionPath, CatalogFunction function) {

		boolean isGeneric = Boolean.valueOf(function.getProperties().get(CatalogConfig.IS_GENERIC));

		// Hive Function does not have properties map
		// thus, use a prefix in class name to distinguish Flink and Hive functions
		String functionClassName = isGeneric ?
			FLINK_FUNCTION_PREFIX + function.getClassName() :
			function.getClassName();

		return new Function(
			// due to https://issues.apache.org/jira/browse/HIVE-22053, we have to normalize function name ourselves
			HiveStringUtils.normalizeIdentifier(functionPath.getObjectName()),
			functionPath.getDatabaseName(),
			functionClassName,
			null,			// Owner name
			PrincipalType.GROUP,	// Temporarily set to GROUP type because it's required by Hive. May change later
			(int) (System.currentTimeMillis() / 1000),
			FunctionType.JAVA,		// FunctionType only has JAVA now
			new ArrayList<>()		// Resource URIs
		);
	}
 
Example 4
public static com.amazonaws.services.glue.model.UserDefinedFunction convertFunction(final Function hiveFunction) {
  if (hiveFunction == null ){
    return null;
  }
  com.amazonaws.services.glue.model.UserDefinedFunction catalogFunction = new com.amazonaws.services.glue.model.UserDefinedFunction();
  catalogFunction.setClassName(hiveFunction.getClassName());
  catalogFunction.setFunctionName(hiveFunction.getFunctionName());
  catalogFunction.setCreateTime(new Date((long) (hiveFunction.getCreateTime()) * 1000));
  catalogFunction.setOwnerName(hiveFunction.getOwnerName());
  if(hiveFunction.getOwnerType() != null) {
    catalogFunction.setOwnerType(hiveFunction.getOwnerType().name());
  }
  catalogFunction.setResourceUris(covertResourceUriList(hiveFunction.getResourceUris()));
  return catalogFunction;
}
 
Example 5
public static UserDefinedFunctionInput convertToUserDefinedFunctionInput(Function hiveFunction) {
  UserDefinedFunctionInput functionInput = new UserDefinedFunctionInput();

  functionInput.setClassName(hiveFunction.getClassName());
  functionInput.setFunctionName(hiveFunction.getFunctionName());
  functionInput.setOwnerName(hiveFunction.getOwnerName());
  if(hiveFunction.getOwnerType() != null) {
    functionInput.setOwnerType(hiveFunction.getOwnerType().name());
  }
  functionInput.setResourceUris(HiveToCatalogConverter.covertResourceUriList(hiveFunction.getResourceUris()));
  return functionInput;
}
 
Example 6
@Test
public void testGetFunctionValid() throws Exception {
  when(glueClient.getUserDefinedFunction(any(GetUserDefinedFunctionRequest.class)))
          .thenReturn(new GetUserDefinedFunctionResult().withUserDefinedFunction(
              HiveToCatalogConverter.convertFunction(testFunction)));
  Function result = metastoreClient.getFunction(testDB.getName(), testFunction.getFunctionName());
  assertEquals(testFunction, result);
}
 
Example 7
@Test
public void testListFunctionsValid() throws Exception {
  List<Function> functions = Lists.newArrayList(testFunction);
  when(glueClient.getUserDefinedFunctions(any(GetUserDefinedFunctionsRequest.class)))
          .thenReturn(new GetUserDefinedFunctionsResult().withUserDefinedFunctions(
              HiveToCatalogConverter.convertFunction(testFunction)));
  List<String> result = metastoreClient.getFunctions(testDB.getName(), ".*");
  assertEquals(functions.size(), result.size());
  assertEquals(functions.get(0).getFunctionName(), result.get(0));
}
 
Example 8
Source Project: waggle-dance   Source File: WaggleDanceIntegrationTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void typicalGetAllFunctions() throws Exception {
  runner = WaggleDanceRunner
      .builder(configLocation)
      .databaseResolution(DatabaseResolution.PREFIXED)
      .primary("primary", localServer.getThriftConnectionUri(), READ_ONLY)
      .withPrimaryPrefix("primary_")
      .federate(SECONDARY_METASTORE_NAME, remoteServer.getThriftConnectionUri(), REMOTE_DATABASE)
      .build();

  runWaggleDance(runner);
  HiveMetaStoreClient proxy = getWaggleDanceClient();
  List<ResourceUri> resourceUris = Lists
      .newArrayList(new ResourceUri(ResourceType.JAR, "hdfs://path/to/my/jar/my.jar"));
  Function localFunction = new Function("fn1", LOCAL_DATABASE, "com.hotels.hive.FN1", "hadoop", PrincipalType.USER, 0,
      FunctionType.JAVA, resourceUris);
  localServer.client().createFunction(localFunction);
  Function remoteFunction = new Function("fn2", REMOTE_DATABASE, "com.hotels.hive.FN1", "hadoop", PrincipalType.USER,
      0, FunctionType.JAVA, resourceUris);
  remoteServer.client().createFunction(remoteFunction);

  GetAllFunctionsResponse allFunctions = proxy.getAllFunctions();
  List<Function> functions = allFunctions.getFunctions();
  assertThat(functions.size(), is(3));
  assertThat(functions.get(0).getFunctionName(), is("fn1"));
  assertThat(functions.get(0).getDbName(), is("primary_" + LOCAL_DATABASE));
  assertThat(functions.get(1).getFunctionName(), is("fn1"));
  assertThat(functions.get(1).getDbName(), is(LOCAL_DATABASE));
  assertThat(functions.get(2).getFunctionName(), is("fn2"));
  assertThat(functions.get(2).getDbName(), is(PREFIXED_REMOTE_DATABASE));
}
 
Example 9
Source Project: waggle-dance   Source File: GetAllFunctionsRequest.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public List<GetAllFunctionsResponse> call() throws Exception {
  GetAllFunctionsResponse response = mapping.getClient().get_all_functions();
  if (response.getFunctions() != null) {
    for (Function function : response.getFunctions()) {
      mapping.transformOutboundFunction(function);
    }
  }
  return Collections.singletonList(response);
}
 
Example 10
Source Project: waggle-dance   Source File: FederatedHMSHandler.java    License: Apache License 2.0 5 votes vote down vote up
@Override
@Loggable(value = Loggable.DEBUG, skipResult = true, name = INVOCATION_LOG_NAME)
public void create_function(Function func)
    throws AlreadyExistsException, InvalidObjectException, MetaException, NoSuchObjectException, TException {
  DatabaseMapping mapping = checkWritePermissions(func.getDbName());
  mapping.getClient().create_function(mapping.transformInboundFunction(func));
}
 
Example 11
Source Project: waggle-dance   Source File: FederatedHMSHandler.java    License: Apache License 2.0 5 votes vote down vote up
@Override
@Loggable(value = Loggable.DEBUG, skipResult = true, name = INVOCATION_LOG_NAME)
public void alter_function(String dbName, String funcName, Function newFunc)
    throws InvalidOperationException, MetaException, TException {
  DatabaseMapping mapping = checkWritePermissions(dbName);
  mapping.checkWritePermissions(newFunc.getDbName());
  mapping
      .getClient()
      .alter_function(mapping.transformInboundDatabaseName(dbName), funcName,
          mapping.transformInboundFunction(newFunc));
}
 
Example 12
Source Project: waggle-dance   Source File: FederatedHMSHandler.java    License: Apache License 2.0 5 votes vote down vote up
@Override
@Loggable(value = Loggable.DEBUG, skipResult = true, name = INVOCATION_LOG_NAME)
public Function get_function(String dbName, String funcName) throws MetaException, NoSuchObjectException, TException {
  DatabaseMapping mapping = databaseMappingService.databaseMapping(dbName);
  return mapping
      .transformOutboundFunction(
          mapping.getClient().get_function(mapping.transformInboundDatabaseName(dbName), funcName));
}
 
Example 13
Source Project: waggle-dance   Source File: DatabaseMappingImplTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void transformOutboundFunction() throws Exception {
  Function function = new Function();
  function.setDbName(DB_NAME);
  Function result = databaseMapping.transformOutboundFunction(function);
  assertThat(result, is(sameInstance(function)));
  assertThat(result.getDbName(), is(OUT_DB_NAME));
}
 
Example 14
Source Project: waggle-dance   Source File: DatabaseMappingImplTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void transformInboundFunction() throws Exception {
  Function function = new Function();
  function.setDbName(DB_NAME);
  Function result = databaseMapping.transformInboundFunction(function);
  assertThat(result, is(sameInstance(function)));
  assertThat(result.getDbName(), is(IN_DB_NAME));
}
 
Example 15
Source Project: waggle-dance   Source File: GetAllFunctionsRequestTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void call() throws Exception {
  GetAllFunctionsRequest request = new GetAllFunctionsRequest(mapping);
  GetAllFunctionsResponse response = new GetAllFunctionsResponse();
  Function function = newFunction("db", "fn1");
  response.addToFunctions(function);

  when(mapping.getClient()).thenReturn(client);
  when(mapping.getClient().get_all_functions()).thenReturn(response);
  List<GetAllFunctionsResponse> responses = request.call();
  assertThat(responses.size(), is(1));
  assertThat(responses.get(0), is(response));
  verify(mapping).transformOutboundFunction(function);
}
 
Example 16
Source Project: waggle-dance   Source File: GetAllFunctionsRequestTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void callFunctionsNotSet() throws Exception {
  GetAllFunctionsRequest request = new GetAllFunctionsRequest(mapping);
  GetAllFunctionsResponse response = new GetAllFunctionsResponse();

  when(mapping.getClient()).thenReturn(client);
  when(mapping.getClient().get_all_functions()).thenReturn(response);
  List<GetAllFunctionsResponse> responses = request.call();
  assertThat(responses.size(), is(1));
  assertThat(responses.get(0), is(response));
  verify(mapping, never()).transformOutboundFunction(any(Function.class));
}
 
Example 17
Source Project: waggle-dance   Source File: GetAllFunctionsRequestTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void callFunctionsEmpty() throws Exception {
  // Not 100% sure if this actually can happen but we cover it anyway.
  GetAllFunctionsRequest request = new GetAllFunctionsRequest(mapping);
  GetAllFunctionsResponse response = new GetAllFunctionsResponse();
  response.setFunctions(Collections.emptyList());

  when(mapping.getClient()).thenReturn(client);
  when(mapping.getClient().get_all_functions()).thenReturn(response);
  List<GetAllFunctionsResponse> responses = request.call();
  assertThat(responses.size(), is(1));
  assertThat(responses.get(0), is(response));
  verify(mapping, never()).transformOutboundFunction(any(Function.class));
}
 
Example 18
Source Project: waggle-dance   Source File: HiveStubs.java    License: Apache License 2.0 5 votes vote down vote up
public static Function newFunction(String databaseName, String functionName) {
  List<ResourceUri> resourceUris = Lists
      .newArrayList(new ResourceUri(ResourceType.JAR, "hdfs://path/to/my/jar/my.jar"));
  Function function = new Function(functionName, databaseName, "com.hotels.hive.FN", "hadoop", PrincipalType.USER, 0,
      FunctionType.JAVA, resourceUris);
  return function;
}
 
Example 19
Source Project: waggle-dance   Source File: FederatedHMSHandlerTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void create_function() throws TException {
  Function function = new Function();
  function.setDbName(DB_P);
  Function inboundFunction = new Function();
  when(primaryMapping.transformInboundFunction(function)).thenReturn(inboundFunction);
  handler.create_function(function);
  verify(primaryMapping).checkWritePermissions(DB_P);
  verify(primaryClient).create_function(inboundFunction);
}
 
Example 20
Source Project: waggle-dance   Source File: FederatedHMSHandlerTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void alter_function() throws TException {
  Function newFunc = new Function();
  newFunc.setDbName(DB_P);
  Function inboundFunction = new Function();
  when(primaryMapping.transformInboundFunction(newFunc)).thenReturn(inboundFunction);
  handler.alter_function(DB_P, "function", newFunc);
  verify(primaryMapping, times(2)).checkWritePermissions(DB_P);
  verify(primaryClient).alter_function(DB_P, "function", inboundFunction);
}
 
Example 21
Source Project: waggle-dance   Source File: FederatedHMSHandlerTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void get_function() throws TException {
  Function fromClient = new Function();
  Function expected = new Function();
  when(primaryClient.get_function(DB_P, "funcName")).thenReturn(fromClient);
  when(primaryMapping.transformOutboundFunction(fromClient)).thenReturn(expected);
  Function result = handler.get_function(DB_P, "funcName");
  assertThat(result, is(expected));
}
 
Example 22
Source Project: flink   Source File: HiveMetastoreClientWrapper.java    License: Apache License 2.0 5 votes vote down vote up
public Function getFunction(String databaseName, String functionName) throws MetaException, TException {
	try {
		// Hive may not throw NoSuchObjectException if function doesn't exist, instead it throws a MetaException
		return client.getFunction(databaseName, functionName);
	} catch (MetaException e) {
		// need to check the cause and message of this MetaException to decide whether it should actually be a NoSuchObjectException
		if (e.getCause() instanceof NoSuchObjectException) {
			throw (NoSuchObjectException) e.getCause();
		}
		if (e.getMessage().startsWith(NoSuchObjectException.class.getSimpleName())) {
			throw new NoSuchObjectException(e.getMessage());
		}
		throw e;
	}
}
 
Example 23
Source Project: flink   Source File: HiveCatalog.java    License: Apache License 2.0 5 votes vote down vote up
private static Function instantiateHiveFunction(ObjectPath functionPath, CatalogFunction function) {

		boolean isGeneric = function.isGeneric();

		// Hive Function does not have properties map
		// thus, use a prefix in class name to distinguish Flink and Hive functions
		String functionClassName;
		if (function.getFunctionLanguage().equals(FunctionLanguage.JAVA)) {
			functionClassName = isGeneric ?
				FLINK_FUNCTION_PREFIX + function.getClassName() :
				function.getClassName();
		} else if (function.getFunctionLanguage().equals(FunctionLanguage.PYTHON)) {
			functionClassName = FLINK_PYTHON_FUNCTION_PREFIX + function.getClassName();
		} else {
			throw new UnsupportedOperationException("HiveCatalog supports only creating" +
				" JAVA or PYTHON based function for now");
		}

		return new Function(
			// due to https://issues.apache.org/jira/browse/HIVE-22053, we have to normalize function name ourselves
			functionPath.getObjectName().trim().toLowerCase(),
			functionPath.getDatabaseName(),
			functionClassName,
			null,			// Owner name
			PrincipalType.GROUP,	// Temporarily set to GROUP type because it's required by Hive. May change later
			(int) (System.currentTimeMillis() / 1000),
			FunctionType.JAVA,		// FunctionType only has JAVA now
			new ArrayList<>()		// Resource URIs
		);
	}
 
Example 24
Source Project: flink   Source File: HiveMetastoreClientWrapper.java    License: Apache License 2.0 4 votes vote down vote up
public void createFunction(Function function) throws InvalidObjectException, MetaException, TException {
	client.createFunction(function);
}
 
Example 25
Source Project: flink   Source File: HiveMetastoreClientWrapper.java    License: Apache License 2.0 4 votes vote down vote up
public void alterFunction(String databaseName, String functionName, Function function)
		throws InvalidObjectException, MetaException, TException {
	client.alterFunction(databaseName, functionName, function);
}
 
Example 26
Source Project: flink   Source File: HiveMetastoreClientWrapper.java    License: Apache License 2.0 4 votes vote down vote up
public Function getFunction(String databaseName, String functionName) throws MetaException, TException {
	HiveShim hiveShim = HiveShimLoader.loadHiveShim(hiveVersion);
	return hiveShim.getFunction(client, databaseName, functionName);
}
 
Example 27
Source Project: flink   Source File: HiveShimV2.java    License: Apache License 2.0 4 votes vote down vote up
@Override
public Function getFunction(IMetaStoreClient client, String dbName, String functionName) throws NoSuchObjectException, TException {
	return client.getFunction(dbName, functionName);
}
 
Example 28
Source Project: waggle-dance   Source File: DatabaseMappingImpl.java    License: Apache License 2.0 4 votes vote down vote up
@Override
public Function transformOutboundFunction(Function function) {
  function.setDbName(metaStoreMapping.transformOutboundDatabaseName(function.getDbName()));
  return function;
}
 
Example 29
Source Project: waggle-dance   Source File: DatabaseMappingImpl.java    License: Apache License 2.0 4 votes vote down vote up
@Override
public Function transformInboundFunction(Function function) {
  function.setDbName(metaStoreMapping.transformInboundDatabaseName(function.getDbName()));
  return function;
}
 
Example 30
Source Project: waggle-dance   Source File: IdentityMapping.java    License: Apache License 2.0 4 votes vote down vote up
@Override
public Function transformOutboundFunction(Function function) {
  return function;
}