org.apache.hadoop.hive.ql.hooks.WriteEntity Java Examples

The following examples show how to use org.apache.hadoop.hive.ql.hooks.WriteEntity. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: SentryHiveAuthorizationTaskFactoryImpl.java    From incubator-sentry with Apache License 2.0 6 votes vote down vote up
@Override
public Task<? extends Serializable> createShowRoleGrantTask(ASTNode ast, Path resultFile,
    HashSet<ReadEntity> inputs, HashSet<WriteEntity> outputs) throws SemanticException {
  ASTNode child = (ASTNode) ast.getChild(0);
  PrincipalType principalType = PrincipalType.USER;
  switch (child.getType()) {
  case HiveParser.TOK_USER:
    principalType = PrincipalType.USER;
    break;
  case HiveParser.TOK_GROUP:
    principalType = PrincipalType.GROUP;
    break;
  case HiveParser.TOK_ROLE:
    principalType = PrincipalType.ROLE;
    break;
  }
  if (principalType != PrincipalType.GROUP) {
    String msg = SentryHiveConstants.GRANT_REVOKE_NOT_SUPPORTED_FOR_PRINCIPAL + principalType;
    throw new SemanticException(msg);
  }
  String principalName = BaseSemanticAnalyzer.unescapeIdentifier(child.getChild(0).getText());
  RoleDDLDesc roleDesc = new RoleDDLDesc(principalName, principalType,
      RoleDDLDesc.RoleOperation.SHOW_ROLE_GRANT, null);
  roleDesc.setResFile(resultFile.toString());
  return createTask(new DDLWork(inputs, outputs,  roleDesc));
}
 
Example #2
Source File: HiveHookIT.java    From atlas with Apache License 2.0 6 votes vote down vote up
private void assertProcessIsNotRegistered(HiveEventContext event) throws Exception {
    try {
        SortedSet<ReadEntity>  sortedHiveInputs  = event.getInputs() == null ? null : new TreeSet<ReadEntity>(entityComparator);
        SortedSet<WriteEntity> sortedHiveOutputs = event.getOutputs() == null ? null : new TreeSet<WriteEntity>(entityComparator);

        if (event.getInputs() != null) {
            sortedHiveInputs.addAll(event.getInputs());
        }

        if (event.getOutputs() != null) {
            sortedHiveOutputs.addAll(event.getOutputs());
        }

        String processQFName = getProcessQualifiedName(hiveMetaStoreBridge, event, sortedHiveInputs, sortedHiveOutputs, getSortedProcessDataSets(event.getInputs()), getSortedProcessDataSets(event.getOutputs()));

        LOG.debug("Searching for process with query {}", processQFName);

        assertEntityIsNotRegistered(HiveDataTypes.HIVE_PROCESS.getName(), ATTRIBUTE_QUALIFIED_NAME, processQFName);
    } catch(Exception e) {
        LOG.error("Exception : ", e);
    }
}
 
Example #3
Source File: HiveHookIT.java    From incubator-atlas with Apache License 2.0 6 votes vote down vote up
@Test(enabled = false)
public void testInsertIntoTempTable() throws Exception {
    String tableName = createTable();
    String insertTableName = createTable(false, false, true);
    assertTableIsRegistered(DEFAULT_DB, tableName);
    assertTableIsNotRegistered(DEFAULT_DB, insertTableName, true);

    String query =
        "insert into " + insertTableName + " select id, name from " + tableName;

    runCommand(query);

    Set<ReadEntity> inputs = getInputs(tableName, Entity.Type.TABLE);
    Set<WriteEntity> outputs = getOutputs(insertTableName, Entity.Type.TABLE);
    outputs.iterator().next().setName(getQualifiedTblName(insertTableName + HiveMetaStoreBridge.TEMP_TABLE_PREFIX + SessionState.get().getSessionId()));
    outputs.iterator().next().setWriteType(WriteEntity.WriteType.INSERT);

    validateProcess(constructEvent(query,  HiveOperation.QUERY, inputs, outputs));

    assertTableIsRegistered(DEFAULT_DB, tableName);
    assertTableIsRegistered(DEFAULT_DB, insertTableName, null, true);
}
 
Example #4
Source File: HiveHookIT.java    From incubator-atlas with Apache License 2.0 6 votes vote down vote up
@Test
public void testTruncateTable() throws Exception {
    String tableName = createTable(false);
    String query = String.format("truncate table %s", tableName);
    runCommand(query);

    Set<WriteEntity> outputs = getOutputs(tableName, Entity.Type.TABLE);

    String tableId = assertTableIsRegistered(DEFAULT_DB, tableName);
    validateProcess(constructEvent(query, HiveOperation.TRUNCATETABLE, null, outputs));

    //Check lineage
    String datasetName = HiveMetaStoreBridge.getTableQualifiedName(CLUSTER_NAME, DEFAULT_DB, tableName);
    JSONObject response = atlasClient.getInputGraph(datasetName);
    JSONObject vertices = response.getJSONObject("values").getJSONObject("vertices");
    //Below should be assertTrue - Fix https://issues.apache.org/jira/browse/ATLAS-653
    Assert.assertFalse(vertices.has(tableId));
}
 
Example #5
Source File: HiveITBase.java    From atlas with Apache License 2.0 6 votes vote down vote up
protected static boolean addQueryType(HiveOperation op, WriteEntity entity) {
    if (entity.getWriteType() != null && HiveOperation.QUERY.equals(op)) {
        switch (entity.getWriteType()) {
            case INSERT:
            case INSERT_OVERWRITE:
            case UPDATE:
            case DELETE:
                return true;
            case PATH_WRITE:
                //Add query type only for DFS paths and ignore local paths since they are not added as outputs
                if ( !Entity.Type.LOCAL_DIR.equals(entity.getType())) {
                    return true;
                }
                break;
            default:
        }
    }
    return false;
}
 
Example #6
Source File: HiveHookIT.java    From incubator-atlas with Apache License 2.0 6 votes vote down vote up
private String assertProcessIsRegistered(final HiveHook.HiveEventContext event, final Set<ReadEntity> inputTbls, final Set<WriteEntity> outputTbls) throws Exception {
    try {
        SortedSet<ReadEntity> sortedHiveInputs = event.getInputs() == null ? null : new TreeSet<ReadEntity>(entityComparator);
        SortedSet<WriteEntity> sortedHiveOutputs = event.getOutputs() == null ? null : new TreeSet<WriteEntity>(entityComparator);
        if ( event.getInputs() != null) {
            sortedHiveInputs.addAll(event.getInputs());
        }
        if ( event.getOutputs() != null) {
            sortedHiveOutputs.addAll(event.getOutputs());
        }
        String processQFName = getProcessQualifiedName(hiveMetaStoreBridge, event, sortedHiveInputs, sortedHiveOutputs, getSortedProcessDataSets(inputTbls), getSortedProcessDataSets(outputTbls));
        LOG.debug("Searching for process with query {}", processQFName);
        return assertEntityIsRegistered(HiveDataTypes.HIVE_PROCESS.getName(), AtlasClient.REFERENCEABLE_ATTRIBUTE_NAME, processQFName, new AssertPredicate() {
            @Override
            public void assertOnEntity(final Referenceable entity) throws Exception {
                List<String> recentQueries = (List<String>) entity.get("recentQueries");
                Assert.assertEquals(recentQueries.get(0), lower(event.getQueryStr()));
            }
        });
    } catch(Exception e) {
        LOG.error("Exception : ", e);
        throw e;
    }
}
 
Example #7
Source File: HiveHookIT.java    From incubator-atlas with Apache License 2.0 6 votes vote down vote up
private void assertProcessIsNotRegistered(HiveHook.HiveEventContext event) throws Exception {
    try {
        SortedSet<ReadEntity> sortedHiveInputs = event.getInputs() == null ? null : new TreeSet<ReadEntity>(entityComparator);
        SortedSet<WriteEntity> sortedHiveOutputs = event.getOutputs() == null ? null : new TreeSet<WriteEntity>(entityComparator);
        if ( event.getInputs() != null) {
            sortedHiveInputs.addAll(event.getInputs());
        }
        if ( event.getOutputs() != null) {
            sortedHiveOutputs.addAll(event.getOutputs());
        }
        String processQFName = getProcessQualifiedName(hiveMetaStoreBridge, event, sortedHiveInputs, sortedHiveOutputs, getSortedProcessDataSets(event.getInputs()), getSortedProcessDataSets(event.getOutputs()));
        LOG.debug("Searching for process with query {}", processQFName);
        assertEntityIsNotRegistered(HiveDataTypes.HIVE_PROCESS.getName(), AtlasClient.REFERENCEABLE_ATTRIBUTE_NAME, processQFName);
    } catch( Exception e) {
        LOG.error("Exception : ", e);
    }
}
 
Example #8
Source File: SentryOnFailureHookContextImpl.java    From incubator-sentry with Apache License 2.0 6 votes vote down vote up
public SentryOnFailureHookContextImpl(String command,
    Set<ReadEntity> inputs, Set<WriteEntity> outputs, HiveOperation hiveOp,
    Database db, Table tab, AccessURI udfURI, AccessURI partitionURI,
    String userName, String ipAddress, AuthorizationException e,
    Configuration conf) {
  this.command = command;
  this.inputs = inputs;
  this.outputs = outputs;
  this.hiveOp = hiveOp;
  this.userName = userName;
  this.ipAddress = ipAddress;
  this.database = db;
  this.table = tab;
  this.udfURI = udfURI;
  this.partitionURI = partitionURI;
  this.authException = e;
  this.conf = conf;
}
 
Example #9
Source File: HiveHookIT.java    From incubator-atlas with Apache License 2.0 6 votes vote down vote up
private Referenceable validateProcess(HiveHook.HiveEventContext event, Set<ReadEntity> inputTables, Set<WriteEntity> outputTables) throws Exception {
    String processId = assertProcessIsRegistered(event, inputTables, outputTables);
    Referenceable process = atlasClient.getEntity(processId);
    if (inputTables == null) {
        Assert.assertNull(process.get(INPUTS));
    } else {
        Assert.assertEquals(((List<Referenceable>) process.get(INPUTS)).size(), inputTables.size());
        validateInputTables(process, inputTables);
    }

    if (outputTables == null) {
        Assert.assertNull(process.get(OUTPUTS));
    } else {
        Assert.assertEquals(((List<Id>) process.get(OUTPUTS)).size(), outputTables.size());
        validateOutputTables(process, outputTables);
    }

    return process;
}
 
Example #10
Source File: HiveHookIT.java    From incubator-atlas with Apache License 2.0 6 votes vote down vote up
private Set<WriteEntity> getOutputs(String inputName, Entity.Type entityType) throws HiveException {
    final WriteEntity entity = new WriteEntity();

    if ( Entity.Type.DFS_DIR.equals(entityType) || Entity.Type.LOCAL_DIR.equals(entityType)) {
        entity.setName(lower(new Path(inputName).toString()));
        entity.setTyp(entityType);
    } else {
        entity.setName(getQualifiedTblName(inputName));
        entity.setTyp(entityType);
    }

    if (entityType == Entity.Type.TABLE) {
        entity.setT(hiveMetaStoreBridge.hiveClient.getTable(DEFAULT_DB, inputName));
    }
    return new LinkedHashSet<WriteEntity>() {{ add(entity); }};
}
 
Example #11
Source File: HiveHook.java    From incubator-atlas with Apache License 2.0 6 votes vote down vote up
private void deleteDatabase(HiveMetaStoreBridge dgiBridge, HiveEventContext event) {
    if (event.getOutputs().size() > 1) {
        LOG.info("Starting deletion of tables and databases with cascade {} ", event.getQueryStr());
    } else {
        LOG.info("Starting deletion of database {} ", event.getQueryStr());
    }

    for (WriteEntity output : event.getOutputs()) {
        if (Type.TABLE.equals(output.getType())) {
            deleteTable(dgiBridge, event, output);
        } else if (Type.DATABASE.equals(output.getType())) {
            final String dbQualifiedName = HiveMetaStoreBridge.getDBQualifiedName(dgiBridge.getClusterName(), output.getDatabase().getName());
            event.addMessage(
                new HookNotification.EntityDeleteRequest(event.getUser(),
                    HiveDataTypes.HIVE_DB.getName(),
                    AtlasClient.REFERENCEABLE_ATTRIBUTE_NAME,
                    dbQualifiedName));
        }
    }
}
 
Example #12
Source File: SentryHiveAuthorizationTaskFactoryImpl.java    From incubator-sentry with Apache License 2.0 6 votes vote down vote up
@Override
public Task<? extends Serializable> createRevokeTask(ASTNode ast, HashSet<ReadEntity> inputs,
    HashSet<WriteEntity> outputs) throws SemanticException {
  List<PrivilegeDesc> privilegeDesc = analyzePrivilegeListDef((ASTNode) ast.getChild(0));
  List<PrincipalDesc> principalDesc = analyzePrincipalListDef((ASTNode) ast.getChild(1));
  PrivilegeObjectDesc privilegeObj = null;
  if (ast.getChildCount() > 2) {
    ASTNode astChild = (ASTNode) ast.getChild(2);
    privilegeObj = analyzePrivilegeObject(astChild);
  }
  if (privilegeObj != null && privilegeObj.getPartSpec() != null) {
    throw new SemanticException(SentryHiveConstants.PARTITION_PRIVS_NOT_SUPPORTED);
  }
  for (PrincipalDesc princ : principalDesc) {
    if (princ.getType() != PrincipalType.ROLE) {
      String msg = SentryHiveConstants.GRANT_REVOKE_NOT_SUPPORTED_FOR_PRINCIPAL + princ.getType();
      throw new SemanticException(msg);
    }
  }
  RevokeDesc revokeDesc = new RevokeDesc(privilegeDesc, principalDesc, privilegeObj);
  return createTask(new DDLWork(inputs, outputs, revokeDesc));
}
 
Example #13
Source File: HiveHook.java    From incubator-atlas with Apache License 2.0 6 votes vote down vote up
private boolean isSelectQuery(HiveEventContext event) {
    if (event.getOperation() == HiveOperation.QUERY) {
        //Select query has only one output
        if (event.getOutputs().size() == 1) {
            WriteEntity output = event.getOutputs().iterator().next();
            /* Strangely select queries have DFS_DIR as the type which seems like a bug in hive. Filter out by checking if the path is a temporary URI
             * Insert into/overwrite queries onto local or dfs paths have DFS_DIR or LOCAL_DIR as the type and WriteType.PATH_WRITE and tempUri = false
             * Insert into a temporary table has isTempURI = false. So will not skip as expected
             */
            if (output.getType() == Type.DFS_DIR || output.getType() == Type.LOCAL_DIR) {
                if (output.getWriteType() == WriteEntity.WriteType.PATH_WRITE &&
                    output.isTempURI()) {
                    return true;
                }
            }
        }
    }
    return false;
}
 
Example #14
Source File: SentryHiveAuthorizationTaskFactoryImpl.java    From incubator-sentry with Apache License 2.0 6 votes vote down vote up
private Task<? extends Serializable> analyzeGrantRevokeRole(boolean isGrant, ASTNode ast,
    HashSet<ReadEntity> inputs, HashSet<WriteEntity> outputs) throws SemanticException {
  List<PrincipalDesc> principalDesc = analyzePrincipalListDef(
      (ASTNode) ast.getChild(0));

  List<String> roles = new ArrayList<String>();
  for (int i = 1; i < ast.getChildCount(); i++) {
    roles.add(BaseSemanticAnalyzer.unescapeIdentifier(ast.getChild(i).getText()));
  }
  String roleOwnerName = "";
  if (SessionState.get() != null
      && SessionState.get().getAuthenticator() != null) {
    roleOwnerName = SessionState.get().getAuthenticator().getUserName();
  }
  for (PrincipalDesc princ : principalDesc) {
    if (princ.getType() != PrincipalType.GROUP) {
      String msg = SentryHiveConstants.GRANT_REVOKE_NOT_SUPPORTED_ON_OBJECT + princ.getType();
      throw new SemanticException(msg);
    }
  }
  GrantRevokeRoleDDL grantRevokeRoleDDL = new GrantRevokeRoleDDL(isGrant,
      roles, principalDesc, roleOwnerName, PrincipalType.USER, false);
  return createTask(new DDLWork(inputs, outputs, grantRevokeRoleDDL));
}
 
Example #15
Source File: HiveHook.java    From incubator-atlas with Apache License 2.0 6 votes vote down vote up
private static boolean addQueryType(HiveOperation op, WriteEntity entity) {
    if (entity.getWriteType() != null && HiveOperation.QUERY.equals(op)) {
        switch (entity.getWriteType()) {
        case INSERT:
        case INSERT_OVERWRITE:
        case UPDATE:
        case DELETE:
            return true;
        case PATH_WRITE:
            //Add query type only for DFS paths and ignore local paths since they are not added as outputs
            if ( !Type.LOCAL_DIR.equals(entity.getType())) {
                return true;
            }
            break;
        default:
        }
    }
    return false;
}
 
Example #16
Source File: HiveHookIT.java    From atlas with Apache License 2.0 6 votes vote down vote up
@Test
public void testCTAS() throws Exception {
    String tableName     = createTable();
    String ctasTableName = "table" + random();
    String query         = "create table " + ctasTableName + " as select * from " + tableName;

    runCommand(query);

    final Set<ReadEntity> readEntities = getInputs(tableName, Entity.Type.TABLE);
    final Set<WriteEntity> writeEntities = getOutputs(ctasTableName, Entity.Type.TABLE);

    HiveEventContext hiveEventContext = constructEvent(query, HiveOperation.CREATETABLE_AS_SELECT, readEntities,
            writeEntities);
    AtlasEntity processEntity1 = validateProcess(hiveEventContext);
    AtlasEntity processExecutionEntity1 = validateProcessExecution(processEntity1, hiveEventContext);
    AtlasObjectId process = toAtlasObjectId(processExecutionEntity1.getRelationshipAttribute(
            BaseHiveEvent.ATTRIBUTE_PROCESS));
    Assert.assertEquals(process.getGuid(), processEntity1.getGuid());

    Assert.assertEquals(numberOfProcessExecutions(processEntity1), 1);
    assertTableIsRegistered(DEFAULT_DB, ctasTableName);
}
 
Example #17
Source File: SentryHiveAuthorizationTaskFactoryImpl.java    From incubator-sentry with Apache License 2.0 6 votes vote down vote up
@Override
public Task<? extends Serializable> createShowRolePrincipalsTask(ASTNode ast, Path resFile,
    HashSet<ReadEntity> inputs, HashSet<WriteEntity> outputs) throws SemanticException {
  String roleName;

  if (ast.getChildCount() == 1) {
    roleName = ast.getChild(0).getText();
  } else {
    // the parser should not allow this
    throw new AssertionError("Unexpected Tokens in SHOW ROLE PRINCIPALS");
  }

  RoleDDLDesc roleDDLDesc = new RoleDDLDesc(roleName, PrincipalType.ROLE,
   RoleDDLDesc.RoleOperation.SHOW_ROLE_PRINCIPALS, null);
  roleDDLDesc.setResFile(resFile.toString());
  return createTask(new DDLWork(inputs, outputs, roleDDLDesc));
  //return TaskFactory.get(new DDLWork(inputs, outputs, roleDDLDesc), conf);
}
 
Example #18
Source File: SentryHiveAuthorizationTaskFactoryImpl.java    From incubator-sentry with Apache License 2.0 5 votes vote down vote up
@Override
public Task<? extends Serializable> createShowGrantTask(ASTNode ast, Path resultFile, HashSet<ReadEntity> inputs,
    HashSet<WriteEntity> outputs) throws SemanticException {
  SentryHivePrivilegeObjectDesc privHiveObj = null;

  ASTNode principal = (ASTNode) ast.getChild(0);
  PrincipalType type = PrincipalType.USER;
  switch (principal.getType()) {
  case HiveParser.TOK_USER:
    type = PrincipalType.USER;
    break;
  case HiveParser.TOK_GROUP:
    type = PrincipalType.GROUP;
    break;
  case HiveParser.TOK_ROLE:
    type = PrincipalType.ROLE;
    break;
  }
  if (type != PrincipalType.ROLE) {
    String msg = SentryHiveConstants.GRANT_REVOKE_NOT_SUPPORTED_FOR_PRINCIPAL + type;
    throw new SemanticException(msg);
  }
  String principalName = BaseSemanticAnalyzer.unescapeIdentifier(principal.getChild(0).getText());
  PrincipalDesc principalDesc = new PrincipalDesc(principalName, type);

  // Partition privileges are not supported by Sentry
  if (ast.getChildCount() > 1) {
    ASTNode child = (ASTNode) ast.getChild(1);
    if (child.getToken().getType() == HiveParser.TOK_PRIV_OBJECT_COL) {
      privHiveObj = analyzePrivilegeObject(child);
    } else {
      throw new SemanticException("Unrecognized Token: " + child.getToken().getType());
    }
  }

  ShowGrantDesc showGrant = new ShowGrantDesc(resultFile.toString(),
      principalDesc, privHiveObj);
  return createTask(new DDLWork(inputs, outputs, showGrant));
}
 
Example #19
Source File: SentryHiveAuthorizationTaskFactoryImpl.java    From incubator-sentry with Apache License 2.0 5 votes vote down vote up
@Override
public Task<? extends Serializable> createShowRolesTask(ASTNode ast, Path resFile,
    HashSet<ReadEntity> inputs, HashSet<WriteEntity> outputs) throws SemanticException {
  RoleDDLDesc showRolesDesc = new RoleDDLDesc(null, null, RoleDDLDesc.RoleOperation.SHOW_ROLES,
      null);
  showRolesDesc.setResFile(resFile.toString());
  return createTask(new DDLWork(inputs, outputs, showRolesDesc));
}
 
Example #20
Source File: HiveAuthzBindingHook.java    From incubator-sentry with Apache License 2.0 5 votes vote down vote up
private boolean filterWriteEntity(WriteEntity writeEntity)
    throws AuthorizationException {
  // skip URI validation for session scratch file URIs
  if (writeEntity.isTempURI()) {
    return true;
  }
  try {
    if (writeEntity.getTyp().equals(Type.DFS_DIR)
        || writeEntity.getTyp().equals(Type.LOCAL_DIR)) {
      HiveConf conf = SessionState.get().getConf();
      String warehouseDir = conf.getVar(ConfVars.METASTOREWAREHOUSE);
      URI scratchURI = new URI(PathUtils.parseDFSURI(warehouseDir,
        conf.getVar(HiveConf.ConfVars.SCRATCHDIR)));
      URI requestURI = new URI(PathUtils.parseDFSURI(warehouseDir,
        writeEntity.getLocation().getPath()));
      LOG.debug("scratchURI = " + scratchURI + ", requestURI = " + requestURI);
      if (PathUtils.impliesURI(scratchURI, requestURI)) {
        return true;
      }
      URI localScratchURI = new URI(PathUtils.parseLocalURI(conf.getVar(HiveConf.ConfVars.LOCALSCRATCHDIR)));
      URI localRequestURI = new URI(PathUtils.parseLocalURI(writeEntity.getLocation().getPath()));
      LOG.debug("localScratchURI = " + localScratchURI + ", localRequestURI = " + localRequestURI);
      if (PathUtils.impliesURI(localScratchURI, localRequestURI)) {
        return true;
      }
    }
  } catch (Exception e) {
    throw new AuthorizationException("Failed to extract uri details", e);
  }
  return false;
}
 
Example #21
Source File: SentryHiveAuthorizationTaskFactoryImpl.java    From incubator-sentry with Apache License 2.0 5 votes vote down vote up
@Override
public Task<? extends Serializable> createShowCurrentRoleTask(HashSet<ReadEntity> inputs,
    HashSet<WriteEntity> outputs, Path resultFile) throws SemanticException {
  RoleDDLDesc ddlDesc = new RoleDDLDesc(null, RoleDDLDesc.RoleOperation.SHOW_CURRENT_ROLE);
  ddlDesc.setResFile(resultFile.toString());
  return createTask(new DDLWork(inputs, outputs, ddlDesc));
}
 
Example #22
Source File: HiveHookIT.java    From incubator-atlas with Apache License 2.0 5 votes vote down vote up
@Test
public void testInsertIntoPartition() throws Exception {
    final boolean isPartitionedTable = true;
    String tableName = createTable(isPartitionedTable);
    String insertTableName = createTable(isPartitionedTable);
    String query =
        "insert into " + insertTableName + " partition(dt = '"+ PART_FILE + "') select id, name from " + tableName
            + " where dt = '"+ PART_FILE + "'";
    runCommand(query);

    final Set<ReadEntity> inputs = getInputs(tableName, Entity.Type.TABLE);
    final Set<WriteEntity> outputs = getOutputs(insertTableName, Entity.Type.TABLE);
    outputs.iterator().next().setWriteType(WriteEntity.WriteType.INSERT);

    final Set<ReadEntity> partitionIps = new LinkedHashSet<ReadEntity>() {
        {
            addAll(inputs);
            add(getPartitionInput());

        }
    };

    final Set<WriteEntity> partitionOps = new LinkedHashSet<WriteEntity>() {
        {
            addAll(outputs);
            add(getPartitionOutput());

        }
    };

    validateProcess(constructEvent(query,  HiveOperation.QUERY, partitionIps, partitionOps), inputs, outputs);

    assertTableIsRegistered(DEFAULT_DB, tableName);
    assertTableIsRegistered(DEFAULT_DB, insertTableName);

    //TODO -Add update test case
}
 
Example #23
Source File: SentryHiveAuthorizationTaskFactoryImpl.java    From incubator-sentry with Apache License 2.0 5 votes vote down vote up
@Override
public Task<? extends Serializable> createCreateRoleTask(ASTNode ast, HashSet<ReadEntity> inputs,
    HashSet<WriteEntity> outputs) throws SemanticException {
  String roleName = BaseSemanticAnalyzer.unescapeIdentifier(ast.getChild(0).getText());
  if (AccessConstants.RESERVED_ROLE_NAMES.contains(roleName.toUpperCase())) {
    String msg = "Roles cannot be one of the reserved roles: " + AccessConstants.RESERVED_ROLE_NAMES;
    throw new SemanticException(msg);
  }
  RoleDDLDesc roleDesc = new RoleDDLDesc(roleName, RoleDDLDesc.RoleOperation.CREATE_ROLE);
  return createTask(new DDLWork(inputs, outputs, roleDesc));
}
 
Example #24
Source File: HiveHookIT.java    From incubator-atlas with Apache License 2.0 5 votes vote down vote up
private String sortEventsAndGetProcessQualifiedName(final HiveHook.HiveEventContext event) throws HiveException{
    SortedSet<ReadEntity> sortedHiveInputs = event.getInputs() == null ? null : new TreeSet<ReadEntity>(entityComparator);
    SortedSet<WriteEntity> sortedHiveOutputs = event.getOutputs() == null ? null : new TreeSet<WriteEntity>(entityComparator);

    if ( event.getInputs() != null) {
        sortedHiveInputs.addAll(event.getInputs());
    }
    if ( event.getOutputs() != null) {
        sortedHiveOutputs.addAll(event.getOutputs());
    }
    return getProcessQualifiedName(hiveMetaStoreBridge, event, sortedHiveInputs, sortedHiveOutputs, getSortedProcessDataSets(event.getInputs()), getSortedProcessDataSets(event.getOutputs()));
}
 
Example #25
Source File: SentryHiveAuthorizationTaskFactoryImpl.java    From incubator-sentry with Apache License 2.0 5 votes vote down vote up
@Override
public Task<? extends Serializable> createGrantTask(ASTNode ast, HashSet<ReadEntity> inputs,
    HashSet<WriteEntity> outputs) throws SemanticException {
  List<PrivilegeDesc> privilegeDesc = analyzePrivilegeListDef(
      (ASTNode) ast.getChild(0));
  List<PrincipalDesc> principalDesc = analyzePrincipalListDef(
      (ASTNode) ast.getChild(1));
  SentryHivePrivilegeObjectDesc privilegeObj = null;
  boolean grantOption = false;
  if (ast.getChildCount() > 2) {
    for (int i = 2; i < ast.getChildCount(); i++) {
      ASTNode astChild = (ASTNode) ast.getChild(i);
      if (astChild.getType() == HiveParser.TOK_GRANT_WITH_OPTION) {
        grantOption = true;
      } else if (astChild.getType() == HiveParser.TOK_PRIV_OBJECT) {
        privilegeObj = analyzePrivilegeObject(astChild);
      }
    }
  }
  String userName = null;
  if (SessionState.get() != null
      && SessionState.get().getAuthenticator() != null) {
    userName = SessionState.get().getAuthenticator().getUserName();
  }
  Preconditions.checkNotNull(privilegeObj, "privilegeObj is null for " + ast.dump());
  if (privilegeObj.getPartSpec() != null) {
    throw new SemanticException(SentryHiveConstants.PARTITION_PRIVS_NOT_SUPPORTED);
  }
  for (PrincipalDesc princ : principalDesc) {
    if (princ.getType() != PrincipalType.ROLE) {
      String msg = SentryHiveConstants.GRANT_REVOKE_NOT_SUPPORTED_FOR_PRINCIPAL + princ.getType();
      throw new SemanticException(msg);
    }
  }
  GrantDesc grantDesc = new GrantDesc(privilegeObj, privilegeDesc,
      principalDesc, userName, PrincipalType.USER, grantOption);
  return createTask(new DDLWork(inputs, outputs, grantDesc));
}
 
Example #26
Source File: HiveHookIT.java    From incubator-atlas with Apache License 2.0 5 votes vote down vote up
@Test
public void testDropAndRecreateCTASOutput() throws Exception {
    String tableName = createTable();
    String ctasTableName = "table" + random();
    String query = "create table " + ctasTableName + " as select * from " + tableName;
    runCommand(query);

    assertTableIsRegistered(DEFAULT_DB, ctasTableName);

    Set<ReadEntity> inputs = getInputs(tableName, Entity.Type.TABLE);
    Set<WriteEntity> outputs =  getOutputs(ctasTableName, Entity.Type.TABLE);

    final HiveHook.HiveEventContext hiveEventContext = constructEvent(query, HiveOperation.CREATETABLE_AS_SELECT, inputs, outputs);
    String processId = assertProcessIsRegistered(hiveEventContext);

    final String drpquery = String.format("drop table %s ", ctasTableName);
    runCommandWithDelay(drpquery, 100);
    assertTableIsNotRegistered(DEFAULT_DB, ctasTableName);

    runCommand(query);
    assertTableIsRegistered(DEFAULT_DB, ctasTableName);
    outputs =  getOutputs(ctasTableName, Entity.Type.TABLE);
    String process2Id = assertProcessIsRegistered(hiveEventContext, inputs, outputs);

    assertNotEquals(process2Id, processId);

    Referenceable processRef = atlasClient.getEntity(processId);
    validateOutputTables(processRef, outputs);
}
 
Example #27
Source File: HiveHookIT.java    From incubator-atlas with Apache License 2.0 5 votes vote down vote up
private HiveHook.HiveEventContext constructEvent(String query, HiveOperation op, Set<ReadEntity> inputs, Set<WriteEntity> outputs) {
    HiveHook.HiveEventContext event = new HiveHook.HiveEventContext();
    event.setQueryStr(query);
    event.setOperation(op);
    event.setInputs(inputs);
    event.setOutputs(outputs);
    return event;
}
 
Example #28
Source File: HiveHookIT.java    From incubator-atlas with Apache License 2.0 5 votes vote down vote up
@Test
public void testCTAS() throws Exception {
    String tableName = createTable();
    String ctasTableName = "table" + random();
    String query = "create table " + ctasTableName + " as select * from " + tableName;
    runCommand(query);

    final Set<ReadEntity> readEntities = getInputs(tableName, Entity.Type.TABLE);
    final Set<WriteEntity> writeEntities = getOutputs(ctasTableName, Entity.Type.TABLE);

    assertProcessIsRegistered(constructEvent(query, HiveOperation.CREATETABLE_AS_SELECT, readEntities, writeEntities));
    assertTableIsRegistered(DEFAULT_DB, ctasTableName);

}
 
Example #29
Source File: HiveHook.java    From incubator-atlas with Apache License 2.0 5 votes vote down vote up
private static void addOutputs(HiveMetaStoreBridge hiveBridge, HiveOperation op, SortedSet<WriteEntity> sortedOutputs, StringBuilder buffer, final Map<WriteEntity, Referenceable> refs, final boolean ignoreHDFSPathsInQFName) throws HiveException {
    if (refs != null) {
        Set<String> dataSetsProcessed = new LinkedHashSet<>();
        if (sortedOutputs != null) {
            for (WriteEntity output : sortedOutputs) {
                final Entity entity = output;
                if (!dataSetsProcessed.contains(output.getName().toLowerCase())) {
                    //HiveOperation.QUERY type encompasses INSERT, INSERT_OVERWRITE, UPDATE, DELETE, PATH_WRITE operations
                    if (addQueryType(op, (WriteEntity) entity)) {
                        buffer.append(SEP);
                        buffer.append(((WriteEntity) entity).getWriteType().name());
                    }
                    if (ignoreHDFSPathsInQFName &&
                        (Type.DFS_DIR.equals(output.getType()) || Type.LOCAL_DIR.equals(output.getType()))) {
                        LOG.debug("Skipping dfs dir output addition to process qualified name {} ", output.getName());
                    } else if (refs.containsKey(output)) {
                        if ( output.getType() == Type.PARTITION || output.getType() == Type.TABLE) {
                            final Date createTime = HiveMetaStoreBridge.getTableCreatedTime(hiveBridge.hiveClient.getTable(output.getTable().getDbName(), output.getTable().getTableName()));
                            addDataset(buffer, refs.get(output), createTime.getTime());
                        } else {
                            addDataset(buffer, refs.get(output));
                        }
                    }
                    dataSetsProcessed.add(output.getName().toLowerCase());
                }
            }
        }
    }
}
 
Example #30
Source File: HiveHook.java    From incubator-atlas with Apache License 2.0 5 votes vote down vote up
private static boolean ignoreHDFSPathsinQFName(final HiveOperation op, final Set<ReadEntity> inputs, final Set<WriteEntity> outputs) {
    switch (op) {
    case LOAD:
    case IMPORT:
        return isPartitionBasedQuery(outputs);
    case EXPORT:
        return isPartitionBasedQuery(inputs);
    case QUERY:
        return true;
    }
    return false;
}