Java Code Examples for org.apache.commons.collections.CollectionUtils#subtract()

@Override
public RangerSecurityZone postUpdate(XXSecurityZone xObj) {
    // Update ServiceVersionInfo for all affected services
    RangerSecurityZone ret = super.postUpdate(xObj);
    Set<String> oldServiceNames = new HashSet(serviceNamesInZones.remove(xObj.getId()));
    Set<String> updatedServiceNames = ret.getServices().keySet();

    Collection<String> newServiceNames = CollectionUtils.subtract(updatedServiceNames, oldServiceNames);
    Collection<String> deletedServiceNames = CollectionUtils.subtract(oldServiceNames, updatedServiceNames);

    try {
        serviceDBStore.createZoneDefaultPolicies(newServiceNames, ret);
        serviceDBStore.deleteZonePolicies(deletedServiceNames, ret.getId());

        oldServiceNames.addAll(updatedServiceNames);
        updateServiceInfos(oldServiceNames);
    } catch (Exception exception) {
        logger.error("postUpdate processing failed for security-zone:[" + ret + "]", exception);
        ret = null;
    }
    return ret;
}
 

public void setLabels(AtlasVertex vertex, Set<String> labels) throws AtlasBaseException {
    final Set<String> currentLabels = getLabels(vertex);
    final Set<String> addedLabels;
    final Set<String> removedLabels;

    if (CollectionUtils.isEmpty(currentLabels)) {
        addedLabels   = labels;
        removedLabels = null;
    } else if (CollectionUtils.isEmpty(labels)) {
        addedLabels   = null;
        removedLabels = currentLabels;
    } else {
        addedLabels   = new HashSet<String>(CollectionUtils.subtract(labels, currentLabels));
        removedLabels = new HashSet<String>(CollectionUtils.subtract(currentLabels, labels));
    }

    updateLabels(vertex, labels);

    entityChangeNotifier.onLabelsUpdatedFromEntity(graphHelper.getGuid(vertex), addedLabels, removedLabels);
}
 

@SuppressWarnings("unchecked")
public void compare() {
    Set<K> keys1 = map1.keySet();
    Set<K> keys2 = map2.keySet();
    Collection<K> addedKeys = CollectionUtils.subtract(keys1, keys2);
    Collection<K> removedKeys = CollectionUtils.subtract(keys2, keys1);
    Collection<K> modifiedKeys = CollectionUtils.intersection(keys1, keys2);

    addedKeys.forEach(k -> added.add(map1.get(k)));
    removedKeys.forEach(k -> removed.add(map2.get(k)));
    modifiedKeys.forEach(k -> {
        if (!map1.get(k).equals(map2.get(k))) {
            modified.add(map1.get(k));
        }
    });
}
 

private void checkNonnullProperty(HugeVertex vertex) {
    Set<Id> keys = vertex.getProperties().keySet();
    VertexLabel vertexLabel = vertex.schemaLabel();
    // Check whether passed all non-null property
    @SuppressWarnings("unchecked")
    Collection<Id> nonNullKeys = CollectionUtils.subtract(
                                 vertexLabel.properties(),
                                 vertexLabel.nullableKeys());
    if (!keys.containsAll(nonNullKeys)) {
        @SuppressWarnings("unchecked")
        Collection<Id> missed = CollectionUtils.subtract(nonNullKeys, keys);
        HugeGraph graph = this.graph();
        E.checkArgument(false, "All non-null property keys %s of " +
                        "vertex label '%s' must be setted, missed keys %s",
                        graph.mapPkId2Name(nonNullKeys), vertexLabel.name(),
                        graph.mapPkId2Name(missed));
    }
}
 

@Scheduled(cron = "0 0/1 * * * ?")
public void serverScanner() {
    CopyOnWriteArraySet<ServerInfo> servers = liveManSetting.getServers();
    List<ServerInfo> list = dynamicServerService.list();
    if (CollectionUtils.isNotEmpty(list)) {
        Collection<ServerInfo> subtract = CollectionUtils.subtract(list, servers);
        for (ServerInfo serverInfo : subtract) {
            // 检查是否可以连接
            try {
                log.info("发现新的服务器资源 " + serverInfo);
                if (broadcastServerService.testServer(serverInfo)) {
                    serverInfo.setAvailable(true);
                    broadcastServerService.addAndInstallServer(serverInfo);
                }
            } catch (Exception ignore) {
            }
        }
    }
}
 

@Test
public void difference_of_sets_apache () {

	@SuppressWarnings("rawtypes")
	Collection possibleFriendRequests = CollectionUtils.subtract(yourFriends, myFriends);
	
	assertEquals(6, possibleFriendRequests.size());
}
 

/**
 * helper function to output log messages if there are tables that are missing read groups that were seen in the other tables
 * @param allReadGroups a set of all of the read groups across inputs
 * @param inputReadGroups a map from file to the read groups that file contains
 */
private static void logTablesWithMissingReadGroups(SortedSet<String> allReadGroups, Map<File, Set<String>> inputReadGroups) {
    // Log the read groups that are missing from specific inputs
    for (final Map.Entry<File, Set<String>> entry: inputReadGroups.entrySet()) {
        final File input = entry.getKey();
        final Set<String> readGroups = entry.getValue();
        if (allReadGroups.size() != readGroups.size()) {
            // Since this is not completely unexpected, more than debug, but less than a proper warning.
            logger.info("Missing read group(s)" + ": " + input.getAbsolutePath());
            for (final Object readGroup: CollectionUtils.subtract(allReadGroups, readGroups)) {
                logger.info("  " + readGroup);
            }
        }
    }
}
 

private static void editNormalMarkSheet(MarkSheetManagementEditBean markSheetManagementEditBean) {
    Collection<MarkSheetEnrolmentEvaluationBean> filteredEnrolmentEvaluationBeansToEditList =
            getEnrolmentEvaluationsWithValidGrades(markSheetManagementEditBean.getEnrolmentEvaluationBeansToEdit());

    Collection<MarkSheetEnrolmentEvaluationBean> enrolmentEvaluationBeansToAppendList =
            getEnrolmentEvaluationsWithValidGrades(markSheetManagementEditBean.getEnrolmentEvaluationBeansToAppend());

    Collection<MarkSheetEnrolmentEvaluationBean> enrolmentEvaluationBeansToRemoveList =
            CollectionUtils.subtract(markSheetManagementEditBean.getEnrolmentEvaluationBeansToEdit(),
                    filteredEnrolmentEvaluationBeansToEditList);

    markSheetManagementEditBean.getMarkSheet().editNormal(filteredEnrolmentEvaluationBeansToEditList,
            enrolmentEvaluationBeansToAppendList, enrolmentEvaluationBeansToRemoveList);
}
 

@SuppressWarnings("rawtypes")
public void compareAndEmit(Set wisb, Set wiri, StreamEvent event) {
    // compare with wisbValues if wisbValues are already there for dynamic
    // type
    Collection noDataValues = CollectionUtils.subtract(wisb, wiri);
    LOG.debug("nodatavalues:" + noDataValues + ", wisb: " + wisb + ", wiri: " + wiri);
    if (noDataValues != null && noDataValues.size() > 0) {
        LOG.info("No data alert is triggered with no data values {} and wisb {}", noDataValues, wisb);

        String is = policyDef.getOutputStreams().get(0);
        StreamDefinition sd = sds.get(is);
        int timestampIndex = sd.getColumnIndex("timestamp");
        int hostIndex = sd.getColumnIndex("host");
        int originalStreamNameIndex = sd.getColumnIndex("originalStreamName");

        for (Object one : noDataValues) {
            Object[] triggerEvent = new Object[sd.getColumns().size()];
            for (int i = 0; i < sd.getColumns().size(); i++) {
                if (i == timestampIndex) {
                    triggerEvent[i] = System.currentTimeMillis();
                } else if (i == hostIndex) {
                    triggerEvent[hostIndex] = ((List) one).get(0);
                } else if (i == originalStreamNameIndex) {
                    triggerEvent[originalStreamNameIndex] = event.getStreamId();
                } else if (sd.getColumns().size() < i) {
                    LOG.error("strema event data have different lenght compare to column definition!");
                } else {
                    triggerEvent[i] = sd.getColumns().get(i).getDefaultValue();
                }
            }
            AlertStreamEvent alertEvent = createAlertEvent(sd, event.getTimestamp(), triggerEvent);
            LOG.info(String.format("Nodata alert %s generated and will be emitted", Joiner.on(",").join(triggerEvent)));
            collector.emit(alertEvent);
        }

    }
}
 

@SuppressWarnings("unchecked")
private void checkNullableKeys(Action action) {
    // Not using switch-case to avoid indent too much
    if (action == Action.ELIMINATE) {
        if (!this.nullableKeys.isEmpty()) {
            throw new NotAllowException(
                      "Not support to eliminate nullableKeys " +
                      "for vertex label currently");
        }
        return;
    }

    VertexLabel vertexLabel = this.vertexLabelOrNull(this.name);
    // The originProps is empty when firstly create vertex label
    List<String> originProps = vertexLabel == null ?
                               ImmutableList.of() :
                               this.graph()
                                   .mapPkId2Name(vertexLabel.properties());
    Set<String> appendProps = this.properties;

    E.checkArgument(CollectionUtil.union(originProps, appendProps)
                                  .containsAll(this.nullableKeys),
                    "The nullableKeys: %s to be created or appended " +
                    "must belong to the origin/new properties: %s/%s",
                    this.nullableKeys, originProps, appendProps);

    E.checkArgument(!CollectionUtil.hasIntersection(this.primaryKeys,
                                                    this.nullableKeys),
                    "The nullableKeys: %s are not allowed to " +
                    "belong to primaryKeys: %s of vertex label '%s'",
                    this.nullableKeys, this.primaryKeys, this.name);

    if (action == Action.APPEND) {
        Collection<String> newAddedProps = CollectionUtils.subtract(
                                           appendProps, originProps);
        E.checkArgument(this.nullableKeys.containsAll(newAddedProps),
                        "The new added properties: %s must be nullable",
                        newAddedProps);
    }
}
 

@SuppressWarnings("unchecked")
private void checkNullableKeys(Action action) {
    // Not using switch-case to avoid indent too much
    if (action == Action.ELIMINATE) {
        if (!this.nullableKeys.isEmpty()) {
            throw new NotAllowException(
                      "Not support to eliminate nullableKeys " +
                      "for edge label currently");
        }
        return;
    }

    EdgeLabel edgeLabel = this.edgeLabelOrNull(this.name);
    // The originProps is empty when firstly create edge label
    List<String> originProps = edgeLabel == null ?
                               ImmutableList.of() :
                               this.graph()
                                   .mapPkId2Name(edgeLabel.properties());
    Set<String> appendProps = this.properties;

    E.checkArgument(CollectionUtil.union(originProps, appendProps)
                    .containsAll(this.nullableKeys),
                    "The nullableKeys: %s to be created or appended " +
                    "must belong to the origin/new properties: %s/%s ",
                    this.nullableKeys, originProps, appendProps);

    E.checkArgument(!CollectionUtil.hasIntersection(this.sortKeys,
                                                    this.nullableKeys),
                    "The nullableKeys: %s are not allowed to " +
                    "belong to sortKeys: %s of edge label '%s'",
                    this.nullableKeys, this.sortKeys, this.name);

    if (action == Action.APPEND) {
        Collection<String> newAddedProps = CollectionUtils.subtract(
                                           appendProps, originProps);
        E.checkArgument(this.nullableKeys.containsAll(newAddedProps),
                        "The new added properties: %s must be nullable",
                        newAddedProps);
    }
}
 

/** {@inheritDoc} */
public Collection getRemovedKeys() {
    Set setValues = set.getElementValues();
    Set preSelectedValues = getPreSelected();
    Collection result = CollectionUtils.subtract(preSelectedValues, setValues);
    return result;
}
 

/** {@inheritDoc} */
public Collection getRemovedKeys() {
    Set setValues = set.getElementValues();
    Set preSelectedValues = getPreSelected();
    Collection result = CollectionUtils.subtract(preSelectedValues, setValues);
    return result;
}
 

/**
 * Compare with metadata snapshot cache to generate diff like added, removed and modified between different versions.
 *
 * @param spec
 */
@SuppressWarnings("unchecked")
@Override
public synchronized void onStreamRouteBoltSpecChange(RouterSpec spec, Map<String, StreamDefinition> sds) {
    sanityCheck(spec);

    // figure out added, removed, modified StreamSortSpec
    Map<StreamPartition, StreamSortSpec> newSSS = spec.makeSSS();

    Set<StreamPartition> newStreamIds = newSSS.keySet();
    Set<StreamPartition> cachedStreamIds = cachedSSS.keySet();
    Collection<StreamPartition> addedStreamIds = CollectionUtils.subtract(newStreamIds, cachedStreamIds);
    Collection<StreamPartition> removedStreamIds = CollectionUtils.subtract(cachedStreamIds, newStreamIds);
    Collection<StreamPartition> modifiedStreamIds = CollectionUtils.intersection(newStreamIds, cachedStreamIds);

    Map<StreamPartition, StreamSortSpec> added = new HashMap<>();
    Map<StreamPartition, StreamSortSpec> removed = new HashMap<>();
    Map<StreamPartition, StreamSortSpec> modified = new HashMap<>();
    addedStreamIds.forEach(s -> added.put(s, newSSS.get(s)));
    removedStreamIds.forEach(s -> removed.put(s, cachedSSS.get(s)));
    modifiedStreamIds.forEach(s -> {
        if (!newSSS.get(s).equals(cachedSSS.get(s))) { // this means StreamSortSpec is changed for one specific streamId
            modified.put(s, newSSS.get(s));
        }
    });
    if (LOG.isDebugEnabled()) {
        LOG.debug("added StreamSortSpec " + added);
        LOG.debug("removed StreamSortSpec " + removed);
        LOG.debug("modified StreamSortSpec " + modified);
    }
    router.onStreamSortSpecChange(added, removed, modified);
    // switch cache
    cachedSSS = newSSS;

    // figure out added, removed, modified StreamRouterSpec
    Map<StreamPartition, List<StreamRouterSpec>> newSRS = spec.makeSRS();

    Set<StreamPartition> newStreamPartitions = newSRS.keySet();
    Set<StreamPartition> cachedStreamPartitions = cachedSRS.keySet();

    Collection<StreamPartition> addedStreamPartitions = CollectionUtils.subtract(newStreamPartitions, cachedStreamPartitions);
    Collection<StreamPartition> removedStreamPartitions = CollectionUtils.subtract(cachedStreamPartitions, newStreamPartitions);
    Collection<StreamPartition> modifiedStreamPartitions = CollectionUtils.intersection(newStreamPartitions, cachedStreamPartitions);

    Collection<StreamRouterSpec> addedRouterSpecs = new ArrayList<>();
    Collection<StreamRouterSpec> removedRouterSpecs = new ArrayList<>();
    Collection<StreamRouterSpec> modifiedRouterSpecs = new ArrayList<>();
    addedStreamPartitions.forEach(s -> addedRouterSpecs.addAll(newSRS.get(s)));
    removedStreamPartitions.forEach(s -> removedRouterSpecs.addAll(cachedSRS.get(s)));
    modifiedStreamPartitions.forEach(s -> {
        if (!CollectionUtils.isEqualCollection(newSRS.get(s), cachedSRS.get(s))) { // this means StreamRouterSpec is changed for one specific StreamPartition
            modifiedRouterSpecs.addAll(newSRS.get(s));
        }
    });

    if (LOG.isDebugEnabled()) {
        LOG.debug("added StreamRouterSpec " + addedRouterSpecs);
        LOG.debug("removed StreamRouterSpec " + removedRouterSpecs);
        LOG.debug("modified StreamRouterSpec " + modifiedRouterSpecs);
    }

    routeCollector.onStreamRouterSpecChange(addedRouterSpecs, removedRouterSpecs, modifiedRouterSpecs, sds);
    // switch cache
    cachedSRS = newSRS;
    sdf = sds;
    specVersion = spec.getVersion();
}
 

@SuppressWarnings("unchecked")
public static Decoder createDecoder(String spec, String[] colnames, ValueType[] schema, FrameBlock meta, int clen) 
{
	Decoder decoder = null;
	
	try
	{
		//parse transform specification
		JSONObject jSpec = new JSONObject(spec);
		List<Decoder> ldecoders = new ArrayList<>();
		
		//create decoders 'recode', 'dummy' and 'pass-through'
		List<Integer> rcIDs = Arrays.asList(ArrayUtils.toObject(
				TfMetaUtils.parseJsonIDList(jSpec, colnames, TfMethod.RECODE.toString())));
		List<Integer> dcIDs = Arrays.asList(ArrayUtils.toObject(
				TfMetaUtils.parseJsonIDList(jSpec, colnames, TfMethod.DUMMYCODE.toString()))); 
		rcIDs = new ArrayList<Integer>(CollectionUtils.union(rcIDs, dcIDs));
		int len = dcIDs.isEmpty() ? Math.min(meta.getNumColumns(), clen) : meta.getNumColumns();
		List<Integer> ptIDs = new ArrayList<Integer>(CollectionUtils
			.subtract(UtilFunctions.getSeqList(1, len, 1), rcIDs));
		
		//create default schema if unspecified (with double columns for pass-through)
		if( schema == null ) {
			schema = UtilFunctions.nCopies(len, ValueType.STRING);
			for( Integer col : ptIDs )
				schema[col-1] = ValueType.FP64;
		}
		
		if( !dcIDs.isEmpty() ) {
			ldecoders.add(new DecoderDummycode(schema, 
				ArrayUtils.toPrimitive(dcIDs.toArray(new Integer[0]))));
		}
		if( !rcIDs.isEmpty() ) {
			ldecoders.add(new DecoderRecode(schema, !dcIDs.isEmpty(),
				ArrayUtils.toPrimitive(rcIDs.toArray(new Integer[0]))));
		}
		if( !ptIDs.isEmpty() ) {
			ldecoders.add(new DecoderPassThrough(schema, 
				ArrayUtils.toPrimitive(ptIDs.toArray(new Integer[0])),
				ArrayUtils.toPrimitive(dcIDs.toArray(new Integer[0]))));
		}
		
		//create composite decoder of all created decoders
		//and initialize with given meta data (recode, dummy, bin)
		decoder = new DecoderComposite(schema, ldecoders);
		decoder.setColnames(colnames);
		decoder.initMetaData(meta);
	}
	catch(Exception ex) {
		throw new DMLRuntimeException(ex);
	}
	
	return decoder;
}
 

@SuppressWarnings("unchecked")
public static Encoder createEncoder(String spec, String[] colnames, ValueType[] schema, FrameBlock meta) {
	Encoder encoder = null;
	int clen = schema.length;
	
	try {
		//parse transform specification
		JSONObject jSpec = new JSONObject(spec);
		List<Encoder> lencoders = new ArrayList<>();
	
		//prepare basic id lists (recode, feature hash, dummycode, pass-through)
		List<Integer> rcIDs = Arrays.asList(ArrayUtils.toObject(
			TfMetaUtils.parseJsonIDList(jSpec, colnames, TfMethod.RECODE.toString())));
		List<Integer>haIDs = Arrays.asList(ArrayUtils.toObject(
			TfMetaUtils.parseJsonIDList(jSpec, colnames, TfMethod.HASH.toString())));
		List<Integer> dcIDs = Arrays.asList(ArrayUtils.toObject(
			TfMetaUtils.parseJsonIDList(jSpec, colnames, TfMethod.DUMMYCODE.toString())));
		List<Integer> binIDs = TfMetaUtils.parseBinningColIDs(jSpec, colnames);
		//note: any dummycode column requires recode as preparation, unless it follows binning
		rcIDs = new ArrayList<Integer>(CollectionUtils.subtract(
			CollectionUtils.union(rcIDs, CollectionUtils.subtract(dcIDs, binIDs)), haIDs));
		List<Integer> ptIDs = new ArrayList<Integer>(CollectionUtils.subtract(
			CollectionUtils.subtract(UtilFunctions.getSeqList(1, clen, 1),
				CollectionUtils.union(rcIDs,haIDs)), binIDs));
		List<Integer> oIDs = Arrays.asList(ArrayUtils.toObject(
			TfMetaUtils.parseJsonIDList(jSpec, colnames, TfMethod.OMIT.toString())));
		List<Integer> mvIDs = Arrays.asList(ArrayUtils.toObject(
			TfMetaUtils.parseJsonObjectIDList(jSpec, colnames, TfMethod.IMPUTE.toString())));
		
		//create individual encoders
		if( !rcIDs.isEmpty() ) {
			EncoderRecode ra = new EncoderRecode(jSpec, colnames, clen);
			ra.setColList(ArrayUtils.toPrimitive(rcIDs.toArray(new Integer[0])));
			lencoders.add(ra);
		}
		if( !haIDs.isEmpty() ) {
			EncoderFeatureHash ha = new EncoderFeatureHash(jSpec, colnames, clen);
			ha.setColList(ArrayUtils.toPrimitive(haIDs.toArray(new Integer[0])));
			lencoders.add(ha);
		}
		if( !ptIDs.isEmpty() )
			lencoders.add(new EncoderPassThrough(
				ArrayUtils.toPrimitive(ptIDs.toArray(new Integer[0])), clen));
		if( !binIDs.isEmpty() )
			lencoders.add(new EncoderBin(jSpec, colnames, schema.length));
		if( !dcIDs.isEmpty() )
			lencoders.add(new EncoderDummycode(jSpec, colnames, schema.length));
		if( !oIDs.isEmpty() )
			lencoders.add(new EncoderOmit(jSpec, colnames, schema.length));
		if( !mvIDs.isEmpty() ) {
			EncoderMVImpute ma = new EncoderMVImpute(jSpec, colnames, schema.length);
			ma.initRecodeIDList(rcIDs);
			lencoders.add(ma);
		}
		
		//create composite decoder of all created encoders
		encoder = new EncoderComposite(lencoders);
		
		//initialize meta data w/ robustness for superset of cols
		if( meta != null ) {
			String[] colnames2 = meta.getColumnNames();
			if( !TfMetaUtils.isIDSpec(jSpec) && colnames!=null && colnames2!=null 
				&& !ArrayUtils.isEquals(colnames, colnames2) ) 
			{
				HashMap<String, Integer> colPos = getColumnPositions(colnames2);
				//create temporary meta frame block w/ shallow column copy
				FrameBlock meta2 = new FrameBlock(meta.getSchema(), colnames2);
				meta2.setNumRows(meta.getNumRows());
				for( int i=0; i<colnames.length; i++ ) {
					if( !colPos.containsKey(colnames[i]) ) {
						throw new DMLRuntimeException("Column name not found in meta data: "
							+colnames[i]+" (meta: "+Arrays.toString(colnames2)+")");
					}
					int pos = colPos.get(colnames[i]);
					meta2.setColumn(i, meta.getColumn(pos));
					meta2.setColumnMetadata(i, meta.getColumnMetadata(pos));
				}
				meta = meta2;
			}
			encoder.initMetaData(meta);
		}
	}
	catch(Exception ex) {
		throw new DMLRuntimeException(ex);
	}
	
	return encoder;
}
 

@SuppressWarnings("unchecked")
public static Decoder createDecoder(String spec, String[] colnames, ValueType[] schema, FrameBlock meta, int clen) 
{
	Decoder decoder = null;
	
	try
	{
		//parse transform specification
		JSONObject jSpec = new JSONObject(spec);
		List<Decoder> ldecoders = new ArrayList<>();
		
		//create decoders 'recode', 'dummy' and 'pass-through'
		List<Integer> rcIDs = Arrays.asList(ArrayUtils.toObject(
				TfMetaUtils.parseJsonIDList(jSpec, colnames, TfMethod.RECODE.toString())));
		List<Integer> dcIDs = Arrays.asList(ArrayUtils.toObject(
				TfMetaUtils.parseJsonIDList(jSpec, colnames, TfMethod.DUMMYCODE.toString()))); 
		rcIDs = new ArrayList<Integer>(CollectionUtils.union(rcIDs, dcIDs));
		int len = dcIDs.isEmpty() ? Math.min(meta.getNumColumns(), clen) : meta.getNumColumns();
		List<Integer> ptIDs = new ArrayList<Integer>(CollectionUtils
			.subtract(UtilFunctions.getSeqList(1, len, 1), rcIDs));
		
		//create default schema if unspecified (with double columns for pass-through)
		if( schema == null ) {
			schema = UtilFunctions.nCopies(len, ValueType.STRING);
			for( Integer col : ptIDs )
				schema[col-1] = ValueType.FP64;
		}
		
		if( !dcIDs.isEmpty() ) {
			ldecoders.add(new DecoderDummycode(schema, 
				ArrayUtils.toPrimitive(dcIDs.toArray(new Integer[0]))));
		}
		if( !rcIDs.isEmpty() ) {
			ldecoders.add(new DecoderRecode(schema, !dcIDs.isEmpty(),
				ArrayUtils.toPrimitive(rcIDs.toArray(new Integer[0]))));
		}
		if( !ptIDs.isEmpty() ) {
			ldecoders.add(new DecoderPassThrough(schema, 
				ArrayUtils.toPrimitive(ptIDs.toArray(new Integer[0])),
				ArrayUtils.toPrimitive(dcIDs.toArray(new Integer[0]))));
		}
		
		//create composite decoder of all created decoders
		//and initialize with given meta data (recode, dummy, bin)
		decoder = new DecoderComposite(schema, ldecoders);
		decoder.setColnames(colnames);
		decoder.initMetaData(meta);
	}
	catch(Exception ex) {
		throw new DMLRuntimeException(ex);
	}
	
	return decoder;
}
 

/**
 * When it is run at the first time, due to cachedPolicies being empty, all existing policies are expected
 * to be addedPolicies.
 */
@SuppressWarnings("unchecked")
@Override
public void run() {
    // we should catch every exception to avoid zombile thread
    try {
        final Stopwatch watch = Stopwatch.createStarted();
        LOG.info("Starting to load policies");
        List<PolicyDefinition> current = client.listPolicies();
        Map<String, PolicyDefinition> currPolicies = new HashMap<>();
        current.forEach(pe -> currPolicies.put(pe.getName(), pe));

        Collection<String> addedPolicies = CollectionUtils.subtract(currPolicies.keySet(), cachedPolicies.keySet());
        Collection<String> removedPolicies = CollectionUtils.subtract(cachedPolicies.keySet(), currPolicies.keySet());
        Collection<String> potentiallyModifiedPolicies = CollectionUtils.intersection(currPolicies.keySet(), cachedPolicies.keySet());

        List<String> reallyModifiedPolicies = new ArrayList<>();
        for (String updatedPolicy : potentiallyModifiedPolicies) {
            if (currPolicies.get(updatedPolicy) != null
                    && !currPolicies.get(updatedPolicy).equals(cachedPolicies.get(updatedPolicy))) {
                reallyModifiedPolicies.add(updatedPolicy);
            }
        }

        boolean policyChanged = false;
        if (addedPolicies.size() != 0
            || removedPolicies.size() != 0
            || reallyModifiedPolicies.size() != 0) {
            policyChanged = true;
        }

        if (!policyChanged) {
            LOG.info("No policy (totally {}) changed since last round", current.size());
            return;
        }

        synchronized (this) {
            for (PolicyChangeListener listener : listeners) {
                listener.onPolicyChange(current, addedPolicies, removedPolicies, reallyModifiedPolicies);
            }
        }

        watch.stop();

        LOG.info("Finished loading {} policies, added: {}, removed: {}, modified: {}, taken: {} ms",
            current.size(), addedPolicies.size(), removedPolicies.size(), potentiallyModifiedPolicies.size(), watch.elapsed(TimeUnit.MILLISECONDS));
        // reset cached policies
        cachedPolicies = currPolicies;
    } catch (Throwable t) {
        LOG.warn("Error loading policy, but continue to run", t);
    }
}
 

@Override
protected void execute(OptionsHelper optionsHelper) throws Exception {
    String action = optionsHelper.getOptionValue(OPTION_ACTION);
    String hybridName = optionsHelper.getOptionValue(OPTION_HYBRID_NAME);
    String projectName = optionsHelper.getOptionValue(OPTION_PROJECT);
    String modelName = optionsHelper.getOptionValue(OPTION_MODEL);
    String cubeNamesStr = optionsHelper.getOptionValue(OPTION_CUBES);
    boolean checkCubeSize = optionsHelper.hasOption(OPTION_CHECK) ? Boolean.parseBoolean(optionsHelper.getOptionValue(OPTION_CHECK)) : true;

    HybridInstance hybridInstance = hybridManager.getHybridInstance(hybridName);

    if ("delete".equals(action)) {
        if (hybridInstance == null) {
            throw new IllegalArgumentException("The Hybrid Cube doesn't exist, could not delete: " + hybridName);
        }
        // Delete the Hybrid
        delete(hybridInstance);
        return;
    }

    String[] cubeNames = new String[] {};
    if (cubeNamesStr != null)
        cubeNames = cubeNamesStr.split(",");
    String owner = null;

    DataModelDesc modelDesc = metadataManager.getDataModelDesc(modelName);
    if (modelDesc == null) {
        throw new IllegalArgumentException("Could not find model: " + modelName);
    }

    List<RealizationEntry> realizationEntries = new ArrayList<RealizationEntry>();
    for (String cubeName : cubeNames) {
        if (StringUtils.isEmpty(cubeName))
            continue;
        CubeInstance cube = cubeManager.getCube(cubeName);
        if (cube == null) {
            throw new IllegalArgumentException("Could not find cube: " + cubeName);
        }
        if (owner == null) {
            owner = cube.getOwner();
        }
        realizationEntries.add(RealizationEntry.create(RealizationType.CUBE, cube.getName()));
    }

    int realizationEntriesLen = realizationEntries.size();
    HashSet<RealizationEntry> hashSet = new HashSet<>();
    for (int i = 0; i < realizationEntriesLen; i++) {
        hashSet.add(realizationEntries.get(i));
    }
    int hashSetLen = hashSet.size();
    if (realizationEntriesLen != hashSetLen) {
        Collection<RealizationEntry> duplicateCubes = CollectionUtils.subtract(realizationEntries, hashSet);
        throw new IllegalArgumentException("The Cubes name does duplicate, could not create: " + duplicateCubes);
    }

    if ("create".equals(action)) {
        if (hybridInstance != null) {
            throw new IllegalArgumentException("The Hybrid Cube does exist, could not create: " + hybridName);
        }
        //Create new Hybrid
        create(hybridName, realizationEntries, projectName, owner);
    } else if ("update".equals(action)) {
        if (hybridInstance == null) {
            throw new IllegalArgumentException("The Hybrid Cube doesn't exist, could not update: " + hybridName);
        }
        // Update the Hybrid
        update(hybridInstance, realizationEntries, projectName, owner, checkCubeSize);
    }
}
 

private Collection<String> getNotSkippedTasks() {
    List all = getExecutedTasks();
    Set skipped = getSkippedTasks();
    return CollectionUtils.subtract(all, skipped);
}