com.carrotsearch.hppc.ObjectIntOpenHashMap Java Examples

The following examples show how to use com.carrotsearch.hppc.ObjectIntOpenHashMap. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: NIFDatasetLoadingTest.java    From gerbil with GNU Affero General Public License v3.0 6 votes vote down vote up
public static void main(String[] args) {
    if (args.length != 1) {
        LOGGER.error(
                "Wrong usage. Need exactly one single argument. Correct usage: 'NIFDatasetLoadingTest <path-to-nif-file>'.");
        return;
    }
    LOGGER.info("Starting loading of given test dataset \"" + args[0] + "\"...");
    FileBasedNIFDataset dataset = new FileBasedNIFDataset(args[0], "Test dataset", Lang.TTL);
    try {
        dataset.init();
    } catch (GerbilException e) {
        LOGGER.error("Got an exception while trying to load the dataset.", e);
    }
    List<Document> documents = dataset.getInstances();
    LOGGER.info("Dataset size: {} documents", documents.size());
    ObjectIntOpenHashMap<String> annotationTypes;
    for (Document document : documents) {
        annotationTypes = listAnnotationTypes(document);
        LOGGER.info("Document {} annotation types: {}", document.getDocumentURI(), annotationTypes.toString());
    }
    IOUtils.closeQuietly(dataset);
    LOGGER.info("Finished loading of given test dataset.");
}
 
Example #2
Source File: Dictionary.java    From arx with Apache License 2.0 6 votes vote down vote up
/**
 * Definalizes the dictionary
 */
@SuppressWarnings("unchecked")
public void definalizeAll() {
    
    // Re-instantiate maps
    maps = new ObjectIntOpenHashMap[mapping.length];
    for (int i = 0; i < maps.length; i++) {
        maps[i] = new ObjectIntOpenHashMap<String>();
    }
    
    // Add from mapping
    for (int i = 0; i < mapping.length; i++) {
        if (mapping[i] != null) {
            for (int j = 0; j < mapping[i].length; j++) {
                maps[i].put(mapping[i][j], j);
            }
        }
    }
    
    // Remove mapping
    for (int i = 0; i < mapping.length; i++) {
        mapping[i] = null;
    }
}
 
Example #3
Source File: FileBasedCachingSameAsRetriever.java    From gerbil with GNU Affero General Public License v3.0 5 votes vote down vote up
@SuppressWarnings("unchecked")
public static FileBasedCachingSameAsRetriever create(SameAsRetriever decoratedRetriever,
        boolean requestEntitiesNotFound, File cacheFile) {
    File tempCacheFile = new File(cacheFile.getAbsolutePath() + "_temp");

    Object objects[] = null;
    // try to read the cache file
    objects = readCacheFile(cacheFile);
    // if this doesn't work, try to read the temp file
    if (objects == null) {
        LOGGER.warn("Couldn't read the cache file. Trying the temporary file...");
        objects = readCacheFile(tempCacheFile);
        // if this worked, rename the temp file to the real file
        if (objects != null) {
            try {
                if (!tempCacheFile.renameTo(cacheFile)) {
                    LOGGER.warn("Reading from the temporary cache file worked, but I couldn't rename it.");
                }
            } catch (Exception e) {
                LOGGER.warn("Reading from the temporary cache file worked, but I couldn't rename it.", e);
            }
        }
    }
    ObjectIntOpenHashMap<String> uriSetIdMapping;
    List<Set<String>> sets;
    // if the reading didn't worked, create new cache objects
    if (objects == null) {
        LOGGER.warn("Couldn't read cache from files. Creating new empty cache.");
        uriSetIdMapping = new ObjectIntOpenHashMap<String>();
        sets = new ArrayList<Set<String>>();
    } else {
        uriSetIdMapping = (ObjectIntOpenHashMap<String>) objects[0];
        sets = (List<Set<String>>) objects[1];
    }
    return new FileBasedCachingSameAsRetriever(decoratedRetriever, uriSetIdMapping, sets, requestEntitiesNotFound,
            cacheFile, tempCacheFile);
}
 
Example #4
Source File: FileBasedCachingSameAsRetriever.java    From gerbil with GNU Affero General Public License v3.0 5 votes vote down vote up
protected FileBasedCachingSameAsRetriever(SameAsRetriever decoratedRetriever,
        ObjectIntOpenHashMap<String> uriSetIdMapping, List<Set<String>> sets, boolean requestEntitiesNotFound,
        File cacheFile, File tempCacheFile) {
    super(decoratedRetriever);
    this.uriSetIdMapping = uriSetIdMapping;
    this.sets = sets;
    this.requestEntitiesNotFound = requestEntitiesNotFound;
    this.cacheFile = cacheFile;
    this.tempCacheFile = tempCacheFile;
}
 
Example #5
Source File: NIFDatasetLoadingTest.java    From gerbil with GNU Affero General Public License v3.0 5 votes vote down vote up
private static ObjectIntOpenHashMap<String> listAnnotationTypes(Document document) {
    ObjectIntOpenHashMap<String> annotationTypes = new ObjectIntOpenHashMap<String>();
    String className;
    for (Marking marking : document.getMarkings()) {
        className = marking.getClass().getSimpleName();
        annotationTypes.putOrAdd(className, 1, 1);
    }
    return annotationTypes;
}
 
Example #6
Source File: AbstractSymbolTable.java    From jopenfst with MIT License 5 votes vote down vote up
protected AbstractSymbolTable(SymbolTable copyFrom) {

    this.symbolToId = new ObjectIntOpenHashMap<>(copyFrom.size());
    this.idToSymbol = new IntObjectOpenHashMap<>(copyFrom.size());
    for (ObjectIntCursor<String> cursor : copyFrom) {
      symbolToId.put(cursor.key, cursor.value);
      idToSymbol.put(cursor.value, cursor.key);
    }
  }
 
Example #7
Source File: DataHandle.java    From arx with Apache License 2.0 5 votes vote down vote up
/**
 * Sets the current header
 * @param header
 */
protected void setHeader(String[] header) {
    this.header = header;
    this.headerMap = new ObjectIntOpenHashMap<String>();
    for (int i = 0; i < header.length; i++) {
        headerMap.put(header[i], i);
    }
}
 
Example #8
Source File: Dictionary.java    From arx with Apache License 2.0 5 votes vote down vote up
/**
 * Instantiates a new dictionary by extracting a projection of the given dictionary
 * 
 * @param dimensions
 */
@SuppressWarnings("unchecked")
public Dictionary(Dictionary input, int[] columns) {
    maps = new ObjectIntOpenHashMap[columns.length];
    mapping = new String[columns.length][];
    suppressed = new int[columns.length];
    for (int i = 0; i < columns.length; i++) {
        maps[i] = input.maps == null ? null : input.maps[columns[i]].clone();
        mapping[i] = input.mapping[columns[i]].clone();
        suppressed[i] = input.suppressed[columns[i]];
    }
}
 
Example #9
Source File: Dictionary.java    From arx with Apache License 2.0 5 votes vote down vote up
/**
 * Instantiates a new dictionary.
 * 
 * @param dimensions
 */
@SuppressWarnings("unchecked")
public Dictionary(final int dimensions) {
    maps = new ObjectIntOpenHashMap[dimensions];
    mapping = new String[dimensions][];
    suppressed = new int[dimensions];
    for (int i = 0; i < dimensions; i++) {
        maps[i] = new ObjectIntOpenHashMap<String>();
    }
}
 
Example #10
Source File: Dictionary.java    From arx with Apache License 2.0 5 votes vote down vote up
/**
 * Registers a new string at the dictionary.
 * 
 * @param dimension
 *            the dimension
 * @param string
 *            the string
 * @return the int
 */
public int register(final int dimension, final String string) {

    // Prepare
    ObjectIntOpenHashMap<String> map = maps[dimension];
    int size = map.size();

    // Return or store
    if (map.putIfAbsent(string, size)) {
        return size;
    } else {
        return map.lget();
    }
}
 
Example #11
Source File: FileBasedCachingSameAsRetrieverTest.java    From gerbil with GNU Affero General Public License v3.0 4 votes vote down vote up
public FileBasedCachingSameAsRetrieverTest() throws IOException {
    super(new RandomNumberReturningSameAsRetriever(), new ObjectIntOpenHashMap<String>(),
            new ArrayList<Set<String>>(), false, File.createTempFile("cache_test_", ".cache"),
            File.createTempFile("cache_test_", ".cache_temp"));
}
 
Example #12
Source File: AbstractSymbolTable.java    From jopenfst with MIT License 4 votes vote down vote up
protected AbstractSymbolTable() {
  this.symbolToId = new ObjectIntOpenHashMap<>();
  this.idToSymbol = new IntObjectOpenHashMap<>();
}
 
Example #13
Source File: HashGroupify.java    From arx with Apache License 2.0 4 votes vote down vote up
/**
 * Returns a data object with microaggregation performed
 * @param microaggregationData
 * @param dictionary
 * @return
 */
public Data performMicroaggregation(DataAggregationInformation microaggregationData,
                                    Dictionary dictionary) {
    
    // Initialize
    int[] indices = microaggregationData.getMicroaggregationIndices();
    DistributionAggregateFunction[] functions = microaggregationData.getMicroaggregationFunctions();
    String[] header = microaggregationData.getMicroaggregationHeader();
    int[] columns = microaggregationData.getMicroaggregationColumns();
    
    // Prepare result
    Data result = Data.createWrapper(new DataMatrix(dataOutput.getNumRows(), indices.length), header, columns, dictionary);

    // TODO: To improve performance, microaggregation and marking of outliers could be performed in one pass
    ObjectIntOpenHashMap<Distribution> cache = new ObjectIntOpenHashMap<Distribution>();
    for (int row = 0; row < dataOutput.getNumRows(); row++) {
        if (privacyModelDefinesSubset == null || privacyModelDefinesSubset.contains(row)) {
            final int hash = dataOutput.hashCode(row);
            final int index = hash & (hashTableBuckets.length - 1);
            HashGroupifyEntry m = hashTableBuckets[index];
            while ((m != null) && ((m.hashcode != hash) || !dataOutput.equalsIgnoringOutliers(row, m.row))) {
                m = m.next;
            }
            if (m == null) { throw new RuntimeException("Invalid state! Groupify the data before performing microaggregation!"); }
            result.getArray().iterator(row);
            for (int i = 0; i < indices.length; i++) {
                int columnIndex = indices[i];
                Distribution distribution = m.distributions[columnIndex];
                int code = cache.getOrDefault(distribution, -1);
                if (code == -1) {
                    String value = functions[i].aggregate(distribution);
                    code = result.getDictionary().register(i, value);
                    cache.put(distribution, code);
                }
                result.getArray().iterator_write(code);
            }
        }
    }
    
    // Finalize
    result.getDictionary().finalizeAll();
    
    // Returns the result
    return result;
}
 
Example #14
Source File: Dictionary.java    From arx with Apache License 2.0 2 votes vote down vote up
/**
 * Returns the map with unfinalized values for the given dimension
 * @param dimension
 * @return
 */
public ObjectIntOpenHashMap<String> getUnfinalizedValues(final int dimension) {
    return maps[dimension];
}