com.thinkaurelius.titan.diskstorage.BackendException Java Examples

@Override
public BerkeleyJETx beginTransaction(final BaseTransactionConfig txCfg) throws BackendException {
    try {
        Transaction tx = null;

        Configuration effectiveCfg =
                new MergedConfiguration(txCfg.getCustomOptions(), getStorageConfig());

        if (transactional) {
            TransactionConfig txnConfig = new TransactionConfig();
            ConfigOption.getEnumValue(effectiveCfg.get(ISOLATION_LEVEL),IsolationLevel.class).configure(txnConfig);
            tx = environment.beginTransaction(null, txnConfig);
        }
        BerkeleyJETx btx = new BerkeleyJETx(tx, ConfigOption.getEnumValue(effectiveCfg.get(LOCK_MODE),LockMode.class), txCfg);

        if (log.isTraceEnabled()) {
            log.trace("Berkeley tx created", new TransactionBegin(btx.toString()));
        }

        return btx;
    } catch (DatabaseException e) {
        throw new PermanentBackendException("Could not start BerkeleyJE transaction", e);
    }
}
 

@Override
public void mutateMany(Map<String, KVMutation> mutations, StoreTransaction txh) throws BackendException {
    for (Map.Entry<String,KVMutation> muts : mutations.entrySet()) {
        BerkeleyJEKeyValueStore store = openDatabase(muts.getKey());
        KVMutation mut = muts.getValue();

        if (!mut.hasAdditions() && !mut.hasDeletions()) {
            log.debug("Empty mutation set for {}, doing nothing", muts.getKey());
        } else {
            log.debug("Mutating {}", muts.getKey());
        }

        if (mut.hasAdditions()) {
            for (KeyValueEntry entry : mut.getAdditions()) {
                store.insert(entry.getKey(),entry.getValue(),txh);
                log.trace("Insertion on {}: {}", muts.getKey(), entry);
            }
        }
        if (mut.hasDeletions()) {
            for (StaticBuffer del : mut.getDeletions()) {
                store.delete(del,txh);
                log.trace("Deletion on {}: {}", muts.getKey(), del);
            }
        }
    }
}
 

/**
 * Titan Geoshapes are converted to a string that gets sent to its respective index. Unfortunately, the string format
 * is not compatible with Solr 4. The GeoToWktConverter transforms the Geoshape's string value into a Well-Known Text
 * format understood by Solr.
 */
@Test
public void testConvertGeoshapePointToWktString() throws BackendException {
    Geoshape p1 = Geoshape.point(35.4, 48.9); //no spaces, no negative values
    Geoshape p2 = Geoshape.point(-35.4,48.9); //negative longitude value
    Geoshape p3 = Geoshape.point(35.4, -48.9); //negative latitude value

    String wkt1 = "POINT(48.9 35.4)";
    String actualWkt1 = GeoToWktConverter.convertToWktString(p1);

    String wkt2 = "POINT(48.9 -35.4)";
    String actualWkt2 = GeoToWktConverter.convertToWktString(p2);

    String wkt3 = "POINT(-48.9 35.4)";
    String actualWkt3 = GeoToWktConverter.convertToWktString(p3);

    assertEquals(wkt1, actualWkt1);
    assertEquals(wkt2, actualWkt2);
    assertEquals(wkt3, actualWkt3);

}
 

@Override
public StaticBuffer get(StaticBuffer key, StoreTransaction txh) throws BackendException {
    Transaction tx = getTransaction(txh);
    try {
        DatabaseEntry dbkey = key.as(ENTRY_FACTORY);
        DatabaseEntry data = new DatabaseEntry();

        log.trace("db={}, op=get, tx={}", name, txh);

        OperationStatus status = db.get(tx, dbkey, data, getLockMode(txh));

        if (status == OperationStatus.SUCCESS) {
            return getBuffer(data);
        } else {
            return null;
        }
    } catch (DatabaseException e) {
        throw new PermanentBackendException(e);
    }
}
 

@Override
public void clearStorage() throws BackendException {
    try {
        Cluster cluster = clusterContext.getClient();

        Keyspace ks = cluster.getKeyspace(keySpaceName);

        // Not a big deal if Keyspace doesn't not exist (dropped manually by user or tests).
        // This is called on per test setup basis to make sure that previous test cleaned
        // everything up, so first invocation would always fail as Keyspace doesn't yet exist.
        if (ks == null)
            return;

        for (ColumnFamilyDefinition cf : cluster.describeKeyspace(keySpaceName).getColumnFamilyList()) {
            ks.truncateColumnFamily(new ColumnFamily<Object, Object>(cf.getName(), null, null));
        }
    } catch (ConnectionException e) {
        throw new PermanentBackendException(e);
    }
}
 

/**
 * Locks with timestamps equal to or numerically greater than the cleaner
 * cutoff timestamp must be preserved. Test that the cleaner reads locks by
 * slicing the store and then does <b>not</b> attempt to write.
 */
@Test
public void testPreservesLocksAtOrAfterCutoff() throws BackendException {
    final Instant cutoff = Instant.ofEpochMilli(10L);

    Entry currentLock = StaticArrayEntry.of(codec.toLockCol(cutoff,
            defaultLockRid, TimestampProviders.MILLI), BufferUtil.getIntBuffer(0));
    Entry futureLock = StaticArrayEntry.of(codec.toLockCol(cutoff.plusMillis(1),
            defaultLockRid, TimestampProviders.MILLI), BufferUtil.getIntBuffer(0));
    EntryList locks = StaticArrayEntryList.of(currentLock, futureLock);

    // Don't increment cutoff: lockCol is exactly at the cutoff timestamp
    del = new StandardLockCleanerRunnable(store, kc, tx, codec, cutoff, TimestampProviders.MILLI);

    expect(store.getSlice(eq(ksq), eq(tx))).andReturn(locks);

    ctrl.replay();
    del.run();
}
 

private static RetryPolicy getRetryPolicy(String serializedRetryPolicy) throws BackendException {
    String[] tokens = serializedRetryPolicy.split(",");
    String policyClassName = tokens[0];
    int argCount = tokens.length - 1;
    Integer[] args = new Integer[argCount];
    for (int i = 1; i < tokens.length; i++) {
        args[i - 1] = Integer.valueOf(tokens[i]);
    }

    try {
        RetryPolicy rp = instantiate(policyClassName, args, serializedRetryPolicy);
        log.debug("Instantiated RetryPolicy object {} from config string \"{}\"", rp, serializedRetryPolicy);
        return rp;
    } catch (Exception e) {
        throw new PermanentBackendException("Failed to instantiate Astyanax Retry Policy class", e);
    }
}
 

@Override
public Map<String, String> getCompressionOptions(String cf) throws BackendException {
    try {
        Keyspace k = keyspaceContext.getClient();

        KeyspaceDefinition kdef = k.describeKeyspace();

        if (null == kdef) {
            throw new PermanentBackendException("Keyspace " + k.getKeyspaceName() + " is undefined");
        }

        ColumnFamilyDefinition cfdef = kdef.getColumnFamily(cf);

        if (null == cfdef) {
            throw new PermanentBackendException("Column family " + cf + " is undefined");
        }

        return cfdef.getCompressionOptions();
    } catch (ConnectionException e) {
        throw new PermanentBackendException(e);
    }
}
 

@Override
public WriteConfiguration getConfig() {
    final KeyColumnValueStoreManager manager = new InMemoryStoreManager(Configuration.EMPTY);
    ModifiableConfiguration config = GraphDatabaseConfiguration.buildGraphConfiguration();
    config.set(GraphDatabaseConfiguration.TIMESTAMP_PROVIDER, TimestampProviders.MICRO);
    try {
        return new KCVSConfiguration(new BackendOperation.TransactionalProvider() {
            @Override
            public StoreTransaction openTx() throws BackendException {
                return manager.beginTransaction(StandardBaseTransactionConfig.of(TimestampProviders.MICRO, manager.getFeatures().getKeyConsistentTxConfig()));
            }

            @Override
            public void close() throws BackendException {
                manager.close();
            }
        }, config, manager.openDatabase("titan"),"general");
    } catch (BackendException e) {
        throw new RuntimeException(e);
    }
}
 

@Test(expectedExceptions = PermanentLockingException.class)
public void shouldThrowExceptionAfterConfiguredRetriesIfLockMediationFails() throws BackendException {
    when(storeManager.getMetaDataSchema("hbase")).thenReturn(new EntryMetaData[] {EntryMetaData.TIMESTAMP});
    when(storeManager.getStorageConfig()).thenReturn(storageConfig);
    when(storageConfig.get(GraphDatabaseConfiguration.LOCK_EXPIRE)).thenReturn(
            new StandardDuration(300L, TimeUnit.MILLISECONDS));
    when(storageConfig.get(GraphDatabaseConfiguration.LOCK_WAIT)).thenReturn(
            new StandardDuration(10L, TimeUnit.MILLISECONDS));
    when(storageConfig.get(GraphDatabaseConfiguration.LOCK_RETRY)).thenReturn(3);
    KeyColumn lockID = new KeyColumn(key, column);
    when(localLockMediator.lock(eq(lockID), eq(transaction), any(Timepoint.class))).
            thenReturn(false).thenReturn(false).thenReturn(false);

    HBaseKeyColumnValueStore hBaseKeyColumnValueStore =
            new HBaseKeyColumnValueStore(storeManager, connectionMask, "titan", "e", "hbase", localLockMediator);
    hBaseKeyColumnValueStore.acquireLock(key, column, expectedValue, transaction);

    fail("Should fail as lock could not be acquired after 3 retries.");
}
 

@Override
public synchronized KCVSLog openLog(final String name) throws BackendException {
    if (openLogs.containsKey(name)) return openLogs.get(name);
    StoreMetaData.Container storeOptions = new StoreMetaData.Container();
    if (0 < indexStoreTTL) {
        storeOptions.put(StoreMetaData.TTL, indexStoreTTL);
    }
    KCVSLog log = new KCVSLog(name,this,storeManager.openDatabase(name, storeOptions),configuration);
    openLogs.put(name,log);
    return log;
}
 

public void open() throws BackendException {
    index = openIndex();
    indexFeatures = index.getFeatures();
    allKeys = getMapping(indexFeatures);
    indexRetriever = getIndexRetriever(allKeys);

    newTx();
}
 

@Override
public void mutateMany(Map<String, Map<StaticBuffer, KCVMutation>> mutations, StoreTransaction txh) throws BackendException {
    ExpectedValueCheckingTransaction etx = (ExpectedValueCheckingTransaction)txh;
    boolean hasAtLeastOneLock = etx.prepareForMutations();
    if (hasAtLeastOneLock) {
        // Force all mutations on this transaction to use strong consistency
        log.debug("Transaction {} holds one or more locks: writing using consistent transaction {} due to held locks", etx, etx.getConsistentTx());
        manager.mutateMany(mutations, etx.getConsistentTx());
    } else {
        log.debug("Transaction {} holds no locks: writing mutations using store transaction {}", etx, etx.getInconsistentTx());
        manager.mutateMany(mutations, etx.getInconsistentTx());
    }
}
 

@Override
public synchronized AstyanaxKeyColumnValueStore openDatabase(String name, StoreMetaData.Container metaData) throws BackendException {
    if (openStores.containsKey(name)) return openStores.get(name);
    else {
        ensureColumnFamilyExists(name);
        AstyanaxKeyColumnValueStore store = new AstyanaxKeyColumnValueStore(name, keyspaceContext.getClient(), this, retryPolicy);
        openStores.put(name, store);
        return store;
    }
}
 

public LocalStoreManager(Configuration storageConfig) throws BackendException {
    super(storageConfig);
    String storageDir = storageConfig.get(STORAGE_DIRECTORY);
    if (null == storageDir) {
        directory = null;
    } else { 
        directory = DirectoryUtil.getOrCreateDataDirectory(storageDir);
    }
}
 

/**
 * {@link com.thinkaurelius.titan.core.attribute.Geoshape} stores Points in the String format: point[X.0,Y.0].
 * Solr needs it to be in Well-Known Text format: POINT(X.0 Y.0)
 */
static String convertToWktString(Geoshape fieldValue) throws BackendException {
    if (fieldValue.getType() == Geoshape.Type.POINT) {
        Geoshape.Point point = fieldValue.getPoint();
        return "POINT(" + point.getLongitude() + " " + point.getLatitude() + ")";
    } else {
        throw new PermanentBackendException("Cannot index " + fieldValue.getType());
    }
}
 

private void closeLogs() {
    try {
        for (LogManager lm : logManagers.values()) lm.close();
        logManagers.clear();
        if (logStoreManager!=null) {
            logStoreManager.close();
            logStoreManager=null;
        }
    } catch (BackendException e) {
        throw new TitanException(e);
    }
}
 

private Object convertValue(Object value) throws BackendException {
    if (value instanceof Geoshape)
        return GeoToWktConverter.convertToWktString((Geoshape) value);
    // in order to serialize/deserialize properly Solr will have to have an
    // access to Titan source which has Decimal type, so for now we simply convert to
    // double and let Solr do the same thing or fail.
    if (value instanceof AbstractDecimal)
        return ((AbstractDecimal) value).doubleValue();
    if (value instanceof UUID)
        return value.toString();
    return value;
}
 

@Override
public void mutate(final StaticBuffer key,
                   final List<Entry> additions,
                   final List<StaticBuffer> deletions,
                   final StoreTransaction txh) throws BackendException {
    runWithMetrics(txh, metricsStoreName, M_MUTATE,
            new StorageCallable<Void>() {
                public Void call() throws BackendException {
                    backend.mutate(key, additions, deletions, txh);
                    return null;
                }
            }
    );
}
 

public static final<V> V executeDirect(Callable<V> exe, Duration totalWaitTime) throws BackendException {
    Preconditions.checkArgument(!totalWaitTime.isZero(),"Need to specify a positive waitTime: %s",totalWaitTime);
    long maxTime = System.currentTimeMillis()+totalWaitTime.toMillis();
    Duration waitTime = pertubateTime(BASE_REATTEMPT_TIME);
    BackendException lastException;
    while (true) {
        try {
            return exe.call();
        } catch (final Throwable e) {
            //Find inner-most StorageException
            Throwable ex = e;
            BackendException storeEx = null;
            do {
                if (ex instanceof BackendException) storeEx = (BackendException)ex;
            } while ((ex=ex.getCause())!=null);
            if (storeEx!=null && storeEx instanceof TemporaryBackendException) {
                lastException = storeEx;
            } else if (e instanceof BackendException) {
                throw (BackendException)e;
            } else {
                throw new PermanentBackendException("Permanent exception while executing backend operation "+exe.toString(),e);
            }
        }
        //Wait and retry
        assert lastException!=null;
        if (System.currentTimeMillis()+waitTime.toMillis()<maxTime) {
            log.info("Temporary exception during backend operation ["+exe.toString()+"]. Attempting backoff retry.",lastException);
            try {
                Thread.sleep(waitTime.toMillis());
            } catch (InterruptedException r) {
                throw new PermanentBackendException("Interrupted while waiting to retry failed backend operation", r);
            }
        } else {
            break;
        }
        waitTime = pertubateTime(waitTime.multipliedBy(2));
    }
    throw new TemporaryBackendException("Could not successfully complete backend operation due to repeated temporary exceptions after "+totalWaitTime,lastException);
}
 

@Override
public void acquireLock(final StaticBuffer key,
                        final StaticBuffer column,
                        final StaticBuffer expectedValue,
                        final StoreTransaction txh) throws BackendException {
    runWithMetrics(txh, metricsStoreName, M_ACQUIRE_LOCK,
        new StorageCallable<Void>() {
            public Void call() throws BackendException {
                backend.acquireLock(key, column, expectedValue, txh);
                return null;
            }
        }
    );
}
 

@Override
public ExpectedValueCheckingTransaction beginTransaction(BaseTransactionConfig configuration) throws BackendException {
    // Get a transaction without any guarantees about strong consistency
    StoreTransaction inconsistentTx = manager.beginTransaction(configuration);

    // Get a transaction that provides global strong consistency
    Configuration customOptions = new MergedConfiguration(storeFeatures.getKeyConsistentTxConfig(), configuration.getCustomOptions());
    BaseTransactionConfig consistentTxCfg = new StandardBaseTransactionConfig.Builder(configuration)
            .customOptions(customOptions).build();
    StoreTransaction strongConsistentTx = manager.beginTransaction(consistentTxCfg);

    // Return a wrapper around both the inconsistent and consistent store transactions
    ExpectedValueCheckingTransaction wrappedTx = new ExpectedValueCheckingTransaction(inconsistentTx, strongConsistentTx, maxReadTime);
    return wrappedTx;
}
 

public void loadStore(int numKeys, int numCols) {
    StoreTransaction tx = getStoreTx();
    try {
        for (int i=1;i<=numKeys;i++) {
            List<Entry> adds = new ArrayList<Entry>(numCols);
            for (int j=1;j<=numCols;j++) adds.add(getEntry(j,j));
            store.mutate(BufferUtil.getIntBuffer(i),adds,KeyColumnValueStore.NO_DELETIONS,tx);
        }
        tx.commit();
    } catch (BackendException e) {
        throw new RuntimeException(e);
    }
}
 

@Override
public synchronized void rollback() throws BackendException {
    super.rollback();
    if (tx == null) return;
    if (log.isTraceEnabled())
        log.trace("{} rolled back", this.toString(), new TransactionClose(this.toString()));
    try {
        closeOpenIterators();
        tx.abort();
        tx = null;
    } catch (DatabaseException e) {
        throw new PermanentBackendException(e);
    }
}
 

@Override
public KeyIterator getKeys(final KeyRangeQuery query, final StoreTransaction txh) throws BackendException {
    return runWithMetrics(txh, metricsStoreName, M_GET_KEYS,
        new StorageCallable<KeyIterator>() {
            public KeyIterator call() throws BackendException {
                KeyIterator ki = backend.getKeys(query, txh);
                if (txh.getConfiguration().hasGroupName()) {
                    return MetricInstrumentedIterator.of(ki,txh.getConfiguration().getGroupName(),metricsStoreName,M_GET_KEYS,M_ITERATOR);
                } else {
                    return ki;
                }
            }
        }
    );
}
 

@Override
public void mutateMany(Map<String, Map<StaticBuffer, KCVMutation>> mutations, StoreTransaction txh) throws BackendException {
    if (!manager.getFeatures().hasStoreTTL()) {
        assert manager.getFeatures().hasCellTTL();
        for (Map.Entry<String,Map<StaticBuffer, KCVMutation>> sentry : mutations.entrySet()) {
            Integer ttl = ttlEnabledStores.get(sentry.getKey());
            if (null != ttl && 0 < ttl) {
                for (KCVMutation mut : sentry.getValue().values()) {
                    if (mut.hasAdditions()) applyTTL(mut.getAdditions(), ttl);
                }
            }
        }
    }
    manager.mutateMany(mutations,txh);
}
 

@Override
public void run() {
    try {
        runWithExceptions();
    } catch (BackendException e) {
        log.warn("Expired lock cleaner failed", e);
    }
}
 

@Test
public void testRemoveRelationIndex() throws InterruptedException, BackendException, ExecutionException {
    tx.commit();
    mgmt.commit();

    // Load the "Graph of the Gods" sample data
    GraphOfTheGodsFactory.loadWithoutMixedIndex(graph, true);

    // Disable the "battlesByTime" index
    TitanManagement m = graph.openManagement();
    RelationType battled = m.getRelationType("battled");
    RelationTypeIndex battlesByTime = m.getRelationIndex(battled, "battlesByTime");
    m.updateIndex(battlesByTime, SchemaAction.DISABLE_INDEX);
    m.commit();
    graph.tx().commit();

    // Block until the SchemaStatus transitions to DISABLED
    assertTrue(ManagementSystem.awaitRelationIndexStatus(graph, "battlesByTime", "battled")
            .status(SchemaStatus.DISABLED).call().getSucceeded());

    // Remove index
    MapReduceIndexManagement mri = new MapReduceIndexManagement(graph);
    m = graph.openManagement();
    battled = m.getRelationType("battled");
    battlesByTime = m.getRelationIndex(battled, "battlesByTime");
    ScanMetrics metrics = mri.updateIndex(battlesByTime, SchemaAction.REMOVE_INDEX).get();

    assertEquals(6, metrics.getCustom(IndexRemoveJob.DELETED_RECORDS_COUNT));
}
 

public BerkeleyJEStoreManager(Configuration configuration) throws BackendException {
        super(configuration);
        stores = new HashMap<String, BerkeleyJEKeyValueStore>();

        int cachePercentage = configuration.get(JVM_CACHE);
        initialize(cachePercentage);

        features = new StandardStoreFeatures.Builder()
                    .orderedScan(true)
                    .transactional(transactional)
                    .keyConsistent(GraphDatabaseConfiguration.buildGraphConfiguration())
                    .locking(true)
                    .keyOrdered(true)
                    .scanTxConfig(GraphDatabaseConfiguration.buildGraphConfiguration()
                            .set(ISOLATION_LEVEL, IsolationLevel.READ_UNCOMMITTED.toString()))
                    .supportsInterruption(false)
                    .build();

//        features = new StoreFeatures();
//        features.supportsOrderedScan = true;
//        features.supportsUnorderedScan = false;
//        features.supportsBatchMutation = false;
//        features.supportsTxIsolation = transactional;
//        features.supportsConsistentKeyOperations = true;
//        features.supportsLocking = true;
//        features.isKeyOrdered = true;
//        features.isDistributed = false;
//        features.hasLocalKeyPartition = false;
//        features.supportsMultiQuery = false;
    }
 

public void clopen() throws BackendException {
    close();
    open();
}