Java Code Examples for com.thinkaurelius.titan.util.system.IOUtils

The following are top voted examples for showing how to use com.thinkaurelius.titan.util.system.IOUtils. These examples are extracted from open source projects. You can vote up the examples you like and your votes will be used in our system to generate more good examples.
Example 1
Project: titan1withtp3.1   File: ElasticsearchStatus.java   Source Code and License 6 votes vote down vote up
public static ElasticsearchStatus read(String path) {

        File pid = new File(path);

        if (!pid.exists()) {
            log.info("ES pidfile {} does not exist", path);
            return null;
        }

        BufferedReader pidReader = null;
        try {
            pidReader = new BufferedReader(new FileReader(pid));
            ElasticsearchStatus s = parsePidFile(pid, pidReader);
            log.info("Read ES pid {} from {}", pid, path);
            return s;
        } catch (IOException e) {
            log.warn("Assuming ES is not running", e);
        } finally {
            IOUtils.closeQuietly(pidReader);
        }

        return null;
    }
 
Example 2
Project: titan1withtp3.1   File: HBaseStoreManager.java   Source Code and License 6 votes vote down vote up
private HTableDescriptor ensureTableExists(String tableName, String initialCFName, int ttlInSeconds) throws BackendException {
    AdminMask adm = null;

    HTableDescriptor desc;

    try { // Create our table, if necessary
        adm = getAdminInterface();
        /*
         * Some HBase versions/impls respond badly to attempts to create a
         * table without at least one CF. See #661. Creating a CF along with
         * the table avoids HBase carping.
         */
        if (adm.tableExists(tableName)) {
            desc = adm.getTableDescriptor(tableName);
        } else {
            desc = createTable(tableName, initialCFName, ttlInSeconds, adm);
        }
    } catch (IOException e) {
        throw new TemporaryBackendException(e);
    } finally {
        IOUtils.closeQuietly(adm);
    }

    return desc;
}
 
Example 3
Project: incubator-atlas   File: HBaseStoreManager.java   Source Code and License 6 votes vote down vote up
private HTableDescriptor ensureTableExists(String tableName, String initialCFName, int ttlInSeconds) throws BackendException {
    AdminMask adm = null;

    HTableDescriptor desc;

    try { // Create our table, if necessary
        adm = getAdminInterface();
        /*
         * Some HBase versions/impls respond badly to attempts to create a
         * table without at least one CF. See #661. Creating a CF along with
         * the table avoids HBase carping.
         */
        if (adm.tableExists(tableName)) {
            desc = adm.getTableDescriptor(tableName);
        } else {
            desc = createTable(tableName, initialCFName, ttlInSeconds, adm);
        }
    } catch (IOException e) {
        throw new TemporaryBackendException(e);
    } finally {
        IOUtils.closeQuietly(adm);
    }

    return desc;
}
 
Example 4
Project: titan1.0.1.kafka   File: ElasticsearchStatus.java   Source Code and License 6 votes vote down vote up
public static ElasticsearchStatus read(String path) {

        File pid = new File(path);

        if (!pid.exists()) {
            log.info("ES pidfile {} does not exist", path);
            return null;
        }

        BufferedReader pidReader = null;
        try {
            pidReader = new BufferedReader(new FileReader(pid));
            ElasticsearchStatus s = parsePidFile(pid, pidReader);
            log.info("Read ES pid {} from {}", pid, path);
            return s;
        } catch (IOException e) {
            log.warn("Assuming ES is not running", e);
        } finally {
            IOUtils.closeQuietly(pidReader);
        }

        return null;
    }
 
Example 5
Project: titan1.0.1.kafka   File: HBaseStoreManager.java   Source Code and License 6 votes vote down vote up
private HTableDescriptor ensureTableExists(String tableName, String initialCFName, int ttlInSeconds) throws BackendException {
    AdminMask adm = null;

    HTableDescriptor desc;

    try { // Create our table, if necessary
        adm = getAdminInterface();
        /*
         * Some HBase versions/impls respond badly to attempts to create a
         * table without at least one CF. See #661. Creating a CF along with
         * the table avoids HBase carping.
         */
        if (adm.tableExists(tableName)) {
            desc = adm.getTableDescriptor(tableName);
        } else {
            desc = createTable(tableName, initialCFName, ttlInSeconds, adm);
        }
    } catch (IOException e) {
        throw new TemporaryBackendException(e);
    } finally {
        IOUtils.closeQuietly(adm);
    }

    return desc;
}
 
Example 6
Project: titan0.5.4-hbase1.1.1-custom   File: ElasticsearchStatus.java   Source Code and License 6 votes vote down vote up
public static ElasticsearchStatus read(String path) {

        File pid = new File(path);

        if (!pid.exists()) {
            log.info("ES pidfile {} does not exist", path);
            return null;
        }

        BufferedReader pidReader = null;
        try {
            pidReader = new BufferedReader(new FileReader(pid));
            ElasticsearchStatus s = parsePidFile(pid, pidReader);
            log.info("Read ES pid {} from {}", pid, path);
            return s;
        } catch (IOException e) {
            log.warn("Assuming ES is not running", e);
        } finally {
            IOUtils.closeQuietly(pidReader);
        }

        return null;
    }
 
Example 7
Project: titan0.5.4-hbase1.1.1-custom   File: StandardTitanGraph.java   Source Code and License 6 votes vote down vote up
private synchronized void shutdownInternal() throws TitanException {
    if (!isOpen) return;
    try {
        //Unregister instance
        ModifiableConfiguration globalConfig = GraphDatabaseConfiguration.getGlobalSystemConfig(backend);
        globalConfig.remove(REGISTRATION_TIME,config.getUniqueGraphId());

        super.shutdown();
        IOUtils.closeQuietly(serializer);
        idAssigner.close();
        backend.close();
        queryCache.close();
    } catch (BackendException e) {
        throw new TitanException("Could not close storage backend", e);
    } finally {
        isOpen = false;
    }
}
 
Example 8
Project: titan0.5.4-hbase1.1.1-custom   File: HBaseStoreManager.java   Source Code and License 6 votes vote down vote up
private HTableDescriptor ensureTableExists(String tableName, String initialCFName, int ttlInSeconds) throws BackendException {
    HBaseAdmin adm = null;

    HTableDescriptor desc;

    try { // Create our table, if necessary
        adm = getAdminInterface();
        /*
         * Some HBase versions/impls respond badly to attempts to create a
         * table without at least one CF. See #661. Creating a CF along with
         * the table avoids HBase carping.
         */
        if (adm.tableExists(tableName)) {
            desc = adm.getTableDescriptor(tableName.getBytes());
        } else {
            desc = createTable(tableName, initialCFName, ttlInSeconds, adm);
        }
    } catch (IOException e) {
        throw new TemporaryBackendException(e);
    } finally {
        IOUtils.closeQuietly(adm);
    }

    return desc;
}
 
Example 9
Project: titan1withtp3.1   File: StorageSetup.java   Source Code and License 5 votes vote down vote up
public static final void deleteHomeDir(String subdir) {
    File homeDirFile = getHomeDirFile(subdir);
    // Make directory if it doesn't exist
    if (!homeDirFile.exists())
        homeDirFile.mkdirs();
    boolean success = IOUtils.deleteFromDirectory(homeDirFile);
    if (!success) throw new IllegalStateException("Could not remove " + homeDirFile);
}
 
Example 10
Project: titan1withtp3.1   File: BerkeleyJEStoreManager.java   Source Code and License 5 votes vote down vote up
@Override
public void clearStorage() throws BackendException {
    if (!stores.isEmpty())
        throw new IllegalStateException("Cannot delete store, since database is open: " + stores.keySet().toString());

    Transaction tx = null;
    for (String db : environment.getDatabaseNames()) {
        environment.removeDatabase(tx, db);
        log.debug("Removed database {} (clearStorage)", db);
    }
    close();
    IOUtils.deleteFromDirectory(directory);
}
 
Example 11
Project: titan1withtp3.1   File: BerkeleyElasticsearchTest.java   Source Code and License 5 votes vote down vote up
/**
 * Test {@link com.thinkaurelius.titan.example.GraphOfTheGodsFactory#create(String)}.
 */
@Test
public void testGraphOfTheGodsFactoryCreate() {
    File bdbtmp = new File("target/gotgfactory");
    IOUtils.deleteDirectory(bdbtmp, true);

    TitanGraph gotg = GraphOfTheGodsFactory.create(bdbtmp.getPath());
    TitanIndexTest.assertGraphOfTheGods(gotg);
    gotg.close();
}
 
Example 12
Project: titan1withtp3.1   File: ElasticsearchStatus.java   Source Code and License 5 votes vote down vote up
public static ElasticsearchStatus write(String path, int pid) {
    File f = new File(path);

    FileOutputStream fos = null;
    try {
        fos = new FileOutputStream(path);
        fos.write(String.format("%d", pid).getBytes());
    } catch (IOException e) {
        throw new RuntimeException(e);
    } finally {
        IOUtils.closeQuietly(fos);
    }

    return new ElasticsearchStatus(f, pid);
}
 
Example 13
Project: titan1withtp3.1   File: MapReduceIndexJobs.java   Source Code and License 5 votes vote down vote up
public static ScanMetrics cassandraRepair(String titanPropertiesPath, String indexName, String relationType, String partitionerName)
        throws InterruptedException, IOException, ClassNotFoundException {
    Properties p = new Properties();
    FileInputStream fis = null;
    try {
        fis = new FileInputStream(titanPropertiesPath);
        p.load(fis);
        return cassandraRepair(p, indexName, relationType, partitionerName);
    } finally {
        IOUtils.closeQuietly(fis);
    }
}
 
Example 14
Project: titan1withtp3.1   File: MapReduceIndexJobs.java   Source Code and License 5 votes vote down vote up
public static ScanMetrics cassandraRemove(String titanPropertiesPath, String indexName, String relationType, String partitionerName)
        throws InterruptedException, IOException, ClassNotFoundException {
    Properties p = new Properties();
    FileInputStream fis = null;
    try {
        fis = new FileInputStream(titanPropertiesPath);
        p.load(fis);
        return cassandraRemove(p, indexName, relationType, partitionerName);
    } finally {
        IOUtils.closeQuietly(fis);
    }
}
 
Example 15
Project: titan1withtp3.1   File: MapReduceIndexJobs.java   Source Code and License 5 votes vote down vote up
public static ScanMetrics hbaseRepair(String titanPropertiesPath, String indexName, String relationType)
        throws InterruptedException, IOException, ClassNotFoundException {
    Properties p = new Properties();
    FileInputStream fis = null;
    try {
        fis = new FileInputStream(titanPropertiesPath);
        p.load(fis);
        return hbaseRepair(p, indexName, relationType);
    } finally {
        IOUtils.closeQuietly(fis);
    }
}
 
Example 16
Project: titan1withtp3.1   File: MapReduceIndexJobs.java   Source Code and License 5 votes vote down vote up
public static ScanMetrics hbaseRemove(String titanPropertiesPath, String indexName, String relationType)
        throws InterruptedException, IOException, ClassNotFoundException {
    Properties p = new Properties();
    FileInputStream fis = null;
    try {
        fis = new FileInputStream(titanPropertiesPath);
        p.load(fis);
        return hbaseRemove(p, indexName, relationType);
    } finally {
        IOUtils.closeQuietly(fis);
    }
}
 
Example 17
Project: titan1withtp3.1   File: KCVSLogManager.java   Source Code and License 5 votes vote down vote up
@Override
public synchronized void close() throws BackendException {
    /* Copying the map is necessary to avoid ConcurrentModificationException.
     * The path to ConcurrentModificationException in the absence of a copy is
     * log.close() -> manager.closedLog(log) -> openLogs.remove(log.getName()).
     */
    for (KCVSLog log : ImmutableMap.copyOf(openLogs).values()) log.close();

    IOUtils.closeQuietly(serializer);
}
 
Example 18
Project: titan1withtp3.1   File: KCVSConfiguration.java   Source Code and License 5 votes vote down vote up
@Override
public void close() {
    try {
        store.close();
        txProvider.close();
        IOUtils.closeQuietly(serializer);
    } catch (BackendException e) {
        throw new TitanException("Could not close configuration store",e);
    }
}
 
Example 19
Project: titan1withtp3.1   File: HBaseStoreManager.java   Source Code and License 5 votes vote down vote up
@Override
public void close() {
    openStores.clear();
    if (logger.isTraceEnabled())
        openManagers.remove(this);
    IOUtils.closeQuietly(cnx);
}
 
Example 20
Project: titan1withtp3.1   File: HBaseStatus.java   Source Code and License 5 votes vote down vote up
public static HBaseStatus write(String path, String hbaseVersion) {
    File f = new File(path);

    FileOutputStream fos = null;
    try {
        fos = new FileOutputStream(path);
        fos.write(String.format("%s", hbaseVersion).getBytes());
    } catch (IOException e) {
        throw new RuntimeException(e);
    } finally {
        IOUtils.closeQuietly(fos);
    }

    return new HBaseStatus(f, hbaseVersion);
}
 
Example 21
Project: incubator-atlas   File: HBaseAdmin0_98.java   Source Code and License 5 votes vote down vote up
@Override
public void clearTable(String tableName, long timestamp) throws IOException
{
    if (!adm.tableExists(tableName)) {
        log.debug("clearStorage() called before table {} was created, skipping.", tableName);
        return;
    }

    // Unfortunately, linear scanning and deleting tables is faster in HBase < 1 when running integration tests than
    // disabling and deleting tables.
    HTable table = null;

    try {
        table = new HTable(adm.getConfiguration(), tableName);

        Scan scan = new Scan();
        scan.setBatch(100);
        scan.setCacheBlocks(false);
        scan.setCaching(2000);
        scan.setTimeRange(0, Long.MAX_VALUE);
        scan.setMaxVersions(1);

        ResultScanner scanner = null;

        try {
            scanner = table.getScanner(scan);

            for (Result res : scanner) {
                Delete d = new Delete(res.getRow());

                d.setTimestamp(timestamp);
                table.delete(d);
            }
        } finally {
            IOUtils.closeQuietly(scanner);
        }
    } finally {
        IOUtils.closeQuietly(table);
    }
}
 
Example 22
Project: incubator-atlas   File: HBaseStoreManager.java   Source Code and License 5 votes vote down vote up
@Override
public void close() {
    openStores.clear();
    if (logger.isTraceEnabled())
        openManagers.remove(this);
    IOUtils.closeQuietly(cnx);
}
 
Example 23
Project: titan1.0.1.kafka   File: StorageSetup.java   Source Code and License 5 votes vote down vote up
public static final void deleteHomeDir(String subdir) {
    File homeDirFile = getHomeDirFile(subdir);
    // Make directory if it doesn't exist
    if (!homeDirFile.exists())
        homeDirFile.mkdirs();
    boolean success = IOUtils.deleteFromDirectory(homeDirFile);
    if (!success) throw new IllegalStateException("Could not remove " + homeDirFile);
}
 
Example 24
Project: titan1.0.1.kafka   File: BerkeleyJEStoreManager.java   Source Code and License 5 votes vote down vote up
@Override
public void clearStorage() throws BackendException {
    if (!stores.isEmpty())
        throw new IllegalStateException("Cannot delete store, since database is open: " + stores.keySet().toString());

    Transaction tx = null;
    for (String db : environment.getDatabaseNames()) {
        environment.removeDatabase(tx, db);
        log.debug("Removed database {} (clearStorage)", db);
    }
    close();
    IOUtils.deleteFromDirectory(directory);
}
 
Example 25
Project: titan1.0.1.kafka   File: BerkeleyElasticsearchTest.java   Source Code and License 5 votes vote down vote up
/**
 * Test {@link com.thinkaurelius.titan.example.GraphOfTheGodsFactory#create(String)}.
 */
@Test
public void testGraphOfTheGodsFactoryCreate() {
    File bdbtmp = new File("target/gotgfactory");
    IOUtils.deleteDirectory(bdbtmp, true);

    TitanGraph gotg = GraphOfTheGodsFactory.create(bdbtmp.getPath());
    TitanIndexTest.assertGraphOfTheGods(gotg);
    gotg.close();
}
 
Example 26
Project: titan1.0.1.kafka   File: ElasticsearchStatus.java   Source Code and License 5 votes vote down vote up
public static ElasticsearchStatus write(String path, int pid) {
    File f = new File(path);

    FileOutputStream fos = null;
    try {
        fos = new FileOutputStream(path);
        fos.write(String.format("%d", pid).getBytes());
    } catch (IOException e) {
        throw new RuntimeException(e);
    } finally {
        IOUtils.closeQuietly(fos);
    }

    return new ElasticsearchStatus(f, pid);
}
 
Example 27
Project: titan1.0.1.kafka   File: MapReduceIndexJobs.java   Source Code and License 5 votes vote down vote up
public static ScanMetrics cassandraRepair(String titanPropertiesPath, String indexName, String relationType, String partitionerName)
        throws InterruptedException, IOException, ClassNotFoundException {
    Properties p = new Properties();
    FileInputStream fis = null;
    try {
        fis = new FileInputStream(titanPropertiesPath);
        p.load(fis);
        return cassandraRepair(p, indexName, relationType, partitionerName);
    } finally {
        IOUtils.closeQuietly(fis);
    }
}
 
Example 28
Project: titan1.0.1.kafka   File: MapReduceIndexJobs.java   Source Code and License 5 votes vote down vote up
public static ScanMetrics cassandraRemove(String titanPropertiesPath, String indexName, String relationType, String partitionerName)
        throws InterruptedException, IOException, ClassNotFoundException {
    Properties p = new Properties();
    FileInputStream fis = null;
    try {
        fis = new FileInputStream(titanPropertiesPath);
        p.load(fis);
        return cassandraRemove(p, indexName, relationType, partitionerName);
    } finally {
        IOUtils.closeQuietly(fis);
    }
}
 
Example 29
Project: titan1.0.1.kafka   File: MapReduceIndexJobs.java   Source Code and License 5 votes vote down vote up
public static ScanMetrics hbaseRepair(String titanPropertiesPath, String indexName, String relationType)
        throws InterruptedException, IOException, ClassNotFoundException {
    Properties p = new Properties();
    FileInputStream fis = null;
    try {
        fis = new FileInputStream(titanPropertiesPath);
        p.load(fis);
        return hbaseRepair(p, indexName, relationType);
    } finally {
        IOUtils.closeQuietly(fis);
    }
}
 
Example 30
Project: titan1.0.1.kafka   File: MapReduceIndexJobs.java   Source Code and License 5 votes vote down vote up
public static ScanMetrics hbaseRemove(String titanPropertiesPath, String indexName, String relationType)
        throws InterruptedException, IOException, ClassNotFoundException {
    Properties p = new Properties();
    FileInputStream fis = null;
    try {
        fis = new FileInputStream(titanPropertiesPath);
        p.load(fis);
        return hbaseRemove(p, indexName, relationType);
    } finally {
        IOUtils.closeQuietly(fis);
    }
}
 
Example 31
Project: titan1.0.1.kafka   File: KCVSLogManager.java   Source Code and License 5 votes vote down vote up
@Override
public synchronized void close() throws BackendException {
    /* Copying the map is necessary to avoid ConcurrentModificationException.
     * The path to ConcurrentModificationException in the absence of a copy is
     * log.close() -> manager.closedLog(log) -> openLogs.remove(log.getName()).
     */
    for (KCVSLog log : ImmutableMap.copyOf(openLogs).values()) log.close();

    IOUtils.closeQuietly(serializer);
}
 
Example 32
Project: titan1.0.1.kafka   File: KCVSConfiguration.java   Source Code and License 5 votes vote down vote up
@Override
public void close() {
    try {
        store.close();
        txProvider.close();
        IOUtils.closeQuietly(serializer);
    } catch (BackendException e) {
        throw new TitanException("Could not close configuration store",e);
    }
}
 
Example 33
Project: titan1.0.1.kafka   File: HBaseStoreManager.java   Source Code and License 5 votes vote down vote up
@Override
public void close() {
    openStores.clear();
    if (logger.isTraceEnabled())
        openManagers.remove(this);
    IOUtils.closeQuietly(cnx);
}
 
Example 34
Project: titan1.0.1.kafka   File: HBaseStatus.java   Source Code and License 5 votes vote down vote up
public static HBaseStatus write(String path, String hbaseVersion) {
    File f = new File(path);

    FileOutputStream fos = null;
    try {
        fos = new FileOutputStream(path);
        fos.write(String.format("%s", hbaseVersion).getBytes());
    } catch (IOException e) {
        throw new RuntimeException(e);
    } finally {
        IOUtils.closeQuietly(fos);
    }

    return new HBaseStatus(f, hbaseVersion);
}
 
Example 35
Project: titan0.5.4-hbase1.1.1-custom   File: StorageSetup.java   Source Code and License 5 votes vote down vote up
public static final void deleteHomeDir(String subdir) {
    File homeDirFile = getHomeDirFile(subdir);
    // Make directory if it doesn't exist
    if (!homeDirFile.exists())
        homeDirFile.mkdirs();
    boolean success = IOUtils.deleteFromDirectory(homeDirFile);
    if (!success) throw new IllegalStateException("Could not remove " + homeDirFile);
}
 
Example 36
Project: titan0.5.4-hbase1.1.1-custom   File: BerkeleyJEStoreManager.java   Source Code and License 5 votes vote down vote up
@Override
public void clearStorage() throws BackendException {
    if (!stores.isEmpty())
        throw new IllegalStateException("Cannot delete store, since database is open: " + stores.keySet().toString());

    Transaction tx = null;
    for (String db : environment.getDatabaseNames()) {
        environment.removeDatabase(tx, db);
        log.debug("Removed database {} (clearStorage)", db);
    }
    close();
    IOUtils.deleteFromDirectory(directory);
}
 
Example 37
Project: titan0.5.4-hbase1.1.1-custom   File: ElasticsearchStatus.java   Source Code and License 5 votes vote down vote up
public static ElasticsearchStatus write(String path, int pid) {
    File f = new File(path);

    FileOutputStream fos = null;
    try {
        fos = new FileOutputStream(path);
        fos.write(String.format("%d", pid).getBytes());
    } catch (IOException e) {
        throw new RuntimeException(e);
    } finally {
        IOUtils.closeQuietly(fos);
    }

    return new ElasticsearchStatus(f, pid);
}
 
Example 38
Project: titan0.5.4-hbase1.1.1-custom   File: LoaderScriptWrapper.java   Source Code and License 5 votes vote down vote up
private static String getScriptString(FileSystem fs, Path scriptPath) throws IOException {
    // Read the Path argument off the FileSystem argument into a string
    InputStreamReader isr = null;
    try {
        isr = new InputStreamReader(fs.open(scriptPath));
        StringWriter wr = new StringWriter();
        org.apache.commons.io.IOUtils.copy(isr, wr);
        return wr.toString();
    } finally {
        IOUtils.closeQuietly(isr);
    }
}
 
Example 39
Project: titan0.5.4-hbase1.1.1-custom   File: KCVSLogManager.java   Source Code and License 5 votes vote down vote up
@Override
public synchronized void close() throws BackendException {
    /* Copying the map is necessary to avoid ConcurrentModificationException.
     * The path to ConcurrentModificationException in the absence of a copy is
     * log.close() -> manager.closedLog(log) -> openLogs.remove(log.getName()).
     */
    for (KCVSLog log : ImmutableMap.copyOf(openLogs).values()) log.close();

    IOUtils.closeQuietly(serializer);
}
 
Example 40
Project: titan0.5.4-hbase1.1.1-custom   File: KCVSConfiguration.java   Source Code and License 5 votes vote down vote up
@Override
public void close() {
    try {
        store.close();
        txProvider.close();
        IOUtils.closeQuietly(serializer);
    } catch (BackendException e) {
        throw new TitanException("Could not close configuration store",e);
    }
}
 
Example 41
Project: titan0.5.4-hbase1.1.1-custom   File: HBaseStoreManager.java   Source Code and License 5 votes vote down vote up
@Override
public void close() {
    openStores.clear();
    if (logger.isTraceEnabled())
        openManagers.remove(this);
    IOUtils.closeQuietly(cnx);
}
 
Example 42
Project: titan0.5.4-hbase1.1.1-custom   File: HBaseStatus.java   Source Code and License 5 votes vote down vote up
public static HBaseStatus write(String path, String hbaseVersion) {
    File f = new File(path);

    FileOutputStream fos = null;
    try {
        fos = new FileOutputStream(path);
        fos.write(String.format("%s", hbaseVersion).getBytes());
    } catch (IOException e) {
        throw new RuntimeException(e);
    } finally {
        IOUtils.closeQuietly(fos);
    }

    return new HBaseStatus(f, hbaseVersion);
}
 
Example 43
Project: titan-experimental   File: PersistitBlueprintsTest.java   Source Code and License 5 votes vote down vote up
@Override
public void cleanUp() throws StorageException {
    //
    synchronized (openGraphs) {
        for (Map.Entry<String, Graph> entry : openGraphs.entrySet()) {
            String dir = entry.getKey();
            File dirFile = new File(dir);
            IOUtils.deleteDirectory(dirFile, true);
            Assert.assertFalse(dirFile.exists() && dirFile.listFiles().length > 0);
        }
    }
}
 
Example 44
Project: titan1withtp3.1   File: GraphOfTheGodsCompatIT.java   Source Code and License 4 votes vote down vote up
/**
 * Download and open databases created by previous versions of Titan.
 * A basic check for cluster-restart forward-compatibility.
 */
@Test
public void testOpenAndQueryCompatibleDatabaseFiles() throws Exception {
    String compatManifest = Joiner.on(File.separator).join("target", "test-classes", "compat.csv");
    BufferedReader compatReader = null;
    int lineNumber = 0;
    int tested = 0;
    try {
        log.info("Opening compat manifest: {}", compatManifest);
        compatReader = new BufferedReader(new FileReader(compatManifest));
        String l;
        while (null != (l = compatReader.readLine())) {
            lineNumber++;
            String tokens[] = l.split(",");
            if (3 != tokens.length) {
                log.warn("Ignoring line {} (splitting on commas yielded {} tokens instead of the expected {}): {}", lineNumber, tokens.length, 3, l);
                continue;
            }
            String version = tokens[0].trim();
            String config = tokens[1].trim();
            String archiveurl = tokens[2].trim();
            if (version.isEmpty()) {
                log.warn("Ignoring line {} due to empty version field", lineNumber);
                continue;
            }
            if (config.isEmpty()) {
                log.warn("Ignoring line {} due to empty config field", lineNumber);
                continue;
            }
            if (archiveurl.isEmpty()) {
                log.warn("Ignoring line {} due to empty archiveurl field", lineNumber);
                continue;
            }

            if (!config.equals(BDB_ES)) {
                log.warn("Ignoring line {} with unknown config string {} (only {} is supported)", lineNumber, config, BDB_ES);
            }

            downloadAndTest(archiveurl);
            tested++;
        }
    } finally {
        IOUtils.closeQuietly(compatReader);
    }

    log.info("Read compat manifest with {} lines ({} tested, {} ignored)",
            lineNumber, tested, lineNumber - tested);
}
 
Example 45
Project: titan1withtp3.1   File: LuceneIndex.java   Source Code and License 4 votes vote down vote up
private void mutateStores(Map.Entry<String, Map<String, IndexMutation>> stores, KeyInformation.IndexRetriever informations) throws IOException, BackendException {
    IndexReader reader = null;
    try {
        String storename = stores.getKey();
        IndexWriter writer = getWriter(storename);
        reader = DirectoryReader.open(writer, true);
        IndexSearcher searcher = new IndexSearcher(reader);
        for (Map.Entry<String, IndexMutation> entry : stores.getValue().entrySet()) {
            String docid = entry.getKey();
            IndexMutation mutation = entry.getValue();

            if (mutation.isDeleted()) {
                if (log.isTraceEnabled())
                    log.trace("Deleted entire document [{}]", docid);

                writer.deleteDocuments(new Term(DOCID, docid));
                continue;
            }

            Pair<Document, Map<String, Shape>> docAndGeo = retrieveOrCreate(docid, searcher);
            Document doc = docAndGeo.getKey();
            Map<String, Shape> geofields = docAndGeo.getValue();

            Preconditions.checkNotNull(doc);
            for (IndexEntry del : mutation.getDeletions()) {
                Preconditions.checkArgument(!del.hasMetaData(), "Lucene index does not support indexing meta data: %s", del);
                String key = del.field;
                if (doc.getField(key) != null) {
                    if (log.isTraceEnabled())
                        log.trace("Removing field [{}] on document [{}]", key, docid);

                    doc.removeFields(key);
                    geofields.remove(key);
                }
            }

            addToDocument(storename, docid, doc, mutation.getAdditions(), geofields, informations);

            //write the old document to the index with the modifications
            writer.updateDocument(new Term(DOCID, docid), doc);
        }
        writer.commit();
    } finally {
        IOUtils.closeQuietly(reader);
    }
}
 
Example 46
Project: titan1withtp3.1   File: LuceneExample.java   Source Code and License 4 votes vote down vote up
@Before
public void setup() {
    if (path.exists()) IOUtils.deleteDirectory(path,false);
    if (!path.exists() && path.isDirectory()) path.mkdirs();
}
 
Example 47
Project: titan1withtp3.1   File: ElasticSearchConfigTest.java   Source Code and License 4 votes vote down vote up
@BeforeClass
public static void killElasticsearch() {
    IOUtils.deleteDirectory(new File("es"), true);
    ElasticsearchRunner esr = new ElasticsearchRunner();
    esr.stop();
}
 
Example 48
Project: titan1withtp3.1   File: HBaseKeyColumnValueStore.java   Source Code and License 4 votes vote down vote up
@Override
public void close() {
    IOUtils.closeQuietly(table);
    isClosed = true;
    logger.debug("RowIterator closed table {}", table);
}
 
Example 49
Project: incubator-atlas   File: HBaseKeyColumnValueStore.java   Source Code and License 4 votes vote down vote up
@Override
public void close() {
    IOUtils.closeQuietly(table);
    isClosed = true;
    logger.debug("RowIterator closed table {}", table);
}
 
Example 50
Project: incubator-atlas   File: HBaseStoreManager.java   Source Code and License 4 votes vote down vote up
@Override
public void mutateMany(Map<String, Map<StaticBuffer, KCVMutation>> mutations, StoreTransaction txh) throws BackendException {
    logger.debug("Enter mutateMany");
    final MaskedTimestamp commitTime = new MaskedTimestamp(txh);
    // In case of an addition and deletion with identical timestamps, the
    // deletion tombstone wins.
    // http://hbase.apache.org/book/versions.html#d244e4250
    Map<StaticBuffer, Pair<Put, Delete>> commandsPerKey =
            convertToCommands(
                    mutations,
                    commitTime.getAdditionTime(times.getUnit()),
                    commitTime.getDeletionTime(times.getUnit()));

    List<Row> batch = new ArrayList<>(commandsPerKey.size()); // actual batch operation

    // convert sorted commands into representation required for 'batch' operation
    for (Pair<Put, Delete> commands : commandsPerKey.values()) {
        if (commands.getFirst() != null)
            batch.add(commands.getFirst());

        if (commands.getSecond() != null)
            batch.add(commands.getSecond());
    }

    try {
        TableMask table = null;

        try {
            table = cnx.getTable(tableName);
            logger.debug("mutateMany : batch mutate started size {} ", batch.size());
            table.batch(batch, new Object[batch.size()]);
            logger.debug("mutateMany : batch mutate finished {} ", batch.size());
        } finally {
            IOUtils.closeQuietly(table);
        }
    } catch (IOException | InterruptedException e) {
        throw new TemporaryBackendException(e);
    }

    sleepAfterWrite(txh, commitTime);
}
 
Example 51
Project: titan1.0.1.kafka   File: GraphOfTheGodsCompatIT.java   Source Code and License 4 votes vote down vote up
/**
 * Download and open databases created by previous versions of Titan.
 * A basic check for cluster-restart forward-compatibility.
 */
@Test
public void testOpenAndQueryCompatibleDatabaseFiles() throws Exception {
    String compatManifest = Joiner.on(File.separator).join("target", "test-classes", "compat.csv");
    BufferedReader compatReader = null;
    int lineNumber = 0;
    int tested = 0;
    try {
        log.info("Opening compat manifest: {}", compatManifest);
        compatReader = new BufferedReader(new FileReader(compatManifest));
        String l;
        while (null != (l = compatReader.readLine())) {
            lineNumber++;
            String tokens[] = l.split(",");
            if (3 != tokens.length) {
                log.warn("Ignoring line {} (splitting on commas yielded {} tokens instead of the expected {}): {}", lineNumber, tokens.length, 3, l);
                continue;
            }
            String version = tokens[0].trim();
            String config = tokens[1].trim();
            String archiveurl = tokens[2].trim();
            if (version.isEmpty()) {
                log.warn("Ignoring line {} due to empty version field", lineNumber);
                continue;
            }
            if (config.isEmpty()) {
                log.warn("Ignoring line {} due to empty config field", lineNumber);
                continue;
            }
            if (archiveurl.isEmpty()) {
                log.warn("Ignoring line {} due to empty archiveurl field", lineNumber);
                continue;
            }

            if (!config.equals(BDB_ES)) {
                log.warn("Ignoring line {} with unknown config string {} (only {} is supported)", lineNumber, config, BDB_ES);
            }

            downloadAndTest(archiveurl);
            tested++;
        }
    } finally {
        IOUtils.closeQuietly(compatReader);
    }

    log.info("Read compat manifest with {} lines ({} tested, {} ignored)",
            lineNumber, tested, lineNumber - tested);
}
 
Example 52
Project: titan1.0.1.kafka   File: LuceneIndex.java   Source Code and License 4 votes vote down vote up
private void mutateStores(Map.Entry<String, Map<String, IndexMutation>> stores, KeyInformation.IndexRetriever informations) throws IOException, BackendException {
    IndexReader reader = null;
    try {
        String storename = stores.getKey();
        IndexWriter writer = getWriter(storename);
        reader = DirectoryReader.open(writer, true);
        IndexSearcher searcher = new IndexSearcher(reader);
        for (Map.Entry<String, IndexMutation> entry : stores.getValue().entrySet()) {
            String docid = entry.getKey();
            IndexMutation mutation = entry.getValue();

            if (mutation.isDeleted()) {
                if (log.isTraceEnabled())
                    log.trace("Deleted entire document [{}]", docid);

                writer.deleteDocuments(new Term(DOCID, docid));
                continue;
            }

            Pair<Document, Map<String, Shape>> docAndGeo = retrieveOrCreate(docid, searcher);
            Document doc = docAndGeo.getKey();
            Map<String, Shape> geofields = docAndGeo.getValue();

            Preconditions.checkNotNull(doc);
            for (IndexEntry del : mutation.getDeletions()) {
                Preconditions.checkArgument(!del.hasMetaData(), "Lucene index does not support indexing meta data: %s", del);
                String key = del.field;
                if (doc.getField(key) != null) {
                    if (log.isTraceEnabled())
                        log.trace("Removing field [{}] on document [{}]", key, docid);

                    doc.removeFields(key);
                    geofields.remove(key);
                }
            }

            addToDocument(storename, docid, doc, mutation.getAdditions(), geofields, informations);

            //write the old document to the index with the modifications
            writer.updateDocument(new Term(DOCID, docid), doc);
        }
        writer.commit();
    } finally {
        IOUtils.closeQuietly(reader);
    }
}
 
Example 53
Project: titan1.0.1.kafka   File: LuceneExample.java   Source Code and License 4 votes vote down vote up
@Before
public void setup() {
    if (path.exists()) IOUtils.deleteDirectory(path,false);
    if (!path.exists() && path.isDirectory()) path.mkdirs();
}
 
Example 54
Project: titan1.0.1.kafka   File: ElasticSearchConfigTest.java   Source Code and License 4 votes vote down vote up
@BeforeClass
public static void killElasticsearch() {
    IOUtils.deleteDirectory(new File("es"), true);
    ElasticsearchRunner esr = new ElasticsearchRunner();
    esr.stop();
}
 
Example 55
Project: titan1.0.1.kafka   File: HBaseAdmin0_94.java   Source Code and License 4 votes vote down vote up
@Override
    public void clearTable(String tableName, long timestamp) throws IOException
    {
        if (!adm.tableExists(tableName)) {
            log.debug("clearStorage() called before table {} was created, skipping.", tableName);
            return;
        }

//        long before = System.currentTimeMillis();
//        try {
//            adm.disableTable(tableName);
//            adm.deleteTable(tableName);
//        } catch (IOException e) {
//            throw new PermanentBackendException(e);
//        }
//        ensureTableExists(tableName, getCfNameForStoreName(GraphDatabaseConfiguration.SYSTEM_PROPERTIES_STORE_NAME), 0);
//        long after = System.currentTimeMillis();
//        logger.debug("Dropped and recreated table {} in {} ms", tableName, after - before);


        // Unfortunately, linear scanning and deleting tables is faster in HBase < 1 when running integration tests than
        // disabling and deleting tables.
        HTable table = null;

        try {
            table = new HTable(adm.getConfiguration(), tableName);

            Scan scan = new Scan();
            scan.setBatch(100);
            scan.setCacheBlocks(false);
            scan.setCaching(2000);
            scan.setTimeRange(0, Long.MAX_VALUE);
            scan.setMaxVersions(1);

            ResultScanner scanner = null;

            try {
                scanner = table.getScanner(scan);

                for (Result res : scanner) {
                    Delete d = new Delete(res.getRow());

                    d.setTimestamp(timestamp);
                    table.delete(d);
                }
            } finally {
                IOUtils.closeQuietly(scanner);
            }
        } finally {
            IOUtils.closeQuietly(table);
        }
    }
 
Example 56
Project: titan1.0.1.kafka   File: HBaseKeyColumnValueStore.java   Source Code and License 4 votes vote down vote up
@Override
public void close() {
    IOUtils.closeQuietly(table);
    isClosed = true;
    logger.debug("RowIterator closed table {}", table);
}
 
Example 57
Project: titan1.0.1.kafka   File: HBaseAdmin0_96.java   Source Code and License 4 votes vote down vote up
@Override
    public void clearTable(String tableName, long timestamp) throws IOException
    {
        if (!adm.tableExists(tableName)) {
            log.debug("clearStorage() called before table {} was created, skipping.", tableName);
            return;
        }

//        long before = System.currentTimeMillis();
//        try {
//            adm.disableTable(tableName);
//            adm.deleteTable(tableName);
//        } catch (IOException e) {
//            throw new PermanentBackendException(e);
//        }
//        ensureTableExists(tableName, getCfNameForStoreName(GraphDatabaseConfiguration.SYSTEM_PROPERTIES_STORE_NAME), 0);
//        long after = System.currentTimeMillis();
//        logger.debug("Dropped and recreated table {} in {} ms", tableName, after - before);


        // Unfortunately, linear scanning and deleting tables is faster in HBase < 1 when running integration tests than
        // disabling and deleting tables.
        HTable table = null;

        try {
            table = new HTable(adm.getConfiguration(), tableName);

            Scan scan = new Scan();
            scan.setBatch(100);
            scan.setCacheBlocks(false);
            scan.setCaching(2000);
            scan.setTimeRange(0, Long.MAX_VALUE);
            scan.setMaxVersions(1);

            ResultScanner scanner = null;

            try {
                scanner = table.getScanner(scan);

                for (Result res : scanner) {
                    Delete d = new Delete(res.getRow());

                    d.setTimestamp(timestamp);
                    table.delete(d);
                }
            } finally {
                IOUtils.closeQuietly(scanner);
            }
        } finally {
            IOUtils.closeQuietly(table);
        }
    }
 
Example 58
Project: titan1.0.1.kafka   File: HBaseAdmin0_98.java   Source Code and License 4 votes vote down vote up
@Override
    public void clearTable(String tableName, long timestamp) throws IOException
    {
        if (!adm.tableExists(tableName)) {
            log.debug("clearStorage() called before table {} was created, skipping.", tableName);
            return;
        }

//        long before = System.currentTimeMillis();
//        try {
//            adm.disableTable(tableName);
//            adm.deleteTable(tableName);
//        } catch (IOException e) {
//            throw new PermanentBackendException(e);
//        }
//        ensureTableExists(tableName, getCfNameForStoreName(GraphDatabaseConfiguration.SYSTEM_PROPERTIES_STORE_NAME), 0);
//        long after = System.currentTimeMillis();
//        logger.debug("Dropped and recreated table {} in {} ms", tableName, after - before);


        // Unfortunately, linear scanning and deleting tables is faster in HBase < 1 when running integration tests than
        // disabling and deleting tables.
        HTable table = null;

        try {
            table = new HTable(adm.getConfiguration(), tableName);

            Scan scan = new Scan();
            scan.setBatch(100);
            scan.setCacheBlocks(false);
            scan.setCaching(2000);
            scan.setTimeRange(0, Long.MAX_VALUE);
            scan.setMaxVersions(1);

            ResultScanner scanner = null;

            try {
                scanner = table.getScanner(scan);

                for (Result res : scanner) {
                    Delete d = new Delete(res.getRow());

                    d.setTimestamp(timestamp);
                    table.delete(d);
                }
            } finally {
                IOUtils.closeQuietly(scanner);
            }
        } finally {
            IOUtils.closeQuietly(table);
        }
    }
 
Example 59
Project: titan0.5.4-hbase1.1.1-custom   File: GraphOfTheGodsCompatIT.java   Source Code and License 4 votes vote down vote up
/**
 * Download and open databases created by previous versions of Titan.
 * A basic check for cluster-restart forward-compatibility.
 */
@Test
public void testOpenAndQueryCompatibleDatabaseFiles() throws Exception {
    String compatManifest = Joiner.on(File.separator).join("target", "test-classes", "compat.csv");
    BufferedReader compatReader = null;
    int lineNumber = 0;
    int tested = 0;
    try {
        log.info("Opening compat manifest: {}", compatManifest);
        compatReader = new BufferedReader(new FileReader(compatManifest));
        String l;
        while (null != (l = compatReader.readLine())) {
            lineNumber++;
            String tokens[] = l.split(",");
            if (3 != tokens.length) {
                log.warn("Ignoring line {} (splitting on commas yielded {} tokens instead of the expected {}): {}", lineNumber, tokens.length, 3, l);
                continue;
            }
            String version = tokens[0].trim();
            String config = tokens[1].trim();
            String archiveurl = tokens[2].trim();
            if (version.isEmpty()) {
                log.warn("Ignoring line {} due to empty version field", lineNumber);
                continue;
            }
            if (config.isEmpty()) {
                log.warn("Ignoring line {} due to empty config field", lineNumber);
                continue;
            }
            if (archiveurl.isEmpty()) {
                log.warn("Ignoring line {} due to empty archiveurl field", lineNumber);
                continue;
            }

            if (!config.equals(BDB_ES)) {
                log.warn("Ignoring line {} with unknown config string {} (only {} is supported)", lineNumber, config, BDB_ES);
            }

            downloadAndTest(archiveurl);
            tested++;
        }
    } finally {
        IOUtils.closeQuietly(compatReader);
    }

    log.info("Read compat manifest with {} lines ({} tested, {} ignored)",
            lineNumber, tested, lineNumber - tested);
}
 
Example 60
Project: titan0.5.4-hbase1.1.1-custom   File: LuceneIndex.java   Source Code and License 4 votes vote down vote up
private void mutateStores(Map.Entry<String, Map<String, IndexMutation>> stores, KeyInformation.IndexRetriever informations) throws IOException, BackendException {
    IndexReader reader = null;
    try {
        String storename = stores.getKey();
        IndexWriter writer = getWriter(storename);
        reader = DirectoryReader.open(writer, true);
        IndexSearcher searcher = new IndexSearcher(reader);
        for (Map.Entry<String, IndexMutation> entry : stores.getValue().entrySet()) {
            String docid = entry.getKey();
            IndexMutation mutation = entry.getValue();

            if (mutation.isDeleted()) {
                if (log.isTraceEnabled())
                    log.trace("Deleted entire document [{}]", docid);

                writer.deleteDocuments(new Term(DOCID, docid));
                continue;
            }

            Pair<Document, Map<String, Shape>> docAndGeo = retrieveOrCreate(docid, searcher);
            Document doc = docAndGeo.getA();
            Map<String, Shape> geofields = docAndGeo.getB();

            Preconditions.checkNotNull(doc);
            for (IndexEntry del : mutation.getDeletions()) {
                Preconditions.checkArgument(!del.hasMetaData(), "Lucene index does not support indexing meta data: %s", del);
                String key = del.field;
                if (doc.getField(key) != null) {
                    if (log.isTraceEnabled())
                        log.trace("Removing field [{}] on document [{}]", key, docid);

                    doc.removeFields(key);
                    geofields.remove(key);
                }
            }

            addToDocument(storename, docid, doc, mutation.getAdditions(), geofields, informations);

            //write the old document to the index with the modifications
            writer.updateDocument(new Term(DOCID, docid), doc);
        }
        writer.commit();
    } finally {
        IOUtils.closeQuietly(reader);
    }
}