Java Code Examples for org.apache.cassandra.io.util.FileUtils

The following examples show how to use org.apache.cassandra.io.util.FileUtils. These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source Project: sasi   Source File: RangeUnionIterator.java    License: Apache License 2.0 6 votes vote down vote up
@Override
protected void performSkipTo(K nextToken)
{
    List<RangeIterator<K, D>> changedRanges = new ArrayList<>();

    while (!ranges.isEmpty())
    {
        if (ranges.peek().getCurrent().compareTo(nextToken) >= 0)
            break;

        RangeIterator<K, D> head = ranges.poll();

        if (head.getMaximum().compareTo(nextToken) >= 0)
        {
            head.skipTo(nextToken);
            changedRanges.add(head);
            continue;
        }

        FileUtils.closeQuietly(head);
    }

    for (RangeIterator<K, D> range : changedRanges)
        ranges.add(range);
}
 
Example 2
Source Project: sasi   Source File: MappedBuffer.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public void close()
{
    if (!FileUtils.isCleanerAvailable())
        return;

    /*
     * Try forcing the unmapping of pages using undocumented unsafe sun APIs.
     * If this fails (non Sun JVM), we'll have to wait for the GC to finalize the mapping.
     * If this works and a thread tries to access any page, hell will unleash on earth.
     */
    try
    {
        for (MappedByteBuffer segment : pages)
            FileUtils.clean(segment);
    }
    catch (Exception e)
    {
        // This is not supposed to happen
    }
}
 
Example 3
Source Project: stratio-cassandra   Source File: InvertedIndex.java    License: Apache License 2.0 6 votes vote down vote up
private static Properties loadProperties()
{
    Properties properties = new Properties();
    InputStream stream = InvertedIndex.class.getClassLoader().getResourceAsStream("InvertedIndex.properties");
    try
    {
        properties.load(stream);
    }
    catch (Exception e)
    {
        throw new RuntimeException(e);
    }
    finally
    {
        FileUtils.closeQuietly(stream);
    }
    logger.info("loaded property file, InvertedIndex.properties");
    return properties;
}
 
Example 4
Source Project: stratio-cassandra   Source File: FBUtilities.java    License: Apache License 2.0 6 votes vote down vote up
public static String getReleaseVersionString()
{
    InputStream in = null;
    try
    {
        in = FBUtilities.class.getClassLoader().getResourceAsStream("org/apache/cassandra/config/version.properties");
        if (in == null)
        {
            return System.getProperty("cassandra.releaseVersion", "Unknown");
        }
        Properties props = new Properties();
        props.load(in);
        return props.getProperty("CassandraVersion");
    }
    catch (Exception e)
    {
        JVMStabilityInspector.inspectThrowable(e);
        logger.warn("Unable to load version.properties", e);
        return "debug version";
    }
    finally
    {
        FileUtils.closeQuietly(in);
    }
}
 
Example 5
Source Project: stratio-cassandra   Source File: FailureDetector.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Dump the inter arrival times for examination if necessary.
 */
public void dumpInterArrivalTimes()
{
    File file = FileUtils.createTempFile("failuredetector-", ".dat");

    OutputStream os = null;
    try
    {
        os = new BufferedOutputStream(new FileOutputStream(file, true));
        os.write(toString().getBytes());
    }
    catch (IOException e)
    {
        throw new FSWriteError(e, file);
    }
    finally
    {
        FileUtils.closeQuietly(os);
    }
}
 
Example 6
Source Project: stratio-cassandra   Source File: CommitLogSegmentManager.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Differs from the above because it can work on any file instead of just existing
 * commit log segments managed by this manager.
 *
 * @param file segment file that is no longer in use.
 */
void recycleSegment(final File file)
{
    if (isCapExceeded()
        || CommitLogDescriptor.fromFileName(file.getName()).getMessagingVersion() != MessagingService.current_version)
    {
        // (don't decrease managed size, since this was never a "live" segment)
        logger.debug("(Unopened) segment {} is no longer needed and will be deleted now", file);
        FileUtils.deleteWithConfirm(file);
        return;
    }

    logger.debug("Recycling {}", file);
    // this wasn't previously a live segment, so add it to the managed size when we make it live
    size.addAndGet(DatabaseDescriptor.getCommitLogSegmentSize());
    segmentManagementTasks.add(new Callable<CommitLogSegment>()
    {
        public CommitLogSegment call()
        {
            return new CommitLogSegment(file.getPath());
        }
    });
}
 
Example 7
Source Project: stratio-cassandra   Source File: DatabaseDescriptor.java    License: Apache License 2.0 6 votes vote down vote up
public static MemtablePool getMemtableAllocatorPool()
{
    long heapLimit = ((long) conf.memtable_heap_space_in_mb) << 20;
    long offHeapLimit = ((long) conf.memtable_offheap_space_in_mb) << 20;
    switch (conf.memtable_allocation_type)
    {
        case unslabbed_heap_buffers:
            return new HeapPool(heapLimit, conf.memtable_cleanup_threshold, new ColumnFamilyStore.FlushLargestColumnFamily());
        case heap_buffers:
            return new SlabPool(heapLimit, 0, conf.memtable_cleanup_threshold, new ColumnFamilyStore.FlushLargestColumnFamily());
        case offheap_buffers:
            if (!FileUtils.isCleanerAvailable())
            {
                logger.error("Could not free direct byte buffer: offheap_buffers is not a safe memtable_allocation_type without this ability, please adjust your config. This feature is only guaranteed to work on an Oracle JVM. Refusing to start.");
                System.exit(-1);
            }
            return new SlabPool(heapLimit, offHeapLimit, conf.memtable_cleanup_threshold, new ColumnFamilyStore.FlushLargestColumnFamily());
        case offheap_objects:
            return new NativePool(heapLimit, offHeapLimit, conf.memtable_cleanup_threshold, new ColumnFamilyStore.FlushLargestColumnFamily());
        default:
            throw new AssertionError();
    }
}
 
Example 8
Source Project: stratio-cassandra   Source File: SSTableReader.java    License: Apache License 2.0 6 votes vote down vote up
private IndexSummary buildSummaryAtLevel(int newSamplingLevel) throws IOException
{
    // we read the positions in a BRAF so we don't have to worry about an entry spanning a mmap boundary.
    RandomAccessReader primaryIndex = RandomAccessReader.open(new File(descriptor.filenameFor(Component.PRIMARY_INDEX)));
    try
    {
        long indexSize = primaryIndex.length();
        try (IndexSummaryBuilder summaryBuilder = new IndexSummaryBuilder(estimatedKeys(), metadata.getMinIndexInterval(), newSamplingLevel))
        {
            long indexPosition;
            while ((indexPosition = primaryIndex.getFilePointer()) != indexSize)
            {
                summaryBuilder.maybeAddEntry(partitioner.decorateKey(ByteBufferUtil.readWithShortLength(primaryIndex)), indexPosition);
                RowIndexEntry.Serializer.skip(primaryIndex);
            }

            return summaryBuilder.build(partitioner);
        }
    }
    finally
    {
        FileUtils.closeQuietly(primaryIndex);
    }
}
 
Example 9
Source Project: stratio-cassandra   Source File: SSTable.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * We use a ReferenceQueue to manage deleting files that have been compacted
 * and for which no more SSTable references exist.  But this is not guaranteed
 * to run for each such file because of the semantics of the JVM gc.  So,
 * we write a marker to `compactedFilename` when a file is compacted;
 * if such a marker exists on startup, the file should be removed.
 *
 * This method will also remove SSTables that are marked as temporary.
 *
 * @return true if the file was deleted
 */
public static boolean delete(Descriptor desc, Set<Component> components)
{
    // remove the DATA component first if it exists
    if (components.contains(Component.DATA))
        FileUtils.deleteWithConfirm(desc.filenameFor(Component.DATA));
    for (Component component : components)
    {
        if (component.equals(Component.DATA) || component.equals(Component.SUMMARY))
            continue;

        FileUtils.deleteWithConfirm(desc.filenameFor(component));
    }
    FileUtils.delete(desc.filenameFor(Component.SUMMARY));

    logger.debug("Deleted {}", desc);
    return true;
}
 
Example 10
Source Project: stratio-cassandra   Source File: SSTable.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Appends new component names to the TOC component.
 */
protected static void appendTOC(Descriptor descriptor, Collection<Component> components)
{
    File tocFile = new File(descriptor.filenameFor(Component.TOC));
    PrintWriter w = null;
    try
    {
        w = new PrintWriter(new FileWriter(tocFile, true));
        for (Component component : components)
            w.println(component.name);
    }
    catch (IOException e)
    {
        throw new FSWriteError(e, tocFile);
    }
    finally
    {
        FileUtils.closeQuietly(w);
    }
}
 
Example 11
Source Project: stratio-cassandra   Source File: IndexHelper.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Skip the index
 * @param in the data input from which the index should be skipped
 * @throws IOException if an I/O error occurs.
 */
public static void skipIndex(DataInput in) throws IOException
{
    /* read only the column index list */
    int columnIndexSize = in.readInt();
    /* skip the column index data */
    if (in instanceof FileDataInput)
    {
        FileUtils.skipBytesFully(in, columnIndexSize);
    }
    else
    {
        // skip bytes
        byte[] skip = new byte[columnIndexSize];
        in.readFully(skip);
    }
}
 
Example 12
Source Project: stratio-cassandra   Source File: BloomFilterTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
@Ignore
public void testHugeBFSerialization() throws IOException
{
    ByteBuffer test = ByteBuffer.wrap(new byte[] {0, 1});

    File file = FileUtils.createTempFile("bloomFilterTest-", ".dat");
    BloomFilter filter = (BloomFilter) FilterFactory.getFilter(((long)Integer.MAX_VALUE / 8) + 1, 0.01d, true);
    filter.add(test);
    DataOutputStreamAndChannel out = new DataOutputStreamAndChannel(new FileOutputStream(file));
    FilterFactory.serialize(filter, out);
    filter.bitset.serialize(out);
    out.close();
    filter.close();
    
    DataInputStream in = new DataInputStream(new FileInputStream(file));
    BloomFilter filter2 = (BloomFilter) FilterFactory.deserialize(in, true);
    Assert.assertTrue(filter2.isPresent(test));
    FileUtils.closeQuietly(in);
}
 
Example 13
Source Project: stratio-cassandra   Source File: IndexSummaryTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testSerialization() throws IOException
{
    Pair<List<DecoratedKey>, IndexSummary> random = generateRandomIndex(100, 1);
    DataOutputBuffer dos = new DataOutputBuffer();
    IndexSummary.serializer.serialize(random.right, dos, false);
    // write junk
    dos.writeUTF("JUNK");
    dos.writeUTF("JUNK");
    FileUtils.closeQuietly(dos);
    DataInputStream dis = new DataInputStream(new ByteArrayInputStream(dos.toByteArray()));
    IndexSummary is = IndexSummary.serializer.deserialize(dis, DatabaseDescriptor.getPartitioner(), false, 1, 1);
    for (int i = 0; i < 100; i++)
        assertEquals(i, is.binarySearch(random.left.get(i)));
    // read the junk
    assertEquals(dis.readUTF(), "JUNK");
    assertEquals(dis.readUTF(), "JUNK");
    is.close();
    FileUtils.closeQuietly(dis);
}
 
Example 14
Source Project: reactor-spring5-example   Source File: CassandraSetup.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Creates the base storage directories required for running a Cassandra instance.
 */
private static void setupCassandraDirectories() throws IOException {
    final Path root = Files.createTempDirectory("cassandra");
    final Path config = root.resolve("cassandra.yaml");
    Files.copy(ReactorExampleApplication.class.getResourceAsStream("/cassandra.yaml"), config);
    System.setProperty("cassandra.config", "file:" + config.toString());
    System.setProperty("cassandra.storagedir", root.toString());
    System.setProperty("cassandra-foreground", "true");
    Stream.of(DatabaseDescriptor.getAllDataFileLocations())
        .map(root::resolve)
        .map(Path::toFile)
        .forEach(FileUtils::createDirectory);
}
 
Example 15
Source Project: sasi   Source File: SSTableAttachedSecondaryIndex.java    License: Apache License 2.0 5 votes vote down vote up
private void completeSSTable(PerSSTableIndexWriter indexWriter, SSTableReader sstable, Collection<ColumnIndex> indexes)
{
    indexWriter.complete();

    for (ColumnIndex index : indexes)
    {
        File tmpIndex = new File(index.getPath(indexWriter.getDescriptor()));
        if (!tmpIndex.exists()) // no data was inserted into the index for given sstable
            continue;

        FileUtils.renameWithConfirm(tmpIndex, new File(index.getPath(sstable.descriptor)));
        index.update(Collections.<SSTableReader>emptyList(), Collections.singletonList(sstable));
    }
}
 
Example 16
Source Project: sasi   Source File: SSTableIndex.java    License: Apache License 2.0 5 votes vote down vote up
public void release()
{
    int n = references.decrementAndGet();
    if (n == 0)
    {
        FileUtils.closeQuietly(index);
        sstable.releaseReference();
        if (obsolete.get() || sstable.isMarkedCompacted())
            FileUtils.delete(index.getIndexPath());
    }
}
 
Example 17
Source Project: sasi   Source File: TermIterator.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public void close()
{
    FileUtils.closeQuietly(union);
    for (SSTableIndex index : referencedIndexes)
        releaseQuietly(index);

    referencedIndexes.clear();
}
 
Example 18
Source Project: sasi   Source File: QueryController.java    License: Apache License 2.0 5 votes vote down vote up
private void releaseIndexes(List<RangeIterator<Long, Token>> indexes)
{
    if (indexes == null)
        return;

    for (RangeIterator<Long, Token> index : indexes)
        FileUtils.closeQuietly(index);
}
 
Example 19
Source Project: sasi   Source File: MappedBufferTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testOpenWithoutPageBits() throws IOException
{
    File tmp = File.createTempFile("mapped-buffer", "tmp");
    tmp.deleteOnExit();

    RandomAccessFile file = new RandomAccessFile(tmp, "rw");

    long numValues = 1000;
    for (long i = 0; i < numValues; i++)
        file.writeLong(i);

    file.getFD().sync();

    try (MappedBuffer buffer = new MappedBuffer(file))
    {
        Assert.assertEquals(numValues * 8, buffer.limit());
        Assert.assertEquals(numValues * 8, buffer.capacity());

        for (long i = 0; i < numValues; i++)
        {
            Assert.assertEquals(i * 8, buffer.position());
            Assert.assertEquals(i, buffer.getLong());
        }
    }
    finally
    {
        FileUtils.closeQuietly(file);
    }
}
 
Example 20
Source Project: sasi   Source File: TokenTreeTest.java    License: Apache License 2.0 5 votes vote down vote up
private static TokenTree generateTree(final long minToken, final long maxToken) throws IOException
{
    final SortedMap<Long, LongSet> toks = new TreeMap<Long, LongSet>()
    {{
            for (long i = minToken; i <= maxToken; i++)
            {
                LongSet offsetSet = new LongOpenHashSet();
                offsetSet.add(i);
                put(i, offsetSet);
            }
    }};

    final TokenTreeBuilder builder = new TokenTreeBuilder(toks).finish();
    final File treeFile = File.createTempFile("token-tree-get-test", "tt");
    treeFile.deleteOnExit();

    final SequentialWriter writer = new SequentialWriter(treeFile, 4096, false);
    builder.write(writer);
    writer.close();

    RandomAccessReader reader = null;

    try
    {
        reader = RandomAccessReader.open(treeFile);
        return new TokenTree(new MappedBuffer(reader));
    }
    finally
    {
        FileUtils.closeQuietly(reader);
    }
}
 
Example 21
Source Project: stratio-cassandra   Source File: CqlBulkRecordWriter.java    License: Apache License 2.0 5 votes vote down vote up
private void prepareWriter() throws IOException
{
    try
    {
        if (writer == null)
        {
            writer = CQLSSTableWriter.builder()
                .forTable(schema)
                .using(insertStatement)
                .withPartitioner(ConfigHelper.getOutputPartitioner(conf))
                .inDirectory(outputDir)
                .withBufferSizeInMB(Integer.parseInt(conf.get(BUFFER_SIZE_IN_MB, "64")))
                .build();
        }
        if (loader == null)
        {
            ExternalClient externalClient = new ExternalClient(conf);
            
            externalClient.addKnownCfs(keyspace, schema);

            this.loader = new SSTableLoader(outputDir, externalClient, new BulkRecordWriter.NullOutputHandler()) {
                @Override
                public void onSuccess(StreamState finalState)
                {
                    if (deleteSrc)
                        FileUtils.deleteRecursive(outputDir);
                }
            };
        }
    }
    catch (Exception e)
    {
        throw new IOException(e);
    }      
}
 
Example 22
Source Project: stratio-cassandra   Source File: ByteBufferUtil.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * @param in data input
 * @return null
 * @throws IOException if an I/O error occurs.
 */
public static ByteBuffer skipShortLength(DataInput in) throws IOException
{
    int skip = readShortLength(in);
    FileUtils.skipBytesFully(in, skip);
    return null;
}
 
Example 23
Source Project: stratio-cassandra   Source File: StorageService.java    License: Apache License 2.0 5 votes vote down vote up
public Map<String, String> getLoadMap()
{
    Map<String, String> map = new HashMap<>();
    for (Map.Entry<InetAddress,Double> entry : LoadBroadcaster.instance.getLoadInfo().entrySet())
    {
        map.put(entry.getKey().getHostAddress(), FileUtils.stringifyFileSize(entry.getValue()));
    }
    // gossiper doesn't see its own updates, so we need to special-case the local node
    map.put(FBUtilities.getBroadcastAddress().getHostAddress(), getLoadString());
    return map;
}
 
Example 24
Source Project: stratio-cassandra   Source File: StorageService.java    License: Apache License 2.0 5 votes vote down vote up
public String[] getAllDataFileLocations()
{
    String[] locations = DatabaseDescriptor.getAllDataFileLocations();
    for (int i = 0; i < locations.length; i++)
        locations[i] = FileUtils.getCanonicalPath(locations[i]);
    return locations;
}
 
Example 25
Source Project: stratio-cassandra   Source File: CommitLogSegment.java    License: Apache License 2.0 5 votes vote down vote up
void internalClose()
{
    try
    {
        if (FileUtils.isCleanerAvailable())
            FileUtils.clean(buffer);
        logFileAccessor.close();
    }
    catch (IOException e)
    {
        throw new FSWriteError(e, getPath());
    }
}
 
Example 26
Source Project: stratio-cassandra   Source File: CompactionManager.java    License: Apache License 2.0 5 votes vote down vote up
public static SSTableWriter createWriter(ColumnFamilyStore cfs,
                                         File compactionFileLocation,
                                         int expectedBloomFilterSize,
                                         long repairedAt,
                                         SSTableReader sstable)
{
    FileUtils.createDirectory(compactionFileLocation);
    return new SSTableWriter(cfs.getTempSSTablePath(compactionFileLocation),
                             expectedBloomFilterSize,
                             repairedAt,
                             cfs.metadata,
                             cfs.partitioner,
                             new MetadataCollector(Collections.singleton(sstable), cfs.metadata.comparator, sstable.getSSTableLevel()));
}
 
Example 27
Source Project: stratio-cassandra   Source File: ColumnSerializer.java    License: Apache License 2.0 5 votes vote down vote up
void skipColumnBody(DataInput in, int mask) throws IOException
{
    if ((mask & COUNTER_MASK) != 0)
        FileUtils.skipBytesFully(in, 16);
    else if ((mask & EXPIRATION_MASK) != 0)
        FileUtils.skipBytesFully(in, 16);
    else
        FileUtils.skipBytesFully(in, 8);

    int length = in.readInt();
    FileUtils.skipBytesFully(in, length);
}
 
Example 28
Source Project: stratio-cassandra   Source File: RowIndexEntry.java    License: Apache License 2.0 5 votes vote down vote up
public static void skipPromotedIndex(DataInput in) throws IOException
{
    int size = in.readInt();
    if (size <= 0)
        return;

    FileUtils.skipBytesFully(in, size);
}
 
Example 29
Source Project: stratio-cassandra   Source File: DataTracker.java    License: Apache License 2.0 5 votes vote down vote up
public void maybeIncrementallyBackup(final SSTableReader sstable)
{
    if (!DatabaseDescriptor.isIncrementalBackupsEnabled())
        return;

    File backupsDir = Directories.getBackupsDirectory(sstable.descriptor);
    sstable.createLinks(FileUtils.getCanonicalPath(backupsDir));
}
 
Example 30
public static void from(final String snapshot, final String ks, final String cf, Map.Entry<String, Pair<Long,Long>> snapshotDetail, TabularDataSupport result)
{
    try
    {
        final String totalSize = FileUtils.stringifyFileSize(snapshotDetail.getValue().left);
        final String liveSize =  FileUtils.stringifyFileSize(snapshotDetail.getValue().right);
        result.put(new CompositeDataSupport(COMPOSITE_TYPE, ITEM_NAMES,
                new Object[]{ snapshot, ks, cf, liveSize, totalSize }));
    }
    catch (OpenDataException e)
    {
        throw new RuntimeException(e);
    }
}