org.apache.nifi.util.file.FileUtils Java Examples

The following examples show how to use org.apache.nifi.util.file.FileUtils. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: PersistentProvenanceRepository.java    From nifi with Apache License 2.0 6 votes vote down vote up
@Override
public long getContainerCapacity(final String containerName) throws IOException {
    Map<String, File> map = configuration.getStorageDirectories();

    File container = map.get(containerName);
    if(container != null) {
        long capacity = FileUtils.getContainerCapacity(container.toPath());
        if(capacity==0) {
            throw new IOException("System returned total space of the partition for " + containerName + " is zero byte. "
                    + "Nifi can not create a zero sized provenance repository.");
        }
        return capacity;
    } else {
        throw new IllegalArgumentException("There is no defined container with name " + containerName);
    }
}
 
Example #2
Source File: AbstractTestRecordReaderWriter.java    From localization_nifi with Apache License 2.0 6 votes vote down vote up
@Test
public void testSingleRecordCompressed() throws IOException {
    final File journalFile = new File("target/storage/" + UUID.randomUUID().toString() + "/testSimpleWrite.gz");
    final File tocFile = TocUtil.getTocFile(journalFile);
    final TocWriter tocWriter = new StandardTocWriter(tocFile, false, false);
    final RecordWriter writer = createWriter(journalFile, tocWriter, true, 8192);

    writer.writeHeader(1L);
    writer.writeRecord(createEvent());
    writer.close();

    final TocReader tocReader = new StandardTocReader(tocFile);

    try (final FileInputStream fis = new FileInputStream(journalFile);
        final RecordReader reader = createReader(fis, journalFile.getName(), tocReader, 2048)) {
        assertEquals(0, reader.getBlockIndex());
        reader.skipToBlock(0);
        final StandardProvenanceEventRecord recovered = reader.nextRecord();
        assertNotNull(recovered);

        assertEquals("nifi://unit-test", recovered.getTransitUri());
        assertNull(reader.nextRecord());
    }

    FileUtils.deleteFile(journalFile.getParentFile(), true);
}
 
Example #3
Source File: AbstractTestRecordReaderWriter.java    From localization_nifi with Apache License 2.0 6 votes vote down vote up
@Test
public void testSimpleWriteWithToc() throws IOException {
    final File journalFile = new File("target/storage/" + UUID.randomUUID().toString() + "/testSimpleWrite");
    final File tocFile = TocUtil.getTocFile(journalFile);
    final TocWriter tocWriter = new StandardTocWriter(tocFile, false, false);
    final RecordWriter writer = createWriter(journalFile, tocWriter, false, 1024 * 1024);

    writer.writeHeader(1L);
    writer.writeRecord(createEvent());
    writer.close();

    final TocReader tocReader = new StandardTocReader(tocFile);

    try (final FileInputStream fis = new FileInputStream(journalFile);
        final RecordReader reader = createReader(fis, journalFile.getName(), tocReader, 2048)) {
        assertEquals(0, reader.getBlockIndex());
        reader.skipToBlock(0);
        final StandardProvenanceEventRecord recovered = reader.nextRecord();
        assertNotNull(recovered);

        assertEquals("nifi://unit-test", recovered.getTransitUri());
        assertNull(reader.nextRecord());
    }

    FileUtils.deleteFile(journalFile.getParentFile(), true);
}
 
Example #4
Source File: FileSystemRepository.java    From nifi with Apache License 2.0 6 votes vote down vote up
@Override
public long getContainerCapacity(final String containerName) throws IOException {
    final Path path = containers.get(containerName);

    if (path == null) {
        throw new IllegalArgumentException("No container exists with name " + containerName);
    }

    long capacity = FileUtils.getContainerCapacity(path);

    if(capacity==0) {
        throw new IOException("System returned total space of the partition for " + containerName + " is zero byte. "
                + "Nifi can not create a zero sized FileSystemRepository.");
    }

    return capacity;
}
 
Example #5
Source File: StandardFlowService.java    From nifi with Apache License 2.0 6 votes vote down vote up
@Override
public void copyCurrentFlow(final OutputStream os) throws IOException {
    readLock.lock();
    try {
        if (!Files.exists(flowXml) || Files.size(flowXml) == 0) {
            return;
        }

        try (final InputStream in = Files.newInputStream(flowXml, StandardOpenOption.READ);
                final InputStream gzipIn = new GZIPInputStream(in)) {
            FileUtils.copy(gzipIn, os);
        }
    } finally {
        readLock.unlock();
    }
}
 
Example #6
Source File: FileBasedClusterNodeFirewall.java    From nifi with Apache License 2.0 6 votes vote down vote up
private void syncWithRestoreDirectory() throws IOException {

        // sanity check that restore directory is a directory, creating it if necessary
        FileUtils.ensureDirectoryExistAndCanAccess(restoreDirectory);

        // check that restore directory is not the same as the primary directory
        if (config.getParentFile().getAbsolutePath().equals(restoreDirectory.getAbsolutePath())) {
            throw new IllegalStateException(
                    String.format("Cluster firewall configuration file '%s' cannot be in the restore directory '%s' ",
                            config.getAbsolutePath(), restoreDirectory.getAbsolutePath()));
        }

        // the restore copy will have same file name, but reside in a different directory
        final File restoreFile = new File(restoreDirectory, config.getName());

        // sync the primary copy with the restore copy
        FileUtils.syncWithRestore(config, restoreFile, logger);

    }
 
Example #7
Source File: CaptureServlet.java    From localization_nifi with Apache License 2.0 6 votes vote down vote up
@Override
protected void doPost(final HttpServletRequest request, final HttpServletResponse response) throws ServletException, IOException {
    final ByteArrayOutputStream baos = new ByteArrayOutputStream();

    // Capture all the headers for reference.  Intentionally choosing to not special handling for headers with multiple values for clarity
    final Enumeration<String> headerNames = request.getHeaderNames();
    lastPostHeaders = new HashMap<>();
    while (headerNames.hasMoreElements()) {
        final String nextHeader = headerNames.nextElement();
        lastPostHeaders.put(nextHeader, request.getHeader(nextHeader));
    }

    try {
        StreamUtils.copy(request.getInputStream(), baos);
        this.lastPost = baos.toByteArray();
    } finally {
        FileUtils.closeQuietly(baos);
    }
    response.setStatus(Status.OK.getStatusCode());
}
 
Example #8
Source File: CaptureServlet.java    From localization_nifi with Apache License 2.0 6 votes vote down vote up
@Override
protected void doPost(final HttpServletRequest request, final HttpServletResponse response) throws ServletException, IOException {
    final ByteArrayOutputStream baos = new ByteArrayOutputStream();

    // Capture all the headers for reference.  Intentionally choosing to not special handling for headers with multiple values for clarity
    final Enumeration<String> headerNames = request.getHeaderNames();
    lastPostHeaders = new HashMap<>();
    while (headerNames.hasMoreElements()) {
        final String nextHeader = headerNames.nextElement();
        lastPostHeaders.put(nextHeader, request.getHeader(nextHeader));
    }

    try {
        StreamUtils.copy(request.getInputStream(), baos);
        this.lastPost = baos.toByteArray();
    } finally {
        FileUtils.closeQuietly(baos);
    }
    response.setStatus(Status.OK.getStatusCode());
}
 
Example #9
Source File: FileBasedClusterNodeFirewall.java    From localization_nifi with Apache License 2.0 6 votes vote down vote up
private void syncWithRestoreDirectory() throws IOException {

        // sanity check that restore directory is a directory, creating it if necessary
        FileUtils.ensureDirectoryExistAndCanAccess(restoreDirectory);

        // check that restore directory is not the same as the primary directory
        if (config.getParentFile().getAbsolutePath().equals(restoreDirectory.getAbsolutePath())) {
            throw new IllegalStateException(
                    String.format("Cluster firewall configuration file '%s' cannot be in the restore directory '%s' ",
                            config.getAbsolutePath(), restoreDirectory.getAbsolutePath()));
        }

        // the restore copy will have same file name, but reside in a different directory
        final File restoreFile = new File(restoreDirectory, config.getName());

        // sync the primary copy with the restore copy
        FileUtils.syncWithRestore(config, restoreFile, logger);

    }
 
Example #10
Source File: StandardFlowService.java    From localization_nifi with Apache License 2.0 6 votes vote down vote up
@Override
public void copyCurrentFlow(final OutputStream os) throws IOException {
    readLock.lock();
    try {
        if (!Files.exists(flowXml) || Files.size(flowXml) == 0) {
            return;
        }

        try (final InputStream in = Files.newInputStream(flowXml, StandardOpenOption.READ);
                final InputStream gzipIn = new GZIPInputStream(in)) {
            FileUtils.copy(gzipIn, os);
        }
    } finally {
        readLock.unlock();
    }
}
 
Example #11
Source File: AbstractTestRecordReaderWriter.java    From nifi with Apache License 2.0 6 votes vote down vote up
@Test
public void testSingleRecordCompressed() throws IOException {
    final File journalFile = new File("target/storage/" + UUID.randomUUID().toString() + "/testSimpleWrite.gz");
    final File tocFile = TocUtil.getTocFile(journalFile);
    final TocWriter tocWriter = new StandardTocWriter(tocFile, false, false);
    final RecordWriter writer = createWriter(journalFile, tocWriter, true, 8192);

    writer.writeHeader(1L);
    writer.writeRecord(createEvent());
    writer.close();

    final TocReader tocReader = new StandardTocReader(tocFile);

    assertRecoveredRecord(journalFile, tocReader, "nifi://unit-test", 0);

    FileUtils.deleteFile(journalFile.getParentFile(), true);
}
 
Example #12
Source File: CaptureServlet.java    From nifi with Apache License 2.0 6 votes vote down vote up
@Override
protected void doPost(final HttpServletRequest request, final HttpServletResponse response) throws ServletException, IOException {
    final ByteArrayOutputStream baos = new ByteArrayOutputStream();

    // Capture all the headers for reference.  Intentionally choosing to not special handling for headers with multiple values for clarity
    final Enumeration<String> headerNames = request.getHeaderNames();
    lastPostHeaders = new HashMap<>();
    while (headerNames.hasMoreElements()) {
        final String nextHeader = headerNames.nextElement();
        lastPostHeaders.put(nextHeader, request.getHeader(nextHeader));
    }

    try {
        StreamUtils.copy(request.getInputStream(), baos);
        this.lastPost = baos.toByteArray();
    } finally {
        FileUtils.closeQuietly(baos);
    }
    response.setStatus(Status.OK.getStatusCode());
}
 
Example #13
Source File: AbstractTestRecordReaderWriter.java    From nifi with Apache License 2.0 6 votes vote down vote up
@Test
public void testSimpleWriteWithToc() throws IOException {
    final File journalFile = new File("target/storage/" + UUID.randomUUID().toString() + "/testSimpleWrite");
    final File tocFile = TocUtil.getTocFile(journalFile);
    final TocWriter tocWriter = new StandardTocWriter(tocFile, false, false);
    final RecordWriter writer = createWriter(journalFile, tocWriter, false, 1024 * 1024);

    writer.writeHeader(1L);
    writer.writeRecord(createEvent());
    writer.close();

    final TocReader tocReader = new StandardTocReader(tocFile);
    final String expectedTransitUri = "nifi://unit-test";
        final int expectedBlockIndex = 0;

    assertRecoveredRecord(journalFile, tocReader, expectedTransitUri, expectedBlockIndex);

    FileUtils.deleteFile(journalFile.getParentFile(), true);
}
 
Example #14
Source File: WriteAheadProvenanceRepository.java    From nifi with Apache License 2.0 6 votes vote down vote up
@Override
public long getContainerCapacity(final String containerName) throws IOException {
    Map<String, File> map = config.getStorageDirectories();

    File container = map.get(containerName);
    if(container != null) {
        long capacity = FileUtils.getContainerCapacity(container.toPath());
        if(capacity==0) {
            throw new IOException("System returned total space of the partition for " + containerName + " is zero byte. "
                    + "Nifi can not create a zero sized provenance repository.");
        }
        return capacity;
    } else {
        throw new IllegalArgumentException("There is no defined container with name " + containerName);
    }
}
 
Example #15
Source File: CaptureServlet.java    From nifi with Apache License 2.0 6 votes vote down vote up
@Override
protected void doPost(final HttpServletRequest request, final HttpServletResponse response) throws ServletException, IOException {
    final ByteArrayOutputStream baos = new ByteArrayOutputStream();

    // Capture all the headers for reference.  Intentionally choosing to not special handling for headers with multiple values for clarity
    final Enumeration<String> headerNames = request.getHeaderNames();
    lastPostHeaders = new HashMap<>();
    while (headerNames.hasMoreElements()) {
        final String nextHeader = headerNames.nextElement();
        lastPostHeaders.put(nextHeader, request.getHeader(nextHeader));
    }

    try {
        StreamUtils.copy(request.getInputStream(), baos);
        this.lastPost = baos.toByteArray();
    } finally {
        FileUtils.closeQuietly(baos);
    }
    response.setStatus(Status.OK.getStatusCode());
}
 
Example #16
Source File: NiFiSinglePortInputOperatorTest.java    From attic-apex-malhar with Apache License 2.0 6 votes vote down vote up
@Before
public void setup() throws IOException
{
  final String windowDataDir = "target/" + this.getClass().getSimpleName();
  final File windowDataDirFile = new File(windowDataDir);
  if (windowDataDirFile.exists()) {
    FileUtils.deleteFile(windowDataDirFile, true);
  }

  Attribute.AttributeMap attributeMap = new Attribute.AttributeMap.DefaultAttributeMap();
  attributeMap.put(DAG.APPLICATION_PATH, windowDataDir);

  context = mockOperatorContext(12345, attributeMap);

  sink = new CollectorTestSink<>();
  builder = new MockSiteToSiteClient.Builder();
  windowDataManager = new FSWindowDataManager();

  operator = new NiFiSinglePortInputOperator(builder, windowDataManager);
  operator.outputPort.setSink(sink);
}
 
Example #17
Source File: NiFiSinglePortOutputOperatorTest.java    From attic-apex-malhar with Apache License 2.0 6 votes vote down vote up
@Before
public void setup() throws IOException
{
  final String windowDataDir = "target/" + this.getClass().getSimpleName();
  final File windowDataDirFile = new File(windowDataDir);
  if (windowDataDirFile.exists()) {
    FileUtils.deleteFile(windowDataDirFile, true);
  }

  Attribute.AttributeMap attributeMap = new Attribute.AttributeMap.DefaultAttributeMap();
  attributeMap.put(DAG.APPLICATION_PATH, windowDataDir);

  context = mockOperatorContext(12345, attributeMap);

  windowDataManager = new FSWindowDataManager();

  stsBuilder = new MockSiteToSiteClient.Builder();
  dpBuilder = new StringNiFiDataPacketBuilder();
  operator = new NiFiSinglePortOutputOperator(stsBuilder, dpBuilder, windowDataManager, 1);
}
 
Example #18
Source File: TestGenerateTableFetch.java    From nifi with Apache License 2.0 5 votes vote down vote up
@BeforeClass
public static void setupBeforeClass() throws IOException {
    System.setProperty("derby.stream.error.file", "target/derby.log");

    // remove previous test database, if any
    final File dbLocation = new File(DB_LOCATION);
    try {
        FileUtils.deleteFile(dbLocation, true);
    } catch (IOException ioe) {
        // Do nothing, may not have existed
    }
}
 
Example #19
Source File: QueryDatabaseTableTest.java    From nifi with Apache License 2.0 5 votes vote down vote up
@AfterClass
public static void cleanUpAfterClass() throws Exception {
    try {
        DriverManager.getConnection("jdbc:derby:" + DB_LOCATION + ";shutdown=true");
    } catch (SQLNonTransientConnectionException e) {
        // Do nothing, this is what happens at Derby shutdown
    }
    // remove previous test database, if any
    final File dbLocation = new File(DB_LOCATION);
    try {
        FileUtils.deleteFile(dbLocation, true);
    } catch (IOException ioe) {
        // Do nothing, may not have existed
    }
}
 
Example #20
Source File: TestSimpleIndexManager.java    From nifi with Apache License 2.0 5 votes vote down vote up
@Test
public void testMultipleWritersSimultaneouslySameIndex() throws IOException {
    final StandardIndexManager mgr = new StandardIndexManager(new RepositoryConfiguration());
    final File dir = new File("target/" + UUID.randomUUID().toString());
    try {
        final EventIndexWriter writer1 = mgr.borrowIndexWriter(dir);
        final EventIndexWriter writer2 = mgr.borrowIndexWriter(dir);

        final Document doc1 = new Document();
        doc1.add(new StringField("id", "1", Store.YES));

        final Document doc2 = new Document();
        doc2.add(new StringField("id", "2", Store.YES));

        writer1.index(doc1, 1000);
        writer2.index(doc2, 1000);
        mgr.returnIndexWriter(writer2);
        mgr.returnIndexWriter(writer1);

        final EventIndexSearcher searcher = mgr.borrowIndexSearcher(dir);
        final TopDocs topDocs = searcher.getIndexSearcher().search(new MatchAllDocsQuery(), 2);
        assertEquals(2, topDocs.totalHits.value);
        mgr.returnIndexSearcher(searcher);
    } finally {
        FileUtils.deleteFile(dir, true);
    }
}
 
Example #21
Source File: AbstractTestRecordReaderWriter.java    From nifi with Apache License 2.0 5 votes vote down vote up
@Test
public void testMultipleRecordsMultipleBlocksCompressed() throws IOException {
    final File journalFile = new File("target/storage/" + UUID.randomUUID().toString() + "/testSimpleWrite.gz");
    final File tocFile = TocUtil.getTocFile(journalFile);
    final TocWriter tocWriter = new StandardTocWriter(tocFile, false, false);
    // new block each 10 bytes
    final RecordWriter writer = createWriter(journalFile, tocWriter, true, 100);

    writer.writeHeader(1L);
    for (int i = 0; i < 10; i++) {
        writer.writeRecord(createEvent());
    }
    writer.close();

    final TocReader tocReader = new StandardTocReader(tocFile);

    try (final FileInputStream fis = new FileInputStream(journalFile);
        final RecordReader reader = createReader(fis, journalFile.getName(), tocReader, 2048)) {
        for (int i = 0; i < 10; i++) {
            final StandardProvenanceEventRecord recovered = reader.nextRecord();
            System.out.println(recovered);
            assertNotNull(recovered);
            assertEquals(i, recovered.getEventId());
            assertEquals("nifi://unit-test", recovered.getTransitUri());

            final Map<String, String> updatedAttrs = recovered.getUpdatedAttributes();
            assertNotNull(updatedAttrs);
            assertEquals(2, updatedAttrs.size());
            assertEquals("1.txt", updatedAttrs.get("filename"));
            assertTrue(updatedAttrs.containsKey("uuid"));
        }

        assertNull(reader.nextRecord());
    }

    FileUtils.deleteFile(journalFile.getParentFile(), true);
}
 
Example #22
Source File: AbstractTestRecordReaderWriter.java    From nifi with Apache License 2.0 5 votes vote down vote up
@Test
public void testMultipleRecordsSameBlockCompressed() throws IOException {
    final File journalFile = new File("target/storage/" + UUID.randomUUID().toString() + "/testSimpleWrite.gz");
    final File tocFile = TocUtil.getTocFile(journalFile);
    final TocWriter tocWriter = new StandardTocWriter(tocFile, false, false);
    // new record each 1 MB of uncompressed data
    final RecordWriter writer = createWriter(journalFile, tocWriter, true, 1024 * 1024);

    writer.writeHeader(1L);
    for (int i = 0; i < 10; i++) {
        writer.writeRecord(createEvent());
    }
    writer.close();

    final TocReader tocReader = new StandardTocReader(tocFile);

    try (final FileInputStream fis = new FileInputStream(journalFile);
        final RecordReader reader = createReader(fis, journalFile.getName(), tocReader, 2048)) {
        for (int i = 0; i < 10; i++) {
            assertEquals(0, reader.getBlockIndex());

            // call skipToBlock half the time to ensure that we can; avoid calling it
            // the other half of the time to ensure that it's okay.
            if (i <= 5) {
                reader.skipToBlock(0);
            }

            final StandardProvenanceEventRecord recovered = reader.nextRecord();
            assertNotNull(recovered);
            assertEquals("nifi://unit-test", recovered.getTransitUri());
        }

        assertNull(reader.nextRecord());
    }

    FileUtils.deleteFile(journalFile.getParentFile(), true);
}
 
Example #23
Source File: StandardFlowSynchronizer.java    From nifi with Apache License 2.0 5 votes vote down vote up
private byte[] readFlowFromDisk() throws IOException {
    final Path flowPath = nifiProperties.getFlowConfigurationFile().toPath();
    if (!Files.exists(flowPath) || Files.size(flowPath) == 0) {
        return new byte[0];
    }

    final ByteArrayOutputStream baos = new ByteArrayOutputStream();

    try (final InputStream in = Files.newInputStream(flowPath, StandardOpenOption.READ);
            final InputStream gzipIn = new GZIPInputStream(in)) {
        FileUtils.copy(gzipIn, baos);
    }

    return baos.toByteArray();
}
 
Example #24
Source File: WriteAheadProvenanceRepository.java    From nifi with Apache License 2.0 5 votes vote down vote up
@Override
public long getContainerUsableSpace(String containerName) throws IOException {
    Map<String, File> map = config.getStorageDirectories();

    File container = map.get(containerName);
    if(container != null) {
        return FileUtils.getContainerUsableSpace(container.toPath());
    } else {
        throw new IllegalArgumentException("There is no defined container with name " + containerName);
    }
}
 
Example #25
Source File: LuceneEventIndex.java    From nifi with Apache License 2.0 5 votes vote down vote up
protected boolean tryDeleteIndex(final File indexDirectory) {
    final long startNanos = System.nanoTime();
    boolean removed = false;
    while (!removed && System.nanoTime() - startNanos < TimeUnit.SECONDS.toNanos(MAX_DELETE_INDEX_WAIT_SECONDS)) {
        removed = indexManager.removeIndex(indexDirectory);

        if (!removed) {
            try {
                Thread.sleep(5000L);
            } catch (final InterruptedException ie) {
                logger.debug("Interrupted when trying to remove index {} from IndexManager; will not remove index", indexDirectory);
                Thread.currentThread().interrupt();
                return false;
            }
        }
    }

    if (removed) {
        try {
            FileUtils.deleteFile(indexDirectory, true);
            logger.debug("Successfully deleted directory {}", indexDirectory);
        } catch (final IOException e) {
            logger.warn("The Lucene Index located at " + indexDirectory + " has expired and contains no Provenance Events that still exist in the respository. "
                + "However, the directory could not be deleted.", e);
        }

        directoryManager.removeDirectory(indexDirectory);
        logger.info("Successfully removed expired Lucene Index {}", indexDirectory);
    } else {
        logger.warn("The Lucene Index located at {} has expired and contains no Provenance Events that still exist in the respository. "
            + "However, the directory could not be deleted because it is still actively being used. Will continue to try to delete "
            + "in a subsequent maintenance cycle.", indexDirectory);
    }

    return removed;
}
 
Example #26
Source File: IndexDirectoryManager.java    From nifi with Apache License 2.0 5 votes vote down vote up
public void replaceDirectory(final File oldIndexDir, final File newIndexDir, final boolean destroyOldIndex) {
    boolean replaced = false;

    synchronized (this) {
        for (final Map.Entry<Long, List<IndexLocation>> entry : indexLocationByTimestamp.entrySet()) {
            final List<IndexLocation> locations = entry.getValue();
            final ListIterator<IndexLocation> itr = locations.listIterator();

            while (itr.hasNext()) {
                final IndexLocation location = itr.next();
                if (location.getIndexDirectory().equals(oldIndexDir)) {
                    final IndexLocation updatedLocation = new IndexLocation(newIndexDir, location.getIndexStartTimestamp(), location.getPartitionName());
                    itr.set(updatedLocation);
                    replaced = true;
                    logger.debug("Replaced {} with {}", location, updatedLocation);
                }
            }
        }
    }

    if (!replaced) {
        insertIndexDirectory(newIndexDir);
    }

    if (destroyOldIndex) {
        try {
            FileUtils.deleteFile(oldIndexDir, true);
        } catch (IOException e) {
            logger.warn("Failed to delete index directory {}; this directory should be cleaned up manually", oldIndexDir, e);
        }
    }

    removeDirectory(oldIndexDir);

    logger.info("Successfully replaced old index directory {} with new index directory {}", oldIndexDir, newIndexDir);
}
 
Example #27
Source File: PersistentProvenanceRepository.java    From nifi with Apache License 2.0 5 votes vote down vote up
@Override
public long getContainerUsableSpace(String containerName) throws IOException {
    Map<String, File> map = configuration.getStorageDirectories();

    File container = map.get(containerName);
    if(container != null) {
        return FileUtils.getContainerUsableSpace(container.toPath());
    } else {
        throw new IllegalArgumentException("There is no defined container with name " + containerName);
    }
}
 
Example #28
Source File: FileSystemRepository.java    From nifi with Apache License 2.0 5 votes vote down vote up
@Override
public long getContainerUsableSpace(String containerName) throws IOException {
    final Path path = containers.get(containerName);

    if (path == null) {
        throw new IllegalArgumentException("No container exists with name " + containerName);
    }

    return FileUtils.getContainerUsableSpace(path);
}
 
Example #29
Source File: ExecuteFlumeSourceTest.java    From nifi with Apache License 2.0 5 votes vote down vote up
@Test
@Ignore("Does not work on Windows")
public void testSourceWithConfig() throws IOException {
    File spoolDirectory = temp.newFolder("spooldir");
    File dst = new File(spoolDirectory, "records.txt");
    FileUtils.copyFile(getClass().getResourceAsStream("/testdata/records.txt"), dst, true, false);

    TestRunner runner = TestRunners.newTestRunner(ExecuteFlumeSource.class);
    runner.setProperty(ExecuteFlumeSource.SOURCE_TYPE, "spooldir");
    runner.setProperty(ExecuteFlumeSink.FLUME_CONFIG,
        "tier1.sources.src-1.spoolDir = " + spoolDirectory.getAbsolutePath());
    runner.run(1, false, true);
    // Because the spool directory source is an event driven source, it may take some time for flow files to get
    // produced. I'm willing to wait up to 5 seconds, but will bail out early if possible. If it takes longer than
    // that then there is likely a bug.
    int numWaits = 10;
    while (runner.getFlowFilesForRelationship(ExecuteFlumeSource.SUCCESS).size() < 4 && --numWaits > 0) {
        try {
            TimeUnit.MILLISECONDS.sleep(500);
        } catch (InterruptedException ex) {
            logger.warn("Sleep interrupted");
        }
    }
    runner.shutdown();
    runner.assertTransferCount(ExecuteFlumeSource.SUCCESS, 4);
    int i = 1;
    for (MockFlowFile flowFile : runner.getFlowFilesForRelationship(ExecuteFlumeSource.SUCCESS)) {
        flowFile.assertContentEquals("record " + i);
        i++;
    }
}
 
Example #30
Source File: TestWriteAheadFlowFileRepository.java    From nifi with Apache License 2.0 5 votes vote down vote up
public void clearRepo() throws IOException {
    final File target = new File("target");
    final File testRepo = new File(target, "test-repo");
    if (testRepo.exists()) {
        FileUtils.deleteFile(testRepo, true);
    }
}